From 09e91f9ad40417bfa0b69f3481328081204e51d7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Sep 2020 11:22:49 +0800 Subject: [PATCH 001/122] TD-1437 --- src/query/src/qAst.c | 6 +- src/tsdb/src/tsdbMemTable.c | 40 +- src/tsdb/src/tsdbMeta.c | 23 +- src/util/CMakeLists.txt | 2 +- src/util/inc/tskiplist.h | 51 +- src/util/src/tskiplist.c | 1016 +++++++++++++++++-------------- src/util/tests/skiplistTest.cpp | 4 +- 7 files changed, 602 insertions(+), 540 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 634f014d97..1e60c71afd 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -411,9 +411,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr } if (cond.start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); } if (cond.start != NULL) { @@ -468,7 +468,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr tSkipListDestroyIter(iter); comp = true; - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index f6a7f1b35c..0b7cc4dca4 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -37,38 +37,25 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { STsdbCfg * pCfg = &pRepo->config; STsdbMeta * pMeta = pRepo->tsdbMeta; - int32_t level = 0; - int32_t headSize = 0; TSKEY key = dataRowKey(row); SMemTable * pMemTable = pRepo->mem; STableData *pTableData = NULL; - SSkipList * pSList = NULL; + // SSkipList * pSList = NULL; - if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && - pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { - pTableData = pMemTable->tData[TABLE_TID(pTable)]; - pSList = pTableData->pData; - } - - tSkipListNewNodeInfo(pSList, &level, &headSize); - - SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *)); - if (pNode == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } + // if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && + // pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { + // pTableData = pMemTable->tData[TABLE_TID(pTable)]; + // pSList = pTableData->pData; + // } void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); if (pRow == NULL) { tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s", REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno)); - free(pNode); return -1; } - pNode->level = level; dataRowCpy(pRow, row); - *(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow; // Operations above may change pRepo->mem, retake those values ASSERT(pRepo->mem != NULL); @@ -77,7 +64,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (TABLE_TID(pTable) >= pMemTable->maxTables) { if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) { tsdbFreeBytes(pRepo, pRow, dataRowLen(row)); - free(pNode); return -1; } } @@ -97,7 +83,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { " to table %s while create new table data object since %s", REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno)); tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); - free(pNode); return -1; } @@ -106,20 +91,19 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); - if (tSkipListPut(pTableData->pData, pNode) == NULL) { + int64_t oldSize = SL_GET_SIZE(pTableData->pData); + if (tSkipListPut(pTableData->pData, (void *)(&pRow), sizeof(void *)) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); - free(pNode); } else { + int64_t deltaSize = SL_GET_SIZE(pTableData->pData) - oldSize; if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; - pMemTable->numOfRows++; + pMemTable->numOfRows += deltaSize; if (pTableData->keyFirst > key) pTableData->keyFirst = key; if (pTableData->keyLast < key) pTableData->keyLast = key; - pTableData->numOfRows++; - - ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); + pTableData->numOfRows += deltaSize; } tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), @@ -439,7 +423,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->numOfRows = 0; pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey); + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], /*SL_DISCARD_DUP_KEY*/ SL_APPEND_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index a84bb69777..dede109928 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -690,7 +690,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { } pTable->tagVal = NULL; STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -892,23 +892,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper pTable->pSuper = pSTable; - int32_t level = 0; - int32_t headSize = 0; + tSkipListPut(pSTable->pIndex, (void *)(&pTable), sizeof(STable *)); - tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize); - - // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the - // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function - SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *)); - if (pNode == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - pNode->level = level; - - memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *)); - - tSkipListPut(pSTable->pIndex, pNode); if (refSuper) T_REF_INC(pSTable); return 0; } @@ -1165,7 +1150,7 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { buf = tdDecodeSchema(buf, &(pTable->tagSchema)); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbFreeTable(pTable); @@ -1191,7 +1176,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); } else { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1)); + tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_GET_SIZE(pTable->pIndex) + 1)); } else { tlen = sizeof(SListNode) + sizeof(SActObj); } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index e63a085cc8..211c5ab320 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -7,7 +7,7 @@ TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) - ADD_SUBDIRECTORY(tests) + # ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 4ba620dce0..f1e09501e9 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -27,6 +27,14 @@ extern "C" { #define MAX_SKIP_LIST_LEVEL 15 #define SKIP_LIST_RECORD_PERFORMANCE 0 +// For key property setting +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists +#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key +#define SL_UPDATA_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert +#define SL_APPEND_DUP_KEY (uint8_t)0x3 // Update duplicate key by append +// For thread safety setting +#define SL_THREAD_SAFE (uint8_t)0x4 + typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); @@ -41,6 +49,9 @@ typedef struct SSkipListNode { uint8_t level; } SSkipListNode; +#define SL_IS_THREAD_SAFE(flags) ((flags)&SL_THREAD_SAFE) +#define SL_DUP_MODE(flags) ((flags) & ((((uint8_t)1) << 2) - 1)) + #define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES) #define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)] @@ -54,6 +65,8 @@ typedef struct SSkipListNode { #define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0))) #define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) +#define SL_GET_SIZE(s) (s)->size +#define SL_GET_TSIZE(s) (s)->tsize /* * @version 0.3 @@ -113,13 +126,16 @@ typedef struct SSkipListKeyInfo { typedef struct SSkipList { __compar_fn_t comparFn; __sl_key_fn_t keyFn; - uint32_t size; - uint8_t maxLevel; - uint8_t level; - SSkipListKeyInfo keyInfo; pthread_rwlock_t *lock; - SSkipListNode * pHead; // point to the first element - SSkipListNode * pTail; // point to the last element + uint16_t len; + uint8_t maxLevel; + uint8_t flags; + uint8_t type; // static info above + uint8_t level; + uint32_t size; // not including duplicate keys + uint32_t tsize; // including duplicate keys + SSkipListNode * pHead; // point to the first element + SSkipListNode * pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif @@ -145,8 +161,7 @@ typedef struct SSkipListIterator { * @param dupKey allow the duplicated key in the skip list * @return */ -SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe, - uint8_t freeNode, __sl_key_fn_t fn); +SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn); /** * @@ -163,6 +178,17 @@ void *tSkipListDestroy(SSkipList *pSkipList); */ void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize); +/** + * put the data into the skiplist + * If failed, NULL will be returned, otherwise, the pNode will be returned. + * + * @param pSkipList + * @param pData + * @param dataLen + * @return + */ +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen); + /** * put the skip list node into the skip list. * If failed, NULL will be returned, otherwise, the pNode will be returned. @@ -171,7 +197,7 @@ void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSiz * @param pNode * @return */ -SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); +SSkipListNode *tSkipListPutNode(SSkipList *pSkipList, SSkipListNode *pNode); /** * get *all* nodes which key are equivalent to pKey @@ -182,13 +208,6 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); */ SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); -/** - * get the size of skip list - * @param pSkipList - * @return - */ -size_t tSkipListGetSize(const SSkipList *pSkipList); - /** * display skip list of the given level, for debug purpose only * @param pSkipList diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 303c2440bf..73f6bf0502 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -13,13 +13,521 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" -#include "tulog.h" #include "tskiplist.h" -#include "tutil.h" +#include "os.h" #include "tcompare.h" +#include "tulog.h" +#include "tutil.h" -UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level +#define DO_MEMSET_PTR_AREA(n) \ + do { \ + int32_t _l = (n)->level; \ + memset(pNode, 0, SL_NODE_HEADER_SIZE(_l)); \ + (n)->level = _l; \ + } while (0) + +static int initForwardBackwardPtr(SSkipList *pSkipList); +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); +static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); +static void tSkipListCorrectLevel(SSkipList *pSkipList); +static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); + +static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); +static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); +static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); +static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); +static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); + +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn) { + SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); + if (pSkipList == NULL) { + return NULL; + } + + if (maxLevel > MAX_SKIP_LIST_LEVEL) { + maxLevel = MAX_SKIP_LIST_LEVEL; + } + + pSkipList->maxLevel = maxLevel; + pSkipList->type = keyType; + pSkipList->len = keyLen; + pSkipList->flags = flags; + pSkipList->keyFn = fn; + pSkipList->comparFn = getKeyComparFunc(keyType); + + // pSkipList->level = 1; // TODO: check if 1 is valid + if (initForwardBackwardPtr(pSkipList) < 0) { + taosTFree(pSkipList); + return NULL; + } + + if (SL_IS_THREAD_SAFE(flags)) { + pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t)); + + if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { + taosTFree(pSkipList->pHead); + taosTFree(pSkipList); + + return NULL; + } + } + + srand((uint32_t)time(NULL)); + +#if SKIP_LIST_RECORD_PERFORMANCE + pSkipList->state.nTotalMemSize += sizeof(SSkipList); +#endif + + return pSkipList; +} + +void *tSkipListDestroy(SSkipList *pSkipList) { + if (pSkipList == NULL) return NULL; + + tSkipListWLock(pSkipList); + + SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); + + while (pNode != pSkipList->pTail) { + SSkipListNode *pTemp = pNode; + pNode = SL_GET_FORWARD_POINTER(pNode, 0); + taosTFree(pTemp); + } + + tSkipListUnlock(pSkipList); + if (pSkipList->lock != NULL) { + pthread_rwlock_destroy(pSkipList->lock); + taosTFree(pSkipList->lock); + } + + taosTFree(pSkipList->pHead); + taosTFree(pSkipList); + return NULL; +} + +void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) { + if (pSkipList == NULL) { + *level = 1; + *headSize = SL_NODE_HEADER_SIZE(*level); + return; + } + + *level = getSkipListRandLevel(pSkipList); + *headSize = SL_NODE_HEADER_SIZE(*level); +} + +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen) { + if (pSkipList == NULL || pData == NULL) return NULL; + + tSkipListWLock(pSkipList); + + int32_t level = getSkipListRandLevel(pSkipList); + + SSkipListNode *pNode = (SSkipListNode *)malloc(SL_NODE_HEADER_SIZE(level) + dataLen); + if (pNode == NULL) { + tSkipListUnlock(pSkipList); + return NULL; + } + + pNode->level = level; + memcpy(SL_GET_NODE_DATA(pNode), pData, dataLen); + + if (tSkipListPutNodeImpl(pSkipList, pNode) == NULL) { + tSkipListUnlock(pSkipList); + taosTFree(pNode); + return NULL; + } + + tSkipListUnlock(pSkipList); + + return pNode; +} + +SSkipListNode *tSkipListPutNode(SSkipList *pSkipList, SSkipListNode *pNode) { + SSkipListNode *pRetNode = NULL; + + if (pSkipList == NULL || pNode == NULL) return NULL; + + tSkipListWLock(pSkipList); + pRetNode = tSkipListPutNodeImpl(pSkipList, pNode); + tSkipListUnlock(pSkipList); + + return pRetNode; +} + +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { + SArray *sa = taosArrayInit(1, POINTER_BYTES); + + tSkipListRLock(pSkipList); + + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + while (1) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + taosArrayPush(sa, &p); + pNode = p; + } + + tSkipListUnlock(pSkipList); + + return sa; +} + +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { + uint32_t count = 0; + + tSkipListWLock(pSkipList); + + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + while (1) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + + tSkipListRemoveNodeImpl(pSkipList, p); + + ++count; + } + + tSkipListCorrectLevel(pSkipList); + + tSkipListUnlock(pSkipList); + + return count; +} + +void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { + tSkipListWLock(pSkipList); + tSkipListRemoveNodeImpl(pSkipList, pNode); + tSkipListCorrectLevel(pSkipList); + tSkipListUnlock(pSkipList); +} + +SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) { + if (pSkipList == NULL) return NULL; + + return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC); +} + +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order) { + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + assert(pSkipList != NULL); + + SSkipListIterator *iter = doCreateSkipListIterator(pSkipList, order); + if (val == NULL) { + return iter; + } + + tSkipListRLock(pSkipList); + + iter->cur = getPriorNode(pSkipList, val, order); + + tSkipListUnlock(pSkipList); + + return iter; +} + +bool tSkipListIterNext(SSkipListIterator *iter) { + if (iter->pSkipList == NULL) return false; + + SSkipList *pSkipList = iter->pSkipList; + uint8_t dupMod = SL_DUP_MODE(pSkipList->flags); + + tSkipListRLock(pSkipList); + + if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate + if (dupMod == SL_APPEND_DUP_KEY) { + if (iter->cur == pSkipList->pHead) { + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + iter->step++; + } else { + while (true) { + iter->step++; + SSkipListNode *pNode = iter->cur; + iter->cur = SL_GET_FORWARD_POINTER(pNode, 0); + + if (iter->cur == pSkipList->pTail) break; + if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { + continue; + } else { + break; + } + } + } + + if (iter->cur != pSkipList->pTail) { + while (true) { + SSkipListNode *pNode = SL_GET_FORWARD_POINTER(iter->cur, 0); + if (pNode == pSkipList->pTail) break; + if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { + iter->step++; + iter->cur = pNode; + } else { + break; + } + } + } + } else { + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + iter->step++; + } + } else { // descending order iterate + if (dupMod == SL_APPEND_DUP_KEY) { + if (iter->cur == pSkipList->pTail) { + iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + iter->step++; + } else { + while (true) { + SSkipListNode *pNode = iter->cur; + iter->cur = SL_GET_BACKWARD_POINTER(pNode, 0); + + if (iter->cur == pSkipList->pHead) break; + if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { + iter->cur = pNode; + continue; + } else { + break; + } + } + } + } else { + iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + iter->step++; + } + } + + tSkipListUnlock(pSkipList); + + return (iter->order == TSDB_ORDER_ASC) ? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead); +} + +SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) { + if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) { + return NULL; + } else { + return iter->cur; + } +} + +void *tSkipListDestroyIter(SSkipListIterator *iter) { + if (iter == NULL) { + return NULL; + } + + taosTFree(iter); + return NULL; +} + +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { + if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) { + return; + } + + SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); + + int32_t id = 1; + char * prev = NULL; + + while (p != pSkipList->pTail) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + if (prev != NULL) { + assert(pSkipList->comparFn(prev, key) < 0); + } + + switch (pSkipList->type) { + case TSDB_DATA_TYPE_INT: + fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BIGINT: + fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key); + break; + case TSDB_DATA_TYPE_BINARY: + fprintf(stdout, "%d: %s \n", id++, key); + break; + case TSDB_DATA_TYPE_DOUBLE: + fprintf(stdout, "%d: %lf \n", id++, *(double *)key); + break; + default: + fprintf(stdout, "\n"); + } + + prev = SL_GET_NODE_KEY(pSkipList, p); + + p = SL_GET_FORWARD_POINTER(p, nlevel - 1); + } +} + +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode, bool hasDupKey) { + uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + + // FIXME: this may cause the level of skiplist change + if (dupMode == SL_UPDATA_DUP_KEY && hasDupKey) { + tSkipListRemoveNodeImpl(pSkipList, forward[0]); + tSkipListCorrectLevel(pSkipList); + } + + DO_MEMSET_PTR_AREA(pNode); + + for (int32_t i = 0; i < pNode->level; ++i) { + SSkipListNode *x = forward[i]; + SL_GET_BACKWARD_POINTER(pNode, i) = x; + + SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); + SL_GET_BACKWARD_POINTER(next, i) = pNode; + + SL_GET_FORWARD_POINTER(pNode, i) = next; + SL_GET_FORWARD_POINTER(x, i) = pNode; + } + + if (!(dupMode == SL_APPEND_DUP_KEY && hasDupKey)) { + pSkipList->size += 1; + } + pSkipList->tsize += 1; +} + +static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { + SSkipListIterator *iter = calloc(1, sizeof(SSkipListIterator)); + + iter->pSkipList = pSkipList; + iter->order = order; + if (order == TSDB_ORDER_ASC) { + iter->cur = pSkipList->pHead; + } else { + iter->cur = pSkipList->pTail; + } + + return iter; +} + +static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) { + if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + return pthread_rwlock_wrlock(pSkipList->lock); + } + return 0; +} + +static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) { + if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + return pthread_rwlock_rdlock(pSkipList->lock); + } + return 0; +} + +static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { + if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + return pthread_rwlock_unlock(pSkipList->lock); + } + return 0; +} + +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { + int compare = 1; + bool hasDupKey = false; + char * pNodeKey = SL_GET_NODE_KEY(pSkipList, pNode); + uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + + if (pSkipList->size == 0) { + for (int i = 0; i < pNode->level; i++) { + forward[i] = pSkipList->pHead; + } + } else { + char *pKey = NULL; + + // Compare min key + pKey = SL_GET_SL_MIN_KEY(pSkipList); + compare = pSkipList->comparFn(pNodeKey, pKey); + if ((dupMode == SL_APPEND_DUP_KEY && compare < 0) || (dupMode != SL_APPEND_DUP_KEY && compare <= 0)) { + for (int i = 0; i < pNode->level; i++) { + forward[i] = pSkipList->pHead; + } + return (compare == 0); + } + + // Compare max key + pKey = SL_GET_SL_MAX_KEY(pSkipList); + compare = pSkipList->comparFn(pNodeKey, pKey); + if ((dupMode == SL_DISCARD_DUP_KEY && compare > 0) || (dupMode != SL_DISCARD_DUP_KEY && compare >= 0)) { + for (int i = 0; i < pNode->level; i++) { + forward[i] = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); + } + + return (compare == 0); + } + + SSkipListNode *px = pSkipList->pHead; + for (int i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); + while (p != pSkipList->pTail) { + pKey = SL_GET_NODE_KEY(pSkipList, p); + + compare = pSkipList->comparFn(pKey, pNodeKey); + if (compare == 0 && hasDupKey == false) hasDupKey = true; + if ((dupMode == SL_APPEND_DUP_KEY && compare > 0) || (dupMode != SL_APPEND_DUP_KEY && compare >= 0)) { + break; + } else { + px = p; + p = SL_GET_FORWARD_POINTER(px, i); + } + } + + forward[i] = px; + } + } + + return hasDupKey; +} + +static bool tSkipListIsNodeDup(SSkipList *pSkipList, SSkipListNode *pNode) { + SSkipListNode *pPrevNode = SL_GET_BACKWARD_POINTER(pNode, 0); + SSkipListNode *pNextNode = SL_GET_FORWARD_POINTER(pNode, 0); + char * pNodeKey = SL_GET_NODE_KEY(pSkipList, pNode); + char * pPrevNodeKey = (pPrevNode == pSkipList->pHead) ? NULL : SL_GET_NODE_KEY(pSkipList, pPrevNode); + char * pNextNodeKey = (pNextNode == pSkipList->pTail) ? NULL : SL_GET_NODE_KEY(pSkipList, pNextNode); + + return ((pPrevNodeKey != NULL && pSkipList->comparFn(pNodeKey, pPrevNodeKey) == 0) || + (pNextNodeKey != NULL && pSkipList->comparFn(pNodeKey, pNextNodeKey) == 0)); +} + +static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { + int32_t level = pNode->level; + uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + + bool sizeReduce = !(dupMode == SL_APPEND_DUP_KEY && tSkipListIsNodeDup(pSkipList, pNode)); + + for (int32_t j = level - 1; j >= 0; --j) { + SSkipListNode *prev = SL_GET_BACKWARD_POINTER(pNode, j); + SSkipListNode *next = SL_GET_FORWARD_POINTER(pNode, j); + + SL_GET_FORWARD_POINTER(prev, j) = next; + SL_GET_BACKWARD_POINTER(next, j) = prev; + } + + taosTFree(pNode); + + if (sizeReduce) pSkipList->size--; + pSkipList->tsize--; +} + +// Function must be called after calling tSkipListRemoveNodeImpl() function +static void tSkipListCorrectLevel(SSkipList *pSkipList) { + while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { + pSkipList->level -= 1; + } +} + +UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, + int32_t level) { // record link count in each level #if SKIP_LIST_RECORD_PERFORMANCE for (int32_t i = 0; i < level; ++i) { pSkipList->state.nLevelNodeCnt[i]++; @@ -47,40 +555,28 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { } static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { - int32_t level = getSkipListNodeRandomHeight(pSkipList); + int32_t level = 0; if (pSkipList->size == 0) { level = 1; - pSkipList->level = 1; } else { + level = getSkipListNodeRandomHeight(pSkipList); if (level > pSkipList->level) { if (pSkipList->level < pSkipList->maxLevel) { - level = (++pSkipList->level); + level = pSkipList->level + 1; } else { level = pSkipList->level; } } } - + assert(level <= pSkipList->maxLevel); return level; } -#define DO_MEMSET_PTR_AREA(n) do {\ -int32_t _l = (n)->level;\ -memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\ -(n)->level = _l;\ -} while(0) - -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); -static SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode); -static SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode); -static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); - - // when order is TSDB_ORDER_ASC, return the last node with key less than val // when order is TSDB_ORDER_DESC, return the first node with key large than val -static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_t order) { - __compar_fn_t comparFn = pSkipList->comparFn; +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order) { + __compar_fn_t comparFn = pSkipList->comparFn; SSkipListNode *pNode = NULL; if (order == TSDB_ORDER_ASC) { @@ -116,214 +612,49 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_ return pNode; } - -static bool initForwardBackwardPtr(SSkipList* pSkipList) { +static int initForwardBackwardPtr(SSkipList *pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; - + // head info - pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2); + pSkipList->pHead = (SSkipListNode *)malloc(SL_NODE_HEADER_SIZE(maxLevel) * 2); if (pSkipList->pHead == NULL) { - return false; + return -1; } - - pSkipList->pHead->level = pSkipList->maxLevel; - + + pSkipList->pHead->level = maxLevel; + // tail info - pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel)); - pSkipList->pTail->level = pSkipList->maxLevel; - + pSkipList->pTail = (SSkipListNode *)POINTER_SHIFT(pSkipList->pHead, SL_NODE_HEADER_SIZE(maxLevel)); + pSkipList->pTail->level = maxLevel; + for (uint32_t i = 0; i < maxLevel; ++i) { SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; } - - return true; + + return 0; } -SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock, - uint8_t freeNode, __sl_key_fn_t fn) { - SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); - if (pSkipList == NULL) { - return NULL; - } - - if (maxLevel > MAX_SKIP_LIST_LEVEL) { - maxLevel = MAX_SKIP_LIST_LEVEL; - } - - pSkipList->keyInfo.type = keyType; - pSkipList->keyInfo.len = keyLen; - pSkipList->keyInfo.dupKey = dupKey; - pSkipList->keyInfo.freeNode = freeNode; - - pSkipList->keyFn = fn; - pSkipList->comparFn = getKeyComparFunc(keyType); - pSkipList->maxLevel = maxLevel; - pSkipList->level = 1; - - if (!initForwardBackwardPtr(pSkipList)) { - taosTFree(pSkipList); - return NULL; - } - - if (lock) { - pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t)); - - if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { - taosTFree(pSkipList->pHead); - taosTFree(pSkipList); - - return NULL; - } - } - - srand((uint32_t)time(NULL)); - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalMemSize += sizeof(SSkipList); -#endif - - return pSkipList; -} - -void *tSkipListDestroy(SSkipList *pSkipList) { - if (pSkipList == NULL) { - return NULL; - } - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - if (pSkipList->keyInfo.freeNode) { - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - - while (pNode != pSkipList->pTail) { - SSkipListNode *pTemp = pNode; - pNode = SL_GET_FORWARD_POINTER(pNode, 0); - taosTFree(pTemp); - } - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - pthread_rwlock_destroy(pSkipList->lock); - - taosTFree(pSkipList->lock); - } - - taosTFree(pSkipList->pHead); - taosTFree(pSkipList); - return NULL; -} - -void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) { - if (pSkipList == NULL) { - *level = 1; - *headSize = SL_NODE_HEADER_SIZE(*level); - return; - } - - *level = getSkipListRandLevel(pSkipList); - *headSize = SL_NODE_HEADER_SIZE(*level); -} - -SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { - if (pSkipList == NULL || pNode == NULL) { - return NULL; - } - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - // if the new key is greater than the maximum key of skip list, push back this node at the end of skip list - char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); - if (pSkipList->size == 0 || pSkipList->comparFn(SL_GET_SL_MAX_KEY(pSkipList), newDatakey) < 0) { - return tSkipListPushBack(pSkipList, pNode); - } - - // if the new key is less than the minimum key of skip list, push front this node at the front of skip list - assert(pSkipList->size > 0); - char* minKey = SL_GET_SL_MIN_KEY(pSkipList); - if (pSkipList->comparFn(newDatakey, minKey) < 0) { - return tSkipListPushFront(pSkipList, pNode); - } - - // find the appropriated position to insert data - SSkipListNode *px = pSkipList->pHead; +static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { + SSkipListNode *pRetNode = NULL; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; - int32_t ret = -1; - for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); - while (p != pSkipList->pTail) { - char *key = SL_GET_NODE_KEY(pSkipList, p); + int hasDupKey = tSkipListGetPosToPut(pSkipList, forward, pNode); + if (SL_DUP_MODE(pSkipList->flags) == SL_DISCARD_DUP_KEY && hasDupKey) { + pRetNode = NULL; + } else { + pRetNode = pNode; + tSkipListDoInsert(pSkipList, forward, pNode, hasDupKey); - // if the forward element is less than the specified key, forward one step - ret = pSkipList->comparFn(key, newDatakey); - if (ret < 0) { - px = p; - p = SL_GET_FORWARD_POINTER(px, i); - } else { - break; - } - } - - forward[i] = px; + if (pNode->level > pSkipList->level) pSkipList->level = pNode->level; } - // if the skip list does not allowed identical key inserted, the new data will be discarded. - if (pSkipList->keyInfo.dupKey == 0 && ret == 0) { - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return NULL; - } - - tSkipListDoInsert(pSkipList, forward, pNode); - return pNode; + return pRetNode; } - - -SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { - SArray* sa = taosArrayInit(1, POINTER_BYTES); - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); - while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); - if (p == pSkipList->pTail) { - break; - } - if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { - break; - } - taosArrayPush(sa, &p); - pNode = p; - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return sa; -} - - - -size_t tSkipListGetSize(const SSkipList* pSkipList) { - if (pSkipList == NULL) { - return 0; - } - - return pSkipList->size; -} +// static void tSkipListSeek(SSkipList *pSkipList, char *key, int order) { +// // TODO +// } // static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey, // int32_t cond, SSkipListNode ***pRes) { @@ -444,158 +775,7 @@ size_t tSkipListGetSize(const SSkipList* pSkipList) { // // return true; //} - -uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { - uint32_t count = 0; - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); - while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); - if (p == pSkipList->pTail) { - break; - } - if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { - break; - } - - for (int32_t j = p->level - 1; j >= 0; --j) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(p, j); - SSkipListNode* next = SL_GET_FORWARD_POINTER(p, j); - SL_GET_FORWARD_POINTER(prev, j) = next; - SL_GET_BACKWARD_POINTER(next, j) = prev; - } - - if (pSkipList->keyInfo.freeNode) { - taosTFree(p); - } - - ++count; - } - - // compress the minimum level of skip list - while (pSkipList->level > 0) { - if (SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) != NULL) { - break; - } - pSkipList->level--; - } - - pSkipList->size -= count; - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return count; -} - -void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { - int32_t level = pNode->level; - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - for (int32_t j = level - 1; j >= 0; --j) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pNode, j); - SSkipListNode* next = SL_GET_FORWARD_POINTER(pNode, j); - - SL_GET_FORWARD_POINTER(prev, j) = next; - SL_GET_BACKWARD_POINTER(next, j) = prev; - } - - if (pSkipList->keyInfo.freeNode) { - taosTFree(pNode); - } - - atomic_sub_fetch_32(&pSkipList->size, 1); - - // compress the minimum level of skip list - while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { - pSkipList->level -= 1; - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } -} - -SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList) { - if (pSkipList == NULL) { - return NULL; - } - - return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC); -} - -SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order) { - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - assert(pSkipList != NULL); - - SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order); - if (val == NULL) { - return iter; - } - - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); - } - - iter->cur = getPriorNode(pSkipList, val, order); - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return iter; -} - -bool tSkipListIterNext(SSkipListIterator *iter) { - if (iter->pSkipList == NULL) { - return false; - } - - SSkipList *pSkipList = iter->pSkipList; - - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); - } - - if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); - } else { // descending order iterate - iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - iter->step += 1; - return (iter->order == TSDB_ORDER_ASC)? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead); -} - -SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) { - if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) { - return NULL; - } else { - return iter->cur; - } -} - -void* tSkipListDestroyIter(SSkipListIterator* iter) { - if (iter == NULL) { - return NULL; - } - - taosTFree(iter); - return NULL; -} - +// // bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) { // SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; // __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); @@ -614,110 +794,4 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // pthread_rwlock_unlock(&pSkipList->lock); // // return ret; -//} - -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { - if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) { - return; - } - - SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); - - int32_t id = 1; - char* prev = NULL; - - while (p != pSkipList->pTail) { - char *key = SL_GET_NODE_KEY(pSkipList, p); - if (prev != NULL) { - assert(pSkipList->comparFn(prev, key) < 0); - } - - switch (pSkipList->keyInfo.type) { - case TSDB_DATA_TYPE_INT: - fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BIGINT: - fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key); - break; - case TSDB_DATA_TYPE_BINARY: - fprintf(stdout, "%d: %s \n", id++, key); - break; - case TSDB_DATA_TYPE_DOUBLE: - fprintf(stdout, "%d: %lf \n", id++, *(double *)key); - break; - default: - fprintf(stdout, "\n"); - } - - prev = SL_GET_NODE_KEY(pSkipList, p); - - p = SL_GET_FORWARD_POINTER(p, nlevel - 1); - } -} - -void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { - DO_MEMSET_PTR_AREA(pNode); - - for (int32_t i = 0; i < pNode->level; ++i) { - SSkipListNode *x = forward[i]; - SL_GET_BACKWARD_POINTER(pNode, i) = x; - - SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); - SL_GET_BACKWARD_POINTER(next, i) = pNode; - - SL_GET_FORWARD_POINTER(pNode, i) = next; - SL_GET_FORWARD_POINTER(x, i) = pNode; - } - - atomic_add_fetch_32(&pSkipList->size, 1); - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } -} - -SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode) { - SSkipListNode* forward[MAX_SKIP_LIST_LEVEL] = {0}; - for(int32_t i = 0; i < pSkipList->level; ++i) { - forward[i] = pSkipList->pHead; - } - - tSkipListDoInsert(pSkipList, forward, pNode); - return pNode; -} - -SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode) { - // do clear pointer area - DO_MEMSET_PTR_AREA(pNode); - - for(int32_t i = 0; i < pNode->level; ++i) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); - SL_GET_FORWARD_POINTER(prev, i) = pNode; - SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; - - SL_GET_BACKWARD_POINTER(pNode, i) = prev; - SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; - } - - atomic_add_fetch_32(&pSkipList->size, 1); - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return pNode; -} - -SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { - SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator)); - - iter->pSkipList = pSkipList; - iter->order = order; - if(order == TSDB_ORDER_ASC) { - iter->cur = pSkipList->pHead; - } else { - iter->cur = pSkipList->pTail; - } - - return iter; -} \ No newline at end of file +//} \ No newline at end of file diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 77174f69fd..cd1fd1f7e1 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -247,7 +247,7 @@ void skiplistPerformanceTest() { printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size); printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level); - assert(tSkipListGetSize(pSkipList) == size); + assert(SL_GET_SIZE(pSkipList) == size); // printf("the level of skiplist is:\n"); // @@ -273,7 +273,7 @@ void skiplistPerformanceTest() { int64_t et = taosGetTimestampMs(); printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st); - assert(tSkipListGetSize(pSkipList) == size); + assert(SL_GET_SIZE(pSkipList) == size); tSkipListDestroy(pSkipList); taosTFree(total); From db37e3cd13cfec79de28496de03b6bc660388bd5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Sep 2020 15:56:40 +0800 Subject: [PATCH 002/122] TD-1438 --- src/common/inc/tdataformat.h | 2 -- src/common/src/tdataformat.c | 11 ++++++-- src/inc/tsdb.h | 1 + src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbMain.c | 5 ++++ src/tsdb/src/tsdbMemTable.c | 14 ++++++---- src/tsdb/src/tsdbRWHelper.c | 53 +++++++++++++++++++----------------- 7 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index cc4afeb3f8..aa39d5b0b2 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -255,8 +255,6 @@ void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows); // ----------------- K-V data row structure /* diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index e7f40442a0..0c446b6d4f 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,6 +18,9 @@ #include "tcoding.h" #include "wchar.h" +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows); + /** * Duplicate the schema and return a new object */ @@ -499,7 +502,9 @@ _err: return -1; } -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) { +// src2 data has more priority than src1 +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows) { tdResetDataCols(target); ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); @@ -509,7 +514,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; - if (key1 <= key2) { + if (key1 < key2) { for (int i = 0; i < src1->numOfCols; i++) { ASSERT(target->cols[i].type == src1->cols[i].type); if (src1->cols[i].len > 0) { @@ -520,7 +525,6 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter1)++; - if (key1 == key2) (*iter2)++; } else { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); @@ -532,6 +536,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter2)++; + if (key1 == key2) (*iter1)++; } } } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 85f9b3bdc7..fe03277763 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -65,6 +65,7 @@ typedef struct { int32_t maxRowsPerFileBlock; // maximum rows per file block int8_t precision; int8_t compression; + int8_t update; } STsdbCfg; // --------- TSDB REPOSITORY USAGE STATISTICS diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 256b8189f8..95b7010038 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -430,7 +430,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem) void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TSKEY* filterKeys, int nFilterKeys); + TSKEY* filterKeys, int nFilterKeys, bool keepDup); static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index a1e6376304..494f17807e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } } + // update check + if (pCfg->update != 0) pCfg->update = 1; + return 0; _err: @@ -923,6 +926,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); tlen += taosEncodeFixedI8(buf, pCfg->precision); tlen += taosEncodeFixedI8(buf, pCfg->compression); + tlen += taosEncodeFixedI8(buf, pCfg->update); return tlen; } @@ -939,6 +943,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); buf = taosDecodeFixedI8(buf, &(pCfg->precision)); buf = taosDecodeFixedI8(buf, &(pCfg->compression)); + buf = taosDecodeFixedI8(buf, &(pCfg->update)); return buf; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0b7cc4dca4..c58460e18f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -278,7 +278,7 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { } int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TSKEY *filterKeys, int nFilterKeys) { + TSKEY *filterKeys, int nFilterKeys, bool keepDup) { ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; STSchema *pSchema = NULL; @@ -319,6 +319,10 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (!keyFiltered) { if (numOfRows >= maxRowsToRead) break; + numOfRows++; + } + + if (!keyFiltered || keepDup) { if (pCols) { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); @@ -329,8 +333,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey tdAppendDataRowToDataCol(row, pSchema, pCols); } - numOfRows++; - } + } } while (tSkipListIterNext(pIter)); return numOfRows; @@ -422,8 +425,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->keyLast = 0; pTableData->numOfRows = 0; - pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], /*SL_DISCARD_DUP_KEY*/ SL_APPEND_DUP_KEY, tsdbGetTsTupleKey); + pTableData->pData = + tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], + pCfg->update ? SL_APPEND_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index bb0f00ef53..f0017d0a14 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -62,7 +62,7 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx); static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows); + TSKEY maxKey, int maxRows, int8_t update); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -1453,7 +1453,7 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, - pDataCols, NULL, 0); + pDataCols, NULL, 0, pCfg->update); ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { @@ -1474,7 +1474,8 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } else { ASSERT(!pHelper->hasOldLastBlock); tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0); + int rowsRead = + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update); ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; @@ -1516,17 +1517,17 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); int rows1 = defaultRowsInBlock - pCompBlock->numOfRows; - int rows2 = - tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows); - if (rows2 == 0) { // all data filtered out + int rows2 = tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, + pDataCols0->numOfRows, pCfg->update); + if (!pCfg->update && rows2 == 0) { // all data filtered out *(pCommitIter->pIter) = slIter; } else { if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows); + pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); + ASSERT(rowsRead == rows2 && rowsRead <= pDataCols->numOfRows && pDataCols->numOfRows > 0); if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; @@ -1535,9 +1536,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int round = 0; int dIter = 0; while (true) { - tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock); + int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, + defaultRowsInBlock, pCfg->update); if (rowsRead == 0) break; if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; @@ -1561,8 +1561,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (keyFirst < blkKeyFirst) { while (true) { tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0); + int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, + NULL, 0, pCfg->update); if (rowsRead == 0) break; ASSERT(rowsRead == pDataCols->numOfRows); @@ -1580,21 +1580,21 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, slIter = *(pCommitIter->pIter); int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows); + pDataCols0->numOfRows, pCfg->update); - if (rows2 == 0) { // all filtered out + if (!pCfg->update && rows2 == 0) { // all filtered out *(pCommitIter->pIter) = slIter; ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { - int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2; + int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0, pCfg->update) + rows2; if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { int rows = (rows1 >= rows3) ? rows3 : rows2; tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows); + pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); + ASSERT(rowsRead == rows && rowsRead <= pDataCols->numOfRows); if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; @@ -1606,12 +1606,11 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int round = 0; int dIter = 0; while (true) { - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); + int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, + defaultRowsInBlock, pCfg->update); if (rowsRead == 0) break; - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) - return -1; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (round == 0) { if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; } else { @@ -1633,7 +1632,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows) { + TSKEY maxKey, int maxRows, int8_t update) { int numOfRows = 0; TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; @@ -1649,14 +1648,17 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte if (key1 == INT64_MAX && key2 == INT64_MAX) break; - if (key1 <= key2) { + if ((key1 < key2) || ((!update) && (key1 == key2))) { for (int i = 0; i < pDataCols->numOfCols; i++) { dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } pTarget->numOfRows++; (*iter)++; - if (key1 == key2) tSkipListIterNext(pCommitIter->pIter); + + if ((!update) && (key1 == key2)) { + tSkipListIterNext(pCommitIter->pIter); + } } else { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); @@ -1665,6 +1667,7 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte tdAppendDataRowToDataCol(row, pSchema, pTarget); tSkipListIterNext(pCommitIter->pIter); + if (key1 == key2) (*iter)++; } numOfRows++; From 5023ce7d36e676270f8b871a389ba4b46b3d9d21 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Sep 2020 15:56:40 +0800 Subject: [PATCH 003/122] TD-1438 --- src/common/inc/tdataformat.h | 2 -- src/common/src/tdataformat.c | 11 ++++++-- src/inc/tsdb.h | 1 + src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbMain.c | 5 ++++ src/tsdb/src/tsdbMemTable.c | 14 ++++++---- src/tsdb/src/tsdbRWHelper.c | 53 +++++++++++++++++++----------------- 7 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index cc4afeb3f8..aa39d5b0b2 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -255,8 +255,6 @@ void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows); // ----------------- K-V data row structure /* diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index e7f40442a0..0c446b6d4f 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,6 +18,9 @@ #include "tcoding.h" #include "wchar.h" +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows); + /** * Duplicate the schema and return a new object */ @@ -499,7 +502,9 @@ _err: return -1; } -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) { +// src2 data has more priority than src1 +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows) { tdResetDataCols(target); ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); @@ -509,7 +514,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; - if (key1 <= key2) { + if (key1 < key2) { for (int i = 0; i < src1->numOfCols; i++) { ASSERT(target->cols[i].type == src1->cols[i].type); if (src1->cols[i].len > 0) { @@ -520,7 +525,6 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter1)++; - if (key1 == key2) (*iter2)++; } else { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); @@ -532,6 +536,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter2)++; + if (key1 == key2) (*iter1)++; } } } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 85f9b3bdc7..fe03277763 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -65,6 +65,7 @@ typedef struct { int32_t maxRowsPerFileBlock; // maximum rows per file block int8_t precision; int8_t compression; + int8_t update; } STsdbCfg; // --------- TSDB REPOSITORY USAGE STATISTICS diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 256b8189f8..95b7010038 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -430,7 +430,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem) void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TSKEY* filterKeys, int nFilterKeys); + TSKEY* filterKeys, int nFilterKeys, bool keepDup); static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index a1e6376304..494f17807e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } } + // update check + if (pCfg->update != 0) pCfg->update = 1; + return 0; _err: @@ -923,6 +926,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); tlen += taosEncodeFixedI8(buf, pCfg->precision); tlen += taosEncodeFixedI8(buf, pCfg->compression); + tlen += taosEncodeFixedI8(buf, pCfg->update); return tlen; } @@ -939,6 +943,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); buf = taosDecodeFixedI8(buf, &(pCfg->precision)); buf = taosDecodeFixedI8(buf, &(pCfg->compression)); + buf = taosDecodeFixedI8(buf, &(pCfg->update)); return buf; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0b7cc4dca4..c58460e18f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -278,7 +278,7 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { } int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TSKEY *filterKeys, int nFilterKeys) { + TSKEY *filterKeys, int nFilterKeys, bool keepDup) { ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; STSchema *pSchema = NULL; @@ -319,6 +319,10 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (!keyFiltered) { if (numOfRows >= maxRowsToRead) break; + numOfRows++; + } + + if (!keyFiltered || keepDup) { if (pCols) { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); @@ -329,8 +333,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey tdAppendDataRowToDataCol(row, pSchema, pCols); } - numOfRows++; - } + } } while (tSkipListIterNext(pIter)); return numOfRows; @@ -422,8 +425,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->keyLast = 0; pTableData->numOfRows = 0; - pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], /*SL_DISCARD_DUP_KEY*/ SL_APPEND_DUP_KEY, tsdbGetTsTupleKey); + pTableData->pData = + tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], + pCfg->update ? SL_APPEND_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index bb0f00ef53..f0017d0a14 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -62,7 +62,7 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx); static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows); + TSKEY maxKey, int maxRows, int8_t update); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -1453,7 +1453,7 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, - pDataCols, NULL, 0); + pDataCols, NULL, 0, pCfg->update); ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { @@ -1474,7 +1474,8 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } else { ASSERT(!pHelper->hasOldLastBlock); tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0); + int rowsRead = + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update); ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; @@ -1516,17 +1517,17 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); int rows1 = defaultRowsInBlock - pCompBlock->numOfRows; - int rows2 = - tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows); - if (rows2 == 0) { // all data filtered out + int rows2 = tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, + pDataCols0->numOfRows, pCfg->update); + if (!pCfg->update && rows2 == 0) { // all data filtered out *(pCommitIter->pIter) = slIter; } else { if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows); + pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); + ASSERT(rowsRead == rows2 && rowsRead <= pDataCols->numOfRows && pDataCols->numOfRows > 0); if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; @@ -1535,9 +1536,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int round = 0; int dIter = 0; while (true) { - tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock); + int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, + defaultRowsInBlock, pCfg->update); if (rowsRead == 0) break; if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; @@ -1561,8 +1561,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (keyFirst < blkKeyFirst) { while (true) { tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0); + int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, + NULL, 0, pCfg->update); if (rowsRead == 0) break; ASSERT(rowsRead == pDataCols->numOfRows); @@ -1580,21 +1580,21 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, slIter = *(pCommitIter->pIter); int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows); + pDataCols0->numOfRows, pCfg->update); - if (rows2 == 0) { // all filtered out + if (!pCfg->update && rows2 == 0) { // all filtered out *(pCommitIter->pIter) = slIter; ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { - int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2; + int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0, pCfg->update) + rows2; if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { int rows = (rows1 >= rows3) ? rows3 : rows2; tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows); + pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); + ASSERT(rowsRead == rows && rowsRead <= pDataCols->numOfRows); if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; @@ -1606,12 +1606,11 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int round = 0; int dIter = 0; while (true) { - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); + int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, + defaultRowsInBlock, pCfg->update); if (rowsRead == 0) break; - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) - return -1; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (round == 0) { if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; } else { @@ -1633,7 +1632,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows) { + TSKEY maxKey, int maxRows, int8_t update) { int numOfRows = 0; TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; @@ -1649,14 +1648,17 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte if (key1 == INT64_MAX && key2 == INT64_MAX) break; - if (key1 <= key2) { + if ((key1 < key2) || ((!update) && (key1 == key2))) { for (int i = 0; i < pDataCols->numOfCols; i++) { dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } pTarget->numOfRows++; (*iter)++; - if (key1 == key2) tSkipListIterNext(pCommitIter->pIter); + + if ((!update) && (key1 == key2)) { + tSkipListIterNext(pCommitIter->pIter); + } } else { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); @@ -1665,6 +1667,7 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte tdAppendDataRowToDataCol(row, pSchema, pTarget); tSkipListIterNext(pCommitIter->pIter); + if (key1 == key2) (*iter)++; } numOfRows++; From aca8e3644b0825ac8c7244a91090fe1492f5df1c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Sep 2020 17:26:08 +0800 Subject: [PATCH 004/122] TD-1439 --- src/tsdb/src/tsdbRead.c | 43 ++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d829a85754..806f0ea828 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -432,7 +432,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { +static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -466,9 +466,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order TSKEY r2 = dataRowKey(rimem); if (r1 == r2) { // data ts are duplicated, ignore the data in mem - tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; - return rimem; + if (!update) { + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + tSkipListIterNext(pCheckInfo->iiter); + pCheckInfo->chosen = 0; + return rmem; + } } else { if (ASCENDING_TRAVERSE(order)) { if (r1 < r2) { @@ -522,6 +528,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { } static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { + STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = -1; @@ -535,7 +542,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update); if (row == NULL) { return false; } @@ -741,11 +748,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){ SQueryFilePos* cur = &pQueryHandle->cur; + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); int32_t code = TSDB_CODE_SUCCESS; /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); assert(cur->pos >= 0 && cur->pos <= binfo.rows); @@ -1208,6 +1216,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; initTableMemIterator(pQueryHandle, pCheckInfo); @@ -1256,7 +1265,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } @@ -1286,7 +1295,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it - moveToNextRowInMem(pCheckInfo); + if (pCfg->update) { + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + numOfRows += 1; + if (cur->win.skey == TSKEY_INITIAL_VAL) { + cur->win.skey = key; + } + + cur->win.ekey = key; + cur->lastKey = key + step; + cur->mixBlock = true; + + moveToNextRowInMem(pCheckInfo); + pos += step; + } else { + moveToNextRowInMem(pCheckInfo); + } } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1765,13 +1789,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbQueryHandle* pQueryHandle) { int numOfRows = 0; int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); STable* pTable = pCheckInfo->pTableObj; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } From 3d2a77ea6d4c8e6c22e80dae08098bdf0d4fd8b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 17 Sep 2020 11:13:06 +0800 Subject: [PATCH 005/122] fix skiplist crash bug --- src/util/src/tskiplist.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 73f6bf0502..c3e8ee41bd 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -466,7 +466,12 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, } SSkipListNode *px = pSkipList->pHead; - for (int i = pSkipList->level - 1; i >= 0; --i) { + for (int i = pNode->level - 1; i >= 0; --i) { + if (i >= pSkipList->level) { + forward[i] = pSkipList->pHead; + continue; + } + SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); while (p != pSkipList->pTail) { pKey = SL_GET_NODE_KEY(pSkipList, p); From 2b1dc53ae1af311ef5763848c3d877785bc30451 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 17 Sep 2020 12:13:19 +0800 Subject: [PATCH 006/122] fix update bug --- src/tsdb/src/tsdbRead.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 806f0ea828..4f902b624b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1321,7 +1321,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* assert(end != -1); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it - moveToNextRowInMem(pCheckInfo); + if (!pCfg->update) { + moveToNextRowInMem(pCheckInfo); + } else { + end -= step; + } } int32_t qstart = 0, qend = 0; From 384b5104fab77cc74998fc105bdc931fa8e31fbd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 18 Sep 2020 10:51:12 +0800 Subject: [PATCH 007/122] set update --- src/vnode/src/vnodeMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index a4e88fb946..9914ecc13b 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -135,6 +135,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.precision = pVnodeCfg->cfg.precision; tsdbCfg.compression = pVnodeCfg->cfg.compression; + tsdbCfg.update = 1; char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); From 4209f666b8f75ff54b11a60242f980be41c1cec8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Sep 2020 12:20:33 +0800 Subject: [PATCH 008/122] [TD-1485] add update options --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 11 ++++++++++ src/inc/taosdef.h | 4 ++++ src/inc/taosmsg.h | 5 ++++- src/mnode/inc/mnodeDef.h | 3 ++- src/mnode/src/mnodeDb.c | 25 +++++++++++++++++++++- src/mnode/src/mnodeVgroup.c | 1 + src/vnode/src/vnodeMain.c | 5 +++-- tests/script/general/http/restful_full.sim | 2 +- 9 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 77e8b76456..b6a2146c13 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -83,6 +83,7 @@ extern int16_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; extern int32_t tsQuorum; +extern int32_t tsUpdate; // balance extern int32_t tsEnableBalance; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 7e46f58a93..f03b0f85b4 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -112,6 +112,7 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; +int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; @@ -774,6 +775,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "update"; + cfg.ptr = &tsUpdate; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_DB_UPDATE; + cfg.maxValue = TSDB_MAX_DB_UPDATE; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "mqttHostName"; cfg.ptr = tsMqttHostName; cfg.valType = TAOS_CFG_VTYPE_STRING; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 9a411d8f85..5d08d1db59 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -350,6 +350,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_WAL_LEVEL 2 #define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 1 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 + #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index e49e2caca1..0c7334afcd 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -531,6 +531,8 @@ typedef struct { int8_t replications; int8_t quorum; int8_t ignoreExist; + int8_t update; + int8_t reserve[9]; } SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { @@ -630,7 +632,8 @@ typedef struct { int8_t replications; int8_t wals; int8_t quorum; - int8_t reserved[16]; + int8_t update; + int8_t reserved[15]; } SMDVnodeCfg; typedef struct { diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 682986b29f..43fe107666 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -171,7 +171,8 @@ typedef struct { int8_t walLevel; int8_t replications; int8_t quorum; - int8_t reserved[12]; + int8_t update; + int8_t reserved[11]; } SDbCfg; typedef struct SDbObj { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 54c049d242..d3bbaebd5d 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { } #endif + if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { + mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + return TSDB_CODE_SUCCESS; } @@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->update < 0) pCfg->update = tsUpdate; } static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { @@ -391,7 +397,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg .compression = pCreate->compression, .walLevel = pCreate->walLevel, .replications = pCreate->replications, - .quorum = pCreate->quorum + .quorum = pCreate->quorum, + .update = pCreate->update }; mnodeSetDefaultDbCfg(&pDb->cfg); @@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "update"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); @@ -745,6 +758,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.update; + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; if (pDb->status == TSDB_DB_STATUS_READY) { const char *src = "ready"; @@ -842,6 +859,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { int8_t replications = pAlter->replications; int8_t quorum = pAlter->quorum; int8_t precision = pAlter->precision; + int8_t update = pAlter->update; terrno = TSDB_CODE_SUCCESS; @@ -944,6 +962,11 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { newCfg.quorum = quorum; } + if (update >= 0 && update != pDb->cfg.update) { + mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update); + newCfg.update = update; + } + return newCfg; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index aa6631ff83..a336bb8781 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -837,6 +837,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { pCfg->replications = (int8_t) pVgroup->numOfVnodes; pCfg->wals = 3; pCfg->quorum = pDb->cfg.quorum; + pCfg->update = pDb->cfg.update; SMDVnodeDesc *pNodes = pVnode->nodes; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 9914ecc13b..7815b32858 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -135,7 +135,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.precision = pVnodeCfg->cfg.precision; tsdbCfg.compression = pVnodeCfg->cfg.compression; - tsdbCfg.update = 1; + tsdbCfg.update = pVnodeCfg->cfg.update; char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); @@ -144,7 +144,8 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { return TSDB_CODE_VND_INIT_FAILED; } - vInfo("vgId:%d, vnode is created, walLevel:%d fsyncPeriod:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel, pVnodeCfg->cfg.fsyncPeriod); + vInfo("vgId:%d, vnode is created, walLevel:%d fsyncPeriod:%d update:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel, + pVnodeCfg->cfg.fsyncPeriod, pVnodeCfg->cfg.update); code = vnodeOpen(pVnodeCfg->cfg.vgId, rootDir); return code; diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index a02140a419..290787261f 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -81,7 +81,7 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then return -1 endi From cf9ba60ef25215375255cf40264c09bac10a97e0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Sep 2020 17:17:31 +0000 Subject: [PATCH 009/122] support option to db --- src/client/src/tscSQLParser.c | 1 + src/inc/ttokendef.h | 411 +-- src/query/inc/qSqlparser.h | 1 + src/query/inc/sql.y | 3 + src/query/src/qTokenizer.c | 3 +- src/query/src/sql.c | 3394 +++++++++-------------- tests/pytest/alter/db_update_options.py | 71 + 7 files changed, 1617 insertions(+), 2267 deletions(-) create mode 100644 tests/pytest/alter/db_update_options.py diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4bc793e79f..4364659ca6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5183,6 +5183,7 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->replications = pCreateDb->replica; pMsg->quorum = pCreateDb->quorum; pMsg->ignoreExist = pCreateDb->ignoreExists; + pMsg->update = pCreateDb->update; } int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index c5831a9b8a..41798133e1 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -16,212 +16,213 @@ #ifndef TDENGINE_TTOKENDEF_H #define TDENGINE_TTOKENDEF_H -#define TK_ID 1 -#define TK_BOOL 2 -#define TK_TINYINT 3 -#define TK_SMALLINT 4 -#define TK_INTEGER 5 -#define TK_BIGINT 6 -#define TK_FLOAT 7 -#define TK_DOUBLE 8 -#define TK_STRING 9 -#define TK_TIMESTAMP 10 -#define TK_BINARY 11 -#define TK_NCHAR 12 -#define TK_OR 13 -#define TK_AND 14 -#define TK_NOT 15 -#define TK_EQ 16 -#define TK_NE 17 -#define TK_ISNULL 18 -#define TK_NOTNULL 19 -#define TK_IS 20 -#define TK_LIKE 21 -#define TK_GLOB 22 -#define TK_BETWEEN 23 -#define TK_IN 24 -#define TK_GT 25 -#define TK_GE 26 -#define TK_LT 27 -#define TK_LE 28 -#define TK_BITAND 29 -#define TK_BITOR 30 -#define TK_LSHIFT 31 -#define TK_RSHIFT 32 -#define TK_PLUS 33 -#define TK_MINUS 34 -#define TK_DIVIDE 35 -#define TK_TIMES 36 -#define TK_STAR 37 -#define TK_SLASH 38 -#define TK_REM 39 -#define TK_CONCAT 40 -#define TK_UMINUS 41 -#define TK_UPLUS 42 -#define TK_BITNOT 43 -#define TK_SHOW 44 -#define TK_DATABASES 45 -#define TK_MNODES 46 -#define TK_DNODES 47 -#define TK_ACCOUNTS 48 -#define TK_USERS 49 -#define TK_MODULES 50 -#define TK_QUERIES 51 -#define TK_CONNECTIONS 52 -#define TK_STREAMS 53 -#define TK_VARIABLES 54 -#define TK_SCORES 55 -#define TK_GRANTS 56 -#define TK_VNODES 57 -#define TK_IPTOKEN 58 -#define TK_DOT 59 -#define TK_TABLES 60 -#define TK_STABLES 61 -#define TK_VGROUPS 62 -#define TK_DROP 63 -#define TK_TABLE 64 -#define TK_DATABASE 65 -#define TK_DNODE 66 -#define TK_USER 67 -#define TK_ACCOUNT 68 -#define TK_USE 69 -#define TK_DESCRIBE 70 -#define TK_ALTER 71 -#define TK_PASS 72 -#define TK_PRIVILEGE 73 -#define TK_LOCAL 74 -#define TK_IF 75 -#define TK_EXISTS 76 -#define TK_CREATE 77 -#define TK_PPS 78 -#define TK_TSERIES 79 -#define TK_DBS 80 -#define TK_STORAGE 81 -#define TK_QTIME 82 -#define TK_CONNS 83 -#define TK_STATE 84 -#define TK_KEEP 85 -#define TK_CACHE 86 -#define TK_REPLICA 87 -#define TK_QUORUM 88 -#define TK_DAYS 89 -#define TK_MINROWS 90 -#define TK_MAXROWS 91 -#define TK_BLOCKS 92 -#define TK_CTIME 93 -#define TK_WAL 94 -#define TK_FSYNC 95 -#define TK_COMP 96 -#define TK_PRECISION 97 -#define TK_LP 98 -#define TK_RP 99 -#define TK_TAGS 100 -#define TK_USING 101 -#define TK_AS 102 -#define TK_COMMA 103 -#define TK_NULL 104 -#define TK_SELECT 105 -#define TK_UNION 106 -#define TK_ALL 107 -#define TK_FROM 108 -#define TK_VARIABLE 109 -#define TK_INTERVAL 110 -#define TK_FILL 111 -#define TK_SLIDING 112 -#define TK_ORDER 113 -#define TK_BY 114 -#define TK_ASC 115 -#define TK_DESC 116 -#define TK_GROUP 117 -#define TK_HAVING 118 -#define TK_LIMIT 119 -#define TK_OFFSET 120 -#define TK_SLIMIT 121 -#define TK_SOFFSET 122 -#define TK_WHERE 123 -#define TK_NOW 124 -#define TK_RESET 125 -#define TK_QUERY 126 -#define TK_ADD 127 -#define TK_COLUMN 128 -#define TK_TAG 129 -#define TK_CHANGE 130 -#define TK_SET 131 -#define TK_KILL 132 -#define TK_CONNECTION 133 -#define TK_STREAM 134 -#define TK_COLON 135 -#define TK_ABORT 136 -#define TK_AFTER 137 -#define TK_ATTACH 138 -#define TK_BEFORE 139 -#define TK_BEGIN 140 -#define TK_CASCADE 141 -#define TK_CLUSTER 142 -#define TK_CONFLICT 143 -#define TK_COPY 144 -#define TK_DEFERRED 145 -#define TK_DELIMITERS 146 -#define TK_DETACH 147 -#define TK_EACH 148 -#define TK_END 149 -#define TK_EXPLAIN 150 -#define TK_FAIL 151 -#define TK_FOR 152 -#define TK_IGNORE 153 -#define TK_IMMEDIATE 154 -#define TK_INITIALLY 155 -#define TK_INSTEAD 156 -#define TK_MATCH 157 -#define TK_KEY 158 -#define TK_OF 159 -#define TK_RAISE 160 -#define TK_REPLACE 161 -#define TK_RESTRICT 162 -#define TK_ROW 163 -#define TK_STATEMENT 164 -#define TK_TRIGGER 165 -#define TK_VIEW 166 -#define TK_COUNT 167 -#define TK_SUM 168 -#define TK_AVG 169 -#define TK_MIN 170 -#define TK_MAX 171 -#define TK_FIRST 172 -#define TK_LAST 173 -#define TK_TOP 174 -#define TK_BOTTOM 175 -#define TK_STDDEV 176 -#define TK_PERCENTILE 177 -#define TK_APERCENTILE 178 -#define TK_LEASTSQUARES 179 -#define TK_HISTOGRAM 180 -#define TK_DIFF 181 -#define TK_SPREAD 182 -#define TK_TWA 183 -#define TK_INTERP 184 -#define TK_LAST_ROW 185 -#define TK_RATE 186 -#define TK_IRATE 187 -#define TK_SUM_RATE 188 -#define TK_SUM_IRATE 189 -#define TK_AVG_RATE 190 -#define TK_AVG_IRATE 191 -#define TK_TBID 192 -#define TK_SEMI 193 -#define TK_NONE 194 -#define TK_PREV 195 -#define TK_LINEAR 196 -#define TK_IMPORT 197 -#define TK_METRIC 198 -#define TK_TBNAME 199 -#define TK_JOIN 200 -#define TK_METRICS 201 -#define TK_STABLE 202 -#define TK_INSERT 203 -#define TK_INTO 204 -#define TK_VALUES 205 +#define TK_ID 1 +#define TK_BOOL 2 +#define TK_TINYINT 3 +#define TK_SMALLINT 4 +#define TK_INTEGER 5 +#define TK_BIGINT 6 +#define TK_FLOAT 7 +#define TK_DOUBLE 8 +#define TK_STRING 9 +#define TK_TIMESTAMP 10 +#define TK_BINARY 11 +#define TK_NCHAR 12 +#define TK_OR 13 +#define TK_AND 14 +#define TK_NOT 15 +#define TK_EQ 16 +#define TK_NE 17 +#define TK_ISNULL 18 +#define TK_NOTNULL 19 +#define TK_IS 20 +#define TK_LIKE 21 +#define TK_GLOB 22 +#define TK_BETWEEN 23 +#define TK_IN 24 +#define TK_GT 25 +#define TK_GE 26 +#define TK_LT 27 +#define TK_LE 28 +#define TK_BITAND 29 +#define TK_BITOR 30 +#define TK_LSHIFT 31 +#define TK_RSHIFT 32 +#define TK_PLUS 33 +#define TK_MINUS 34 +#define TK_DIVIDE 35 +#define TK_TIMES 36 +#define TK_STAR 37 +#define TK_SLASH 38 +#define TK_REM 39 +#define TK_CONCAT 40 +#define TK_UMINUS 41 +#define TK_UPLUS 42 +#define TK_BITNOT 43 +#define TK_SHOW 44 +#define TK_DATABASES 45 +#define TK_MNODES 46 +#define TK_DNODES 47 +#define TK_ACCOUNTS 48 +#define TK_USERS 49 +#define TK_MODULES 50 +#define TK_QUERIES 51 +#define TK_CONNECTIONS 52 +#define TK_STREAMS 53 +#define TK_VARIABLES 54 +#define TK_SCORES 55 +#define TK_GRANTS 56 +#define TK_VNODES 57 +#define TK_IPTOKEN 58 +#define TK_DOT 59 +#define TK_TABLES 60 +#define TK_STABLES 61 +#define TK_VGROUPS 62 +#define TK_DROP 63 +#define TK_TABLE 64 +#define TK_DATABASE 65 +#define TK_DNODE 66 +#define TK_USER 67 +#define TK_ACCOUNT 68 +#define TK_USE 69 +#define TK_DESCRIBE 70 +#define TK_ALTER 71 +#define TK_PASS 72 +#define TK_PRIVILEGE 73 +#define TK_LOCAL 74 +#define TK_IF 75 +#define TK_EXISTS 76 +#define TK_CREATE 77 +#define TK_PPS 78 +#define TK_TSERIES 79 +#define TK_DBS 80 +#define TK_STORAGE 81 +#define TK_QTIME 82 +#define TK_CONNS 83 +#define TK_STATE 84 +#define TK_KEEP 85 +#define TK_CACHE 86 +#define TK_REPLICA 87 +#define TK_QUORUM 88 +#define TK_DAYS 89 +#define TK_MINROWS 90 +#define TK_MAXROWS 91 +#define TK_BLOCKS 92 +#define TK_CTIME 93 +#define TK_WAL 94 +#define TK_FSYNC 95 +#define TK_COMP 96 +#define TK_PRECISION 97 +#define TK_UPDATE 98 +#define TK_LP 99 +#define TK_RP 100 +#define TK_TAGS 101 +#define TK_USING 102 +#define TK_AS 103 +#define TK_COMMA 104 +#define TK_NULL 105 +#define TK_SELECT 106 +#define TK_UNION 107 +#define TK_ALL 108 +#define TK_FROM 109 +#define TK_VARIABLE 110 +#define TK_INTERVAL 111 +#define TK_FILL 112 +#define TK_SLIDING 113 +#define TK_ORDER 114 +#define TK_BY 115 +#define TK_ASC 116 +#define TK_DESC 117 +#define TK_GROUP 118 +#define TK_HAVING 119 +#define TK_LIMIT 120 +#define TK_OFFSET 121 +#define TK_SLIMIT 122 +#define TK_SOFFSET 123 +#define TK_WHERE 124 +#define TK_NOW 125 +#define TK_RESET 126 +#define TK_QUERY 127 +#define TK_ADD 128 +#define TK_COLUMN 129 +#define TK_TAG 130 +#define TK_CHANGE 131 +#define TK_SET 132 +#define TK_KILL 133 +#define TK_CONNECTION 134 +#define TK_STREAM 135 +#define TK_COLON 136 +#define TK_ABORT 137 +#define TK_AFTER 138 +#define TK_ATTACH 139 +#define TK_BEFORE 140 +#define TK_BEGIN 141 +#define TK_CASCADE 142 +#define TK_CLUSTER 143 +#define TK_CONFLICT 144 +#define TK_COPY 145 +#define TK_DEFERRED 146 +#define TK_DELIMITERS 147 +#define TK_DETACH 148 +#define TK_EACH 149 +#define TK_END 150 +#define TK_EXPLAIN 151 +#define TK_FAIL 152 +#define TK_FOR 153 +#define TK_IGNORE 154 +#define TK_IMMEDIATE 155 +#define TK_INITIALLY 156 +#define TK_INSTEAD 157 +#define TK_MATCH 158 +#define TK_KEY 159 +#define TK_OF 160 +#define TK_RAISE 161 +#define TK_REPLACE 162 +#define TK_RESTRICT 163 +#define TK_ROW 164 +#define TK_STATEMENT 165 +#define TK_TRIGGER 166 +#define TK_VIEW 167 +#define TK_COUNT 168 +#define TK_SUM 169 +#define TK_AVG 170 +#define TK_MIN 171 +#define TK_MAX 172 +#define TK_FIRST 173 +#define TK_LAST 174 +#define TK_TOP 175 +#define TK_BOTTOM 176 +#define TK_STDDEV 177 +#define TK_PERCENTILE 178 +#define TK_APERCENTILE 179 +#define TK_LEASTSQUARES 180 +#define TK_HISTOGRAM 181 +#define TK_DIFF 182 +#define TK_SPREAD 183 +#define TK_TWA 184 +#define TK_INTERP 185 +#define TK_LAST_ROW 186 +#define TK_RATE 187 +#define TK_IRATE 188 +#define TK_SUM_RATE 189 +#define TK_SUM_IRATE 190 +#define TK_AVG_RATE 191 +#define TK_AVG_IRATE 192 +#define TK_TBID 193 +#define TK_SEMI 194 +#define TK_NONE 195 +#define TK_PREV 196 +#define TK_LINEAR 197 +#define TK_IMPORT 198 +#define TK_METRIC 199 +#define TK_TBNAME 200 +#define TK_JOIN 201 +#define TK_METRICS 202 +#define TK_STABLE 203 +#define TK_INSERT 204 +#define TK_INTO 205 +#define TK_VALUES 206 #define TK_SPACE 300 #define TK_COMMENT 301 diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index d6664577a3..01250f9233 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -129,6 +129,7 @@ typedef struct SCreateDBInfo { int32_t compressionLevel; SStrToken precision; bool ignoreExists; + int8_t update; tVariantList *keep; } SCreateDBInfo; diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 8a5b9d61e2..6fcf647f4f 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -229,6 +229,7 @@ wal(Y) ::= WAL INTEGER(X). { Y = X; } fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } comp(Y) ::= COMP INTEGER(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; } +update(Y) ::= UPDATE INTEGER(X). { Y = X; } %type db_optr {SCreateDBInfo} db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} @@ -246,6 +247,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } +db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } %type alter_db_optr {SCreateDBInfo} alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} @@ -257,6 +259,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 0c9f92786f..0c6ee25f13 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -155,6 +155,7 @@ static SKeyword keywordTable[] = { {"INSERT", TK_INSERT}, {"INTO", TK_INTO}, {"VALUES", TK_VALUES}, + {"UPDATE", TK_UPDATE}, {"RESET", TK_RESET}, {"QUERY", TK_QUERY}, {"ADD", TK_ADD}, @@ -661,4 +662,4 @@ bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, l void taosCleanupKeywordsTable() { taosHashCleanup(KeywordHashTable); -} \ No newline at end of file +} diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 3145f39a5e..be5bcaae82 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -1,30 +1,9 @@ -/* -** 2000-05-29 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Driver template for the LEMON parser generator. -** -** The "lemon" program processes an LALR(1) input grammar file, then uses -** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the -** interstitial "-" characters) contained in this template is changed into -** the value of the %name directive from the grammar. Otherwise, the content -** of this template is copied straight through into the generate parser -** source file. -** -** The following is the concatenation of all %include directives from the -** input grammar file: +/* Driver template for the LEMON parser generator. +** The author disclaims copyright to this source code. */ +/* First off, code is included that follows the "include" declaration +** in the input grammar file. */ #include -#include -/************ Begin %include sections from the grammar ************************/ #include #include @@ -37,119 +16,95 @@ #include "ttokendef.h" #include "tutil.h" #include "tvariant.h" -/**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ - -/* The next sections is a series of control #defines. -** various aspects of the generated parser. -** YYCODETYPE is the data type used to store the integer codes -** that represent terminal and non-terminal symbols. -** "unsigned char" is used if there are fewer than -** 256 symbols. Larger types otherwise. -** YYNOCODE is a number of type YYCODETYPE that is not used for -** any terminal or nonterminal symbol. -** YYFALLBACK If defined, this indicates that one or more tokens -** (also known as: "terminal symbols") have fall-back -** values which should be used if the original symbol -** would not parse. This permits keywords to sometimes -** be used as identifiers, for example. -** YYACTIONTYPE is the data type used for "action codes" - numbers -** that indicate what to do in response to the next -** token. -** ParseTOKENTYPE is the data type used for minor type for terminal -** symbols. Background: A "minor type" is a semantic -** value associated with a terminal or non-terminal -** symbols. For example, for an "ID" terminal symbol, -** the minor type might be the name of the identifier. -** Each non-terminal can have a different minor type. -** Terminal symbols all have the same minor type, though. -** This macros defines the minor type for terminal -** symbols. -** YYMINORTYPE is the data type used for all minor types. -** This is typically a union of many types, one of -** which is ParseTOKENTYPE. The entry in the union -** for terminal symbols is called "yy0". -** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() -** ParseARG_SDECL A static variable declaration for the %extra_argument -** ParseARG_PDECL A parameter declaration for the %extra_argument -** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter -** ParseARG_STORE Code to store %extra_argument into yypParser -** ParseARG_FETCH Code to extract %extra_argument from yypParser -** ParseCTX_* As ParseARG_ except for %extra_context -** YYERRORSYMBOL is the code number of the error symbol. If not -** defined, then do no error processing. -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar -** YYNTOKEN Number of terminal symbols -** YY_MAX_SHIFT Maximum value for shift actions -** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions -** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_ERROR_ACTION The yy_action[] code for syntax error -** YY_ACCEPT_ACTION The yy_action[] code for accept -** YY_NO_ACTION The yy_action[] code for no-op -** YY_MIN_REDUCE Minimum value for reduce actions -** YY_MAX_REDUCE Maximum value for reduce actions +/* Next is all token values, in a form suitable for use by makeheaders. +** This section will be null unless lemon is run with the -m switch. +*/ +/* +** These constants (all generated automatically by the parser generator) +** specify the various kinds of tokens (terminals) that the parser +** understands. +** +** Each symbol here is a terminal symbol in the grammar. +*/ +/* Make sure the INTERFACE macro is defined. */ #ifndef INTERFACE # define INTERFACE 1 #endif -/************* Begin control #defines *****************************************/ +/* The next thing included is series of defines which control +** various aspects of the generated parser. +** YYCODETYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 terminals +** and nonterminals. "int" is used otherwise. +** YYNOCODE is a number of type YYCODETYPE which corresponds +** to no legal terminal or nonterminal number. This +** number is used to fill in empty slots of the hash +** table. +** YYFALLBACK If defined, this indicates that one or more tokens +** have fall-back values which should be used if the +** original value of the token will not parse. +** YYACTIONTYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 rules and +** states combined. "int" is used otherwise. +** ParseTOKENTYPE is the data type used for minor tokens given +** directly to the parser from the tokenizer. +** YYMINORTYPE is the data type used for all minor tokens. +** This is typically a union of many types, one of +** which is ParseTOKENTYPE. The entry in the union +** for base tokens is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** ParseARG_SDECL A static variable declaration for the %extra_argument +** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_STORE Code to store %extra_argument into yypParser +** ParseARG_FETCH Code to extract %extra_argument from yypParser +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +*/ #define YYCODETYPE unsigned short int -#define YYNOCODE 270 +#define YYNOCODE 274 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - int yy112; - SCreateDBInfo yy118; - tVariantList* yy156; - tSQLExprList* yy158; - tSQLExpr* yy190; - SSubclauseInfo* yy333; - SIntervalVal yy340; - TAOS_FIELD yy343; - int64_t yy369; - SCreateTableSQL* yy398; - SLimitVal yy414; - SQuerySQL* yy444; - SCreateAcctSQL yy479; - tVariant yy506; - tFieldList* yy511; + int yy46; + tSQLExpr* yy64; + tVariant yy134; + SCreateAcctSQL yy149; + int64_t yy207; + SLimitVal yy216; + TAOS_FIELD yy223; + SSubclauseInfo* yy231; + SCreateDBInfo yy268; + tSQLExprList* yy290; + SQuerySQL* yy414; + SCreateTableSQL* yy470; + tVariantList* yy498; + tFieldList* yy523; + SIntervalVal yy532; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif #define ParseARG_SDECL SSqlInfo* pInfo; #define ParseARG_PDECL ,SSqlInfo* pInfo -#define ParseARG_PARAM ,pInfo -#define ParseARG_FETCH SSqlInfo* pInfo=yypParser->pInfo; -#define ParseARG_STORE yypParser->pInfo=pInfo; -#define ParseCTX_SDECL -#define ParseCTX_PDECL -#define ParseCTX_PARAM -#define ParseCTX_FETCH -#define ParseCTX_STORE +#define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo +#define ParseARG_STORE yypParser->pInfo = pInfo +#define YYNSTATE 414 +#define YYNRULE 231 #define YYFALLBACK 1 -#define YYNSTATE 248 -#define YYNRULE 228 -#define YYNRULE_WITH_ACTION 228 -#define YYNTOKEN 206 -#define YY_MAX_SHIFT 247 -#define YY_MIN_SHIFTREDUCE 410 -#define YY_MAX_SHIFTREDUCE 637 -#define YY_ERROR_ACTION 638 -#define YY_ACCEPT_ACTION 639 -#define YY_NO_ACTION 640 -#define YY_MIN_REDUCE 641 -#define YY_MAX_REDUCE 868 -/************* End control #defines *******************************************/ -#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) +#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) +#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) +#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) + +/* The yyzerominor constant is used to initialize instances of +** YYMINORTYPE objects to zero. */ +static const YYMINORTYPE yyzerominor = { 0 }; /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -172,35 +127,33 @@ typedef union { ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** 0 <= N < YYNSTATE Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then -** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. ** -** N == YY_ERROR_ACTION A syntax error has occurred. +** N == YYNSTATE+YYNRULE A syntax error has occurred. ** -** N == YY_ACCEPT_ACTION The parser accepts its input. +** N == YYNSTATE+YYNRULE+1 The parser accepts its input. ** -** N == YY_NO_ACTION No such action. Denotes unused +** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused ** slots in the yy_action[] table. ** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE -** ** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as either: +** Given state S and lookahead X, the action is computed as ** -** (A) N = yy_action[ yy_shift_ofst[S] + X ] -** (B) N = yy_default[S] +** yy_action[ yy_shift_ofst[S] + X ] ** -** The (A) formula is preferred. The B formula is used instead if -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** If the index value yy_shift_ofst[S]+X is out of range or if the value +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] +** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table +** and that yy_default[S] should be used instead. ** -** The formulas above are for computing the action when the lookahead is +** The formula above is for computing the action when the lookahead is ** a terminal symbol. If the lookahead is a non-terminal (as occurs after ** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array. +** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of +** YY_SHIFT_USE_DFLT. ** ** The following are the tables generated in this section: ** @@ -212,226 +165,244 @@ typedef union { ** yy_reduce_ofst[] For each state, the offset into yy_action for ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. -** -*********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (560) +*/ +#define YY_ACTTAB_COUNT (665) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 741, 451, 11, 739, 740, 639, 247, 451, 742, 452, - /* 10 */ 744, 745, 743, 35, 36, 452, 37, 38, 156, 245, - /* 20 */ 167, 29, 138, 137, 202, 41, 39, 43, 40, 106, - /* 30 */ 517, 162, 856, 34, 33, 782, 138, 32, 31, 30, - /* 40 */ 35, 36, 771, 37, 38, 161, 856, 167, 29, 771, - /* 50 */ 106, 202, 41, 39, 43, 40, 187, 159, 223, 222, - /* 60 */ 34, 33, 138, 157, 32, 31, 30, 35, 36, 451, - /* 70 */ 37, 38, 855, 142, 167, 29, 760, 452, 202, 41, - /* 80 */ 39, 43, 40, 199, 78, 60, 779, 34, 33, 234, - /* 90 */ 234, 32, 31, 30, 21, 41, 39, 43, 40, 32, - /* 100 */ 31, 30, 56, 34, 33, 852, 808, 32, 31, 30, - /* 110 */ 21, 21, 106, 411, 412, 413, 414, 415, 416, 417, - /* 120 */ 418, 419, 420, 421, 422, 246, 591, 171, 36, 757, - /* 130 */ 37, 38, 225, 50, 167, 29, 21, 62, 202, 41, - /* 140 */ 39, 43, 40, 172, 221, 757, 757, 34, 33, 27, - /* 150 */ 51, 32, 31, 30, 8, 37, 38, 63, 116, 167, - /* 160 */ 29, 101, 758, 202, 41, 39, 43, 40, 809, 226, - /* 170 */ 197, 757, 34, 33, 170, 851, 32, 31, 30, 16, - /* 180 */ 214, 240, 239, 213, 212, 211, 238, 210, 237, 236, - /* 190 */ 235, 209, 737, 760, 725, 726, 727, 728, 729, 730, - /* 200 */ 731, 732, 733, 734, 735, 736, 166, 604, 12, 241, - /* 210 */ 595, 17, 598, 190, 601, 559, 166, 604, 26, 103, - /* 220 */ 595, 597, 598, 600, 601, 34, 33, 151, 760, 32, - /* 230 */ 31, 30, 21, 90, 89, 145, 572, 573, 163, 164, - /* 240 */ 173, 150, 201, 76, 80, 85, 88, 79, 163, 164, - /* 250 */ 166, 604, 549, 82, 595, 106, 598, 100, 601, 244, - /* 260 */ 243, 97, 17, 16, 26, 240, 239, 756, 680, 26, - /* 270 */ 238, 129, 237, 236, 235, 119, 120, 70, 66, 69, - /* 280 */ 203, 165, 163, 164, 689, 533, 180, 129, 530, 186, - /* 290 */ 531, 850, 532, 184, 183, 596, 153, 599, 133, 131, - /* 300 */ 93, 92, 91, 42, 174, 546, 681, 220, 219, 129, - /* 310 */ 18, 61, 541, 42, 603, 593, 175, 176, 563, 189, - /* 320 */ 3, 47, 46, 537, 603, 538, 564, 623, 605, 602, - /* 330 */ 14, 13, 13, 154, 523, 75, 74, 522, 46, 602, - /* 340 */ 48, 22, 207, 535, 155, 536, 22, 42, 10, 9, - /* 350 */ 140, 594, 87, 86, 141, 143, 144, 148, 603, 149, - /* 360 */ 147, 136, 146, 139, 865, 759, 819, 818, 168, 607, - /* 370 */ 815, 814, 169, 602, 751, 224, 781, 773, 786, 788, - /* 380 */ 102, 801, 117, 800, 115, 118, 26, 534, 188, 691, - /* 390 */ 208, 134, 24, 217, 688, 218, 864, 72, 863, 861, - /* 400 */ 121, 95, 709, 25, 23, 135, 678, 81, 558, 676, - /* 410 */ 83, 191, 84, 674, 158, 673, 195, 177, 130, 671, - /* 420 */ 52, 670, 770, 669, 49, 44, 668, 107, 108, 200, - /* 430 */ 667, 194, 659, 132, 665, 663, 198, 196, 192, 661, - /* 440 */ 28, 57, 58, 802, 216, 77, 227, 228, 229, 230, - /* 450 */ 231, 232, 205, 233, 242, 53, 637, 178, 179, 636, - /* 460 */ 152, 64, 67, 182, 181, 672, 635, 628, 94, 96, - /* 470 */ 185, 666, 124, 55, 123, 710, 122, 125, 126, 128, - /* 480 */ 127, 1, 2, 189, 755, 543, 59, 560, 111, 109, - /* 490 */ 112, 110, 104, 113, 114, 160, 19, 193, 5, 565, - /* 500 */ 105, 6, 606, 4, 20, 15, 204, 7, 608, 65, - /* 510 */ 206, 492, 488, 486, 485, 484, 481, 455, 215, 68, - /* 520 */ 45, 71, 22, 519, 73, 518, 516, 54, 476, 474, - /* 530 */ 466, 472, 468, 470, 464, 462, 491, 490, 489, 487, - /* 540 */ 483, 482, 46, 453, 426, 424, 641, 640, 640, 640, - /* 550 */ 640, 640, 640, 640, 640, 640, 640, 640, 98, 99, + /* 0 */ 398, 76, 80, 85, 88, 79, 341, 173, 397, 263, + /* 10 */ 180, 82, 35, 36, 18, 37, 38, 184, 183, 167, + /* 20 */ 29, 87, 86, 202, 41, 39, 43, 40, 10, 9, + /* 30 */ 646, 248, 34, 33, 223, 222, 32, 31, 30, 35, + /* 40 */ 36, 8, 37, 38, 63, 116, 167, 29, 101, 21, + /* 50 */ 202, 41, 39, 43, 40, 245, 244, 97, 414, 34, + /* 60 */ 33, 75, 74, 32, 31, 30, 35, 36, 256, 37, + /* 70 */ 38, 401, 174, 167, 29, 220, 219, 202, 41, 39, + /* 80 */ 43, 40, 187, 226, 207, 337, 34, 33, 22, 398, + /* 90 */ 32, 31, 30, 36, 307, 37, 38, 397, 22, 167, + /* 100 */ 29, 190, 56, 202, 41, 39, 43, 40, 32, 31, + /* 110 */ 30, 361, 34, 33, 363, 362, 32, 31, 30, 360, + /* 120 */ 306, 358, 357, 359, 46, 356, 297, 400, 133, 131, + /* 130 */ 93, 92, 91, 413, 412, 411, 410, 409, 408, 407, + /* 140 */ 406, 405, 404, 403, 402, 247, 375, 138, 374, 106, + /* 150 */ 37, 38, 50, 99, 167, 29, 161, 279, 202, 41, + /* 160 */ 39, 43, 40, 106, 373, 165, 372, 34, 33, 51, + /* 170 */ 203, 32, 31, 30, 311, 98, 323, 322, 321, 320, + /* 180 */ 319, 318, 317, 316, 315, 314, 313, 312, 310, 16, + /* 190 */ 214, 241, 240, 213, 212, 211, 239, 210, 238, 237, + /* 200 */ 236, 209, 235, 166, 285, 3, 270, 294, 197, 289, + /* 210 */ 293, 288, 292, 166, 285, 47, 78, 294, 199, 289, + /* 220 */ 60, 288, 234, 166, 285, 12, 291, 294, 290, 289, + /* 230 */ 46, 288, 21, 255, 48, 163, 164, 34, 33, 257, + /* 240 */ 189, 32, 31, 30, 151, 163, 164, 296, 370, 201, + /* 250 */ 90, 89, 145, 284, 392, 163, 164, 13, 150, 369, + /* 260 */ 299, 41, 39, 43, 40, 368, 221, 138, 337, 34, + /* 270 */ 33, 100, 21, 32, 31, 30, 162, 279, 26, 16, + /* 280 */ 17, 241, 240, 295, 275, 274, 239, 26, 238, 237, + /* 290 */ 236, 280, 235, 377, 21, 13, 380, 398, 379, 266, + /* 300 */ 378, 42, 106, 14, 21, 397, 172, 186, 337, 267, + /* 310 */ 106, 42, 286, 46, 153, 119, 120, 70, 66, 69, + /* 320 */ 225, 42, 286, 62, 175, 176, 242, 287, 171, 265, + /* 330 */ 337, 354, 286, 103, 129, 27, 331, 287, 355, 129, + /* 340 */ 343, 129, 256, 17, 138, 170, 371, 287, 159, 258, + /* 350 */ 26, 338, 156, 246, 278, 367, 157, 366, 365, 61, + /* 360 */ 364, 353, 352, 351, 350, 371, 349, 271, 371, 348, + /* 370 */ 347, 346, 22, 54, 340, 342, 339, 73, 71, 45, + /* 380 */ 68, 330, 215, 329, 328, 327, 326, 325, 65, 206, + /* 390 */ 204, 7, 324, 15, 4, 298, 376, 277, 6, 5, + /* 400 */ 105, 260, 20, 19, 268, 193, 160, 264, 104, 185, + /* 410 */ 55, 253, 252, 251, 59, 182, 181, 250, 179, 189, + /* 420 */ 249, 178, 2, 1, 96, 393, 243, 95, 94, 386, + /* 430 */ 232, 123, 127, 128, 305, 228, 230, 126, 231, 233, + /* 440 */ 124, 125, 229, 152, 122, 53, 67, 205, 216, 344, + /* 450 */ 64, 26, 114, 227, 28, 113, 77, 192, 198, 194, + /* 460 */ 44, 196, 273, 52, 49, 195, 276, 58, 57, 399, + /* 470 */ 396, 647, 395, 132, 394, 391, 390, 112, 389, 111, + /* 480 */ 388, 110, 387, 109, 130, 108, 269, 188, 200, 177, + /* 490 */ 107, 647, 385, 308, 234, 115, 304, 384, 84, 254, + /* 500 */ 224, 647, 647, 158, 191, 647, 647, 647, 647, 272, + /* 510 */ 647, 83, 647, 169, 383, 303, 647, 647, 647, 302, + /* 520 */ 168, 647, 301, 647, 81, 382, 135, 647, 23, 25, + /* 530 */ 345, 121, 336, 335, 72, 334, 218, 647, 332, 217, + /* 540 */ 24, 381, 134, 208, 309, 118, 117, 102, 262, 261, + /* 550 */ 259, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 560 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 570 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 580 */ 647, 647, 647, 647, 647, 647, 647, 300, 647, 647, + /* 590 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 600 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 610 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 620 */ 647, 647, 647, 333, 647, 647, 647, 647, 647, 647, + /* 630 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 139, + /* 640 */ 647, 647, 647, 647, 647, 647, 146, 647, 647, 136, + /* 650 */ 147, 149, 148, 144, 143, 141, 140, 155, 154, 283, + /* 660 */ 282, 281, 142, 137, 11, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 225, 1, 259, 228, 229, 206, 207, 1, 233, 9, - /* 10 */ 235, 236, 237, 13, 14, 9, 16, 17, 208, 209, - /* 20 */ 20, 21, 259, 259, 24, 25, 26, 27, 28, 209, - /* 30 */ 5, 268, 269, 33, 34, 209, 259, 37, 38, 39, - /* 40 */ 13, 14, 243, 16, 17, 268, 269, 20, 21, 243, - /* 50 */ 209, 24, 25, 26, 27, 28, 257, 226, 33, 34, - /* 60 */ 33, 34, 259, 257, 37, 38, 39, 13, 14, 1, - /* 70 */ 16, 17, 269, 259, 20, 21, 245, 9, 24, 25, - /* 80 */ 26, 27, 28, 263, 72, 265, 260, 33, 34, 78, - /* 90 */ 78, 37, 38, 39, 209, 25, 26, 27, 28, 37, - /* 100 */ 38, 39, 102, 33, 34, 259, 265, 37, 38, 39, - /* 110 */ 209, 209, 209, 45, 46, 47, 48, 49, 50, 51, - /* 120 */ 52, 53, 54, 55, 56, 57, 99, 242, 14, 244, - /* 130 */ 16, 17, 209, 103, 20, 21, 209, 246, 24, 25, - /* 140 */ 26, 27, 28, 242, 242, 244, 244, 33, 34, 258, - /* 150 */ 120, 37, 38, 39, 98, 16, 17, 101, 102, 20, - /* 160 */ 21, 209, 239, 24, 25, 26, 27, 28, 265, 242, - /* 170 */ 267, 244, 33, 34, 226, 259, 37, 38, 39, 85, - /* 180 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 190 */ 96, 97, 225, 245, 227, 228, 229, 230, 231, 232, - /* 200 */ 233, 234, 235, 236, 237, 238, 1, 2, 44, 226, - /* 210 */ 5, 98, 7, 261, 9, 99, 1, 2, 105, 103, - /* 220 */ 5, 5, 7, 7, 9, 33, 34, 63, 245, 37, - /* 230 */ 38, 39, 209, 69, 70, 71, 115, 116, 33, 34, - /* 240 */ 63, 77, 37, 64, 65, 66, 67, 68, 33, 34, - /* 250 */ 1, 2, 37, 74, 5, 209, 7, 98, 9, 60, - /* 260 */ 61, 62, 98, 85, 105, 87, 88, 244, 213, 105, - /* 270 */ 92, 216, 94, 95, 96, 64, 65, 66, 67, 68, - /* 280 */ 15, 59, 33, 34, 213, 2, 126, 216, 5, 125, - /* 290 */ 7, 259, 9, 133, 134, 5, 132, 7, 64, 65, - /* 300 */ 66, 67, 68, 98, 127, 103, 213, 130, 131, 216, - /* 310 */ 108, 265, 99, 98, 109, 1, 33, 34, 99, 106, - /* 320 */ 98, 103, 103, 5, 109, 7, 99, 99, 99, 124, - /* 330 */ 103, 103, 103, 259, 99, 128, 129, 99, 103, 124, - /* 340 */ 122, 103, 99, 5, 259, 7, 103, 98, 128, 129, - /* 350 */ 259, 37, 72, 73, 259, 259, 259, 259, 109, 259, - /* 360 */ 259, 259, 259, 259, 245, 245, 240, 240, 240, 104, - /* 370 */ 240, 240, 240, 124, 241, 240, 209, 243, 209, 209, - /* 380 */ 209, 266, 209, 266, 247, 209, 105, 104, 243, 209, - /* 390 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209, - /* 400 */ 209, 59, 209, 209, 209, 209, 209, 209, 109, 209, - /* 410 */ 209, 262, 209, 209, 262, 209, 262, 209, 209, 209, - /* 420 */ 119, 209, 256, 209, 121, 118, 209, 255, 254, 113, - /* 430 */ 209, 111, 209, 209, 209, 209, 117, 112, 110, 209, - /* 440 */ 123, 210, 210, 210, 75, 84, 83, 49, 80, 82, - /* 450 */ 53, 81, 210, 79, 75, 210, 5, 135, 5, 5, - /* 460 */ 210, 214, 214, 5, 135, 210, 5, 86, 211, 211, - /* 470 */ 126, 210, 218, 107, 222, 224, 223, 221, 219, 217, - /* 480 */ 220, 215, 212, 106, 243, 99, 103, 99, 251, 253, - /* 490 */ 250, 252, 98, 249, 248, 1, 103, 98, 114, 99, - /* 500 */ 98, 114, 99, 98, 103, 98, 100, 98, 104, 72, - /* 510 */ 100, 9, 5, 5, 5, 5, 5, 76, 15, 72, - /* 520 */ 16, 129, 103, 5, 129, 5, 99, 98, 5, 5, - /* 530 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 540 */ 5, 5, 103, 76, 59, 58, 0, 270, 270, 270, - /* 550 */ 270, 270, 270, 270, 270, 270, 270, 270, 21, 21, - /* 560 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 570 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 580 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 590 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 600 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 610 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 620 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 630 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 640 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 650 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 660 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 670 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 680 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 690 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 700 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 710 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 720 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 730 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 740 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 750 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 760 */ 270, 270, 270, 270, 270, 270, + /* 0 */ 1, 64, 65, 66, 67, 68, 5, 63, 9, 104, + /* 10 */ 127, 74, 13, 14, 109, 16, 17, 134, 135, 20, + /* 20 */ 21, 72, 73, 24, 25, 26, 27, 28, 129, 130, + /* 30 */ 208, 209, 33, 34, 33, 34, 37, 38, 39, 13, + /* 40 */ 14, 99, 16, 17, 102, 103, 20, 21, 211, 211, + /* 50 */ 24, 25, 26, 27, 28, 60, 61, 62, 0, 33, + /* 60 */ 34, 129, 130, 37, 38, 39, 13, 14, 246, 16, + /* 70 */ 17, 58, 128, 20, 21, 131, 132, 24, 25, 26, + /* 80 */ 27, 28, 260, 245, 100, 247, 33, 34, 104, 1, + /* 90 */ 37, 38, 39, 14, 100, 16, 17, 9, 104, 20, + /* 100 */ 21, 264, 103, 24, 25, 26, 27, 28, 37, 38, + /* 110 */ 39, 227, 33, 34, 230, 231, 37, 38, 39, 235, + /* 120 */ 100, 237, 238, 239, 104, 241, 100, 59, 64, 65, + /* 130 */ 66, 67, 68, 45, 46, 47, 48, 49, 50, 51, + /* 140 */ 52, 53, 54, 55, 56, 57, 5, 262, 7, 211, + /* 150 */ 16, 17, 104, 21, 20, 21, 271, 272, 24, 25, + /* 160 */ 26, 27, 28, 211, 5, 59, 7, 33, 34, 121, + /* 170 */ 15, 37, 38, 39, 227, 21, 229, 230, 231, 232, + /* 180 */ 233, 234, 235, 236, 237, 238, 239, 240, 241, 85, + /* 190 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 200 */ 96, 97, 98, 1, 2, 99, 268, 5, 270, 7, + /* 210 */ 5, 9, 7, 1, 2, 104, 72, 5, 266, 7, + /* 220 */ 268, 9, 78, 1, 2, 44, 5, 5, 7, 7, + /* 230 */ 104, 9, 211, 100, 123, 33, 34, 33, 34, 37, + /* 240 */ 107, 37, 38, 39, 63, 33, 34, 1, 5, 37, + /* 250 */ 69, 70, 71, 100, 76, 33, 34, 104, 77, 5, + /* 260 */ 105, 25, 26, 27, 28, 5, 245, 262, 247, 33, + /* 270 */ 34, 99, 211, 37, 38, 39, 271, 272, 106, 85, + /* 280 */ 99, 87, 88, 37, 116, 117, 92, 106, 94, 95, + /* 290 */ 96, 100, 98, 2, 211, 104, 5, 1, 7, 100, + /* 300 */ 9, 99, 211, 104, 211, 9, 245, 126, 247, 100, + /* 310 */ 211, 99, 110, 104, 133, 64, 65, 66, 67, 68, + /* 320 */ 211, 99, 110, 249, 33, 34, 228, 125, 245, 100, + /* 330 */ 247, 215, 110, 104, 218, 261, 215, 125, 215, 218, + /* 340 */ 247, 218, 246, 99, 262, 228, 248, 125, 228, 211, + /* 350 */ 106, 242, 210, 211, 272, 5, 260, 5, 5, 268, + /* 360 */ 5, 5, 5, 5, 5, 248, 5, 268, 248, 5, + /* 370 */ 5, 5, 104, 99, 5, 100, 5, 130, 130, 16, + /* 380 */ 72, 76, 15, 5, 5, 5, 5, 5, 72, 101, + /* 390 */ 101, 99, 9, 99, 99, 105, 105, 100, 115, 115, + /* 400 */ 99, 263, 104, 104, 100, 99, 1, 100, 99, 127, + /* 410 */ 108, 100, 86, 5, 104, 5, 136, 5, 5, 107, + /* 420 */ 5, 136, 214, 217, 213, 212, 75, 59, 213, 212, + /* 430 */ 81, 224, 222, 219, 246, 49, 82, 221, 53, 79, + /* 440 */ 220, 223, 80, 212, 225, 212, 216, 212, 75, 226, + /* 450 */ 216, 106, 251, 83, 124, 252, 84, 111, 118, 112, + /* 460 */ 119, 113, 212, 120, 122, 265, 269, 212, 212, 211, + /* 470 */ 211, 273, 211, 211, 211, 211, 211, 253, 211, 254, + /* 480 */ 211, 255, 211, 256, 211, 257, 110, 246, 114, 211, + /* 490 */ 258, 273, 211, 244, 78, 250, 259, 211, 211, 246, + /* 500 */ 243, 273, 273, 265, 265, 273, 273, 273, 273, 269, + /* 510 */ 273, 211, 273, 243, 211, 243, 273, 273, 273, 243, + /* 520 */ 243, 273, 243, 273, 211, 211, 211, 273, 211, 211, + /* 530 */ 211, 211, 211, 211, 211, 211, 211, 273, 211, 211, + /* 540 */ 211, 248, 211, 211, 211, 211, 211, 211, 211, 211, + /* 550 */ 211, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 570 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 580 */ 273, 273, 273, 273, 273, 273, 273, 243, 273, 273, + /* 590 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 600 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 610 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 620 */ 273, 273, 273, 248, 273, 273, 273, 273, 273, 273, + /* 630 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 262, + /* 640 */ 273, 273, 273, 273, 273, 273, 262, 273, 273, 262, + /* 650 */ 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, + /* 660 */ 262, 262, 262, 262, 262, }; -#define YY_SHIFT_COUNT (247) -#define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (546) -static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 164, 94, 178, 205, 249, 6, 6, 6, 6, 6, - /* 10 */ 6, 0, 68, 249, 283, 283, 283, 113, 6, 6, - /* 20 */ 6, 6, 6, 12, 11, 11, 560, 215, 249, 249, - /* 30 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 40 */ 249, 249, 249, 249, 249, 283, 283, 25, 25, 25, - /* 50 */ 25, 25, 25, 56, 25, 159, 6, 6, 6, 6, - /* 60 */ 121, 121, 202, 6, 6, 6, 6, 6, 6, 6, - /* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 100 */ 281, 342, 342, 299, 299, 299, 342, 301, 303, 307, - /* 110 */ 316, 319, 325, 320, 328, 317, 281, 342, 342, 369, - /* 120 */ 369, 342, 361, 363, 398, 368, 367, 397, 370, 374, - /* 130 */ 342, 379, 342, 379, 560, 560, 27, 54, 54, 54, - /* 140 */ 114, 139, 70, 70, 70, 179, 192, 192, 192, 192, - /* 150 */ 211, 234, 177, 160, 62, 62, 199, 213, 116, 219, - /* 160 */ 227, 228, 229, 216, 290, 314, 222, 265, 218, 30, - /* 170 */ 235, 238, 243, 207, 220, 318, 338, 280, 451, 322, - /* 180 */ 453, 454, 329, 458, 461, 381, 344, 377, 386, 366, - /* 190 */ 383, 388, 394, 494, 399, 400, 402, 393, 384, 401, - /* 200 */ 387, 403, 405, 404, 407, 406, 409, 410, 437, 502, - /* 210 */ 507, 508, 509, 510, 511, 441, 503, 447, 504, 392, - /* 220 */ 395, 419, 518, 520, 427, 429, 419, 523, 524, 525, - /* 230 */ 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, - /* 240 */ 536, 439, 467, 537, 538, 485, 487, 546, +#define YY_SHIFT_USE_DFLT (-118) +#define YY_SHIFT_COUNT (248) +#define YY_SHIFT_MIN (-117) +#define YY_SHIFT_MAX (416) +static const short yy_shift_ofst[] = { + /* 0 */ 181, 104, 194, 212, 222, 296, 296, 296, 296, 296, + /* 10 */ 296, -1, 88, 222, 291, 291, 291, 244, 296, 296, + /* 20 */ 296, 296, 296, 144, 416, 416, -118, 202, 222, 222, + /* 30 */ 222, 222, 222, 222, 222, 222, 222, 222, 222, 222, + /* 40 */ 222, 222, 222, 222, 222, 291, 291, 1, 1, 1, + /* 50 */ 1, 1, 1, -58, 1, 172, 296, 296, 296, 296, + /* 60 */ 168, 168, -95, 296, 296, 296, 296, 296, 296, 296, + /* 70 */ 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, + /* 80 */ 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, + /* 90 */ 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, + /* 100 */ 345, 368, 368, 376, 376, 376, 368, 343, 342, 341, + /* 110 */ 374, 340, 348, 347, 346, 330, 345, 368, 368, 373, + /* 120 */ 373, 368, 372, 370, 386, 362, 354, 385, 349, 360, + /* 130 */ 368, 351, 368, 351, -118, -118, 26, 53, 53, 53, + /* 140 */ 79, 134, 236, 236, 236, -63, 204, 204, 204, 204, + /* 150 */ 251, 64, -56, -117, 71, 71, -5, 133, 229, 209, + /* 160 */ 199, 191, 153, 221, 205, 246, 106, 155, 111, 48, + /* 170 */ 20, -6, -16, -68, -101, 159, 141, -51, 415, 285, + /* 180 */ 413, 412, 280, 410, 408, 326, 282, 312, 311, 302, + /* 190 */ 310, 307, 309, 405, 306, 304, 301, 299, 284, 298, + /* 200 */ 283, 297, 295, 290, 294, 289, 292, 288, 316, 383, + /* 210 */ 382, 381, 380, 379, 378, 305, 367, 308, 363, 248, + /* 220 */ 247, 268, 371, 369, 275, 274, 268, 366, 365, 364, + /* 230 */ 361, 359, 358, 357, 356, 355, 353, 352, 350, 260, + /* 240 */ 254, 243, 126, 178, 154, 132, 68, 13, 58, }; +#define YY_REDUCE_USE_DFLT (-179) #define YY_REDUCE_COUNT (135) -#define YY_REDUCE_MIN (-257) -#define YY_REDUCE_MAX (270) +#define YY_REDUCE_MIN (-178) +#define YY_REDUCE_MAX (402) static const short yy_reduce_ofst[] = { - /* 0 */ -201, -33, -225, -237, -223, -97, -180, -115, -99, -98, - /* 10 */ -73, -174, -190, -197, -169, -52, -17, -194, -48, -159, - /* 20 */ 46, -77, 23, 55, 71, 93, -109, -257, -236, -186, - /* 30 */ -154, -84, 32, 74, 85, 91, 95, 96, 97, 98, - /* 40 */ 100, 101, 102, 103, 104, 119, 120, 126, 127, 128, - /* 50 */ 130, 131, 132, 133, 135, 134, 167, 169, 170, 171, - /* 60 */ 115, 117, 137, 173, 176, 180, 181, 182, 183, 184, - /* 70 */ 185, 186, 187, 188, 189, 190, 191, 193, 194, 195, - /* 80 */ 196, 197, 198, 200, 201, 203, 204, 206, 208, 209, - /* 90 */ 210, 212, 214, 217, 221, 223, 224, 225, 226, 230, - /* 100 */ 145, 231, 232, 149, 152, 154, 233, 166, 172, 174, - /* 110 */ 236, 239, 237, 240, 244, 246, 241, 242, 245, 247, - /* 120 */ 248, 250, 251, 253, 252, 254, 256, 259, 260, 262, - /* 130 */ 255, 257, 261, 258, 266, 270, + /* 0 */ -178, -53, -116, 5, -115, -62, -48, 83, 61, 21, + /* 10 */ -162, 138, 142, 82, 120, 117, 98, 96, -163, 99, + /* 20 */ 91, 109, 93, 123, 121, 116, 74, 402, 401, 400, + /* 30 */ 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, + /* 40 */ 389, 388, 387, 384, 377, 375, 293, 344, 279, 277, + /* 50 */ 276, 272, 270, 249, 257, 253, 339, 338, 337, 336, + /* 60 */ 240, 197, 245, 335, 334, 333, 332, 331, 329, 328, + /* 70 */ 327, 325, 324, 323, 322, 321, 320, 319, 318, 317, + /* 80 */ 315, 314, 313, 303, 300, 287, 286, 281, 278, 273, + /* 90 */ 271, 269, 267, 265, 264, 263, 262, 261, 259, 258, + /* 100 */ 241, 256, 255, 239, 238, 200, 250, 237, 232, 228, + /* 110 */ 227, 226, 225, 224, 203, 201, 188, 235, 233, 234, + /* 120 */ 230, 231, 223, 219, 207, 220, 218, 216, 210, 214, + /* 130 */ 217, 215, 213, 211, 206, 208, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 638, 690, 679, 858, 858, 638, 638, 638, 638, 638, - /* 10 */ 638, 783, 656, 858, 638, 638, 638, 638, 638, 638, - /* 20 */ 638, 638, 638, 692, 692, 692, 778, 638, 638, 638, - /* 30 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 40 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 50 */ 638, 638, 638, 638, 638, 638, 638, 785, 787, 638, - /* 60 */ 805, 805, 776, 638, 638, 638, 638, 638, 638, 638, - /* 70 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 80 */ 638, 677, 638, 675, 638, 638, 638, 638, 638, 638, - /* 90 */ 638, 638, 638, 638, 638, 638, 638, 664, 638, 638, - /* 100 */ 638, 658, 658, 638, 638, 638, 658, 812, 816, 810, - /* 110 */ 798, 806, 797, 793, 792, 820, 638, 658, 658, 687, - /* 120 */ 687, 658, 708, 706, 704, 696, 702, 698, 700, 694, - /* 130 */ 658, 685, 658, 685, 724, 738, 638, 821, 857, 811, - /* 140 */ 847, 846, 853, 845, 844, 638, 840, 841, 843, 842, - /* 150 */ 638, 638, 638, 638, 849, 848, 638, 638, 638, 638, - /* 160 */ 638, 638, 638, 638, 638, 638, 823, 638, 817, 813, - /* 170 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 180 */ 638, 638, 638, 638, 638, 638, 638, 775, 638, 638, - /* 190 */ 784, 638, 638, 638, 638, 638, 638, 807, 638, 799, - /* 200 */ 638, 638, 638, 638, 638, 638, 638, 752, 638, 638, - /* 210 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 220 */ 638, 862, 638, 638, 638, 746, 860, 638, 638, 638, - /* 230 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 240 */ 638, 711, 638, 662, 660, 638, 654, 638, + /* 0 */ 645, 463, 452, 634, 634, 645, 645, 645, 645, 645, + /* 10 */ 645, 559, 429, 634, 645, 645, 645, 645, 645, 645, + /* 20 */ 645, 645, 645, 465, 465, 465, 554, 645, 645, 645, + /* 30 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 40 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 50 */ 645, 645, 645, 645, 645, 645, 645, 561, 563, 645, + /* 60 */ 581, 581, 552, 645, 645, 645, 645, 645, 645, 645, + /* 70 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 80 */ 645, 450, 645, 448, 645, 645, 645, 645, 645, 645, + /* 90 */ 645, 645, 645, 645, 645, 645, 645, 437, 645, 645, + /* 100 */ 645, 431, 431, 645, 645, 645, 431, 588, 592, 586, + /* 110 */ 574, 582, 573, 569, 568, 596, 645, 431, 431, 460, + /* 120 */ 460, 431, 481, 479, 477, 469, 475, 471, 473, 467, + /* 130 */ 431, 458, 431, 458, 498, 513, 645, 597, 633, 587, + /* 140 */ 623, 622, 629, 621, 620, 645, 616, 617, 619, 618, + /* 150 */ 645, 645, 645, 645, 625, 624, 645, 645, 645, 645, + /* 160 */ 645, 645, 645, 645, 645, 645, 599, 645, 593, 589, + /* 170 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 180 */ 645, 645, 645, 645, 645, 645, 645, 551, 645, 645, + /* 190 */ 560, 645, 645, 645, 645, 645, 645, 583, 645, 575, + /* 200 */ 645, 645, 645, 645, 645, 645, 645, 528, 645, 645, + /* 210 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 220 */ 645, 638, 645, 645, 645, 522, 636, 645, 645, 645, + /* 230 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 240 */ 645, 645, 484, 645, 435, 433, 645, 427, 645, 644, + /* 250 */ 643, 642, 635, 550, 549, 548, 547, 556, 558, 557, + /* 260 */ 555, 562, 564, 553, 567, 566, 571, 570, 572, 565, + /* 270 */ 585, 584, 577, 578, 580, 579, 576, 613, 631, 632, + /* 280 */ 630, 628, 627, 626, 612, 611, 610, 609, 608, 605, + /* 290 */ 607, 604, 606, 603, 602, 601, 600, 598, 615, 614, + /* 300 */ 595, 594, 591, 590, 546, 531, 530, 529, 527, 464, + /* 310 */ 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, + /* 320 */ 502, 501, 500, 499, 496, 492, 490, 489, 488, 485, + /* 330 */ 459, 462, 461, 641, 640, 639, 637, 533, 534, 526, + /* 340 */ 525, 524, 523, 532, 483, 482, 480, 478, 470, 476, + /* 350 */ 472, 474, 468, 466, 454, 453, 521, 520, 519, 518, + /* 360 */ 517, 516, 515, 514, 497, 495, 494, 493, 491, 487, + /* 370 */ 486, 536, 545, 544, 543, 542, 541, 540, 539, 538, + /* 380 */ 537, 535, 451, 449, 447, 446, 445, 444, 443, 442, + /* 390 */ 441, 440, 457, 439, 432, 438, 436, 456, 455, 434, + /* 400 */ 430, 428, 426, 425, 424, 423, 422, 421, 420, 419, + /* 410 */ 418, 417, 416, 415, }; -/********** End of lemon-generated parsing tables *****************************/ -/* The next table maps tokens (terminal symbols) into fallback tokens. -** If a construct like the following: +/* The next table maps tokens into fallback tokens. If a construct +** like the following: ** ** %fallback ID X Y Z. ** @@ -439,10 +410,6 @@ static const YYACTIONTYPE yy_default[] = { ** and Z. Whenever one of the tokens X, Y, or Z is input to the parser ** but it does not parse, the type of the token is changed to ID and ** the parse is retried before an error is thrown. -** -** This feature can be used, for example, to cause some keywords in a language -** to revert to identifiers if they keyword does not apply in the context where -** it appears. */ #ifdef YYFALLBACK static const YYCODETYPE yyFallback[] = { @@ -544,6 +511,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* FSYNC => nothing */ 0, /* COMP => nothing */ 0, /* PRECISION => nothing */ + 0, /* UPDATE => nothing */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* TAGS => nothing */ @@ -666,13 +634,9 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. -** -** After the "shift" half of a SHIFTREDUCE action, the stateno field -** actually contains the reduce action for the second half of the -** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYACTIONTYPE stateno; /* The state-number */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -683,22 +647,17 @@ typedef struct yyStackEntry yyStackEntry; /* The state of the parser is completely contained in an instance of ** the following structure */ struct yyParser { - yyStackEntry *yytos; /* Pointer to top element of the stack */ + int yyidx; /* Index of top element in stack */ #ifdef YYTRACKMAXSTACKDEPTH - int yyhwm; /* High-water mark of the stack */ + int yyidxMax; /* Maximum value of yyidx */ #endif -#ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ -#endif ParseARG_SDECL /* A place to hold %extra_argument */ - ParseCTX_SDECL /* A place to hold %extra_context */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ #else yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ #endif }; typedef struct yyParser yyParser; @@ -735,282 +694,81 @@ void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ } #endif /* NDEBUG */ -#if defined(YYCOVERAGE) || !defined(NDEBUG) +#ifndef NDEBUG /* For tracing shifts, the names of all terminals and nonterminals ** are required. The following table supplies these names */ static const char *const yyTokenName[] = { - /* 0 */ "$", - /* 1 */ "ID", - /* 2 */ "BOOL", - /* 3 */ "TINYINT", - /* 4 */ "SMALLINT", - /* 5 */ "INTEGER", - /* 6 */ "BIGINT", - /* 7 */ "FLOAT", - /* 8 */ "DOUBLE", - /* 9 */ "STRING", - /* 10 */ "TIMESTAMP", - /* 11 */ "BINARY", - /* 12 */ "NCHAR", - /* 13 */ "OR", - /* 14 */ "AND", - /* 15 */ "NOT", - /* 16 */ "EQ", - /* 17 */ "NE", - /* 18 */ "ISNULL", - /* 19 */ "NOTNULL", - /* 20 */ "IS", - /* 21 */ "LIKE", - /* 22 */ "GLOB", - /* 23 */ "BETWEEN", - /* 24 */ "IN", - /* 25 */ "GT", - /* 26 */ "GE", - /* 27 */ "LT", - /* 28 */ "LE", - /* 29 */ "BITAND", - /* 30 */ "BITOR", - /* 31 */ "LSHIFT", - /* 32 */ "RSHIFT", - /* 33 */ "PLUS", - /* 34 */ "MINUS", - /* 35 */ "DIVIDE", - /* 36 */ "TIMES", - /* 37 */ "STAR", - /* 38 */ "SLASH", - /* 39 */ "REM", - /* 40 */ "CONCAT", - /* 41 */ "UMINUS", - /* 42 */ "UPLUS", - /* 43 */ "BITNOT", - /* 44 */ "SHOW", - /* 45 */ "DATABASES", - /* 46 */ "MNODES", - /* 47 */ "DNODES", - /* 48 */ "ACCOUNTS", - /* 49 */ "USERS", - /* 50 */ "MODULES", - /* 51 */ "QUERIES", - /* 52 */ "CONNECTIONS", - /* 53 */ "STREAMS", - /* 54 */ "VARIABLES", - /* 55 */ "SCORES", - /* 56 */ "GRANTS", - /* 57 */ "VNODES", - /* 58 */ "IPTOKEN", - /* 59 */ "DOT", - /* 60 */ "TABLES", - /* 61 */ "STABLES", - /* 62 */ "VGROUPS", - /* 63 */ "DROP", - /* 64 */ "TABLE", - /* 65 */ "DATABASE", - /* 66 */ "DNODE", - /* 67 */ "USER", - /* 68 */ "ACCOUNT", - /* 69 */ "USE", - /* 70 */ "DESCRIBE", - /* 71 */ "ALTER", - /* 72 */ "PASS", - /* 73 */ "PRIVILEGE", - /* 74 */ "LOCAL", - /* 75 */ "IF", - /* 76 */ "EXISTS", - /* 77 */ "CREATE", - /* 78 */ "PPS", - /* 79 */ "TSERIES", - /* 80 */ "DBS", - /* 81 */ "STORAGE", - /* 82 */ "QTIME", - /* 83 */ "CONNS", - /* 84 */ "STATE", - /* 85 */ "KEEP", - /* 86 */ "CACHE", - /* 87 */ "REPLICA", - /* 88 */ "QUORUM", - /* 89 */ "DAYS", - /* 90 */ "MINROWS", - /* 91 */ "MAXROWS", - /* 92 */ "BLOCKS", - /* 93 */ "CTIME", - /* 94 */ "WAL", - /* 95 */ "FSYNC", - /* 96 */ "COMP", - /* 97 */ "PRECISION", - /* 98 */ "LP", - /* 99 */ "RP", - /* 100 */ "TAGS", - /* 101 */ "USING", - /* 102 */ "AS", - /* 103 */ "COMMA", - /* 104 */ "NULL", - /* 105 */ "SELECT", - /* 106 */ "UNION", - /* 107 */ "ALL", - /* 108 */ "FROM", - /* 109 */ "VARIABLE", - /* 110 */ "INTERVAL", - /* 111 */ "FILL", - /* 112 */ "SLIDING", - /* 113 */ "ORDER", - /* 114 */ "BY", - /* 115 */ "ASC", - /* 116 */ "DESC", - /* 117 */ "GROUP", - /* 118 */ "HAVING", - /* 119 */ "LIMIT", - /* 120 */ "OFFSET", - /* 121 */ "SLIMIT", - /* 122 */ "SOFFSET", - /* 123 */ "WHERE", - /* 124 */ "NOW", - /* 125 */ "RESET", - /* 126 */ "QUERY", - /* 127 */ "ADD", - /* 128 */ "COLUMN", - /* 129 */ "TAG", - /* 130 */ "CHANGE", - /* 131 */ "SET", - /* 132 */ "KILL", - /* 133 */ "CONNECTION", - /* 134 */ "STREAM", - /* 135 */ "COLON", - /* 136 */ "ABORT", - /* 137 */ "AFTER", - /* 138 */ "ATTACH", - /* 139 */ "BEFORE", - /* 140 */ "BEGIN", - /* 141 */ "CASCADE", - /* 142 */ "CLUSTER", - /* 143 */ "CONFLICT", - /* 144 */ "COPY", - /* 145 */ "DEFERRED", - /* 146 */ "DELIMITERS", - /* 147 */ "DETACH", - /* 148 */ "EACH", - /* 149 */ "END", - /* 150 */ "EXPLAIN", - /* 151 */ "FAIL", - /* 152 */ "FOR", - /* 153 */ "IGNORE", - /* 154 */ "IMMEDIATE", - /* 155 */ "INITIALLY", - /* 156 */ "INSTEAD", - /* 157 */ "MATCH", - /* 158 */ "KEY", - /* 159 */ "OF", - /* 160 */ "RAISE", - /* 161 */ "REPLACE", - /* 162 */ "RESTRICT", - /* 163 */ "ROW", - /* 164 */ "STATEMENT", - /* 165 */ "TRIGGER", - /* 166 */ "VIEW", - /* 167 */ "COUNT", - /* 168 */ "SUM", - /* 169 */ "AVG", - /* 170 */ "MIN", - /* 171 */ "MAX", - /* 172 */ "FIRST", - /* 173 */ "LAST", - /* 174 */ "TOP", - /* 175 */ "BOTTOM", - /* 176 */ "STDDEV", - /* 177 */ "PERCENTILE", - /* 178 */ "APERCENTILE", - /* 179 */ "LEASTSQUARES", - /* 180 */ "HISTOGRAM", - /* 181 */ "DIFF", - /* 182 */ "SPREAD", - /* 183 */ "TWA", - /* 184 */ "INTERP", - /* 185 */ "LAST_ROW", - /* 186 */ "RATE", - /* 187 */ "IRATE", - /* 188 */ "SUM_RATE", - /* 189 */ "SUM_IRATE", - /* 190 */ "AVG_RATE", - /* 191 */ "AVG_IRATE", - /* 192 */ "TBID", - /* 193 */ "SEMI", - /* 194 */ "NONE", - /* 195 */ "PREV", - /* 196 */ "LINEAR", - /* 197 */ "IMPORT", - /* 198 */ "METRIC", - /* 199 */ "TBNAME", - /* 200 */ "JOIN", - /* 201 */ "METRICS", - /* 202 */ "STABLE", - /* 203 */ "INSERT", - /* 204 */ "INTO", - /* 205 */ "VALUES", - /* 206 */ "program", - /* 207 */ "cmd", - /* 208 */ "dbPrefix", - /* 209 */ "ids", - /* 210 */ "cpxName", - /* 211 */ "ifexists", - /* 212 */ "alter_db_optr", - /* 213 */ "acct_optr", - /* 214 */ "ifnotexists", - /* 215 */ "db_optr", - /* 216 */ "pps", - /* 217 */ "tseries", - /* 218 */ "dbs", - /* 219 */ "streams", - /* 220 */ "storage", - /* 221 */ "qtime", - /* 222 */ "users", - /* 223 */ "conns", - /* 224 */ "state", - /* 225 */ "keep", - /* 226 */ "tagitemlist", - /* 227 */ "cache", - /* 228 */ "replica", - /* 229 */ "quorum", - /* 230 */ "days", - /* 231 */ "minrows", - /* 232 */ "maxrows", - /* 233 */ "blocks", - /* 234 */ "ctime", - /* 235 */ "wal", - /* 236 */ "fsync", - /* 237 */ "comp", - /* 238 */ "prec", - /* 239 */ "typename", - /* 240 */ "signed", - /* 241 */ "create_table_args", - /* 242 */ "columnlist", - /* 243 */ "select", - /* 244 */ "column", - /* 245 */ "tagitem", - /* 246 */ "selcollist", - /* 247 */ "from", - /* 248 */ "where_opt", - /* 249 */ "interval_opt", - /* 250 */ "fill_opt", - /* 251 */ "sliding_opt", - /* 252 */ "groupby_opt", - /* 253 */ "orderby_opt", - /* 254 */ "having_opt", - /* 255 */ "slimit_opt", - /* 256 */ "limit_opt", - /* 257 */ "union", - /* 258 */ "sclp", - /* 259 */ "expr", - /* 260 */ "as", - /* 261 */ "tablelist", - /* 262 */ "tmvar", - /* 263 */ "sortlist", - /* 264 */ "sortitem", - /* 265 */ "item", - /* 266 */ "sortorder", - /* 267 */ "grouplist", - /* 268 */ "exprlist", - /* 269 */ "expritem", + "$", "ID", "BOOL", "TINYINT", + "SMALLINT", "INTEGER", "BIGINT", "FLOAT", + "DOUBLE", "STRING", "TIMESTAMP", "BINARY", + "NCHAR", "OR", "AND", "NOT", + "EQ", "NE", "ISNULL", "NOTNULL", + "IS", "LIKE", "GLOB", "BETWEEN", + "IN", "GT", "GE", "LT", + "LE", "BITAND", "BITOR", "LSHIFT", + "RSHIFT", "PLUS", "MINUS", "DIVIDE", + "TIMES", "STAR", "SLASH", "REM", + "CONCAT", "UMINUS", "UPLUS", "BITNOT", + "SHOW", "DATABASES", "MNODES", "DNODES", + "ACCOUNTS", "USERS", "MODULES", "QUERIES", + "CONNECTIONS", "STREAMS", "VARIABLES", "SCORES", + "GRANTS", "VNODES", "IPTOKEN", "DOT", + "TABLES", "STABLES", "VGROUPS", "DROP", + "TABLE", "DATABASE", "DNODE", "USER", + "ACCOUNT", "USE", "DESCRIBE", "ALTER", + "PASS", "PRIVILEGE", "LOCAL", "IF", + "EXISTS", "CREATE", "PPS", "TSERIES", + "DBS", "STORAGE", "QTIME", "CONNS", + "STATE", "KEEP", "CACHE", "REPLICA", + "QUORUM", "DAYS", "MINROWS", "MAXROWS", + "BLOCKS", "CTIME", "WAL", "FSYNC", + "COMP", "PRECISION", "UPDATE", "LP", + "RP", "TAGS", "USING", "AS", + "COMMA", "NULL", "SELECT", "UNION", + "ALL", "FROM", "VARIABLE", "INTERVAL", + "FILL", "SLIDING", "ORDER", "BY", + "ASC", "DESC", "GROUP", "HAVING", + "LIMIT", "OFFSET", "SLIMIT", "SOFFSET", + "WHERE", "NOW", "RESET", "QUERY", + "ADD", "COLUMN", "TAG", "CHANGE", + "SET", "KILL", "CONNECTION", "STREAM", + "COLON", "ABORT", "AFTER", "ATTACH", + "BEFORE", "BEGIN", "CASCADE", "CLUSTER", + "CONFLICT", "COPY", "DEFERRED", "DELIMITERS", + "DETACH", "EACH", "END", "EXPLAIN", + "FAIL", "FOR", "IGNORE", "IMMEDIATE", + "INITIALLY", "INSTEAD", "MATCH", "KEY", + "OF", "RAISE", "REPLACE", "RESTRICT", + "ROW", "STATEMENT", "TRIGGER", "VIEW", + "COUNT", "SUM", "AVG", "MIN", + "MAX", "FIRST", "LAST", "TOP", + "BOTTOM", "STDDEV", "PERCENTILE", "APERCENTILE", + "LEASTSQUARES", "HISTOGRAM", "DIFF", "SPREAD", + "TWA", "INTERP", "LAST_ROW", "RATE", + "IRATE", "SUM_RATE", "SUM_IRATE", "AVG_RATE", + "AVG_IRATE", "TBID", "SEMI", "NONE", + "PREV", "LINEAR", "IMPORT", "METRIC", + "TBNAME", "JOIN", "METRICS", "STABLE", + "INSERT", "INTO", "VALUES", "error", + "program", "cmd", "dbPrefix", "ids", + "cpxName", "ifexists", "alter_db_optr", "acct_optr", + "ifnotexists", "db_optr", "pps", "tseries", + "dbs", "streams", "storage", "qtime", + "users", "conns", "state", "keep", + "tagitemlist", "cache", "replica", "quorum", + "days", "minrows", "maxrows", "blocks", + "ctime", "wal", "fsync", "comp", + "prec", "update", "typename", "signed", + "create_table_args", "columnlist", "select", "column", + "tagitem", "selcollist", "from", "where_opt", + "interval_opt", "fill_opt", "sliding_opt", "groupby_opt", + "orderby_opt", "having_opt", "slimit_opt", "limit_opt", + "union", "sclp", "expr", "as", + "tablelist", "tmvar", "sortlist", "sortitem", + "item", "sortorder", "grouplist", "exprlist", + "expritem", }; -#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ +#endif /* NDEBUG */ #ifndef NDEBUG /* For tracing reduce actions, the names of all rules are required. @@ -1099,226 +857,181 @@ static const char *const yyRuleName[] = { /* 80 */ "fsync ::= FSYNC INTEGER", /* 81 */ "comp ::= COMP INTEGER", /* 82 */ "prec ::= PRECISION STRING", - /* 83 */ "db_optr ::=", - /* 84 */ "db_optr ::= db_optr cache", - /* 85 */ "db_optr ::= db_optr replica", - /* 86 */ "db_optr ::= db_optr quorum", - /* 87 */ "db_optr ::= db_optr days", - /* 88 */ "db_optr ::= db_optr minrows", - /* 89 */ "db_optr ::= db_optr maxrows", - /* 90 */ "db_optr ::= db_optr blocks", - /* 91 */ "db_optr ::= db_optr ctime", - /* 92 */ "db_optr ::= db_optr wal", - /* 93 */ "db_optr ::= db_optr fsync", - /* 94 */ "db_optr ::= db_optr comp", - /* 95 */ "db_optr ::= db_optr prec", - /* 96 */ "db_optr ::= db_optr keep", - /* 97 */ "alter_db_optr ::=", - /* 98 */ "alter_db_optr ::= alter_db_optr replica", - /* 99 */ "alter_db_optr ::= alter_db_optr quorum", - /* 100 */ "alter_db_optr ::= alter_db_optr keep", - /* 101 */ "alter_db_optr ::= alter_db_optr blocks", - /* 102 */ "alter_db_optr ::= alter_db_optr comp", - /* 103 */ "alter_db_optr ::= alter_db_optr wal", - /* 104 */ "alter_db_optr ::= alter_db_optr fsync", - /* 105 */ "typename ::= ids", - /* 106 */ "typename ::= ids LP signed RP", - /* 107 */ "signed ::= INTEGER", - /* 108 */ "signed ::= PLUS INTEGER", - /* 109 */ "signed ::= MINUS INTEGER", - /* 110 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", - /* 111 */ "create_table_args ::= LP columnlist RP", - /* 112 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", - /* 113 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", - /* 114 */ "create_table_args ::= AS select", - /* 115 */ "columnlist ::= columnlist COMMA column", - /* 116 */ "columnlist ::= column", - /* 117 */ "column ::= ids typename", - /* 118 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 119 */ "tagitemlist ::= tagitem", - /* 120 */ "tagitem ::= INTEGER", - /* 121 */ "tagitem ::= FLOAT", - /* 122 */ "tagitem ::= STRING", - /* 123 */ "tagitem ::= BOOL", - /* 124 */ "tagitem ::= NULL", - /* 125 */ "tagitem ::= MINUS INTEGER", - /* 126 */ "tagitem ::= MINUS FLOAT", - /* 127 */ "tagitem ::= PLUS INTEGER", - /* 128 */ "tagitem ::= PLUS FLOAT", - /* 129 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", - /* 130 */ "union ::= select", - /* 131 */ "union ::= LP union RP", - /* 132 */ "union ::= union UNION ALL select", - /* 133 */ "union ::= union UNION ALL LP select RP", - /* 134 */ "cmd ::= union", - /* 135 */ "select ::= SELECT selcollist", - /* 136 */ "sclp ::= selcollist COMMA", - /* 137 */ "sclp ::=", - /* 138 */ "selcollist ::= sclp expr as", - /* 139 */ "selcollist ::= sclp STAR", - /* 140 */ "as ::= AS ids", - /* 141 */ "as ::= ids", - /* 142 */ "as ::=", - /* 143 */ "from ::= FROM tablelist", - /* 144 */ "tablelist ::= ids cpxName", - /* 145 */ "tablelist ::= ids cpxName ids", - /* 146 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 147 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 148 */ "tmvar ::= VARIABLE", - /* 149 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 150 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", - /* 151 */ "interval_opt ::=", - /* 152 */ "fill_opt ::=", - /* 153 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 154 */ "fill_opt ::= FILL LP ID RP", - /* 155 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 156 */ "sliding_opt ::=", - /* 157 */ "orderby_opt ::=", - /* 158 */ "orderby_opt ::= ORDER BY sortlist", - /* 159 */ "sortlist ::= sortlist COMMA item sortorder", - /* 160 */ "sortlist ::= item sortorder", - /* 161 */ "item ::= ids cpxName", - /* 162 */ "sortorder ::= ASC", - /* 163 */ "sortorder ::= DESC", - /* 164 */ "sortorder ::=", - /* 165 */ "groupby_opt ::=", - /* 166 */ "groupby_opt ::= GROUP BY grouplist", - /* 167 */ "grouplist ::= grouplist COMMA item", - /* 168 */ "grouplist ::= item", - /* 169 */ "having_opt ::=", - /* 170 */ "having_opt ::= HAVING expr", - /* 171 */ "limit_opt ::=", - /* 172 */ "limit_opt ::= LIMIT signed", - /* 173 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 174 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 175 */ "slimit_opt ::=", - /* 176 */ "slimit_opt ::= SLIMIT signed", - /* 177 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 178 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 179 */ "where_opt ::=", - /* 180 */ "where_opt ::= WHERE expr", - /* 181 */ "expr ::= LP expr RP", - /* 182 */ "expr ::= ID", - /* 183 */ "expr ::= ID DOT ID", - /* 184 */ "expr ::= ID DOT STAR", - /* 185 */ "expr ::= INTEGER", - /* 186 */ "expr ::= MINUS INTEGER", - /* 187 */ "expr ::= PLUS INTEGER", - /* 188 */ "expr ::= FLOAT", - /* 189 */ "expr ::= MINUS FLOAT", - /* 190 */ "expr ::= PLUS FLOAT", - /* 191 */ "expr ::= STRING", - /* 192 */ "expr ::= NOW", - /* 193 */ "expr ::= VARIABLE", - /* 194 */ "expr ::= BOOL", - /* 195 */ "expr ::= ID LP exprlist RP", - /* 196 */ "expr ::= ID LP STAR RP", - /* 197 */ "expr ::= expr IS NULL", - /* 198 */ "expr ::= expr IS NOT NULL", - /* 199 */ "expr ::= expr LT expr", - /* 200 */ "expr ::= expr GT expr", - /* 201 */ "expr ::= expr LE expr", - /* 202 */ "expr ::= expr GE expr", - /* 203 */ "expr ::= expr NE expr", - /* 204 */ "expr ::= expr EQ expr", - /* 205 */ "expr ::= expr AND expr", - /* 206 */ "expr ::= expr OR expr", - /* 207 */ "expr ::= expr PLUS expr", - /* 208 */ "expr ::= expr MINUS expr", - /* 209 */ "expr ::= expr STAR expr", - /* 210 */ "expr ::= expr SLASH expr", - /* 211 */ "expr ::= expr REM expr", - /* 212 */ "expr ::= expr LIKE expr", - /* 213 */ "expr ::= expr IN LP exprlist RP", - /* 214 */ "exprlist ::= exprlist COMMA expritem", - /* 215 */ "exprlist ::= expritem", - /* 216 */ "expritem ::= expr", - /* 217 */ "expritem ::=", - /* 218 */ "cmd ::= RESET QUERY CACHE", - /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 221 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 222 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 223 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 224 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 225 */ "cmd ::= KILL CONNECTION INTEGER", - /* 226 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 227 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 83 */ "update ::= UPDATE INTEGER", + /* 84 */ "db_optr ::=", + /* 85 */ "db_optr ::= db_optr cache", + /* 86 */ "db_optr ::= db_optr replica", + /* 87 */ "db_optr ::= db_optr quorum", + /* 88 */ "db_optr ::= db_optr days", + /* 89 */ "db_optr ::= db_optr minrows", + /* 90 */ "db_optr ::= db_optr maxrows", + /* 91 */ "db_optr ::= db_optr blocks", + /* 92 */ "db_optr ::= db_optr ctime", + /* 93 */ "db_optr ::= db_optr wal", + /* 94 */ "db_optr ::= db_optr fsync", + /* 95 */ "db_optr ::= db_optr comp", + /* 96 */ "db_optr ::= db_optr prec", + /* 97 */ "db_optr ::= db_optr keep", + /* 98 */ "db_optr ::= db_optr update", + /* 99 */ "alter_db_optr ::=", + /* 100 */ "alter_db_optr ::= alter_db_optr replica", + /* 101 */ "alter_db_optr ::= alter_db_optr quorum", + /* 102 */ "alter_db_optr ::= alter_db_optr keep", + /* 103 */ "alter_db_optr ::= alter_db_optr blocks", + /* 104 */ "alter_db_optr ::= alter_db_optr comp", + /* 105 */ "alter_db_optr ::= alter_db_optr wal", + /* 106 */ "alter_db_optr ::= alter_db_optr fsync", + /* 107 */ "alter_db_optr ::= alter_db_optr update", + /* 108 */ "typename ::= ids", + /* 109 */ "typename ::= ids LP signed RP", + /* 110 */ "signed ::= INTEGER", + /* 111 */ "signed ::= PLUS INTEGER", + /* 112 */ "signed ::= MINUS INTEGER", + /* 113 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", + /* 114 */ "create_table_args ::= LP columnlist RP", + /* 115 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", + /* 116 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", + /* 117 */ "create_table_args ::= AS select", + /* 118 */ "columnlist ::= columnlist COMMA column", + /* 119 */ "columnlist ::= column", + /* 120 */ "column ::= ids typename", + /* 121 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 122 */ "tagitemlist ::= tagitem", + /* 123 */ "tagitem ::= INTEGER", + /* 124 */ "tagitem ::= FLOAT", + /* 125 */ "tagitem ::= STRING", + /* 126 */ "tagitem ::= BOOL", + /* 127 */ "tagitem ::= NULL", + /* 128 */ "tagitem ::= MINUS INTEGER", + /* 129 */ "tagitem ::= MINUS FLOAT", + /* 130 */ "tagitem ::= PLUS INTEGER", + /* 131 */ "tagitem ::= PLUS FLOAT", + /* 132 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 133 */ "union ::= select", + /* 134 */ "union ::= LP union RP", + /* 135 */ "union ::= union UNION ALL select", + /* 136 */ "union ::= union UNION ALL LP select RP", + /* 137 */ "cmd ::= union", + /* 138 */ "select ::= SELECT selcollist", + /* 139 */ "sclp ::= selcollist COMMA", + /* 140 */ "sclp ::=", + /* 141 */ "selcollist ::= sclp expr as", + /* 142 */ "selcollist ::= sclp STAR", + /* 143 */ "as ::= AS ids", + /* 144 */ "as ::= ids", + /* 145 */ "as ::=", + /* 146 */ "from ::= FROM tablelist", + /* 147 */ "tablelist ::= ids cpxName", + /* 148 */ "tablelist ::= ids cpxName ids", + /* 149 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 150 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 151 */ "tmvar ::= VARIABLE", + /* 152 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 153 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 154 */ "interval_opt ::=", + /* 155 */ "fill_opt ::=", + /* 156 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 157 */ "fill_opt ::= FILL LP ID RP", + /* 158 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 159 */ "sliding_opt ::=", + /* 160 */ "orderby_opt ::=", + /* 161 */ "orderby_opt ::= ORDER BY sortlist", + /* 162 */ "sortlist ::= sortlist COMMA item sortorder", + /* 163 */ "sortlist ::= item sortorder", + /* 164 */ "item ::= ids cpxName", + /* 165 */ "sortorder ::= ASC", + /* 166 */ "sortorder ::= DESC", + /* 167 */ "sortorder ::=", + /* 168 */ "groupby_opt ::=", + /* 169 */ "groupby_opt ::= GROUP BY grouplist", + /* 170 */ "grouplist ::= grouplist COMMA item", + /* 171 */ "grouplist ::= item", + /* 172 */ "having_opt ::=", + /* 173 */ "having_opt ::= HAVING expr", + /* 174 */ "limit_opt ::=", + /* 175 */ "limit_opt ::= LIMIT signed", + /* 176 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 177 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 178 */ "slimit_opt ::=", + /* 179 */ "slimit_opt ::= SLIMIT signed", + /* 180 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 181 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 182 */ "where_opt ::=", + /* 183 */ "where_opt ::= WHERE expr", + /* 184 */ "expr ::= LP expr RP", + /* 185 */ "expr ::= ID", + /* 186 */ "expr ::= ID DOT ID", + /* 187 */ "expr ::= ID DOT STAR", + /* 188 */ "expr ::= INTEGER", + /* 189 */ "expr ::= MINUS INTEGER", + /* 190 */ "expr ::= PLUS INTEGER", + /* 191 */ "expr ::= FLOAT", + /* 192 */ "expr ::= MINUS FLOAT", + /* 193 */ "expr ::= PLUS FLOAT", + /* 194 */ "expr ::= STRING", + /* 195 */ "expr ::= NOW", + /* 196 */ "expr ::= VARIABLE", + /* 197 */ "expr ::= BOOL", + /* 198 */ "expr ::= ID LP exprlist RP", + /* 199 */ "expr ::= ID LP STAR RP", + /* 200 */ "expr ::= expr IS NULL", + /* 201 */ "expr ::= expr IS NOT NULL", + /* 202 */ "expr ::= expr LT expr", + /* 203 */ "expr ::= expr GT expr", + /* 204 */ "expr ::= expr LE expr", + /* 205 */ "expr ::= expr GE expr", + /* 206 */ "expr ::= expr NE expr", + /* 207 */ "expr ::= expr EQ expr", + /* 208 */ "expr ::= expr AND expr", + /* 209 */ "expr ::= expr OR expr", + /* 210 */ "expr ::= expr PLUS expr", + /* 211 */ "expr ::= expr MINUS expr", + /* 212 */ "expr ::= expr STAR expr", + /* 213 */ "expr ::= expr SLASH expr", + /* 214 */ "expr ::= expr REM expr", + /* 215 */ "expr ::= expr LIKE expr", + /* 216 */ "expr ::= expr IN LP exprlist RP", + /* 217 */ "exprlist ::= exprlist COMMA expritem", + /* 218 */ "exprlist ::= expritem", + /* 219 */ "expritem ::= expr", + /* 220 */ "expritem ::=", + /* 221 */ "cmd ::= RESET QUERY CACHE", + /* 222 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 223 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 224 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 225 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 226 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 227 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 228 */ "cmd ::= KILL CONNECTION INTEGER", + /* 229 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 230 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ #if YYSTACKDEPTH<=0 /* -** Try to increase the size of the parser stack. Return the number -** of errors. Return 0 on success. +** Try to increase the size of the parser stack. */ -static int yyGrowStack(yyParser *p){ +static void yyGrowStack(yyParser *p){ int newSize; - int idx; yyStackEntry *pNew; newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; - }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); - } + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); if( pNew ){ p->yystack = pNew; - p->yytos = &p->yystack[idx]; + p->yystksz = newSize; #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); + fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", + yyTracePrompt, p->yystksz); } #endif - p->yystksz = newSize; } - return pNew==0; } #endif -/* Datatype of the argument to the memory allocated passed as the -** second argument to ParseAlloc() below. This can be changed by -** putting an appropriate #define in the %include section of the input -** grammar. -*/ -#ifndef YYMALLOCARGTYPE -# define YYMALLOCARGTYPE size_t -#endif - -/* Initialize a new parser that has already been allocated. -*/ -void ParseInit(void *yypRawParser ParseCTX_PDECL){ - yyParser *yypParser = (yyParser*)yypRawParser; - ParseCTX_STORE -#ifdef YYTRACKMAXSTACKDEPTH - yypParser->yyhwm = 0; -#endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yypParser->yytos = yypParser->yystack; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif -} - -#ifndef Parse_ENGINEALWAYSONSTACK /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -1331,32 +1044,34 @@ void ParseInit(void *yypRawParser ParseCTX_PDECL){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ - yyParser *yypParser; - yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( yypParser ){ - ParseCTX_STORE - ParseInit(yypParser ParseCTX_PARAM); +void *ParseAlloc(void *(*mallocProc)(size_t)){ + yyParser *pParser; + pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + if( pParser ){ + pParser->yyidx = -1; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyidxMax = 0; +#endif +#if YYSTACKDEPTH<=0 + pParser->yystack = NULL; + pParser->yystksz = 0; + yyGrowStack(pParser); +#endif } - return (void*)yypParser; + return pParser; } -#endif /* Parse_ENGINEALWAYSONSTACK */ - -/* The following function deletes the "minor type" or semantic value -** associated with a symbol. The symbol can be either a terminal -** or nonterminal. "yymajor" is the symbol code, and "yypminor" is -** a pointer to the value to be deleted. The code used to do the -** deletions is derived from the %destructor and/or %token_destructor -** directives of the input grammar. +/* The following function deletes the value associated with a +** symbol. The symbol can be either a terminal or nonterminal. +** "yymajor" is the symbol code, and "yypminor" is a pointer to +** the value. */ static void yy_destructor( yyParser *yypParser, /* The parser */ YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1365,57 +1080,55 @@ static void yy_destructor( ** being destroyed before it is finished parsing. ** ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are *not* used + ** which appear on the RHS of the rule, but which are not used ** inside the C code. */ -/********* Begin destructor definitions ***************************************/ - case 225: /* keep */ - case 226: /* tagitemlist */ - case 250: /* fill_opt */ - case 252: /* groupby_opt */ - case 253: /* orderby_opt */ - case 263: /* sortlist */ - case 267: /* grouplist */ + case 227: /* keep */ + case 228: /* tagitemlist */ + case 253: /* fill_opt */ + case 255: /* groupby_opt */ + case 256: /* orderby_opt */ + case 266: /* sortlist */ + case 270: /* grouplist */ { -tVariantListDestroy((yypminor->yy156)); +tVariantListDestroy((yypminor->yy498)); } break; - case 242: /* columnlist */ + case 245: /* columnlist */ { -tFieldListDestroy((yypminor->yy511)); +tFieldListDestroy((yypminor->yy523)); } break; - case 243: /* select */ + case 246: /* select */ { -doDestroyQuerySql((yypminor->yy444)); +doDestroyQuerySql((yypminor->yy414)); } break; - case 246: /* selcollist */ - case 258: /* sclp */ - case 268: /* exprlist */ + case 249: /* selcollist */ + case 261: /* sclp */ + case 271: /* exprlist */ { -tSQLExprListDestroy((yypminor->yy158)); +tSQLExprListDestroy((yypminor->yy290)); } break; - case 248: /* where_opt */ - case 254: /* having_opt */ - case 259: /* expr */ - case 269: /* expritem */ + case 251: /* where_opt */ + case 257: /* having_opt */ + case 262: /* expr */ + case 272: /* expritem */ { -tSQLExprDestroy((yypminor->yy190)); +tSQLExprDestroy((yypminor->yy64)); } break; - case 257: /* union */ + case 260: /* union */ { -destroyAllSelectClause((yypminor->yy333)); +destroyAllSelectClause((yypminor->yy231)); } break; - case 264: /* sortitem */ + case 267: /* sortitem */ { -tVariantDestroy(&(yypminor->yy506)); +tVariantDestroy(&(yypminor->yy134)); } break; -/********* End destructor definitions *****************************************/ default: break; /* If no destructor action specified: do nothing */ } } @@ -1425,53 +1138,51 @@ tVariantDestroy(&(yypminor->yy506)); ** ** If there is a destructor routine associated with the token which ** is popped from the stack, then call it. +** +** Return the major token number for the symbol popped. */ -static void yy_pop_parser_stack(yyParser *pParser){ - yyStackEntry *yytos; - assert( pParser->yytos!=0 ); - assert( pParser->yytos > pParser->yystack ); - yytos = pParser->yytos--; +static int yy_pop_parser_stack(yyParser *pParser){ + YYCODETYPE yymajor; + yyStackEntry *yytos = &pParser->yystack[pParser->yyidx]; + + if( pParser->yyidx<0 ) return 0; #ifndef NDEBUG - if( yyTraceFILE ){ + if( yyTraceFILE && pParser->yyidx>=0 ){ fprintf(yyTraceFILE,"%sPopping %s\n", yyTracePrompt, yyTokenName[yytos->major]); } #endif - yy_destructor(pParser, yytos->major, &yytos->minor); + yymajor = yytos->major; + yy_destructor(pParser, yymajor, &yytos->minor); + pParser->yyidx--; + return yymajor; } -/* -** Clear all secondary memory allocations from the parser -*/ -void ParseFinalize(void *p){ - yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); -#endif -} - -#ifndef Parse_ENGINEALWAYSONSTACK /* -** Deallocate and destroy a parser. Destructors are called for +** Deallocate and destroy a parser. Destructors are all called for ** all stack elements before shutting the parser down. ** -** If the YYPARSEFREENEVERNULL macro exists (for example because it -** is defined in a %include section of the input grammar) then it is -** assumed that the input pointer is never NULL. +** Inputs: +**
    +**
  • A pointer to the parser. This should be a pointer +** obtained from ParseAlloc. +**
  • A pointer to a function used to reclaim memory obtained +** from malloc. +**
*/ void ParseFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ -#ifndef YYPARSEFREENEVERNULL - if( p==0 ) return; + yyParser *pParser = (yyParser*)p; + if( pParser==0 ) return; + while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + free(pParser->yystack); #endif - ParseFinalize(p); - (*freeProc)(p); + (*freeProc)((void*)pParser); } -#endif /* Parse_ENGINEALWAYSONSTACK */ /* ** Return the peak depth of the stack for a parser. @@ -1479,118 +1190,85 @@ void ParseFree( #ifdef YYTRACKMAXSTACKDEPTH int ParseStackPeak(void *p){ yyParser *pParser = (yyParser*)p; - return pParser->yyhwm; -} -#endif - -/* This array of booleans keeps track of the parser statement -** coverage. The element yycoverage[X][Y] is set when the parser -** is in state X and has a lookahead token Y. In a well-tested -** systems, every element of this matrix should end up being set. -*/ -#if defined(YYCOVERAGE) -static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; -#endif - -/* -** Write into out a description of every state/lookahead combination that -** -** (1) has not been used by the parser, and -** (2) is not a syntax error. -** -** Return the number of missed state/lookahead combinations. -*/ -#if defined(YYCOVERAGE) -int ParseCoverage(FILE *out){ - int stateno, iLookAhead, i; - int nMissed = 0; - for(stateno=0; statenoyyidxMax; } #endif /* ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. +** +** If the look-ahead token is YYNOCODE, then check to see if the action is +** independent of the look-ahead. If it is, return the action, otherwise +** return YY_NO_ACTION. */ -static YYACTIONTYPE yy_find_shift_action( - YYCODETYPE iLookAhead, /* The look-ahead token */ - YYACTIONTYPE stateno /* Current state number */ +static int yy_find_shift_action( + yyParser *pParser, /* The parser */ + YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; - - if( stateno>YY_MAX_SHIFT ) return stateno; - assert( stateno <= YY_SHIFT_COUNT ); -#if defined(YYCOVERAGE) - yycoverage[stateno][iLookAhead] = 1; -#endif - do{ - i = yy_shift_ofst[stateno]; - assert( i>=0 ); - assert( i<=YY_ACTTAB_COUNT ); - assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); - assert( iLookAhead!=YYNOCODE ); - assert( iLookAhead < YYNTOKEN ); - i += iLookAhead; - assert( i<(int)YY_NLOOKAHEAD ); - if( yy_lookahead[i]!=iLookAhead ){ + int stateno = pParser->yystack[pParser->yyidx].stateno; + + if( stateno>YY_SHIFT_COUNT + || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ + return yy_default[stateno]; + } + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + if( iLookAhead>0 ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - assert( iLookAhead %s\n", yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); } #endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; + return yy_find_shift_action(pParser, iFallback); } #endif #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); - if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]); } #endif /* NDEBUG */ return yy_action[j]; } } #endif /* YYWILDCARD */ - return yy_default[stateno]; - }else{ - assert( i>=0 && iyyidx--; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); } #endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser ** stack every overflows */ -/******** Begin %stack_overflow code ******************************************/ -/******** End %stack_overflow code ********************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument var */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } -/* -** Print tracing information for a SHIFT action -*/ -#ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ - if( yyTraceFILE ){ - if( yyNewStateyytos->major], - yyNewState); - }else{ - fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", - yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], - yyNewState - YY_MIN_REDUCE); - } - } -} -#else -# define yyTraceShift(X,Y,Z) -#endif - /* ** Perform a shift action. */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ - YYACTIONTYPE yyNewState, /* The new state to shift in */ - YYCODETYPE yyMajor, /* The major token to shift in */ - ParseTOKENTYPE yyMinor /* The minor token to shift in */ + int yyNewState, /* The new state to shift in */ + int yyMajor, /* The major token to shift in */ + YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ ){ yyStackEntry *yytos; - yypParser->yytos++; + yypParser->yyidx++; #ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + if( yypParser->yyidx>yypParser->yyidxMax ){ + yypParser->yyidxMax = yypParser->yyidx; } #endif #if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); + if( yypParser->yyidx>=YYSTACKDEPTH ){ + yyStackOverflow(yypParser, yypMinor); return; } #else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ - if( yyGrowStack(yypParser) ){ - yypParser->yytos--; - yyStackOverflow(yypParser); + if( yypParser->yyidx>=yypParser->yystksz ){ + yyGrowStack(yypParser); + if( yypParser->yyidx>=yypParser->yystksz ){ + yyStackOverflow(yypParser, yypMinor); return; } } #endif - if( yyNewState > YY_MAX_SHIFT ){ - yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yytos = &yypParser->yystack[yypParser->yyidx]; + yytos->stateno = (YYACTIONTYPE)yyNewState; + yytos->major = (YYCODETYPE)yyMajor; + yytos->minor = *yypMinor; +#ifndef NDEBUG + if( yyTraceFILE && yypParser->yyidx>0 ){ + int i; + fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); + fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); + for(i=1; i<=yypParser->yyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); } - yytos = yypParser->yytos; - yytos->stateno = yyNewState; - yytos->major = yyMajor; - yytos->minor.yy0 = yyMinor; - yyTraceShift(yypParser, yyNewState, "Shift"); +#endif } -/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side -** of that rule */ -static const YYCODETYPE yyRuleInfoLhs[] = { - 206, /* (0) program ::= cmd */ - 207, /* (1) cmd ::= SHOW DATABASES */ - 207, /* (2) cmd ::= SHOW MNODES */ - 207, /* (3) cmd ::= SHOW DNODES */ - 207, /* (4) cmd ::= SHOW ACCOUNTS */ - 207, /* (5) cmd ::= SHOW USERS */ - 207, /* (6) cmd ::= SHOW MODULES */ - 207, /* (7) cmd ::= SHOW QUERIES */ - 207, /* (8) cmd ::= SHOW CONNECTIONS */ - 207, /* (9) cmd ::= SHOW STREAMS */ - 207, /* (10) cmd ::= SHOW VARIABLES */ - 207, /* (11) cmd ::= SHOW SCORES */ - 207, /* (12) cmd ::= SHOW GRANTS */ - 207, /* (13) cmd ::= SHOW VNODES */ - 207, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - 208, /* (15) dbPrefix ::= */ - 208, /* (16) dbPrefix ::= ids DOT */ - 210, /* (17) cpxName ::= */ - 210, /* (18) cpxName ::= DOT ids */ - 207, /* (19) cmd ::= SHOW dbPrefix TABLES */ - 207, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - 207, /* (21) cmd ::= SHOW dbPrefix STABLES */ - 207, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - 207, /* (23) cmd ::= SHOW dbPrefix VGROUPS */ - 207, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */ - 207, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */ - 207, /* (26) cmd ::= DROP DATABASE ifexists ids */ - 207, /* (27) cmd ::= DROP DNODE ids */ - 207, /* (28) cmd ::= DROP USER ids */ - 207, /* (29) cmd ::= DROP ACCOUNT ids */ - 207, /* (30) cmd ::= USE ids */ - 207, /* (31) cmd ::= DESCRIBE ids cpxName */ - 207, /* (32) cmd ::= ALTER USER ids PASS ids */ - 207, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */ - 207, /* (34) cmd ::= ALTER DNODE ids ids */ - 207, /* (35) cmd ::= ALTER DNODE ids ids ids */ - 207, /* (36) cmd ::= ALTER LOCAL ids */ - 207, /* (37) cmd ::= ALTER LOCAL ids ids */ - 207, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */ - 207, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */ - 207, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - 209, /* (41) ids ::= ID */ - 209, /* (42) ids ::= STRING */ - 211, /* (43) ifexists ::= IF EXISTS */ - 211, /* (44) ifexists ::= */ - 214, /* (45) ifnotexists ::= IF NOT EXISTS */ - 214, /* (46) ifnotexists ::= */ - 207, /* (47) cmd ::= CREATE DNODE ids */ - 207, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - 207, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - 207, /* (50) cmd ::= CREATE USER ids PASS ids */ - 216, /* (51) pps ::= */ - 216, /* (52) pps ::= PPS INTEGER */ - 217, /* (53) tseries ::= */ - 217, /* (54) tseries ::= TSERIES INTEGER */ - 218, /* (55) dbs ::= */ - 218, /* (56) dbs ::= DBS INTEGER */ - 219, /* (57) streams ::= */ - 219, /* (58) streams ::= STREAMS INTEGER */ - 220, /* (59) storage ::= */ - 220, /* (60) storage ::= STORAGE INTEGER */ - 221, /* (61) qtime ::= */ - 221, /* (62) qtime ::= QTIME INTEGER */ - 222, /* (63) users ::= */ - 222, /* (64) users ::= USERS INTEGER */ - 223, /* (65) conns ::= */ - 223, /* (66) conns ::= CONNS INTEGER */ - 224, /* (67) state ::= */ - 224, /* (68) state ::= STATE ids */ - 213, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - 225, /* (70) keep ::= KEEP tagitemlist */ - 227, /* (71) cache ::= CACHE INTEGER */ - 228, /* (72) replica ::= REPLICA INTEGER */ - 229, /* (73) quorum ::= QUORUM INTEGER */ - 230, /* (74) days ::= DAYS INTEGER */ - 231, /* (75) minrows ::= MINROWS INTEGER */ - 232, /* (76) maxrows ::= MAXROWS INTEGER */ - 233, /* (77) blocks ::= BLOCKS INTEGER */ - 234, /* (78) ctime ::= CTIME INTEGER */ - 235, /* (79) wal ::= WAL INTEGER */ - 236, /* (80) fsync ::= FSYNC INTEGER */ - 237, /* (81) comp ::= COMP INTEGER */ - 238, /* (82) prec ::= PRECISION STRING */ - 215, /* (83) db_optr ::= */ - 215, /* (84) db_optr ::= db_optr cache */ - 215, /* (85) db_optr ::= db_optr replica */ - 215, /* (86) db_optr ::= db_optr quorum */ - 215, /* (87) db_optr ::= db_optr days */ - 215, /* (88) db_optr ::= db_optr minrows */ - 215, /* (89) db_optr ::= db_optr maxrows */ - 215, /* (90) db_optr ::= db_optr blocks */ - 215, /* (91) db_optr ::= db_optr ctime */ - 215, /* (92) db_optr ::= db_optr wal */ - 215, /* (93) db_optr ::= db_optr fsync */ - 215, /* (94) db_optr ::= db_optr comp */ - 215, /* (95) db_optr ::= db_optr prec */ - 215, /* (96) db_optr ::= db_optr keep */ - 212, /* (97) alter_db_optr ::= */ - 212, /* (98) alter_db_optr ::= alter_db_optr replica */ - 212, /* (99) alter_db_optr ::= alter_db_optr quorum */ - 212, /* (100) alter_db_optr ::= alter_db_optr keep */ - 212, /* (101) alter_db_optr ::= alter_db_optr blocks */ - 212, /* (102) alter_db_optr ::= alter_db_optr comp */ - 212, /* (103) alter_db_optr ::= alter_db_optr wal */ - 212, /* (104) alter_db_optr ::= alter_db_optr fsync */ - 239, /* (105) typename ::= ids */ - 239, /* (106) typename ::= ids LP signed RP */ - 240, /* (107) signed ::= INTEGER */ - 240, /* (108) signed ::= PLUS INTEGER */ - 240, /* (109) signed ::= MINUS INTEGER */ - 207, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ - 241, /* (111) create_table_args ::= LP columnlist RP */ - 241, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ - 241, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ - 241, /* (114) create_table_args ::= AS select */ - 242, /* (115) columnlist ::= columnlist COMMA column */ - 242, /* (116) columnlist ::= column */ - 244, /* (117) column ::= ids typename */ - 226, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */ - 226, /* (119) tagitemlist ::= tagitem */ - 245, /* (120) tagitem ::= INTEGER */ - 245, /* (121) tagitem ::= FLOAT */ - 245, /* (122) tagitem ::= STRING */ - 245, /* (123) tagitem ::= BOOL */ - 245, /* (124) tagitem ::= NULL */ - 245, /* (125) tagitem ::= MINUS INTEGER */ - 245, /* (126) tagitem ::= MINUS FLOAT */ - 245, /* (127) tagitem ::= PLUS INTEGER */ - 245, /* (128) tagitem ::= PLUS FLOAT */ - 243, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - 257, /* (130) union ::= select */ - 257, /* (131) union ::= LP union RP */ - 257, /* (132) union ::= union UNION ALL select */ - 257, /* (133) union ::= union UNION ALL LP select RP */ - 207, /* (134) cmd ::= union */ - 243, /* (135) select ::= SELECT selcollist */ - 258, /* (136) sclp ::= selcollist COMMA */ - 258, /* (137) sclp ::= */ - 246, /* (138) selcollist ::= sclp expr as */ - 246, /* (139) selcollist ::= sclp STAR */ - 260, /* (140) as ::= AS ids */ - 260, /* (141) as ::= ids */ - 260, /* (142) as ::= */ - 247, /* (143) from ::= FROM tablelist */ - 261, /* (144) tablelist ::= ids cpxName */ - 261, /* (145) tablelist ::= ids cpxName ids */ - 261, /* (146) tablelist ::= tablelist COMMA ids cpxName */ - 261, /* (147) tablelist ::= tablelist COMMA ids cpxName ids */ - 262, /* (148) tmvar ::= VARIABLE */ - 249, /* (149) interval_opt ::= INTERVAL LP tmvar RP */ - 249, /* (150) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - 249, /* (151) interval_opt ::= */ - 250, /* (152) fill_opt ::= */ - 250, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - 250, /* (154) fill_opt ::= FILL LP ID RP */ - 251, /* (155) sliding_opt ::= SLIDING LP tmvar RP */ - 251, /* (156) sliding_opt ::= */ - 253, /* (157) orderby_opt ::= */ - 253, /* (158) orderby_opt ::= ORDER BY sortlist */ - 263, /* (159) sortlist ::= sortlist COMMA item sortorder */ - 263, /* (160) sortlist ::= item sortorder */ - 265, /* (161) item ::= ids cpxName */ - 266, /* (162) sortorder ::= ASC */ - 266, /* (163) sortorder ::= DESC */ - 266, /* (164) sortorder ::= */ - 252, /* (165) groupby_opt ::= */ - 252, /* (166) groupby_opt ::= GROUP BY grouplist */ - 267, /* (167) grouplist ::= grouplist COMMA item */ - 267, /* (168) grouplist ::= item */ - 254, /* (169) having_opt ::= */ - 254, /* (170) having_opt ::= HAVING expr */ - 256, /* (171) limit_opt ::= */ - 256, /* (172) limit_opt ::= LIMIT signed */ - 256, /* (173) limit_opt ::= LIMIT signed OFFSET signed */ - 256, /* (174) limit_opt ::= LIMIT signed COMMA signed */ - 255, /* (175) slimit_opt ::= */ - 255, /* (176) slimit_opt ::= SLIMIT signed */ - 255, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */ - 255, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */ - 248, /* (179) where_opt ::= */ - 248, /* (180) where_opt ::= WHERE expr */ - 259, /* (181) expr ::= LP expr RP */ - 259, /* (182) expr ::= ID */ - 259, /* (183) expr ::= ID DOT ID */ - 259, /* (184) expr ::= ID DOT STAR */ - 259, /* (185) expr ::= INTEGER */ - 259, /* (186) expr ::= MINUS INTEGER */ - 259, /* (187) expr ::= PLUS INTEGER */ - 259, /* (188) expr ::= FLOAT */ - 259, /* (189) expr ::= MINUS FLOAT */ - 259, /* (190) expr ::= PLUS FLOAT */ - 259, /* (191) expr ::= STRING */ - 259, /* (192) expr ::= NOW */ - 259, /* (193) expr ::= VARIABLE */ - 259, /* (194) expr ::= BOOL */ - 259, /* (195) expr ::= ID LP exprlist RP */ - 259, /* (196) expr ::= ID LP STAR RP */ - 259, /* (197) expr ::= expr IS NULL */ - 259, /* (198) expr ::= expr IS NOT NULL */ - 259, /* (199) expr ::= expr LT expr */ - 259, /* (200) expr ::= expr GT expr */ - 259, /* (201) expr ::= expr LE expr */ - 259, /* (202) expr ::= expr GE expr */ - 259, /* (203) expr ::= expr NE expr */ - 259, /* (204) expr ::= expr EQ expr */ - 259, /* (205) expr ::= expr AND expr */ - 259, /* (206) expr ::= expr OR expr */ - 259, /* (207) expr ::= expr PLUS expr */ - 259, /* (208) expr ::= expr MINUS expr */ - 259, /* (209) expr ::= expr STAR expr */ - 259, /* (210) expr ::= expr SLASH expr */ - 259, /* (211) expr ::= expr REM expr */ - 259, /* (212) expr ::= expr LIKE expr */ - 259, /* (213) expr ::= expr IN LP exprlist RP */ - 268, /* (214) exprlist ::= exprlist COMMA expritem */ - 268, /* (215) exprlist ::= expritem */ - 269, /* (216) expritem ::= expr */ - 269, /* (217) expritem ::= */ - 207, /* (218) cmd ::= RESET QUERY CACHE */ - 207, /* (219) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - 207, /* (220) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - 207, /* (221) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - 207, /* (222) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - 207, /* (223) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - 207, /* (224) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - 207, /* (225) cmd ::= KILL CONNECTION INTEGER */ - 207, /* (226) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - 207, /* (227) cmd ::= KILL QUERY INTEGER COLON INTEGER */ -}; - -/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number -** of symbols on the right-hand side of that rule. */ -static const signed char yyRuleInfoNRhs[] = { - -1, /* (0) program ::= cmd */ - -2, /* (1) cmd ::= SHOW DATABASES */ - -2, /* (2) cmd ::= SHOW MNODES */ - -2, /* (3) cmd ::= SHOW DNODES */ - -2, /* (4) cmd ::= SHOW ACCOUNTS */ - -2, /* (5) cmd ::= SHOW USERS */ - -2, /* (6) cmd ::= SHOW MODULES */ - -2, /* (7) cmd ::= SHOW QUERIES */ - -2, /* (8) cmd ::= SHOW CONNECTIONS */ - -2, /* (9) cmd ::= SHOW STREAMS */ - -2, /* (10) cmd ::= SHOW VARIABLES */ - -2, /* (11) cmd ::= SHOW SCORES */ - -2, /* (12) cmd ::= SHOW GRANTS */ - -2, /* (13) cmd ::= SHOW VNODES */ - -3, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - 0, /* (15) dbPrefix ::= */ - -2, /* (16) dbPrefix ::= ids DOT */ - 0, /* (17) cpxName ::= */ - -2, /* (18) cpxName ::= DOT ids */ - -3, /* (19) cmd ::= SHOW dbPrefix TABLES */ - -5, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - -3, /* (21) cmd ::= SHOW dbPrefix STABLES */ - -5, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - -3, /* (23) cmd ::= SHOW dbPrefix VGROUPS */ - -4, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */ - -5, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */ - -4, /* (26) cmd ::= DROP DATABASE ifexists ids */ - -3, /* (27) cmd ::= DROP DNODE ids */ - -3, /* (28) cmd ::= DROP USER ids */ - -3, /* (29) cmd ::= DROP ACCOUNT ids */ - -2, /* (30) cmd ::= USE ids */ - -3, /* (31) cmd ::= DESCRIBE ids cpxName */ - -5, /* (32) cmd ::= ALTER USER ids PASS ids */ - -5, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */ - -4, /* (34) cmd ::= ALTER DNODE ids ids */ - -5, /* (35) cmd ::= ALTER DNODE ids ids ids */ - -3, /* (36) cmd ::= ALTER LOCAL ids */ - -4, /* (37) cmd ::= ALTER LOCAL ids ids */ - -4, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */ - -4, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */ - -6, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - -1, /* (41) ids ::= ID */ - -1, /* (42) ids ::= STRING */ - -2, /* (43) ifexists ::= IF EXISTS */ - 0, /* (44) ifexists ::= */ - -3, /* (45) ifnotexists ::= IF NOT EXISTS */ - 0, /* (46) ifnotexists ::= */ - -3, /* (47) cmd ::= CREATE DNODE ids */ - -6, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - -5, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - -5, /* (50) cmd ::= CREATE USER ids PASS ids */ - 0, /* (51) pps ::= */ - -2, /* (52) pps ::= PPS INTEGER */ - 0, /* (53) tseries ::= */ - -2, /* (54) tseries ::= TSERIES INTEGER */ - 0, /* (55) dbs ::= */ - -2, /* (56) dbs ::= DBS INTEGER */ - 0, /* (57) streams ::= */ - -2, /* (58) streams ::= STREAMS INTEGER */ - 0, /* (59) storage ::= */ - -2, /* (60) storage ::= STORAGE INTEGER */ - 0, /* (61) qtime ::= */ - -2, /* (62) qtime ::= QTIME INTEGER */ - 0, /* (63) users ::= */ - -2, /* (64) users ::= USERS INTEGER */ - 0, /* (65) conns ::= */ - -2, /* (66) conns ::= CONNS INTEGER */ - 0, /* (67) state ::= */ - -2, /* (68) state ::= STATE ids */ - -9, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - -2, /* (70) keep ::= KEEP tagitemlist */ - -2, /* (71) cache ::= CACHE INTEGER */ - -2, /* (72) replica ::= REPLICA INTEGER */ - -2, /* (73) quorum ::= QUORUM INTEGER */ - -2, /* (74) days ::= DAYS INTEGER */ - -2, /* (75) minrows ::= MINROWS INTEGER */ - -2, /* (76) maxrows ::= MAXROWS INTEGER */ - -2, /* (77) blocks ::= BLOCKS INTEGER */ - -2, /* (78) ctime ::= CTIME INTEGER */ - -2, /* (79) wal ::= WAL INTEGER */ - -2, /* (80) fsync ::= FSYNC INTEGER */ - -2, /* (81) comp ::= COMP INTEGER */ - -2, /* (82) prec ::= PRECISION STRING */ - 0, /* (83) db_optr ::= */ - -2, /* (84) db_optr ::= db_optr cache */ - -2, /* (85) db_optr ::= db_optr replica */ - -2, /* (86) db_optr ::= db_optr quorum */ - -2, /* (87) db_optr ::= db_optr days */ - -2, /* (88) db_optr ::= db_optr minrows */ - -2, /* (89) db_optr ::= db_optr maxrows */ - -2, /* (90) db_optr ::= db_optr blocks */ - -2, /* (91) db_optr ::= db_optr ctime */ - -2, /* (92) db_optr ::= db_optr wal */ - -2, /* (93) db_optr ::= db_optr fsync */ - -2, /* (94) db_optr ::= db_optr comp */ - -2, /* (95) db_optr ::= db_optr prec */ - -2, /* (96) db_optr ::= db_optr keep */ - 0, /* (97) alter_db_optr ::= */ - -2, /* (98) alter_db_optr ::= alter_db_optr replica */ - -2, /* (99) alter_db_optr ::= alter_db_optr quorum */ - -2, /* (100) alter_db_optr ::= alter_db_optr keep */ - -2, /* (101) alter_db_optr ::= alter_db_optr blocks */ - -2, /* (102) alter_db_optr ::= alter_db_optr comp */ - -2, /* (103) alter_db_optr ::= alter_db_optr wal */ - -2, /* (104) alter_db_optr ::= alter_db_optr fsync */ - -1, /* (105) typename ::= ids */ - -4, /* (106) typename ::= ids LP signed RP */ - -1, /* (107) signed ::= INTEGER */ - -2, /* (108) signed ::= PLUS INTEGER */ - -2, /* (109) signed ::= MINUS INTEGER */ - -6, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ - -3, /* (111) create_table_args ::= LP columnlist RP */ - -7, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ - -7, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ - -2, /* (114) create_table_args ::= AS select */ - -3, /* (115) columnlist ::= columnlist COMMA column */ - -1, /* (116) columnlist ::= column */ - -2, /* (117) column ::= ids typename */ - -3, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */ - -1, /* (119) tagitemlist ::= tagitem */ - -1, /* (120) tagitem ::= INTEGER */ - -1, /* (121) tagitem ::= FLOAT */ - -1, /* (122) tagitem ::= STRING */ - -1, /* (123) tagitem ::= BOOL */ - -1, /* (124) tagitem ::= NULL */ - -2, /* (125) tagitem ::= MINUS INTEGER */ - -2, /* (126) tagitem ::= MINUS FLOAT */ - -2, /* (127) tagitem ::= PLUS INTEGER */ - -2, /* (128) tagitem ::= PLUS FLOAT */ - -12, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - -1, /* (130) union ::= select */ - -3, /* (131) union ::= LP union RP */ - -4, /* (132) union ::= union UNION ALL select */ - -6, /* (133) union ::= union UNION ALL LP select RP */ - -1, /* (134) cmd ::= union */ - -2, /* (135) select ::= SELECT selcollist */ - -2, /* (136) sclp ::= selcollist COMMA */ - 0, /* (137) sclp ::= */ - -3, /* (138) selcollist ::= sclp expr as */ - -2, /* (139) selcollist ::= sclp STAR */ - -2, /* (140) as ::= AS ids */ - -1, /* (141) as ::= ids */ - 0, /* (142) as ::= */ - -2, /* (143) from ::= FROM tablelist */ - -2, /* (144) tablelist ::= ids cpxName */ - -3, /* (145) tablelist ::= ids cpxName ids */ - -4, /* (146) tablelist ::= tablelist COMMA ids cpxName */ - -5, /* (147) tablelist ::= tablelist COMMA ids cpxName ids */ - -1, /* (148) tmvar ::= VARIABLE */ - -4, /* (149) interval_opt ::= INTERVAL LP tmvar RP */ - -6, /* (150) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - 0, /* (151) interval_opt ::= */ - 0, /* (152) fill_opt ::= */ - -6, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - -4, /* (154) fill_opt ::= FILL LP ID RP */ - -4, /* (155) sliding_opt ::= SLIDING LP tmvar RP */ - 0, /* (156) sliding_opt ::= */ - 0, /* (157) orderby_opt ::= */ - -3, /* (158) orderby_opt ::= ORDER BY sortlist */ - -4, /* (159) sortlist ::= sortlist COMMA item sortorder */ - -2, /* (160) sortlist ::= item sortorder */ - -2, /* (161) item ::= ids cpxName */ - -1, /* (162) sortorder ::= ASC */ - -1, /* (163) sortorder ::= DESC */ - 0, /* (164) sortorder ::= */ - 0, /* (165) groupby_opt ::= */ - -3, /* (166) groupby_opt ::= GROUP BY grouplist */ - -3, /* (167) grouplist ::= grouplist COMMA item */ - -1, /* (168) grouplist ::= item */ - 0, /* (169) having_opt ::= */ - -2, /* (170) having_opt ::= HAVING expr */ - 0, /* (171) limit_opt ::= */ - -2, /* (172) limit_opt ::= LIMIT signed */ - -4, /* (173) limit_opt ::= LIMIT signed OFFSET signed */ - -4, /* (174) limit_opt ::= LIMIT signed COMMA signed */ - 0, /* (175) slimit_opt ::= */ - -2, /* (176) slimit_opt ::= SLIMIT signed */ - -4, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */ - -4, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */ - 0, /* (179) where_opt ::= */ - -2, /* (180) where_opt ::= WHERE expr */ - -3, /* (181) expr ::= LP expr RP */ - -1, /* (182) expr ::= ID */ - -3, /* (183) expr ::= ID DOT ID */ - -3, /* (184) expr ::= ID DOT STAR */ - -1, /* (185) expr ::= INTEGER */ - -2, /* (186) expr ::= MINUS INTEGER */ - -2, /* (187) expr ::= PLUS INTEGER */ - -1, /* (188) expr ::= FLOAT */ - -2, /* (189) expr ::= MINUS FLOAT */ - -2, /* (190) expr ::= PLUS FLOAT */ - -1, /* (191) expr ::= STRING */ - -1, /* (192) expr ::= NOW */ - -1, /* (193) expr ::= VARIABLE */ - -1, /* (194) expr ::= BOOL */ - -4, /* (195) expr ::= ID LP exprlist RP */ - -4, /* (196) expr ::= ID LP STAR RP */ - -3, /* (197) expr ::= expr IS NULL */ - -4, /* (198) expr ::= expr IS NOT NULL */ - -3, /* (199) expr ::= expr LT expr */ - -3, /* (200) expr ::= expr GT expr */ - -3, /* (201) expr ::= expr LE expr */ - -3, /* (202) expr ::= expr GE expr */ - -3, /* (203) expr ::= expr NE expr */ - -3, /* (204) expr ::= expr EQ expr */ - -3, /* (205) expr ::= expr AND expr */ - -3, /* (206) expr ::= expr OR expr */ - -3, /* (207) expr ::= expr PLUS expr */ - -3, /* (208) expr ::= expr MINUS expr */ - -3, /* (209) expr ::= expr STAR expr */ - -3, /* (210) expr ::= expr SLASH expr */ - -3, /* (211) expr ::= expr REM expr */ - -3, /* (212) expr ::= expr LIKE expr */ - -5, /* (213) expr ::= expr IN LP exprlist RP */ - -3, /* (214) exprlist ::= exprlist COMMA expritem */ - -1, /* (215) exprlist ::= expritem */ - -1, /* (216) expritem ::= expr */ - 0, /* (217) expritem ::= */ - -3, /* (218) cmd ::= RESET QUERY CACHE */ - -7, /* (219) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (220) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - -7, /* (221) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - -7, /* (222) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - -8, /* (223) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - -9, /* (224) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - -3, /* (225) cmd ::= KILL CONNECTION INTEGER */ - -5, /* (226) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - -5, /* (227) cmd ::= KILL QUERY INTEGER COLON INTEGER */ +/* The following table contains information about every rule that +** is used during the reduce. +*/ +static const struct { + YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ + unsigned char nrhs; /* Number of right-hand side symbols in the rule */ +} yyRuleInfo[] = { + { 208, 1 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 3 }, + { 210, 0 }, + { 210, 2 }, + { 212, 0 }, + { 212, 2 }, + { 209, 3 }, + { 209, 5 }, + { 209, 3 }, + { 209, 5 }, + { 209, 3 }, + { 209, 4 }, + { 209, 5 }, + { 209, 4 }, + { 209, 3 }, + { 209, 3 }, + { 209, 3 }, + { 209, 2 }, + { 209, 3 }, + { 209, 5 }, + { 209, 5 }, + { 209, 4 }, + { 209, 5 }, + { 209, 3 }, + { 209, 4 }, + { 209, 4 }, + { 209, 4 }, + { 209, 6 }, + { 211, 1 }, + { 211, 1 }, + { 213, 2 }, + { 213, 0 }, + { 216, 3 }, + { 216, 0 }, + { 209, 3 }, + { 209, 6 }, + { 209, 5 }, + { 209, 5 }, + { 218, 0 }, + { 218, 2 }, + { 219, 0 }, + { 219, 2 }, + { 220, 0 }, + { 220, 2 }, + { 221, 0 }, + { 221, 2 }, + { 222, 0 }, + { 222, 2 }, + { 223, 0 }, + { 223, 2 }, + { 224, 0 }, + { 224, 2 }, + { 225, 0 }, + { 225, 2 }, + { 226, 0 }, + { 226, 2 }, + { 215, 9 }, + { 227, 2 }, + { 229, 2 }, + { 230, 2 }, + { 231, 2 }, + { 232, 2 }, + { 233, 2 }, + { 234, 2 }, + { 235, 2 }, + { 236, 2 }, + { 237, 2 }, + { 238, 2 }, + { 239, 2 }, + { 240, 2 }, + { 241, 2 }, + { 217, 0 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 214, 0 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 242, 1 }, + { 242, 4 }, + { 243, 1 }, + { 243, 2 }, + { 243, 2 }, + { 209, 6 }, + { 244, 3 }, + { 244, 7 }, + { 244, 7 }, + { 244, 2 }, + { 245, 3 }, + { 245, 1 }, + { 247, 2 }, + { 228, 3 }, + { 228, 1 }, + { 248, 1 }, + { 248, 1 }, + { 248, 1 }, + { 248, 1 }, + { 248, 1 }, + { 248, 2 }, + { 248, 2 }, + { 248, 2 }, + { 248, 2 }, + { 246, 12 }, + { 260, 1 }, + { 260, 3 }, + { 260, 4 }, + { 260, 6 }, + { 209, 1 }, + { 246, 2 }, + { 261, 2 }, + { 261, 0 }, + { 249, 3 }, + { 249, 2 }, + { 263, 2 }, + { 263, 1 }, + { 263, 0 }, + { 250, 2 }, + { 264, 2 }, + { 264, 3 }, + { 264, 4 }, + { 264, 5 }, + { 265, 1 }, + { 252, 4 }, + { 252, 6 }, + { 252, 0 }, + { 253, 0 }, + { 253, 6 }, + { 253, 4 }, + { 254, 4 }, + { 254, 0 }, + { 256, 0 }, + { 256, 3 }, + { 266, 4 }, + { 266, 2 }, + { 268, 2 }, + { 269, 1 }, + { 269, 1 }, + { 269, 0 }, + { 255, 0 }, + { 255, 3 }, + { 270, 3 }, + { 270, 1 }, + { 257, 0 }, + { 257, 2 }, + { 259, 0 }, + { 259, 2 }, + { 259, 4 }, + { 259, 4 }, + { 258, 0 }, + { 258, 2 }, + { 258, 4 }, + { 258, 4 }, + { 251, 0 }, + { 251, 2 }, + { 262, 3 }, + { 262, 1 }, + { 262, 3 }, + { 262, 3 }, + { 262, 1 }, + { 262, 2 }, + { 262, 2 }, + { 262, 1 }, + { 262, 2 }, + { 262, 2 }, + { 262, 1 }, + { 262, 1 }, + { 262, 1 }, + { 262, 1 }, + { 262, 4 }, + { 262, 4 }, + { 262, 3 }, + { 262, 4 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 5 }, + { 271, 3 }, + { 271, 1 }, + { 272, 1 }, + { 272, 0 }, + { 209, 3 }, + { 209, 7 }, + { 209, 7 }, + { 209, 7 }, + { 209, 7 }, + { 209, 8 }, + { 209, 9 }, + { 209, 3 }, + { 209, 5 }, + { 209, 5 }, }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2169,76 +1602,43 @@ static void yy_accept(yyParser*); /* Forward Declaration */ /* ** Perform a reduce action and the shift that must immediately ** follow the reduce. -** -** The yyLookahead and yyLookaheadToken parameters provide reduce actions -** access to the lookahead token (if any). The yyLookahead will be YYNOCODE -** if the lookahead token has already been consumed. As this procedure is -** only called from one place, optimizing compilers will in-line it, which -** means that the extra parameters have no performance impact. */ -static YYACTIONTYPE yy_reduce( +static void yy_reduce( yyParser *yypParser, /* The parser */ - unsigned int yyruleno, /* Number of the rule by which to reduce */ - int yyLookahead, /* Lookahead token, or YYNOCODE if none */ - ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ - ParseCTX_PDECL /* %extra_context */ + int yyruleno /* Number of the rule by which to reduce */ ){ int yygoto; /* The next state */ - YYACTIONTYPE yyact; /* The next action */ + int yyact; /* The next action */ + YYMINORTYPE yygotominor; /* The LHS of the rule reduced */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - ParseARG_FETCH - (void)yyLookahead; - (void)yyLookaheadToken; - yymsp = yypParser->yytos; + ParseARG_FETCH; + yymsp = &yypParser->yystack[yypParser->yyidx]; #ifndef NDEBUG - if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfoNRhs[yyruleno]; - if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", - yyTracePrompt, - yyruleno, yyRuleName[yyruleno], - yyruleno=0 + && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, + yyRuleName[yyruleno]); } #endif /* NDEBUG */ - /* Check that the stack is large enough to grow by a single entry - ** if the RHS of the rule is empty. This ensures that there is room - ** enough on the stack to push the LHS value */ - if( yyRuleInfoNRhs[yyruleno]==0 ){ -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; - } - yymsp = yypParser->yytos; - } -#endif - } + /* Silence complaints from purify about yygotominor being uninitialized + ** in some cases when it is copied into the stack after the following + ** switch. yygotominor is uninitialized when a rule reduces that does + ** not set the value of its left-hand side nonterminal. Leaving the + ** value of the nonterminal uninitialized is utterly harmless as long + ** as the value is never used. So really the only thing this code + ** accomplishes is to quieten purify. + ** + ** 2007-01-16: The wireshark project (www.wireshark.org) reports that + ** without this code, their parser segfaults. I'm not sure what there + ** parser is doing to make this happen. This is the second bug report + ** from wireshark this week. Clearly they are stressing Lemon in ways + ** that it has not been previously stressed... (SQLite ticket #2172) + */ + /*memset(&yygotominor, 0, sizeof(yygotominor));*/ + yygotominor = yyzerominor; + switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -2249,8 +1649,6 @@ static YYACTIONTYPE yy_reduce( ** #line ** break; */ -/********** Begin reduce actions **********************************************/ - YYMINORTYPE yylhsminor; case 0: /* program ::= cmd */ {} break; @@ -2297,17 +1695,16 @@ static YYACTIONTYPE yy_reduce( { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &yymsp[0].minor.yy0, 0); } break; case 15: /* dbPrefix ::= */ -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.type = 0;} +{yygotominor.yy0.n = 0; yygotominor.yy0.type = 0;} break; case 16: /* dbPrefix ::= ids DOT */ -{yylhsminor.yy0 = yymsp[-1].minor.yy0; } - yymsp[-1].minor.yy0 = yylhsminor.yy0; +{yygotominor.yy0 = yymsp[-1].minor.yy0; } break; case 17: /* cpxName ::= */ -{yymsp[1].minor.yy0.n = 0; } +{yygotominor.yy0.n = 0; } break; case 18: /* cpxName ::= DOT ids */ -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n += 1; } +{yygotominor.yy0 = yymsp[0].minor.yy0; yygotominor.yy0.n += 1; } break; case 19: /* cmd ::= SHOW dbPrefix TABLES */ { @@ -2391,37 +1788,34 @@ static YYACTIONTYPE yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy118, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} break; case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy479);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} break; case 40: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy479);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} break; case 41: /* ids ::= ID */ case 42: /* ids ::= STRING */ yytestcase(yyruleno==42); -{yylhsminor.yy0 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy0 = yylhsminor.yy0; +{yygotominor.yy0 = yymsp[0].minor.yy0; } break; case 43: /* ifexists ::= IF EXISTS */ -{yymsp[-1].minor.yy0.n = 1;} + case 45: /* ifnotexists ::= IF NOT EXISTS */ yytestcase(yyruleno==45); +{yygotominor.yy0.n = 1;} break; case 44: /* ifexists ::= */ case 46: /* ifnotexists ::= */ yytestcase(yyruleno==46); -{yymsp[1].minor.yy0.n = 0;} - break; - case 45: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy0.n = 1;} +{yygotominor.yy0.n = 0;} break; case 47: /* cmd ::= CREATE DNODE ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; case 48: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy479);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} break; case 49: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy118, &yymsp[-2].minor.yy0);} +{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);} break; case 50: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} @@ -2435,7 +1829,7 @@ static YYACTIONTYPE yy_reduce( case 63: /* users ::= */ yytestcase(yyruleno==63); case 65: /* conns ::= */ yytestcase(yyruleno==65); case 67: /* state ::= */ yytestcase(yyruleno==67); -{yymsp[1].minor.yy0.n = 0; } +{yygotominor.yy0.n = 0; } break; case 52: /* pps ::= PPS INTEGER */ case 54: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==54); @@ -2446,24 +1840,23 @@ static YYACTIONTYPE yy_reduce( case 64: /* users ::= USERS INTEGER */ yytestcase(yyruleno==64); case 66: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==66); case 68: /* state ::= STATE ids */ yytestcase(yyruleno==68); -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } +{yygotominor.yy0 = yymsp[0].minor.yy0; } break; case 69: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy479.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy479.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy479.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy479.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy479.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy479.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy479.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy479.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy479.stat = yymsp[0].minor.yy0; + yygotominor.yy149.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yygotominor.yy149.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yygotominor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yygotominor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yygotominor.yy149.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yygotominor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yygotominor.yy149.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yygotominor.yy149.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yygotominor.yy149.stat = yymsp[0].minor.yy0; } - yymsp[-8].minor.yy479 = yylhsminor.yy479; break; case 70: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy156 = yymsp[0].minor.yy156; } +{ yygotominor.yy498 = yymsp[0].minor.yy498; } break; case 71: /* cache ::= CACHE INTEGER */ case 72: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==72); @@ -2477,540 +1870,461 @@ static YYACTIONTYPE yy_reduce( case 80: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==80); case 81: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==81); case 82: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==82); -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } + case 83: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==83); +{ yygotominor.yy0 = yymsp[0].minor.yy0; } break; - case 83: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy118);} + case 84: /* db_optr ::= */ +{setDefaultCreateDbOption(&yygotominor.yy268);} break; - case 84: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 85: /* db_optr ::= db_optr cache */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 85: /* db_optr ::= db_optr replica */ - case 98: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==98); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 86: /* db_optr ::= db_optr replica */ + case 100: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==100); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 86: /* db_optr ::= db_optr quorum */ - case 99: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==99); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 87: /* db_optr ::= db_optr quorum */ + case 101: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==101); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 87: /* db_optr ::= db_optr days */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 88: /* db_optr ::= db_optr days */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 88: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 89: /* db_optr ::= db_optr minrows */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } break; - case 89: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 90: /* db_optr ::= db_optr maxrows */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } break; - case 90: /* db_optr ::= db_optr blocks */ - case 101: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==101); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 91: /* db_optr ::= db_optr blocks */ + case 103: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==103); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 91: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 92: /* db_optr ::= db_optr ctime */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 92: /* db_optr ::= db_optr wal */ - case 103: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==103); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 93: /* db_optr ::= db_optr wal */ + case 105: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==105); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 93: /* db_optr ::= db_optr fsync */ - case 104: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==104); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 94: /* db_optr ::= db_optr fsync */ + case 106: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==106); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 94: /* db_optr ::= db_optr comp */ - case 102: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==102); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 95: /* db_optr ::= db_optr comp */ + case 104: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==104); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 95: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 96: /* db_optr ::= db_optr prec */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.precision = yymsp[0].minor.yy0; } break; - case 96: /* db_optr ::= db_optr keep */ - case 100: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==100); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.keep = yymsp[0].minor.yy156; } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 97: /* db_optr ::= db_optr keep */ + case 102: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==102); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.keep = yymsp[0].minor.yy498; } break; - case 97: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy118);} + case 98: /* db_optr ::= db_optr update */ + case 107: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==107); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 105: /* typename ::= ids */ + case 99: /* alter_db_optr ::= */ +{ setDefaultCreateDbOption(&yygotominor.yy268);} + break; + case 108: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSQLSetColumnType (&yylhsminor.yy343, &yymsp[0].minor.yy0); + tSQLSetColumnType (&yygotominor.yy223, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy343 = yylhsminor.yy343; break; - case 106: /* typename ::= ids LP signed RP */ + case 109: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy369 <= 0) { + if (yymsp[-1].minor.yy207 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSQLSetColumnType(&yylhsminor.yy343, &yymsp[-3].minor.yy0); + tSQLSetColumnType(&yygotominor.yy223, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy369; // negative value of name length - tSQLSetColumnType(&yylhsminor.yy343, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207; // negative value of name length + tSQLSetColumnType(&yygotominor.yy223, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy343 = yylhsminor.yy343; break; - case 107: /* signed ::= INTEGER */ -{ yylhsminor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy369 = yylhsminor.yy369; + case 110: /* signed ::= INTEGER */ + case 111: /* signed ::= PLUS INTEGER */ yytestcase(yyruleno==111); +{ yygotominor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 108: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 112: /* signed ::= MINUS INTEGER */ +{ yygotominor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 109: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy369 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} - break; - case 110: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + case 113: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0); } break; - case 111: /* create_table_args ::= LP columnlist RP */ + case 114: /* create_table_args ::= LP columnlist RP */ { - yymsp[-2].minor.yy398 = tSetCreateSQLElems(yymsp[-1].minor.yy511, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, yymsp[-2].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); + yygotominor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy523, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); + setSQLInfo(pInfo, yygotominor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 112: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + case 115: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ { - yymsp[-6].minor.yy398 = tSetCreateSQLElems(yymsp[-5].minor.yy511, yymsp[-1].minor.yy511, NULL, NULL, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); + yygotominor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy523, yymsp[-1].minor.yy523, NULL, NULL, NULL, TSQL_CREATE_STABLE); + setSQLInfo(pInfo, yygotominor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 113: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + case 116: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yymsp[-6].minor.yy398 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy156, NULL, TSQL_CREATE_TABLE_FROM_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); + yygotominor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy498, NULL, TSQL_CREATE_TABLE_FROM_STABLE); + setSQLInfo(pInfo, yygotominor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 114: /* create_table_args ::= AS select */ + case 117: /* create_table_args ::= AS select */ { - yymsp[-1].minor.yy398 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy444, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, yymsp[-1].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); + yygotominor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM); + setSQLInfo(pInfo, yygotominor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 115: /* columnlist ::= columnlist COMMA column */ -{yylhsminor.yy511 = tFieldListAppend(yymsp[-2].minor.yy511, &yymsp[0].minor.yy343); } - yymsp[-2].minor.yy511 = yylhsminor.yy511; + case 118: /* columnlist ::= columnlist COMMA column */ +{yygotominor.yy523 = tFieldListAppend(yymsp[-2].minor.yy523, &yymsp[0].minor.yy223); } break; - case 116: /* columnlist ::= column */ -{yylhsminor.yy511 = tFieldListAppend(NULL, &yymsp[0].minor.yy343);} - yymsp[0].minor.yy511 = yylhsminor.yy511; + case 119: /* columnlist ::= column */ +{yygotominor.yy523 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);} break; - case 117: /* column ::= ids typename */ + case 120: /* column ::= ids typename */ { - tSQLSetColumnInfo(&yylhsminor.yy343, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy343); + tSQLSetColumnInfo(&yygotominor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223); } - yymsp[-1].minor.yy343 = yylhsminor.yy343; break; - case 118: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy156 = tVariantListAppend(yymsp[-2].minor.yy156, &yymsp[0].minor.yy506, -1); } - yymsp[-2].minor.yy156 = yylhsminor.yy156; + case 121: /* tagitemlist ::= tagitemlist COMMA tagitem */ +{ yygotominor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); } break; - case 119: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[0].minor.yy506, -1); } - yymsp[0].minor.yy156 = yylhsminor.yy156; + case 122: /* tagitemlist ::= tagitem */ +{ yygotominor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } break; - case 120: /* tagitem ::= INTEGER */ - case 121: /* tagitem ::= FLOAT */ yytestcase(yyruleno==121); - case 122: /* tagitem ::= STRING */ yytestcase(yyruleno==122); - case 123: /* tagitem ::= BOOL */ yytestcase(yyruleno==123); -{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy506, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy506 = yylhsminor.yy506; + case 123: /* tagitem ::= INTEGER */ + case 124: /* tagitem ::= FLOAT */ yytestcase(yyruleno==124); + case 125: /* tagitem ::= STRING */ yytestcase(yyruleno==125); + case 126: /* tagitem ::= BOOL */ yytestcase(yyruleno==126); +{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yygotominor.yy134, &yymsp[0].minor.yy0); } break; - case 124: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy506, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy506 = yylhsminor.yy506; + case 127: /* tagitem ::= NULL */ +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yygotominor.yy134, &yymsp[0].minor.yy0); } break; - case 125: /* tagitem ::= MINUS INTEGER */ - case 126: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==126); - case 127: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==127); - case 128: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==128); + case 128: /* tagitem ::= MINUS INTEGER */ + case 129: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==129); + case 130: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==130); + case 131: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==131); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy506, &yymsp[-1].minor.yy0); + tVariantCreate(&yygotominor.yy134, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy506 = yylhsminor.yy506; break; - case 129: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 132: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy444 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy158, yymsp[-9].minor.yy156, yymsp[-8].minor.yy190, yymsp[-4].minor.yy156, yymsp[-3].minor.yy156, &yymsp[-7].minor.yy340, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy156, &yymsp[0].minor.yy414, &yymsp[-1].minor.yy414); + yygotominor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy498, yymsp[-8].minor.yy64, yymsp[-4].minor.yy498, yymsp[-3].minor.yy498, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy498, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216); } - yymsp[-11].minor.yy444 = yylhsminor.yy444; break; - case 130: /* union ::= select */ -{ yylhsminor.yy333 = setSubclause(NULL, yymsp[0].minor.yy444); } - yymsp[0].minor.yy333 = yylhsminor.yy333; + case 133: /* union ::= select */ +{ yygotominor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); } break; - case 131: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy333 = yymsp[-1].minor.yy333; } + case 134: /* union ::= LP union RP */ +{ yygotominor.yy231 = yymsp[-1].minor.yy231; } break; - case 132: /* union ::= union UNION ALL select */ -{ yylhsminor.yy333 = appendSelectClause(yymsp[-3].minor.yy333, yymsp[0].minor.yy444); } - yymsp[-3].minor.yy333 = yylhsminor.yy333; + case 135: /* union ::= union UNION ALL select */ +{ yygotominor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); } break; - case 133: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy333 = appendSelectClause(yymsp[-5].minor.yy333, yymsp[-1].minor.yy444); } - yymsp[-5].minor.yy333 = yylhsminor.yy333; + case 136: /* union ::= union UNION ALL LP select RP */ +{ yygotominor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); } break; - case 134: /* cmd ::= union */ -{ setSQLInfo(pInfo, yymsp[0].minor.yy333, NULL, TSDB_SQL_SELECT); } + case 137: /* cmd ::= union */ +{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); } break; - case 135: /* select ::= SELECT selcollist */ + case 138: /* select ::= SELECT selcollist */ { - yylhsminor.yy444 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy158, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yygotominor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy444 = yylhsminor.yy444; break; - case 136: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy158 = yymsp[-1].minor.yy158;} - yymsp[-1].minor.yy158 = yylhsminor.yy158; + case 139: /* sclp ::= selcollist COMMA */ +{yygotominor.yy290 = yymsp[-1].minor.yy290;} break; - case 137: /* sclp ::= */ -{yymsp[1].minor.yy158 = 0;} + case 140: /* sclp ::= */ +{yygotominor.yy290 = 0;} break; - case 138: /* selcollist ::= sclp expr as */ + case 141: /* selcollist ::= sclp expr as */ { - yylhsminor.yy158 = tSQLExprListAppend(yymsp[-2].minor.yy158, yymsp[-1].minor.yy190, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yygotominor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-2].minor.yy158 = yylhsminor.yy158; break; - case 139: /* selcollist ::= sclp STAR */ + case 142: /* selcollist ::= sclp STAR */ { tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy158 = tSQLExprListAppend(yymsp[-1].minor.yy158, pNode, 0); + yygotominor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; break; - case 140: /* as ::= AS ids */ -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } + case 143: /* as ::= AS ids */ + case 144: /* as ::= ids */ yytestcase(yyruleno==144); +{ yygotominor.yy0 = yymsp[0].minor.yy0; } break; - case 141: /* as ::= ids */ -{ yylhsminor.yy0 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy0 = yylhsminor.yy0; + case 145: /* as ::= */ +{ yygotominor.yy0.n = 0; } break; - case 142: /* as ::= */ -{ yymsp[1].minor.yy0.n = 0; } + case 146: /* from ::= FROM tablelist */ + case 161: /* orderby_opt ::= ORDER BY sortlist */ yytestcase(yyruleno==161); + case 169: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==169); +{yygotominor.yy498 = yymsp[0].minor.yy498;} break; - case 143: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy156 = yymsp[0].minor.yy156;} - break; - case 144: /* tablelist ::= ids cpxName */ + case 147: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy156 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[-1].minor.yy0, -1); // table alias name + yygotominor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yygotominor.yy498, &yymsp[-1].minor.yy0, -1); // table alias name } - yymsp[-1].minor.yy156 = yylhsminor.yy156; break; - case 145: /* tablelist ::= ids cpxName ids */ + case 148: /* tablelist ::= ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy156 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[0].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yygotominor.yy498, &yymsp[0].minor.yy0, -1); } - yymsp[-2].minor.yy156 = yylhsminor.yy156; break; - case 146: /* tablelist ::= tablelist COMMA ids cpxName */ + case 149: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy156 = tVariantListAppendToken(yymsp[-3].minor.yy156, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[-1].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yygotominor.yy498, &yymsp[-1].minor.yy0, -1); } - yymsp[-3].minor.yy156 = yylhsminor.yy156; break; - case 147: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 150: /* tablelist ::= tablelist COMMA ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy156 = tVariantListAppendToken(yymsp[-4].minor.yy156, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[0].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yymsp[-4].minor.yy498, &yymsp[-2].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yygotominor.yy498, &yymsp[0].minor.yy0, -1); } - yymsp[-4].minor.yy156 = yylhsminor.yy156; break; - case 148: /* tmvar ::= VARIABLE */ -{yylhsminor.yy0 = yymsp[0].minor.yy0;} - yymsp[0].minor.yy0 = yylhsminor.yy0; + case 151: /* tmvar ::= VARIABLE */ +{yygotominor.yy0 = yymsp[0].minor.yy0;} break; - case 149: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy340.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy340.offset.n = 0; yymsp[-3].minor.yy340.offset.z = NULL; yymsp[-3].minor.yy340.offset.type = 0;} + case 152: /* interval_opt ::= INTERVAL LP tmvar RP */ +{yygotominor.yy532.interval = yymsp[-1].minor.yy0; yygotominor.yy532.offset.n = 0; yygotominor.yy532.offset.z = NULL; yygotominor.yy532.offset.type = 0;} break; - case 150: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy340.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy340.offset = yymsp[-1].minor.yy0;} + case 153: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yygotominor.yy532.interval = yymsp[-3].minor.yy0; yygotominor.yy532.offset = yymsp[-1].minor.yy0;} break; - case 151: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy340, 0, sizeof(yymsp[1].minor.yy340));} + case 154: /* interval_opt ::= */ +{memset(&yygotominor.yy532, 0, sizeof(yygotominor.yy532));} break; - case 152: /* fill_opt ::= */ -{yymsp[1].minor.yy156 = 0; } + case 155: /* fill_opt ::= */ +{yygotominor.yy498 = 0; } break; - case 153: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 156: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy156, &A, -1, 0); - yymsp[-5].minor.yy156 = yymsp[-1].minor.yy156; + tVariantListInsert(yymsp[-1].minor.yy498, &A, -1, 0); + yygotominor.yy498 = yymsp[-1].minor.yy498; } break; - case 154: /* fill_opt ::= FILL LP ID RP */ + case 157: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy156 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 155: /* sliding_opt ::= SLIDING LP tmvar RP */ -{yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } + case 158: /* sliding_opt ::= SLIDING LP tmvar RP */ +{yygotominor.yy0 = yymsp[-1].minor.yy0; } break; - case 156: /* sliding_opt ::= */ -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } + case 159: /* sliding_opt ::= */ +{yygotominor.yy0.n = 0; yygotominor.yy0.z = NULL; yygotominor.yy0.type = 0; } break; - case 157: /* orderby_opt ::= */ - case 165: /* groupby_opt ::= */ yytestcase(yyruleno==165); -{yymsp[1].minor.yy156 = 0;} + case 160: /* orderby_opt ::= */ + case 168: /* groupby_opt ::= */ yytestcase(yyruleno==168); +{yygotominor.yy498 = 0;} break; - case 158: /* orderby_opt ::= ORDER BY sortlist */ - case 166: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==166); -{yymsp[-2].minor.yy156 = yymsp[0].minor.yy156;} - break; - case 159: /* sortlist ::= sortlist COMMA item sortorder */ + case 162: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy156 = tVariantListAppend(yymsp[-3].minor.yy156, &yymsp[-1].minor.yy506, yymsp[0].minor.yy112); + yygotominor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } - yymsp[-3].minor.yy156 = yylhsminor.yy156; break; - case 160: /* sortlist ::= item sortorder */ + case 163: /* sortlist ::= item sortorder */ { - yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[-1].minor.yy506, yymsp[0].minor.yy112); + yygotominor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } - yymsp[-1].minor.yy156 = yylhsminor.yy156; break; - case 161: /* item ::= ids cpxName */ + case 164: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy506, &yymsp[-1].minor.yy0); + tVariantCreate(&yygotominor.yy134, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy506 = yylhsminor.yy506; break; - case 162: /* sortorder ::= ASC */ -{yymsp[0].minor.yy112 = TSDB_ORDER_ASC; } + case 165: /* sortorder ::= ASC */ +{yygotominor.yy46 = TSDB_ORDER_ASC; } break; - case 163: /* sortorder ::= DESC */ -{yymsp[0].minor.yy112 = TSDB_ORDER_DESC;} + case 166: /* sortorder ::= DESC */ +{yygotominor.yy46 = TSDB_ORDER_DESC;} break; - case 164: /* sortorder ::= */ -{yymsp[1].minor.yy112 = TSDB_ORDER_ASC;} + case 167: /* sortorder ::= */ +{yygotominor.yy46 = TSDB_ORDER_ASC;} break; - case 167: /* grouplist ::= grouplist COMMA item */ + case 170: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy156 = tVariantListAppend(yymsp[-2].minor.yy156, &yymsp[0].minor.yy506, -1); + yygotominor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); } - yymsp[-2].minor.yy156 = yylhsminor.yy156; break; - case 168: /* grouplist ::= item */ + case 171: /* grouplist ::= item */ { - yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[0].minor.yy506, -1); + yygotominor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } - yymsp[0].minor.yy156 = yylhsminor.yy156; break; - case 169: /* having_opt ::= */ - case 179: /* where_opt ::= */ yytestcase(yyruleno==179); - case 217: /* expritem ::= */ yytestcase(yyruleno==217); -{yymsp[1].minor.yy190 = 0;} + case 172: /* having_opt ::= */ + case 182: /* where_opt ::= */ yytestcase(yyruleno==182); + case 220: /* expritem ::= */ yytestcase(yyruleno==220); +{yygotominor.yy64 = 0;} break; - case 170: /* having_opt ::= HAVING expr */ - case 180: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==180); -{yymsp[-1].minor.yy190 = yymsp[0].minor.yy190;} + case 173: /* having_opt ::= HAVING expr */ + case 183: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==183); + case 219: /* expritem ::= expr */ yytestcase(yyruleno==219); +{yygotominor.yy64 = yymsp[0].minor.yy64;} break; - case 171: /* limit_opt ::= */ - case 175: /* slimit_opt ::= */ yytestcase(yyruleno==175); -{yymsp[1].minor.yy414.limit = -1; yymsp[1].minor.yy414.offset = 0;} + case 174: /* limit_opt ::= */ + case 178: /* slimit_opt ::= */ yytestcase(yyruleno==178); +{yygotominor.yy216.limit = -1; yygotominor.yy216.offset = 0;} break; - case 172: /* limit_opt ::= LIMIT signed */ - case 176: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==176); -{yymsp[-1].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-1].minor.yy414.offset = 0;} + case 175: /* limit_opt ::= LIMIT signed */ + case 179: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==179); +{yygotominor.yy216.limit = yymsp[0].minor.yy207; yygotominor.yy216.offset = 0;} break; - case 173: /* limit_opt ::= LIMIT signed OFFSET signed */ - case 177: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==177); -{yymsp[-3].minor.yy414.limit = yymsp[-2].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[0].minor.yy369;} + case 176: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 180: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==180); +{yygotominor.yy216.limit = yymsp[-2].minor.yy207; yygotominor.yy216.offset = yymsp[0].minor.yy207;} break; - case 174: /* limit_opt ::= LIMIT signed COMMA signed */ - case 178: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==178); -{yymsp[-3].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[-2].minor.yy369;} + case 177: /* limit_opt ::= LIMIT signed COMMA signed */ + case 181: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==181); +{yygotominor.yy216.limit = yymsp[0].minor.yy207; yygotominor.yy216.offset = yymsp[-2].minor.yy207;} break; - case 181: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy190 = yymsp[-1].minor.yy190; } + case 184: /* expr ::= LP expr RP */ +{yygotominor.yy64 = yymsp[-1].minor.yy64; } break; - case 182: /* expr ::= ID */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 185: /* expr ::= ID */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} break; - case 183: /* expr ::= ID DOT ID */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 186: /* expr ::= ID DOT ID */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} break; - case 184: /* expr ::= ID DOT STAR */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 187: /* expr ::= ID DOT STAR */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} break; - case 185: /* expr ::= INTEGER */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 188: /* expr ::= INTEGER */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} break; - case 186: /* expr ::= MINUS INTEGER */ - case 187: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==187); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy190 = yylhsminor.yy190; + case 189: /* expr ::= MINUS INTEGER */ + case 190: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==190); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} break; - case 188: /* expr ::= FLOAT */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 191: /* expr ::= FLOAT */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} break; - case 189: /* expr ::= MINUS FLOAT */ - case 190: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==190); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy190 = yylhsminor.yy190; + case 192: /* expr ::= MINUS FLOAT */ + case 193: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==193); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} break; - case 191: /* expr ::= STRING */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 194: /* expr ::= STRING */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} break; - case 192: /* expr ::= NOW */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 195: /* expr ::= NOW */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } break; - case 193: /* expr ::= VARIABLE */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 196: /* expr ::= VARIABLE */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} break; - case 194: /* expr ::= BOOL */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 197: /* expr ::= BOOL */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} break; - case 195: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy190 = tSQLExprCreateFunction(yymsp[-1].minor.yy158, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy190 = yylhsminor.yy190; + case 198: /* expr ::= ID LP exprlist RP */ +{ yygotominor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } break; - case 196: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy190 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy190 = yylhsminor.yy190; + case 199: /* expr ::= ID LP STAR RP */ +{ yygotominor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } break; - case 197: /* expr ::= expr IS NULL */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, NULL, TK_ISNULL);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 200: /* expr ::= expr IS NULL */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, NULL, TK_ISNULL);} break; - case 198: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-3].minor.yy190, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy190 = yylhsminor.yy190; + case 201: /* expr ::= expr IS NOT NULL */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-3].minor.yy64, NULL, TK_NOTNULL);} break; - case 199: /* expr ::= expr LT expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LT);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 202: /* expr ::= expr LT expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);} break; - case 200: /* expr ::= expr GT expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_GT);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 203: /* expr ::= expr GT expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);} break; - case 201: /* expr ::= expr LE expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LE);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 204: /* expr ::= expr LE expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);} break; - case 202: /* expr ::= expr GE expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_GE);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 205: /* expr ::= expr GE expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);} break; - case 203: /* expr ::= expr NE expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_NE);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 206: /* expr ::= expr NE expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);} break; - case 204: /* expr ::= expr EQ expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_EQ);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 207: /* expr ::= expr EQ expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);} break; - case 205: /* expr ::= expr AND expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_AND);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 208: /* expr ::= expr AND expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);} break; - case 206: /* expr ::= expr OR expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_OR); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 209: /* expr ::= expr OR expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); } break; - case 207: /* expr ::= expr PLUS expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_PLUS); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 210: /* expr ::= expr PLUS expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); } break; - case 208: /* expr ::= expr MINUS expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_MINUS); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 211: /* expr ::= expr MINUS expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); } break; - case 209: /* expr ::= expr STAR expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_STAR); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 212: /* expr ::= expr STAR expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); } break; - case 210: /* expr ::= expr SLASH expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_DIVIDE);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 213: /* expr ::= expr SLASH expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);} break; - case 211: /* expr ::= expr REM expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_REM); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 214: /* expr ::= expr REM expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); } break; - case 212: /* expr ::= expr LIKE expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LIKE); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 215: /* expr ::= expr LIKE expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); } break; - case 213: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-4].minor.yy190, (tSQLExpr*)yymsp[-1].minor.yy158, TK_IN); } - yymsp[-4].minor.yy190 = yylhsminor.yy190; + case 216: /* expr ::= expr IN LP exprlist RP */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); } break; - case 214: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy158 = tSQLExprListAppend(yymsp[-2].minor.yy158,yymsp[0].minor.yy190,0);} - yymsp[-2].minor.yy158 = yylhsminor.yy158; + case 217: /* exprlist ::= exprlist COMMA expritem */ +{yygotominor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);} break; - case 215: /* exprlist ::= expritem */ -{yylhsminor.yy158 = tSQLExprListAppend(0,yymsp[0].minor.yy190,0);} - yymsp[0].minor.yy158 = yylhsminor.yy158; + case 218: /* exprlist ::= expritem */ +{yygotominor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);} break; - case 216: /* expritem ::= expr */ -{yylhsminor.yy190 = yymsp[0].minor.yy190;} - yymsp[0].minor.yy190 = yylhsminor.yy190; - break; - case 218: /* cmd ::= RESET QUERY CACHE */ + case 221: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 219: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 222: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy511, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 220: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 223: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3021,14 +2335,14 @@ static YYACTIONTYPE yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 221: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 224: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy511, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 222: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 225: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3039,7 +2353,7 @@ static YYACTIONTYPE yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 223: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 226: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3053,49 +2367,55 @@ static YYACTIONTYPE yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 224: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 227: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy506, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1); SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 225: /* cmd ::= KILL CONNECTION INTEGER */ + case 228: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 226: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 229: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 227: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 230: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: break; -/********** End reduce actions ************************************************/ }; - assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); - - /* It is not possible for a REDUCE to be followed by an error */ - assert( yyact!=YY_ERROR_ACTION ); - - yymsp += yysize+1; - yypParser->yytos = yymsp; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact, "... then shift"); - return yyact; + yygoto = yyRuleInfo[yyruleno].lhs; + yysize = yyRuleInfo[yyruleno].nrhs; + yypParser->yyidx -= yysize; + yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); + if( yyact < YYNSTATE ){ +#ifdef NDEBUG + /* If we are not debugging and the reduce action popped at least + ** one element off the stack, then we can push the new element back + ** onto the stack here, and skip the stack overflow test in yy_shift(). + ** That gives a significant speed improvement. */ + if( yysize ){ + yypParser->yyidx++; + yymsp -= yysize-1; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yymsp->minor = yygotominor; + }else +#endif + { + yy_shift(yypParser,yyact,yygoto,&yygotominor); + } + }else{ + assert( yyact == YYNSTATE + YYNRULE + 1 ); + yy_accept(yypParser); + } } /* @@ -3105,20 +2425,16 @@ static YYACTIONTYPE yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); } #endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ -/************ Begin %parse_failure code ***************************************/ -/************ End %parse_failure code *****************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } #endif /* YYNOERRORRECOVERY */ @@ -3128,12 +2444,10 @@ static void yy_parse_failed( static void yy_syntax_error( yyParser *yypParser, /* The parser */ int yymajor, /* The major type of the error token */ - ParseTOKENTYPE yyminor /* The minor type of the error token */ + YYMINORTYPE yyminor /* The minor type of the error token */ ){ - ParseARG_FETCH - ParseCTX_FETCH -#define TOKEN yyminor -/************ Begin %syntax_error code ****************************************/ + ParseARG_FETCH; +#define TOKEN (yyminor.yy0) pInfo->valid = false; int32_t outputBufLen = tListLen(pInfo->pzErrMsg); @@ -3156,9 +2470,7 @@ static void yy_syntax_error( } assert(len <= outputBufLen); -/************ End %syntax_error code ******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } /* @@ -3167,24 +2479,17 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); } #endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - assert( yypParser->yytos==yypParser->yystack ); + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser accepts */ -/*********** Begin %parse_accept code *****************************************/ -/*********** End %parse_accept code *******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } /* The main parser program. @@ -3213,54 +2518,50 @@ void Parse( ParseARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - YYACTIONTYPE yyact; /* The parser action. */ -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyact; /* The parser action. */ int yyendofinput; /* True if we are at the end of input */ -#endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif - yyParser *yypParser = (yyParser*)yyp; /* The parser */ - ParseCTX_FETCH - ParseARG_STORE + yyParser *yypParser; /* The parser */ - assert( yypParser->yytos!=0 ); -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - yyendofinput = (yymajor==0); + /* (re)initialize the parser, if necessary */ + yypParser = (yyParser*)yyp; + if( yypParser->yyidx<0 ){ +#if YYSTACKDEPTH<=0 + if( yypParser->yystksz <=0 ){ + /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ + yyminorunion = yyzerominor; + yyStackOverflow(yypParser, &yyminorunion); + return; + } #endif + yypParser->yyidx = 0; + yypParser->yyerrcnt = -1; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; + } + yyminorunion.yy0 = yyminor; + yyendofinput = (yymajor==0); + ParseARG_STORE; - yyact = yypParser->yytos->stateno; #ifndef NDEBUG if( yyTraceFILE ){ - if( yyact < YY_MIN_REDUCE ){ - fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact); - }else{ - fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); - } + fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); } #endif do{ - assert( yyact==yypParser->yytos->stateno ); - yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); - if( yyact >= YY_MIN_REDUCE ){ - yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, - yyminor ParseCTX_PARAM); - }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); -#ifndef YYNOERRORRECOVERY + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + if( yyactyyerrcnt--; -#endif - break; - }else if( yyact==YY_ACCEPT_ACTION ){ - yypParser->yytos--; - yy_accept(yypParser); - return; + yymajor = YYNOCODE; + }else if( yyact < YYNSTATE + YYNRULE ){ + yy_reduce(yypParser,yyact-YYNSTATE); }else{ assert( yyact == YY_ERROR_ACTION ); - yyminorunion.yy0 = yyminor; #ifdef YYERRORSYMBOL int yymx; #endif @@ -3290,9 +2591,9 @@ void Parse( ** */ if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminor); + yy_syntax_error(yypParser,yymajor,yyminorunion); } - yymx = yypParser->yytos->major; + yymx = yypParser->yystack[yypParser->yyidx].major; if( yymx==YYERRORSYMBOL || yyerrorhit ){ #ifndef NDEBUG if( yyTraceFILE ){ @@ -3300,31 +2601,30 @@ void Parse( yyTracePrompt,yyTokenName[yymajor]); } #endif - yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); yymajor = YYNOCODE; }else{ - while( yypParser->yytos >= yypParser->yystack - && (yyact = yy_find_reduce_action( - yypParser->yytos->stateno, - YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE + while( + yypParser->yyidx >= 0 && + yymx != YYERRORSYMBOL && + (yyact = yy_find_reduce_action( + yypParser->yystack[yypParser->yyidx].stateno, + YYERRORSYMBOL)) >= YYNSTATE ){ yy_pop_parser_stack(yypParser); } - if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ + if( yypParser->yyidx < 0 || yymajor==0 ){ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ - yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + YYMINORTYPE u2; + u2.YYERRSYMDT = 0; + yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); } } yypParser->yyerrcnt = 3; yyerrorhit = 1; - if( yymajor==YYNOCODE ) break; - yyact = yypParser->yytos->stateno; #elif defined(YYNOERRORRECOVERY) /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to ** do any kind of error recovery. Instead, simply invoke the syntax @@ -3333,9 +2633,10 @@ void Parse( ** Applications can set this macro (for example inside %include) if ** they intend to abandon the parse upon the first syntax error seen. */ - yy_syntax_error(yypParser,yymajor, yyminor); + yy_syntax_error(yypParser,yymajor,yyminorunion); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - break; + yymajor = YYNOCODE; + #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -3347,45 +2648,16 @@ void Parse( ** three input tokens have been successfully shifted. */ if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor, yyminor); + yy_syntax_error(yypParser,yymajor,yyminorunion); } yypParser->yyerrcnt = 3; yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); if( yyendofinput ){ yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif } - break; + yymajor = YYNOCODE; #endif } - }while( yypParser->yytos>yypParser->yystack ); -#ifndef NDEBUG - if( yyTraceFILE ){ - yyStackEntry *i; - char cDiv = '['; - fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); - for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ - fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); - cDiv = ' '; - } - fprintf(yyTraceFILE,"]\n"); - } -#endif + }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); return; } - -/* -** Return the fallback token corresponding to canonical token iToken, or -** 0 if iToken has no fallback. -*/ -int ParseFallback(int iToken){ -#ifdef YYFALLBACK - assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); - return yyFallback[iToken]; -#else - (void)iToken; - return 0; -#endif -} diff --git a/tests/pytest/alter/db_update_options.py b/tests/pytest/alter/db_update_options.py new file mode 100644 index 0000000000..224e0f25b0 --- /dev/null +++ b/tests/pytest/alter/db_update_options.py @@ -0,0 +1,71 @@ + +# -*- coding: utf-8 -*- + +import random +import string +import subprocess +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + def run(self): + tdLog.debug("check database") + tdSql.prepare() + + # check default update value + sql = "create database if not exists db" + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + sql = "alter database db update 1" + + # check update value + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,1) + + + sql = "alter database db update 0" + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + sql = "alter database db update -1" + tdSql.error(sql) + + sql = "alter database db update 100" + tdSql.error(sql) + + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + tdSql.execute('drop database db') + tdSql.error('create database db update 100') + tdSql.error('create database db update -1') + + tdSql.execute('create database db update 1') + + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,1) + + tdSql.execute('drop database db') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 4a83966f5c7cce0fcaf86dbe3e1410fdf8b6e338 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Sep 2020 10:02:23 +0800 Subject: [PATCH 010/122] TD-1194 --- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbMemTable.c | 10 +- src/tsdb/src/tsdbMeta.c | 6 +- src/tsdb/src/tsdbRead.c | 18 +- src/util/inc/tskiplist.h | 188 +++-------------- src/util/src/tskiplist.c | 405 ++++++++++++++---------------------- 6 files changed, 210 insertions(+), 419 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 95b7010038..b515752bc3 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -438,7 +438,7 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) return NULL; - return *(SDataRow *)SL_GET_NODE_DATA(node); + return (SDataRow)SL_GET_NODE_DATA(node); } static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index c58460e18f..9c76a09ca3 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -91,11 +91,11 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); - int64_t oldSize = SL_GET_SIZE(pTableData->pData); - if (tSkipListPut(pTableData->pData, (void *)(&pRow), sizeof(void *)) == NULL) { + int64_t oldSize = SL_SIZE(pTableData->pData); + if (tSkipListPut(pTableData->pData, pRow) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); } else { - int64_t deltaSize = SL_GET_SIZE(pTableData->pData) - oldSize; + int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize; if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; @@ -427,7 +427,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], - pCfg->update ? SL_APPEND_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); + pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -447,7 +447,7 @@ static void tsdbFreeTableData(STableData *pTableData) { } } -static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); } +static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); } static void *tsdbCommitData(void *arg) { STsdbRepo * pRepo = (STsdbRepo *)arg; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 76f9d51b87..391cb55985 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -643,7 +643,7 @@ static void tsdbOrgMeta(void *pHandle) { } static char *getTagIndexKey(const void *pData) { - STable *pTable = *(STable **)pData; + STable *pTable = (STable *)pData; STSchema *pSchema = tsdbGetTableTagSchema(pTable); STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); @@ -900,7 +900,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper pTable->pSuper = pSTable; - tSkipListPut(pSTable->pIndex, (void *)(&pTable), sizeof(STable *)); + tSkipListPut(pSTable->pIndex, (void *)pTable); if (refSuper) T_REF_INC(pSTable); return 0; @@ -1182,7 +1182,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); } else { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_GET_SIZE(pTable->pIndex) + 1)); + tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1)); } else { tlen = sizeof(SListNode) + sizeof(SActObj); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 320f1c3765..f390208af1 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -386,7 +386,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); assert(node != NULL); - SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); + SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", @@ -408,7 +408,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); assert(node != NULL); - SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); + SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", @@ -438,14 +438,14 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { - rmem = *(SDataRow *)SL_GET_NODE_DATA(node); + rmem = (SDataRow)SL_GET_NODE_DATA(node); } } if (pCheckInfo->iiter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { - rimem = *(SDataRow *)SL_GET_NODE_DATA(node); + rimem = (SDataRow)SL_GET_NODE_DATA(node); } } @@ -1358,8 +1358,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; @@ -1864,9 +1864,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) { while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode); - STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(list, &info); } @@ -2434,7 +2434,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC static bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; - STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); + STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); char* val = NULL; diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index f1e09501e9..1eccac6646 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -30,43 +30,27 @@ extern "C" { // For key property setting #define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists #define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key -#define SL_UPDATA_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert -#define SL_APPEND_DUP_KEY (uint8_t)0x3 // Update duplicate key by append +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert // For thread safety setting #define SL_THREAD_SAFE (uint8_t)0x4 typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); -/** - * the skip list node is located in a consecutive memory area, - * the format of skip list node is as follows: - * +------------+-----------------------+------------------------+-----+------+ - * | node level | forward pointer array | backward pointer array | key | data | - * +------------+-----------------------+------------------------+-----+------+ - */ typedef struct SSkipListNode { - uint8_t level; + uint8_t level; + uint8_t flags; + void * pData; + struct SSkipListNode *forwards[]; } SSkipListNode; -#define SL_IS_THREAD_SAFE(flags) ((flags)&SL_THREAD_SAFE) -#define SL_DUP_MODE(flags) ((flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_NODE_DELETED_FLAG (uint8_t)0x1 -#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES) - -#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)] -#define SL_GET_BACKWARD_POINTER(n, _l) \ - ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)] - -#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level)) -#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n))) - -#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0))) -#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0))) - -#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) -#define SL_GET_SIZE(s) (s)->size -#define SL_GET_TSIZE(s) (s)->tsize +#define SL_GET_NODE_DATA(n) (n)->pData +#define SL_IS_NODE_DELETED(n) ((n)->flags & SL_NODE_DELETED_FLAG) +#define SL_SET_NODE_DELETED(n) (n)->flags |= SL_NODE_DELETED_FLAG +#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] +#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] /* * @version 0.3 @@ -116,13 +100,6 @@ typedef struct tSkipListState { uint64_t nTotalElapsedTimeForInsert; } tSkipListState; -typedef struct SSkipListKeyInfo { - uint8_t dupKey : 2; // if allow duplicated key in the skip list - uint8_t type : 4; // key type - uint8_t freeNode:2; // free node when destroy the skiplist - uint8_t len; // maximum key length, used in case of string key -} SSkipListKeyInfo; - typedef struct SSkipList { __compar_fn_t comparFn; __sl_key_fn_t keyFn; @@ -130,10 +107,10 @@ typedef struct SSkipList { uint16_t len; uint8_t maxLevel; uint8_t flags; - uint8_t type; // static info above + uint8_t type; // static info above uint8_t level; - uint32_t size; // not including duplicate keys - uint32_t tsize; // including duplicate keys + uint32_t size; // semantic meaning of size + uint32_t tsize; // # of all skiplist nodes in this SL SSkipListNode * pHead; // point to the first element SSkipListNode * pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE @@ -141,130 +118,33 @@ typedef struct SSkipList { #endif } SSkipList; -/* - * iterate the skiplist - * this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may - * continue iterating the skiplist, so add the reference count for skiplist - * TODO add the ref for skip list when one iterator is created - */ typedef struct SSkipListIterator { SSkipList * pSkipList; SSkipListNode *cur; - int32_t step; // the number of nodes that have been checked already - int32_t order; // order of the iterator + int32_t step; // the number of nodes that have been checked already + int32_t order; // order of the iterator } SSkipListIterator; -/** - * - * @param nMaxLevel maximum skip list level - * @param keyType type of key - * @param dupKey allow the duplicated key in the skip list - * @return - */ -SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn); +#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) +#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) +#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) +#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) +#define SL_SIZE(s) (s)->size +#define SL_TSIZE(s) (s)->tsize -/** - * - * @param pSkipList - * @return NULL will always be returned - */ -void *tSkipListDestroy(SSkipList *pSkipList); - -/** - * - * @param pSkipList - * @param level - * @param headSize - */ -void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize); - -/** - * put the data into the skiplist - * If failed, NULL will be returned, otherwise, the pNode will be returned. - * - * @param pSkipList - * @param pData - * @param dataLen - * @return - */ -SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen); - -/** - * put the skip list node into the skip list. - * If failed, NULL will be returned, otherwise, the pNode will be returned. - * - * @param pSkipList - * @param pNode - * @return - */ -SSkipListNode *tSkipListPutNode(SSkipList *pSkipList, SSkipListNode *pNode); - -/** - * get *all* nodes which key are equivalent to pKey - * - * @param pSkipList - * @param pKey - * @return - */ -SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); - -/** - * display skip list of the given level, for debug purpose only - * @param pSkipList - * @param nlevel - */ -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); - -/** - * create skiplist iterator - * @param pSkipList - * @return - */ +SSkipList * tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn); +void tSkipListDestroy(SSkipList *pSkipList); +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); +SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); - -/** - * create skip list iterator from the given node and specified the order - * @param pSkipList - * @param pNode start position, instead of the first node in skip list - * @param order traverse order of the iterator - * @return - */ -SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order); - -/** - * forward the skip list iterator - * @param iter - * @return - */ -bool tSkipListIterNext(SSkipListIterator *iter); - -/** - * get the element of skip list node - * @param iter - * @return - */ -SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); - -/** - * destroy the skip list node - * @param iter - * @return - */ -void *tSkipListDestroyIter(SSkipListIterator *iter); - -/* - * remove nodes of the pKey value. - * If more than one node has the same value, all will be removed - * - * @Return - * the count of removed nodes - */ -uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); - -/* - * remove the specified node in parameters - */ -void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); +bool tSkipListIterNext(SSkipListIterator *iter); +SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); +void * tSkipListDestroyIter(SSkipListIterator *iter); +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); +void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); #ifdef __cplusplus } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index c3e8ee41bd..3da137140f 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -19,30 +19,26 @@ #include "tulog.h" #include "tutil.h" -#define DO_MEMSET_PTR_AREA(n) \ - do { \ - int32_t _l = (n)->level; \ - memset(pNode, 0, SL_NODE_HEADER_SIZE(_l)); \ - (n)->level = _l; \ - } while (0) +#define DO_MEMSET_PTR_AREA(n) memset((n)->forwards, 0, ((n)->level * 2)) -static int initForwardBackwardPtr(SSkipList *pSkipList); -static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); -static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); -static void tSkipListCorrectLevel(SSkipList *pSkipList); +static int initForwardBackwardPtr(SSkipList *pSkipList); +static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); +static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); +static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData); +static SSkipListNode * tSkipListNewNode(uint8_t level); +#define tSkipListFreeNode(n) taosTFree((n)) static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); -static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn) { SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); - if (pSkipList == NULL) { - return NULL; - } + if (pSkipList == NULL) return NULL; if (maxLevel > MAX_SKIP_LIST_LEVEL) { maxLevel = MAX_SKIP_LIST_LEVEL; @@ -55,19 +51,20 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, u pSkipList->keyFn = fn; pSkipList->comparFn = getKeyComparFunc(keyType); - // pSkipList->level = 1; // TODO: check if 1 is valid if (initForwardBackwardPtr(pSkipList) < 0) { - taosTFree(pSkipList); + tSkipListDestroy(pSkipList); return NULL; } - if (SL_IS_THREAD_SAFE(flags)) { + if (SL_IS_THREAD_SAFE(pSkipList)) { pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t)); + if (pSkipList->lock == NULL) { + tSkipListDestroy(pSkipList); + return NULL; + } if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { - taosTFree(pSkipList->pHead); - taosTFree(pSkipList); - + tSkipListDestroy(pSkipList); return NULL; } } @@ -81,17 +78,17 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, u return pSkipList; } -void *tSkipListDestroy(SSkipList *pSkipList) { - if (pSkipList == NULL) return NULL; +void tSkipListDestroy(SSkipList *pSkipList) { + if (pSkipList == NULL) return; tSkipListWLock(pSkipList); - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); + SSkipListNode *pNode = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, 0); while (pNode != pSkipList->pTail) { SSkipListNode *pTemp = pNode; - pNode = SL_GET_FORWARD_POINTER(pNode, 0); - taosTFree(pTemp); + pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0); + tSkipListFreeNode(pTemp); } tSkipListUnlock(pSkipList); @@ -100,42 +97,35 @@ void *tSkipListDestroy(SSkipList *pSkipList) { taosTFree(pSkipList->lock); } - taosTFree(pSkipList->pHead); + tSkipListFreeNode(pSkipList->pHead); + tSkipListFreeNode(pSkipList->pTail); taosTFree(pSkipList); - return NULL; } -void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) { - if (pSkipList == NULL) { - *level = 1; - *headSize = SL_NODE_HEADER_SIZE(*level); - return; - } - - *level = getSkipListRandLevel(pSkipList); - *headSize = SL_NODE_HEADER_SIZE(*level); -} - -SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen) { +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { if (pSkipList == NULL || pData == NULL) return NULL; + SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; + uint8_t dupMode = SL_DUP_MODE(pSkipList); + SSkipListNode *pNode = NULL; + tSkipListWLock(pSkipList); - int32_t level = getSkipListRandLevel(pSkipList); + bool hasDup = tSkipListGetPosToPut(pSkipList, forward, pData); - SSkipListNode *pNode = (SSkipListNode *)malloc(SL_NODE_HEADER_SIZE(level) + dataLen); - if (pNode == NULL) { - tSkipListUnlock(pSkipList); - return NULL; - } + if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) { + if (dupMode == SL_UPDATE_DUP_KEY) { + pNode = SL_NODE_GET_FORWARD_POINTER(forward[0], 0); + atomic_store_ptr(&(pNode->pData), pData); + pNode->flags &= (~(SL_NODE_DELETED_FLAG)); + } + } else { + pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList)); + if (pNode != NULL) { + pNode->pData = pData; - pNode->level = level; - memcpy(SL_GET_NODE_DATA(pNode), pData, dataLen); - - if (tSkipListPutNodeImpl(pSkipList, pNode) == NULL) { - tSkipListUnlock(pSkipList); - taosTFree(pNode); - return NULL; + tSkipListDoInsert(pSkipList, forward, pNode); + } } tSkipListUnlock(pSkipList); @@ -143,41 +133,6 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen) { return pNode; } -SSkipListNode *tSkipListPutNode(SSkipList *pSkipList, SSkipListNode *pNode) { - SSkipListNode *pRetNode = NULL; - - if (pSkipList == NULL || pNode == NULL) return NULL; - - tSkipListWLock(pSkipList); - pRetNode = tSkipListPutNodeImpl(pSkipList, pNode); - tSkipListUnlock(pSkipList); - - return pRetNode; -} - -SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { - SArray *sa = taosArrayInit(1, POINTER_BYTES); - - tSkipListRLock(pSkipList); - - SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); - while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); - if (p == pSkipList->pTail) { - break; - } - if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { - break; - } - taosArrayPush(sa, &p); - pNode = p; - } - - tSkipListUnlock(pSkipList); - - return sa; -} - uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { uint32_t count = 0; @@ -185,7 +140,7 @@ uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); if (p == pSkipList->pTail) { break; } @@ -205,6 +160,31 @@ uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { return count; } +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { + SArray *sa = taosArrayInit(1, POINTER_BYTES); + + tSkipListRLock(pSkipList); + + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + while (1) { + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + if (!SL_IS_NODE_DELETED(p)) { + taosArrayPush(sa, &p); + } + pNode = p; + } + + tSkipListUnlock(pSkipList); + + return sa; +} + void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { tSkipListWLock(pSkipList); tSkipListRemoveNodeImpl(pSkipList, pNode); @@ -240,68 +220,22 @@ bool tSkipListIterNext(SSkipListIterator *iter) { if (iter->pSkipList == NULL) return false; SSkipList *pSkipList = iter->pSkipList; - uint8_t dupMod = SL_DUP_MODE(pSkipList->flags); tSkipListRLock(pSkipList); - if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate - if (dupMod == SL_APPEND_DUP_KEY) { - if (iter->cur == pSkipList->pHead) { - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); - iter->step++; - } else { - while (true) { - iter->step++; - SSkipListNode *pNode = iter->cur; - iter->cur = SL_GET_FORWARD_POINTER(pNode, 0); - - if (iter->cur == pSkipList->pTail) break; - if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { - continue; - } else { - break; - } - } - } - - if (iter->cur != pSkipList->pTail) { - while (true) { - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(iter->cur, 0); - if (pNode == pSkipList->pTail) break; - if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { - iter->step++; - iter->cur = pNode; - } else { - break; - } - } - } - } else { - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + if (iter->order == TSDB_ORDER_ASC) { + while (true) { + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); iter->step++; + if (iter->cur == pSkipList->pTail) break; + if (!SL_IS_NODE_DELETED(iter->cur)) break; } - } else { // descending order iterate - if (dupMod == SL_APPEND_DUP_KEY) { - if (iter->cur == pSkipList->pTail) { - iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); - iter->step++; - } else { - while (true) { - SSkipListNode *pNode = iter->cur; - iter->cur = SL_GET_BACKWARD_POINTER(pNode, 0); - - if (iter->cur == pSkipList->pHead) break; - if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { - iter->cur = pNode; - continue; - } else { - break; - } - } - } - } else { - iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + } else { + while (true) { + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); iter->step++; + if (iter->cur == pSkipList->pHead) break; + if (!SL_IS_NODE_DELETED(iter->cur)) break; } } @@ -332,7 +266,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { return; } - SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); int32_t id = 1; char * prev = NULL; @@ -364,35 +298,34 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { prev = SL_GET_NODE_KEY(pSkipList, p); - p = SL_GET_FORWARD_POINTER(p, nlevel - 1); + p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1); } } -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode, bool hasDupKey) { - uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); - - // FIXME: this may cause the level of skiplist change - if (dupMode == SL_UPDATA_DUP_KEY && hasDupKey) { - tSkipListRemoveNodeImpl(pSkipList, forward[0]); - tSkipListCorrectLevel(pSkipList); - } - +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { DO_MEMSET_PTR_AREA(pNode); for (int32_t i = 0; i < pNode->level; ++i) { - SSkipListNode *x = forward[i]; - SL_GET_BACKWARD_POINTER(pNode, i) = x; + if (i >= pSkipList->level) { + SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; + SL_NODE_GET_BACKWARD_POINTER(pNode, i) = pSkipList->pHead; + SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; + SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; + } else { + SSkipListNode *x = forward[i]; + SL_NODE_GET_BACKWARD_POINTER(pNode, i) = x; - SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); - SL_GET_BACKWARD_POINTER(next, i) = pNode; + SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(x, i); + SL_NODE_GET_BACKWARD_POINTER(next, i) = pNode; - SL_GET_FORWARD_POINTER(pNode, i) = next; - SL_GET_FORWARD_POINTER(x, i) = pNode; + SL_NODE_GET_FORWARD_POINTER(pNode, i) = next; + SL_NODE_GET_FORWARD_POINTER(x, i) = pNode; + } } - if (!(dupMode == SL_APPEND_DUP_KEY && hasDupKey)) { - pSkipList->size += 1; - } + if (pSkipList->level < pNode->level) pSkipList->level = pNode->level; + + pSkipList->size += 1; pSkipList->tsize += 1; } @@ -411,78 +344,73 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t } static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) { - if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + if (pSkipList->lock) { return pthread_rwlock_wrlock(pSkipList->lock); } return 0; } static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) { - if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + if (pSkipList->lock) { return pthread_rwlock_rdlock(pSkipList->lock); } return 0; } static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { - if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + if (pSkipList->lock) { return pthread_rwlock_unlock(pSkipList->lock); } return 0; } -static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData) { int compare = 1; bool hasDupKey = false; - char * pNodeKey = SL_GET_NODE_KEY(pSkipList, pNode); - uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + char * pDataKey = pSkipList->keyFn(pData); if (pSkipList->size == 0) { - for (int i = 0; i < pNode->level; i++) { + for (int i = 0; i < pSkipList->level; i++) { forward[i] = pSkipList->pHead; } } else { char *pKey = NULL; // Compare min key - pKey = SL_GET_SL_MIN_KEY(pSkipList); - compare = pSkipList->comparFn(pNodeKey, pKey); - if ((dupMode == SL_APPEND_DUP_KEY && compare < 0) || (dupMode != SL_APPEND_DUP_KEY && compare <= 0)) { - for (int i = 0; i < pNode->level; i++) { + pKey = SL_GET_MIN_KEY(pSkipList); + compare = pSkipList->comparFn(pDataKey, pKey); + if (compare <= 0) { + for (int i = 0; i < pSkipList->level; i++) { forward[i] = pSkipList->pHead; } + return (compare == 0); } // Compare max key - pKey = SL_GET_SL_MAX_KEY(pSkipList); - compare = pSkipList->comparFn(pNodeKey, pKey); - if ((dupMode == SL_DISCARD_DUP_KEY && compare > 0) || (dupMode != SL_DISCARD_DUP_KEY && compare >= 0)) { - for (int i = 0; i < pNode->level; i++) { - forward[i] = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); + pKey = SL_GET_MAX_KEY(pSkipList); + compare = pSkipList->comparFn(pDataKey, pKey); + if (compare > 0) { + for (int i = 0; i < pSkipList->level; i++) { + forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i); } return (compare == 0); } SSkipListNode *px = pSkipList->pHead; - for (int i = pNode->level - 1; i >= 0; --i) { - if (i >= pSkipList->level) { - forward[i] = pSkipList->pHead; - continue; - } - - SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); + for (int i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(px, i); while (p != pSkipList->pTail) { pKey = SL_GET_NODE_KEY(pSkipList, p); - compare = pSkipList->comparFn(pKey, pNodeKey); - if (compare == 0 && hasDupKey == false) hasDupKey = true; - if ((dupMode == SL_APPEND_DUP_KEY && compare > 0) || (dupMode != SL_APPEND_DUP_KEY && compare >= 0)) { + compare = pSkipList->comparFn(pKey, pDataKey); + if (compare >= 0) { + if (compare == 0 && !hasDupKey) hasDupKey = true; break; } else { px = p; - p = SL_GET_FORWARD_POINTER(px, i); + p = SL_NODE_GET_FORWARD_POINTER(px, i); } } @@ -493,40 +421,35 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, return hasDupKey; } -static bool tSkipListIsNodeDup(SSkipList *pSkipList, SSkipListNode *pNode) { - SSkipListNode *pPrevNode = SL_GET_BACKWARD_POINTER(pNode, 0); - SSkipListNode *pNextNode = SL_GET_FORWARD_POINTER(pNode, 0); - char * pNodeKey = SL_GET_NODE_KEY(pSkipList, pNode); - char * pPrevNodeKey = (pPrevNode == pSkipList->pHead) ? NULL : SL_GET_NODE_KEY(pSkipList, pPrevNode); - char * pNextNodeKey = (pNextNode == pSkipList->pTail) ? NULL : SL_GET_NODE_KEY(pSkipList, pNextNode); - - return ((pPrevNodeKey != NULL && pSkipList->comparFn(pNodeKey, pPrevNodeKey) == 0) || - (pNextNodeKey != NULL && pSkipList->comparFn(pNodeKey, pNextNodeKey) == 0)); -} - static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { int32_t level = pNode->level; - uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + uint8_t dupMode = SL_DUP_MODE(pSkipList); - bool sizeReduce = !(dupMode == SL_APPEND_DUP_KEY && tSkipListIsNodeDup(pSkipList, pNode)); + if (dupMode == SL_UPDATE_DUP_KEY) { + if (SL_IS_NODE_DELETED(pNode)) { + return; + } else { + SL_SET_NODE_DELETED(pNode); + pSkipList->size--; + } + } else { + for (int32_t j = level - 1; j >= 0; --j) { + SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j); + SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j); - for (int32_t j = level - 1; j >= 0; --j) { - SSkipListNode *prev = SL_GET_BACKWARD_POINTER(pNode, j); - SSkipListNode *next = SL_GET_FORWARD_POINTER(pNode, j); + SL_NODE_GET_FORWARD_POINTER(prev, j) = next; + SL_NODE_GET_BACKWARD_POINTER(next, j) = prev; + } - SL_GET_FORWARD_POINTER(prev, j) = next; - SL_GET_BACKWARD_POINTER(next, j) = prev; + tSkipListFreeNode(pNode); + pSkipList->size--; + pSkipList->tsize--; } - - taosTFree(pNode); - - if (sizeReduce) pSkipList->size--; - pSkipList->tsize--; } // Function must be called after calling tSkipListRemoveNodeImpl() function static void tSkipListCorrectLevel(SSkipList *pSkipList) { - while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { + while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { pSkipList->level -= 1; } } @@ -587,12 +510,12 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ if (order == TSDB_ORDER_ASC) { pNode = pSkipList->pHead; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, i); while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); if (comparFn(key, val) < 0) { pNode = p; - p = SL_GET_FORWARD_POINTER(p, i); + p = SL_NODE_GET_FORWARD_POINTER(p, i); } else { break; } @@ -601,12 +524,12 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ } else { pNode = pSkipList->pTail; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_BACKWARD_POINTER(pNode, i); + SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(pNode, i); while (p != pSkipList->pHead) { char *key = SL_GET_NODE_KEY(pSkipList, p); if (comparFn(key, val) > 0) { pNode = p; - p = SL_GET_BACKWARD_POINTER(p, i); + p = SL_NODE_GET_BACKWARD_POINTER(p, i); } else { break; } @@ -621,46 +544,34 @@ static int initForwardBackwardPtr(SSkipList *pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; // head info - pSkipList->pHead = (SSkipListNode *)malloc(SL_NODE_HEADER_SIZE(maxLevel) * 2); - if (pSkipList->pHead == NULL) { + pSkipList->pHead = tSkipListNewNode(maxLevel); + if (pSkipList->pHead == NULL) return -1; + + // tail info + pSkipList->pTail = tSkipListNewNode(maxLevel); + if (pSkipList->pTail == NULL) { + tSkipListFreeNode(pSkipList->pHead); return -1; } - pSkipList->pHead->level = maxLevel; - - // tail info - pSkipList->pTail = (SSkipListNode *)POINTER_SHIFT(pSkipList->pHead, SL_NODE_HEADER_SIZE(maxLevel)); - pSkipList->pTail->level = maxLevel; - for (uint32_t i = 0; i < maxLevel; ++i) { - SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; - SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; + SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; + SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; } return 0; } -static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { - SSkipListNode *pRetNode = NULL; - SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; +static SSkipListNode *tSkipListNewNode(uint8_t level) { + int32_t tsize = sizeof(SSkipListNode) + sizeof(SSkipListNode *) * level * 2; - int hasDupKey = tSkipListGetPosToPut(pSkipList, forward, pNode); - if (SL_DUP_MODE(pSkipList->flags) == SL_DISCARD_DUP_KEY && hasDupKey) { - pRetNode = NULL; - } else { - pRetNode = pNode; - tSkipListDoInsert(pSkipList, forward, pNode, hasDupKey); + SSkipListNode *pNode = (SSkipListNode *)calloc(1, tsize); + if (pNode == NULL) return NULL; - if (pNode->level > pSkipList->level) pSkipList->level = pNode->level; - } - - return pRetNode; + pNode->level = level; + return pNode; } -// static void tSkipListSeek(SSkipList *pSkipList, char *key, int order) { -// // TODO -// } - // static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey, // int32_t cond, SSkipListNode ***pRes) { // pthread_rwlock_rdlock(&pSkipList->lock); @@ -774,7 +685,7 @@ static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SS // } // // // compress the minimum level of skip list -// while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { +// while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { // pSkipList->level -= 1; // } // From 5ea611f2c8284c22f5895cdb3ae1cc69a53a505d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Sep 2020 13:44:22 +0800 Subject: [PATCH 011/122] TD-1194 --- src/tsdb/src/tsdbMemTable.c | 7 ------ src/util/inc/tskiplist.h | 2 ++ src/util/src/tskiplist.c | 43 ++++++++++++++++++++++++++++--------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 9c76a09ca3..e139b29d5e 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -40,13 +40,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { TSKEY key = dataRowKey(row); SMemTable * pMemTable = pRepo->mem; STableData *pTableData = NULL; - // SSkipList * pSList = NULL; - - // if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && - // pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { - // pTableData = pMemTable->tData[TABLE_TID(pTable)]; - // pSList = pTableData->pData; - // } void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); if (pRow == NULL) { diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 1eccac6646..030a9d69c1 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -145,6 +145,8 @@ SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); void * tSkipListDestroyIter(SSkipListIterator *iter); uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); +SSkipListKey tSkipListGetMinKey(SSkipList *pSkipList); +SSkipListKey tSkipListGetMaxKey(SSkipList *pSkipList); #ifdef __cplusplus } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 3da137140f..603a9759b8 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -19,8 +19,6 @@ #include "tulog.h" #include "tutil.h" -#define DO_MEMSET_PTR_AREA(n) memset((n)->forwards, 0, ((n)->level * 2)) - static int initForwardBackwardPtr(SSkipList *pSkipList); static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); @@ -117,7 +115,10 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { if (dupMode == SL_UPDATE_DUP_KEY) { pNode = SL_NODE_GET_FORWARD_POINTER(forward[0], 0); atomic_store_ptr(&(pNode->pData), pData); - pNode->flags &= (~(SL_NODE_DELETED_FLAG)); + if (SL_IS_NODE_DELETED(pNode)) { + pNode->flags &= (~(SL_NODE_DELETED_FLAG)); + pSkipList->size++; + } } } else { pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList)); @@ -136,6 +137,8 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { uint32_t count = 0; + if (SL_DUP_MODE(pSkipList) == SL_DISCARD_DUP_KEY) return 0; + tSkipListWLock(pSkipList); SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); @@ -199,8 +202,8 @@ SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) { } SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order) { - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - assert(pSkipList != NULL); + ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + ASSERT(pSkipList != NULL); SSkipListIterator *iter = doCreateSkipListIterator(pSkipList, order); if (val == NULL) { @@ -274,7 +277,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); if (prev != NULL) { - assert(pSkipList->comparFn(prev, key) < 0); + ASSERT(pSkipList->comparFn(prev, key) < 0); } switch (pSkipList->type) { @@ -302,9 +305,29 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { } } -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { - DO_MEMSET_PTR_AREA(pNode); +SSkipListKey tSkipListGetMinKey(SSkipList *pSkipList) { + if (pSkipList == NULL || SL_SIZE(pSkipList) == 0) return NULL; + SSkipListNode *pNode = pSkipList->pHead; + while ((pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0)) != pSkipList->pTail) { + if (!SL_IS_NODE_DELETED(pNode)) return pSkipList->keyFn(pNode->pData); + } + + return NULL; +} + +SSkipListKey tSkipListGetMaxKey(SSkipList *pSkipList) { + if (pSkipList == NULL || SL_SIZE(pSkipList) == 0) return NULL; + + SSkipListNode *pNode = pSkipList->pTail; + while ((pNode = SL_NODE_GET_BACKWARD_POINTER(pNode, 0)) != pSkipList->pHead) { + if (!SL_IS_NODE_DELETED(pNode)) return pSkipList->keyFn(pNode->pData); + } + + return NULL; +} + +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { for (int32_t i = 0; i < pNode->level; ++i) { if (i >= pSkipList->level) { SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; @@ -365,7 +388,7 @@ static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { } static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData) { - int compare = 1; + int compare = 0; bool hasDupKey = false; char * pDataKey = pSkipList->keyFn(pData); @@ -497,7 +520,7 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { } } - assert(level <= pSkipList->maxLevel); + ASSERT(level <= pSkipList->maxLevel); return level; } From e118dc87a590f6e0167b75494a606bbe13adee7f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Sep 2020 14:48:36 +0800 Subject: [PATCH 012/122] TD-1194 --- src/query/src/qAst.c | 20 ++++++++++---------- src/tsdb/src/tsdbMeta.c | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index ef98dbeda7..ca33f6a067 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -432,7 +432,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr break; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal @@ -450,7 +450,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_GREATER) { continue; } else { - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); comp = false; } @@ -465,7 +465,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr continue; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } @@ -476,11 +476,11 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { + if (comp) { continue; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } @@ -504,7 +504,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_LESS) { continue; } else { - STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); comp = false; // no need to compare anymore } @@ -518,7 +518,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { - STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } } @@ -684,7 +684,7 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki while (tSkipListIterNext(iter)) { SSkipListNode *pNode = tSkipListIterGet(iter); if (filterItem(pExpr, pNode, param)) { - taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); + taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); } } tSkipListDestroyIter(iter); @@ -699,7 +699,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SSkipListNode *pNode = tSkipListIterGet(iter); char * pData = SL_GET_NODE_DATA(pNode); - tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); + tstr *name = (tstr*) tsdbGetTableName((void*) pData); // todo speed up by using hash if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { @@ -713,7 +713,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, } if (addToResult) { - STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(res, &info); } } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 391cb55985..1914ff08ed 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -925,7 +925,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { SSkipListNode *pNode = taosArrayGetP(res, i); // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); - if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need + if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need tSkipListRemoveNode(pSTable->pIndex, pNode); } } @@ -1229,7 +1229,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) { } while (tSkipListIterNext(pIter)) { - STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE); pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable); } @@ -1254,7 +1254,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { tsdbWLockRepoMeta(pRepo); while (tSkipListIterNext(pIter)) { - STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); tsdbRemoveTableFromMeta(pRepo, tTable, false, false); } From bbf367772976725144942c9b7367abe644c098af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 27 Sep 2020 10:27:53 +0800 Subject: [PATCH 013/122] TD-1548 --- src/common/inc/tdataformat.h | 74 +++++-- src/common/src/tdataformat.c | 69 ++++--- src/inc/taosdef.h | 1 + src/tsdb/inc/tsdbMain.h | 22 +- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbMemTable.c | 192 ++++++++++++----- src/tsdb/src/tsdbMeta.c | 11 +- src/tsdb/src/tsdbRWHelper.c | 387 ++++++++++++++++++++--------------- src/util/inc/tskiplist.h | 22 +- src/util/src/tskiplist.c | 87 +++----- 10 files changed, 516 insertions(+), 351 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index aa39d5b0b2..8124075e6c 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -119,6 +119,33 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); +// ----------------- Semantic timestamp key definition +typedef uint64_t TKEY; + +#define TKEY_INVALID UINT64_MAX +#define TKEY_NULL TKEY_INVALID +#define TKEY_NEGATIVE_FLAG (((TKEY)1) << (sizeof(TKEY) * 8 - 1)) +#define TKEY_DELETE_FLAG (((TKEY)1) << (sizeof(TKEY) * 8 - 2)) +#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) + +#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) +#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) +#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) +#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) +#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) + +static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { + TSKEY key1 = tdGetKey(*(TKEY *)tkey1); + TSKEY key2 = tdGetKey(*(TKEY *)tkey2); + + if (key1 < key2) { + return -1; + } else if (key1 > key2) { + return 1; + } else { + return 0; + } +} // ----------------- Data row structure /* A data row, the format is like below: @@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); * +----------+----------+---------------------------------+---------------------------------+ * | len | sversion | First part | Second part | * +----------+----------+---------------------------------+---------------------------------+ + * + * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ typedef void *SDataRow; @@ -137,11 +166,13 @@ typedef void *SDataRow; #define dataRowLen(r) (*(uint16_t *)(r)) #define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) -#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r))) +#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) +#define dataRowKey(r) tdGetKey(dataRowTKey(r)) #define dataRowSetLen(r, l) (dataRowLen(r) = (l)) #define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) +#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) SDataRow tdNewDataRowFromSchema(STSchema *pSchema); void tdFreeDataRow(SDataRow row); @@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); - switch (type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); - memcpy(ptr, value, varDataTLen(value)); - dataRowLen(row) += varDataTLen(value); - break; - default: + if (IS_VAR_DATA_TYPE(type)) { + *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); + memcpy(ptr, value, varDataTLen(value)); + dataRowLen(row) += varDataTLen(value); + } else { + if (offset == 0) { + ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); + TKEY tvalue = tdGetTKEY(*(TSKEY *)value); + memcpy(POINTER_SHIFT(row, toffset), &tvalue, TYPE_BYTES[type]); + } else { memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); - break; + } } return 0; @@ -171,12 +204,10 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i // NOTE: offset here including the header size static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { - switch (type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - default: - return POINTER_SHIFT(row, offset); + if (IS_VAR_DATA_TYPE(type)) { + return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); + } else { + return POINTER_SHIFT(row, offset); } } @@ -243,9 +274,14 @@ typedef struct { } SDataCols; #define keyCol(pCols) (&((pCols)->cols[0])) // Key column -#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)] -#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0) -#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) +#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] +#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) +#define dataColsTKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0) +#define dataColsKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TSDB_DATA_BIGINT_NULL : dataColsKeyAt(pCols, 0) +#define dataColsTKeyLast(pCols) \ + (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) +#define dataColsKeyLast(pCols) \ + (((pCols)->numOfRows == 0) ? TSDB_DATA_BIGINT_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 0c446b6d4f..b45cbd796d 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -423,30 +423,41 @@ void tdResetDataCols(SDataCols *pCols) { } void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { - ASSERT(dataColsKeyLast(pCols) < dataRowKey(row)); + ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); int rcol = 0; int dcol = 0; - while (dcol < pCols->numOfCols) { - SDataCol *pDataCol = &(pCols->cols[dcol]); - if (rcol >= schemaNCols(pSchema)) { - dataColSetNullAt(pDataCol, pCols->numOfRows); - dcol++; - continue; + if (dataRowDeleted(row)) { + for (; dcol < pCols->numOfCols; dcol++) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (dcol == 0) { + dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints); + } else { + dataColSetNullAt(pDataCol, pCols->numOfRows); + } } + } else { + while (dcol < pCols->numOfCols) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (rcol >= schemaNCols(pSchema)) { + dataColSetNullAt(pDataCol, pCols->numOfRows); + dcol++; + continue; + } - STColumn *pRowCol = schemaColAt(pSchema, rcol); - if (pRowCol->colId == pDataCol->colId) { - void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE); - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); - dcol++; - rcol++; - } else if (pRowCol->colId < pDataCol->colId) { - rcol++; - } else { - dataColSetNullAt(pDataCol, pCols->numOfRows); - dcol++; + STColumn *pRowCol = schemaColAt(pSchema, rcol); + if (pRowCol->colId == pDataCol->colId) { + void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE); + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); + dcol++; + rcol++; + } else if (pRowCol->colId < pDataCol->colId) { + rcol++; + } else { + dataColSetNullAt(pDataCol, pCols->numOfRows); + dcol++; + } } } pCols->numOfRows++; @@ -511,8 +522,12 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i while (target->numOfRows < tRows) { if (*iter1 >= limit1 && *iter2 >= limit2) break; - TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; - TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; + TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1); + TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); + TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2); + TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2); + + ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); if (key1 < key2) { for (int i = 0; i < src1->numOfCols; i++) { @@ -525,12 +540,14 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i target->numOfRows++; (*iter1)++; - } else { - for (int i = 0; i < src2->numOfCols; i++) { - ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, - target->maxPoints); + } else if (key1 >= key2) { + if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { + for (int i = 0; i < src2->numOfCols; i++) { + ASSERT(target->cols[i].type == src2->cols[i].type); + if (src2->cols[i].len > 0) { + dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, + target->maxPoints); + } } } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index e9aa6a7050..a6cf76c473 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -75,6 +75,7 @@ extern const int32_t TYPE_BYTES[11]; #define TSDB_DATA_SMALLINT_NULL 0x8000 #define TSDB_DATA_INT_NULL 0x80000000 #define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL #define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN #define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index b515752bc3..7d40d7f00a 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -320,6 +320,15 @@ typedef struct { void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; +typedef struct { + int rowsInserted; + int rowsUpdated; + int rowsDeleteSucceed; + int rowsDeleteFailed; + int nOperations; + TSKEY keyFirst; + TSKEY keyLast; +} SMergeInfo; // ------------------ tsdbScan.c typedef struct { SFileGroup fGroup; @@ -422,7 +431,7 @@ void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); // ------------------ tsdbMemTable.c -int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); +int tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); @@ -430,7 +439,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem) void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TSKEY* filterKeys, int nFilterKeys, bool keepDup); + TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; @@ -443,11 +452,18 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) return -1; + if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; return dataRowKey(row); } +static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { + SDataRow row = tsdbNextIterRow(pIter); + if (row == NULL) return TKEY_NULL; + + return dataRowTKey(row); +} + static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { ASSERT(pRepo != NULL); if (pRepo->mem == NULL) return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 494f17807e..a0ff1a6b95 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -765,7 +765,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY return -1; } - if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1; + if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1; (*affectedrows)++; points++; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index e139b29d5e..cd688d46a7 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -32,14 +32,31 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row); // ---------------- INTERNAL FUNCTIONS ---------------- -int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { +int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { STsdbCfg * pCfg = &pRepo->config; STsdbMeta * pMeta = pRepo->tsdbMeta; + TKEY tkey = dataRowTKey(row); TSKEY key = dataRowKey(row); SMemTable * pMemTable = pRepo->mem; STableData *pTableData = NULL; + bool isRowDelete = TKEY_IS_DELETED(tkey); + + if (isRowDelete) { + if (!pCfg->update) { + tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo)); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + return -1; + } + + if (key > TABLE_LASTKEY(pTable)) { + tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64, + REPO_ID(pRepo), key, TABLE_LASTKEY(pTable)); + return 0; + } + } void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); if (pRow == NULL) { @@ -88,8 +105,10 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (tSkipListPut(pTableData->pData, pRow) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); } else { + // TODO: may need to refact here int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize; - if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; + if ((!isRowDelete) && (TABLE_LASTKEY(pTable) < key)) TABLE_LASTKEY(pTable) = key; + if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; pMemTable->numOfRows += deltaSize; @@ -99,8 +118,9 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { pTableData->numOfRows += deltaSize; } - tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key); + tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), + isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), + key); return 0; } @@ -270,66 +290,120 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { return 0; } +/** + * This is an important function to load data or try to load data from memory skiplist iterator. + * + * This function load memory data until: + * 1. iterator ends + * 2. data key exceeds maxKey + * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead + * 4. operations in pCols not exceeds its max capacity if pCols is given + * + * The function try to move as mush as possible. + */ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TSKEY *filterKeys, int nFilterKeys, bool keepDup) { - ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); + TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { + ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL); if (pIter == NULL) return 0; STSchema *pSchema = NULL; - int numOfRows = 0; - TSKEY keyNext = 0; + TSKEY rowKey = 0; + TSKEY fKey = 0; + bool isRowDel = false; int filterIter = 0; + SDataRow row = NULL; - if (nFilterKeys != 0) { // for filter purpose - ASSERT(filterKeys != NULL); - keyNext = tsdbNextIterKey(pIter); - if (keyNext < 0 || keyNext > maxKey) return numOfRows; - void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE); - filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY))); + memset(pMergeInfo, 0, sizeof(*pMergeInfo)); + pMergeInfo->keyFirst = INT64_MAX; + pMergeInfo->keyLast = INT64_MIN; + + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); } - do { - SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) break; + if (nFilterKeys == 0 || filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } - keyNext = dataRowKey(row); - if (keyNext > maxKey) break; + while (true) { + if (fKey == INT64_MAX && rowKey == INT64_MAX) break; - bool keyFiltered = false; - if (nFilterKeys != 0) { - while (true) { - if (filterIter >= nFilterKeys) break; - if (keyNext == filterKeys[filterIter]) { - keyFiltered = true; - filterIter++; - break; - } else if (keyNext < filterKeys[filterIter]) { - break; + if (fKey < rowKey) { + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + } else if (fKey > rowKey) { + if (isRowDel) { + pMergeInfo->rowsDeleteFailed++; + } else { + if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsInserted++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } + } else { + if (isRowDel) { + ASSERT(!keepDup); + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsDeleteSucceed++; + pMergeInfo->nOperations++; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } else { + if (keepDup) { + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsUpdated++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } else { - filterIter++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); } } - } - if (!keyFiltered) { - if (numOfRows >= maxRowsToRead) break; - numOfRows++; - } - - if (!keyFiltered || keepDup) { - if (pCols) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); - if (pSchema == NULL) { - ASSERT(0); - } - } - - tdAppendDataRowToDataCol(row, pSchema, pCols); + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); } - } - } while (tSkipListIterNext(pIter)); - return numOfRows; + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + } + } + + return 0; } // ---------------- LOCAL FUNCTIONS ---------------- @@ -420,7 +494,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], - pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); + tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -562,7 +636,7 @@ static void tsdbEndCommit(STsdbRepo *pRepo) { static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { for (int i = 0; i < nIters; i++) { TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); - if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; + if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1; } return 0; } @@ -758,5 +832,21 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { taosTFree(tData); + return 0; +} + +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) { + if (pCols) { + if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) { + *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); + if (*ppSchema == NULL) { + ASSERT(false); + return -1; + } + } + + tdAppendDataRowToDataCol(row, *ppSchema, pCols); + } + return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 1914ff08ed..53a1b3f5bf 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { if (pTable != NULL) { tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); - return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + goto _err; } if (pCfg->type == TSDB_CHILD_TABLE) { @@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { } pTable->tagVal = NULL; STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), SL_ALLOW_DUP_KEY, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -745,7 +746,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { T_REF_INC(pTable); - tsdbTrace("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), + tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); return pTable; @@ -1155,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { buf = tdDecodeSchema(buf, &(pTable->tagSchema)); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), SL_ALLOW_DUP_KEY, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, + SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbFreeTable(pTable); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 6c6a738899..c6d4ae7a5e 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -27,6 +27,7 @@ #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) #define TSDB_KEY_COL_OFFSET 0 #define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock)) +#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last) static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock, @@ -34,7 +35,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD static int compareKeyBlock(const void *arg1, const void *arg2); static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize); static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo); static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static void tsdbResetHelperFileImpl(SRWHelper *pHelper); static int tsdbInitHelperFile(SRWHelper *pHelper); @@ -61,8 +62,10 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock); static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx); -static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update); +static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps); +static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -279,7 +282,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols while (true) { ASSERT(blkIdx <= (int)pIdx->numOfBlocks); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - if (keyFirst < 0 || keyFirst > maxKey) break; // iter over + if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) { if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1; @@ -925,7 +928,7 @@ _err: return -1; } -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) { +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) { ASSERT(pCompBlock->numOfSubBlocks == 0); SCompIdx *pIdx = &(pHelper->curCompIdx); @@ -958,9 +961,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pSCompBlock->numOfSubBlocks++; ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); pSCompBlock->len += sizeof(SCompBlock); - pSCompBlock->numOfRows += rowsAdded; - pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst); - pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast); + pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; + pSCompBlock->keyFirst = pMergeInfo->keyFirst; + pSCompBlock->keyLast = pMergeInfo->keyLast; pIdx->len += sizeof(SCompBlock); } else { // Need to create two sub-blocks void *ptr = NULL; @@ -989,11 +992,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId ((SCompBlock *)ptr)[1] = *pCompBlock; pSCompBlock->numOfSubBlocks = 2; - pSCompBlock->numOfRows += rowsAdded; + pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo); pSCompBlock->len = sizeof(SCompBlock) * 2; - pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst); - pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast); + pSCompBlock->keyFirst = pMergeInfo->keyFirst; + pSCompBlock->keyLast = pMergeInfo->keyLast; pIdx->len += (sizeof(SCompBlock) * 2); } @@ -1047,6 +1050,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int return 0; } +static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { + SCompIdx *pCompIdx = &(pHelper->curCompIdx); + + ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks); + + SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx); + SCompBlock compBlock = *pCompBlock; + ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); + + if (pCompIdx->numOfBlocks == 1) { + memset(pCompIdx, 0, sizeof(*pCompIdx)); + } else { + int tsize = 0; + + if (compBlock.numOfSubBlocks > 1) { + tsize = pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks); + + ASSERT(tsize > 0); + memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset), + POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks), + tsize); + + pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks; + } + + tsize = pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo); + ASSERT(tsize > 0); + memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize); + + pCompIdx->len -= sizeof(SCompBlock); + + pCompIdx->numOfBlocks--; + pCompIdx->hasLast = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last; + pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast; + } + + return 0; +} + static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { pHelper->idxH.numOfIdx = 0; pHelper->idxH.curIdx = 0; @@ -1439,12 +1481,14 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { } static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SCompBlock compBlock = {0}; + STsdbCfg * pCfg = &(pHelper->pRepo->config); + STable * pTable = pCommitIter->pTable; + SCompIdx * pIdx = &(pHelper->curCompIdx); + TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); + int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; + SCompBlock compBlock = {0}; + SMergeInfo mergeInfo = {0}; + SMergeInfo *pMergeInfo = &mergeInfo; ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey); if (pIdx->hasLast) { // append to with last block @@ -1452,39 +1496,47 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, - pDataCols, NULL, 0, pCfg->update); - ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); - if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && - pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1; - } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols, + NULL, 0, pCfg->update, pMergeInfo); - if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; + if (pDataCols->numOfRows > 0) { + ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); + + if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && + pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; + if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1; + } else { + if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); + + if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows); + + if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; + } + + if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; } - - if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; } else { ASSERT(!pHelper->hasOldLastBlock); tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update); - ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; + if (pDataCols->numOfRows > 0) { + ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); + if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; + if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; + } } #ifndef NDEBUG TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter); - ASSERT(keyNext < 0 || keyNext > pIdx->maxKey); + ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey); #endif return 0; @@ -1492,13 +1544,15 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - SCompBlock compBlock = {0}; - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SDataCols *pDataCols0 = pHelper->pDataCols[0]; + STsdbCfg * pCfg = &(pHelper->pRepo->config); + STable * pTable = pCommitIter->pTable; + SCompIdx * pIdx = &(pHelper->curCompIdx); + SCompBlock compBlock = {0}; + TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); + int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; + SDataCols * pDataCols0 = pHelper->pDataCols[0]; + SMergeInfo mergeInfo = {0}; + SMergeInfo *pMergeInfo = &mergeInfo; SSkipListIterator slIter = {0}; @@ -1509,120 +1563,82 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pCompBlock != NULL); int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); - if (pCompBlock->last) { - ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1); - int16_t colId = 0; - slIter = *(pCommitIter->pIter); - if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; - ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); + ASSERT((!TSDB_IS_LAST_BLOCK(pCompBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); - int rows1 = defaultRowsInBlock - pCompBlock->numOfRows; - int rows2 = tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows, pCfg->update); - if (!pCfg->update && rows2 == 0) { // all data filtered out - *(pCommitIter->pIter) = slIter; - } else { - if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock && - pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); - ASSERT(rowsRead == rows2 && rowsRead <= pDataCols->numOfRows && pDataCols->numOfRows > 0); - if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; - tblkIdx++; - } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - int round = 0; - int dIter = 0; - while (true) { - int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, - defaultRowsInBlock, pCfg->update); - if (rowsRead == 0) break; + if ((!TSDB_IS_LAST_BLOCK(pCompBlock)) && keyFirst < pCompBlock->keyFirst) { + // Loop to write data until pCompBlock->keyFirst-1 + while (true) { + tdResetDataCols(pDataCols); + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, pCompBlock->keyLast - 1, defaultRowsInBlock, pDataCols, NULL, 0, + pCfg->update, pMergeInfo); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); + if (pDataCols->numOfRows == 0) break; - if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - if (round == 0) { - if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } else { - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } - - tblkIdx++; - round++; - } - } - if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; + if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; + tblkIdx++; } + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { - TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1); - TSKEY blkKeyFirst = pCompBlock->keyFirst; - TSKEY blkKeyLast = pCompBlock->keyLast; + int16_t colId = 0; + if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; - if (keyFirst < blkKeyFirst) { - while (true) { - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, - NULL, 0, pCfg->update); - if (rowsRead == 0) break; + TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1); - ASSERT(rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - tblkIdx++; + slIter = *(pCommitIter->pIter); + tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows, + pCfg->update, pMergeInfo); + + if (pMergeInfo->nOperations == 0) { + // Do nothing + ASSERT(pMergeInfo->rowsDeleteFailed > 0); + *(pCommitIter->pIter) = slIter; + tblkIdx++; + } else if (pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { + // Delete the block and do some stuff + ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN); + if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1; + *pCommitIter->pIter = slIter; + if (pCompBlock->last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + } else if (tsdbCheckAddSubBlockCond(pHelper, pCompBlock, pMergeInfo, pDataCols->maxPoints)) { + // Append as a sub-block of the searched block + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData, + pDataCols0->numOfRows, pCfg->update, pMergeInfo); + ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0); + if (tsdbWriteBlockToFile(pHelper, pCompBlock->last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, + &compBlock, pCompBlock->last, false) < 0) { + return -1; } - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) { + return -1; + } + tblkIdx++; } else { - ASSERT(keyFirst <= blkKeyLast); - int16_t colId = 0; - if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; + // load the block data, merge with the memory data + if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + int round = 0; + int dIter = 0; + while (true) { + tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock, + pCfg->update); - slIter = *(pCommitIter->pIter); - int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); - int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows, pCfg->update); - - if (!pCfg->update && rows2 == 0) { // all filtered out - *(pCommitIter->pIter) = slIter; - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); - } else { - int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0, pCfg->update) + rows2; - - if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { - int rows = (rows1 >= rows3) ? rows3 : rows2; - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); - ASSERT(rowsRead == rows && rowsRead <= pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) - return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; - tblkIdx++; - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + if (pDataCols->numOfRows == 0) break; + if (tblkIdx == pIdx->numOfBlocks - 1) { + if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - int round = 0; - int dIter = 0; - while (true) { - int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, - defaultRowsInBlock, pCfg->update); - if (rowsRead == 0) break; - - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - if (round == 0) { - if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } else { - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } - - round++; - tblkIdx++; - } - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; } + + if (round == 0) { + if (pCompBlock->last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; + } else { + if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; + } + + round++; + tblkIdx++; } } } @@ -1631,9 +1647,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, return 0; } -static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update) { - int numOfRows = 0; TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; STSchema *pSchema = NULL; @@ -1643,39 +1658,60 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); + bool isRowDel = false; SDataRow row = tsdbNextIterRow(pCommitIter->pIter); - key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row); + if (row == NULL || dataRowKey(row) > maxKey) { + key2 = INT64_MAX; + } else { + key2 = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } if (key1 == INT64_MAX && key2 == INT64_MAX) break; - if ((key1 < key2) || ((!update) && (key1 == key2))) { + if (key1 < key2) { for (int i = 0; i < pDataCols->numOfCols; i++) { dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } + pTarget->numOfRows++; (*iter)++; + } else if (key1 > key2) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } - if ((!update) && (key1 == key2)) { - tSkipListIterNext(pCommitIter->pIter); - } - } else { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); - ASSERT(pSchema != NULL); + tdAppendDataRowToDataCol(row, pSchema, pTarget); } - tdAppendDataRowToDataCol(row, pSchema, pTarget); tSkipListIterNext(pCommitIter->pIter); - if (key1 == key2) (*iter)++; + } else { + if (update) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendDataRowToDataCol(row, pSchema, pTarget); + } + } else { + ASSERT(!isRowDel); + + for (int i = 0; i < pDataCols->numOfCols; i++) { + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + pTarget->numOfRows++; + } + (*iter)++; + tSkipListIterNext(pCommitIter->pIter); } - - numOfRows++; - if (numOfRows >= maxRows) break; - ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints); } - - return numOfRows; } static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) { @@ -1698,3 +1734,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, return 0; } + +static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) { + STsdbCfg *pCfg = &(pHelper->pRepo->config); + int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; + + ASSERT(mergeRows > 0); + + if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) { + if (pCompBlock->last) { + if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true; + } else { + if (mergeRows < pCfg->maxRowsPerFileBlock) return true; + } + } + + return false; +} \ No newline at end of file diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 030a9d69c1..579483dbff 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -39,16 +39,11 @@ typedef char *(*__sl_key_fn_t)(const void *); typedef struct SSkipListNode { uint8_t level; - uint8_t flags; void * pData; struct SSkipListNode *forwards[]; } SSkipListNode; -#define SL_NODE_DELETED_FLAG (uint8_t)0x1 - #define SL_GET_NODE_DATA(n) (n)->pData -#define SL_IS_NODE_DELETED(n) ((n)->flags & SL_NODE_DELETED_FLAG) -#define SL_SET_NODE_DELETED(n) (n)->flags |= SL_NODE_DELETED_FLAG #define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] #define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] @@ -109,8 +104,7 @@ typedef struct SSkipList { uint8_t flags; uint8_t type; // static info above uint8_t level; - uint32_t size; // semantic meaning of size - uint32_t tsize; // # of all skiplist nodes in this SL + uint32_t size; SSkipListNode * pHead; // point to the first element SSkipListNode * pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE @@ -131,13 +125,13 @@ typedef struct SSkipListIterator { #define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) #define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) #define SL_SIZE(s) (s)->size -#define SL_TSIZE(s) (s)->tsize -SSkipList * tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn); -void tSkipListDestroy(SSkipList *pSkipList); -SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); -SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, + __sl_key_fn_t fn); +void tSkipListDestroy(SSkipList *pSkipList); +SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData); +SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); bool tSkipListIterNext(SSkipListIterator *iter); @@ -145,8 +139,6 @@ SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); void * tSkipListDestroyIter(SSkipListIterator *iter); uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); -SSkipListKey tSkipListGetMinKey(SSkipList *pSkipList); -SSkipListKey tSkipListGetMaxKey(SSkipList *pSkipList); #ifdef __cplusplus } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 603a9759b8..ea166e4c25 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -34,7 +34,8 @@ static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); -SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn) { +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, + __sl_key_fn_t fn) { SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); if (pSkipList == NULL) return NULL; @@ -47,7 +48,11 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, u pSkipList->len = keyLen; pSkipList->flags = flags; pSkipList->keyFn = fn; - pSkipList->comparFn = getKeyComparFunc(keyType); + if (comparFn == NULL) { + pSkipList->comparFn = getKeyComparFunc(keyType); + } else { + pSkipList->comparFn = comparFn; + } if (initForwardBackwardPtr(pSkipList) < 0) { tSkipListDestroy(pSkipList); @@ -115,10 +120,6 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { if (dupMode == SL_UPDATE_DUP_KEY) { pNode = SL_NODE_GET_FORWARD_POINTER(forward[0], 0); atomic_store_ptr(&(pNode->pData), pData); - if (SL_IS_NODE_DELETED(pNode)) { - pNode->flags &= (~(SL_NODE_DELETED_FLAG)); - pSkipList->size++; - } } } else { pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList)); @@ -137,8 +138,6 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { uint32_t count = 0; - if (SL_DUP_MODE(pSkipList) == SL_DISCARD_DUP_KEY) return 0; - tSkipListWLock(pSkipList); SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); @@ -177,9 +176,7 @@ SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { break; } - if (!SL_IS_NODE_DELETED(p)) { - taosArrayPush(sa, &p); - } + taosArrayPush(sa, &p); pNode = p; } @@ -227,19 +224,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) { tSkipListRLock(pSkipList); if (iter->order == TSDB_ORDER_ASC) { - while (true) { - iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); - iter->step++; - if (iter->cur == pSkipList->pTail) break; - if (!SL_IS_NODE_DELETED(iter->cur)) break; - } + if (iter->cur == pSkipList->pTail) return false; + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); + iter->step++; } else { - while (true) { - iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); - iter->step++; - if (iter->cur == pSkipList->pHead) break; - if (!SL_IS_NODE_DELETED(iter->cur)) break; - } + if (iter->cur == pSkipList->pHead) return false; + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); + iter->step++; } tSkipListUnlock(pSkipList); @@ -305,28 +296,6 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { } } -SSkipListKey tSkipListGetMinKey(SSkipList *pSkipList) { - if (pSkipList == NULL || SL_SIZE(pSkipList) == 0) return NULL; - - SSkipListNode *pNode = pSkipList->pHead; - while ((pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0)) != pSkipList->pTail) { - if (!SL_IS_NODE_DELETED(pNode)) return pSkipList->keyFn(pNode->pData); - } - - return NULL; -} - -SSkipListKey tSkipListGetMaxKey(SSkipList *pSkipList) { - if (pSkipList == NULL || SL_SIZE(pSkipList) == 0) return NULL; - - SSkipListNode *pNode = pSkipList->pTail; - while ((pNode = SL_NODE_GET_BACKWARD_POINTER(pNode, 0)) != pSkipList->pHead) { - if (!SL_IS_NODE_DELETED(pNode)) return pSkipList->keyFn(pNode->pData); - } - - return NULL; -} - static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { for (int32_t i = 0; i < pNode->level; ++i) { if (i >= pSkipList->level) { @@ -349,7 +318,6 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSk if (pSkipList->level < pNode->level) pSkipList->level = pNode->level; pSkipList->size += 1; - pSkipList->tsize += 1; } static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { @@ -447,27 +415,18 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { int32_t level = pNode->level; uint8_t dupMode = SL_DUP_MODE(pSkipList); + ASSERT(dupMode != SL_DISCARD_DUP_KEY && dupMode != SL_UPDATE_DUP_KEY); - if (dupMode == SL_UPDATE_DUP_KEY) { - if (SL_IS_NODE_DELETED(pNode)) { - return; - } else { - SL_SET_NODE_DELETED(pNode); - pSkipList->size--; - } - } else { - for (int32_t j = level - 1; j >= 0; --j) { - SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j); - SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j); + for (int32_t j = level - 1; j >= 0; --j) { + SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j); + SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j); - SL_NODE_GET_FORWARD_POINTER(prev, j) = next; - SL_NODE_GET_BACKWARD_POINTER(next, j) = prev; - } - - tSkipListFreeNode(pNode); - pSkipList->size--; - pSkipList->tsize--; + SL_NODE_GET_FORWARD_POINTER(prev, j) = next; + SL_NODE_GET_BACKWARD_POINTER(next, j) = prev; } + + tSkipListFreeNode(pNode); + pSkipList->size--; } // Function must be called after calling tSkipListRemoveNodeImpl() function From 47ead850543d2483160a887761a1830450510197 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 27 Sep 2020 10:47:52 +0800 Subject: [PATCH 014/122] TD-1548 --- src/tsdb/src/tsdbRWHelper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index c6d4ae7a5e..f56f12e48a 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1065,7 +1065,7 @@ static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { int tsize = 0; if (compBlock.numOfSubBlocks > 1) { - tsize = pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks); + tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks)); ASSERT(tsize > 0); memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset), @@ -1075,14 +1075,14 @@ static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks; } - tsize = pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo); + tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo)); ASSERT(tsize > 0); memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize); pCompIdx->len -= sizeof(SCompBlock); pCompIdx->numOfBlocks--; - pCompIdx->hasLast = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last; + pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last); pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast; } From 1f942936444c16982965e3595be3c0b2be7ffcb0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 27 Sep 2020 13:27:28 +0800 Subject: [PATCH 015/122] fix coredump --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index f56f12e48a..2fff744b5c 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1592,7 +1592,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (pMergeInfo->nOperations == 0) { // Do nothing - ASSERT(pMergeInfo->rowsDeleteFailed > 0); + ASSERT(pMergeInfo->rowsDeleteFailed >= 0); *(pCommitIter->pIter) = slIter; tblkIdx++; } else if (pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { From f879d6b0f601ddc8365479de054a86b338cf0db1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Oct 2020 18:19:19 +0800 Subject: [PATCH 016/122] merge develop --- src/util/src/tskiplist.c | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 9f00480555..c8e0febd7a 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -20,7 +20,7 @@ #include "tutil.h" static int initForwardBackwardPtr(SSkipList *pSkipList); -static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); +static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); @@ -140,7 +140,7 @@ uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { tSkipListWLock(pSkipList); - SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); while (1) { SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); if (p == pSkipList->pTail) { @@ -167,7 +167,7 @@ SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { tSkipListRLock(pSkipList); - SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); while (1) { SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); if (p == pSkipList->pTail) { @@ -209,7 +209,7 @@ SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char * tSkipListRLock(pSkipList); - iter->cur = getPriorNode(pSkipList, val, order); + iter->cur = getPriorNode(pSkipList, val, order, &(iter->next)); tSkipListUnlock(pSkipList); @@ -226,10 +226,24 @@ bool tSkipListIterNext(SSkipListIterator *iter) { if (iter->order == TSDB_ORDER_ASC) { if (iter->cur == pSkipList->pTail) return false; iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); + + // a new node is inserted into between iter->cur and iter->next, ignore it + if (iter->cur != iter->next && (iter->next != NULL)) { + iter->cur = iter->next; + } + + iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); iter->step++; } else { if (iter->cur == pSkipList->pHead) return false; iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); + + // a new node is inserted into between iter->cur and iter->next, ignore it + if (iter->cur != iter->next && (iter->next != NULL)) { + iter->cur = iter->next; + } + + iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); iter->step++; } @@ -327,8 +341,10 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t iter->order = order; if (order == TSDB_ORDER_ASC) { iter->cur = pSkipList->pHead; + iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); } else { iter->cur = pSkipList->pTail; + iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); } return iter; @@ -485,9 +501,12 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { // when order is TSDB_ORDER_ASC, return the last node with key less than val // when order is TSDB_ORDER_DESC, return the first node with key large than val -static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order) { +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur) { __compar_fn_t comparFn = pSkipList->comparFn; SSkipListNode *pNode = NULL; + if (pCur != NULL) { + *pCur = NULL; + } if (order == TSDB_ORDER_ASC) { pNode = pSkipList->pHead; @@ -499,6 +518,9 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ pNode = p; p = SL_NODE_GET_FORWARD_POINTER(p, i); } else { + if (pCur != NULL) { + *pCur = p; + } break; } } @@ -513,6 +535,9 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ pNode = p; p = SL_NODE_GET_BACKWARD_POINTER(p, i); } else { + if (pCur != NULL) { + *pCur = p; + } break; } } From 57e93db1499ad83d10614f5ad9182d4a8e68ddf8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Oct 2020 15:55:59 +0800 Subject: [PATCH 017/122] fix some bug --- src/common/inc/tdataformat.h | 32 +++++---------- src/common/src/tdataformat.c | 80 ++++++++---------------------------- src/tsdb/src/tsdbMemTable.c | 15 +++++-- src/tsdb/src/tsdbRWHelper.c | 27 ++++++------ src/util/inc/tskiplist.h | 6 +-- 5 files changed, 58 insertions(+), 102 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 8124075e6c..f6683e4843 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -193,7 +193,7 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i if (offset == 0) { ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); TKEY tvalue = tdGetTKEY(*(TSKEY *)value); - memcpy(POINTER_SHIFT(row, toffset), &tvalue, TYPE_BYTES[type]); + memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]); } else { memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); } @@ -227,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints); -void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); @@ -235,28 +234,20 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); - break; - - default: - return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); - break; + if (IS_VAR_DATA_TYPE(pCol->type)) { + return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); + } else { + return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); } } static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { ASSERT(rows > 0); - switch (pDataCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); - break; - default: - return TYPE_BYTES[pDataCol->type] * rows; + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); + } else { + return TYPE_BYTES[pDataCol->type] * rows; } } @@ -277,11 +268,11 @@ typedef struct { #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) #define dataColsTKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0) -#define dataColsKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TSDB_DATA_BIGINT_NULL : dataColsKeyAt(pCols, 0) +#define dataColsKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0) #define dataColsTKeyLast(pCols) \ (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) #define dataColsKeyLast(pCols) \ - (((pCols)->numOfRows == 0) ? TSDB_DATA_BIGINT_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) + (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); @@ -289,7 +280,6 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); -void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); // ----------------- K-V data row structure diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 4596adaf14..49d21d9275 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -205,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE; pDataCol->len = 0; - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pDataCol->type)) { pDataCol->dataOff = (VarDataOffsetT *)(*pBuf); pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints); pDataCol->spaceSize = pDataCol->bytes * maxPoints; @@ -218,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) } } +// value from timestamp should be TKEY here instead of TSKEY void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); - // Update the length - pCol->len += varDataTLen(value); - break; - default: - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); - pCol->len += pCol->bytes; - break; - } -} - -void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) { - int pointsLeft = numOfRows - pointsToPop; - - ASSERT(pointsLeft > 0); - - if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) { - ASSERT(pCol->len > 0); - VarDataOffsetT toffset = pCol->dataOff[pointsToPop]; - pCol->len = pCol->len - toffset; - ASSERT(pCol->len > 0); - memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len); - dataColSetOffset(pCol, pointsLeft); + if (IS_VAR_DATA_TYPE(pCol->type)) { + // set offset + pCol->dataOff[numOfRows] = pCol->len; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); + // Update the length + pCol->len += varDataTLen(value); } else { ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - pCol->len = TYPE_BYTES[pCol->type] * pointsLeft; - memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); + pCol->len += pCol->bytes; } } bool isNEleNull(SDataCol *pCol, int nEle) { - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - for (int i = 0; i < nEle; i++) { - if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; - } - return true; - default: - for (int i = 0; i < nEle; i++) { - if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; - } - return true; + for (int i = 0; i < nEle; i++) { + if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; } + return true; } void dataColSetNullAt(SDataCol *pCol, int index) { @@ -393,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize; pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf))); - if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { ASSERT(pDataCols->cols[i].dataOff != NULL); pRet->cols[i].dataOff = (int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf))); @@ -403,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->cols[i].len = pDataCols->cols[i].len; if (pDataCols->cols[i].len > 0) { memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); - if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints); } } @@ -463,21 +432,6 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) pCols->numOfRows++; } -// Pop pointsToPop points from the SDataCols -void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) { - int pointsLeft = pCols->numOfRows - pointsToPop; - if (pointsLeft <= 0) { - tdResetDataCols(pCols); - return; - } - - for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { - SDataCol *pCol = pCols->cols + iCol; - dataColPopPoints(pCol, pointsToPop, pCols->numOfRows); - } - pCols->numOfRows = pointsLeft; -} - int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); @@ -549,9 +503,9 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i target->maxPoints); } } + target->numOfRows++; } - target->numOfRows++; (*iter2)++; if (key1 == key2) (*iter1)++; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 7adf16443e..2bdb95163d 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -105,8 +105,14 @@ int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (tSkipListPut(pTableData->pData, pRow) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); } else { - // TODO: may need to refact here int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize; + if (isRowDelete) { + if (TABLE_LASTKEY(pTable) == key) { + // TODO: need to update table last key here (may from file) + } + } else { + if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; + } if ((!isRowDelete) && (TABLE_LASTKEY(pTable) < key)) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; @@ -301,7 +307,7 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead * 4. operations in pCols not exceeds its max capacity if pCols is given * - * The function try to move as mush as possible. + * The function tries to procceed AS MUSH AS POSSIBLE. */ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { @@ -321,12 +327,13 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) { rowKey = INT64_MAX; + isRowDel = false; } else { rowKey = dataRowKey(row); isRowDel = dataRowDeleted(row); } - if (nFilterKeys == 0 || filterIter >= nFilterKeys) { + if (filterIter >= nFilterKeys) { fKey = INT64_MAX; } else { fKey = tdGetKey(filterKeys[filterIter]); @@ -362,6 +369,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) { rowKey = INT64_MAX; + isRowDel = false; } else { rowKey = dataRowKey(row); isRowDel = dataRowDeleted(row); @@ -391,6 +399,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) { rowKey = INT64_MAX; + isRowDel = false; } else { rowKey = dataRowKey(row); isRowDel = dataRowDeleted(row); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 2fff744b5c..eb1193110d 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1507,6 +1507,8 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast); if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1; } else { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; @@ -1553,6 +1555,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols * pDataCols0 = pHelper->pDataCols[0]; SMergeInfo mergeInfo = {0}; SMergeInfo *pMergeInfo = &mergeInfo; + SCompBlock oBlock = {0}; SSkipListIterator slIter = {0}; @@ -1562,14 +1565,14 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE); ASSERT(pCompBlock != NULL); int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); + oBlock = *pCompBlock; - ASSERT((!TSDB_IS_LAST_BLOCK(pCompBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); + ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); - if ((!TSDB_IS_LAST_BLOCK(pCompBlock)) && keyFirst < pCompBlock->keyFirst) { - // Loop to write data until pCompBlock->keyFirst-1 + if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { while (true) { tdResetDataCols(pDataCols); - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, pCompBlock->keyLast - 1, defaultRowsInBlock, pDataCols, NULL, 0, + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); if (pDataCols->numOfRows == 0) break; @@ -1582,7 +1585,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { int16_t colId = 0; - if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; + if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1; TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1); @@ -1595,19 +1598,19 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pMergeInfo->rowsDeleteFailed >= 0); *(pCommitIter->pIter) = slIter; tblkIdx++; - } else if (pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { + } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { // Delete the block and do some stuff ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN); if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1; *pCommitIter->pIter = slIter; - if (pCompBlock->last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - } else if (tsdbCheckAddSubBlockCond(pHelper, pCompBlock, pMergeInfo, pDataCols->maxPoints)) { + if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) { // Append as a sub-block of the searched block tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update, pMergeInfo); ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0); - if (tsdbWriteBlockToFile(pHelper, pCompBlock->last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, - &compBlock, pCompBlock->last, false) < 0) { + if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, + &compBlock, oBlock.last, false) < 0) { return -1; } if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) { @@ -1616,7 +1619,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, tblkIdx++; } else { // load the block data, merge with the memory data - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1; int round = 0; int dIter = 0; while (true) { @@ -1631,7 +1634,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } if (round == 0) { - if (pCompBlock->last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; } else { if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 1c5b02a502..65b6482520 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -28,9 +28,9 @@ extern "C" { #define SKIP_LIST_RECORD_PERFORMANCE 0 // For key property setting -#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists -#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key -#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) +#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case) // For thread safety setting #define SL_THREAD_SAFE (uint8_t)0x4 From e4868728968595f5236a4c07b2150d509ed6de31 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 30 Oct 2020 10:43:40 +0800 Subject: [PATCH 018/122] initialize database update option --- src/query/src/qParserImpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 7e8128f200..6e4b08316d 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -872,5 +872,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->quorum = -1; pDBInfo->keep = NULL; + pDBInfo->update = -1; memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } From 3060fafd9f7b254b995a994b04de2a4ccddddee3 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Fri, 30 Oct 2020 04:42:03 +0000 Subject: [PATCH 019/122] Now locking create table operation, resolving TD-1471 --- tests/pytest/crash_gen/crash_gen.py | 109 +++++++++++++++++++++------- tests/pytest/crash_gen/misc.py | 3 + 2 files changed, 84 insertions(+), 28 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen.py b/tests/pytest/crash_gen/crash_gen.py index 8d2b0080bc..9bc701aa8b 100755 --- a/tests/pytest/crash_gen/crash_gen.py +++ b/tests/pytest/crash_gen/crash_gen.py @@ -243,7 +243,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 180 # one minute + WORKER_THREAD_TIMEOUT = 120 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 @@ -1177,6 +1177,8 @@ class Task(): instead. But a task is always associated with a DB ''' taskSn = 100 + _lock = threading.Lock() + _tableLocks: Dict[str, threading.Lock] = {} @classmethod def allocTaskNum(cls): @@ -1198,6 +1200,8 @@ class Task(): self._execStats = execStats self._db = db # A task is always associated/for a specific DB + + def isSuccess(self): return self._err is None @@ -1351,6 +1355,24 @@ class Task(): def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread return wt.getQueryResult() + def lockTable(self, ftName): # full table name + # print(" <<" + ftName + '_', end="", flush=True) + with Task._lock: + if not ftName in Task._tableLocks: + Task._tableLocks[ftName] = threading.Lock() + + Task._tableLocks[ftName].acquire() + + def unlockTable(self, ftName): + # print('_' + ftName + ">> ", end="", flush=True) + with Task._lock: + if not ftName in self._tableLocks: + raise RuntimeError("Corrupt state, no such lock") + lock = Task._tableLocks[ftName] + if not lock.locked(): + raise RuntimeError("Corrupte state, already unlocked") + lock.release() + class ExecutionStats: def __init__(self): @@ -1461,7 +1483,7 @@ class StateTransitionTask(Task): _baseTableNumber = None - _endState = None + _endState = None # TODO: no longter used? @classmethod def getInfo(cls): # each sub class should supply their own information @@ -1486,7 +1508,7 @@ class StateTransitionTask(Task): @classmethod def getRegTableName(cls, i): - if ( StateTransitionTask._baseTableNumber is None): + if ( StateTransitionTask._baseTableNumber is None): # Set it one time StateTransitionTask._baseTableNumber = Dice.throw( 999) if gConfig.dynamic_db_table_names else 0 return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i) @@ -1583,14 +1605,23 @@ class TdSuperTable: def hasRegTables(self, dbc: DbConn, dbName: str): return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0 - def ensureTable(self, dbc: DbConn, dbName: str, regTableName: str): + def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str): sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName) if dbc.query(sql) >= 1 : # reg table exists already return - sql = "CREATE TABLE {}.{} USING {}.{} tags ({})".format( - dbName, regTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) - ) - dbc.execute(sql) + + # acquire a lock first, so as to be able to *verify*. More details in TD-1471 + fullTableName = dbName + '.' + regTableName + task.lockTable(fullTableName) + Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table + print("(" + fullTableName[-3:] + ")", end="", flush=True) + try: + sql = "CREATE TABLE {} USING {}.{} tags ({})".format( + fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) + ) + dbc.execute(sql) + finally: + task.unlockTable(fullTableName) # no matter what def _getTagStrForSql(self, dbc, dbName: str) : tags = self._getTags(dbc, dbName) @@ -1862,7 +1893,12 @@ class TaskAddData(StateTransitionTask): sTable = db.getFixedSuperTable() regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) - sTable.ensureTable(wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists + + + fullTableName = db.getName() + '.' + regTableName + # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" + sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists + # self._unlockTable(fullTableName) for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table nextInt = db.getNextInt() @@ -1872,27 +1908,29 @@ class TaskAddData(StateTransitionTask): self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) self.fAddLogReady.flush() os.fsync(self.fAddLogReady) - sql = "insert into {}.{} values ('{}', {});".format( # removed: tags ('{}', {}) - db.getName(), - regTableName, - # ds.getFixedSuperTableName(), - # ds.getNextBinary(), ds.getNextFloat(), - nextTick, nextInt) - dbc.execute(sql) - # Successfully wrote the data into the DB, let's record it - # somehow - te.recordDataMark(nextInt) - if gConfig.record_ops: - self.fAddLogDone.write( - "Wrote {} to {}\n".format( - nextInt, regTableName)) - self.fAddLogDone.flush() - os.fsync(self.fAddLogDone) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + if gConfig.verify_data: + self.lockTable(fullTableName) + # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written + + try: + sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {}) + fullTableName, + # ds.getFixedSuperTableName(), + # ds.getNextBinary(), ds.getNextFloat(), + nextTick, nextInt) + dbc.execute(sql) + except: # Any exception at all + if gConfig.verify_data: + self.unlockTable(fullTableName) + raise # Now read it back and verify, we might encounter an error if table is dropped if gConfig.verify_data: # only if command line asks for it try: - readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts= '{}'". + readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". format(db.getName(), regTableName, nextTick)) if readBack != nextInt : raise taos.error.ProgrammingError( @@ -1905,8 +1943,23 @@ class TaskAddData(StateTransitionTask): "Failed to read back same data for tick: {}, wrote: {}, read: {}" .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), errno) - # Re-throw no matter what - raise + elif errno in [0x218, 0x362]: # table doesn't exist + # do nothing + dummy = 0 + else: + # Re-throw otherwise + raise + finally: + self.unlockTable(fullTableName) # Unlock the table no matter what + + # Successfully wrote the data into the DB, let's record it somehow + te.recordDataMark(nextInt) + + if gConfig.record_ops: + self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) + self.fAddLogDone.flush() + os.fsync(self.fAddLogDone) + self.activeTable.discard(i) # not raising an error, unlike remove diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index 34a33c6af6..4dc053dae3 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -166,6 +166,8 @@ class Progress: SERVICE_RECONNECT_START = 4 SERVICE_RECONNECT_SUCCESS = 5 SERVICE_RECONNECT_FAILURE = 6 + CREATE_TABLE_ATTEMPT = 7 + tokens = { STEP_BOUNDARY: '.', BEGIN_THREAD_STEP: '[', @@ -174,6 +176,7 @@ class Progress: SERVICE_RECONNECT_START: '', SERVICE_RECONNECT_FAILURE: '.xr>', + CREATE_TABLE_ATTEMPT: '_c', } @classmethod From 45fcefe80c094471f85cfa15b3ebba5dd0246569 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 30 Oct 2020 13:27:51 +0800 Subject: [PATCH 020/122] refactor part of code --- src/common/inc/tdataformat.h | 4 ++-- src/tsdb/src/tsdbMemTable.c | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index f6683e4843..27052f08db 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -124,8 +124,8 @@ typedef uint64_t TKEY; #define TKEY_INVALID UINT64_MAX #define TKEY_NULL TKEY_INVALID -#define TKEY_NEGATIVE_FLAG (((TKEY)1) << (sizeof(TKEY) * 8 - 1)) -#define TKEY_DELETE_FLAG (((TKEY)1) << (sizeof(TKEY) * 8 - 2)) +#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) +#define TKEY_DELETE_FLAG (((TKEY)1) << 62) #define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 2bdb95163d..19f59f3080 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -113,7 +113,6 @@ int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { } else { if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; } - if ((!isRowDelete) && (TABLE_LASTKEY(pTable) < key)) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; From fb76d7378207041c7f7dcd20ef495e898b006ad7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 3 Nov 2020 14:13:28 +0800 Subject: [PATCH 021/122] [TD-1557] add test cases for update --- tests/pytest/update/allow_update.py | 67 +++++++ tests/pytest/update/append_commit_data.py | 33 ++++ tests/pytest/update/append_commit_last.py | 42 +++++ tests/pytest/update/merge_commit_data.py | 206 ++++++++++++++++++++++ tests/pytest/update/merge_commit_data2.py | 153 ++++++++++++++++ tests/pytest/update/merge_commit_last.py | 187 ++++++++++++++++++++ 6 files changed, 688 insertions(+) create mode 100644 tests/pytest/update/allow_update.py create mode 100644 tests/pytest/update/append_commit_data.py create mode 100644 tests/pytest/update/append_commit_last.py create mode 100644 tests/pytest/update/merge_commit_data.py create mode 100644 tests/pytest/update/merge_commit_data2.py create mode 100644 tests/pytest/update/merge_commit_last.py diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py new file mode 100644 index 0000000000..cc6fab5fc5 --- /dev/null +++ b/tests/pytest/update/allow_update.py @@ -0,0 +1,67 @@ +# For step 1, we set all the values as 1 +# for step 2, we set all the values as 2 +# and so on +# check query result here means not + +# clear env set up + +# create database db update 1 + +# create table t (ts timestamp, a int) + +# set an init time (such as $t0 = 1604295582000) + +# Step 1 +insert into t values ($t0, 1); +insert into t values ($t0 - 3, 1); +insert into t values ($t0 + 3, 1); + +check query result + +# Step 2 +insert into t values ($t0, 2); +insert into t values ($t0 - 3, 2); +insert into t values ($t0 + 3, 2); + +check query result + +# Step 3 +insert into t values ($t0 - 4, 3); +insert into t values ($t0 - 2, 3); +insert into t values ($t0 + 2, 3); +insert into t values ($t0 + 4, 3); + +check query result + +# Step 4 +insert into t values ($t0 - 4, 4); +insert into t values ($t0 - 2, 4); +insert into t values ($t0 + 2, 4); +insert into t values ($t0 + 4, 4); + +check query result + +# Step 4 +insert into t values ($t0 - 1, 5); +insert into t values ($t0 + 1, 5); + +check query result + +# Step 4 +insert into t values ($t0 - 4, 6); +insert into t values ($t0 - 3, 6); +insert into t values ($t0 - 2, 6); +insert into t values ($t0 - 1, 6); +insert into t values ($t0, 6); +insert into t values ($t0 + 1, 6); +insert into t values ($t0 + 2, 6); +insert into t values ($t0 + 3, 6); +insert into t values ($t0 + 4, 6); + +check query result + +# Step 4 + +restart to commit + +check query result \ No newline at end of file diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py new file mode 100644 index 0000000000..c907acb932 --- /dev/null +++ b/tests/pytest/update/append_commit_data.py @@ -0,0 +1,33 @@ +# clear env set up + +create database db update 1 +$t0 = 1604298064000 + +## Step 1 +create table t1 (ts timestamp, a int); + +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($k = 1; $k <= 100; $k++) { + for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $k * 200 $i, 1); + } + + restart to commit + check query result +} + +## Step 2 +create table t2 (ts timestamp, a int); + +for ($i = 0; $i < 20000; $i++) { + insert into t2 values ($t0 + $i, 1); +} + +restart to commit +check query result \ No newline at end of file diff --git a/tests/pytest/update/append_commit_last.py b/tests/pytest/update/append_commit_last.py new file mode 100644 index 0000000000..9b31adccec --- /dev/null +++ b/tests/pytest/update/append_commit_last.py @@ -0,0 +1,42 @@ +# clear env set up + +create database db update 1; + +## Step 1 +$loops = 1000 +#t0 = 1604298064000 + +create table t1 (ts timestamp, a int); +for ($i = 0; $i < $loops; $i++) { + insert into t1 values ($t0 + $i, 1); + restart to commit + check query result +} + +## Step 2 +create table t2 (ts timestamp, a int); + +insert into t2 ($t0, 1); +restart to commit +check query result + +for ($i = 1; $i <= 150; $i++) { + insert into t2 values ($t0 + $i, 1); +} +restart to commit +check query result + +## Step 3 +create table t3 (ts timestamp, a int); + +insert into t3 ($t0, 1); +restart to commit +check query result + +for ($i = 0; $i < 8; $i++) { + for ($j = 1; $j <= 10; $j++) { + insert into t3 values ($t0 + $i * 10 + $j , 1); + } +} +restart to commit +check query result \ No newline at end of file diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py new file mode 100644 index 0000000000..e9e2596821 --- /dev/null +++ b/tests/pytest/update/merge_commit_data.py @@ -0,0 +1,206 @@ +# clear env set up + +$t0 = 1603152000000 + +create database db update 1 days 30; + +## Step 1. UPDATE THE WHOLE DATA BLOCK REPEATEDLY +create table t1 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($k = 0; $k < 10; $k++) { + for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 1); + } + + check query result + restart to commit + check query result +} + +## Step 2. PREPEND DATA +create table t2 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t2 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = -100; $i < 0; $i++) { + insert into t2 values ($t0 + $i, 1); +} + +check query result +restart to commit +check query result + +## Step 3. PREPEND MASSIVE DATA +create table t3 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t3 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = -6000; $i < 0; $i++) { + insert into t3 values ($t0 + $i, 1); +} + +check query result +restart to commit +check query result + +## Step 4. APPEND DATA +create table t4 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t4 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 100; $i++) { + insert into t4 values ($t0 + 200 + $i, 1); +} + +check query result +restart to commit +check query result + +## Step 5. APPEND MASSIVE DATA +create table t5 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t5 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 6000; $i++) { + insert into t5 values ($t0 + 200 + $i, 1); +} + +check query result +restart to commit +check query result + +## Step 6. UPDATE BLOCK IN TWO STEP +create table t6 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t6 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 100; $i++) { + insert into t6 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +for ($i = 100; $i < 200; $i++) { + insert into t6 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 7. UPDATE LAST HALF AND INSERT LITTLE DATA +create table t7 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t7 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 100; $i < 300; $i++) { + insert into t7 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 8. UPDATE LAST HALF AND INSERT MASSIVE DATA +create table t8 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t8 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 100; $i < 6000; $i++) { + insert into t8 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 9. UPDATE FIRST HALF AND PREPEND LITTLE DATA +create table t9 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t9 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = -100; $i < 100; $i++) { + insert into t9 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 10. UPDATE FIRST HALF AND PREPEND MASSIVE DATA +create table t10 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t10 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = -6000; $i < 100; $i++) { + insert into t10 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 11. UPDATE FIRST HALF AND APPEND MASSIVE DATA +create table t11 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t11 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 100; $i++) { + insert into t11 values ($t0 + $i, 2); +} + +for ($i = 200; $i < 6000; $i++) { + insert into t11 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result \ No newline at end of file diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py new file mode 100644 index 0000000000..e9dd084eb0 --- /dev/null +++ b/tests/pytest/update/merge_commit_data2.py @@ -0,0 +1,153 @@ +# clear env set up + +$t0 = 1603152000000 + +create database db update 1 days 30; + +## Step 1. UPDATE TWO WHOLE DATA BLOCK REPEATEDLY +create table t1 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + 5000 + $i, 1); +} + +check query result +restart to commit +check query result + +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 2); +} +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + 5000 + $i, 1); +} +check query result +restart to commit +check query result + +## Step 2. INSERT IN MIDDLE LITTLE DATA REPEATEDLY +create table t2 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t2 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t2 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($k = 0; $k < 10; $k++) { + for ($i = 0; $i < 10; $i++) { + insert into t2 values ($t0 + 200 + $k * 10 + $i, 1); + } + check query result + restart to commit + check query result +} + +## Step 3. INSERT IN MIDDLE AND UPDATE TWO BLOCKS +create table t3 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t3 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t3 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = 0; $i < 5200; $i++) { + insert into t3 values ($t0 + $i, 2); +} +check query result +restart to commit +check query result + +## Step 4. INSERT IN MIDDLE AND UPDATE FIRST HALF OF TWO BLOCKS +create table t4 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t4 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t4 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = 0; $i < 100; $i++) { + insert into t4 values ($t0 + $i, 2); +} +for ($i = 200; $i < 5000; $i++) { + insert into t4 values ($t0 + $i, 2); +} +for ($i = 0; $i < 100; $i++) { + insert into t4 values ($t0 + 5000 + $i, 2); +} +check query result +restart to commit +check query result + +## Step 5. INSERT IN MIDDLE AND UPDATE last HALF OF TWO BLOCKS +create table t5 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t5 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t5 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = 100; $i < 200; $i++) { + insert into t5 values ($t0 + $i, 2); +} +for ($i = 200; $i < 5000; $i++) { + insert into t5 values ($t0 + $i, 2); +} +for ($i = 100; $i < 200; $i++) { + insert into t5 values ($t0 + 5000 + $i, 2); +} +check query result +restart to commit +check query result + +## Step 6. INSERT MASSIVE DATA AND UPDATE ALL BLOCKS +create table t6 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t6 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t6 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = -1000; $i < 10000; $i++) { + insert into t6 values ($t0 + $i, 2); +} +check query result +restart to commit +check query result + +## Step 7. SPLIT DATA BLOCK +create table t7 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t7 values ($t0 + $i, 1); +} +for ($i = 0; $i < 200; $i++) { + insert into t7 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = -1000; $i < 10000; $i++) { + insert into t7 values ($t0 + $i, 2); +} +check query result +restart to commit +check query result \ No newline at end of file diff --git a/tests/pytest/update/merge_commit_last.py b/tests/pytest/update/merge_commit_last.py new file mode 100644 index 0000000000..9d832d8984 --- /dev/null +++ b/tests/pytest/update/merge_commit_last.py @@ -0,0 +1,187 @@ +# clear env set up + +$t0 = 1603152000000 + +create database db update 1 days 30; + +## STEP 1: UPDATE THE LAST RECORD REPEATEDLY +create table t1 (ts timestamp, a int); + +for ($i = 0; $i < 100; $i++) { + insert into t1 values ($t0, $i); + + restart to commit + check query result +} + +## STEP 2: UPDATE THE WHOLE LAST BLOCK +create table t2 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t2 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 0; $i < 50; $i++) { + insert into t2 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 3: UPDATE PART OF THE LAST BLOCK +create table t3 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t3 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 0; $i < 25; $i++) { + insert into t3 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +for ($i = 25; $i < 50; $i++) { + insert into t3 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 4: UPDATE AND INSERT APPEND AT END OF DATA +create table t4 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t4 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 0; $i < 25; $i++) { + insert into t4 values ($t0 + $i, 2); +} + +for ($i = 50; $i < 60; $i++) { + insert into t4 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 5: UPDATE AND INSERT PREPEND SOME DATA +create table t5 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t5 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = -10; $i < 0; $i++) { + insert into t4 values ($t0 + $i, 2); +} + +for ($i = 0; $i < 25; $i++) { + insert into t5 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +for ($i = -10; $i < 0; $i++) { + insert into t4 values ($t0 + $i, 3); +} + +for ($i = 25; $i < 50; $i++) { + insert into t5 values ($t0 + $i, 3); +} + +check query result +restart to commit +check query result + +## STEP 6: INSERT AHEAD A LOT OF DATA +create table t6 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t6 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = -1000; $i < 0; $i++) { + insert into t6 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 7: INSERT AHEAD A LOT AND UPDATE +create table t7 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t7 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = -1000; $i < 25; $i++) { + insert into t7 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 8: INSERT AHEAD A LOT AND UPDATE +create table t8 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t8 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 25; $i < 6000; $i++) { + insert into t8 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 9: UPDATE ONLY MIDDLE +create table t9 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t9 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 20; $i < 30; $i++) { + insert into t9 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 10: A LOT OF DATA COVER THE WHOLE BLOCK +create table t10 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t10 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = -4000; $i < 4000; $i++) { + insert into t10 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result \ No newline at end of file From 19ae9714bc46d01312a3a2030d3fa7af65d5ad41 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 14:48:20 +0800 Subject: [PATCH 022/122] add for update test --- tests/pytest/update/append_commit_data.py | 105 +++++++++++++++++----- 1 file changed, 81 insertions(+), 24 deletions(-) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index c907acb932..e0cda0cd90 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -1,33 +1,90 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -create database db update 1 -$t0 = 1604298064000 +# -*- coding: utf-8 -*- -## Step 1 -create table t1 (ts timestamp, a int); +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 1); -} -restart to commit -check query result +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) -for ($k = 1; $k <= 100; $k++) { - for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $k * 200 $i, 1); - } + def run(self): + print("==========step1") + print("create table && insert data") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db update 1' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') - restart to commit - check query result -} + insertRows = 200 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0+i)) + print("==========step2") + print("restart to commit ") + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows) + for k in range(0,100): + tdLog.info("insert %d rows" % (insertRows)) + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into db.t1 values(%d,1)' % + (t0+k*200+i) + ) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows+200*k) + print("==========step2") + print("insert into another table ") + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 20000 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d, 1)' % + (t0+i)) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) -## Step 2 -create table t2 (ts timestamp, a int); + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) -for ($i = 0; $i < 20000; $i++) { - insert into t2 values ($t0 + $i, 1); -} -restart to commit -check query result \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From fb7b1a7fd9c0e94377b34d7acb66c877de204f0e Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 14:54:52 +0800 Subject: [PATCH 023/122] test for update --- tests/pytest/update/append_commit_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index e0cda0cd90..9c23609414 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -63,7 +63,7 @@ class TDTestCase: tdDnodes.start(1) time.sleep(5) tdSql.query("select * from db.t1") - tdSql.checkRows(insertRows+200*k) + tdSql.checkRows(insertRows+200*(k+1)) print("==========step2") print("insert into another table ") s = 'use db' From 3d51199f5d77a53089cd11648f31e2ee1caf3bb6 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 3 Nov 2020 15:25:59 +0800 Subject: [PATCH 024/122] [add cases for update feature] --- tests/pytest/update/allow_update-0.py | 170 ++++++++++++++++++ tests/pytest/update/allow_update.py | 237 ++++++++++++++++++-------- 2 files changed, 340 insertions(+), 67 deletions(-) create mode 100644 tests/pytest/update/allow_update-0.py diff --git a/tests/pytest/update/allow_update-0.py b/tests/pytest/update/allow_update-0.py new file mode 100644 index 0000000000..61295ec4b7 --- /dev/null +++ b/tests/pytest/update/allow_update-0.py @@ -0,0 +1,170 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numOfRecords = 10 + self.ts = 1604295582000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.execute("use udb") + + def run(self): + tdSql.prepare() + startTs = self.ts + + print("==============step1") + tdSql.execute("create database udb update 0") + tdSql.execute("use udb") + tdSql.execute("create table t (ts timestamp, a int)") + tdSql.execute("insert into t values (%d, 1)" % (startTs)) + tdSql.execute("insert into t values (%d, 1)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 1)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step2") + tdSql.execute("insert into t values (%d, 2)" % (startTs)) + tdSql.execute("insert into t values (%d, 2)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 2)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step3") + tdSql.execute("insert into t values (%d, 3)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 3)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 3) + + print("==============step4") + tdSql.execute("insert into t values (%d, 4)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 4)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 3) + + print("==============step5") + tdSql.execute("insert into t values (%d, 5)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 5)" % (startTs + 1)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + print("==============step6") + tdSql.execute("insert into t values (%d, 6)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py index cc6fab5fc5..2879fc6f79 100644 --- a/tests/pytest/update/allow_update.py +++ b/tests/pytest/update/allow_update.py @@ -1,67 +1,170 @@ -# For step 1, we set all the values as 1 -# for step 2, we set all the values as 2 -# and so on -# check query result here means not - -# clear env set up - -# create database db update 1 - -# create table t (ts timestamp, a int) - -# set an init time (such as $t0 = 1604295582000) - -# Step 1 -insert into t values ($t0, 1); -insert into t values ($t0 - 3, 1); -insert into t values ($t0 + 3, 1); - -check query result - -# Step 2 -insert into t values ($t0, 2); -insert into t values ($t0 - 3, 2); -insert into t values ($t0 + 3, 2); - -check query result - -# Step 3 -insert into t values ($t0 - 4, 3); -insert into t values ($t0 - 2, 3); -insert into t values ($t0 + 2, 3); -insert into t values ($t0 + 4, 3); - -check query result - -# Step 4 -insert into t values ($t0 - 4, 4); -insert into t values ($t0 - 2, 4); -insert into t values ($t0 + 2, 4); -insert into t values ($t0 + 4, 4); - -check query result - -# Step 4 -insert into t values ($t0 - 1, 5); -insert into t values ($t0 + 1, 5); - -check query result - -# Step 4 -insert into t values ($t0 - 4, 6); -insert into t values ($t0 - 3, 6); -insert into t values ($t0 - 2, 6); -insert into t values ($t0 - 1, 6); -insert into t values ($t0, 6); -insert into t values ($t0 + 1, 6); -insert into t values ($t0 + 2, 6); -insert into t values ($t0 + 3, 6); -insert into t values ($t0 + 4, 6); - -check query result - -# Step 4 - -restart to commit - -check query result \ No newline at end of file +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numOfRecords = 10 + self.ts = 1604295582000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.execute("use udb") + + def run(self): + tdSql.prepare() + startTs = self.ts + + print("==============step1") + tdSql.execute("create database udb update 1") + tdSql.execute("use udb") + tdSql.execute("create table t (ts timestamp, a int)") + tdSql.execute("insert into t values (%d, 1)" % (startTs)) + tdSql.execute("insert into t values (%d, 1)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 1)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step2") + tdSql.execute("insert into t values (%d, 2)" % (startTs)) + tdSql.execute("insert into t values (%d, 2)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 2)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 2) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 2) + + print("==============step3") + tdSql.execute("insert into t values (%d, 3)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 3)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 2) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 2) + tdSql.checkData(6, 0, 3) + + print("==============step4") + tdSql.execute("insert into t values (%d, 4)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 4)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 4) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 4) + tdSql.checkData(3, 0, 2) + tdSql.checkData(4, 0, 4) + tdSql.checkData(5, 0, 2) + tdSql.checkData(6, 0, 4) + + print("==============step5") + tdSql.execute("insert into t values (%d, 5)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 5)" % (startTs + 1)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 4) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 4) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 2) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 4) + tdSql.checkData(7, 0, 2) + tdSql.checkData(8, 0, 4) + + print("==============step6") + tdSql.execute("insert into t values (%d, 6)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 6) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 6) + tdSql.checkData(3, 0, 6) + tdSql.checkData(4, 0, 6) + tdSql.checkData(5, 0, 6) + tdSql.checkData(6, 0, 6) + tdSql.checkData(7, 0, 6) + tdSql.checkData(8, 0, 6) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 6) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 6) + tdSql.checkData(3, 0, 6) + tdSql.checkData(4, 0, 6) + tdSql.checkData(5, 0, 6) + tdSql.checkData(6, 0, 6) + tdSql.checkData(7, 0, 6) + tdSql.checkData(8, 0, 6) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From c208bf23caf0939aa9c7aa9079bb3fcc6789f9b2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 Nov 2020 15:38:35 +0800 Subject: [PATCH 025/122] fix TD-1900 --- src/common/inc/tdataformat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 27052f08db..43968a6108 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -267,8 +267,8 @@ typedef struct { #define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) -#define dataColsTKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0) -#define dataColsKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0) +#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0)) +#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0)) #define dataColsTKeyLast(pCols) \ (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) #define dataColsKeyLast(pCols) \ From d76703f84d45c9c03a8a9591369669eee95a23fe Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 15:46:47 +0800 Subject: [PATCH 026/122] add test case for update --- tests/pytest/update/append_commit_data.py | 2 +- tests/pytest/update/merge_commit_data.py | 585 ++++++++++++++-------- 2 files changed, 389 insertions(+), 198 deletions(-) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index 9c23609414..e0cda0cd90 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -63,7 +63,7 @@ class TDTestCase: tdDnodes.start(1) time.sleep(5) tdSql.query("select * from db.t1") - tdSql.checkRows(insertRows+200*(k+1)) + tdSql.checkRows(insertRows+200*k) print("==========step2") print("insert into another table ") s = 'use db' diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index e9e2596821..3cc3ba828b 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -1,206 +1,397 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -$t0 = 1603152000000 +# -*- coding: utf-8 -*- -create database db update 1 days 30; +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * -## Step 1. UPDATE THE WHOLE DATA BLOCK REPEATEDLY -create table t1 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 1); -} -restart to commit -check query result +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) -for ($k = 0; $k < 10; $k++) { - for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 1); - } + def run(self): + print("==========step1") + print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db update 1 days 30' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') - check query result - restart to commit - check query result -} + insertRows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0 + i)) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) -## Step 2. PREPEND DATA -create table t2 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t2 values ($t0 + $i, 1); -} + for k in range(0,10): + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into t1 values(%d,1)' % + (t0+i) + ) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + print("==========step2") + print("PREPEND DATA ") + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + for i in range(-100,0): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + print("==========step3") + print("PREPEND MASSIVE DATA ") + ret = tdSql.execute('create table t3 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + for i in range(-6000,0): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + print("==========step4") + print("APPEND DATA") + ret = tdSql.execute('create table t4 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + print("==========step5") + print("APPEND DATA") + ret = tdSql.execute('create table t5 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + for i in range(0,6000): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + print("==========step6") + print("UPDATE BLOCK IN TWO STEP") + ret = tdSql.execute('create table t6 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t6 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows+100) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'300') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows+100) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'300') + for i in range(0,200): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows+200) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows+200) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'400') + print("==========step7") + print("UPDATE LAST HALF AND INSERT LITTLE DATA") + ret = tdSql.execute('create table t7 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t7 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + for i in range(100,300): + ret = tdSql.execute( + 'insert into t7 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows+200) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'500') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows+100) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'500') + print("==========step8") + print("UPDATE LAST HALF AND INSERT MASSIVE DATA") + ret = tdSql.execute('create table t8 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t8 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + for i in range(6000): + ret = tdSql.execute( + 'insert into t8 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'12000') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'12000') + print("==========step9") + print("UPDATE FIRST HALF AND PREPEND LITTLE DATA") + ret = tdSql.execute('create table t9 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t9 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + for i in range(-100,100): + ret = tdSql.execute( + 'insert into t9 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'500') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'500') + print("==========step10") + print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA") + ret = tdSql.execute('create table t10 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t10 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + for i in range(-6000,100): + ret = tdSql.execute( + 'insert into t10 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12300') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12300') + print("==========step11") + print("UPDATE FIRST HALF AND APPEND MASSIVE DATA") + ret = tdSql.execute('create table t11 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t11 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + for i in range(100): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + for i in range(200,6000): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11500') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11500') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) -restart to commit -check query result -for ($i = -100; $i < 0; $i++) { - insert into t2 values ($t0 + $i, 1); -} - -check query result -restart to commit -check query result - -## Step 3. PREPEND MASSIVE DATA -create table t3 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t3 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = -6000; $i < 0; $i++) { - insert into t3 values ($t0 + $i, 1); -} - -check query result -restart to commit -check query result - -## Step 4. APPEND DATA -create table t4 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t4 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 0; $i < 100; $i++) { - insert into t4 values ($t0 + 200 + $i, 1); -} - -check query result -restart to commit -check query result - -## Step 5. APPEND MASSIVE DATA -create table t5 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t5 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 0; $i < 6000; $i++) { - insert into t5 values ($t0 + 200 + $i, 1); -} - -check query result -restart to commit -check query result - -## Step 6. UPDATE BLOCK IN TWO STEP -create table t6 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t6 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 0; $i < 100; $i++) { - insert into t6 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -for ($i = 100; $i < 200; $i++) { - insert into t6 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 7. UPDATE LAST HALF AND INSERT LITTLE DATA -create table t7 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t7 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 100; $i < 300; $i++) { - insert into t7 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 8. UPDATE LAST HALF AND INSERT MASSIVE DATA -create table t8 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t8 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 100; $i < 6000; $i++) { - insert into t8 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 9. UPDATE FIRST HALF AND PREPEND LITTLE DATA -create table t9 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t9 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = -100; $i < 100; $i++) { - insert into t9 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 10. UPDATE FIRST HALF AND PREPEND MASSIVE DATA -create table t10 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t10 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = -6000; $i < 100; $i++) { - insert into t10 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 11. UPDATE FIRST HALF AND APPEND MASSIVE DATA -create table t11 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t11 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 0; $i < 100; $i++) { - insert into t11 values ($t0 + $i, 2); -} - -for ($i = 200; $i < 6000; $i++) { - insert into t11 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 6e4702f0168841da8ecded9bc3f3a5dd9a11c62c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 16:08:27 +0800 Subject: [PATCH 027/122] add test case for update --- tests/pytest/update/append_commit_data.py | 6 --- tests/pytest/update/merge_commit_data.py | 45 ----------------------- 2 files changed, 51 deletions(-) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index e0cda0cd90..3169b748e0 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -46,9 +46,7 @@ class TDTestCase: print("==========step2") print("restart to commit ") tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from db.t1") tdSql.checkRows(insertRows) for k in range(0,100): @@ -59,9 +57,7 @@ class TDTestCase: (t0+k*200+i) ) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from db.t1") tdSql.checkRows(insertRows+200*k) print("==========step2") @@ -75,9 +71,7 @@ class TDTestCase: 'insert into t2 values (%d, 1)' % (t0+i)) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t2") tdSql.checkRows(insertRows) diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index 3cc3ba828b..d2b425b955 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -44,9 +44,7 @@ class TDTestCase: 'insert into t1 values (%d , 1)' % (t0 + i)) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t1") tdSql.checkRows(insertRows) @@ -75,9 +73,6 @@ class TDTestCase: tdSql.query("select * from t2") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) - tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t2") tdSql.checkRows(insertRows) for i in range(-100,0): @@ -87,9 +82,7 @@ class TDTestCase: tdSql.query("select * from t2") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t2") tdSql.checkRows(insertRows+100) print("==========step3") @@ -103,9 +96,7 @@ class TDTestCase: tdSql.query("select * from t3") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t3") tdSql.checkRows(insertRows) for i in range(-6000,0): @@ -115,9 +106,7 @@ class TDTestCase: tdSql.query("select * from t3") tdSql.checkRows(insertRows+6000) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t3") tdSql.checkRows(insertRows+6000) print("==========step4") @@ -131,9 +120,7 @@ class TDTestCase: tdSql.query("select * from t4") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t4") tdSql.checkRows(insertRows) for i in range(0,100): @@ -143,9 +130,7 @@ class TDTestCase: tdSql.query("select * from t4") tdSql.checkRows(insertRows+100) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t4") tdSql.checkRows(insertRows+100) print("==========step5") @@ -159,9 +144,7 @@ class TDTestCase: tdSql.query("select * from t5") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t5") tdSql.checkRows(insertRows) for i in range(0,6000): @@ -171,9 +154,7 @@ class TDTestCase: tdSql.query("select * from t5") tdSql.checkRows(insertRows+6000) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t5") tdSql.checkRows(insertRows+6000) print("==========step6") @@ -187,9 +168,7 @@ class TDTestCase: tdSql.query("select * from t6") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t6") tdSql.checkRows(insertRows) for i in range(0,100): @@ -201,9 +180,7 @@ class TDTestCase: tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'300') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t6") tdSql.checkRows(insertRows+100) tdSql.query("select sum(a) from t6") @@ -217,9 +194,7 @@ class TDTestCase: tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'400') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t6") tdSql.checkRows(insertRows+200) tdSql.query("select sum(a) from t6") @@ -235,9 +210,7 @@ class TDTestCase: tdSql.query("select * from t7") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t7") tdSql.checkRows(insertRows) for i in range(100,300): @@ -249,9 +222,7 @@ class TDTestCase: tdSql.query("select sum(a) from t7") tdSql.checkData(0,0,'500') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t7") tdSql.checkRows(insertRows+100) tdSql.query("select sum(a) from t7") @@ -267,9 +238,7 @@ class TDTestCase: tdSql.query("select * from t8") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t8") tdSql.checkRows(insertRows) for i in range(6000): @@ -281,9 +250,7 @@ class TDTestCase: tdSql.query("select sum(a) from t8") tdSql.checkData(0,0,'12000') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t8") tdSql.checkRows(6000) tdSql.query("select sum(a) from t8") @@ -299,9 +266,7 @@ class TDTestCase: tdSql.query("select * from t9") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t9") tdSql.checkRows(insertRows) for i in range(-100,100): @@ -313,9 +278,7 @@ class TDTestCase: tdSql.query("select sum(a) from t9") tdSql.checkData(0,0,'500') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t9") tdSql.checkRows(300) tdSql.query("select sum(a) from t9") @@ -331,9 +294,7 @@ class TDTestCase: tdSql.query("select * from t10") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t10") tdSql.checkRows(insertRows) for i in range(-6000,100): @@ -345,9 +306,7 @@ class TDTestCase: tdSql.query("select sum(a) from t10") tdSql.checkData(0,0,'12300') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t10") tdSql.checkRows(6200) tdSql.query("select sum(a) from t10") @@ -363,9 +322,7 @@ class TDTestCase: tdSql.query("select * from t11") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t11") tdSql.checkRows(insertRows) for i in range(100): @@ -381,9 +338,7 @@ class TDTestCase: tdSql.query("select sum(a) from t11") tdSql.checkData(0,0,'11500') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t11") tdSql.checkRows(6000) tdSql.query("select sum(a) from t11") From e84adb0c53ff57806b2064adb106985ce2a60baf Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 16:13:55 +0800 Subject: [PATCH 028/122] add test case for update --- tests/pytest/update/merge_commit_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index d2b425b955..9b3fea1865 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -57,9 +57,7 @@ class TDTestCase: tdSql.query("select * from t1") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t1") tdSql.checkRows(insertRows) print("==========step2") @@ -73,6 +71,7 @@ class TDTestCase: tdSql.query("select * from t2") tdSql.checkRows(insertRows) tdDnodes.stop(1) + tdDnodes.start(1) tdSql.query("select * from t2") tdSql.checkRows(insertRows) for i in range(-100,0): From 97ba24b9365e515d70a3dcc2770853c2a93c61d1 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 3 Nov 2020 16:34:56 +0800 Subject: [PATCH 029/122] [add test case] --- tests/pytest/update/allow_update.py | 71 ++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py index 2879fc6f79..cb77ea6a31 100644 --- a/tests/pytest/update/allow_update.py +++ b/tests/pytest/update/allow_update.py @@ -36,10 +36,11 @@ class TDTestCase: tdSql.prepare() startTs = self.ts - print("==============step1") tdSql.execute("create database udb update 1") tdSql.execute("use udb") tdSql.execute("create table t (ts timestamp, a int)") + + print("==============step1") tdSql.execute("insert into t values (%d, 1)" % (startTs)) tdSql.execute("insert into t values (%d, 1)" % (startTs - 3)) tdSql.execute("insert into t values (%d, 1)" % (startTs + 3)) @@ -159,7 +160,73 @@ class TDTestCase: tdSql.checkData(5, 0, 6) tdSql.checkData(6, 0, 6) tdSql.checkData(7, 0, 6) - tdSql.checkData(8, 0, 6) + tdSql.checkData(8, 0, 6) + + tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)") + + print("==============step1") + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs)) + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3)) + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3)) + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(2, 1, None) + tdSql.checkData(0, 2, 'c-3') + tdSql.checkData(1, 2, 'c+0') + tdSql.checkData(2, 2, 'c+3') + tdSql.checkData(0, 3, None) + tdSql.checkData(1, 3, None) + tdSql.checkData(2, 3, None) + + print("==============step2") + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs)) + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3)) + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3)) + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(0, 1, 2.0) + tdSql.checkData(1, 1, 2.0) + tdSql.checkData(2, 1, 2.0) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(0, 3, 1) + tdSql.checkData(1, 3, 1) + tdSql.checkData(2, 3, 0) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(0, 1, 2.0) + tdSql.checkData(1, 1, 2.0) + tdSql.checkData(2, 1, 2.0) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(0, 3, 1) + tdSql.checkData(1, 3, 1) + tdSql.checkData(2, 3, 0) def stop(self): tdSql.close() From 8640626418aeaba7603d6bb727c8220b0e2268ee Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 16:35:12 +0800 Subject: [PATCH 030/122] add test case for update --- tests/pytest/update/merge_commit_data.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index 9b3fea1865..4fb6765361 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -79,7 +79,7 @@ class TDTestCase: 'insert into t2 values (%d , 1)' % (t0+i)) tdSql.query("select * from t2") - tdSql.checkRows(insertRows) + tdSql.checkRows(insertRows+100) tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t2") @@ -175,13 +175,13 @@ class TDTestCase: 'insert into t6 values (%d , 2)' % (t0+i)) tdSql.query("select * from t6") - tdSql.checkRows(insertRows+100) + tdSql.checkRows(insertRows) tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'300') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t6") - tdSql.checkRows(insertRows+100) + tdSql.checkRows(insertRows) tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'300') for i in range(0,200): @@ -189,13 +189,13 @@ class TDTestCase: 'insert into t6 values (%d , 2)' % (t0+i)) tdSql.query("select * from t6") - tdSql.checkRows(insertRows+200) + tdSql.checkRows(insertRows) tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'400') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t6") - tdSql.checkRows(insertRows+200) + tdSql.checkRows(insertRows) tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'400') print("==========step7") @@ -217,13 +217,13 @@ class TDTestCase: 'insert into t7 values (%d , 2)' % (t0+i)) tdSql.query("select * from t7") - tdSql.checkRows(insertRows+200) + tdSql.checkRows(300) tdSql.query("select sum(a) from t7") tdSql.checkData(0,0,'500') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t7") - tdSql.checkRows(insertRows+100) + tdSql.checkRows(300) tdSql.query("select sum(a) from t7") tdSql.checkData(0,0,'500') print("==========step8") @@ -335,13 +335,13 @@ class TDTestCase: tdSql.query("select * from t11") tdSql.checkRows(6000) tdSql.query("select sum(a) from t11") - tdSql.checkData(0,0,'11500') + tdSql.checkData(0,0,'11900') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t11") tdSql.checkRows(6000) tdSql.query("select sum(a) from t11") - tdSql.checkData(0,0,'11500') + tdSql.checkData(0,0,'11900') def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 9d18f77db53cfcf180b274f32537b76bc8ee9566 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 3 Nov 2020 17:31:39 +0800 Subject: [PATCH 031/122] [NONE] --- tests/pytest/update/allow_update.py | 33 +++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py index cb77ea6a31..5871197f02 100644 --- a/tests/pytest/update/allow_update.py +++ b/tests/pytest/update/allow_update.py @@ -164,7 +164,7 @@ class TDTestCase: tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)") - print("==============step1") + print("==============step7") tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs)) tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3)) tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3)) @@ -186,7 +186,7 @@ class TDTestCase: tdSql.checkData(1, 3, None) tdSql.checkData(2, 3, None) - print("==============step2") + print("==============step8") tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs)) tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3)) tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3)) @@ -227,6 +227,35 @@ class TDTestCase: tdSql.checkData(0, 3, 1) tdSql.checkData(1, 3, 1) tdSql.checkData(2, 3, 0) + + + + tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))") + + print("==============step9") + insertRows = 20000 + for i in range(0, insertRows): + tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i)) + + tdSql.query("select * from ct") + tdSql.checkRows(insertRows) + + for i in range(0, insertRows): + tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows)) + + tdSql.query("select * from ct") + tdSql.checkRows(insertRows) + + tdSql.query("select a,b from ct limit 3") + tdSql.checkData(0, 0, insertRows+0) + tdSql.checkData(1, 0, insertRows+1) + tdSql.checkData(2, 0, insertRows+2) + + tdSql.checkData(0, 1, insertRows+0) + tdSql.checkData(1, 1, insertRows+1) + tdSql.checkData(2, 1, insertRows+2) + + def stop(self): tdSql.close() From 4746305dbdbbd926590ba470526098916bdb6e00 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 Nov 2020 17:35:02 +0800 Subject: [PATCH 032/122] fix TD-1902 --- src/tsdb/src/tsdbRWHelper.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index eb1193110d..c3f92e2bd1 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1572,7 +1572,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { while (true) { tdResetDataCols(pDataCols); - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst, defaultRowsInBlock, pDataCols, NULL, 0, + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); if (pDataCols->numOfRows == 0) break; @@ -1714,6 +1714,8 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt (*iter)++; tSkipListIterNext(pCommitIter->pIter); } + + if (pTarget->numOfRows >= maxRows) break; } } From 9c289dd7626e6be6d6f666acb7baad81f2b66707 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 3 Nov 2020 18:37:03 +0800 Subject: [PATCH 033/122] add update test case --- tests/pytest/update/append_commit_last.py | 109 ++++-- tests/pytest/update/merge_commit_last.py | 430 ++++++++++++++-------- 2 files changed, 358 insertions(+), 181 deletions(-) diff --git a/tests/pytest/update/append_commit_last.py b/tests/pytest/update/append_commit_last.py index 9b31adccec..013983f940 100644 --- a/tests/pytest/update/append_commit_last.py +++ b/tests/pytest/update/append_commit_last.py @@ -1,42 +1,85 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -create database db update 1; +# -*- coding: utf-8 -*- -## Step 1 -$loops = 1000 -#t0 = 1604298064000 +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * -create table t1 (ts timestamp, a int); -for ($i = 0; $i < $loops; $i++) { - insert into t1 values ($t0 + $i, 1); - restart to commit - check query result -} -## Step 2 -create table t2 (ts timestamp, a int); +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) -insert into t2 ($t0, 1); -restart to commit -check query result + self.ts = 1604298064000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use udb") -for ($i = 1; $i <= 150; $i++) { - insert into t2 values ($t0 + $i, 1); -} -restart to commit -check query result + def run(self): + tdSql.prepare() -## Step 3 -create table t3 (ts timestamp, a int); + print("==============step1") + tdSql.execute("create database udb update 1") + tdSql.execute("use udb") + tdSql.execute("create table t1 (ts timestamp, a int)") -insert into t3 ($t0, 1); -restart to commit -check query result + for i in range(10): + tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(i + 1) -for ($i = 0; $i < 8; $i++) { - for ($j = 1; $j <= 10; $j++) { - insert into t3 values ($t0 + $i * 10 + $j , 1); - } -} -restart to commit -check query result \ No newline at end of file + + print("==============step2") + tdSql.execute("create table t2 (ts timestamp, a int)") + tdSql.execute("insert into t2 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(1) + + for i in range(1, 151): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(151) + + + print("==============step3") + tdSql.execute("create table t3 (ts timestamp, a int)") + tdSql.execute("insert into t3 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(1) + + for i in range(8): + for j in range(1, 11): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j)) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(81) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_last.py b/tests/pytest/update/merge_commit_last.py index 9d832d8984..183cca0a1e 100644 --- a/tests/pytest/update/merge_commit_last.py +++ b/tests/pytest/update/merge_commit_last.py @@ -1,187 +1,321 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -$t0 = 1603152000000 +# -*- coding: utf-8 -*- -create database db update 1 days 30; +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * -## STEP 1: UPDATE THE LAST RECORD REPEATEDLY -create table t1 (ts timestamp, a int); -for ($i = 0; $i < 100; $i++) { - insert into t1 values ($t0, $i); +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) - restart to commit - check query result -} + self.ts = 1603152000000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use udb") -## STEP 2: UPDATE THE WHOLE LAST BLOCK -create table t2 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t2 values ($t0 + $i, 1); -} -restart to commit -check query result + def run(self): + tdSql.prepare() -for ($i = 0; $i < 50; $i++) { - insert into t2 values ($t0 + $i, 2); -} + tdSql.execute("create database udb update 1 days 30") + tdSql.execute("use udb") -check query result -restart to commit -check query result + print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY") + tdSql.execute("create table t1 (ts timestamp, a int)") -## STEP 3: UPDATE PART OF THE LAST BLOCK -create table t3 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t3 values ($t0 + $i, 1); -} -restart to commit -check query result + for i in range(5): + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, i) -for ($i = 0; $i < 25; $i++) { - insert into t3 values ($t0 + $i, 2); -} + print("==============step 2: UPDATE THE WHOLE LAST BLOCK") + tdSql.execute("create table t2 (ts timestamp, a int)") -check query result -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) -for ($i = 25; $i < 50; $i++) { - insert into t3 values ($t0 + $i, 2); -} + for i in range(50): + tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i)) + tdSql.query("select * from t2") + for i in range(50): + tdSql.checkData(i, 1, 2) -check query result -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 2) -## STEP 4: UPDATE AND INSERT APPEND AT END OF DATA -create table t4 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t4 values ($t0 + $i, 1); -} -restart to commit -check query result + print("==============step 3: UPDATE PART OF THE LAST BLOCK") + tdSql.execute("create table t3 (ts timestamp, a int)") -for ($i = 0; $i < 25; $i++) { - insert into t4 values ($t0 + $i, 2); -} + for i in range(50): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) -for ($i = 50; $i < 60; $i++) { - insert into t4 values ($t0 + $i, 2); -} + for i in range(25): + tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t3") + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) -check query result -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + + print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA") + tdSql.execute("create table t4 (ts timestamp, a int)") -## STEP 5: UPDATE AND INSERT PREPEND SOME DATA -create table t5 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t5 values ($t0 + $i, 1); -} -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) -for ($i = -10; $i < 0; $i++) { - insert into t4 values ($t0 + $i, 2); -} + for i in range(25): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + for i in range(50, 60): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t4") + tdSql.checkRows(60) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + for i in range(50, 60): + tdSql.checkData(i, 1, 2) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(60) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + for i in range(50, 60): + tdSql.checkData(i, 1, 2) -for ($i = 0; $i < 25; $i++) { - insert into t5 values ($t0 + $i, 2); -} + print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA") + tdSql.execute("create table t5 (ts timestamp, a int)") -check query result -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + for i in range(25): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 95) -for ($i = -10; $i < 0; $i++) { - insert into t4 values ($t0 + $i, 3); -} + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 95) -for ($i = 25; $i < 50; $i++) { - insert into t5 values ($t0 + $i, 3); -} + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + for i in range(25, 50): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) -check query result -restart to commit -check query result + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 155) -## STEP 6: INSERT AHEAD A LOT OF DATA -create table t6 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t6 values ($t0 + $i, 1); -} -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 155) + -for ($i = -1000; $i < 0; $i++) { - insert into t6 values ($t0 + $i, 2); -} + print("==============step 6: INSERT AHEAD A LOT OF DATA") + tdSql.execute("create table t6 (ts timestamp, a int)") -check query result -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i)) -## STEP 7: INSERT AHEAD A LOT AND UPDATE -create table t7 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t7 values ($t0 + $i, 1); -} -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 0): + tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) -for ($i = -1000; $i < 25; $i++) { - insert into t7 values ($t0 + $i, 2); -} + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) -check query result -restart to commit -check query result + print("==============step 7: INSERT AHEAD A LOT AND UPDATE") + tdSql.execute("create table t7 (ts timestamp, a int)") -## STEP 8: INSERT AHEAD A LOT AND UPDATE -create table t8 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t8 values ($t0 + $i, 1); -} -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i)) -for ($i = 25; $i < 6000; $i++) { - insert into t8 values ($t0 + $i, 2); -} + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 25): + tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2075) -check query result -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2075) -## STEP 9: UPDATE ONLY MIDDLE -create table t9 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t9 values ($t0 + $i, 1); -} -restart to commit -check query result + print("==============step 8: INSERT AFTER A LOT AND UPDATE") + tdSql.execute("create table t8 (ts timestamp, a int)") -for ($i = 20; $i < 30; $i++) { - insert into t9 values ($t0 + $i, 2); -} + for i in range(50): + tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i)) -check query result -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 50) + + for i in range(25, 6000): + tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11975) -## STEP 10: A LOT OF DATA COVER THE WHOLE BLOCK -create table t10 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t10 values ($t0 + $i, 1); -} -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11975) -for ($i = -4000; $i < 4000; $i++) { - insert into t10 values ($t0 + $i, 2); -} + print("==============step 9: UPDATE ONLY MIDDLE") + tdSql.execute("create table t9 (ts timestamp, a int)") -check query result -restart to commit -check query result \ No newline at end of file + for i in range(50): + tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + for i in range(20, 30): + tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 60) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 60) + + print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK") + tdSql.execute("create table t10 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 50) + + for i in range(-4000, 4000): + tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 16000) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 16000) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 428524bd08a3daddb10974b82dca0f99c4e58112 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 3 Nov 2020 14:05:21 +0000 Subject: [PATCH 034/122] first version for taos file --- src/util/inc/tfile.h | 43 ++++++++ src/util/inc/tref.h | 37 ++++--- src/util/src/tfile.c | 94 +++++++++++++++++ src/util/src/tref.c | 246 +++++++++++++++++++++---------------------- 4 files changed, 278 insertions(+), 142 deletions(-) create mode 100644 src/util/inc/tfile.h create mode 100644 src/util/src/tfile.c diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h new file mode 100644 index 0000000000..00b2fd6c32 --- /dev/null +++ b/src/util/inc/tfile.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TFILE_H +#define TDENGINE_TFILE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +// the same syntax as UNIX standard open/close/read/write +// but FD is int64_t and will never be reused + +int64_t tfopen(const char *pathname, int flags); +int64_t tfclose(int64_t tfd); +ssize_t tfwrite(int64_t tfd, const void *buf, size_t count); +ssize_t tfread(int64_t tfd, void *buf, size_t count); + +// init taos file module +int tfinit(); + +// clean up taos fle module +void tfcleanup(); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TREF_H diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h index ead8e2eb90..3e5db33cf7 100644 --- a/src/util/inc/tref.h +++ b/src/util/inc/tref.h @@ -21,38 +21,45 @@ extern "C" { #endif -// open an instance, return refId which will be used by other APIs -int taosOpenRef(int max, void (*fp)(void *)); +// open a reference set, max is the mod used by hash, fp is the pointer to free resource function +// return rsetId which will be used by other APIs. On error, -1 is returned, and terrno is set appropriately +int taosOpenRef(int max, void (*fp)(void *)); -// close the Ref instance -void taosCloseRef(int refId); +// close the reference set, refId is the return value by taosOpenRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosCloseRef(int refId); // add ref, p is the pointer to resource or pointer ID -int taosAddRef(int refId, void *p); +// return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately +int64_t taosAddRef(int refId, void *p); #define taosRemoveRef taosReleaseRef -// acquire ref, p is the pointer to resource or pointer ID -int taosAcquireRef(int refId, void *p); +// acquire ref, rid is the reference ID returned by taosAddRef +// return the resource p. On error, NULL is returned, and terrno is set appropriately +void *taosAcquireRef(int rsetId, int64_t rid); -// release ref, p is the pointer to resource or pinter ID -void taosReleaseRef(int refId, void *p); +// release ref, rid is the reference ID returned by taosAddRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosReleaseRef(int rsetId, int64_t rid); -// return the first if p is null, otherwise return the next after p -void *taosIterateRef(int refId, void *p); +// return the first reference if rid is 0, otherwise return the next after current reference. +// if return value is NULL, it means list is over(if terrno is set, it means error happens) +void *taosIterateRef(int rsetId, int64_t rid); // return the number of references in system int taosListRef(); /* sample code to iterate the refs -void demoIterateRefs(int refId) { +void demoIterateRefs(int rsetId) { - void *p = taosIterateRef(refId, NULL); + void *p = taosIterateRef(refId, 0); while (p) { - // process P + + // get the rid from p - p = taosIterateRef(refId, p); + p = taosIterateRef(rsetId, rid); } } diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c new file mode 100644 index 0000000000..ea699c2436 --- /dev/null +++ b/src/util/src/tfile.c @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taoserror.h" +#include "tulog.h" +#include "tutil.h" +#include "tref.h" + +static int tsFileRsetId = -1; + +static void taosCloseFile(void *p) { + close((int)(uintptr_t)p); +} + +int tfinit() { + + tsFileRsetId = taosOpenRef(2000, taosCloseFile); + return tsFileRsetId; +} + +void tfcleanup() { + + if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); + tsFileRsetId = -1; +} + +int64_t tfopen(const char *pathname, int flags) { + int fd = open(pathname, flags); + + if (fd < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + int64_t rid = taosAddRef(tsFileRsetId, (void *)(long)fd); + if (rid < 0) { + close(fd); + return -1; + } + + return rid; +} + +int64_t tfclose(int64_t tfd) { + return taosReleaseRef(tsFileRsetId, tfd); +} + +ssize_t tfwrite(int64_t tfd, const void *buf, size_t count) { + + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return terrno; + + int fd = (int)(uintptr_t)p; + + ssize_t ret = write(fd, buf, count); + if (ret < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + taosReleaseRef(tsFileRsetId, tfd); + return ret; +} + +ssize_t tfread(int64_t tfd, void *buf, size_t count) { + + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return terrno; + + int fd = (int)(uintptr_t)p; + + ssize_t ret = read(fd, buf, count); + if (ret < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + taosReleaseRef(tsFileRsetId, tfd); + return ret; +} + diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 23a7210e99..c2adb6d15b 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -24,19 +24,21 @@ #define TSDB_REF_STATE_DELETED 2 typedef struct SRefNode { - struct SRefNode *prev; - struct SRefNode *next; - void *p; - int32_t count; + struct SRefNode *prev; // previous node + struct SRefNode *next; // next node + void *p; // pointer to resource protected, + int64_t rid; // reference ID + int32_t count; // number of references } SRefNode; typedef struct { - SRefNode **nodeList; - int state; // 0: empty, 1: active; 2: deleted - int refId; - int max; - int32_t count; // total number of SRefNodes in this set - int64_t *lockedBy; + SRefNode **nodeList; // array of SRefNode linked list + int state; // 0: empty, 1: active; 2: deleted + int rsetId; // refSet ID, global unique + int64_t rid; // increase by one for each new reference + int max; // mod + int32_t count; // total number of SRefNodes in this set + int64_t *lockedBy; void (*fp)(void *); } SRefSet; @@ -47,7 +49,6 @@ static int tsRefSetNum = 0; static int tsNextId = 0; static void taosInitRefModule(void); -static int taosHashRef(SRefSet *pSet, void *p); static void taosLockList(int64_t *lockedBy); static void taosUnlockList(int64_t *lockedBy); static void taosIncRefCount(SRefSet *pSet); @@ -58,19 +59,21 @@ int taosOpenRef(int max, void (*fp)(void *)) SRefNode **nodeList; SRefSet *pSet; int64_t *lockedBy; - int i, refId; + int i, rsetId; pthread_once(&tsRefModuleInit, taosInitRefModule); nodeList = calloc(sizeof(SRefNode *), (size_t)max); - if (nodeList == NULL) { - return TSDB_CODE_REF_NO_MEMORY; + if (nodeList == NULL) { + terrno = TSDB_CODE_REF_NO_MEMORY; + return -1; } lockedBy = calloc(sizeof(int64_t), (size_t)max); if (lockedBy == NULL) { free(nodeList); - return TSDB_CODE_REF_NO_MEMORY; + terrno = TSDB_CODE_REF_NO_MEMORY; + return -1; } pthread_mutex_lock(&tsRefMutex); @@ -81,20 +84,21 @@ int taosOpenRef(int max, void (*fp)(void *)) } if (i < TSDB_REF_OBJECTS) { - refId = tsNextId; - pSet = tsRefSetList + refId; + rsetId = tsNextId; + pSet = tsRefSetList + rsetId; taosIncRefCount(pSet); pSet->max = max; pSet->nodeList = nodeList; pSet->lockedBy = lockedBy; pSet->fp = fp; + pSet->rid = 1; + pSet->rsetId = rsetId; pSet->state = TSDB_REF_STATE_ACTIVE; - pSet->refId = refId; tsRefSetNum++; - uTrace("refId:%d is opened, max:%d, fp:%p refSetNum:%d", refId, max, fp, tsRefSetNum); + uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum); } else { - refId = TSDB_CODE_REF_FULL; + rsetId = TSDB_CODE_REF_FULL; free (nodeList); free (lockedBy); uTrace("run out of Ref ID, maximum:%d refSetNum:%d", TSDB_REF_OBJECTS, tsRefSetNum); @@ -102,121 +106,116 @@ int taosOpenRef(int max, void (*fp)(void *)) pthread_mutex_unlock(&tsRefMutex); - return refId; + return rsetId; } -void taosCloseRef(int refId) +int taosCloseRef(int rsetId) { SRefSet *pSet; int deleted = 0; - if (refId < 0 || refId >= TSDB_REF_OBJECTS) { - uTrace("refId:%d is invalid, out of range", refId); - return; + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d is invalid, out of range", rsetId); + terrno = TSDB_CODE_REF_INVALID_ID; + return -1; } - pSet = tsRefSetList + refId; + pSet = tsRefSetList + rsetId; pthread_mutex_lock(&tsRefMutex); if (pSet->state == TSDB_REF_STATE_ACTIVE) { pSet->state = TSDB_REF_STATE_DELETED; deleted = 1; - uTrace("refId:%d is closed, count:%d", refId, pSet->count); + uTrace("rsetId:%d is closed, count:%d", rsetId, pSet->count); } else { - uTrace("refId:%d is already closed, count:%d", refId, pSet->count); + uTrace("rsetId:%d is already closed, count:%d", rsetId, pSet->count); } pthread_mutex_unlock(&tsRefMutex); if (deleted) taosDecRefCount(pSet); + + return 0; } -int taosAddRef(int refId, void *p) +int64_t taosAddRef(int rsetId, void *p) { int hash; SRefNode *pNode; SRefSet *pSet; + int64_t rid = 0; - if (refId < 0 || refId >= TSDB_REF_OBJECTS) { - uTrace("refId:%d p:%p failed to add, refId not valid", refId, p); - return TSDB_CODE_REF_INVALID_ID; + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d p:%p failed to add, rsetId not valid", rsetId, p); + terrno = TSDB_CODE_REF_INVALID_ID; + return -1; } - pSet = tsRefSetList + refId; + pSet = tsRefSetList + rsetId; taosIncRefCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { taosDecRefCount(pSet); - uTrace("refId:%d p:%p failed to add, not active", refId, p); - return TSDB_CODE_REF_ID_REMOVED; + uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p); + terrno = TSDB_CODE_REF_ID_REMOVED; + return -1; } - int code = 0; - hash = taosHashRef(pSet, p); + pNode = calloc(sizeof(SRefNode), 1); + if (pNode == NULL) { + terrno = TSDB_CODE_REF_NO_MEMORY; + return -1; + } + rid = atomic_add_fetch_64(&pSet->rid, 1); + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); - pNode = pSet->nodeList[hash]; - while (pNode) { - if (pNode->p == p) - break; + pNode->p = p; + pNode->rid = rid; + pNode->count = 1; - pNode = pNode->next; - } + pNode->prev = NULL; + pNode->next = pSet->nodeList[hash]; + if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode; + pSet->nodeList[hash] = pNode; - if (pNode) { - code = TSDB_CODE_REF_ALREADY_EXIST; - uTrace("refId:%d p:%p is already there, faild to add", refId, p); - } else { - pNode = calloc(sizeof(SRefNode), 1); - if (pNode) { - pNode->p = p; - pNode->count = 1; - pNode->prev = 0; - pNode->next = pSet->nodeList[hash]; - if (pSet->nodeList[hash]) pSet->nodeList[hash]->prev = pNode; - pSet->nodeList[hash] = pNode; - uTrace("refId:%d p:%p is added, count:%d malloc mem: %p", refId, p, pSet->count, pNode); - } else { - code = TSDB_CODE_REF_NO_MEMORY; - uTrace("refId:%d p:%p is not added, since no memory", refId, p); - } - } - - if (code < 0) taosDecRefCount(pSet); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is added, count:%d", rsetId, p, rid, pSet->count); taosUnlockList(pSet->lockedBy+hash); - return code; + return rid; } -int taosAcquireRef(int refId, void *p) +void *taosAcquireRef(int rsetId, int64_t rid) { - int hash, code = 0; + int hash; SRefNode *pNode; SRefSet *pSet; + void *p = NULL; - if (refId < 0 || refId >= TSDB_REF_OBJECTS) { - uTrace("refId:%d p:%p failed to acquire, refId not valid", refId, p); - return TSDB_CODE_REF_INVALID_ID; + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rsetId not valid", rsetId, rid); + terrno = TSDB_CODE_REF_INVALID_ID; + return NULL; } - pSet = tsRefSetList + refId; + pSet = tsRefSetList + rsetId; taosIncRefCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { - uTrace("refId:%d p:%p failed to acquire, not active", refId, p); + uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, not active", rsetId, rid); taosDecRefCount(pSet); - return TSDB_CODE_REF_ID_REMOVED; + terrno = TSDB_CODE_REF_ID_REMOVED; + return NULL; } - hash = taosHashRef(pSet, p); - + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); pNode = pSet->nodeList[hash]; while (pNode) { - if (pNode->p == p) { + if (pNode->rid == rid) { break; } @@ -225,44 +224,47 @@ int taosAcquireRef(int refId, void *p) if (pNode) { pNode->count++; - uTrace("refId:%d p:%p is acquired", refId, p); + p = pNode->p; + uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid); } else { - code = TSDB_CODE_REF_NOT_EXIST; - uTrace("refId:%d p:%p is not there, failed to acquire", refId, p); + terrno = TSDB_CODE_REF_NOT_EXIST; + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid); } taosUnlockList(pSet->lockedBy+hash); taosDecRefCount(pSet); - return code; + return p; } -void taosReleaseRef(int refId, void *p) +int taosReleaseRef(int rsetId, int64_t rid) { int hash; SRefNode *pNode; SRefSet *pSet; int released = 0; - if (refId < 0 || refId >= TSDB_REF_OBJECTS) { - uTrace("refId:%d p:%p failed to release, refId not valid", refId, p); - return; + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d rid:%" PRId64 " failed to release, rsetId not valid", rsetId, rid); + terrno = TSDB_CODE_REF_INVALID_ID; + return -1; } - pSet = tsRefSetList + refId; + pSet = tsRefSetList + rsetId; if (pSet->state == TSDB_REF_STATE_EMPTY) { - uTrace("refId:%d p:%p failed to release, cleaned", refId, p); - return; + uTrace("rsetId:%d rid:%" PRId64 " failed to release, cleaned", rsetId, rid); + terrno = TSDB_CODE_REF_ID_REMOVED; + return -1; } - hash = taosHashRef(pSet, p); - + terrno = 0; + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); pNode = pSet->nodeList[hash]; while (pNode) { - if (pNode->p == p) + if (pNode->rid == rid) break; pNode = pNode->next; @@ -284,57 +286,63 @@ void taosReleaseRef(int refId, void *p) (*pSet->fp)(pNode->p); + uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); free(pNode); released = 1; - uTrace("refId:%d p:%p is removed, count:%d, free mem: %p", refId, p, pSet->count, pNode); } else { - uTrace("refId:%d p:%p is released", refId, p); + uTrace("rsetId:%d p:%p rid:%" PRId64 "is released", rsetId, pNode->p, rid); } } else { - uTrace("refId:%d p:%p is not there, failed to release", refId, p); + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; } taosUnlockList(pSet->lockedBy+hash); if (released) taosDecRefCount(pSet); + + return terrno; } -// if p is NULL, return the first p in hash list, otherwise, return the next after p -void *taosIterateRef(int refId, void *p) { +// if rid is 0, return the first p in hash list, otherwise, return the next after current rid +void *taosIterateRef(int rsetId, int64_t rid) { SRefNode *pNode = NULL; SRefSet *pSet; - if (refId < 0 || refId >= TSDB_REF_OBJECTS) { - uTrace("refId:%d p:%p failed to iterate, refId not valid", refId, p); + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rsetId not valid", rsetId, rid); + terrno = TSDB_CODE_REF_INVALID_ID; return NULL; } - pSet = tsRefSetList + refId; + pSet = tsRefSetList + rsetId; taosIncRefCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { - uTrace("refId:%d p:%p failed to iterate, not active", refId, p); + uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rset not active", rsetId, rid); + terrno = TSDB_CODE_REF_ID_REMOVED; taosDecRefCount(pSet); return NULL; } int hash = 0; - if (p) { - hash = taosHashRef(pSet, p); + if (rid > 0) { + hash = rid % pSet->max; taosLockList(pSet->lockedBy+hash); pNode = pSet->nodeList[hash]; while (pNode) { - if (pNode->p == p) break; + if (pNode->rid == rid) break; pNode = pNode->next; } if (pNode == NULL) { - uError("refId:%d p:%p not there, quit", refId, p); + uError("rsetId:%d rid:%" PRId64 " not there, quit", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; taosUnlockList(pSet->lockedBy+hash); return NULL; } - // p is there + // rid is there pNode = pNode->next; if (pNode == NULL) { taosUnlockList(pSet->lockedBy+hash); @@ -356,12 +364,12 @@ void *taosIterateRef(int refId, void *p) { pNode->count++; // acquire it newP = pNode->p; taosUnlockList(pSet->lockedBy+hash); - uTrace("refId:%d p:%p is returned", refId, p); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); } else { - uTrace("refId:%d p:%p the list is over", refId, p); + uTrace("rsetId:%d the list is over", rsetId); } - if (p) taosReleaseRef(refId, p); // release the current one + if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one taosDecRefCount(pSet); @@ -381,13 +389,13 @@ int taosListRef() { if (pSet->state == TSDB_REF_STATE_EMPTY) continue; - uInfo("refId:%d state:%d count::%d", i, pSet->state, pSet->count); + uInfo("rsetId:%d state:%d count::%d", i, pSet->state, pSet->count); for (int j=0; j < pSet->max; ++j) { pNode = pSet->nodeList[j]; while (pNode) { - uInfo("refId:%d p:%p count:%d", i, pNode->p, pNode->count); + uInfo("rsetId:%d p:%p rid:%" PRId64 "count:%d", i, pNode->p, pNode->rid, pNode->count); pNode = pNode->next; num++; } @@ -399,22 +407,6 @@ int taosListRef() { return num; } -static int taosHashRef(SRefSet *pSet, void *p) -{ - int hash = 0; - int64_t v = (int64_t)p; - - for (int i = 0; i < sizeof(v); ++i) { - hash += (int)(v & 0xFFFF); - v = v >> 16; - i = i + 2; - } - - hash = hash % pSet->max; - - return hash; -} - static void taosLockList(int64_t *lockedBy) { int64_t tid = taosGetPthreadId(); int i = 0; @@ -438,12 +430,12 @@ static void taosInitRefModule(void) { static void taosIncRefCount(SRefSet *pSet) { atomic_add_fetch_32(&pSet->count, 1); - uTrace("refId:%d inc count:%d", pSet->refId, pSet->count); + uTrace("rsetId:%d inc count:%d", pSet->rsetId, pSet->count); } static void taosDecRefCount(SRefSet *pSet) { int32_t count = atomic_sub_fetch_32(&pSet->count, 1); - uTrace("refId:%d dec count:%d", pSet->refId, pSet->count); + uTrace("rsetId:%d dec count:%d", pSet->rsetId, pSet->count); if (count > 0) return; @@ -458,7 +450,7 @@ static void taosDecRefCount(SRefSet *pSet) { taosTFree(pSet->lockedBy); tsRefSetNum--; - uTrace("refId:%d is cleaned, refSetNum:%d count:%d", pSet->refId, tsRefSetNum, pSet->count); + uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count); } pthread_mutex_unlock(&tsRefMutex); From a8e272d249518da1c897ac05aabeec286681254c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 Nov 2020 05:30:42 +0000 Subject: [PATCH 035/122] fix TD-1934 --- src/common/src/tdataformat.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 49d21d9275..40b524488a 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -434,12 +434,12 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); - ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); ASSERT(target->numOfCols == source->numOfCols); SDataCols *pTarget = NULL; if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap + ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); for (int i = 0; i < rowsToMerge; i++) { for (int j = 0; j < source->numOfCols; j++) { if (source->cols[j].len > 0) { @@ -509,6 +509,8 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i (*iter2)++; if (key1 == key2) (*iter1)++; } + + ASSERT(target->numOfRows <= target->maxPoints); } } From 43701df54a7186a28fd0547af76a66ca839d4841 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 4 Nov 2020 15:28:32 +0800 Subject: [PATCH 036/122] add same test cases with update flag 0 --- tests/pytest/update/append_commit_last-0.py | 90 ++++++ tests/pytest/update/merge_commit_last-0.py | 309 ++++++++++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 tests/pytest/update/append_commit_last-0.py create mode 100644 tests/pytest/update/merge_commit_last-0.py diff --git a/tests/pytest/update/append_commit_last-0.py b/tests/pytest/update/append_commit_last-0.py new file mode 100644 index 0000000000..c884207f2b --- /dev/null +++ b/tests/pytest/update/append_commit_last-0.py @@ -0,0 +1,90 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1604298064000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use db") + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table t1 (ts timestamp, a int)") + + for i in range(10): + tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(i + 1) + tdSql.query("select sum(a) from t1") + tdSql.checkData(0, 0, i + 1) + + print("==============step2") + tdSql.execute("create table t2 (ts timestamp, a int)") + tdSql.execute("insert into t2 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + for i in range(1, 151): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(151) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 151) + + + print("==============step3") + tdSql.execute("create table t3 (ts timestamp, a int)") + tdSql.execute("insert into t3 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + for i in range(8): + for j in range(1, 11): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j)) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(81) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 81) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_last-0.py b/tests/pytest/update/merge_commit_last-0.py new file mode 100644 index 0000000000..8a247f3809 --- /dev/null +++ b/tests/pytest/update/merge_commit_last-0.py @@ -0,0 +1,309 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1603152000000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use db") + + def run(self): + tdSql.prepare() + + print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY") + tdSql.execute("create table t1 (ts timestamp, a int)") + + for i in range(5): + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 0) + + print("==============step 2: UPDATE THE WHOLE LAST BLOCK") + tdSql.execute("create table t2 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + for i in range(50): + tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i)) + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + print("==============step 3: UPDATE PART OF THE LAST BLOCK") + tdSql.execute("create table t3 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + for i in range(25): + tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA") + tdSql.execute("create table t4 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 50) + + for i in range(25): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + for i in range(50, 60): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t4") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 70) + + print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA") + tdSql.execute("create table t5 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 50) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + for i in range(25): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + for i in range(25, 50): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + + print("==============step 6: INSERT AHEAD A LOT OF DATA") + tdSql.execute("create table t6 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 0): + tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) + + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) + + print("==============step 7: INSERT AHEAD A LOT AND UPDATE") + tdSql.execute("create table t7 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 25): + tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2050) + + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2050) + + print("==============step 8: INSERT AFTER A LOT AND UPDATE") + tdSql.execute("create table t8 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 50) + + for i in range(25, 6000): + tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11950) + + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11950) + + print("==============step 9: UPDATE ONLY MIDDLE") + tdSql.execute("create table t9 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + for i in range(20, 30): + tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK") + tdSql.execute("create table t10 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 50) + + for i in range(-4000, 4000): + tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 15950) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 15950) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From f605be1018cdd3d3e961afdb80be270f363933c8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 Nov 2020 07:31:21 +0000 Subject: [PATCH 037/122] fix TD-1931 --- src/tsdb/src/tsdbMemTable.c | 1 + src/tsdb/src/tsdbRWHelper.c | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 19f59f3080..d68a97eb13 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -322,6 +322,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey memset(pMergeInfo, 0, sizeof(*pMergeInfo)); pMergeInfo->keyFirst = INT64_MAX; pMergeInfo->keyLast = INT64_MIN; + if (pCols) tdResetDataCols(pCols); row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index e02a7bbfe5..6cbd119961 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1495,7 +1495,6 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pIdx->len > 0); SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); - tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols, NULL, 0, pCfg->update, pMergeInfo); @@ -1525,7 +1524,6 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } } else { ASSERT(!pHelper->hasOldLastBlock); - tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); @@ -1571,7 +1569,6 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { while (true) { - tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); From 7f88bc2c8e0c345b14a794bc639a37b69a443d5b Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 4 Nov 2020 15:34:31 +0800 Subject: [PATCH 038/122] add updatetest.sh file --- tests/pytest/updatetest.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/pytest/updatetest.sh diff --git a/tests/pytest/updatetest.sh b/tests/pytest/updatetest.sh new file mode 100644 index 0000000000..ade1180553 --- /dev/null +++ b/tests/pytest/updatetest.sh @@ -0,0 +1,10 @@ +# update +python3 ./test.py -f update/allow_update.py +python3 ./test.py -f update/allow_update-0.py +python3 ./test.py -f update/append_commit_data.py +python3 ./test.py -f update/append_commit_last-0.py +python3 ./test.py -f update/append_commit_last.py +python3 ./test.py -f update/merge_commit_data.py +python3 ./test.py -f update/merge_commit_data2.py +python3 ./test.py -f update/merge_commit_last-0.py +python3 ./test.py -f update/merge_commit_last.py \ No newline at end of file From 13dda6b431d4bb5f509dcc2566fe881484bd0fb4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 Nov 2020 09:17:27 +0000 Subject: [PATCH 039/122] TD-1932 --- src/mnode/src/mnodeDb.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 812b1c609f..fb4793c056 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -969,8 +969,13 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { } if (update >= 0 && update != pDb->cfg.update) { +#if 0 mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update); newCfg.update = update; +#else + mError("db:%s, can't alter update option", pDb->name); + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; +#endif } return newCfg; From b636d216b9dbed55649b0d078888e8c31b394a2a Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Wed, 4 Nov 2020 18:07:43 +0800 Subject: [PATCH 040/122] add testcase for update --- tests/pytest/query/kill_query.py | 70 +++ tests/pytest/update/merge_commit_data2.py | 467 +++++++++++++----- .../update/merge_commit_data2_update0.py | 384 ++++++++++++++ 3 files changed, 787 insertions(+), 134 deletions(-) create mode 100644 tests/pytest/query/kill_query.py create mode 100644 tests/pytest/update/merge_commit_data2_update0.py diff --git a/tests/pytest/query/kill_query.py b/tests/pytest/query/kill_query.py new file mode 100644 index 0000000000..211e49c9ee --- /dev/null +++ b/tests/pytest/query/kill_query.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + + +class TDTestCase: + """ + kill query + """ + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create database if not exists demo;"); + tdSql.execute("use demo;") + tdSql.execute("CREATE TABLE IF NOT EXISTS demo1 (ts TIMESTAMP, f1 int);") + ts = 1300000000000 + + for i in range(1000): + tdSql.execute(" insert into demo1 values({ts}, {data});".format(ts=ts, data=i)) + ts += 1 + + print("==============insert into test1 and test2 form test file") + + + + print("==============step2") + while True: + tdSql.query('select * from demo1;') + print('==============', len(tdSql.queryResult)) + with open(ordered_csv) as f1: + num1 = len(f1.readlines()) + tdSql.checkRows(num1) + + + tdSql.query('select * from test2;') + with open(disordered_csv) as f2: + num2 = len(f2.readlines()) + tdSql.checkRows(num2) + print("=============execute select count(*) from xxx") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index e9dd084eb0..3f0fc718ad 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -1,153 +1,352 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -$t0 = 1603152000000 +# -*- coding: utf-8 -*- -create database db update 1 days 30; +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time -## Step 1. UPDATE TWO WHOLE DATA BLOCK REPEATEDLY -create table t1 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 1); -} -restart to commit -check query result +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def restart_taosd(self,db): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use %s;" % db) -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + 5000 + $i, 1); -} + def date_to_timestamp_microseconds(self, date): + datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f") + obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0) + return obj_stamp -check query result -restart to commit -check query result + def timestamp_microseconds_to_date(self, timestamp): + d = datetime.datetime.fromtimestamp(timestamp/1000) + str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f") + return str1 -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 2); -} -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + 5000 + $i, 1); -} -check query result -restart to commit -check query result -## Step 2. INSERT IN MIDDLE LITTLE DATA REPEATEDLY -create table t2 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t2 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t2 values ($t0 + 5000 + $i, 1); -} -restart to commit -for ($k = 0; $k < 10; $k++) { - for ($i = 0; $i < 10; $i++) { - insert into t2 values ($t0 + 200 + $k * 10 + $i, 1); - } - check query result - restart to commit - check query result -} + def run(self): + print("==========step1") + print("create table && insert data") + sql = 'reset query cache' + tdSql.execute(sql) + sql = 'drop database if exists db' + tdSql.execute(sql) + sql = 'create database db update 1 days 30;' + tdSql.execute(sql) + sql = 'use db;' + tdSql.execute(sql) + tdSql.execute('create table t1 (ts timestamp, a int)') -## Step 3. INSERT IN MIDDLE AND UPDATE TWO BLOCKS -create table t3 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t3 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t3 values ($t0 + 5000 + $i, 1); -} -restart to commit -for ($i = 0; $i < 5200; $i++) { - insert into t3 values ($t0 + $i, 2); -} -check query result -restart to commit -check query result + print("==================================1 start") + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i)) + print("==========step2") + print("restart to commit ") + self.restart_taosd('db') -## Step 4. INSERT IN MIDDLE AND UPDATE FIRST HALF OF TWO BLOCKS -create table t4 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t4 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t4 values ($t0 + 5000 + $i, 1); -} -restart to commit + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) -for ($i = 0; $i < 100; $i++) { - insert into t4 values ($t0 + $i, 2); -} -for ($i = 200; $i < 5000; $i++) { - insert into t4 values ($t0 + $i, 2); -} -for ($i = 0; $i < 100; $i++) { - insert into t4 values ($t0 + 5000 + $i, 2); -} -check query result -restart to commit -check query result + print("==========step3") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + print('check query result before restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) -## Step 5. INSERT IN MIDDLE AND UPDATE last HALF OF TWO BLOCKS -create table t5 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t5 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t5 values ($t0 + 5000 + $i, 1); -} -restart to commit + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) -for ($i = 100; $i < 200; $i++) { - insert into t5 values ($t0 + $i, 2); -} -for ($i = 200; $i < 5000; $i++) { - insert into t5 values ($t0 + $i, 2); -} -for ($i = 100; $i < 200; $i++) { - insert into t5 values ($t0 + 5000 + $i, 2); -} -check query result -restart to commit -check query result + print("==========step4") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 2)' %(t0+i)) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) -## Step 6. INSERT MASSIVE DATA AND UPDATE ALL BLOCKS -create table t6 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t6 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t6 values ($t0 + 5000 + $i, 1); -} -restart to commit + print('check query result before restart') + tdSql.query('select * from db.t1;') + print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 2) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) -for ($i = -1000; $i < 10000; $i++) { - insert into t6 values ($t0 + $i, 2); -} -check query result -restart to commit -check query result + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t1;') + # print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 2) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) -## Step 7. SPLIT DATA BLOCK -create table t7 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t7 values ($t0 + $i, 1); -} -for ($i = 0; $i < 200; $i++) { - insert into t7 values ($t0 + 5000 + $i, 1); -} -restart to commit + print("==================================2 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t2 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') -for ($i = -1000; $i < 10000; $i++) { - insert into t7 values ($t0 + $i, 2); -} -check query result -restart to commit -check query result \ No newline at end of file + for k in range(10): + for i in range(10): + tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + + + print("==================================3 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t3 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(5200): + tdSql.execute('insert into t3 values (%d , 2)' %(t0+i)) + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t3;') + for i in range(5200): + tdSql.checkData(i, 1, 2) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t3;') + for i in range(5200): + tdSql.checkData(i, 1, 2) + + print("==================================4 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t4 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t4;') + for i in range(100): + tdSql.checkData(i, 1, 2) + for i in range(100, 200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t4;') + for i in range(100): + tdSql.checkData(i, 1, 2) + for i in range(100, 200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================5 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t5 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t5;') + for i in range(100): + tdSql.checkData(i, 1, 1) + for i in range(100, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5100): + tdSql.checkData(i, 1, 1) + for i in range(5100, 5200): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t5;') + for i in range(100): + tdSql.checkData(i, 1, 1) + for i in range(100, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5100): + tdSql.checkData(i, 1, 1) + for i in range(5100, 5200): + tdSql.checkData(i, 1, 2) + + print("==================================6 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t6 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t6 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + + print("==================================7 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t7 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i)) + + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t7 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data2_update0.py b/tests/pytest/update/merge_commit_data2_update0.py new file mode 100644 index 0000000000..def50e0466 --- /dev/null +++ b/tests/pytest/update/merge_commit_data2_update0.py @@ -0,0 +1,384 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def restart_taosd(self,db): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use %s;" % db) + + def date_to_timestamp_microseconds(self, date): + datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f") + obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0) + return obj_stamp + + def timestamp_microseconds_to_date(self, timestamp): + d = datetime.datetime.fromtimestamp(timestamp/1000) + str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f") + return str1 + + + + def run(self): + print("==========step1") + print("create table && insert data") + sql = 'reset query cache' + tdSql.execute(sql) + sql = 'drop database if exists db' + tdSql.execute(sql) + sql = 'create database db update 0 days 30;' + tdSql.execute(sql) + sql = 'use db;' + tdSql.execute(sql) + tdSql.execute('create table t1 (ts timestamp, a int)') + + + print("==================================1 start") + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i)) + print("==========step2") + print("restart to commit ") + self.restart_taosd('db') + + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + + print("==========step3") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + print('check query result before restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print("==========step4") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 2)' %(t0+i)) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t1;') + print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t1;') + # print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print("==================================2 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t2 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for k in range(10): + for i in range(10): + tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + # print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + + + print("==================================3 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t3 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(5200): + tdSql.execute('insert into t3 values (%d , 2)' %(t0+i)) + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t3;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t3;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================4 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t4 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t4;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t4;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + # + print("==================================5 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t5 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t5;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t5;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================6 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t6 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t6 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + + print("==================================7 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t7 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i)) + + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t7 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 1b4674bb063c6944e44d85507db4032a4faa7dc5 Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Wed, 4 Nov 2020 18:40:19 +0800 Subject: [PATCH 041/122] remove kill query.py --- tests/pytest/query/kill_query.py | 70 -------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 tests/pytest/query/kill_query.py diff --git a/tests/pytest/query/kill_query.py b/tests/pytest/query/kill_query.py deleted file mode 100644 index 211e49c9ee..0000000000 --- a/tests/pytest/query/kill_query.py +++ /dev/null @@ -1,70 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import taos -from util.log import tdLog -from util.cases import tdCases -from util.sql import tdSql -from util.dnodes import tdDnodes - - -class TDTestCase: - """ - kill query - """ - - def init(self, conn, logSql): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) - - def run(self): - tdSql.prepare() - - print("==============step1") - tdSql.execute("create database if not exists demo;"); - tdSql.execute("use demo;") - tdSql.execute("CREATE TABLE IF NOT EXISTS demo1 (ts TIMESTAMP, f1 int);") - ts = 1300000000000 - - for i in range(1000): - tdSql.execute(" insert into demo1 values({ts}, {data});".format(ts=ts, data=i)) - ts += 1 - - print("==============insert into test1 and test2 form test file") - - - - print("==============step2") - while True: - tdSql.query('select * from demo1;') - print('==============', len(tdSql.queryResult)) - with open(ordered_csv) as f1: - num1 = len(f1.readlines()) - tdSql.checkRows(num1) - - - tdSql.query('select * from test2;') - with open(disordered_csv) as f2: - num2 = len(f2.readlines()) - tdSql.checkRows(num2) - print("=============execute select count(*) from xxx") - - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) From 6f0f199a758a40fef8f5f5909d78a90bb96cb452 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 4 Nov 2020 18:58:57 +0800 Subject: [PATCH 042/122] add test case for update --- tests/pytest/update/append_commit_data-0.py | 84 +++++ tests/pytest/update/merge_commit_data-0.py | 351 ++++++++++++++++++++ 2 files changed, 435 insertions(+) create mode 100644 tests/pytest/update/append_commit_data-0.py create mode 100644 tests/pytest/update/merge_commit_data-0.py diff --git a/tests/pytest/update/append_commit_data-0.py b/tests/pytest/update/append_commit_data-0.py new file mode 100644 index 0000000000..b844a50a08 --- /dev/null +++ b/tests/pytest/update/append_commit_data-0.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + print("==========step1") + print("create table && insert data") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') + + insertRows = 200 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0+i)) + print("==========step2") + print("restart to commit ") + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows) + for k in range(0,100): + tdLog.info("insert %d rows" % (insertRows)) + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into db.t1 values(%d,1)' % + (t0+k*200+i) + ) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows+200*k) + print("==========step2") + print("insert into another table ") + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 20000 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d, 1)' % + (t0+i)) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data-0.py b/tests/pytest/update/merge_commit_data-0.py new file mode 100644 index 0000000000..14d435f7f2 --- /dev/null +++ b/tests/pytest/update/merge_commit_data-0.py @@ -0,0 +1,351 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + print("==========step1") + print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db days 30' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') + + insertRows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0 + i)) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + + for k in range(0,10): + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into t1 values(%d,1)' % + (t0+i) + ) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + print("==========step2") + print("PREPEND DATA ") + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + for i in range(-100,0): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + print("==========step3") + print("PREPEND MASSIVE DATA ") + ret = tdSql.execute('create table t3 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + for i in range(-6000,0): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + print("==========step4") + print("APPEND DATA") + ret = tdSql.execute('create table t4 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + print("==========step5") + print("APPEND DATA") + ret = tdSql.execute('create table t5 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + for i in range(0,6000): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + print("==========step6") + print("UPDATE BLOCK IN TWO STEP") + ret = tdSql.execute('create table t6 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t6 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + for i in range(0,200): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + print("==========step7") + print("UPDATE LAST HALF AND INSERT LITTLE DATA") + ret = tdSql.execute('create table t7 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t7 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + for i in range(100,300): + ret = tdSql.execute( + 'insert into t7 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t7") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'400') + print("==========step8") + print("UPDATE LAST HALF AND INSERT MASSIVE DATA") + ret = tdSql.execute('create table t8 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t8 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + for i in range(6000): + ret = tdSql.execute( + 'insert into t8 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'11800') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'11800') + print("==========step9") + print("UPDATE FIRST HALF AND PREPEND LITTLE DATA") + ret = tdSql.execute('create table t9 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t9 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + for i in range(-100,100): + ret = tdSql.execute( + 'insert into t9 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'400') + print("==========step10") + print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA") + ret = tdSql.execute('create table t10 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t10 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + for i in range(-6000,100): + ret = tdSql.execute( + 'insert into t10 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12200') + print("==========step11") + print("UPDATE FIRST HALF AND APPEND MASSIVE DATA") + ret = tdSql.execute('create table t11 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t11 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + for i in range(100): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + for i in range(200,6000): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11800') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11800') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From b43c5ba92630ccfdc1930b8d0fbeb29886fe4d01 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 4 Nov 2020 21:15:47 +0000 Subject: [PATCH 043/122] Refactoring of crash_gen tool --- .../{crash_gen.py => crash_gen_main.py} | 81 ++++++++++++------ tests/pytest/crash_gen/db.py | 6 ++ tests/pytest/crash_gen/misc.py | 8 +- tests/pytest/crash_gen/service_manager.py | 85 +++++++++++++------ tests/pytest/crash_gen_bootstrap.py | 2 +- 5 files changed, 129 insertions(+), 53 deletions(-) rename tests/pytest/crash_gen/{crash_gen.py => crash_gen_main.py} (97%) diff --git a/tests/pytest/crash_gen/crash_gen.py b/tests/pytest/crash_gen/crash_gen_main.py similarity index 97% rename from tests/pytest/crash_gen/crash_gen.py rename to tests/pytest/crash_gen/crash_gen_main.py index 9bc701aa8b..fee51b98c0 100755 --- a/tests/pytest/crash_gen/crash_gen.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -38,9 +38,9 @@ import resource from guppy import hpy import gc -from .service_manager import ServiceManager, TdeInstance -from .misc import Logging, Status, CrashGenError, Dice, Helper, Progress -from .db import DbConn, MyTDSql, DbConnNative, DbManager +from crash_gen.service_manager import ServiceManager, TdeInstance +from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress +from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager import taos import requests @@ -435,7 +435,7 @@ class ThreadCoordinator: Logging.debug("\r\n\n--> Main thread ready to finish up...") Logging.debug("Main thread joining all threads") self._pool.joinAll() # Get all threads to finish - Logging.info("\nAll worker threads finished") + Logging.info(". . . All worker threads finished") # No CR/LF before self._execStats.endExec() def cleanup(self): # free resources @@ -1072,17 +1072,18 @@ class Database: t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above - Logging.info("Setting up TICKS to start from: {}".format(t4)) + Logging.debug("Setting up TICKS to start from: {}".format(t4)) return t4 @classmethod def getNextTick(cls): with cls._clsLock: # prevent duplicate tick - if cls._lastLaggingTick==0: + if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized # 10k at 1/20 chance, should be enough to avoid overlaps - cls._lastLaggingTick = cls.setupLastTick() + datetime.timedelta(0, -10000) - if cls._lastTick==0: # should be quite a bit into the future - cls._lastTick = cls.setupLastTick() + tick = cls.setupLastTick() + cls._lastTick = tick + cls._lastLaggingTick = tick + datetime.timedelta(0, -10000) + # if : # should be quite a bit into the future if Dice.throw(20) == 0: # 1 in 20 chance, return lagging tick cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds @@ -1322,7 +1323,7 @@ class Task(): self._err = err self._aborted = True except Exception as e: - self.logInfo("Non-TAOS exception encountered") + Logging.info("Non-TAOS exception encountered with: {}".format(self.__class__.__name__)) self._err = e self._aborted = True traceback.print_exc() @@ -1566,8 +1567,11 @@ class TaskCreateSuperTable(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place + sTable.create(wt.getDbConn(), self._db.getName(), - {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'}) + {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'}, + dropIfExists = True + ) # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) # No need to create the regular tables, INSERT will do that # automatically @@ -1580,14 +1584,41 @@ class TdSuperTable: def getName(self): return self._stName + def drop(self, dbc, dbName, skipCheck = False): + if self.exists(dbc, dbName) : # if myself exists + fullTableName = dbName + '.' + self._stName + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: + if not skipCheck: + raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) + + def exists(self, dbc, dbName): + dbc.execute("USE " + dbName) + return dbc.existsSuperTable(self._stName) + # TODO: odd semantic, create() method is usually static? - def create(self, dbc, dbName, cols: dict, tags: dict): + def create(self, dbc, dbName, cols: dict, tags: dict, + dropIfExists = False + ): + '''Creating a super table''' - sql = "CREATE TABLE {}.{} ({}) TAGS ({})".format( - dbName, - self._stName, - ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]), - ",".join(['%s %s'%(k,v) for (k,v) in tags.items()]) + dbc.execute("USE " + dbName) + fullTableName = dbName + '.' + self._stName + if dbc.existsSuperTable(self._stName): + if dropIfExists: + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: # error + raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) + + # Now let's create + sql = "CREATE TABLE {} ({})".format( + fullTableName, + ",".join(['%s %s'%(k,v) for (k,v) in cols.items()])) + if tags is None : + sql += " TAGS (dummy int) " + else: + sql += " TAGS ({})".format( + ",".join(['%s %s'%(k,v) for (k,v) in tags.items()]) ) dbc.execute(sql) @@ -1611,17 +1642,19 @@ class TdSuperTable: return # acquire a lock first, so as to be able to *verify*. More details in TD-1471 - fullTableName = dbName + '.' + regTableName - task.lockTable(fullTableName) + fullTableName = dbName + '.' + regTableName + if task is not None: # optional lock + task.lockTable(fullTableName) Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table - print("(" + fullTableName[-3:] + ")", end="", flush=True) + # print("(" + fullTableName[-3:] + ")", end="", flush=True) try: sql = "CREATE TABLE {} USING {}.{} tags ({})".format( fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) ) dbc.execute(sql) finally: - task.unlockTable(fullTableName) # no matter what + if task is not None: + task.unlockTable(fullTableName) # no matter what def _getTagStrForSql(self, dbc, dbName: str) : tags = self._getTags(dbc, dbName) @@ -1840,7 +1873,7 @@ class TaskRestartService(StateTransitionTask): with self._classLock: if self._isRunning: - print("Skipping restart task, another running already") + Logging.info("Skipping restart task, another running already") return self._isRunning = True @@ -1999,7 +2032,7 @@ class ThreadStacks: # stack info for all threads class ClientManager: def __init__(self): - print("Starting service manager") + Logging.info("Starting service manager") # signal.signal(signal.SIGTERM, self.sigIntHandler) # signal.signal(signal.SIGINT, self.sigIntHandler) @@ -2101,7 +2134,7 @@ class ClientManager: thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps) self.tc = ThreadCoordinator(thPool, dbManager) - print("Starting client instance to: {}".format(tInst)) + Logging.info("Starting client instance: {}".format(tInst)) self.tc.run() # print("exec stats: {}".format(self.tc.getExecStats())) # print("TC failed = {}".format(self.tc.isFailed())) diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index 43c855647c..2a4b362f82 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -95,6 +95,11 @@ class DbConn: # print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName))) return dbName in dbs # TODO: super weird type mangling seen, once here + def existsSuperTable(self, stName): + self.query("show stables") + sts = [v[0] for v in self.getQueryResult()] + return stName in sts + def hasTables(self): return self.query("show tables") > 0 @@ -240,6 +245,7 @@ class MyTDSql: def _execInternal(self, sql): startTime = time.time() + # Logging.debug("Executing SQL: " + sql) ret = self._cursor.execute(sql) # print("\nSQL success: {}".format(sql)) queryTime = time.time() - startTime diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index 4dc053dae3..2d2ce99d95 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -27,7 +27,7 @@ class LoggingFilter(logging.Filter): class MyLoggingAdapter(logging.LoggerAdapter): def process(self, msg, kwargs): - return "[{}] {}".format(threading.get_ident() % 10000, msg), kwargs + return "[{:04d}] {}".format(threading.get_ident() % 10000, msg), kwargs # return '[%s] %s' % (self.extra['connid'], msg), kwargs @@ -51,7 +51,7 @@ class Logging: _logger.addHandler(ch) # Logging adapter, to be used as a logger - print("setting logger variable") + # print("setting logger variable") # global logger cls.logger = MyLoggingAdapter(_logger, []) @@ -166,7 +166,8 @@ class Progress: SERVICE_RECONNECT_START = 4 SERVICE_RECONNECT_SUCCESS = 5 SERVICE_RECONNECT_FAILURE = 6 - CREATE_TABLE_ATTEMPT = 7 + SERVICE_START_NAP = 7 + CREATE_TABLE_ATTEMPT = 8 tokens = { STEP_BOUNDARY: '.', @@ -176,6 +177,7 @@ class Progress: SERVICE_RECONNECT_START: '', SERVICE_RECONNECT_FAILURE: '.xr>', + SERVICE_START_NAP: '_zz', CREATE_TABLE_ATTEMPT: '_c', } diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index 196e9d944a..d249abc439 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -47,6 +47,17 @@ class TdeInstance(): .format(selfPath, projPath)) return buildPath + @classmethod + def prepareGcovEnv(cls, env): + # Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html + bPath = cls._getBuildPath() # build PATH + numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3 + numSegments = numSegments - 1 # DEBUG only + env['GCOV_PREFIX'] = bPath + '/svc_gcov' + env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings + Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format( + numSegments, env['GCOV_PREFIX'] )) + def __init__(self, subdir='test', tInstNum=0, port=6030, fepPort=6030): self._buildDir = self._getBuildPath() self._subdir = '/' + subdir # TODO: tolerate "/" @@ -217,6 +228,11 @@ class TdeSubProcess: # raise CrashGenError("Empty instance not allowed in TdeSubProcess") # self._tInst = tInst # Default create at ServiceManagerThread + def __repr__(self): + if self.subProcess is None: + return '[TdeSubProc: Empty]' + return '[TdeSubProc: pid = {}]'.format(self.getPid()) + def getStdOut(self): return self.subProcess.stdout @@ -235,17 +251,30 @@ class TdeSubProcess: # Sanity check if self.subProcess: # already there raise RuntimeError("Corrupt process state") - + + # Prepare environment variables for coverage information + # Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment + myEnv = os.environ.copy() + TdeInstance.prepareGcovEnv(myEnv) + + # print(myEnv) + # print(myEnv.items()) + # print("Starting TDengine via Shell: {}".format(cmdLineStr)) + + useShell = True self.subProcess = subprocess.Popen( - cmdLine, - shell=False, + ' '.join(cmdLine) if useShell else cmdLine, + shell=useShell, # svcCmdSingle, shell=True, # capture core dump? stdout=subprocess.PIPE, stderr=subprocess.PIPE, # bufsize=1, # not supported in binary mode - close_fds=ON_POSIX + close_fds=ON_POSIX, + env=myEnv ) # had text=True, which interferred with reading EOF + STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process? + def stop(self): """ Stop a sub process, and try to return a meaningful return code. @@ -267,7 +296,7 @@ class TdeSubProcess: SIGUSR2 12 """ if not self.subProcess: - print("Sub process already stopped") + Logging.error("Sub process already stopped") return # -1 retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N) @@ -278,20 +307,25 @@ class TdeSubProcess: return retCode # process still alive, let's interrupt it - print("Terminate running process, send SIG_INT and wait...") - # sub process should end, then IPC queue should end, causing IO thread to end - # sig = signal.SIGINT - sig = signal.SIGKILL - self.subProcess.send_signal(sig) # SIGNINT or SIGKILL + Logging.info("Terminate running process, send SIG_{} and wait...".format(self.STOP_SIGNAL)) + # sub process should end, then IPC queue should end, causing IO thread to end + topSubProc = psutil.Process(self.subProcess.pid) + for child in topSubProc.children(recursive=True): # or parent.children() for recursive=False + child.send_signal(self.STOP_SIGNAL) + time.sleep(0.2) # 200 ms + # topSubProc.send_signal(sig) # now kill the main sub process (likely the Shell) + + self.subProcess.send_signal(self.STOP_SIGNAL) # main sub process (likely the Shell) self.subProcess.wait(20) retCode = self.subProcess.returncode # should always be there # May throw subprocess.TimeoutExpired exception above, therefore # The process is guranteed to have ended by now self.subProcess = None if retCode != 0: # != (- signal.SIGINT): - Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(sig, retCode)) + Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format( + self.STOP_SIGNAL, retCode)) else: - Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(sig)) + Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(self.STOP_SIGNAL)) return - retCode class ServiceManager: @@ -439,7 +473,7 @@ class ServiceManager: time.sleep(self.PAUSE_BETWEEN_IPC_CHECK) # pause, before next round # raise CrashGenError("dummy") - print("Service Manager Thread (with subprocess) ended, main thread exiting...") + Logging.info("Service Manager Thread (with subprocess) ended, main thread exiting...") def _getFirstInstance(self): return self._tInsts[0] @@ -452,7 +486,7 @@ class ServiceManager: # Find if there's already a taosd service, and then kill it for proc in psutil.process_iter(): if proc.name() == 'taosd': - print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt") + Logging.info("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt") time.sleep(2.0) proc.kill() # print("Process: {}".format(proc.name())) @@ -559,7 +593,8 @@ class ServiceManagerThread: for i in range(0, 100): time.sleep(1.0) # self.procIpcBatch() # don't pump message during start up - print("_zz_", end="", flush=True) + Progress.emit(Progress.SERVICE_START_NAP) + # print("_zz_", end="", flush=True) if self._status.isRunning(): Logging.info("[] TDengine service READY to process requests") Logging.info("[] TAOS service started: {}".format(self)) @@ -595,12 +630,12 @@ class ServiceManagerThread: def stop(self): # can be called from both main thread or signal handler - print("Terminating TDengine service running as the sub process...") + Logging.info("Terminating TDengine service running as the sub process...") if self.getStatus().isStopped(): - print("Service already stopped") + Logging.info("Service already stopped") return if self.getStatus().isStopping(): - print("Service is already being stopped") + Logging.info("Service is already being stopped") return # Linux will send Control-C generated SIGINT to the TDengine process # already, ref: @@ -616,10 +651,10 @@ class ServiceManagerThread: if retCode == signal.SIGSEGV : # SGV Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)") except subprocess.TimeoutExpired as err: - print("Time out waiting for TDengine service process to exit") + Logging.info("Time out waiting for TDengine service process to exit") else: if self._tdeSubProcess.isRunning(): # still running, should now never happen - print("FAILED to stop sub process, it is still running... pid = {}".format( + Logging.error("FAILED to stop sub process, it is still running... pid = {}".format( self._tdeSubProcess.getPid())) else: self._tdeSubProcess = None # not running any more @@ -683,9 +718,9 @@ class ServiceManagerThread: return # we are done with THIS BATCH else: # got line, printing out if forceOutput: - Logging.info(line) + Logging.info('[TAOSD] ' + line) else: - Logging.debug(line) + Logging.debug('[TAOSD] ' + line) print(">", end="", flush=True) _ProgressBars = ["--", "//", "||", "\\\\"] @@ -728,11 +763,11 @@ class ServiceManagerThread: # queue.put(line) # meaning sub process must have died - Logging.info("\nEnd of stream detected for TDengine STDOUT: {}".format(self)) + Logging.info("EOF for TDengine STDOUT: {}".format(self)) out.close() def svcErrorReader(self, err: IO, queue): for line in iter(err.readline, b''): - print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line)) - Logging.info("\nEnd of stream detected for TDengine STDERR: {}".format(self)) + Logging.info("TDengine STDERR: {}".format(line)) + Logging.info("EOF for TDengine STDERR: {}".format(self)) err.close() \ No newline at end of file diff --git a/tests/pytest/crash_gen_bootstrap.py b/tests/pytest/crash_gen_bootstrap.py index a3417d21a8..fd12284b9d 100644 --- a/tests/pytest/crash_gen_bootstrap.py +++ b/tests/pytest/crash_gen_bootstrap.py @@ -11,7 +11,7 @@ ################################################################### import sys -from crash_gen.crash_gen import MainExec +from crash_gen.crash_gen_main import MainExec if __name__ == "__main__": From faa8ff7cc24740859f231d1a7ba29f594d7abc14 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 5 Nov 2020 03:43:26 +0000 Subject: [PATCH 044/122] [TD-1937]:client crash on percentile --- src/client/src/tscLocalMerge.c | 3 +++ src/query/src/qHistogram.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index c67edf5b5a..f49c4da0cb 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -97,6 +97,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->param[2].i64Key = pQueryInfo->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[1].i64Key = pQueryInfo->order.orderColId; + } else if (functionId == TSDB_FUNC_APERCT) { + pCtx->param[0].i64Key = pExpr->param[0].i64Key; + pCtx->param[0].nType = pExpr->param[0].nType; } pCtx->interBufBytes = pExpr->interBytes; diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 703ee2c521..35e5906d1f 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { (*pHisto)->numOfEntries += 1; } } else { /* insert a new slot */ - if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) { + if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { if (idx > 0) { assert((*pHisto)->elems[idx - 1].val <= val); } @@ -661,4 +661,4 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 free(pHistoBins); return pResHistogram; -} \ No newline at end of file +} From 5b735ad09170dea7fce07a0f897f859fe0867746 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 5 Nov 2020 07:28:20 +0000 Subject: [PATCH 045/122] second version --- src/rpc/src/rpcMain.c | 15 +-- src/util/inc/tref.h | 5 +- src/util/src/tref.c | 197 +++++++++++++++++++++++--------------- src/util/tests/trefTest.c | 117 ++++++++++++---------- 4 files changed, 195 insertions(+), 139 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index d01c34c810..05330ebff8 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -82,6 +82,7 @@ typedef struct { int8_t oldInUse; // server EP inUse passed by app int8_t redirect; // flag to indicate redirect int8_t connType; // connection type + int32_t rid; // refId returned by taosAddRef SRpcMsg *pRsp; // for synchronous API tsem_t *pSem; // for synchronous API SRpcEpSet *pSet; // for synchronous API @@ -374,7 +375,7 @@ void *rpcReallocCont(void *ptr, int contLen) { return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } -void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { +int64_t rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { SRpcInfo *pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; @@ -403,10 +404,11 @@ void rpcSendRequest(void *shandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg) { // set the handle to pContext, so app can cancel the request if (pMsg->handle) *((void **)pMsg->handle) = pContext; - taosAddRef(tsRpcRefId, pContext); + pContext->rid = taosAddRef(tsRpcRefId, pContext); + rpcSendReqToServer(pRpc, pContext); - return; + return pContext->rid; } void rpcSendResponse(const SRpcMsg *pRsp) { @@ -551,11 +553,10 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) { return code; } -void rpcCancelRequest(void *handle) { - SRpcReqContext *pContext = handle; +void rpcCancelRequest(int64_t rid) { - int code = taosAcquireRef(tsRpcRefId, pContext); - if (code < 0) return; + SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid); + if (pContext == NULL) return; rpcCloseConn(pContext->pConn); diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h index 3e5db33cf7..cd5092f30a 100644 --- a/src/util/inc/tref.h +++ b/src/util/inc/tref.h @@ -32,7 +32,10 @@ int taosCloseRef(int refId); // add ref, p is the pointer to resource or pointer ID // return Reference ID(rid) allocated. On error, -1 is returned, and terrno is set appropriately int64_t taosAddRef(int refId, void *p); -#define taosRemoveRef taosReleaseRef + +// remove ref, rid is the reference ID returned by taosAddRef +// return 0 if success. On error, -1 is returned, and terrno is set appropriately +int taosRemoveRef(int rsetId, int64_t rid); // acquire ref, rid is the reference ID returned by taosAddRef // return the resource p. On error, NULL is returned, and terrno is set appropriately diff --git a/src/util/src/tref.c b/src/util/src/tref.c index c2adb6d15b..915ed53193 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -29,6 +29,7 @@ typedef struct SRefNode { void *p; // pointer to resource protected, int64_t rid; // reference ID int32_t count; // number of references + int removed; // 1: removed } SRefNode; typedef struct { @@ -51,8 +52,9 @@ static int tsNextId = 0; static void taosInitRefModule(void); static void taosLockList(int64_t *lockedBy); static void taosUnlockList(int64_t *lockedBy); -static void taosIncRefCount(SRefSet *pSet); -static void taosDecRefCount(SRefSet *pSet); +static void taosIncRsetCount(SRefSet *pSet); +static void taosDecRsetCount(SRefSet *pSet); +static int taosDecRefCount(int rsetId, int64_t rid, int remove); int taosOpenRef(int max, void (*fp)(void *)) { @@ -86,7 +88,7 @@ int taosOpenRef(int max, void (*fp)(void *)) if (i < TSDB_REF_OBJECTS) { rsetId = tsNextId; pSet = tsRefSetList + rsetId; - taosIncRefCount(pSet); + taosIncRsetCount(pSet); pSet->max = max; pSet->nodeList = nodeList; pSet->lockedBy = lockedBy; @@ -134,7 +136,7 @@ int taosCloseRef(int rsetId) pthread_mutex_unlock(&tsRefMutex); - if (deleted) taosDecRefCount(pSet); + if (deleted) taosDecRsetCount(pSet); return 0; } @@ -153,9 +155,9 @@ int64_t taosAddRef(int rsetId, void *p) } pSet = tsRefSetList + rsetId; - taosIncRefCount(pSet); + taosIncRsetCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { - taosDecRefCount(pSet); + taosDecRsetCount(pSet); uTrace("rsetId:%d p:%p failed to add, not active", rsetId, p); terrno = TSDB_CODE_REF_ID_REMOVED; return -1; @@ -187,6 +189,12 @@ int64_t taosAddRef(int rsetId, void *p) return rid; } +int taosRemoveRef(int rsetId, int64_t rid) +{ + return taosDecRefCount(rsetId, rid, 1); +} + +// if rid is 0, return the first p in hash list, otherwise, return the next after current rid void *taosAcquireRef(int rsetId, int64_t rid) { int hash; @@ -200,11 +208,17 @@ void *taosAcquireRef(int rsetId, int64_t rid) return NULL; } + if (rid <= 0) { + uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, rid not valid", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + return NULL; + } + pSet = tsRefSetList + rsetId; - taosIncRefCount(pSet); + taosIncRsetCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { uTrace("rsetId:%d rid:%" PRId64 " failed to acquire, not active", rsetId, rid); - taosDecRefCount(pSet); + taosDecRsetCount(pSet); terrno = TSDB_CODE_REF_ID_REMOVED; return NULL; } @@ -223,9 +237,14 @@ void *taosAcquireRef(int rsetId, int64_t rid) } if (pNode) { - pNode->count++; - p = pNode->p; - uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid); + if (pNode->removed == 0) { + pNode->count++; + p = pNode->p; + uTrace("rsetId:%d p:%p rid:%" PRId64 " is acquired", rsetId, pNode->p, rid); + } else { + terrno = TSDB_CODE_REF_NOT_EXIST; + uTrace("rsetId:%d p:%p rid:%" PRId64 " is already removed, failed to acquire", rsetId, pNode->p, rid); + } } else { terrno = TSDB_CODE_REF_NOT_EXIST; uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to acquire", rsetId, rid); @@ -233,75 +252,14 @@ void *taosAcquireRef(int rsetId, int64_t rid) taosUnlockList(pSet->lockedBy+hash); - taosDecRefCount(pSet); + taosDecRsetCount(pSet); return p; } int taosReleaseRef(int rsetId, int64_t rid) { - int hash; - SRefNode *pNode; - SRefSet *pSet; - int released = 0; - - if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { - uTrace("rsetId:%d rid:%" PRId64 " failed to release, rsetId not valid", rsetId, rid); - terrno = TSDB_CODE_REF_INVALID_ID; - return -1; - } - - pSet = tsRefSetList + rsetId; - if (pSet->state == TSDB_REF_STATE_EMPTY) { - uTrace("rsetId:%d rid:%" PRId64 " failed to release, cleaned", rsetId, rid); - terrno = TSDB_CODE_REF_ID_REMOVED; - return -1; - } - - terrno = 0; - hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); - - pNode = pSet->nodeList[hash]; - while (pNode) { - if (pNode->rid == rid) - break; - - pNode = pNode->next; - } - - if (pNode) { - pNode->count--; - - if (pNode->count == 0) { - if ( pNode->prev ) { - pNode->prev->next = pNode->next; - } else { - pSet->nodeList[hash] = pNode->next; - } - - if ( pNode->next ) { - pNode->next->prev = pNode->prev; - } - - (*pSet->fp)(pNode->p); - - uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); - free(pNode); - released = 1; - } else { - uTrace("rsetId:%d p:%p rid:%" PRId64 "is released", rsetId, pNode->p, rid); - } - } else { - uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release", rsetId, rid); - terrno = TSDB_CODE_REF_NOT_EXIST; - } - - taosUnlockList(pSet->lockedBy+hash); - - if (released) taosDecRefCount(pSet); - - return terrno; + return taosDecRefCount(rsetId, rid, 0); } // if rid is 0, return the first p in hash list, otherwise, return the next after current rid @@ -315,12 +273,18 @@ void *taosIterateRef(int rsetId, int64_t rid) { return NULL; } + if (rid <= 0) { + uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rid not valid", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + return NULL; + } + pSet = tsRefSetList + rsetId; - taosIncRefCount(pSet); + taosIncRsetCount(pSet); if (pSet->state != TSDB_REF_STATE_ACTIVE) { uTrace("rsetId:%d rid:%" PRId64 " failed to iterate, rset not active", rsetId, rid); terrno = TSDB_CODE_REF_ID_REMOVED; - taosDecRefCount(pSet); + taosDecRsetCount(pSet); return NULL; } @@ -371,7 +335,7 @@ void *taosIterateRef(int rsetId, int64_t rid) { if (rid > 0) taosReleaseRef(rsetId, rid); // release the current one - taosDecRefCount(pSet); + taosDecRsetCount(pSet); return newP; } @@ -407,6 +371,81 @@ int taosListRef() { return num; } +static int taosDecRefCount(int rsetId, int64_t rid, int remove) { + int hash; + SRefSet *pSet; + SRefNode *pNode; + int released = 0; + int code = 0; + + if (rsetId < 0 || rsetId >= TSDB_REF_OBJECTS) { + uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rsetId not valid", rsetId, rid); + terrno = TSDB_CODE_REF_INVALID_ID; + return -1; + } + + if (rid <= 0) { + uTrace("rsetId:%d rid:%" PRId64 " failed to remove, rid not valid", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + return -1; + } + + pSet = tsRefSetList + rsetId; + if (pSet->state == TSDB_REF_STATE_EMPTY) { + uTrace("rsetId:%d rid:%" PRId64 " failed to remove, cleaned", rsetId, rid); + terrno = TSDB_CODE_REF_ID_REMOVED; + return -1; + } + + terrno = 0; + hash = rid % pSet->max; + + taosLockList(pSet->lockedBy+hash); + + pNode = pSet->nodeList[hash]; + while (pNode) { + if (pNode->rid == rid) + break; + + pNode = pNode->next; + } + + if (pNode) { + pNode->count--; + if (remove) pNode->removed = 1; + + if (pNode->count <= 0) { + if (pNode->prev) { + pNode->prev->next = pNode->next; + } else { + pSet->nodeList[hash] = pNode->next; + } + + if (pNode->next) { + pNode->next->prev = pNode->prev; + } + + (*pSet->fp)(pNode->p); + + uTrace("rsetId:%d p:%p rid:%" PRId64 "is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); + free(pNode); + released = 1; + } else { + uTrace("rsetId:%d p:%p rid:%" PRId64 "is released, count:%d", rsetId, pNode->p, rid, pNode->count); + } + } else { + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); + terrno = TSDB_CODE_REF_NOT_EXIST; + code = -1; + } + + taosUnlockList(pSet->lockedBy+hash); + + if (released) taosDecRsetCount(pSet); + + return code; +} + static void taosLockList(int64_t *lockedBy) { int64_t tid = taosGetPthreadId(); int i = 0; @@ -428,12 +467,12 @@ static void taosInitRefModule(void) { pthread_mutex_init(&tsRefMutex, NULL); } -static void taosIncRefCount(SRefSet *pSet) { +static void taosIncRsetCount(SRefSet *pSet) { atomic_add_fetch_32(&pSet->count, 1); uTrace("rsetId:%d inc count:%d", pSet->rsetId, pSet->count); } -static void taosDecRefCount(SRefSet *pSet) { +static void taosDecRsetCount(SRefSet *pSet) { int32_t count = atomic_sub_fetch_32(&pSet->count, 1); uTrace("rsetId:%d dec count:%d", pSet->rsetId, pSet->count); diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c index 09ffccd7b5..6887b24abd 100644 --- a/src/util/tests/trefTest.c +++ b/src/util/tests/trefTest.c @@ -11,106 +11,119 @@ #include "tulog.h" typedef struct { - int refNum; - int steps; - int refId; - void **p; + int refNum; + int steps; + int rsetId; + int64_t rid; + void **p; } SRefSpace; -void iterateRefs(int refId) { +void iterateRefs(int rsetId) { int count = 0; - void *p = taosIterateRef(refId, NULL); + void *p = taosIterateRef(rsetId, NULL); while (p) { // process P count++; - p = taosIterateRef(refId, p); + p = taosIterateRef(rsetId, p); } printf(" %d ", count); } -void *takeRefActions(void *param) { +void *addRef(void *param) { SRefSpace *pSpace = (SRefSpace *)param; - int code, id; + int id; + int64_t rid; for (int i=0; i < pSpace->steps; ++i) { - printf("s"); + printf("a"); id = random() % pSpace->refNum; - code = taosAddRef(pSpace->refId, pSpace->p[id]); - usleep(1); - - id = random() % pSpace->refNum; - code = taosAcquireRef(pSpace->refId, pSpace->p[id]); - if (code >= 0) { - usleep(id % 5 + 1); - taosReleaseRef(pSpace->refId, pSpace->p[id]); + if (pSpace->rid[id] <= 0) { + pSpace->p[id] = malloc(128); + pSpace->rid[id] = taosAddRef(pSpace->rsetId, pSpace->p[id]); } - - id = random() % pSpace->refNum; - taosRemoveRef(pSpace->refId, pSpace->p[id]); - usleep(id %5 + 1); - - id = random() % pSpace->refNum; - code = taosAcquireRef(pSpace->refId, pSpace->p[id]); - if (code >= 0) { - usleep(id % 5 + 1); - taosReleaseRef(pSpace->refId, pSpace->p[id]); - } - - id = random() % pSpace->refNum; - iterateRefs(id); + usleep(100); } - for (int i=0; i < pSpace->refNum; ++i) { - taosRemoveRef(pSpace->refId, pSpace->p[i]); - } - - //uInfo("refId:%d thread exits", pSpace->refId); + return NULL; +} + +void *removeRef(void *param) { + SRefSpace *pSpace = (SRefSpace *)param; + int id; + int64_t rid; + + for (int i=0; i < pSpace->steps; ++i) { + printf("d"); + id = random() % pSpace->refNum; + if (pSpace->rid[id] > 0) { + code = taosRemoveRef(pSpace->rsetId, pSpace->rid[id]); + if (code == 0) pSpace->rid[id] = 0; + } + + usleep(100); + } + + return NULL; +} + +void *acquireRelease(void *param) { + SRefSpace *pSpace = (SRefSpace *)param; + int id; + int64_t rid; + + for (int i=0; i < pSpace->steps; ++i) { + printf("a"); + + id = random() % pSpace->refNum; + code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); + if (code >= 0) { + usleep(id % 5 + 1); + taosReleaseRef(pSpace->rsetId, pSpace->p[id]); + } + } return NULL; } void myfree(void *p) { - return; + free(p); } void *openRefSpace(void *param) { SRefSpace *pSpace = (SRefSpace *)param; printf("c"); - pSpace->refId = taosOpenRef(50, myfree); + pSpace->rsetId = taosOpenRef(50, myfree); - if (pSpace->refId < 0) { - printf("failed to open ref, reson:%s\n", tstrerror(pSpace->refId)); + if (pSpace->rsetId < 0) { + printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId)); return NULL; } pSpace->p = (void **) calloc(sizeof(void *), pSpace->refNum); - for (int i=0; irefNum; ++i) { - pSpace->p[i] = (void *) malloc(128); - } pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); pthread_t thread1, thread2, thread3; - pthread_create(&(thread1), &thattr, takeRefActions, (void *)(pSpace)); - pthread_create(&(thread2), &thattr, takeRefActions, (void *)(pSpace)); - pthread_create(&(thread3), &thattr, takeRefActions, (void *)(pSpace)); + pthread_create(&(thread1), &thattr, addRef, (void *)(pSpace)); + pthread_create(&(thread2), &thattr, removeRef, (void *)(pSpace)); + pthread_create(&(thread3), &thattr, acquireRelease, (void *)(pSpace)); pthread_join(thread1, NULL); pthread_join(thread2, NULL); pthread_join(thread3, NULL); - taosCloseRef(pSpace->refId); - for (int i=0; irefNum; ++i) { - free(pSpace->p[i]); + taosRemoveRef(pSpace->rsetId, pSpace->rid[i]); } - uInfo("refId:%d main thread exit", pSpace->refId); + taosCloseRef(pSpace->rsetId); + + uInfo("rsetId:%d main thread exit", pSpace->rsetId); free(pSpace->p); pSpace->p = NULL; @@ -140,7 +153,7 @@ int main(int argc, char *argv[]) { printf("\nusage: %s [options] \n", argv[0]); printf(" [-n]: number of references, default: %d\n", refNum); printf(" [-s]: steps to run for each reference, default: %d\n", steps); - printf(" [-t]: number of refIds running in parallel, default: %d\n", threads); + printf(" [-t]: number of rsetIds running in parallel, default: %d\n", threads); printf(" [-l]: number of loops, default: %d\n", loops); printf(" [-d]: debugFlag, default: %d\n", uDebugFlag); exit(0); From fdaf1e470ce98293ceec682b6c251343066c1003 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 17:18:01 +0800 Subject: [PATCH 046/122] [TD-1954]: a inport/export tool for logical backup --- packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 2 ++ packaging/tools/makeclient.sh | 2 +- packaging/tools/makepkg.sh | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index a7bb22f345..edc7de9692 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 54c3c9b279..afec1eaf9a 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include @@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 69fa53c087..83a9cb1ced 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index a6d868ed1d..00a92cb063 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" From a9593a44727b775bcb9f50a9bde188659a6610d2 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 5 Nov 2020 18:07:26 +0800 Subject: [PATCH 047/122] [TD-1939] add test case for TD-1939 --- tests/pytest/query/queryLike.py | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/pytest/query/queryLike.py diff --git a/tests/pytest/query/queryLike.py b/tests/pytest/query/queryLike.py new file mode 100644 index 0000000000..aa943c4f1f --- /dev/null +++ b/tests/pytest/query/queryLike.py @@ -0,0 +1,45 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.execute("create table cars(ts timestamp, c nchar(2)) tags(t1 nchar(2))") + tdSql.execute("insert into car0 using cars tags('aa') values(now, 'bb');") + tdSql.query("select count(*) from cars where t1 like '%50 90 30 04 00 00%'") + tdSql.checkRows(0) + + tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))") + tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');") + tdSql.query("select count(*) from test_cars where t1 like '%50 90 30 04 00 00%'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 5ba5336756c6130ee9df1494b648517e96dd90fb Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 5 Nov 2020 18:38:23 +0800 Subject: [PATCH 048/122] update Jenkinksfile --- Jenkinsfile | 1 + tests/pytest/query/queryLike.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8bf7e435fd..834321240f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { cd debug cmake .. > /dev/null make > /dev/null + make install > /dev/null cd ${WKC}/tests #./test-all.sh smoke ./test-all.sh pytest diff --git a/tests/pytest/query/queryLike.py b/tests/pytest/query/queryLike.py index aa943c4f1f..3c3b030f8f 100644 --- a/tests/pytest/query/queryLike.py +++ b/tests/pytest/query/queryLike.py @@ -33,7 +33,7 @@ class TDTestCase: tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))") tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');") - tdSql.query("select count(*) from test_cars where t1 like '%50 90 30 04 00 00%'") + tdSql.query("select * from test_cars where t1 like '%50 90 30 04 00 00%'") tdSql.checkRows(1) def stop(self): From 6ca2968862967cf55357e604cb7295c8d2809d48 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 5 Nov 2020 10:58:28 +0000 Subject: [PATCH 049/122] TD-1944 --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index d01c34c810..d89d3e275d 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1622,7 +1622,7 @@ static void rpcDecRef(SRpcInfo *pRpc) int count = atomic_sub_fetch_32(&tsRpcNum, 1); if (count == 0) { - taosCloseRef(tsRpcRefId); + // taosCloseRef(tsRpcRefId); // tsRpcInit = PTHREAD_ONCE_INIT; // windows compliling error } } From ea374b6f868ef2cfe6efa51238bafadb03042359 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 11:34:40 +0000 Subject: [PATCH 050/122] TD-1915 --- src/dnode/src/dnodeVRead.c | 48 +++++++------------ src/inc/vnode.h | 17 ++++--- src/vnode/src/vnodeMain.c | 15 ------ src/vnode/src/vnodeRead.c | 97 +++++++++++++++++++++++++------------- src/vnode/src/vnodeWrite.c | 2 +- 5 files changed, 94 insertions(+), 85 deletions(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index f571cfda9f..34df11adcc 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -92,33 +92,23 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { pHead->vgId = htonl(pHead->vgId); pHead->contLen = htonl(pHead->contLen); - taos_queue queue = vnodeAcquireRqueue(pHead->vgId); - - if (queue == NULL) { - leftLen -= pHead->contLen; - pCont -= pHead->contLen; - continue; + void *pVnode = vnodeAcquire(pHead->vgId); + if (pVnode != NULL) { + int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; + vnodeRelease(pVnode); } - // put message into queue - SVReadMsg *pRead = taosAllocateQitem(sizeof(SVReadMsg)); - pRead->rpcMsg = *pMsg; - pRead->pCont = pCont; - pRead->contLen = pHead->contLen; - - // next vnode leftLen -= pHead->contLen; pCont -= pHead->contLen; - queuedMsgNum++; - - taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); } if (queuedMsgNum == 0) { SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); } + + rpcFreeCont(pMsg->pCont); } void *dnodeAllocVReadQueue(void *pVnode) { @@ -162,50 +152,48 @@ void dnodeFreeVReadQueue(void *rqueue) { void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { SRpcMsg rpcRsp = { - .handle = pRead->rpcMsg.handle, + .handle = pRead->rpcHandle, .pCont = pRead->rspRet.rsp, .contLen = pRead->rspRet.len, .code = code, }; rpcSendResponse(&rpcRsp); - rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); } void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) { - rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); } static void *dnodeProcessReadQueue(void *param) { - SVReadMsg *pReadMsg; + SVReadMsg *pRead; int32_t qtype; void * pVnode; while (1) { - if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pReadMsg, &pVnode) == 0) { + if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) { dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset); break; } - dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle, - taosMsg[pReadMsg->rpcMsg.msgType], qtype, pReadMsg); + dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead, + taosMsg[pRead->msgType], qtype); - int32_t code = vnodeProcessRead(pVnode, pReadMsg); + int32_t code = vnodeProcessRead(pVnode, pRead); if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { - dnodeSendRpcVReadRsp(pVnode, pReadMsg, code); + dnodeSendRpcVReadRsp(pVnode, pRead, code); } else { if (code == TSDB_CODE_QRY_HAS_RSP) { - dnodeSendRpcVReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); + dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code); } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client - assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5)); - dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); + assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5)); + dnodeDispatchNonRspMsg(pVnode, pRead, code); } } - taosFreeQitem(pReadMsg); + taosFreeQitem(pRead); } return NULL; diff --git a/src/inc/vnode.h b/src/inc/vnode.h index e77bec926a..018e96e193 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -37,10 +37,15 @@ typedef struct { } SRspRet; typedef struct { - SRspRet rspRet; - void * pCont; + int32_t code; int32_t contLen; - SRpcMsg rpcMsg; + void * rpcHandle; + void * rpcAhandle; + void * qhandle; + int8_t qtype; + int8_t msgType; + SRspRet rspRet; + char pCont[]; } SVReadMsg; typedef struct { @@ -62,13 +67,11 @@ int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); int32_t vnodeClose(int32_t vgId); void* vnodeAcquire(int32_t vgId); // add refcount -void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam); int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam); -int32_t vnodeCheckWrite(void *pVnode); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); void vnodeBuildStatusMsg(void *param); void vnodeConfirmForward(void *param, uint64_t version, int32_t code); @@ -77,8 +80,8 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); int32_t vnodeInitResources(); void vnodeCleanupResources(); -int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pReadMsg); -int32_t vnodeCheckRead(void *pVnode); +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam); +int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); #ifdef __cplusplus } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 2dcdba5d7c..128da72623 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -468,21 +468,6 @@ void *vnodeAcquire(int32_t vgId) { return *ppVnode; } -void *vnodeAcquireRqueue(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) return NULL; - - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]); - vnodeRelease(pVnode); - return NULL; - } - - return pVnode->rqueue; -} - void *vnodeGetWal(void *pVnode) { return ((SVnodeObj *)pVnode)->wal; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index faa35b0e02..fb984f6750 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -46,7 +46,7 @@ void vnodeInitReadFp(void) { // int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) { SVnodeObj *pVnode = (SVnodeObj *)param; - int msgType = pReadMsg->rpcMsg.msgType; + int32_t msgType = pReadMsg->msgType; if (vnodeProcessReadMsgFp[msgType] == NULL) { vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]); @@ -56,7 +56,7 @@ int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -int32_t vnodeCheckRead(void *param) { +static int32_t vnodeCheckRead(void *param) { SVnodeObj *pVnode = param; if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], @@ -78,24 +78,58 @@ int32_t vnodeCheckRead(void *param) { return TSDB_CODE_SUCCESS; } -static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; - SVReadMsg *pRead = (SVReadMsg *)taosAllocateQitem(sizeof(SVReadMsg)); - pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; - pRead->pCont = qhandle; - pRead->contLen = 0; - pRead->rpcMsg.ahandle = ahandle; +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + + if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) { + int32_t code = vnodeCheckRead(pVnode); + if (code != TSDB_CODE_SUCCESS) return code; + } + + int32_t size = sizeof(SVReadMsg) + contLen; + SVReadMsg *pRead = taosAllocateQitem(size); + if (pRead == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + if (rparam != NULL) { + SRpcMsg *pRpcMsg = rparam; + pRead->rpcHandle = pRpcMsg->handle; + pRead->rpcAhandle = pRpcMsg->ahandle; + pRead->msgType = pRpcMsg->msgType; + pRead->code = pRpcMsg->code; + } + + if (contLen != 0) { + pRead->contLen = contLen; + memcpy(pRead->pCont, pCont, contLen); + } else { + pRead->qhandle = pCont; + } + + pRead->qtype = qtype; atomic_add_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, get vnode rqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead); - taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); - + taosWriteQitem(pVnode->rqueue, qtype, pRead); return TSDB_CODE_SUCCESS; } +static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { + SRpcMsg rpcMsg = {0}; + rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; + rpcMsg.ahandle = ahandle; + + int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg); + if (code == TSDB_CODE_SUCCESS) { + vDebug("QInfo:%p add to vread queue for exec query", *qhandle); + } + + return code; +} + /** * * @param pRet response message object @@ -155,18 +189,18 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { memset(pRet, 0, sizeof(SRspRet)); // qHandle needs to be freed correctly - if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (pReadMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont; killQueryMsg->free = htons(killQueryMsg->free); killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle); - vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); - assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1); + vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcHandle); + assert(pReadMsg->contLen > 0 && killQueryMsg->free == 1); void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle); if (qhandle == NULL || *qhandle == NULL) { vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); } else { assert(*qhandle == (void *)killQueryMsg->qhandle); @@ -208,9 +242,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } if (handle != NULL && - vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); return pRsp->code; @@ -221,7 +255,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); - code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle); + code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcHandle); if (code != TSDB_CODE_SUCCESS) { pRsp->code = code; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -230,7 +264,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } } else { assert(pCont != NULL); - void **qhandle = (void **)pCont; + void **qhandle = (void **)pReadMsg->qhandle; vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); @@ -242,14 +276,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { // build query rsp, the retrieve request has reached here already if (buildRes) { // update the connection info according to the retrieve connection - pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle); - assert(pReadMsg->rpcMsg.handle != NULL); + pReadMsg->rpcHandle = qGetResultRetrieveMsg(*qhandle); + assert(pReadMsg->rpcHandle != NULL); vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); // set the real rsp error code - pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle); + pReadMsg->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcHandle); // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; @@ -283,7 +317,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { pRetrieve->qhandle = htobe64(pRetrieve->qhandle); vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, - pRetrieve->free, pReadMsg->rpcMsg.handle); + pRetrieve->free, pReadMsg->rpcHandle); memset(pRet, 0, sizeof(SRspRet)); @@ -314,9 +348,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } // register the qhandle to connect to quit query immediate if connection is broken - if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcMsg.handle); + if (vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcHandle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -326,7 +359,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { bool freeHandle = true; bool buildRes = false; - code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle); + code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcHandle); if (code != TSDB_CODE_SUCCESS) { // TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -337,7 +370,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { assert(buildRes == true); #if _NON_BLOCKING_RETRIEVE if (!buildRes) { - assert(pReadMsg->rpcMsg.handle != NULL); + assert(pReadMsg->rpcHandle != NULL); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); return TSDB_CODE_QRY_NOT_READY; @@ -345,7 +378,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { #endif // ahandle is the sqlObj pointer - code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle); + code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcHandle); } // If qhandle is not added into vread queue, the query should be completed already or paused with error. diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 80c68b0917..3caee2fb0c 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -97,7 +97,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara return syncCode; } -int32_t vnodeCheckWrite(void *param) { +static int32_t vnodeCheckWrite(void *param) { SVnodeObj *pVnode = param; if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); From acc5c53b126e079f25613ab15c3c1354ce031343 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 5 Nov 2020 13:15:23 +0000 Subject: [PATCH 051/122] make changes on app --- src/client/inc/tsclient.h | 3 +- src/client/src/tscServer.c | 24 ++++----- src/client/src/tscSql.c | 24 ++++----- src/client/src/tscUtil.c | 2 +- src/inc/trpc.h | 4 +- src/inc/tsync.h | 14 ++--- src/mnode/src/mnodeSdb.c | 6 +-- src/rpc/src/rpcMain.c | 6 +-- src/sync/inc/syncInt.h | 1 + src/sync/src/syncMain.c | 104 +++++++++++++++++-------------------- src/sync/test/syncServer.c | 2 +- src/vnode/inc/vnodeInt.h | 2 +- src/vnode/src/vnodeMain.c | 20 +++---- src/wal/inc/walInt.h | 1 + src/wal/src/walMgmt.c | 9 ++-- 15 files changed, 106 insertions(+), 116 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index 1c2821638b..4c85af4919 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -330,6 +330,7 @@ typedef struct STscObj { char writeAuth : 1; char superAuth : 1; uint32_t connId; + uint64_t rid; // ref ID returned by taosAddRef struct SSqlObj * pHb; struct SSqlObj * sqlList; struct SSqlStream *streamList; @@ -348,7 +349,7 @@ typedef struct SSqlObj { void *signature; pthread_t owner; // owner of sql object, by which it is executed STscObj *pTscObj; - void *pRpcCtx; + int64_t rpcRid; void (*fp)(); void (*fetchFp)(); void *param; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 1584afe706..a3e2eba168 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -182,27 +182,23 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { int32_t waitingDuring = tsShellActivityTimer * 500; tscDebug("%p send heartbeat in %dms", pSql, waitingDuring); - taosTmrReset(tscProcessActivityTimer, waitingDuring, pObj, tscTmr, &pObj->pTimer); + taosTmrReset(tscProcessActivityTimer, waitingDuring, (void *)pObj->rid, tscTmr, &pObj->pTimer); } else { tscDebug("%p start to close tscObj:%p, not send heartbeat again", pSql, pObj); } } void tscProcessActivityTimer(void *handle, void *tmrId) { - STscObj *pObj = (STscObj *)handle; - - int ret = taosAcquireRef(tscRefId, pObj); - if (ret < 0) { - tscTrace("%p failed to acquire TSC obj, reason:%s", pObj, tstrerror(ret)); - return; - } + int64_t rid = (int64_t) handle; + STscObj *pObj = taosAcquireRef(tscRefId, rid); + if (pObj == NULL) return; SSqlObj* pHB = pObj->pHb; void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE)); if (p == NULL) { tscWarn("%p HB object has been released already", pHB); - taosReleaseRef(tscRefId, pObj); + taosReleaseRef(tscRefId, pObj->rid); return; } @@ -216,7 +212,7 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code)); } - taosReleaseRef(tscRefId, pObj); + taosReleaseRef(tscRefId, rid); } int tscSendMsgToServer(SSqlObj *pSql) { @@ -241,7 +237,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .pCont = pMsg, .contLen = pSql->cmd.payloadLen, .ahandle = pSql, - .handle = &pSql->pRpcCtx, + .handle = NULL, .code = 0 }; @@ -249,7 +245,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); + pSql->rpcRid = rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); return TSDB_CODE_SUCCESS; } @@ -269,7 +265,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { SSqlCmd *pCmd = &pSql->cmd; assert(*pSql->self == pSql); - pSql->pRpcCtx = NULL; + pSql->rpcRid = -1; if (pObj->signature != pObj) { tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); @@ -2026,7 +2022,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { createHBObj(pObj); //launch a timer to send heartbeat to maintain the connection and send status to mnode - taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, pObj, tscTmr, &pObj->pTimer); + taosTmrReset(tscProcessActivityTimer, tsShellActivityTimer * 500, (void *)pObj->rid, tscTmr, &pObj->pTimer); return 0; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 89dfa24e8f..bdc46c5446 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -161,7 +161,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa registerSqlObj(pSql); tsInsertHeadSize = sizeof(SMsgDesc) + sizeof(SSubmitMsg); - taosAddRef(tscRefId, pObj); + pObj->rid = taosAddRef(tscRefId, pObj); return pSql; } @@ -279,9 +279,9 @@ void taos_close(TAOS *taos) { SSqlObj* pHb = pObj->pHb; if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) { - if (pHb->pRpcCtx != NULL) { // wait for rsp from dnode - rpcCancelRequest(pHb->pRpcCtx); - pHb->pRpcCtx = NULL; + if (pHb->rpcRid > 0) { // wait for rsp from dnode + rpcCancelRequest(pHb->rpcRid); + pHb->rpcRid = -1; } tscDebug("%p HB is freed", pHb); @@ -298,7 +298,7 @@ void taos_close(TAOS *taos) { tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn); - taosRemoveRef(tscRefId, pObj); + taosRemoveRef(tscRefId, pObj->rid); } void waitForQueryRsp(void *param, TAOS_RES *tres, int code) { @@ -748,9 +748,9 @@ static void tscKillSTableQuery(SSqlObj *pSql) { assert(pSubObj->self == (SSqlObj**) p); pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - if (pSubObj->pRpcCtx != NULL) { - rpcCancelRequest(pSubObj->pRpcCtx); - pSubObj->pRpcCtx = NULL; + if (pSubObj->rpcRid > 0) { + rpcCancelRequest(pSubObj->rpcRid); + pSubObj->rpcRid = -1; } tscQueueAsyncRes(pSubObj); @@ -775,7 +775,7 @@ void taos_stop_query(TAOS_RES *res) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - assert(pSql->pRpcCtx == NULL); + assert(pSql->rpcRid <= 0); tscKillSTableQuery(pSql); } else { if (pSql->cmd.command < TSDB_SQL_LOCAL) { @@ -784,9 +784,9 @@ void taos_stop_query(TAOS_RES *res) { * reset and freed in the processMsgFromServer function, and causes the invalid * write problem for rpcCancelRequest. */ - if (pSql->pRpcCtx != NULL) { - rpcCancelRequest(pSql->pRpcCtx); - pSql->pRpcCtx = NULL; + if (pSql->rpcRid > 0) { + rpcCancelRequest(pSql->rpcRid); + pSql->rpcRid = -1; } tscQueueAsyncRes(pSql); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 879eeeaded..2aee90653d 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -376,7 +376,7 @@ void tscFreeRegisteredSqlObj(void *pSql) { tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref); if (ref == 0) { tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj); - taosRemoveRef(tscRefId, pTscObj); + taosRemoveRef(tscRefId, pTscObj->rid); } } diff --git a/src/inc/trpc.h b/src/inc/trpc.h index bdee917b5e..e430a43807 100644 --- a/src/inc/trpc.h +++ b/src/inc/trpc.h @@ -83,13 +83,13 @@ void rpcClose(void *); void *rpcMallocCont(int contLen); void rpcFreeCont(void *pCont); void *rpcReallocCont(void *ptr, int contLen); -void rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg); +int64_t rpcSendRequest(void *thandle, const SRpcEpSet *pEpSet, SRpcMsg *pMsg); void rpcSendResponse(const SRpcMsg *pMsg); void rpcSendRedirectRsp(void *pConn, const SRpcEpSet *pEpSet); int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo); void rpcSendRecv(void *shandle, SRpcEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); int rpcReportProgress(void *pConn, char *pCont, int contLen); -void rpcCancelRequest(void *pContext); +void rpcCancelRequest(int64_t rid); #ifdef __cplusplus } diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 671adefab8..1b16fef84c 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -106,13 +106,13 @@ typedef void* tsync_h; int32_t syncInit(); void syncCleanUp(); -tsync_h syncStart(const SSyncInfo *); -void syncStop(tsync_h shandle); -int32_t syncReconfig(tsync_h shandle, const SSyncCfg *); -int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype); -void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code); -void syncRecover(tsync_h shandle); // recover from other nodes: -int syncGetNodesRole(tsync_h shandle, SNodesRole *); +int64_t syncStart(const SSyncInfo *); +void syncStop(int64_t rid); +int32_t syncReconfig(int64_t rid, const SSyncCfg *); +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype); +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code); +void syncRecover(int64_t rid); // recover from other nodes: +int syncGetNodesRole(int64_t rid, SNodesRole *); extern char *syncRole[]; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 8fb0b33060..f6acb6826c 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -72,7 +72,7 @@ typedef struct { ESyncRole role; ESdbStatus status; int64_t version; - void * sync; + int64_t sync; void * wal; SSyncCfg cfg; int32_t numOfTables; @@ -212,7 +212,7 @@ static void sdbRestoreTables() { } void sdbUpdateMnodeRoles() { - if (tsSdbObj.sync == NULL) return; + if (tsSdbObj.sync <= 0) return; SNodesRole roles = {0}; syncGetNodesRole(tsSdbObj.sync, &roles); @@ -433,7 +433,7 @@ void sdbCleanUp() { if (tsSdbObj.sync) { syncStop(tsSdbObj.sync); - tsSdbObj.sync = NULL; + tsSdbObj.sync = -1; } if (tsSdbObj.wal) { diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 05330ebff8..dc67f6a80f 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -560,7 +560,7 @@ void rpcCancelRequest(int64_t rid) { rpcCloseConn(pContext->pConn); - taosReleaseRef(tsRpcRefId, pContext); + taosReleaseRef(tsRpcRefId, rid); } static void rpcFreeMsg(void *msg) { @@ -629,7 +629,7 @@ static void rpcReleaseConn(SRpcConn *pConn) { // if there is an outgoing message, free it if (pConn->outType && pConn->pReqMsg) { if (pConn->pContext) pConn->pContext->pConn = NULL; - taosRemoveRef(tsRpcRefId, pConn->pContext); + taosRemoveRef(tsRpcRefId, pConn->pContext->rid); } } @@ -1110,7 +1110,7 @@ static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { } // free the request message - taosRemoveRef(tsRpcRefId, pContext); + taosRemoveRef(tsRpcRefId, pContext->rid); } static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) { diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index f681810646..8808a82c46 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -141,6 +141,7 @@ typedef struct SSyncNode { int8_t replica; int8_t quorum; uint32_t vgId; + int64_t rid; void *ahandle; int8_t selfIndex; SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 6f5e3be8ab..74e327183b 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -142,14 +142,14 @@ void syncCleanUp() { sInfo("sync module is cleaned up"); } -void *syncStart(const SSyncInfo *pInfo) { +int64_t syncStart(const SSyncInfo *pInfo) { const SSyncCfg *pCfg = &pInfo->syncCfg; SSyncNode *pNode = (SSyncNode *)calloc(sizeof(SSyncNode), 1); if (pNode == NULL) { sError("no memory to allocate syncNode"); terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; + return -1; } tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path)); @@ -170,10 +170,10 @@ void *syncStart(const SSyncInfo *pInfo) { pNode->quorum = pCfg->quorum; if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica; - int ret = taosAddRef(tsSyncRefId, pNode); - if (ret < 0) { + pNode->rid = taosAddRef(tsSyncRefId, pNode); + if (pNode->rid < 0) { syncFreeNode(pNode); - return NULL; + return -1; } for (int i = 0; i < pCfg->replica; ++i) { @@ -187,8 +187,8 @@ void *syncStart(const SSyncInfo *pInfo) { if (pNode->selfIndex < 0) { sInfo("vgId:%d, this node is not configured", pNode->vgId); terrno = TSDB_CODE_SYN_INVALID_CONFIG; - syncStop(pNode); - return NULL; + syncStop(pNode->rid); + return -1; } nodeVersion = pInfo->version; // set the initial version @@ -200,15 +200,15 @@ void *syncStart(const SSyncInfo *pInfo) { if (pNode->pSyncFwds == NULL) { sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId); terrno = TAOS_SYSTEM_ERROR(errno); - syncStop(pNode); - return NULL; + syncStop(pNode->rid); + return -1; } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl); if (pNode->pFwdTimer == NULL) { sError("vgId:%d, failed to allocate timer", pNode->vgId); - syncStop(pNode); - return NULL; + syncStop(pNode->rid); + return -1; } syncAddArbitrator(pNode); @@ -218,15 +218,14 @@ void *syncStart(const SSyncInfo *pInfo) { (*pNode->notifyRole)(pNode->ahandle, nodeRole); } - return pNode; + return pNode->rid; } -void syncStop(void *param) { - SSyncNode *pNode = param; +void syncStop(int64_t rid) { SSyncPeer *pPeer; - int ret = taosAcquireRef(tsSyncRefId, pNode); - if (ret < 0) return; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return; sInfo("vgId:%d, cleanup sync", pNode->vgId); @@ -245,16 +244,15 @@ void syncStop(void *param) { pthread_mutex_unlock(&(pNode->mutex)); - taosReleaseRef(tsSyncRefId, pNode); - taosRemoveRef(tsSyncRefId, pNode); + taosReleaseRef(tsSyncRefId, rid); + taosRemoveRef(tsSyncRefId, rid); } -int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { - SSyncNode *pNode = param; +int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { int i, j; - int ret = taosAcquireRef(tsSyncRefId, pNode); - if (ret < 0) return TSDB_CODE_SYN_INVALID_CONFIG; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, pNode->replica); @@ -318,29 +316,25 @@ int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) { syncRole[nodeRole]); syncBroadcastStatus(pNode); - taosReleaseRef(tsSyncRefId, pNode); + taosReleaseRef(tsSyncRefId, rid); return 0; } -int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype) { - SSyncNode *pNode = param; - - int ret = taosAcquireRef(tsSyncRefId, pNode); - if (ret < 0) return 0; +int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int qtype) { + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return 0; int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype); - taosReleaseRef(tsSyncRefId, pNode); + taosReleaseRef(tsSyncRefId, rid); return code; } -void syncConfirmForward(void *param, uint64_t version, int32_t code) { - SSyncNode *pNode = param; - - int ret = taosAcquireRef(tsSyncRefId, pNode); - if (ret < 0) return; +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return; SSyncPeer *pPeer = pNode->pMaster; if (pPeer && pNode->quorum > 1) { @@ -365,15 +359,14 @@ void syncConfirmForward(void *param, uint64_t version, int32_t code) { } } - taosReleaseRef(tsSyncRefId, pNode); + taosReleaseRef(tsSyncRefId, rid); } -void syncRecover(void *param) { - SSyncNode *pNode = param; +void syncRecover(int64_t rid) { SSyncPeer *pPeer; - int ret = taosAcquireRef(tsSyncRefId, pNode); - if (ret < 0) return; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return; // to do: add a few lines to check if recover is OK // if take this node to unsync state, the whole system may not work @@ -393,14 +386,12 @@ void syncRecover(void *param) { pthread_mutex_unlock(&(pNode->mutex)); - taosReleaseRef(tsSyncRefId, pNode); + taosReleaseRef(tsSyncRefId, rid); } -int syncGetNodesRole(void *param, SNodesRole *pNodesRole) { - SSyncNode *pNode = param; - - int ret = taosAcquireRef(tsSyncRefId, pNode); - if (ret < 0) return -1; +int syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return -1; pNodesRole->selfIndex = pNode->selfIndex; for (int i = 0; i < pNode->replica; ++i) { @@ -408,7 +399,7 @@ int syncGetNodesRole(void *param, SNodesRole *pNodesRole) { pNodesRole->role[i] = pNode->peerInfo[i]->role; } - taosReleaseRef(tsSyncRefId, pNode); + taosReleaseRef(tsSyncRefId, rid); return 0; } @@ -455,7 +446,7 @@ void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); int syncDecPeerRef(SSyncPeer *pPeer) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { - taosReleaseRef(tsSyncRefId, pPeer->pSyncNode); + taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); sDebug("%s, resource is freed", pPeer->id); taosTFree(pPeer->watchFd); @@ -512,7 +503,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer); } - taosAcquireRef(tsSyncRefId, pNode); + taosAcquireRef(tsSyncRefId, pNode->rid); return pPeer; } @@ -1105,7 +1096,7 @@ static void syncProcessBrokenLink(void *param) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - if (taosAcquireRef(tsSyncRefId, pNode) < 0) return; + if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return; pthread_mutex_lock(&(pNode->mutex)); sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno)); @@ -1116,7 +1107,7 @@ static void syncProcessBrokenLink(void *param) { } pthread_mutex_unlock(&(pNode->mutex)); - taosReleaseRef(tsSyncRefId, pNode); + taosReleaseRef(tsSyncRefId, pNode->rid); } static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { @@ -1184,10 +1175,9 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code } static void syncMonitorFwdInfos(void *param, void *tmrId) { - SSyncNode *pNode = param; - - int ret = taosAcquireRef(tsSyncRefId, pNode); - if ( ret < 0) return; + int64_t rid = (int64_t) param; + SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); + if (pNode == NULL) return; SSyncFwds *pSyncFwds = pNode->pSyncFwds; @@ -1206,10 +1196,10 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { pthread_mutex_unlock(&(pNode->mutex)); } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl); } - taosReleaseRef(tsSyncRefId, pNode); + taosReleaseRef(tsSyncRefId, rid); } static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) { diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 0cf752da97..9dd3feb461 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -30,7 +30,7 @@ int dataFd = -1; void * qhandle = NULL; int walNum = 0; uint64_t tversion = 0; -void * syncHandle; +int64_t syncHandle; int role; int nodeId; char path[256]; diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 169334c611..9a7722c57d 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -47,7 +47,7 @@ typedef struct { void *rqueue; void *wal; void *tsdb; - void *sync; + int64_t sync; void *events; void *cq; // continuous query int32_t cfgVersion; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index e206933116..ab1341a0da 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -44,12 +44,12 @@ static void vnodeCtrlFlow(void *handle, int32_t mseconds); static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); #ifndef _SYNC -tsync_h syncStart(const SSyncInfo *info) { return NULL; } -int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; } -void syncStop(tsync_h shandle) {} -int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; } -int syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; } -void syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {} +int64_t syncStart(const SSyncInfo *info) { return NULL; } +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; } +void syncStop(int64_t rid) {} +int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; } +int syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; } +void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {} #endif char* vnodeStatus[] = { @@ -330,7 +330,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { #ifndef _SYNC pVnode->role = TAOS_SYNC_ROLE_MASTER; #else - if (pVnode->sync == NULL) { + if (pVnode->sync <= 0) { vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); vnodeCleanUp(pVnode); @@ -589,9 +589,9 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { } // stop replication module - if (pVnode->sync) { - void *sync = pVnode->sync; - pVnode->sync = NULL; + if (pVnode->sync > 0) { + int64_t sync = pVnode->sync; + pVnode->sync = -1; syncStop(sync); } diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h index 5273eb5b1c..d1e9772259 100644 --- a/src/wal/inc/walInt.h +++ b/src/wal/inc/walInt.h @@ -43,6 +43,7 @@ extern int32_t wDebugFlag; typedef struct { uint64_t version; int64_t fileId; + int64_t rid; int32_t vgId; int32_t fd; int32_t keep; diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index c8f0274174..2c4349ed3a 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -78,7 +78,8 @@ void *walOpen(char *path, SWalCfg *pCfg) { return NULL; } - if (taosAddRef(tsWal.refId, pWal) != TSDB_CODE_SUCCESS) { + pWal->rid = taosAddRef(tsWal.refId, pWal); + if (pWal->rid < 0) { walFreeObj(pWal); return NULL; } @@ -143,7 +144,7 @@ void walClose(void *handle) { } pthread_mutex_unlock(&pWal->mutex); - taosRemoveRef(tsWal.refId, pWal); + taosRemoveRef(tsWal.refId, pWal->rid); } static int32_t walInitObj(SWal *pWal) { @@ -185,7 +186,7 @@ static void walUpdateSeq() { } static void walFsyncAll() { - SWal *pWal = taosIterateRef(tsWal.refId, NULL); + SWal *pWal = taosIterateRef(tsWal.refId, 0); while (pWal) { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq); @@ -194,7 +195,7 @@ static void walFsyncAll() { wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code)); } } - pWal = taosIterateRef(tsWal.refId, pWal); + pWal = taosIterateRef(tsWal.refId, pWal->rid); } } From cb0fcf10d5df213eae20da0d4e96f9e47cee0e1a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 5 Nov 2020 14:40:20 +0000 Subject: [PATCH 052/122] minor changes --- src/rpc/src/rpcMain.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index a7123f250c..a24894c2bf 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -221,8 +221,7 @@ static void rpcFree(void *p) { free(p); } -static void rpcInit(void) { - +void rpcInit(void) { tsProgressTimer = tsRpcTimer/2; tsRpcMaxRetry = tsRpcMaxTime * 1000/tsProgressTimer; tsRpcHeadSize = RPC_MSG_OVERHEAD; @@ -231,6 +230,11 @@ static void rpcInit(void) { tsRpcRefId = taosOpenRef(200, rpcFree); } +void rpcCleanup(void) { + taosCloseRef(tsRpcRefId); + tsRpcRefId = -1; +} + void *rpcOpen(const SRpcInit *pInit) { SRpcInfo *pRpc; @@ -1621,11 +1625,7 @@ static void rpcDecRef(SRpcInfo *pRpc) tDebug("%s rpc resources are released", pRpc->label); taosTFree(pRpc); - int count = atomic_sub_fetch_32(&tsRpcNum, 1); - if (count == 0) { - // taosCloseRef(tsRpcRefId); - // tsRpcInit = PTHREAD_ONCE_INIT; // windows compliling error - } + atomic_sub_fetch_32(&tsRpcNum, 1); } } From 9bf73b64c3d7641355532db32c31dc33d12d4e1f Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 5 Nov 2020 15:18:34 +0000 Subject: [PATCH 053/122] minor changes on debug info --- src/client/src/tscSystem.c | 2 +- src/util/src/tref.c | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 839d5889f3..01046a2840 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -36,7 +36,7 @@ void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; int tsInsertHeadSize; -int tscRefId; +int tscRefId = -1; int tscNumOfThreads; diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 915ed53193..99d566e2ab 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -82,13 +82,13 @@ int taosOpenRef(int max, void (*fp)(void *)) for (i = 0; i < TSDB_REF_OBJECTS; ++i) { tsNextId = (tsNextId + 1) % TSDB_REF_OBJECTS; + if (tsNextId == 0) tsNextId = 1; // dont use 0 as rsetId if (tsRefSetList[tsNextId].state == TSDB_REF_STATE_EMPTY) break; } if (i < TSDB_REF_OBJECTS) { rsetId = tsNextId; pSet = tsRefSetList + rsetId; - taosIncRsetCount(pSet); pSet->max = max; pSet->nodeList = nodeList; pSet->lockedBy = lockedBy; @@ -96,6 +96,7 @@ int taosOpenRef(int max, void (*fp)(void *)) pSet->rid = 1; pSet->rsetId = rsetId; pSet->state = TSDB_REF_STATE_ACTIVE; + taosIncRsetCount(pSet); tsRefSetNum++; uTrace("rsetId:%d is opened, max:%d, fp:%p refSetNum:%d", rsetId, max, fp, tsRefSetNum); @@ -469,12 +470,12 @@ static void taosInitRefModule(void) { static void taosIncRsetCount(SRefSet *pSet) { atomic_add_fetch_32(&pSet->count, 1); - uTrace("rsetId:%d inc count:%d", pSet->rsetId, pSet->count); + // uTrace("rsetId:%d inc count:%d", pSet->rsetId, count); } static void taosDecRsetCount(SRefSet *pSet) { int32_t count = atomic_sub_fetch_32(&pSet->count, 1); - uTrace("rsetId:%d dec count:%d", pSet->rsetId, pSet->count); + // uTrace("rsetId:%d dec count:%d", pSet->rsetId, count); if (count > 0) return; From 0d57114b776dd077999dbf10129f86ecd2ab6bb0 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 5 Nov 2020 22:23:14 +0000 Subject: [PATCH 054/122] Minor refactoring of crash_gen tool, now doing multi-record insertion some times --- tests/pytest/crash_gen/crash_gen_main.py | 167 ++++++++++++----------- 1 file changed, 91 insertions(+), 76 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index fee51b98c0..8a074500e1 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -388,9 +388,9 @@ class ThreadCoordinator: self._syncAtBarrier() # For now just cross the barrier Progress.emit(Progress.END_THREAD_STEP) except threading.BrokenBarrierError as err: - Logging.info("Main loop aborted, caused by worker thread time-out") + Logging.info("Main loop aborted, caused by worker thread(s) time-out") self._execStats.registerFailure("Aborted due to worker thread timeout") - print("\n\nWorker Thread time-out detected, important thread info:") + print("\n\nWorker Thread time-out detected, TAOS related threads are:") ts = ThreadStacks() ts.print(filterInternal=True) workerTimeout = True @@ -1242,6 +1242,7 @@ class Task(): 0x0B, # Unable to establish connection, more details in TD-1648 0x200, # invalid SQL， TODO: re-examine with TD-934 0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776 + 0x213, # "Disconnected from service", result of "kill connection ???" 0x217, # "db not selected", client side defined error code # 0x218, # "Table does not exist" client side defined error code 0x360, # Table already exists @@ -1911,13 +1912,88 @@ class TaskAddData(StateTransitionTask): def canBeginFrom(cls, state: AnyState): return state.canAddData() + def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor): + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + fullTableName = db.getName() + '.' + regTableName + + sql = "insert into {} values ".format(fullTableName) + for j in range(numRecords): # number of records per table + nextInt = db.getNextInt() + nextTick = db.getNextTick() + sql += "('{}', {});".format(nextTick, nextInt) + dbc.execute(sql) + + def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + + for j in range(numRecords): # number of records per table + nextInt = db.getNextInt() + nextTick = db.getNextTick() + if gConfig.record_ops: + self.prepToRecordOps() + self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + if gConfig.verify_data: + self.lockTable(fullTableName) + # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written + + try: + sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {}) + fullTableName, + # ds.getFixedSuperTableName(), + # ds.getNextBinary(), ds.getNextFloat(), + nextTick, nextInt) + dbc.execute(sql) + except: # Any exception at all + if gConfig.verify_data: + self.unlockTable(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if gConfig.verify_data: # only if command line asks for it + try: + readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + if readBack != nextInt : + raise taos.error.ProgrammingError( + "Failed to read back same data, wrote: {}, read: {}" + .format(nextInt, readBack), 0x999) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x991, 0x992] : # not a single result + raise taos.error.ProgrammingError( + "Failed to read back same data for tick: {}, wrote: {}, read: {}" + .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), + errno) + elif errno in [0x218, 0x362]: # table doesn't exist + # do nothing + dummy = 0 + else: + # Re-throw otherwise + raise + finally: + self.unlockTable(fullTableName) # Unlock the table no matter what + + # Successfully wrote the data into the DB, let's record it somehow + te.recordDataMark(nextInt) + + if gConfig.record_ops: + self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) + self.fAddLogDone.flush() + os.fsync(self.fAddLogDone) + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access db = self._db dbc = wt.getDbConn() - tblSeq = list(range( - self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES)) - random.shuffle(tblSeq) + numTables = self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + tblSeq = list(range(numTables )) + random.shuffle(tblSeq) # now we have random sequence for i in tblSeq: if (i in self.activeTable): # wow already active print("x", end="", flush=True) # concurrent insertion @@ -1925,82 +2001,20 @@ class TaskAddData(StateTransitionTask): self.activeTable.add(i) # marking it active sTable = db.getFixedSuperTable() - regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) - - + regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) fullTableName = db.getName() + '.' + regTableName # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists # self._unlockTable(fullTableName) - for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table - nextInt = db.getNextInt() - nextTick = db.getNextTick() - if gConfig.record_ops: - self.prepToRecordOps() - self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) - self.fAddLogReady.flush() - os.fsync(self.fAddLogReady) - - # TODO: too ugly trying to lock the table reliably, refactor... - fullTableName = db.getName() + '.' + regTableName - if gConfig.verify_data: - self.lockTable(fullTableName) - # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written - - try: - sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {}) - fullTableName, - # ds.getFixedSuperTableName(), - # ds.getNextBinary(), ds.getNextFloat(), - nextTick, nextInt) - dbc.execute(sql) - except: # Any exception at all - if gConfig.verify_data: - self.unlockTable(fullTableName) - raise - - # Now read it back and verify, we might encounter an error if table is dropped - if gConfig.verify_data: # only if command line asks for it - try: - readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". - format(db.getName(), regTableName, nextTick)) - if readBack != nextInt : - raise taos.error.ProgrammingError( - "Failed to read back same data, wrote: {}, read: {}" - .format(nextInt, readBack), 0x999) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x991, 0x992] : # not a single result - raise taos.error.ProgrammingError( - "Failed to read back same data for tick: {}, wrote: {}, read: {}" - .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), - errno) - elif errno in [0x218, 0x362]: # table doesn't exist - # do nothing - dummy = 0 - else: - # Re-throw otherwise - raise - finally: - self.unlockTable(fullTableName) # Unlock the table no matter what - - # Successfully wrote the data into the DB, let's record it somehow - te.recordDataMark(nextInt) - - if gConfig.record_ops: - self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) - self.fAddLogDone.flush() - os.fsync(self.fAddLogDone) - - + if Dice.throw(1) == 0: # 1 in 2 chance + self._addData(db, dbc, regTableName, te) + else: + self._addDataInBatch(db, dbc, regTableName, te) self.activeTable.discard(i) # not raising an error, unlike remove - - - class ThreadStacks: # stack info for all threads def __init__(self): self._allStacks = {} @@ -2022,13 +2036,14 @@ class ThreadStacks: # stack info for all threads '__init__']: # the thread that extracted the stack continue # ignore # Now print - print("\n<----- Thread Info for ID: {}".format(thNid)) + print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid)) + stackFrame = 0 for frame in stack: # print(frame) - print("File {filename}, line {lineno}, in {name}".format( - filename=frame.filename, lineno=frame.lineno, name=frame.name)) + print("[{sf}] File {filename}, line {lineno}, in {name}".format( + sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name)) print(" {}".format(frame.line)) - print("-----> End of Thread Info\n") + print("-----> End of Thread Info ----->\n") class ClientManager: def __init__(self): From 10e951eb2ea5fc1230418cb9a933549771117d05 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 6 Nov 2020 00:39:39 +0000 Subject: [PATCH 055/122] fix bug in tfile --- src/util/src/tfile.c | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index ea699c2436..c3becef82c 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -26,13 +26,11 @@ static void taosCloseFile(void *p) { } int tfinit() { - tsFileRsetId = taosOpenRef(2000, taosCloseFile); return tsFileRsetId; } void tfcleanup() { - if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); tsFileRsetId = -1; } @@ -45,17 +43,15 @@ int64_t tfopen(const char *pathname, int flags) { return -1; } - int64_t rid = taosAddRef(tsFileRsetId, (void *)(long)fd); - if (rid < 0) { - close(fd); - return -1; - } + void *p = (void *)(long)fd; + int64_t rid = taosAddRef(tsFileRsetId, p); + if (rid < 0) close(fd); return rid; } int64_t tfclose(int64_t tfd) { - return taosReleaseRef(tsFileRsetId, tfd); + return taosRemoveRef(tsFileRsetId, tfd); } ssize_t tfwrite(int64_t tfd, const void *buf, size_t count) { @@ -66,10 +62,7 @@ ssize_t tfwrite(int64_t tfd, const void *buf, size_t count) { int fd = (int)(uintptr_t)p; ssize_t ret = write(fd, buf, count); - if (ret < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseRef(tsFileRsetId, tfd); return ret; @@ -83,10 +76,7 @@ ssize_t tfread(int64_t tfd, void *buf, size_t count) { int fd = (int)(uintptr_t)p; ssize_t ret = read(fd, buf, count); - if (ret < 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } + if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseRef(tsFileRsetId, tfd); return ret; From 54a50d66e5372a87f6a90a22214a897d081bcfe9 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 6 Nov 2020 00:54:51 +0000 Subject: [PATCH 056/122] type conversion --- src/util/src/tfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index c3becef82c..0324a5bae8 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -43,7 +43,7 @@ int64_t tfopen(const char *pathname, int flags) { return -1; } - void *p = (void *)(long)fd; + void *p = (void *)(int64_t)fd; int64_t rid = taosAddRef(tsFileRsetId, p); if (rid < 0) close(fd); From 5ffa312614b2561beae816c6d87260afbdd0a742 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 6 Nov 2020 01:10:08 +0000 Subject: [PATCH 057/122] return -1 if error happens --- src/util/src/tfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 0324a5bae8..b9a25ffe23 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -57,7 +57,7 @@ int64_t tfclose(int64_t tfd) { ssize_t tfwrite(int64_t tfd, const void *buf, size_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return terrno; + if (p == NULL) return -1; int fd = (int)(uintptr_t)p; @@ -71,7 +71,7 @@ ssize_t tfwrite(int64_t tfd, const void *buf, size_t count) { ssize_t tfread(int64_t tfd, void *buf, size_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); - if (p == NULL) return terrno; + if (p == NULL) return -1; int fd = (int)(uintptr_t)p; From ecdd067900db71449648a8182673acf31ec986e8 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 6 Nov 2020 01:20:53 +0000 Subject: [PATCH 058/122] size_t type conversion --- src/util/src/tfile.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index b9a25ffe23..4807fea0d0 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -61,7 +61,7 @@ ssize_t tfwrite(int64_t tfd, const void *buf, size_t count) { int fd = (int)(uintptr_t)p; - ssize_t ret = write(fd, buf, count); + ssize_t ret = write(fd, buf, (uint32_t)count); if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseRef(tsFileRsetId, tfd); @@ -75,7 +75,7 @@ ssize_t tfread(int64_t tfd, void *buf, size_t count) { int fd = (int)(uintptr_t)p; - ssize_t ret = read(fd, buf, count); + ssize_t ret = read(fd, buf, (uint32_t)count); if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseRef(tsFileRsetId, tfd); From 4902effb55d878fd009a3c99ed41948870950542 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 6 Nov 2020 01:27:37 +0000 Subject: [PATCH 059/122] data type for rid --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index a24894c2bf..05275c28b0 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -82,7 +82,7 @@ typedef struct { int8_t oldInUse; // server EP inUse passed by app int8_t redirect; // flag to indicate redirect int8_t connType; // connection type - int32_t rid; // refId returned by taosAddRef + int64_t rid; // refId returned by taosAddRef SRpcMsg *pRsp; // for synchronous API tsem_t *pSem; // for synchronous API SRpcEpSet *pSet; // for synchronous API From d553f4083a5e46936f426cc4decc4a42dc34ed92 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 6 Nov 2020 09:30:40 +0800 Subject: [PATCH 060/122] [TD-1961] modify return when call taos_query() --- tests/examples/c/asyncdemo.c | 41 +++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/tests/examples/c/asyncdemo.c b/tests/examples/c/asyncdemo.c index 1e523bd7fe..225c4f7541 100644 --- a/tests/examples/c/asyncdemo.c +++ b/tests/examples/c/asyncdemo.c @@ -46,6 +46,34 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code); void taos_select_call_back(void *param, TAOS_RES *tres, int code); void taos_error(TAOS *taos); +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + int main(int argc, char *argv[]) { TAOS *taos; @@ -78,16 +106,14 @@ int main(int argc, char *argv[]) printf("success to connect to server\n"); - sprintf(sql, "drop database %s", db); - taos_query(taos, sql); + sprintf(sql, "drop database if exists %s", db); + queryDB(taos, sql); sprintf(sql, "create database %s", db); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); sprintf(sql, "use %s", db); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); strcpy(prefix, "asytbl_"); for (i = 0; i < numOfTables; ++i) { @@ -95,8 +121,7 @@ int main(int argc, char *argv[]) tableList[i].taos = taos; sprintf(tableList[i].name, "%s%d", prefix, i); sprintf(sql, "create table %s%d (ts timestamp, volume bigint)", prefix, i); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); } gettimeofday(&systemTime, NULL); From 4bc9284a76e1e656f1d9673f90452dfae948353f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 Nov 2020 09:31:29 +0800 Subject: [PATCH 061/122] TD-1915 --- src/dnode/inc/dnodeMPeer.h | 10 +-- src/dnode/inc/dnodeMRead.h | 10 +-- src/dnode/inc/dnodeMWrite.h | 10 +-- src/dnode/src/dnodeMPeer.c | 60 +++++++++--------- src/dnode/src/dnodeMRead.c | 76 +++++++++++------------ src/dnode/src/dnodeMWrite.c | 74 +++++++++++----------- src/dnode/src/dnodeMain.c | 26 ++++---- src/dnode/src/dnodePeer.c | 42 ++++++------- src/dnode/src/dnodeShell.c | 88 +++++++++++++-------------- src/dnode/src/dnodeVWrite.c | 2 +- src/inc/dnode.h | 32 +++++----- src/mnode/src/mnodeMain.c | 12 ++-- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeTable.c | 24 ++++---- src/mnode/src/mnodeVgroup.c | 10 +-- src/plugins/monitor/src/monitorMain.c | 64 +++++++++---------- src/vnode/src/vnodeCfg.c | 1 - src/vnode/src/vnodeMain.c | 30 ++++----- src/vnode/src/vnodeRead.c | 67 ++++++++++---------- src/vnode/src/vnodeWrite.c | 3 +- 20 files changed, 316 insertions(+), 327 deletions(-) diff --git a/src/dnode/inc/dnodeMPeer.h b/src/dnode/inc/dnodeMPeer.h index 9a48703110..00221baa22 100644 --- a/src/dnode/inc/dnodeMPeer.h +++ b/src/dnode/inc/dnodeMPeer.h @@ -20,11 +20,11 @@ extern "C" { #endif -int32_t dnodeInitMnodePeer(); -void dnodeCleanupMnodePeer(); -int32_t dnodeAllocateMnodePqueue(); -void dnodeFreeMnodePqueue(); -void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg); +int32_t dnodeInitMPeer(); +void dnodeCleanupMPeer(); +int32_t dnodeAllocateMPeerQueue(); +void dnodeFreeMPeerQueue(); +void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeMRead.h b/src/dnode/inc/dnodeMRead.h index 4e93838b79..8a8e71227d 100644 --- a/src/dnode/inc/dnodeMRead.h +++ b/src/dnode/inc/dnodeMRead.h @@ -20,11 +20,11 @@ extern "C" { #endif -int32_t dnodeInitMnodeRead(); -void dnodeCleanupMnodeRead(); -int32_t dnodeAllocateMnodeRqueue(); -void dnodeFreeMnodeRqueue(); -void dnodeDispatchToMnodeReadQueue(SRpcMsg *rpcMsg); +int32_t dnodeInitMRead(); +void dnodeCleanupMRead(); +int32_t dnodeAllocMReadQueue(); +void dnodeFreeMReadQueue(); +void dnodeDispatchToMReadQueue(SRpcMsg *rpcMsg); #ifdef __cplusplus } diff --git a/src/dnode/inc/dnodeMWrite.h b/src/dnode/inc/dnodeMWrite.h index 498fea81c5..6a3d41bc81 100644 --- a/src/dnode/inc/dnodeMWrite.h +++ b/src/dnode/inc/dnodeMWrite.h @@ -20,11 +20,11 @@ extern "C" { #endif -int32_t dnodeInitMnodeWrite(); -void dnodeCleanupMnodeWrite(); -int32_t dnodeAllocateMnodeWqueue(); -void dnodeFreeMnodeWqueue(); -void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg); +int32_t dnodeInitMWrite(); +void dnodeCleanupMWrite(); +int32_t dnodeAllocMWritequeue(); +void dnodeFreeMWritequeue(); +void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg); #ifdef __cplusplus } diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 0da7ab74e4..5c49dc259c 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -35,44 +35,44 @@ typedef struct { typedef struct { int32_t curNum; int32_t maxNum; - SMPeerWorker *peerWorker; + SMPeerWorker *worker; } SMPeerWorkerPool; -static SMPeerWorkerPool tsMPeerPool; +static SMPeerWorkerPool tsMPeerWP; static taos_qset tsMPeerQset; static taos_queue tsMPeerQueue; -static void *dnodeProcessMnodePeerQueue(void *param); +static void *dnodeProcessMPeerQueue(void *param); -int32_t dnodeInitMnodePeer() { +int32_t dnodeInitMPeer() { tsMPeerQset = taosOpenQset(); - tsMPeerPool.maxNum = 1; - tsMPeerPool.curNum = 0; - tsMPeerPool.peerWorker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerPool.maxNum); + tsMPeerWP.maxNum = 1; + tsMPeerWP.curNum = 0; + tsMPeerWP.worker = (SMPeerWorker *)calloc(sizeof(SMPeerWorker), tsMPeerWP.maxNum); - if (tsMPeerPool.peerWorker == NULL) return -1; - for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i; + if (tsMPeerWP.worker == NULL) return -1; + for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { + SMPeerWorker *pWorker = tsMPeerWP.worker + i; pWorker->workerId = i; dDebug("dnode mpeer worker:%d is created", i); } - dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerPool.maxNum, tsMPeerQset); + dDebug("dnode mpeer is initialized, workers:%d qset:%p", tsMPeerWP.maxNum, tsMPeerQset); return 0; } -void dnodeCleanupMnodePeer() { - for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i; +void dnodeCleanupMPeer() { + for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { + SMPeerWorker *pWorker = tsMPeerWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(tsMPeerQset); } dDebug("dnode mpeer worker:%d is closed", i); } - for (int32_t i = 0; i < tsMPeerPool.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i; + for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) { + SMPeerWorker *pWorker = tsMPeerWP.worker + i; dDebug("dnode mpeer worker:%d start to join", i); if (pWorker->thread) { pthread_join(pWorker->thread, NULL); @@ -84,44 +84,44 @@ void dnodeCleanupMnodePeer() { taosCloseQset(tsMPeerQset); tsMPeerQset = NULL; - taosTFree(tsMPeerPool.peerWorker); + taosTFree(tsMPeerWP.worker); } -int32_t dnodeAllocateMnodePqueue() { +int32_t dnodeAllocateMPeerQueue() { tsMPeerQueue = taosOpenQueue(); if (tsMPeerQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; taosAddIntoQset(tsMPeerQset, tsMPeerQueue, NULL); - for (int32_t i = tsMPeerPool.curNum; i < tsMPeerPool.maxNum; ++i) { - SMPeerWorker *pWorker = tsMPeerPool.peerWorker + i; + for (int32_t i = tsMPeerWP.curNum; i < tsMPeerWP.maxNum; ++i) { + SMPeerWorker *pWorker = tsMPeerWP.worker + i; pWorker->workerId = i; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodePeerQueue, pWorker) != 0) { + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMPeerQueue, pWorker) != 0) { dError("failed to create thread to process mpeer queue, reason:%s", strerror(errno)); } pthread_attr_destroy(&thAttr); - tsMPeerPool.curNum = i + 1; - dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerPool.maxNum); + tsMPeerWP.curNum = i + 1; + dDebug("dnode mpeer worker:%d is launched, total:%d", pWorker->workerId, tsMPeerWP.maxNum); } dDebug("dnode mpeer queue:%p is allocated", tsMPeerQueue); return TSDB_CODE_SUCCESS; } -void dnodeFreeMnodePqueue() { +void dnodeFreeMPeerQueue() { dDebug("dnode mpeer queue:%p is freed", tsMPeerQueue); taosCloseQueue(tsMPeerQueue); tsMPeerQueue = NULL; } -void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) { +void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMPeerQueue == NULL) { dnodeSendRedirectMsg(pMsg, false); rpcFreeCont(pMsg->pCont); @@ -133,12 +133,12 @@ void dnodeDispatchToMnodePeerQueue(SRpcMsg *pMsg) { taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); } -static void dnodeFreeMnodePeerMsg(SMnodeMsg *pPeer) { +static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) { mnodeCleanupMsg(pPeer); taosFreeQitem(pPeer); } -static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) { +static void dnodeSendRpcMPeerRsp(SMnodeMsg *pPeer, int32_t code) { if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; SRpcMsg rpcRsp = { @@ -149,10 +149,10 @@ static void dnodeSendRpcMnodePeerRsp(SMnodeMsg *pPeer, int32_t code) { }; rpcSendResponse(&rpcRsp); - dnodeFreeMnodePeerMsg(pPeer); + dnodeFreeMPeerMsg(pPeer); } -static void *dnodeProcessMnodePeerQueue(void *param) { +static void *dnodeProcessMPeerQueue(void *param) { SMnodeMsg *pPeerMsg; int32_t type; void * unUsed; @@ -165,7 +165,7 @@ static void *dnodeProcessMnodePeerQueue(void *param) { dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]); int32_t code = mnodeProcessPeerReq(pPeerMsg); - dnodeSendRpcMnodePeerRsp(pPeerMsg, code); + dnodeSendRpcMPeerRsp(pPeerMsg, code); } return NULL; diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 4ad787f26e..8208a66e1b 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -35,46 +35,46 @@ typedef struct { typedef struct { int32_t curNum; int32_t maxNum; - SMReadWorker *readWorker; + SMReadWorker *worker; } SMReadWorkerPool; -static SMReadWorkerPool tsMReadPool; +static SMReadWorkerPool tsMReadWP; static taos_qset tsMReadQset; static taos_queue tsMReadQueue; -static void *dnodeProcessMnodeReadQueue(void *param); +static void *dnodeProcessMReadQueue(void *param); -int32_t dnodeInitMnodeRead() { +int32_t dnodeInitMRead() { tsMReadQset = taosOpenQset(); - tsMReadPool.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2; - tsMReadPool.maxNum = MAX(2, tsMReadPool.maxNum); - tsMReadPool.maxNum = MIN(4, tsMReadPool.maxNum); - tsMReadPool.curNum = 0; - tsMReadPool.readWorker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadPool.maxNum); + tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2; + tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum); + tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum); + tsMReadWP.curNum = 0; + tsMReadWP.worker = (SMReadWorker *)calloc(sizeof(SMReadWorker), tsMReadWP.maxNum); - if (tsMReadPool.readWorker == NULL) return -1; - for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadPool.readWorker + i; + if (tsMReadWP.worker == NULL) return -1; + for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { + SMReadWorker *pWorker = tsMReadWP.worker + i; pWorker->workerId = i; dDebug("dnode mread worker:%d is created", i); } - dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadPool.maxNum, tsMReadQset); + dDebug("dnode mread is initialized, workers:%d qset:%p", tsMReadWP.maxNum, tsMReadQset); return 0; } -void dnodeCleanupMnodeRead() { - for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadPool.readWorker + i; +void dnodeCleanupMRead() { + for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { + SMReadWorker *pWorker = tsMReadWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(tsMReadQset); } dDebug("dnode mread worker:%d is closed", i); } - for (int32_t i = 0; i < tsMReadPool.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadPool.readWorker + i; + for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) { + SMReadWorker *pWorker = tsMReadWP.worker + i; dDebug("dnode mread worker:%d start to join", i); if (pWorker->thread) { pthread_join(pWorker->thread, NULL); @@ -86,43 +86,43 @@ void dnodeCleanupMnodeRead() { taosCloseQset(tsMReadQset); tsMReadQset = NULL; - free(tsMReadPool.readWorker); + free(tsMReadWP.worker); } -int32_t dnodeAllocateMnodeRqueue() { +int32_t dnodeAllocMReadQueue() { tsMReadQueue = taosOpenQueue(); if (tsMReadQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; taosAddIntoQset(tsMReadQset, tsMReadQueue, NULL); - for (int32_t i = tsMReadPool.curNum; i < tsMReadPool.maxNum; ++i) { - SMReadWorker *pWorker = tsMReadPool.readWorker + i; + for (int32_t i = tsMReadWP.curNum; i < tsMReadWP.maxNum; ++i) { + SMReadWorker *pWorker = tsMReadWP.worker + i; pWorker->workerId = i; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeReadQueue, pWorker) != 0) { + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMReadQueue, pWorker) != 0) { dError("failed to create thread to process mread queue, reason:%s", strerror(errno)); } pthread_attr_destroy(&thAttr); - tsMReadPool.curNum = i + 1; - dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadPool.maxNum); + tsMReadWP.curNum = i + 1; + dDebug("dnode mread worker:%d is launched, total:%d", pWorker->workerId, tsMReadWP.maxNum); } dDebug("dnode mread queue:%p is allocated", tsMReadQueue); return TSDB_CODE_SUCCESS; } -void dnodeFreeMnodeRqueue() { +void dnodeFreeMReadQueue() { dDebug("dnode mread queue:%p is freed", tsMReadQueue); taosCloseQueue(tsMReadQueue); tsMReadQueue = NULL; } -void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) { +void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMReadQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); rpcFreeCont(pMsg->pCont); @@ -134,16 +134,16 @@ void dnodeDispatchToMnodeReadQueue(SRpcMsg *pMsg) { taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); } -static void dnodeFreeMnodeReadMsg(SMnodeMsg *pRead) { +static void dnodeFreeMReadMsg(SMnodeMsg *pRead) { mnodeCleanupMsg(pRead); taosFreeQitem(pRead); } -static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) { +static void dnodeSendRpcMReadRsp(SMnodeMsg *pRead, int32_t code) { if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { // may be a auto create req, should put into write queue - dnodeReprocessMnodeWriteMsg(pRead); + dnodeReprocessMWriteMsg(pRead); return; } @@ -155,23 +155,23 @@ static void dnodeSendRpcMnodeReadRsp(SMnodeMsg *pRead, int32_t code) { }; rpcSendResponse(&rpcRsp); - dnodeFreeMnodeReadMsg(pRead); + dnodeFreeMReadMsg(pRead); } -static void *dnodeProcessMnodeReadQueue(void *param) { - SMnodeMsg *pReadMsg; +static void *dnodeProcessMReadQueue(void *param) { + SMnodeMsg *pRead; int32_t type; void * unUsed; - + while (1) { - if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pReadMsg, &unUsed) == 0) { + if (taosReadQitemFromQset(tsMReadQset, &type, (void **)&pRead, &unUsed) == 0) { dDebug("qset:%p, mnode read got no message from qset, exiting", tsMReadQset); break; } - dDebug("%p, msg:%s will be processed in mread queue", pReadMsg->rpcMsg.ahandle, taosMsg[pReadMsg->rpcMsg.msgType]); - int32_t code = mnodeProcessRead(pReadMsg); - dnodeSendRpcMnodeReadRsp(pReadMsg, code); + dDebug("%p, msg:%s will be processed in mread queue", pRead->rpcMsg.ahandle, taosMsg[pRead->rpcMsg.msgType]); + int32_t code = mnodeProcessRead(pRead); + dnodeSendRpcMReadRsp(pRead, code); } return NULL; diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 600688b9fd..e628ea1f81 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -36,45 +36,45 @@ typedef struct { typedef struct { int32_t curNum; int32_t maxNum; - SMWriteWorker *writeWorker; + SMWriteWorker *worker; } SMWriteWorkerPool; -static SMWriteWorkerPool tsMWritePool; +static SMWriteWorkerPool tsMWriteWP; static taos_qset tsMWriteQset; static taos_queue tsMWriteQueue; extern void * tsDnodeTmr; -static void *dnodeProcessMnodeWriteQueue(void *param); +static void *dnodeProcessMWriteQueue(void *param); -int32_t dnodeInitMnodeWrite() { +int32_t dnodeInitMWrite() { tsMWriteQset = taosOpenQset(); - tsMWritePool.maxNum = 1; - tsMWritePool.curNum = 0; - tsMWritePool.writeWorker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWritePool.maxNum); + tsMWriteWP.maxNum = 1; + tsMWriteWP.curNum = 0; + tsMWriteWP.worker = (SMWriteWorker *)calloc(sizeof(SMWriteWorker), tsMWriteWP.maxNum); - if (tsMWritePool.writeWorker == NULL) return -1; - for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWritePool.writeWorker + i; + if (tsMWriteWP.worker == NULL) return -1; + for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { + SMWriteWorker *pWorker = tsMWriteWP.worker + i; pWorker->workerId = i; dDebug("dnode mwrite worker:%d is created", i); } - dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWritePool.maxNum, tsMWriteQset); + dDebug("dnode mwrite is initialized, workers:%d qset:%p", tsMWriteWP.maxNum, tsMWriteQset); return 0; } -void dnodeCleanupMnodeWrite() { - for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWritePool.writeWorker + i; +void dnodeCleanupMWrite() { + for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { + SMWriteWorker *pWorker = tsMWriteWP.worker + i; if (pWorker->thread) { taosQsetThreadResume(tsMWriteQset); } dDebug("dnode mwrite worker:%d is closed", i); } - for (int32_t i = 0; i < tsMWritePool.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWritePool.writeWorker + i; + for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) { + SMWriteWorker *pWorker = tsMWriteWP.worker + i; dDebug("dnode mwrite worker:%d start to join", i); if (pWorker->thread) { pthread_join(pWorker->thread, NULL); @@ -86,43 +86,43 @@ void dnodeCleanupMnodeWrite() { taosCloseQset(tsMWriteQset); tsMWriteQset = NULL; - taosTFree(tsMWritePool.writeWorker); + taosTFree(tsMWriteWP.worker); } -int32_t dnodeAllocateMnodeWqueue() { +int32_t dnodeAllocMWritequeue() { tsMWriteQueue = taosOpenQueue(); if (tsMWriteQueue == NULL) return TSDB_CODE_DND_OUT_OF_MEMORY; taosAddIntoQset(tsMWriteQset, tsMWriteQueue, NULL); - for (int32_t i = tsMWritePool.curNum; i < tsMWritePool.maxNum; ++i) { - SMWriteWorker *pWorker = tsMWritePool.writeWorker + i; + for (int32_t i = tsMWriteWP.curNum; i < tsMWriteWP.maxNum; ++i) { + SMWriteWorker *pWorker = tsMWriteWP.worker + i; pWorker->workerId = i; pthread_attr_t thAttr; pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMnodeWriteQueue, pWorker) != 0) { + if (pthread_create(&pWorker->thread, &thAttr, dnodeProcessMWriteQueue, pWorker) != 0) { dError("failed to create thread to process mwrite queue, reason:%s", strerror(errno)); } pthread_attr_destroy(&thAttr); - tsMWritePool.curNum = i + 1; - dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWritePool.maxNum); + tsMWriteWP.curNum = i + 1; + dDebug("dnode mwrite worker:%d is launched, total:%d", pWorker->workerId, tsMWriteWP.maxNum); } dDebug("dnode mwrite queue:%p is allocated", tsMWriteQueue); return TSDB_CODE_SUCCESS; } -void dnodeFreeMnodeWqueue() { +void dnodeFreeMWritequeue() { dDebug("dnode mwrite queue:%p is freed", tsMWriteQueue); taosCloseQueue(tsMWriteQueue); tsMWriteQueue = NULL; } -void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) { +void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMWriteQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); rpcFreeCont(pMsg->pCont); @@ -137,7 +137,7 @@ void dnodeDispatchToMnodeWriteQueue(SRpcMsg *pMsg) { taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } -static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) { +static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { dDebug("app:%p:%p, msg:%s is freed from mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); @@ -145,12 +145,12 @@ static void dnodeFreeMnodeWriteMsg(SMnodeMsg *pWrite) { taosFreeQitem(pWrite); } -void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) { +void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code) { SMnodeMsg *pWrite = pMsg; if (pWrite == NULL) return; if (code == TSDB_CODE_MND_ACTION_IN_PROGRESS) return; if (code == TSDB_CODE_MND_ACTION_NEED_REPROCESSED) { - dnodeReprocessMnodeWriteMsg(pWrite); + dnodeReprocessMWriteMsg(pWrite); return; } @@ -162,10 +162,10 @@ void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code) { }; rpcSendResponse(&rpcRsp); - dnodeFreeMnodeWriteMsg(pWrite); + dnodeFreeMWriteMsg(pWrite); } -static void *dnodeProcessMnodeWriteQueue(void *param) { +static void *dnodeProcessMWriteQueue(void *param) { SMnodeMsg *pWrite; int32_t type; void * unUsed; @@ -180,13 +180,13 @@ static void *dnodeProcessMnodeWriteQueue(void *param) { taosMsg[pWrite->rpcMsg.msgType]); int32_t code = mnodeProcessWrite(pWrite); - dnodeSendRpcMnodeWriteRsp(pWrite, code); + dnodeSendRpcMWriteRsp(pWrite, code); } return NULL; } -void dnodeReprocessMnodeWriteMsg(void *pMsg) { +void dnodeReprocessMWriteMsg(void *pMsg) { SMnodeMsg *pWrite = pMsg; if (!mnodeIsRunning() || tsMWriteQueue == NULL) { @@ -194,7 +194,7 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) { taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); dnodeSendRedirectMsg(pMsg, true); - dnodeFreeMnodeWriteMsg(pWrite); + dnodeFreeMWriteMsg(pWrite); } else { dDebug("app:%p:%p, msg:%s is reput into mwrite queue:%p, retry times:%d", pWrite->rpcMsg.ahandle, pWrite, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry); @@ -203,12 +203,12 @@ void dnodeReprocessMnodeWriteMsg(void *pMsg) { } } -static void dnodeDoDelayReprocessMnodeWriteMsg(void *param, void *tmrId) { - dnodeReprocessMnodeWriteMsg(param); +static void dnodeDoDelayReprocessMWriteMsg(void *param, void *tmrId) { + dnodeReprocessMWriteMsg(param); } -void dnodeDelayReprocessMnodeWriteMsg(void *pMsg) { +void dnodeDelayReprocessMWriteMsg(void *pMsg) { SMnodeMsg *mnodeMsg = pMsg; void *unUsed = NULL; - taosTmrReset(dnodeDoDelayReprocessMnodeWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed); + taosTmrReset(dnodeDoDelayReprocessMWriteMsg, 300, mnodeMsg, tsDnodeTmr, &unUsed); } diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 5fde4f972b..f4c0ee565e 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -37,11 +37,11 @@ #include "dnodeShell.h" #include "dnodeTelemetry.h" -static SDnodeRunStatus tsDnodeRunStatus = TSDB_DNODE_RUN_STATUS_STOPPED; +static SRunStatus tsRunStatus = TSDB_RUN_STATUS_STOPPED; static int32_t dnodeInitStorage(); static void dnodeCleanupStorage(); -static void dnodeSetRunStatus(SDnodeRunStatus status); +static void dnodeSetRunStatus(SRunStatus status); static void dnodeCheckDataDirOpenned(char *dir); static int32_t dnodeInitComponents(); static void dnodeCleanupComponents(int32_t stepId); @@ -63,9 +63,9 @@ static const SDnodeComponent tsDnodeComponents[] = { {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! {"vread", dnodeInitVRead, dnodeCleanupVRead}, {"vwrite", dnodeInitVWrite, dnodeCleanupVWrite}, - {"mread", dnodeInitMnodeRead, dnodeCleanupMnodeRead}, - {"mwrite", dnodeInitMnodeWrite, dnodeCleanupMnodeWrite}, - {"mpeer", dnodeInitMnodePeer, dnodeCleanupMnodePeer}, + {"mread", dnodeInitMRead, dnodeCleanupMRead}, + {"mwrite", dnodeInitMWrite, dnodeCleanupMWrite}, + {"mpeer", dnodeInitMPeer, dnodeCleanupMPeer}, {"client", dnodeInitClient, dnodeCleanupClient}, {"server", dnodeInitServer, dnodeCleanupServer}, {"mgmt", dnodeInitMgmt, dnodeCleanupMgmt}, @@ -104,7 +104,7 @@ static int32_t dnodeInitComponents() { } int32_t dnodeInitSystem() { - dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_INITIALIZE); + dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE); tscEmbedded = 1; taosBlockSIGPIPE(); taosResolveCRC(); @@ -137,7 +137,7 @@ int32_t dnodeInitSystem() { } dnodeStartModules(); - dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_RUNING); + dnodeSetRunStatus(TSDB_RUN_STATUS_RUNING); dInfo("TDengine is initialized successfully"); @@ -145,20 +145,20 @@ int32_t dnodeInitSystem() { } void dnodeCleanUpSystem() { - if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_STOPPED) { - dnodeSetRunStatus(TSDB_DNODE_RUN_STATUS_STOPPED); + if (dnodeGetRunStatus() != TSDB_RUN_STATUS_STOPPED) { + dnodeSetRunStatus(TSDB_RUN_STATUS_STOPPED); dnodeCleanupComponents(sizeof(tsDnodeComponents) / sizeof(tsDnodeComponents[0]) - 1); taos_cleanup(); taosCloseLog(); } } -SDnodeRunStatus dnodeGetRunStatus() { - return tsDnodeRunStatus; +SRunStatus dnodeGetRunStatus() { + return tsRunStatus; } -static void dnodeSetRunStatus(SDnodeRunStatus status) { - tsDnodeRunStatus = status; +static void dnodeSetRunStatus(SRunStatus status) { + tsRunStatus = status; } static void dnodeCheckDataDirOpenned(char *dir) { diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index c6fc2b9e36..cbdbc54c25 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -34,8 +34,8 @@ static void (*dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *); static void (*dnodeProcessRspMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *rpcMsg); static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet); -static void *tsDnodeServerRpc = NULL; -static void *tsDnodeClientRpc = NULL; +static void *tsServerRpc = NULL; +static void *tsClientRpc = NULL; int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_TABLE] = dnodeDispatchToVWriteQueue; @@ -50,11 +50,11 @@ int32_t dnodeInitServer() { dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CONFIG_DNODE] = dnodeDispatchToMgmtQueue; dnodeProcessReqMsgFp[TSDB_MSG_TYPE_MD_CREATE_MNODE] = dnodeDispatchToMgmtQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMnodePeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMnodePeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMnodePeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMnodePeerQueue; - dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMnodePeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_TABLE] = dnodeDispatchToMPeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_CONFIG_VNODE] = dnodeDispatchToMPeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_AUTH] = dnodeDispatchToMPeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_GRANT] = dnodeDispatchToMPeerQueue; + dnodeProcessReqMsgFp[TSDB_MSG_TYPE_DM_STATUS] = dnodeDispatchToMPeerQueue; SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); @@ -66,8 +66,8 @@ int32_t dnodeInitServer() { rpcInit.connType = TAOS_CONN_SERVER; rpcInit.idleTime = tsShellActivityTimer * 1000; - tsDnodeServerRpc = rpcOpen(&rpcInit); - if (tsDnodeServerRpc == NULL) { + tsServerRpc = rpcOpen(&rpcInit); + if (tsServerRpc == NULL) { dError("failed to init inter-dnodes RPC server"); return -1; } @@ -77,9 +77,9 @@ int32_t dnodeInitServer() { } void dnodeCleanupServer() { - if (tsDnodeServerRpc) { - rpcClose(tsDnodeServerRpc); - tsDnodeServerRpc = NULL; + if (tsServerRpc) { + rpcClose(tsServerRpc); + tsServerRpc = NULL; dInfo("inter-dnodes RPC server is closed"); } } @@ -93,7 +93,7 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pMsg->pCont == NULL) return; - if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) { + if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { rspMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rspMsg); rpcFreeCont(pMsg->pCont); @@ -131,8 +131,8 @@ int32_t dnodeInitClient() { rpcInit.ckey = "key"; rpcInit.secret = secret; - tsDnodeClientRpc = rpcOpen(&rpcInit); - if (tsDnodeClientRpc == NULL) { + tsClientRpc = rpcOpen(&rpcInit); + if (tsClientRpc == NULL) { dError("failed to init mnode rpc client"); return -1; } @@ -142,9 +142,9 @@ int32_t dnodeInitClient() { } void dnodeCleanupClient() { - if (tsDnodeClientRpc) { - rpcClose(tsDnodeClientRpc); - tsDnodeClientRpc = NULL; + if (tsClientRpc) { + rpcClose(tsClientRpc); + tsClientRpc = NULL; dInfo("dnode inter-dnodes rpc client is closed"); } } @@ -168,15 +168,15 @@ void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)) { } void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { - rpcSendRequest(tsDnodeClientRpc, epSet, rpcMsg); + rpcSendRequest(tsClientRpc, epSet, rpcMsg); } void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { SRpcEpSet epSet = {0}; dnodeGetEpSetForPeer(&epSet); - rpcSendRecv(tsDnodeClientRpc, &epSet, rpcMsg, rpcRsp); + rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp); } void dnodeSendMsgToDnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp, SRpcEpSet *epSet) { - rpcSendRecv(tsDnodeClientRpc, epSet, rpcMsg, rpcRsp); + rpcSendRecv(tsClientRpc, epSet, rpcMsg, rpcRsp); } \ No newline at end of file diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 2f6d844ff1..89f657f789 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -33,9 +33,9 @@ static void (*dnodeProcessShellMsgFp[TSDB_MSG_TYPE_MAX])(SRpcMsg *); static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *); static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey); -static void * tsDnodeShellRpc = NULL; -static int32_t tsDnodeQueryReqNum = 0; -static int32_t tsDnodeSubmitReqNum = 0; +static void * tsShellRpc = NULL; +static int32_t tsQueryReqNum = 0; +static int32_t tsSubmitReqNum = 0; int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_SUBMIT] = dnodeDispatchToVWriteQueue; @@ -44,35 +44,35 @@ int32_t dnodeInitShell() { dnodeProcessShellMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = dnodeDispatchToVWriteQueue; // the following message shall be treated as mnode write - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMnodeWriteQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMnodeWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_ACCT] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_USER] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DNODE]= dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DNODE] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_DB] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CREATE_TABLE]= dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_DROP_TABLE] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_TABLE] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_ALTER_STREAM]= dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_QUERY] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_STREAM] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_KILL_CONN] = dnodeDispatchToMWriteQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONFIG_DNODE]= dnodeDispatchToMWriteQueue; // the following message shall be treated as mnode query - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMnodeReadQueue; - dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMnodeReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_HEARTBEAT] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_CONNECT] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_USE_DB] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLE_META] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_STABLE_VGROUP]= dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_TABLES_META] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_SHOW] = dnodeDispatchToMReadQueue; + dnodeProcessShellMsgFp[TSDB_MSG_TYPE_CM_RETRIEVE] = dnodeDispatchToMReadQueue; int32_t numOfThreads = tsNumOfCores * tsNumOfThreadsPerCore; numOfThreads = (int32_t) ((1.0 - tsRatioOfQueryThreads) * numOfThreads / 2.0); @@ -91,8 +91,8 @@ int32_t dnodeInitShell() { rpcInit.idleTime = tsShellActivityTimer * 1000; rpcInit.afp = dnodeRetrieveUserAuthInfo; - tsDnodeShellRpc = rpcOpen(&rpcInit); - if (tsDnodeShellRpc == NULL) { + tsShellRpc = rpcOpen(&rpcInit); + if (tsShellRpc == NULL) { dError("failed to init shell rpc server"); return -1; } @@ -102,13 +102,13 @@ int32_t dnodeInitShell() { } void dnodeCleanupShell() { - if (tsDnodeShellRpc) { - rpcClose(tsDnodeShellRpc); - tsDnodeShellRpc = NULL; + if (tsShellRpc) { + rpcClose(tsShellRpc); + tsShellRpc = NULL; } } -void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { +static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { SRpcMsg rpcMsg = { .handle = pMsg->handle, .pCont = NULL, @@ -117,7 +117,7 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { if (pMsg->pCont == NULL) return; - if (dnodeGetRunStatus() != TSDB_DNODE_RUN_STATUS_RUNING) { + if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { dError("RPC %p, shell msg:%s is ignored since dnode not running", pMsg->handle, taosMsg[pMsg->msgType]); rpcMsg.code = TSDB_CODE_APP_NOT_READY; rpcSendResponse(&rpcMsg); @@ -126,9 +126,9 @@ void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { } if (pMsg->msgType == TSDB_MSG_TYPE_QUERY) { - atomic_fetch_add_32(&tsDnodeQueryReqNum, 1); + atomic_fetch_add_32(&tsQueryReqNum, 1); } else if (pMsg->msgType == TSDB_MSG_TYPE_SUBMIT) { - atomic_fetch_add_32(&tsDnodeSubmitReqNum, 1); + atomic_fetch_add_32(&tsSubmitReqNum, 1); } else {} if ( dnodeProcessShellMsgFp[pMsg->msgType] ) { @@ -211,12 +211,12 @@ void *dnodeSendCfgTableToRecv(int32_t vgId, int32_t tid) { } } -SDnodeStatisInfo dnodeGetStatisInfo() { - SDnodeStatisInfo info = {0}; - if (dnodeGetRunStatus() == TSDB_DNODE_RUN_STATUS_RUNING) { +SStatisInfo dnodeGetStatisInfo() { + SStatisInfo info = {0}; + if (dnodeGetRunStatus() == TSDB_RUN_STATUS_RUNING) { info.httpReqNum = httpGetReqCount(); - info.queryReqNum = atomic_exchange_32(&tsDnodeQueryReqNum, 0); - info.submitReqNum = atomic_exchange_32(&tsDnodeSubmitReqNum, 0); + info.queryReqNum = atomic_exchange_32(&tsQueryReqNum, 0); + info.submitReqNum = atomic_exchange_32(&tsSubmitReqNum, 0); } return info; diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index c28ad66b65..9b7497d3ec 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -113,7 +113,7 @@ void dnodeDispatchToVWriteQueue(SRpcMsg *pRpcMsg) { void *dnodeAllocVWriteQueue(void *pVnode) { pthread_mutex_lock(&tsVWriteWP.mutex); SVWriteWorker *pWorker = tsVWriteWP.worker + tsVWriteWP.nextId; - void *queue = taosOpenQueue(); + taos_queue *queue = taosOpenQueue(); if (queue == NULL) { pthread_mutex_unlock(&tsVWriteWP.mutex); return NULL; diff --git a/src/inc/dnode.h b/src/inc/dnode.h index 6032d8cc0a..9454b97e38 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -27,16 +27,16 @@ typedef struct { int32_t queryReqNum; int32_t submitReqNum; int32_t httpReqNum; -} SDnodeStatisInfo; +} SStatisInfo; typedef enum { - TSDB_DNODE_RUN_STATUS_INITIALIZE, - TSDB_DNODE_RUN_STATUS_RUNING, - TSDB_DNODE_RUN_STATUS_STOPPED -} SDnodeRunStatus; + TSDB_RUN_STATUS_INITIALIZE, + TSDB_RUN_STATUS_RUNING, + TSDB_RUN_STATUS_STOPPED +} SRunStatus; -SDnodeRunStatus dnodeGetRunStatus(); -SDnodeStatisInfo dnodeGetStatisInfo(); +SRunStatus dnodeGetRunStatus(); +SStatisInfo dnodeGetStatisInfo(); bool dnodeIsFirstDeploy(); bool dnodeIsMasterEp(char *ep); @@ -59,15 +59,15 @@ void dnodeSendRpcVWriteRsp(void *pVnode, void *param, int32_t code); void *dnodeAllocVReadQueue(void *pVnode); void dnodeFreeVReadQueue(void *rqueue); -int32_t dnodeAllocateMnodePqueue(); -void dnodeFreeMnodePqueue(); -int32_t dnodeAllocateMnodeRqueue(); -void dnodeFreeMnodeRqueue(); -int32_t dnodeAllocateMnodeWqueue(); -void dnodeFreeMnodeWqueue(); -void dnodeSendRpcMnodeWriteRsp(void *pMsg, int32_t code); -void dnodeReprocessMnodeWriteMsg(void *pMsg); -void dnodeDelayReprocessMnodeWriteMsg(void *pMsg); +int32_t dnodeAllocateMPeerQueue(); +void dnodeFreeMPeerQueue(); +int32_t dnodeAllocMReadQueue(); +void dnodeFreeMReadQueue(); +int32_t dnodeAllocMWritequeue(); +void dnodeFreeMWritequeue(); +void dnodeSendRpcMWriteRsp(void *pMsg, int32_t code); +void dnodeReprocessMWriteMsg(void *pMsg); +void dnodeDelayReprocessMWriteMsg(void *pMsg); void dnodeSendStatusMsgToMnode(); diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 2bb8a81056..1f5ad42bde 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -96,9 +96,9 @@ int32_t mnodeStartSystem() { return -1; } - dnodeAllocateMnodeWqueue(); - dnodeAllocateMnodeRqueue(); - dnodeAllocateMnodePqueue(); + dnodeAllocMWritequeue(); + dnodeAllocMReadQueue(); + dnodeAllocateMPeerQueue(); if (mnodeInitComponents() != 0) { return -1; @@ -127,9 +127,9 @@ void mnodeCleanupSystem() { mInfo("starting to clean up mnode"); tsMgmtIsRunning = false; - dnodeFreeMnodeWqueue(); - dnodeFreeMnodeRqueue(); - dnodeFreeMnodePqueue(); + dnodeFreeMWritequeue(); + dnodeFreeMReadQueue(); + dnodeFreeMPeerQueue(); mnodeCleanupTimer(); mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 4e370ca028..6bc144d1e1 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -295,7 +295,7 @@ static void sdbConfirmForward(void *ahandle, void *param, int32_t code) { if (pOper->writeCb != NULL) { pOper->retCode = (*pOper->writeCb)(pMsg, pOper->retCode); } - dnodeSendRpcMnodeWriteRsp(pMsg, pOper->retCode); + dnodeSendRpcMWriteRsp(pMsg, pOper->retCode); // if ahandle, means this func is called by sdb write if (ahandle == NULL) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 28dc2c6cff..00234e33b2 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -1679,12 +1679,12 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { pTable->info.tableId, pMsg->rpcMsg.handle); pMsg->retry = 0; - dnodeReprocessMnodeWriteMsg(pMsg); + dnodeReprocessMWriteMsg(pMsg); } else { mDebug("app:%p:%p, table:%s, created in dnode, thandle:%p", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, pMsg->rpcMsg.handle); - dnodeSendRpcMnodeWriteRsp(pMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(pMsg, TSDB_CODE_SUCCESS); } return TSDB_CODE_MND_ACTION_IN_PROGRESS; } else { @@ -2351,14 +2351,14 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { mError("app:%p:%p, table:%s, failed to drop in dnode, vgId:%d sid:%d uid:%" PRIu64 ", reason:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code)); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); return; } if (mnodeMsg->pVgroup == NULL) mnodeMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (mnodeMsg->pVgroup == NULL) { mError("app:%p:%p, table:%s, failed to get vgroup", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST); + dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_MND_VGROUP_NOT_EXIST); return; } @@ -2368,7 +2368,7 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { mnodeDropVgroup(mnodeMsg->pVgroup, NULL); } - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); } /* @@ -2399,7 +2399,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeSendDropChildTableMsg(mnodeMsg, false); rpcMsg->code = TSDB_CODE_SUCCESS; - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); return; } @@ -2416,7 +2416,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pTable = NULL; mnodeDestroyChildTable(pTable); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, code); + dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { if (mnodeMsg->retry++ < 10) { @@ -2425,7 +2425,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid, pTable->uid, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); - dnodeDelayReprocessMnodeWriteMsg(mnodeMsg); + dnodeDelayReprocessMWriteMsg(mnodeMsg); } else { mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, @@ -2434,7 +2434,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable}; sdbDeleteRow(&oper); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); } } } @@ -2452,18 +2452,18 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { mDebug("app:%p:%p, ctable:%s, altered in dnode, thandle:%p result:%s", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->rpcMsg.handle, tstrerror(rpcMsg->code)); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); + dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); } else { if (mnodeMsg->retry++ < 3) { mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); - dnodeDelayReprocessMnodeWriteMsg(mnodeMsg); + dnodeDelayReprocessMWriteMsg(mnodeMsg); } else { mError("app:%p:%p, table:%s, failed to alter in dnode, result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, rpcMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); } } } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 1d1f21c3d7..fada760cb7 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -529,7 +529,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; (void)sdbUpdateRow(&desc); - dnodeReprocessMnodeWriteMsg(pMsg); + dnodeReprocessMWriteMsg(pMsg); return TSDB_CODE_MND_ACTION_IN_PROGRESS; // if (pVgroup->status == TAOS_VG_STATUS_CREATING || pVgroup->status == TAOS_VG_STATUS_READY) { // mInfo("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d", pMsg->rpcMsg.ahandle, pMsg, pVgroup->vgId, @@ -537,7 +537,7 @@ static int32_t mnodeCreateVgroupCb(SMnodeMsg *pMsg, int32_t code) { // pVgroup->status = TAOS_VG_STATUS_READY; // SSdbOper desc = {.type = SDB_OPER_GLOBAL, .pObj = pVgroup, .table = tsVgroupSdb}; // (void)sdbUpdateRow(&desc); - // dnodeReprocessMnodeWriteMsg(pMsg); + // dnodeReprocessMWriteMsg(pMsg); // return TSDB_CODE_MND_ACTION_IN_PROGRESS; // } else { // mError("app:%p:%p, vgId:%d, is created in sdb, db:%s replica:%d, but vgroup is dropping", pMsg->rpcMsg.ahandle, @@ -969,7 +969,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mnodeMsg->pVgroup = NULL; mnodeDestroyVgroup(pVgroup); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, code); + dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { SSdbOper oper = { @@ -978,7 +978,7 @@ static void mnodeProcessCreateVnodeRsp(SRpcMsg *rpcMsg) { .pObj = pVgroup }; sdbDeleteRow(&oper); - dnodeSendRpcMnodeWriteRsp(mnodeMsg, mnodeMsg->code); + dnodeSendRpcMWriteRsp(mnodeMsg, mnodeMsg->code); } } @@ -1040,7 +1040,7 @@ static void mnodeProcessDropVnodeRsp(SRpcMsg *rpcMsg) { code = TSDB_CODE_MND_SDB_ERROR; } - dnodeReprocessMnodeWriteMsg(mnodeMsg); + dnodeReprocessMWriteMsg(mnodeMsg); } static int32_t mnodeProcessVnodeCfgMsg(SMnodeMsg *pMsg) { diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monitorMain.c index 048f839b72..de1e0e233c 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monitorMain.c @@ -27,12 +27,12 @@ #include "monitor.h" #include "taoserror.h" -#define monitorFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} -#define monitorError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} -#define monitorWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} -#define monitorInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }} -#define monitorDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} -#define monitorTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} +#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} +#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} +#define mnWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} +#define mnInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }} +#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} +#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} #define SQL_LENGTH 1030 #define LOG_LEN_STR 100 @@ -91,12 +91,12 @@ int32_t monitorInitSystem() { pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) { - monitorError("failed to create thread to for monitor module, reason:%s", strerror(errno)); + mnError("failed to create thread to for monitor module, reason:%s", strerror(errno)); return -1; } pthread_attr_destroy(&thAttr); - monitorDebug("monitor thread is launched"); + mnDebug("monitor thread is launched"); monitorStartSystemFp = monitorStartSystem; monitorStopSystemFp = monitorStopSystem; @@ -107,12 +107,12 @@ int32_t monitorStartSystem() { taos_init(); tsMonitor.start = 1; monitorExecuteSQLFp = monitorExecuteSQL; - monitorInfo("monitor module start"); + mnInfo("monitor module start"); return 0; } static void *monitorThreadFunc(void *param) { - monitorDebug("starting to initialize monitor module ..."); + mnDebug("starting to initialize monitor module ..."); while (1) { static int32_t accessTimes = 0; @@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) { if (tsMonitor.quiting) { tsMonitor.state = MON_STATE_NOT_INIT; - monitorInfo("monitor thread will quit, for taosd is quiting"); + mnInfo("monitor thread will quit, for taosd is quiting"); break; } else { taosGetDisk(); @@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) { } if (dnodeGetDnodeId() <= 0) { - monitorDebug("dnode not initialized, waiting for 3000 ms to start monitor module"); + mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module"); continue; } @@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) { tsMonitor.state = MON_STATE_NOT_INIT; tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0); if (tsMonitor.conn == NULL) { - monitorError("failed to connect to database, reason:%s", tstrerror(terrno)); + mnError("failed to connect to database, reason:%s", tstrerror(terrno)); continue; } else { - monitorDebug("connect to database success"); + mnDebug("connect to database success"); } } @@ -155,10 +155,10 @@ static void *monitorThreadFunc(void *param) { taos_free_result(res); if (code != 0) { - monitorError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code)); + mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code)); break; } else { - monitorDebug("successfully to exec sql:%s", tsMonitor.sql); + mnDebug("successfully to exec sql:%s", tsMonitor.sql); } } @@ -174,7 +174,7 @@ static void *monitorThreadFunc(void *param) { } } - monitorInfo("monitor thread is stopped"); + mnInfo("monitor thread is stopped"); return NULL; } @@ -238,7 +238,7 @@ void monitorStopSystem() { tsMonitor.start = 0; tsMonitor.state = MON_STATE_NOT_INIT; monitorExecuteSQLFp = NULL; - monitorInfo("monitor module stopped"); + mnInfo("monitor module stopped"); } void monitorCleanUpSystem() { @@ -249,7 +249,7 @@ void monitorCleanUpSystem() { taos_close(tsMonitor.conn); tsMonitor.conn = NULL; } - monitorInfo("monitor module is cleaned up"); + mnInfo("monitor module is cleaned up"); } // unit is MB @@ -257,13 +257,13 @@ static int32_t monitorBuildMemorySql(char *sql) { float sysMemoryUsedMB = 0; bool suc = taosGetSysMemory(&sysMemoryUsedMB); if (!suc) { - monitorDebug("failed to get sys memory info"); + mnDebug("failed to get sys memory info"); } float procMemoryUsedMB = 0; suc = taosGetProcMemory(&procMemoryUsedMB); if (!suc) { - monitorDebug("failed to get proc memory info"); + mnDebug("failed to get proc memory info"); } return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB); @@ -274,7 +274,7 @@ static int32_t monitorBuildCpuSql(char *sql) { float sysCpuUsage = 0, procCpuUsage = 0; bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); if (!suc) { - monitorDebug("failed to get cpu usage"); + mnDebug("failed to get cpu usage"); } if (sysCpuUsage <= procCpuUsage) { @@ -294,14 +294,14 @@ static int32_t monitorBuildBandSql(char *sql) { float bandSpeedKb = 0; bool suc = taosGetBandSpeed(&bandSpeedKb); if (!suc) { - monitorDebug("failed to get bandwidth speed"); + mnDebug("failed to get bandwidth speed"); } return sprintf(sql, ", %f", bandSpeedKb); } static int32_t monitorBuildReqSql(char *sql) { - SDnodeStatisInfo info = dnodeGetStatisInfo(); + SStatisInfo info = dnodeGetStatisInfo(); return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum); } @@ -309,7 +309,7 @@ static int32_t monitorBuildIoSql(char *sql) { float readKB = 0, writeKB = 0; bool suc = taosGetProcIO(&readKB, &writeKB); if (!suc) { - monitorDebug("failed to get io info"); + mnDebug("failed to get io info"); } return sprintf(sql, ", %f, %f", readKB, writeKB); @@ -332,19 +332,19 @@ static void monitorSaveSystemInfo() { taos_free_result(res); if (code != 0) { - monitorError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); + mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); } else { - monitorDebug("successfully to save system info, sql:%s", tsMonitor.sql); + mnDebug("successfully to save system info, sql:%s", tsMonitor.sql); } } static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) { int32_t c = taos_errno(result); if (c != TSDB_CODE_SUCCESS) { - monitorError("save %s failed, reason:%s", (char *)param, tstrerror(c)); + mnError("save %s failed, reason:%s", (char *)param, tstrerror(c)); } else { int32_t rows = taos_affected_rows(result); - monitorDebug("save %s succ, rows:%d", (char *)param, rows); + mnDebug("save %s succ, rows:%d", (char *)param, rows); } taos_free_result(result); @@ -380,7 +380,7 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) { pMon->totalConns, pMon->maxConns, pMon->accessState); - monitorDebug("save account info, sql:%s", sql); + mnDebug("save account info, sql:%s", sql); taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info"); } @@ -401,13 +401,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) { len += sprintf(sql + len, "', '%s')", tsLocalEp); sql[len++] = 0; - monitorDebug("save log, sql: %s", sql); + mnDebug("save log, sql: %s", sql); taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log"); } void monitorExecuteSQL(char *sql) { if (tsMonitor.state != MON_STATE_INITED) return; - monitorDebug("execute sql:%s", sql); + mnDebug("execute sql:%s", sql); taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql"); } diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index f0040f8cdf..e8dd44b48f 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -22,7 +22,6 @@ #include "tsdb.h" #include "dnode.h" #include "vnodeInt.h" -#include "vnodeVersion.h" #include "vnodeCfg.h" static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 128da72623..e60173e066 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -15,17 +15,11 @@ #define _DEFAULT_SOURCE #include "os.h" - -#include "tcache.h" -#include "cJSON.h" -#include "dnode.h" -#include "hash.h" #include "taoserror.h" #include "taosmsg.h" #include "tglobal.h" #include "trpc.h" #include "tsdb.h" -#include "ttimer.h" #include "tutil.h" #include "vnode.h" #include "vnodeInt.h" @@ -34,7 +28,7 @@ #include "vnodeCfg.h" #include "vnodeVersion.h" -static SHashObj*tsDnodeVnodesHash; +static SHashObj*tsVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); static int vnodeProcessTsdbStatus(void *arg, int status); static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); @@ -67,8 +61,8 @@ int32_t vnodeInitResources() { vnodeInitWriteFp(); vnodeInitReadFp(); - tsDnodeVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); - if (tsDnodeVnodesHash == NULL) { + tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + if (tsVnodesHash == NULL) { vError("failed to init vnode list"); return TSDB_CODE_VND_OUT_OF_MEMORY; } @@ -77,10 +71,10 @@ int32_t vnodeInitResources() { } void vnodeCleanupResources() { - if (tsDnodeVnodesHash != NULL) { + if (tsVnodesHash != NULL) { vDebug("vnode list is cleanup"); - taosHashCleanup(tsDnodeVnodesHash); - tsDnodeVnodesHash = NULL; + taosHashCleanup(tsVnodesHash); + tsVnodesHash = NULL; } syncCleanUp(); @@ -348,7 +342,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); - taosHashPut(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); + taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); return TSDB_CODE_SUCCESS; } @@ -440,7 +434,7 @@ void vnodeRelease(void *pVnodeRaw) { tsem_destroy(&pVnode->sem); free(pVnode); - int32_t count = taosHashGetSize(tsDnodeVnodesHash); + int32_t count = taosHashGetSize(tsVnodesHash); vDebug("vgId:%d, vnode is destroyed, vnodes:%d", vgId, count); } @@ -457,7 +451,7 @@ static void vnodeIncRef(void *ptNode) { } void *vnodeAcquire(int32_t vgId) { - SVnodeObj **ppVnode = taosHashGetCB(tsDnodeVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); + SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *)); if (ppVnode == NULL || *ppVnode == NULL) { terrno = TSDB_CODE_VND_INVALID_VGROUP_ID; @@ -496,7 +490,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { } int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash); + SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); while (taosHashIterNext(pIter)) { SVnodeObj **pVnode = taosHashIterGet(pIter); if (pVnode == NULL) continue; @@ -517,7 +511,7 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { void vnodeBuildStatusMsg(void *param) { SStatusMsg *pStatus = param; - SHashMutableIterator *pIter = taosHashCreateIter(tsDnodeVnodesHash); + SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); while (taosHashIterNext(pIter)) { SVnodeObj **pVnode = taosHashIterGet(pIter); @@ -546,7 +540,7 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { static void vnodeCleanUp(SVnodeObj *pVnode) { // remove from hash, so new messages wont be consumed - taosHashRemove(tsDnodeVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); + taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); if (pVnode->status != TAOS_VN_STATUS_INIT) { // it may be in updateing or reset state, then it shall wait diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index fb984f6750..b91f5ba0ab 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -15,13 +15,10 @@ #define _DEFAULT_SOURCE #define _NON_BLOCKING_RETRIEVE 0 - #include "os.h" - #include "tglobal.h" #include "taoserror.h" #include "taosmsg.h" -#include "tcache.h" #include "query.h" #include "trpc.h" #include "tsdb.h" @@ -29,9 +26,9 @@ #include "vnodeInt.h" #include "tqueue.h" -static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pReadMsg); -static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg); -static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg); +static int32_t (*vnodeProcessReadMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *pVnode, SVReadMsg *pRead); +static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead); +static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead); static int32_t vnodeNotifyCurrentQhandle(void* handle, void* qhandle, int32_t vgId); void vnodeInitReadFp(void) { @@ -44,16 +41,16 @@ void vnodeInitReadFp(void) { // still required, or there will be a deadlock, so we don’t do any check here, but put the check codes before the // request enters the queue // -int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) { +int32_t vnodeProcessRead(void *param, SVReadMsg *pRead) { SVnodeObj *pVnode = (SVnodeObj *)param; - int32_t msgType = pReadMsg->msgType; + int32_t msgType = pRead->msgType; if (vnodeProcessReadMsgFp[msgType] == NULL) { vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]); return TSDB_CODE_VND_MSG_NOT_PROCESSED; } - return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); + return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead); } static int32_t vnodeCheckRead(void *param) { @@ -180,27 +177,27 @@ static void vnodeBuildNoResultQueryRsp(SRspRet *pRet) { pRsp->completed = true; } -static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { - void * pCont = pReadMsg->pCont; - int32_t contLen = pReadMsg->contLen; - SRspRet *pRet = &pReadMsg->rspRet; +static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { + void * pCont = pRead->pCont; + int32_t contLen = pRead->contLen; + SRspRet *pRet = &pRead->rspRet; SQueryTableMsg *pQueryTableMsg = (SQueryTableMsg *)pCont; memset(pRet, 0, sizeof(SRspRet)); // qHandle needs to be freed correctly - if (pReadMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont; + if (pRead->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pRead->pCont; killQueryMsg->free = htons(killQueryMsg->free); killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle); - vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcHandle); - assert(pReadMsg->contLen > 0 && killQueryMsg->free == 1); + vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pRead->rpcHandle); + assert(pRead->contLen > 0 && killQueryMsg->free == 1); void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle); if (qhandle == NULL || *qhandle == NULL) { vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle, - pReadMsg->rpcHandle); + pRead->rpcHandle); } else { assert(*qhandle == (void *)killQueryMsg->qhandle); @@ -242,9 +239,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } if (handle != NULL && - vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcHandle); + pRead->rpcHandle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); return pRsp->code; @@ -255,7 +252,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); - code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcHandle); + code = vnodePutItemIntoReadQueue(pVnode, handle, pRead->rpcHandle); if (code != TSDB_CODE_SUCCESS) { pRsp->code = code; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -264,7 +261,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } } else { assert(pCont != NULL); - void **qhandle = (void **)pReadMsg->qhandle; + void **qhandle = (void **)pRead->qhandle; vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); @@ -276,14 +273,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { // build query rsp, the retrieve request has reached here already if (buildRes) { // update the connection info according to the retrieve connection - pReadMsg->rpcHandle = qGetResultRetrieveMsg(*qhandle); - assert(pReadMsg->rpcHandle != NULL); + pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle); + assert(pRead->rpcHandle != NULL); vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, - pReadMsg->rpcHandle); + pRead->rpcHandle); // set the real rsp error code - pReadMsg->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcHandle); + pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle); // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; @@ -308,16 +305,16 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { return code; } -static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { - void * pCont = pReadMsg->pCont; - SRspRet *pRet = &pReadMsg->rspRet; +static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { + void * pCont = pRead->pCont; + SRspRet *pRet = &pRead->rspRet; SRetrieveTableMsg *pRetrieve = pCont; pRetrieve->free = htons(pRetrieve->free); pRetrieve->qhandle = htobe64(pRetrieve->qhandle); vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, - pRetrieve->free, pReadMsg->rpcHandle); + pRetrieve->free, pRead->rpcHandle); memset(pRet, 0, sizeof(SRspRet)); @@ -348,8 +345,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } // register the qhandle to connect to quit query immediate if connection is broken - if (vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcHandle); + if (vnodeNotifyCurrentQhandle(pRead->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pRead->rpcHandle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -359,7 +356,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { bool freeHandle = true; bool buildRes = false; - code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcHandle); + code = qRetrieveQueryResultInfo(*handle, &buildRes, pRead->rpcHandle); if (code != TSDB_CODE_SUCCESS) { // TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -370,7 +367,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { assert(buildRes == true); #if _NON_BLOCKING_RETRIEVE if (!buildRes) { - assert(pReadMsg->rpcHandle != NULL); + assert(pRead->rpcHandle != NULL); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); return TSDB_CODE_QRY_NOT_READY; @@ -378,7 +375,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { #endif // ahandle is the sqlObj pointer - code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcHandle); + code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle); } // If qhandle is not added into vread queue, the query should be completed already or paused with error. diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 3caee2fb0c..f42b359b26 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -19,7 +19,6 @@ #include "taoserror.h" #include "tqueue.h" #include "trpc.h" -#include "tutil.h" #include "tsdb.h" #include "twal.h" #include "tsync.h" @@ -185,7 +184,7 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { SDropSTableMsg *pTable = pCont; - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; vDebug("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId); From 1e34d8ca9a173078e1a82759061c32affbd7b61f Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 6 Nov 2020 01:37:59 +0000 Subject: [PATCH 062/122] minor changes, dont set terrno --- src/util/src/tref.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 99d566e2ab..b998dfd43c 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -398,9 +398,7 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { return -1; } - terrno = 0; hash = rid % pSet->max; - taosLockList(pSet->lockedBy+hash); pNode = pSet->nodeList[hash]; From 73b23e8b8d0506e77799e676fe3adeeaa0c7a794 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 6 Nov 2020 09:48:57 +0800 Subject: [PATCH 063/122] [TD-1961] modify return when call taos_query() --- tests/examples/c/demo.c | 59 ++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 8f8a66a325..59b9c74827 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -22,6 +22,34 @@ #include #include // TAOS header file +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + int main(int argc, char *argv[]) { TAOS * taos; char qstr[1024]; @@ -44,22 +72,26 @@ int main(int argc, char *argv[]) { printf("success to connect to server\n"); - taos_query(taos, "drop database demo"); + //taos_query(taos, "drop database demo"); + queryDB(taos, "drop database if exists demo"); - result = taos_query(taos, "create database demo"); - if (result == NULL) { - printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/); - exit(1); - } + //result = taos_query(taos, "create database demo"); + //if (result == NULL) { + // printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/); + // exit(1); + //} + queryDB(taos, "create database demo"); printf("success to create database\n"); - taos_query(taos, "use demo"); + //taos_query(taos, "use demo"); + queryDB(taos, "use demo"); // create table - if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); - exit(1); - } + //if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) { + // printf("failed to create table, reason:%s\n", taos_errstr(result)); + // exit(1); + //} + queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))"); printf("success to create table\n"); // sleep for one second to make sure table is created on data node @@ -80,8 +112,10 @@ int main(int argc, char *argv[]) { printf("insert row: %i\n", i); } else { printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/); + taos_free_result(result); exit(1); } + taos_free_result(result); //sleep(1); } @@ -91,7 +125,8 @@ int main(int argc, char *argv[]) { sprintf(qstr, "SELECT * FROM m1"); result = taos_query(taos, qstr); if (result == NULL || taos_errno(result) != 0) { - printf("failed to select, reason:%s\n", taos_errstr(result)); + printf("failed to select, reason:%s\n", taos_errstr(result)); + taos_free_result(result); exit(1); } From 86041893003f5b68f444ce2393e55c4ca3af7249 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 6 Nov 2020 10:05:07 +0800 Subject: [PATCH 064/122] [TD-1961] modify return when call taos_query() --- tests/examples/c/apitest.c | 76 +++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 759e16d1de..a2754913b3 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -9,26 +9,40 @@ static void prepare_data(TAOS* taos) { - taos_query(taos, "drop database if exists test;"); + TAOS_RES *result; + result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); usleep(100000); - taos_query(taos, "create database test;"); + result = taos_query(taos, "create database test;"); + taos_free_result(result); usleep(100000); taos_select_db(taos, "test"); - taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + taos_free_result(result); - taos_query(taos, "create table t0 using meters tags(0);"); - taos_query(taos, "create table t1 using meters tags(1);"); - taos_query(taos, "create table t2 using meters tags(2);"); - taos_query(taos, "create table t3 using meters tags(3);"); - taos_query(taos, "create table t4 using meters tags(4);"); - taos_query(taos, "create table t5 using meters tags(5);"); - taos_query(taos, "create table t6 using meters tags(6);"); - taos_query(taos, "create table t7 using meters tags(7);"); - taos_query(taos, "create table t8 using meters tags(8);"); - taos_query(taos, "create table t9 using meters tags(9);"); + result = taos_query(taos, "create table t0 using meters tags(0);"); + taos_free_result(result); + result = taos_query(taos, "create table t1 using meters tags(1);"); + taos_free_result(result); + result = taos_query(taos, "create table t2 using meters tags(2);"); + taos_free_result(result); + result = taos_query(taos, "create table t3 using meters tags(3);"); + taos_free_result(result); + result = taos_query(taos, "create table t4 using meters tags(4);"); + taos_free_result(result); + result = taos_query(taos, "create table t5 using meters tags(5);"); + taos_free_result(result); + result = taos_query(taos, "create table t6 using meters tags(6);"); + taos_free_result(result); + result = taos_query(taos, "create table t7 using meters tags(7);"); + taos_free_result(result); + result = taos_query(taos, "create table t8 using meters tags(8);"); + taos_free_result(result); + result = taos_query(taos, "create table t9 using meters tags(9);"); + taos_free_result(result); - TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" + result = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" " ('2020-01-01 00:01:00.000', 0)" " ('2020-01-01 00:02:00.000', 0)" " t1 values('2020-01-01 00:00:00.000', 0)" @@ -46,10 +60,11 @@ static void prepare_data(TAOS* taos) { " t7 values('2020-01-01 00:01:02.000', 0)" " t8 values('2020-01-01 00:01:02.000', 0)" " t9 values('2020-01-01 00:01:02.000', 0)"); - int affected = taos_affected_rows(res); + int affected = taos_affected_rows(result); if (affected != 18) { printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected); } + taos_free_result(result); // super tables subscription usleep(1000000); } @@ -153,23 +168,30 @@ static void verify_subscribe(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); - taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + TAOS_RES *result; + result = taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); + taos_free_result(result); + result = taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + result = taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); + taos_free_result(result); + result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); // keep progress information and restart subscription taos_unsubscribe(tsub, 1); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + taos_free_result(result); tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); res = taos_consume(tsub); check_row_count(__LINE__, res, 24); @@ -196,7 +218,8 @@ static void verify_subscribe(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); @@ -205,7 +228,8 @@ static void verify_subscribe(TAOS* taos) { int blockFetch = 0; tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000); usleep(2000000); - taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); + taos_free_result(result); usleep(2000000); taos_unsubscribe(tsub, 0); } @@ -213,8 +237,9 @@ static void verify_subscribe(TAOS* taos) { void verify_prepare(TAOS* taos) { TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); usleep(100000); - taos_query(taos, "create database test;"); + result = taos_query(taos, "create database test;"); int code = taos_errno(result); if (code != 0) { @@ -429,7 +454,8 @@ void verify_stream(TAOS* taos) { NULL); printf("waiting for stream data\n"); usleep(100000); - taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); + TAOS_RES* result = taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); + taos_free_result(result); usleep(200000000); taos_close_stream(strm); } From c034940cc84ebfa74bb3391afeebd002376b6459 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 6 Nov 2020 10:07:51 +0800 Subject: [PATCH 065/122] [TD-1961] modify return when call taos_query() --- tests/examples/c/apitest.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index a2754913b3..be60a88ad7 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -150,6 +150,7 @@ static void verify_query(TAOS* taos) { res = taos_query(taos, "select * from meters"); taos_stop_query(res); + taos_free_result(res); } From e4e86609b1e1d16d0f5c1e884ab516e338a41432 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 Nov 2020 10:40:38 +0800 Subject: [PATCH 066/122] TD-1915 --- src/dnode/src/dnodeMPeer.c | 2 +- src/dnode/src/dnodeMRead.c | 2 +- src/dnode/src/dnodeMWrite.c | 2 +- src/dnode/src/dnodePeer.c | 1 + src/inc/mnode.h | 16 +-- src/inc/taosdef.h | 59 +++++------ src/mnode/inc/mnodeDef.h | 8 +- src/mnode/inc/mnodeTable.h | 4 +- src/mnode/inc/mnodeVgroup.h | 4 +- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeTable.c | 174 +++++++++++++++---------------- src/mnode/src/mnodeVgroup.c | 6 +- src/plugins/http/src/httpQueue.c | 2 +- 13 files changed, 142 insertions(+), 140 deletions(-) diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 5c49dc259c..536ebdb268 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -128,7 +128,7 @@ void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) { return; } - SMnodeMsg *pPeer = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg)); + SMnodeMsg *pPeer = taosAllocateQitem(sizeof(SMnodeMsg)); mnodeCreateMsg(pPeer, pMsg); taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); } diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 8208a66e1b..1a54dd1ecd 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -129,7 +129,7 @@ void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) { return; } - SMnodeMsg *pRead = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg)); + SMnodeMsg *pRead = taosAllocateQitem(sizeof(SMnodeMsg)); mnodeCreateMsg(pRead, pMsg); taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); } diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index e628ea1f81..5bf485d610 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -129,7 +129,7 @@ void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { return; } - SMnodeMsg *pWrite = (SMnodeMsg *)taosAllocateQitem(sizeof(SMnodeMsg)); + SMnodeMsg *pWrite = taosAllocateQitem(sizeof(SMnodeMsg)); mnodeCreateMsg(pWrite, pMsg); dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index cbdbc54c25..afa712a965 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -19,6 +19,7 @@ * to dnode. All theses messages are handled from here */ +#define _DEFAULT_SOURCE #include "os.h" #include "taosmsg.h" #include "tglobal.h" diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 5bef7402e3..2bf7718058 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -35,7 +35,13 @@ typedef struct { } SMnodeRsp; typedef struct SMnodeMsg { - SRpcMsg rpcMsg; + struct SAcctObj * pAcct; + struct SDnodeObj *pDnode; + struct SUserObj * pUser; + struct SDbObj * pDb; + struct SVgObj * pVgroup; + struct STableObj *pTable; + struct SSTableObj*pSTable; SMnodeRsp rpcRsp; int8_t received; int8_t successed; @@ -43,13 +49,7 @@ typedef struct SMnodeMsg { int8_t retry; int32_t code; void * pObj; - struct SAcctObj * pAcct; - struct SDnodeObj *pDnode; - struct SUserObj * pUser; - struct SDbObj * pDb; - struct SVgObj * pVgroup; - struct STableObj *pTable; - struct SSuperTableObj *pSTable; + SRpcMsg rpcMsg; } SMnodeMsg; void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index aee60da201..42d4424ebb 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -424,42 +424,43 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_PORT_DNODEDNODE 5 #define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 -#define TSDB_PORT_ARBITRATOR 12 - -#define TAOS_QTYPE_RPC 0 -#define TAOS_QTYPE_FWD 1 -#define TAOS_QTYPE_WAL 2 -#define TAOS_QTYPE_CQ 3 -#define TAOS_QTYPE_QUERY 4 +#define TSDB_PORT_ARBITRATOR 12 typedef enum { - TSDB_SUPER_TABLE = 0, // super table - TSDB_CHILD_TABLE = 1, // table created from super table - TSDB_NORMAL_TABLE = 2, // ordinary table - TSDB_STREAM_TABLE = 3, // table created from stream computing - TSDB_TABLE_MAX = 4 + TAOS_QTYPE_RPC = 0, + TAOS_QTYPE_FWD = 1, + TAOS_QTYPE_WAL = 2, + TAOS_QTYPE_CQ = 3, + TAOS_QTYPE_QUERY = 4 +} EQType; + +typedef enum { + TSDB_SUPER_TABLE = 0, // super table + TSDB_CHILD_TABLE = 1, // table created from super table + TSDB_NORMAL_TABLE = 2, // ordinary table + TSDB_STREAM_TABLE = 3, // table created from stream computing + TSDB_TABLE_MAX = 4 } ETableType; typedef enum { - TSDB_MOD_MNODE, - TSDB_MOD_HTTP, - TSDB_MOD_MONITOR, - TSDB_MOD_MQTT, - TSDB_MOD_MAX + TSDB_MOD_MNODE = 0, + TSDB_MOD_HTTP = 1, + TSDB_MOD_MONITOR = 2, + TSDB_MOD_MQTT = 3, + TSDB_MOD_MAX = 4 } EModuleType; - typedef enum { - TSDB_CHECK_ITEM_NETWORK, - TSDB_CHECK_ITEM_MEM, - TSDB_CHECK_ITEM_CPU, - TSDB_CHECK_ITEM_DISK, - TSDB_CHECK_ITEM_OS, - TSDB_CHECK_ITEM_ACCESS, - TSDB_CHECK_ITEM_VERSION, - TSDB_CHECK_ITEM_DATAFILE, - TSDB_CHECK_ITEM_MAX - } ECheckItemType; - +typedef enum { + TSDB_CHECK_ITEM_NETWORK, + TSDB_CHECK_ITEM_MEM, + TSDB_CHECK_ITEM_CPU, + TSDB_CHECK_ITEM_DISK, + TSDB_CHECK_ITEM_OS, + TSDB_CHECK_ITEM_ACCESS, + TSDB_CHECK_ITEM_VERSION, + TSDB_CHECK_ITEM_DATAFILE, + TSDB_CHECK_ITEM_MAX +} ECheckItemType; #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 4bc840f026..f0227be3ec 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -89,7 +89,7 @@ typedef struct STableObj { int8_t type; } STableObj; -typedef struct SSuperTableObj { +typedef struct SSTableObj { STableObj info; int8_t reserved0[9]; // for fill struct STableObj to 4byte align int16_t nextColId; @@ -104,7 +104,7 @@ typedef struct SSuperTableObj { int32_t numOfTables; SSchema * schema; void * vgHash; -} SSuperTableObj; +} SSTableObj; typedef struct { STableObj info; @@ -122,8 +122,8 @@ typedef struct { int32_t refCount; char* sql; //used by normal table SSchema* schema; //used by normal table - SSuperTableObj *superTable; -} SChildTableObj; + SSTableObj*superTable; +} SCTableObj; typedef struct { int32_t dnodeId; diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index ed0dbe4ecf..7c0077aa60 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -29,8 +29,8 @@ int64_t mnodeGetChildTableNum(); void * mnodeGetTable(char *tableId); void mnodeIncTableRef(void *pTable); void mnodeDecTableRef(void *pTable); -void * mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable); -void * mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable); +void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable); +void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 7aa662b81c..0e6d9dfde4 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -43,8 +43,8 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle); void mnodeAlterVgroup(SVgObj *pVgroup, void *ahandle); int32_t mnodeGetAvailableVgroup(struct SMnodeMsg *pMsg, SVgObj **pVgroup, int32_t *sid); -void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable); -void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable); +void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable); +void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable); void mnodeSendDropVnodeMsg(int32_t vgId, SRpcEpSet *epSet, void *ahandle); void mnodeSendCreateVgroupMsg(SVgObj *pVgroup, void *ahandle); void mnodeSendAlterVgroupMsg(SVgObj *pVgroup); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 6bc144d1e1..6130503558 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -1043,7 +1043,7 @@ void sdbFreeWritequeue() { int32_t sdbWriteToQueue(void *param, void *data, int32_t qtype, void *pMsg) { SWalHead *pHead = data; int32_t size = sizeof(SWalHead) + pHead->len; - SWalHead *pWal = (SWalHead *)taosAllocateQitem(size); + SWalHead *pWal = taosAllocateQitem(size); memcpy(pWal, pHead, size); taosWriteQitem(tsSdbWriteQueue, qtype, pWal); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 00234e33b2..e3a0b38d7c 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -52,9 +52,9 @@ static int32_t tsSuperTableUpdateSize; static void * mnodeGetChildTable(char *tableId); static void * mnodeGetSuperTable(char *tableId); static void * mnodeGetSuperTableByUid(uint64_t uid); -static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable); -static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable); -static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable); +static void mnodeDropAllChildTablesInStable(SSTableObj *pStable); +static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable); +static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable); static int32_t mnodeGetShowTableMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows, void *pConn); @@ -86,9 +86,9 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg); static int32_t mnodeProcessAlterTableMsg(SMnodeMsg *pMsg); static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg); -static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName); +static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName); -static void mnodeDestroyChildTable(SChildTableObj *pTable) { +static void mnodeDestroyChildTable(SCTableObj *pTable) { taosTFree(pTable->info.tableId); taosTFree(pTable->schema); taosTFree(pTable->sql); @@ -101,7 +101,7 @@ static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) { } static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) { - SChildTableObj *pTable = pOper->pObj; + SCTableObj *pTable = pOper->pObj; SVgObj *pVgroup = mnodeGetVgroup(pTable->vgId); if (pVgroup == NULL) { @@ -150,7 +150,7 @@ static int32_t mnodeChildTableActionInsert(SSdbOper *pOper) { } static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) { - SChildTableObj *pTable = pOper->pObj; + SCTableObj *pTable = pOper->pObj; if (pTable->vgId == 0) { return TSDB_CODE_MND_VGROUP_NOT_EXIST; } @@ -186,8 +186,8 @@ static int32_t mnodeChildTableActionDelete(SSdbOper *pOper) { } static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { - SChildTableObj *pNew = pOper->pObj; - SChildTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); + SCTableObj *pNew = pOper->pObj; + SCTableObj *pTable = mnodeGetChildTable(pNew->info.tableId); if (pTable != pNew) { void *oldTableId = pTable->info.tableId; void *oldSql = pTable->sql; @@ -195,7 +195,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { void *oldSTable = pTable->superTable; int32_t oldRefCount = pTable->refCount; - memcpy(pTable, pNew, sizeof(SChildTableObj)); + memcpy(pTable, pNew, sizeof(SCTableObj)); pTable->refCount = oldRefCount; pTable->sql = pNew->sql; @@ -213,7 +213,7 @@ static int32_t mnodeChildTableActionUpdate(SSdbOper *pOper) { } static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) { - SChildTableObj *pTable = pOper->pObj; + SCTableObj *pTable = pOper->pObj; assert(pTable != NULL && pOper->rowData != NULL); int32_t len = strlen(pTable->info.tableId); @@ -244,7 +244,7 @@ static int32_t mnodeChildTableActionEncode(SSdbOper *pOper) { static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) { assert(pOper->rowData != NULL); - SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); + SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; int32_t len = strlen(pOper->rowData); @@ -284,7 +284,7 @@ static int32_t mnodeChildTableActionDecode(SSdbOper *pOper) { static int32_t mnodeChildTableActionRestored() { void *pIter = NULL; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; while (1) { pIter = mnodeGetNextChildTable(pIter, &pTable); @@ -323,7 +323,7 @@ static int32_t mnodeChildTableActionRestored() { } if (pTable->info.type == TSDB_CHILD_TABLE) { - SSuperTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid); + SSTableObj *pSuperTable = mnodeGetSuperTableByUid(pTable->suid); if (pSuperTable == NULL) { mError("ctable:%s, stable:%" PRIu64 " not exist", pTable->info.tableId, pTable->suid); pTable->vgId = 0; @@ -344,14 +344,14 @@ static int32_t mnodeChildTableActionRestored() { } static int32_t mnodeInitChildTables() { - SChildTableObj tObj; + SCTableObj tObj; tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_CTABLE, .tableName = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, - .maxRowSize = sizeof(SChildTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, + .maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, .insertFp = mnodeChildTableActionInsert, @@ -386,7 +386,7 @@ int64_t mnodeGetChildTableNum() { return sdbGetNumOfRows(tsChildTableSdb); } -static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { +static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) { atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { @@ -402,7 +402,7 @@ static void mnodeAddTableIntoStable(SSuperTableObj *pStable, SChildTableObj *pCt } } -static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj *pCtable) { +static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable) { atomic_sub_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) return; @@ -416,7 +416,7 @@ static void mnodeRemoveTableFromStable(SSuperTableObj *pStable, SChildTableObj * mnodeDecVgroupRef(pVgroup); } -static void mnodeDestroySuperTable(SSuperTableObj *pStable) { +static void mnodeDestroySuperTable(SSTableObj *pStable) { if (pStable->vgHash != NULL) { taosHashCleanup(pStable->vgHash); pStable->vgHash = NULL; @@ -432,7 +432,7 @@ static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) { } static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) { - SSuperTableObj *pStable = pOper->pObj; + SSTableObj *pStable = pOper->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL && pDb->status == TSDB_DB_STATUS_READY) { mnodeAddSuperTableIntoDb(pDb); @@ -443,11 +443,11 @@ static int32_t mnodeSuperTableActionInsert(SSdbOper *pOper) { } static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) { - SSuperTableObj *pStable = pOper->pObj; + SSTableObj *pStable = pOper->pObj; SDbObj *pDb = mnodeGetDbByTableId(pStable->info.tableId); if (pDb != NULL) { mnodeRemoveSuperTableFromDb(pDb); - mnodeDropAllChildTablesInStable((SSuperTableObj *)pStable); + mnodeDropAllChildTablesInStable((SSTableObj *)pStable); } mnodeDecDbRef(pDb); @@ -455,8 +455,8 @@ static int32_t mnodeSuperTableActionDelete(SSdbOper *pOper) { } static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { - SSuperTableObj *pNew = pOper->pObj; - SSuperTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); + SSTableObj *pNew = pOper->pObj; + SSTableObj *pTable = mnodeGetSuperTable(pNew->info.tableId); if (pTable != NULL && pTable != pNew) { void *oldTableId = pTable->info.tableId; void *oldSchema = pTable->schema; @@ -464,7 +464,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { int32_t oldRefCount = pTable->refCount; int32_t oldNumOfTables = pTable->numOfTables; - memcpy(pTable, pNew, sizeof(SSuperTableObj)); + memcpy(pTable, pNew, sizeof(SSTableObj)); pTable->vgHash = oldVgHash; pTable->refCount = oldRefCount; @@ -480,7 +480,7 @@ static int32_t mnodeSuperTableActionUpdate(SSdbOper *pOper) { } static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) { - SSuperTableObj *pStable = pOper->pObj; + SSTableObj *pStable = pOper->pObj; assert(pOper->pObj != NULL && pOper->rowData != NULL); int32_t len = strlen(pStable->info.tableId); @@ -504,7 +504,7 @@ static int32_t mnodeSuperTableActionEncode(SSdbOper *pOper) { static int32_t mnodeSuperTableActionDecode(SSdbOper *pOper) { assert(pOper->rowData != NULL); - SSuperTableObj *pStable = (SSuperTableObj *) calloc(1, sizeof(SSuperTableObj)); + SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj)); if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY; int32_t len = strlen(pOper->rowData); @@ -537,14 +537,14 @@ static int32_t mnodeSuperTableActionRestored() { } static int32_t mnodeInitSuperTables() { - SSuperTableObj tObj; + SSTableObj tObj; tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; SSdbTableDesc tableDesc = { .tableId = SDB_TABLE_STABLE, .tableName = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, - .maxRowSize = sizeof(SSuperTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, + .maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN, .refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj, .keyType = SDB_KEY_VAR_STRING, .insertFp = mnodeSuperTableActionInsert, @@ -615,7 +615,7 @@ static void *mnodeGetSuperTable(char *tableId) { } static void *mnodeGetSuperTableByUid(uint64_t uid) { - SSuperTableObj *pStable = NULL; + SSTableObj *pStable = NULL; void *pIter = NULL; while (1) { @@ -647,11 +647,11 @@ void *mnodeGetTable(char *tableId) { return NULL; } -void *mnodeGetNextChildTable(void *pIter, SChildTableObj **pTable) { +void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) { return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable); } -void *mnodeGetNextSuperTable(void *pIter, SSuperTableObj **pTable) { +void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) { return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable); } @@ -765,12 +765,12 @@ static int32_t mnodeProcessDropTableMsg(SMnodeMsg *pMsg) { } if (pMsg->pTable->type == TSDB_SUPER_TABLE) { - SSuperTableObj *pSTable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pSTable = (SSTableObj *)pMsg->pTable; mInfo("app:%p:%p, table:%s, start to drop stable, uid:%" PRIu64 ", numOfChildTables:%d, sizeOfVgList:%d", pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pSTable->uid, pSTable->numOfTables, (int32_t)taosHashGetSize(pSTable->vgHash)); return mnodeProcessDropSuperTableMsg(pMsg); } else { - SChildTableObj *pCTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pCTable = (SCTableObj *)pMsg->pTable; mInfo("app:%p:%p, table:%s, start to drop ctable, vgId:%d tid:%d uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, pDrop->tableId, pCTable->vgId, pCTable->tid, pCTable->uid); return mnodeProcessDropChildTableMsg(pMsg); @@ -816,7 +816,7 @@ static int32_t mnodeProcessTableMetaMsg(SMnodeMsg *pMsg) { } static int32_t mnodeCreateSuperTableCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pTable = (SSTableObj *)pMsg->pTable; assert(pTable); if (code == TSDB_CODE_SUCCESS) { @@ -835,7 +835,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; - SSuperTableObj * pStable = calloc(1, sizeof(SSuperTableObj)); + SSTableObj * pStable = calloc(1, sizeof(SSTableObj)); if (pStable == NULL) { mError("app:%p:%p, table:%s, failed to create, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -878,7 +878,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { .type = SDB_OPER_GLOBAL, .table = tsSuperTableSdb, .pObj = pStable, - .rowSize = sizeof(SSuperTableObj) + schemaSize, + .rowSize = sizeof(SSTableObj) + schemaSize, .pMsg = pMsg, .writeCb = mnodeCreateSuperTableCb }; @@ -894,7 +894,7 @@ static int32_t mnodeProcessCreateSuperTableMsg(SMnodeMsg *pMsg) { } static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pTable = (SSTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, stable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); } else { @@ -907,7 +907,7 @@ static int32_t mnodeDropSuperTableCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { if (pMsg == NULL) return TSDB_CODE_MND_APP_ERROR; - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash); while (taosHashIterNext(pIter)) { @@ -950,7 +950,7 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { return code; } -static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char *tagName) { +static int32_t mnodeFindSuperTableTagIndex(SSTableObj *pStable, const char *tagName) { SSchema *schema = (SSchema *) pStable->schema; for (int32_t tag = 0; tag < pStable->numOfTags; tag++) { if (strcasecmp(schema[pStable->numOfColumns + tag].name, tagName) == 0) { @@ -962,7 +962,7 @@ static int32_t mnodeFindSuperTableTagIndex(SSuperTableObj *pStable, const char * } static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, add tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); @@ -970,7 +970,7 @@ static int32_t mnodeAddSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { } static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t ntags) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (pStable->numOfTags + ntags > TSDB_MAX_TAGS) { mError("app:%p:%p, stable:%s, add tag, too many tags", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId); return TSDB_CODE_MND_TOO_MANY_TAGS; @@ -1018,14 +1018,14 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t } static int32_t mnodeDropSuperTableTagCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, drop tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; } static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableTagIndex(pStable, tagName); if (col < 0) { mError("app:%p:%p, stable:%s, drop tag, tag:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, @@ -1052,14 +1052,14 @@ static int32_t mnodeDropSuperTableTag(SMnodeMsg *pMsg, char *tagName) { } static int32_t mnodeModifySuperTableTagNameCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, modify tag result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; } static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, char *newTagName) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableTagIndex(pStable, oldTagName); if (col < 0) { mError("app:%p:%p, stable:%s, failed to modify table tag, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, @@ -1095,7 +1095,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c return sdbUpdateRow(&oper); } -static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *colName) { +static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName) { SSchema *schema = (SSchema *) pStable->schema; for (int32_t col = 0; col < pStable->numOfColumns; col++) { if (strcasecmp(schema[col].name, colName) == 0) { @@ -1107,7 +1107,7 @@ static int32_t mnodeFindSuperTableColumnIndex(SSuperTableObj *pStable, char *col } static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, add column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; @@ -1115,7 +1115,7 @@ static int32_t mnodeAddSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) { SDbObj *pDb = pMsg->pDb; - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (ncols <= 0) { mError("app:%p:%p, stable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, ncols); return TSDB_CODE_MND_APP_ERROR; @@ -1170,7 +1170,7 @@ static int32_t mnodeAddSuperTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32 } static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, delete column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; @@ -1178,7 +1178,7 @@ static int32_t mnodeDropSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { SDbObj *pDb = pMsg->pDb; - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableColumnIndex(pStable, colName); if (col <= 0) { mError("app:%p:%p, stable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, @@ -1215,14 +1215,14 @@ static int32_t mnodeDropSuperTableColumn(SMnodeMsg *pMsg, char *colName) { } static int32_t mnodeChangeSuperTableColumnCb(SMnodeMsg *pMsg, int32_t code) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; mLInfo("app:%p:%p, stable %s, change column result:%s", pMsg->rpcMsg.ahandle, pMsg, pStable->info.tableId, tstrerror(code)); return code; } static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) { - SSuperTableObj *pStable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pStable = (SSTableObj *)pMsg->pTable; int32_t col = mnodeFindSuperTableColumnIndex(pStable, oldName); if (col < 0) { mError("app:%p:%p, stable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, @@ -1321,7 +1321,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, int32_t numOfRows = 0; char * pWrite; int32_t cols = 0; - SSuperTableObj *pTable = NULL; + SSTableObj *pTable = NULL; char prefix[64] = {0}; int32_t prefixLen; @@ -1399,7 +1399,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows, void mnodeDropAllSuperTables(SDbObj *pDropDb) { void * pIter= NULL; int32_t numOfTables = 0; - SSuperTableObj *pTable = NULL; + SSTableObj *pTable = NULL; char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); @@ -1430,7 +1430,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } -static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pTable) { +static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns + pTable->numOfTags; assert(numOfCols <= TSDB_MAX_COLUMNS); @@ -1446,7 +1446,7 @@ static int32_t mnodeSetSchemaFromSuperTable(SSchema *pSchema, SSuperTableObj *pT } static int32_t mnodeGetSuperTableMeta(SMnodeMsg *pMsg) { - SSuperTableObj *pTable = (SSuperTableObj *)pMsg->pTable; + SSTableObj *pTable = (SSTableObj *)pMsg->pTable; STableMetaMsg *pMeta = rpcMallocCont(sizeof(STableMetaMsg) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16)); if (pMeta == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -1479,7 +1479,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { int32_t contLen = sizeof(SSTableVgroupRspMsg) + 32 * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg); for (int32_t i = 0; i < numOfTable; ++i) { char *stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; - SSuperTableObj *pTable = mnodeGetSuperTable(stableName); + SSTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable != NULL && pTable->vgHash != NULL) { contLen += (taosHashGetSize(pTable->vgHash) * sizeof(SVgroupMsg) + sizeof(SVgroupsMsg)); } @@ -1497,7 +1497,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { for (int32_t i = 0; i < numOfTable; ++i) { char * stableName = (char *)pInfo + sizeof(SSTableVgroupMsg) + (TSDB_TABLE_FNAME_LEN)*i; - SSuperTableObj *pTable = mnodeGetSuperTable(stableName); + SSTableObj *pTable = mnodeGetSuperTable(stableName); if (pTable == NULL) { mError("app:%p:%p, stable:%s, not exist while get stable vgroup info", pMsg->rpcMsg.ahandle, pMsg, stableName); mnodeDecTableRef(pTable); @@ -1569,7 +1569,7 @@ static void mnodeProcessDropSuperTableRsp(SRpcMsg *rpcMsg) { mInfo("drop stable rsp received, result:%s", tstrerror(rpcMsg->code)); } -static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableObj *pTable) { +static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SCTableObj *pTable) { STagData * pTagData = NULL; int32_t tagDataLen = 0; int32_t totalCols = 0; @@ -1643,7 +1643,7 @@ static void *mnodeBuildCreateChildTableMsg(SCMCreateTableMsg *pMsg, SChildTableO } static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; assert(pTable); mDebug("app:%p:%p, table:%s, created in mnode, vgId:%d sid:%d, uid:%" PRIu64, pMsg->rpcMsg.ahandle, pMsg, @@ -1669,7 +1669,7 @@ static int32_t mnodeDoCreateChildTableFp(SMnodeMsg *pMsg) { } static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; assert(pTable); @@ -1699,7 +1699,7 @@ static int32_t mnodeDoCreateChildTableCb(SMnodeMsg *pMsg, int32_t code) { static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) { SVgObj *pVgroup = pMsg->pVgroup; SCMCreateTableMsg *pCreate = pMsg->rpcMsg.pCont; - SChildTableObj *pTable = calloc(1, sizeof(SChildTableObj)); + SCTableObj *pTable = calloc(1, sizeof(SCTableObj)); if (pTable == NULL) { mError("app:%p:%p, table:%s, failed to alloc memory", pMsg->rpcMsg.ahandle, pMsg, pCreate->tableId); return TSDB_CODE_MND_OUT_OF_MEMORY; @@ -1842,7 +1842,7 @@ static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) { } static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; mLInfo("app:%p:%p, ctable:%s, is dropped from sdb", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); SMDDropTableMsg *pDrop = rpcMallocCont(sizeof(SMDDropTableMsg)); @@ -1880,7 +1880,7 @@ static int32_t mnodeSendDropChildTableMsg(SMnodeMsg *pMsg, bool needReturn) { static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { if (code != TSDB_CODE_SUCCESS) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; mError("app:%p:%p, ctable:%s, failed to drop, sdb error", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId); return code; } @@ -1889,7 +1889,7 @@ static int32_t mnodeDropChildTableCb(SMnodeMsg *pMsg, int32_t code) { } static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; if (pMsg->pVgroup == NULL) pMsg->pVgroup = mnodeGetVgroup(pTable->vgId); if (pMsg->pVgroup == NULL) { mError("app:%p:%p, table:%s, failed to drop ctable, vgroup not exist", pMsg->rpcMsg.ahandle, pMsg, @@ -1914,7 +1914,7 @@ static int32_t mnodeProcessDropChildTableMsg(SMnodeMsg *pMsg) { return code; } -static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *colName) { +static int32_t mnodeFindNormalTableColumnIndex(SCTableObj *pTable, char *colName) { SSchema *schema = (SSchema *) pTable->schema; for (int32_t col = 0; col < pTable->numOfColumns; col++) { if (strcasecmp(schema[col].name, colName) == 0) { @@ -1926,7 +1926,7 @@ static int32_t mnodeFindNormalTableColumnIndex(SChildTableObj *pTable, char *col } static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; if (code != TSDB_CODE_SUCCESS) { mError("app:%p:%p, ctable %s, failed to alter column, reason:%s", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, tstrerror(code)); @@ -1965,7 +1965,7 @@ static int32_t mnodeAlterNormalTableColumnCb(SMnodeMsg *pMsg, int32_t code) { } static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int32_t ncols) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; SDbObj *pDb = pMsg->pDb; if (ncols <= 0) { mError("app:%p:%p, ctable:%s, add column, ncols:%d <= 0", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, ncols); @@ -2014,7 +2014,7 @@ static int32_t mnodeAddNormalTableColumn(SMnodeMsg *pMsg, SSchema schema[], int3 static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { SDbObj *pDb = pMsg->pDb; - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; int32_t col = mnodeFindNormalTableColumnIndex(pTable, colName); if (col <= 0) { mError("app:%p:%p, ctable:%s, drop column, column:%s not exist", pMsg->rpcMsg.ahandle, pMsg, pTable->info.tableId, @@ -2046,7 +2046,7 @@ static int32_t mnodeDropNormalTableColumn(SMnodeMsg *pMsg, char *colName) { } static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char *newName) { - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; int32_t col = mnodeFindNormalTableColumnIndex(pTable, oldName); if (col < 0) { mError("app:%p:%p, ctable:%s, change column, oldName: %s, newName: %s", pMsg->rpcMsg.ahandle, pMsg, @@ -2082,7 +2082,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char return sdbUpdateRow(&oper); } -static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *pTable) { +static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SCTableObj *pTable) { int32_t numOfCols = pTable->numOfColumns; for (int32_t i = 0; i < numOfCols; ++i) { strcpy(pSchema->name, pTable->schema[i].name); @@ -2097,7 +2097,7 @@ static int32_t mnodeSetSchemaFromNormalTable(SSchema *pSchema, SChildTableObj *p static int32_t mnodeDoGetChildTableMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta) { SDbObj *pDb = pMsg->pDb; - SChildTableObj *pTable = (SChildTableObj *)pMsg->pTable; + SCTableObj *pTable = (SCTableObj *)pMsg->pTable; pMeta->uid = htobe64(pTable->uid); pMeta->tid = htonl(pTable->tid); @@ -2203,7 +2203,7 @@ static int32_t mnodeGetChildTableMeta(SMnodeMsg *pMsg) { void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { void * pIter = NULL; int32_t numOfTables = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; mInfo("vgId:%d, all child tables will be dropped from sdb", pVgroup->vgId); @@ -2231,7 +2231,7 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { void mnodeDropAllChildTables(SDbObj *pDropDb) { void * pIter = NULL; int32_t numOfTables = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; char prefix[64] = {0}; tstrncpy(prefix, pDropDb->name, 64); @@ -2261,10 +2261,10 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); } -static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) { +static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { void * pIter = NULL; int32_t numOfTables = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; mInfo("stable:%s uid:%" PRIu64 ", all child tables:%d will be dropped from sdb", pStable->info.tableId, pStable->uid, pStable->numOfTables); @@ -2292,11 +2292,11 @@ static void mnodeDropAllChildTablesInStable(SSuperTableObj *pStable) { } #if 0 -static SChildTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { +static SCTableObj* mnodeGetTableByPos(int32_t vnode, int32_t tid) { SVgObj *pVgroup = mnodeGetVgroup(vnode); if (pVgroup == NULL) return NULL; - SChildTableObj *pTable = pVgroup->tableList[tid - 1]; + SCTableObj *pTable = pVgroup->tableList[tid - 1]; mnodeIncTableRef((STableObj *)pTable); mnodeDecVgroupRef(pVgroup); @@ -2314,7 +2314,7 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { mDebug("app:%p:%p, dnode:%d, vgId:%d sid:%d, receive table config msg", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId, pCfg->vgId, pCfg->sid); - SChildTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid); + SCTableObj *pTable = mnodeGetTableByPos(pCfg->vgId, pCfg->sid); if (pTable == NULL) { mError("app:%p:%p, dnode:%d, vgId:%d sid:%d, table not found", pMsg->rpcMsg.ahandle, pMsg, pCfg->dnodeId, pCfg->vgId, pCfg->sid); @@ -2322,7 +2322,7 @@ static int32_t mnodeProcessTableCfgMsg(SMnodeMsg *pMsg) { } SMDCreateTableMsg *pCreate = NULL; - pCreate = mnodeBuildCreateChildTableMsg(NULL, (SChildTableObj *)pTable); + pCreate = mnodeBuildCreateChildTableMsg(NULL, (SCTableObj *)pTable); mnodeDecTableRef(pTable); if (pCreate == NULL) return terrno; @@ -2340,7 +2340,7 @@ static void mnodeProcessDropChildTableRsp(SRpcMsg *rpcMsg) { SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; - SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; assert(pTable); mInfo("app:%p:%p, table:%s, drop table rsp received, vgId:%d sid:%d uid:%" PRIu64 ", thandle:%p result:%s", @@ -2381,7 +2381,7 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; - SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; assert(pTable); // If the table is deleted by another thread during creation, stop creating and send drop msg to vnode @@ -2445,7 +2445,7 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { SMnodeMsg *mnodeMsg = rpcMsg->ahandle; mnodeMsg->received++; - SChildTableObj *pTable = (SChildTableObj *)mnodeMsg->pTable; + SCTableObj *pTable = (SCTableObj *)mnodeMsg->pTable; assert(pTable); if (rpcMsg->code == TSDB_CODE_SUCCESS || rpcMsg->code == TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { @@ -2483,7 +2483,7 @@ static int32_t mnodeProcessMultiTableMetaMsg(SMnodeMsg *pMsg) { for (int32_t t = 0; t < pInfo->numOfTables; ++t) { char * tableId = (char *)(pInfo->tableIds + t * TSDB_TABLE_FNAME_LEN); - SChildTableObj *pTable = mnodeGetChildTable(tableId); + SCTableObj *pTable = mnodeGetChildTable(tableId); if (pTable == NULL) continue; if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDbByTableId(tableId); @@ -2607,7 +2607,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows int32_t cols = 0; int32_t numOfRows = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; @@ -2843,7 +2843,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro } int32_t numOfRows = 0; - SChildTableObj *pTable = NULL; + SCTableObj *pTable = NULL; SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER; char prefix[64] = {0}; diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index fada760cb7..b7ebe39635 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -694,7 +694,7 @@ static bool mnodeFilterVgroups(SVgObj *pVgroup, STableObj *pTable) { return true; } - SChildTableObj *pCTable = (SChildTableObj *)pTable; + SCTableObj *pCTable = (SCTableObj *)pTable; if (pVgroup->vgId == pCTable->vgId) { return true; } else { @@ -791,7 +791,7 @@ static int32_t mnodeRetrieveVgroups(SShowObj *pShow, char *data, int32_t rows, v return numOfRows; } -void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { +void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SCTableObj *pTable) { int32_t idPoolSize = taosIdPoolMaxSize(pVgroup->idPool); if (pTable->tid > idPoolSize) { mnodeAllocVgroupIdPool(pVgroup); @@ -807,7 +807,7 @@ void mnodeAddTableIntoVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { } } -void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SChildTableObj *pTable) { +void mnodeRemoveTableFromVgroup(SVgObj *pVgroup, SCTableObj *pTable) { if (pTable->tid >= 1) { taosFreeId(pVgroup->idPool, pTable->tid); pVgroup->numOfTables--; diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 43a8ddbd1a..0fb055972a 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -49,7 +49,7 @@ static taos_queue tsHttpQueue; void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) { if (tsHttpQueue != NULL) { - SHttpResult *pMsg = (SHttpResult *)taosAllocateQitem(sizeof(SHttpResult)); + SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult)); pMsg->param = param; pMsg->result = result; pMsg->numOfRows = numOfRows; From d78c54fb0b1cd8433dbdb3cfd3354fb313c179b7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 Nov 2020 11:40:12 +0800 Subject: [PATCH 067/122] TD-1915 --- src/dnode/src/dnodeMPeer.c | 9 +++--- src/dnode/src/dnodeMRead.c | 9 +++--- src/dnode/src/dnodeMWrite.c | 14 ++++----- src/dnode/src/dnodeMgmt.c | 60 ++++++++++++++++++++++--------------- src/inc/mnode.h | 3 +- src/mnode/src/mnodeInt.c | 14 ++++++--- 6 files changed, 62 insertions(+), 47 deletions(-) diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 536ebdb268..b5ecc4930f 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -124,13 +124,12 @@ void dnodeFreeMPeerQueue() { void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMPeerQueue == NULL) { dnodeSendRedirectMsg(pMsg, false); - rpcFreeCont(pMsg->pCont); - return; + } else { + SMnodeMsg *pPeer = mnodeCreateMsg(pMsg); + taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); } - SMnodeMsg *pPeer = taosAllocateQitem(sizeof(SMnodeMsg)); - mnodeCreateMsg(pPeer, pMsg); - taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); + rpcFreeCont(pMsg->pCont); } static void dnodeFreeMPeerMsg(SMnodeMsg *pPeer) { diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 1a54dd1ecd..c14c7a8158 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -125,13 +125,12 @@ void dnodeFreeMReadQueue() { void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMReadQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); - rpcFreeCont(pMsg->pCont); - return; + } else { + SMnodeMsg *pRead = mnodeCreateMsg(pMsg); + taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); } - SMnodeMsg *pRead = taosAllocateQitem(sizeof(SMnodeMsg)); - mnodeCreateMsg(pRead, pMsg); - taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); + rpcFreeCont(pMsg->pCont); } static void dnodeFreeMReadMsg(SMnodeMsg *pRead) { diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 5bf485d610..3940a251d4 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -125,16 +125,14 @@ void dnodeFreeMWritequeue() { void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { if (!mnodeIsRunning() || tsMWriteQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); - rpcFreeCont(pMsg->pCont); - return; + } else { + SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); + dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, + taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); + taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } - SMnodeMsg *pWrite = taosAllocateQitem(sizeof(SMnodeMsg)); - mnodeCreateMsg(pWrite, pMsg); - - dDebug("app:%p:%p, msg:%s is put into mwrite queue:%p", pWrite->rpcMsg.ahandle, pWrite, - taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); - taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); + rpcFreeCont(pMsg->pCont); } static void dnodeFreeMWriteMsg(SMnodeMsg *pWrite) { diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 4a5dc31a9f..dcb48f7833 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -47,6 +47,11 @@ typedef struct { int32_t * vnodeList; } SOpenVnodeThread; +typedef struct { + SRpcMsg rpcMsg; + char pCont[]; +} SMgmtMsg; + void * tsDnodeTmr = NULL; static void * tsStatusTimer = NULL; static uint32_t tsRebootTime; @@ -172,38 +177,46 @@ void dnodeCleanupMgmt() { vnodeCleanupResources(); } -void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) { - void *item; - - item = taosAllocateQitem(sizeof(SRpcMsg)); - if (item) { - memcpy(item, pMsg, sizeof(SRpcMsg)); - taosWriteQitem(tsMgmtQueue, 1, item); - } else { - SRpcMsg rsp = { - .handle = pMsg->handle, - .pCont = NULL, - .code = TSDB_CODE_DND_OUT_OF_MEMORY - }; - - rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); +static int32_t dnodeWriteToMgmtQueue(SRpcMsg *pMsg) { + int32_t size = sizeof(SMgmtMsg) + pMsg->contLen; + SMgmtMsg *pMgmt = taosAllocateQitem(size); + if (pMgmt == NULL) { + return TSDB_CODE_DND_OUT_OF_MEMORY; } + + pMgmt->rpcMsg = *pMsg; + pMgmt->rpcMsg.pCont = pMgmt->pCont; + memcpy(pMgmt->pCont, pMsg->pCont, pMsg->contLen); + taosWriteQitem(tsMgmtQueue, TAOS_QTYPE_RPC, pMgmt); + + return TSDB_CODE_SUCCESS; +} + +void dnodeDispatchToMgmtQueue(SRpcMsg *pMsg) { + int32_t code = dnodeWriteToMgmtQueue(pMsg); + if (code != TSDB_CODE_SUCCESS) { + SRpcMsg rsp = {.handle = pMsg->handle, .code = code}; + rpcSendResponse(&rsp); + } + + rpcFreeCont(pMsg->pCont); } static void *dnodeProcessMgmtQueue(void *param) { - SRpcMsg *pMsg; - SRpcMsg rsp = {0}; - int type; - void * handle; + SMgmtMsg *pMgmt; + SRpcMsg * pMsg; + SRpcMsg rsp = {0}; + int32_t qtype; + void * handle; while (1) { - if (taosReadQitemFromQset(tsMgmtQset, &type, (void **) &pMsg, &handle) == 0) { + if (taosReadQitemFromQset(tsMgmtQset, &qtype, (void **)&pMgmt, &handle) == 0) { dDebug("qset:%p, dnode mgmt got no message from qset, exit", tsMgmtQset); break; } - dDebug("%p, msg:%s will be processed", pMsg->ahandle, taosMsg[pMsg->msgType]); + pMsg = &pMgmt->rpcMsg; + dDebug("%p, msg:%p:%s will be processed", pMsg->ahandle, pMgmt, taosMsg[pMsg->msgType]); if (dnodeProcessMgmtMsgFp[pMsg->msgType]) { rsp.code = (*dnodeProcessMgmtMsgFp[pMsg->msgType])(pMsg); } else { @@ -211,10 +224,9 @@ static void *dnodeProcessMgmtQueue(void *param) { } rsp.handle = pMsg->handle; - rsp.pCont = NULL; + rsp.pCont = NULL; rpcSendResponse(&rsp); - rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 2bf7718058..128e4d35a4 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -50,9 +50,10 @@ typedef struct SMnodeMsg { int32_t code; void * pObj; SRpcMsg rpcMsg; + char pCont[]; } SMnodeMsg; -void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg); +void * mnodeCreateMsg(SRpcMsg *pRpcMsg); int32_t mnodeInitMsg(SMnodeMsg *pMsg); void mnodeCleanupMsg(SMnodeMsg *pMsg); diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c index fb1b8741a9..98ba9d1e14 100644 --- a/src/mnode/src/mnodeInt.c +++ b/src/mnode/src/mnodeInt.c @@ -18,7 +18,7 @@ #include "taosmsg.h" #include "taoserror.h" #include "trpc.h" -#include "tcache.h" +#include "tqueue.h" #include "mnode.h" #include "dnode.h" #include "mnodeDef.h" @@ -34,8 +34,15 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -void mnodeCreateMsg(SMnodeMsg *pMsg, SRpcMsg *rpcMsg) { - pMsg->rpcMsg = *rpcMsg; +void *mnodeCreateMsg(SRpcMsg *pRpcMsg) { + int32_t size = sizeof(SMnodeMsg) + pRpcMsg->contLen; + SMnodeMsg *pMsg = taosAllocateQitem(size); + + pMsg->rpcMsg = *pRpcMsg; + pMsg->rpcMsg.pCont = pMsg->pCont; + memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen); + + return pMsg; } int32_t mnodeInitMsg(SMnodeMsg *pMsg) { @@ -54,7 +61,6 @@ int32_t mnodeInitMsg(SMnodeMsg *pMsg) { void mnodeCleanupMsg(SMnodeMsg *pMsg) { if (pMsg != NULL) { - if (pMsg->rpcMsg.pCont) rpcFreeCont(pMsg->rpcMsg.pCont); if (pMsg->pUser) mnodeDecUserRef(pMsg->pUser); if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb); if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup); From a191917452d021d1741b1a3359f6a8fbd99bc4f2 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 6 Nov 2020 03:59:43 +0000 Subject: [PATCH 068/122] change head file --- src/util/inc/tfile.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index 00b2fd6c32..981ee7cecb 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -22,19 +22,19 @@ extern "C" { #include +// init taos file module +int tfinit(); + +// clean up taos file module +void tfcleanup(); + // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused - int64_t tfopen(const char *pathname, int flags); int64_t tfclose(int64_t tfd); ssize_t tfwrite(int64_t tfd, const void *buf, size_t count); ssize_t tfread(int64_t tfd, void *buf, size_t count); -// init taos file module -int tfinit(); - -// clean up taos fle module -void tfcleanup(); #ifdef __cplusplus } From 7fa7be7a8a10a1d36bdc88afca8256e4314036fc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 Nov 2020 13:28:12 +0800 Subject: [PATCH 069/122] TD-1915 --- src/mnode/src/mnodeInt.c | 3 +++ src/mnode/src/mnodeTable.c | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c index 98ba9d1e14..d3f2407386 100644 --- a/src/mnode/src/mnodeInt.c +++ b/src/mnode/src/mnodeInt.c @@ -61,6 +61,9 @@ int32_t mnodeInitMsg(SMnodeMsg *pMsg) { void mnodeCleanupMsg(SMnodeMsg *pMsg) { if (pMsg != NULL) { + if (pMsg->rpcMsg.pCont != pMsg->pCont) { + tfree(pMsg->rpcMsg.pCont); + } if (pMsg->pUser) mnodeDecUserRef(pMsg->pUser); if (pMsg->pDb) mnodeDecDbRef(pMsg->pDb); if (pMsg->pVgroup) mnodeDecVgroupRef(pMsg->pVgroup); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index e3a0b38d7c..d27688f8a9 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -2156,7 +2156,7 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { } int32_t contLen = sizeof(SCMCreateTableMsg) + offsetof(STagData, data) + tagLen; - SCMCreateTableMsg *pCreateMsg = rpcMallocCont(contLen); + SCMCreateTableMsg *pCreateMsg = calloc(1, contLen); if (pCreateMsg == NULL) { mError("app:%p:%p, table:%s, failed to create table while get meta info, no enough memory", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId); @@ -2174,11 +2174,13 @@ static int32_t mnodeAutoCreateChildTable(SMnodeMsg *pMsg) { mDebug("app:%p:%p, table:%s, start to create on demand, tagLen:%d stable:%s", pMsg->rpcMsg.ahandle, pMsg, pInfo->tableId, tagLen, pTags->name); - rpcFreeCont(pMsg->rpcMsg.pCont); + if (pMsg->rpcMsg.pCont != pMsg->pCont) { + tfree(pMsg->rpcMsg.pCont); + } pMsg->rpcMsg.msgType = TSDB_MSG_TYPE_CM_CREATE_TABLE; pMsg->rpcMsg.pCont = pCreateMsg; pMsg->rpcMsg.contLen = contLen; - + return TSDB_CODE_MND_ACTION_NEED_REPROCESSED; } From 4735b224ef6a52267a8ca5e9a7a9924c188adb05 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 Nov 2020 06:25:06 +0000 Subject: [PATCH 070/122] change read/write to taosRead/Write --- src/util/inc/tfile.h | 9 ++++----- src/util/src/tfile.c | 25 +++++++++++-------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index 981ee7cecb..10b7c1df35 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -23,18 +23,17 @@ extern "C" { #include // init taos file module -int tfinit(); +int32_t tfinit(); // clean up taos file module void tfcleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused -int64_t tfopen(const char *pathname, int flags); +int64_t tfopen(const char *pathname, int32_t flags); int64_t tfclose(int64_t tfd); -ssize_t tfwrite(int64_t tfd, const void *buf, size_t count); -ssize_t tfread(int64_t tfd, void *buf, size_t count); - +int64_t tfwrite(int64_t tfd, void *buf, int64_t count); +int64_t tfread(int64_t tfd, void *buf, int64_t count); #ifdef __cplusplus } diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 4807fea0d0..27ba30fe81 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -19,13 +19,13 @@ #include "tutil.h" #include "tref.h" -static int tsFileRsetId = -1; +static int32_t tsFileRsetId = -1; static void taosCloseFile(void *p) { - close((int)(uintptr_t)p); + close((int32_t)(uintptr_t)p); } -int tfinit() { +int32_t tfinit() { tsFileRsetId = taosOpenRef(2000, taosCloseFile); return tsFileRsetId; } @@ -35,8 +35,8 @@ void tfcleanup() { tsFileRsetId = -1; } -int64_t tfopen(const char *pathname, int flags) { - int fd = open(pathname, flags); +int64_t tfopen(const char *pathname, int32_t flags) { + int32_t fd = open(pathname, flags); if (fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -54,31 +54,28 @@ int64_t tfclose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } -ssize_t tfwrite(int64_t tfd, const void *buf, size_t count) { - +int64_t tfwrite(int64_t tfd, void *buf, int64_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; - int fd = (int)(uintptr_t)p; + int32_t fd = (int32_t)(uintptr_t)p; - ssize_t ret = write(fd, buf, (uint32_t)count); + int64_t ret = taosWrite(fd, buf, count); if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseRef(tsFileRsetId, tfd); return ret; } -ssize_t tfread(int64_t tfd, void *buf, size_t count) { - +int64_t tfread(int64_t tfd, void *buf, int64_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; - int fd = (int)(uintptr_t)p; + int32_t fd = (int32_t)(uintptr_t)p; - ssize_t ret = read(fd, buf, (uint32_t)count); + int64_t ret = taosRead(fd, buf, count); if (ret < 0) terrno = TAOS_SYSTEM_ERROR(errno); taosReleaseRef(tsFileRsetId, tfd); return ret; } - From a98a25e45ba853726e1ddc92790cd0fad100fa06 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 6 Nov 2020 14:26:52 +0800 Subject: [PATCH 071/122] [TD-1966]fix test case error about valgrind --- Jenkinsfile | 1 - tests/pytest/handle_crash_gen_val_log.sh | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8bf7e435fd..eb4b56ad21 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -89,7 +89,6 @@ pipeline { catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { sh ''' cd ${WKC}/tests/pytest - ./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4 ./handle_crash_gen_val_log.sh ''' } diff --git a/tests/pytest/handle_crash_gen_val_log.sh b/tests/pytest/handle_crash_gen_val_log.sh index 1a4c12a16c..2d48de65c9 100755 --- a/tests/pytest/handle_crash_gen_val_log.sh +++ b/tests/pytest/handle_crash_gen_val_log.sh @@ -5,7 +5,9 @@ GREEN='\033[1;32m' GREEN_DARK='\033[0;32m' GREEN_UNDERLINE='\033[4;32m' NC='\033[0m' - +nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null & +./crash_gen.sh --valgrind -p -t 10 -s 100 -b 4 +pidof taosd|xargs kill grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'` From 0670f28138ffa157a31fff1a01b743b4c171a1be Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 6 Nov 2020 14:37:53 +0800 Subject: [PATCH 072/122] [TD-1937] add test case for TD-1937 --- tests/pytest/fulltest.sh | 16 ++- tests/pytest/functions/function_percentile.py | 13 +- tests/pytest/updatetest.sh | 10 -- .../script/general/parser/join_multivnode.sim | 114 +----------------- tests/script/general/parser/stream_on_sys.sim | 12 +- 5 files changed, 36 insertions(+), 129 deletions(-) delete mode 100644 tests/pytest/updatetest.sh diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index b14321a4ef..b555a27c93 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -161,7 +161,7 @@ python3 ./test.py -f stream/metric_1.py python3 ./test.py -f stream/new.py python3 ./test.py -f stream/stream1.py python3 ./test.py -f stream/stream2.py -python3 ./test.py -f stream/parser.py +#python3 ./test.py -f stream/parser.py python3 ./test.py -f stream/history.py #alter table @@ -207,3 +207,17 @@ python3 test.py -f tools/taosdemo.py python3 test.py -f subscribe/singlemeter.py #python3 test.py -f subscribe/stability.py python3 test.py -f subscribe/supertable.py + + +# update +python3 ./test.py -f update/allow_update.py +python3 ./test.py -f update/allow_update-0.py +python3 ./test.py -f update/append_commit_data.py +python3 ./test.py -f update/append_commit_last-0.py +python3 ./test.py -f update/append_commit_last.py +python3 ./test.py -f update/merge_commit_data.py +python3 ./test.py -f update/merge_commit_data-0.py +python3 ./test.py -f update/merge_commit_data2.py +python3 ./test.py -f update/merge_commit_data2_update0.py +python3 ./test.py -f update/merge_commit_last-0.py +python3 ./test.py -f update/merge_commit_last.py diff --git a/tests/pytest/functions/function_percentile.py b/tests/pytest/functions/function_percentile.py index aaeb94372e..e63d65f2e6 100644 --- a/tests/pytest/functions/function_percentile.py +++ b/tests/pytest/functions/function_percentile.py @@ -130,8 +130,19 @@ class TDTestCase: tdSql.query("select percentile(col6, 100) from test") tdSql.checkData(0, 0, np.percentile(floatData, 100)) tdSql.query("select apercentile(col6, 100) from test") - print("apercentile result: %s" % tdSql.getData(0, 0)) + print("apercentile result: %s" % tdSql.getData(0, 0)) + tdSql.execute("create table meters (ts timestamp, voltage int) tags(loc nchar(20))") + tdSql.execute("create table t0 using meters tags('beijing')") + tdSql.execute("create table t1 using meters tags('shanghai')") + for i in range(self.rowNum): + tdSql.execute("insert into t0 values(%d, %d)" % (self.ts + i, i + 1)) + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts + i, i + 1)) + + tdSql.error("select percentile(voltage, 20) from meters") + tdSql.query("select apercentile(voltage, 20) from meters") + print("apercentile result: %s" % tdSql.getData(0, 0)) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/pytest/updatetest.sh b/tests/pytest/updatetest.sh deleted file mode 100644 index ade1180553..0000000000 --- a/tests/pytest/updatetest.sh +++ /dev/null @@ -1,10 +0,0 @@ -# update -python3 ./test.py -f update/allow_update.py -python3 ./test.py -f update/allow_update-0.py -python3 ./test.py -f update/append_commit_data.py -python3 ./test.py -f update/append_commit_last-0.py -python3 ./test.py -f update/append_commit_last.py -python3 ./test.py -f update/merge_commit_data.py -python3 ./test.py -f update/merge_commit_data2.py -python3 ./test.py -f update/merge_commit_last-0.py -python3 ./test.py -f update/merge_commit_last.py \ No newline at end of file diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index c5fcf575ae..76230f79f0 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -134,66 +134,8 @@ sql select join_mt0.ts, join_mt1.t1, join_mt0.t1, join_mt1.tbname, join_mt0.tbna #1970-01-01 08:01:40.800 | 10 | 45.000000000 | 0 | true | false | 0 | #1970-01-01 08:01:40.790 | 10 | 945.000000000 | 90 | true | true | 0 | -sql select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc limit 20 offset 19; -if $rows != 20 then - return -1 -endi +sql_error select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc limit 20 offset 19; -if $data00 != @70-01-01 08:01:40.800@ then - return -1 -endi - -if $data01 != 10 then - return -1 -endi - -if $data02 != 45.000000000 then - return -1 -endi - -if $data03 != 0 then - return -1 -endi - -if $data04 != 1 then - return -1 -endi - -if $data05 != 0 then - return -1 -endi - -if $data06 != 0 then - return -1 -endi - -if $data10 != @70-01-01 08:01:40.790@ then - return -1 -endi - -if $data11 != 10 then - return -1 -endi - -if $data12 != 945.000000000 then - return -1 -endi - -if $data13 != 90 then - return -1 -endi - -if $data14 != 1 then - return -1 -endi - -if $data15 != 1 then - return -1 -endi - -if $data16 != 0 then - return -1 -endi sql select count(join_mt0.c1), sum(join_mt0.c2)/count(*), avg(c2), first(join_mt0.c5), last(c7) from join_mt0 interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; if $rows != 100 then @@ -261,59 +203,9 @@ if $data16 != 2 then endi # this function will cause shell crash -sql select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; -if $rows != 100 then - return -1 -endi +sql_error select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc; -if $data00 != @70-01-01 08:01:40.990@ then - return -1 -endi - -if $data01 != 10 then - return -1 -endi - -if $data02 != 90 then - return -1 -endi - -if $data03 != 0 then - return -1 -endi - -if $data11 != 10 then - return -1 -endi - -if $data12 != 80 then - return -1 -endi - -if $data13 != 0 then - return -1 -endi - -sql select last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10m) group by join_mt0.t1 order by join_mt0.ts asc; -if $rows != 1 then - return -1 -endi - -if $data00 != @70-01-01 08:00:00.000@ then - return -1 -endi - -if $data01 != 1 then - return -1 -endi - -if $data02 != 0 then - return -1 -endi - -if $data03 != 0 then - return -1 -endi +sql_error select last(join_mt1.c7), first(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10m) group by join_mt0.t1 order by join_mt0.ts asc; sql_error select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); sql select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); diff --git a/tests/script/general/parser/stream_on_sys.sim b/tests/script/general/parser/stream_on_sys.sim index 5507b4db48..845a484488 100644 --- a/tests/script/general/parser/stream_on_sys.sim +++ b/tests/script/general/parser/stream_on_sys.sim @@ -22,12 +22,12 @@ $i = 0 sql use $db -sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) sliding(2s) -sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) sliding(2s) -sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) sliding(2s) -sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) sliding(2s) -sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) sliding(2s) -sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sliding(2s) +sql create table cpustrm as select count(*), avg(cpu_taosd), max(cpu_taosd), min(cpu_taosd), avg(cpu_system), max(cpu_cores), min(cpu_cores), last(cpu_cores) from log.dn1 interval(4s) +sql create table memstrm as select count(*), avg(mem_taosd), max(mem_taosd), min(mem_taosd), avg(mem_system), first(mem_total), last(mem_total) from log.dn1 interval(4s) +sql create table diskstrm as select count(*), avg(disk_used), last(disk_used), avg(disk_total), first(disk_total) from log.dn1 interval(4s) +sql create table bandstrm as select count(*), avg(band_speed), last(band_speed) from log.dn1 interval(4s) +sql create table reqstrm as select count(*), avg(req_http), last(req_http), avg(req_select), last(req_select), avg(req_insert), last(req_insert) from log.dn1 interval(4s) +sql create table iostrm as select count(*), avg(io_read), last(io_read), avg(io_write), last(io_write) from log.dn1 interval(4s) sleep 120000 sql select * from cpustrm if $rows <= 0 then From 86287ddbcb070a74c38cd62c46882ab41488d12e Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Fri, 6 Nov 2020 15:23:16 +0800 Subject: [PATCH 073/122] [TD-1592]: add test case for kill qeury> --- tests/pytest/query/kill_query.py | 82 ++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 tests/pytest/query/kill_query.py diff --git a/tests/pytest/query/kill_query.py b/tests/pytest/query/kill_query.py new file mode 100644 index 0000000000..8975eea268 --- /dev/null +++ b/tests/pytest/query/kill_query.py @@ -0,0 +1,82 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes +import os +import threading +import time + + +class TDTestCase: + """ + kill query + """ + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def query(self): + conn = taos.connect(host='127.0.0.1', user='root', password='taosdata', config='/etc/config') + cursor = conn.cursor() + while True: + cursor.execute('show queries;') + print('show queries!') + temp = cursor.fetchall() + if temp: + print(temp[0][0]) + cursor.execute('kill query %s ;' % temp[0][0]) + print('kill query success') + break + time.sleep(0.5) + cursor.close() + conn.close() + + def run(self): + tdSql.prepare() + + print("==============step1") + os.system('yes | sudo taosdemo -n 100') + print('insert into test.meters 10000000 rows') + + + t1 = threading.Thread(target=self.query) + t1.setDaemon(True) + t1.start() + + print("==============step2") + tdSql.execute('use test;') + try: + print('============begin select * from 10000000 rows') + tdSql.query('select * from test.meters;') + # print(tdSql.queryResult) + except Exception as e: + if not "ProgrammingError('Query terminated'" in str(e): + raise Exception('fail') + + print('success') + print('kill query success') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) + From 1eb017791f82826bcef970e2c79f3fd6a91b01cc Mon Sep 17 00:00:00 2001 From: Steven Li Date: Fri, 6 Nov 2020 09:21:02 +0000 Subject: [PATCH 074/122] Correcting a tiny display bug in the crash_gen tool --- tests/pytest/crash_gen/crash_gen_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 8a074500e1..8d68457ec8 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2043,6 +2043,7 @@ class ThreadStacks: # stack info for all threads print("[{sf}] File {filename}, line {lineno}, in {name}".format( sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name)) print(" {}".format(frame.line)) + stackFrame += 1 print("-----> End of Thread Info ----->\n") class ClientManager: From 855ecf59fe589902388031c5e44306fd0cfa6f67 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 6 Nov 2020 17:49:42 +0800 Subject: [PATCH 075/122] scripts --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1b2fe37c71..a5d04f79a6 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -57,8 +57,8 @@ cd ../../../debug; make ./test.sh -f general/db/delete_reuse2.sim ./test.sh -f general/db/delete_reusevnode.sim ./test.sh -f general/db/delete_reusevnode2.sim -./test.sh -f general/db/delete_writing1.sim -./test.sh -f general/db/delete_writing2.sim +#./test.sh -f general/db/delete_writing1.sim +#./test.sh -f general/db/delete_writing2.sim ./test.sh -f general/db/delete.sim ./test.sh -f general/db/len.sim ./test.sh -f general/db/repeat.sim From 41dce83a0378f08a3391ae0ffedc9181be69943b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 6 Nov 2020 10:16:20 +0000 Subject: [PATCH 076/122] [TD-1970]: column name for select last(*)/first(*)/last_row(*) is not correct --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 029e1fdb5a..87f158a2d9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2037,7 +2037,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = {.tableIndex = j, .columnIndex = i}; char name[TSDB_COL_NAME_LEN] = {0}; - SStrToken t = {.z = pSchema->name, .n = (uint32_t)strnlen(pSchema->name, TSDB_COL_NAME_LEN)}; + SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)}; setResultColName(name, pItem, cvtFunc.originFuncId, &t); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { From 67c90d60dffb1d2b535aa07280c3eddb1a20ac65 Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 6 Nov 2020 18:23:27 +0800 Subject: [PATCH 077/122] change --- packaging/deb/makedeb.sh | 2 +- packaging/rpm/tdengine.spec | 4 ++-- packaging/tools/makeclient.sh | 3 ++- packaging/tools/makepkg.sh | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index edc7de9692..516d91258a 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -48,7 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin -cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin +#cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index afec1eaf9a..19d932e074 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -58,7 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin -cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin +#cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include @@ -135,7 +135,7 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : - ${csudo} rm -f ${bin_link_dir}/taosdump || : + #${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 83a9cb1ced..4f4c6dee1d 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,8 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" + #bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 00a92cb063..2be60709aa 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,7 +36,8 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" + #bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" From ff690a8c63b91e57bc063b16de260a7873996287 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 7 Nov 2020 20:46:51 +0800 Subject: [PATCH 078/122] [TD-1974] : fix big number result output issue. --- src/kit/taosdemo/taosdemo.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 699e96428e..b12850be09 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -793,8 +793,8 @@ int main(int argc, char *argv[]) { (ntables * nrecords_per_table) / (t * nrecords_per_request), t * 1000); - printf("Spent %.4f seconds to insert %d records with %d record(s) per request: %.2f records/second\n", - t, ntables * nrecords_per_table, nrecords_per_request, + printf("Spent %.4f seconds to insert %lld records with %d record(s) per request: %.2f records/second\n", + t, (long long int)ntables * nrecords_per_table, nrecords_per_request, ntables * nrecords_per_table / t); for (int i = 0; i < threads; i++) { From c1905a921aad6854be57ef0936148ffb6bca417a Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 7 Nov 2020 21:27:08 +0800 Subject: [PATCH 079/122] fix one more place. --- src/kit/taosdemo/taosdemo.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index b12850be09..a866a677ad 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -795,7 +795,7 @@ int main(int argc, char *argv[]) { printf("Spent %.4f seconds to insert %lld records with %d record(s) per request: %.2f records/second\n", t, (long long int)ntables * nrecords_per_table, nrecords_per_request, - ntables * nrecords_per_table / t); + ((long long int)ntables * nrecords_per_table) / t); for (int i = 0; i < threads; i++) { info *t_info = infos + i; From 774356add1a2616d182d4dd8b08392c0be0b86bc Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 7 Nov 2020 21:40:33 +0800 Subject: [PATCH 080/122] [TD-1975] : fix build issue on CentOS 7. --- src/client/src/tscSQLParser.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 029e1fdb5a..f14f094e0b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -16,6 +16,7 @@ #define _BSD_SOURCE #define _XOPEN_SOURCE 500 #define _DEFAULT_SOURCE +#define _POSIX_C_SOURCE 200809L #include "os.h" #include "qAst.h" From 604e2304fb8bdc885f8571ef3ebd587814f67605 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 7 Nov 2020 22:19:56 +0800 Subject: [PATCH 081/122] use _GNU_SOURCE instead of _POSIX_C_SOURCE --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f14f094e0b..34ea37d114 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -16,7 +16,7 @@ #define _BSD_SOURCE #define _XOPEN_SOURCE 500 #define _DEFAULT_SOURCE -#define _POSIX_C_SOURCE 200809L +#define _GNU_SOURCE #include "os.h" #include "qAst.h" From d6f527d89fddb5c3f80a78e3a46961f33a2748a7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 8 Nov 2020 03:18:55 +0000 Subject: [PATCH 082/122] [TD-1952]: tsim crash while run background scripts --- tests/tsim/inc/sim.h | 89 +++++++-------- tests/tsim/inc/simParse.h | 2 +- tests/tsim/src/simExe.c | 227 +++++++++++++++++++------------------ tests/tsim/src/simMain.c | 10 +- tests/tsim/src/simParse.c | 204 ++++++++++++++++----------------- tests/tsim/src/simSystem.c | 47 ++++---- 6 files changed, 292 insertions(+), 287 deletions(-) diff --git a/tests/tsim/inc/sim.h b/tests/tsim/inc/sim.h index 58e58a442c..01e5016557 100644 --- a/tests/tsim/inc/sim.h +++ b/tests/tsim/inc/sim.h @@ -100,7 +100,7 @@ typedef struct _cmd_t { int16_t cmdno; int16_t nlen; char name[MAX_SIM_CMD_NAME_LEN]; - bool (*parseCmd)(char *, struct _cmd_t *, int); + bool (*parseCmd)(char *, struct _cmd_t *, int32_t); bool (*executeCmd)(struct _script_t *script, char *option); struct _cmd_t *next; } SCommand; @@ -111,7 +111,7 @@ typedef struct { int16_t errorJump; // sql jump flag, while '-x' exist in sql cmd, this flag // will be SQL_JUMP_TRUE, otherwise is SQL_JUMP_FALSE */ int16_t lineNum; // correspodning line number in original file - int optionOffset; // relative option offset + int32_t optionOffset;// relative option offset } SCmdLine; typedef struct _var_t { @@ -121,59 +121,56 @@ typedef struct _var_t { } SVariable; typedef struct _script_t { - int type; - bool killed; - - void *taos; - char rows[12]; // number of rows data retrieved - char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM] - [MAX_QUERY_VALUE_LEN]; // query results - char system_exit_code[12]; - char system_ret_content[MAX_SYSTEM_RESULT_LEN]; - - int varLen; - int linePos; // current cmd position - int numOfLines; // number of lines in the script - int bgScriptLen; - char fileName[MAX_FILE_NAME_LEN]; // script file name - char error[MAX_ERROR_LEN]; - char *optionBuffer; + int32_t type; + bool killed; + void * taos; + char rows[12]; // number of rows data retrieved + char data[MAX_QUERY_ROW_NUM][MAX_QUERY_COL_NUM][MAX_QUERY_VALUE_LEN]; // query results + char system_exit_code[12]; + char system_ret_content[MAX_SYSTEM_RESULT_LEN]; + int32_t varLen; + int32_t linePos; // current cmd position + int32_t numOfLines; // number of lines in the script + int32_t bgScriptLen; + char fileName[MAX_FILE_NAME_LEN]; // script file name + char error[MAX_ERROR_LEN]; + char * optionBuffer; SCmdLine *lines; // command list SVariable variables[MAX_VAR_LEN]; + pthread_t bgPid; + char auth[128]; struct _script_t *bgScripts[MAX_BACKGROUND_SCRIPT_NUM]; - char auth[128]; } SScript; extern SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; extern SCommand simCmdList[]; -extern int simScriptPos; -extern int simScriptSucced; -extern int simDebugFlag; -extern char tsScriptDir[]; -extern bool simAsyncQuery; +extern int32_t simScriptPos; +extern int32_t simScriptSucced; +extern int32_t simDebugFlag; +extern char tsScriptDir[]; +extern bool simAsyncQuery; SScript *simParseScript(char *fileName); - SScript *simProcessCallOver(SScript *script); -void *simExecuteScript(void *script); -void simInitsimCmdList(); -bool simSystemInit(); -void simSystemCleanUp(); -char *simGetVariable(SScript *script, char *varName, int varLen); -bool simExecuteExpCmd(SScript *script, char *option); -bool simExecuteTestCmd(SScript *script, char *option); -bool simExecuteGotoCmd(SScript *script, char *option); -bool simExecuteRunCmd(SScript *script, char *option); -bool simExecuteRunBackCmd(SScript *script, char *option); -bool simExecuteSystemCmd(SScript *script, char *option); -bool simExecuteSystemContentCmd(SScript *script, char *option); -bool simExecutePrintCmd(SScript *script, char *option); -bool simExecuteSleepCmd(SScript *script, char *option); -bool simExecuteReturnCmd(SScript *script, char *option); -bool simExecuteSqlCmd(SScript *script, char *option); -bool simExecuteSqlErrorCmd(SScript *script, char *rest); -bool simExecuteSqlSlowCmd(SScript *script, char *option); -bool simExecuteRestfulCmd(SScript *script, char *rest); -void simVisuallizeOption(SScript *script, char *src, char *dst); +void * simExecuteScript(void *script); +void simInitsimCmdList(); +bool simSystemInit(); +void simSystemCleanUp(); +char * simGetVariable(SScript *script, char *varName, int32_t varLen); +bool simExecuteExpCmd(SScript *script, char *option); +bool simExecuteTestCmd(SScript *script, char *option); +bool simExecuteGotoCmd(SScript *script, char *option); +bool simExecuteRunCmd(SScript *script, char *option); +bool simExecuteRunBackCmd(SScript *script, char *option); +bool simExecuteSystemCmd(SScript *script, char *option); +bool simExecuteSystemContentCmd(SScript *script, char *option); +bool simExecutePrintCmd(SScript *script, char *option); +bool simExecuteSleepCmd(SScript *script, char *option); +bool simExecuteReturnCmd(SScript *script, char *option); +bool simExecuteSqlCmd(SScript *script, char *option); +bool simExecuteSqlErrorCmd(SScript *script, char *rest); +bool simExecuteSqlSlowCmd(SScript *script, char *option); +bool simExecuteRestfulCmd(SScript *script, char *rest); +void simVisuallizeOption(SScript *script, char *src, char *dst); #endif \ No newline at end of file diff --git a/tests/tsim/inc/simParse.h b/tests/tsim/inc/simParse.h index d3f92add71..ef7d8e5ce7 100644 --- a/tests/tsim/inc/simParse.h +++ b/tests/tsim/inc/simParse.h @@ -50,6 +50,6 @@ typedef struct { char sexpLen[MAX_NUM_BLOCK]; /*switch expression length */ } SBlock; -bool simParseExpression(char *token, int lineNum); +bool simParseExpression(char *token, int32_t lineNum); #endif \ No newline at end of file diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index 7f786dfaa9..2db750cdd3 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "sim.h" #include "taos.h" @@ -38,30 +39,28 @@ void simLogSql(char *sql, bool useSharp) { } else { fprintf(fp, "%s;\n", sql); } - + fflush(fp); } char *simParseArbitratorName(char *varName); char *simParseHostName(char *varName); -char *simGetVariable(SScript *script, char *varName, int varLen) { +char *simGetVariable(SScript *script, char *varName, int32_t varLen) { if (strncmp(varName, "hostname", 8) == 0) { return simParseHostName(varName); } if (strncmp(varName, "arbitrator", 10) == 0) { - return simParseArbitratorName(varName); + return simParseArbitratorName(varName); } if (strncmp(varName, "error", varLen) == 0) return script->error; if (strncmp(varName, "rows", varLen) == 0) return script->rows; - if (strncmp(varName, "system_exit", varLen) == 0) - return script->system_exit_code; + if (strncmp(varName, "system_exit", varLen) == 0) return script->system_exit_code; - if (strncmp(varName, "system_content", varLen) == 0) - return script->system_ret_content; + if (strncmp(varName, "system_content", varLen) == 0) return script->system_ret_content; // variable like data2_192.168.0.1 if (strncmp(varName, "data", 4) == 0) { @@ -70,16 +69,16 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } if (varName[5] == '_') { - int col = varName[4] - '0'; + int32_t col = varName[4] - '0'; if (col < 0 || col >= MAX_QUERY_COL_NUM) { return "null"; } - char *keyName; - int keyLen; + char * keyName; + int32_t keyLen; paGetToken(varName + 6, &keyName, &keyLen); - for (int i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { if (strncmp(keyName, script->data[i][0], keyLen) == 0) { simDebug("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); return script->data[i][col]; @@ -87,16 +86,16 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } return "null"; } else if (varName[6] == '_') { - int col = (varName[4] - '0') * 10 + (varName[5] - '0'); + int32_t col = (varName[4] - '0') * 10 + (varName[5] - '0'); if (col < 0 || col >= MAX_QUERY_COL_NUM) { return "null"; } - char *keyName; - int keyLen; + char * keyName; + int32_t keyLen; paGetToken(varName + 7, &keyName, &keyLen); - for (int i = 0; i < MAX_QUERY_ROW_NUM; ++i) { + for (int32_t i = 0; i < MAX_QUERY_ROW_NUM; ++i) { if (strncmp(keyName, script->data[i][0], keyLen) == 0) { simTrace("script:%s, keyName:%s, keyValue:%s", script->fileName, script->data[i][0], script->data[i][col]); return script->data[i][col]; @@ -104,8 +103,8 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } return "null"; } else { - int row = varName[4] - '0'; - int col = varName[5] - '0'; + int32_t row = varName[4] - '0'; + int32_t col = varName[5] - '0'; if (row < 0 || row >= MAX_QUERY_ROW_NUM) { return "null"; } @@ -118,7 +117,7 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { } } - for (int i = 0; i < script->varLen; ++i) { + for (int32_t i = 0; i < script->varLen; ++i) { SVariable *var = &script->variables[i]; if (var->varNameLen != varLen) { continue; @@ -144,11 +143,11 @@ char *simGetVariable(SScript *script, char *varName, int varLen) { return var->varValue; } -int simExecuteExpression(SScript *script, char *exp) { - char *op1, *op2, *var1, *var2, *var3, *rest; - int op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; - char t0[512], t1[512], t2[512], t3[1024]; - int result; +int32_t simExecuteExpression(SScript *script, char *exp) { + char * op1, *op2, *var1, *var2, *var3, *rest; + int32_t op1Len, op2Len, var1Len, var2Len, var3Len, val0, val1; + char t0[512], t1[512], t2[512], t3[1024]; + int32_t result; rest = paGetToken(exp, &var1, &var1Len); rest = paGetToken(rest, &op1, &op1Len); @@ -234,7 +233,7 @@ bool simExecuteExpCmd(SScript *script, char *option) { } bool simExecuteTestCmd(SScript *script, char *option) { - int result; + int32_t result; result = simExecuteExpression(script, option); if (result >= 0) @@ -285,13 +284,12 @@ bool simExecuteRunBackCmd(SScript *script, char *option) { sprintf(script->error, "lineNum:%d. parse file:%s error", script->lines[script->linePos].lineNum, fileName); return false; } - simInfo("script:%s, start to execute in background", newScript->fileName); newScript->type = SIM_SCRIPT_TYPE_BACKGROUND; script->bgScripts[script->bgScriptLen++] = newScript; + simInfo("script:%s, start to execute in background,", newScript->fileName); - pthread_t pid; - if (pthread_create(&pid, NULL, simExecuteScript, (void *)newScript) != 0) { + if (pthread_create(&newScript->bgPid, NULL, simExecuteScript, (void *)newScript) != 0) { sprintf(script->error, "lineNum:%d. create background thread failed", script->lines[script->linePos].lineNum); return false; } @@ -307,13 +305,13 @@ bool simExecuteSystemCmd(SScript *script, char *option) { simVisuallizeOption(script, option, buf + strlen(buf)); simLogSql(buf, true); - int code = system(buf); - int repeatTimes = 0; + int32_t code = system(buf); + int32_t repeatTimes = 0; while (code < 0) { - simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", - script->fileName, buf, code, errno, strerror(errno), repeatTimes); + simError("script:%s, failed to execute %s , code %d, errno:%d %s, repeatTimes:%d", script->fileName, buf, code, + errno, strerror(errno), repeatTimes); taosMsleep(1000); -#ifdef LINUX +#ifdef LINUX signal(SIGCHLD, SIG_DFL); #endif if (repeatTimes++ >= 10) { @@ -368,8 +366,8 @@ bool simExecutePrintCmd(SScript *script, char *rest) { } bool simExecuteSleepCmd(SScript *script, char *option) { - int delta; - char buf[1024]; + int32_t delta; + char buf[1024]; simVisuallizeOption(script, option, buf); option = buf; @@ -395,7 +393,7 @@ bool simExecuteReturnCmd(SScript *script, char *option) { simVisuallizeOption(script, option, buf); option = buf; - int ret = 1; + int32_t ret = 1; if (option && option[0] != 0) ret = atoi(option); if (ret < 0) { @@ -411,8 +409,8 @@ bool simExecuteReturnCmd(SScript *script, char *option) { } void simVisuallizeOption(SScript *script, char *src, char *dst) { - char *var, *token, *value; - int dstLen, srcLen, tokenLen; + char * var, *token, *value; + int32_t dstLen, srcLen, tokenLen; dst[0] = 0, dstLen = 0; @@ -420,14 +418,14 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { var = strchr(src, '$'); if (var == NULL) break; if (var && ((var - src - 1) > 0) && *(var - 1) == '\\') { - srcLen = (int)(var - src - 1); + srcLen = (int32_t)(var - src - 1); memcpy(dst + dstLen, src, srcLen); dstLen += srcLen; src = var; break; } - srcLen = (int)(var - src); + srcLen = (int32_t)(var - src); memcpy(dst + dstLen, src, srcLen); dstLen += srcLen; @@ -435,13 +433,13 @@ void simVisuallizeOption(SScript *script, char *src, char *dst) { value = simGetVariable(script, token, tokenLen); strcpy(dst + dstLen, value); - dstLen += (int)strlen(value); + dstLen += (int32_t)strlen(value); } strcpy(dst + dstLen, src); } -void simCloseRestFulConnect(SScript *script) { +void simCloseRestFulConnect(SScript *script) { memset(script->auth, 0, sizeof(script->auth)); } @@ -465,7 +463,7 @@ void simCloseTaosdConnect(SScript *script) { // {"status":"succ","code":0,"desc":"/KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04"} // {"status":"succ","head":["affected_rows"],"data":[[1]],"rows":1} // {"status":"succ","head":["ts","i"],"data":[["2017-12-25 21:28:41.022",1],["2017-12-25 21:28:42.022",2],["2017-12-25 21:28:43.022",3],["2017-12-25 21:28:44.022",4],["2017-12-25 21:28:45.022",5],["2017-12-25 21:28:46.022",6],["2017-12-25 21:28:47.022",7],["2017-12-25 21:28:48.022",8],["2017-12-25 21:28:49.022",9],["2017-12-25 21:28:50.022",10]],"rows":10} -int simParseHttpCommandResult(SScript *script, char *command) { +int32_t simParseHttpCommandResult(SScript *script, char *command) { cJSON* root = cJSON_Parse(command); if (root == NULL) { simError("script:%s, failed to parse json, response:%s", script->fileName, command); @@ -492,14 +490,15 @@ int simParseHttpCommandResult(SScript *script, char *command) { cJSON_Delete(root); return -1; } - int retcode = (int)code->valueint; + int32_t retcode = (int32_t)code->valueint; if (retcode != 1017) { - simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring, command); + simError("script:%s, json:status:%s not equal to succ, response:%s", script->fileName, status->valuestring, + command); cJSON_Delete(root); return retcode; } else { simDebug("script:%s, json:status:%s not equal to succ, but code is %d, response:%s", script->fileName, - status->valuestring, retcode, command); + status->valuestring, retcode, command); cJSON_Delete(root); return 0; } @@ -524,27 +523,27 @@ int simParseHttpCommandResult(SScript *script, char *command) { return -1; } - int rowsize = cJSON_GetArraySize(data); + int32_t rowsize = cJSON_GetArraySize(data); if (rowsize < 0) { simError("script:%s, failed to parse json:data, data size %d, response:%s", script->fileName, rowsize, command); cJSON_Delete(root); return -1; } - int rowIndex = 0; + int32_t rowIndex = 0; sprintf(script->rows, "%d", rowsize); - for (int r = 0; r < rowsize; ++r) { + for (int32_t r = 0; r < rowsize; ++r) { cJSON *row = cJSON_GetArrayItem(data, r); if (row == NULL) continue; if (rowIndex++ >= 10) break; - int colsize = cJSON_GetArraySize(row); + int32_t colsize = cJSON_GetArraySize(row); if (colsize < 0) { break; } colsize = MIN(10, colsize); - for (int c = 0; c < colsize; ++c) { + for (int32_t c = 0; c < colsize; ++c) { cJSON *col = cJSON_GetArrayItem(row, c); if (col->valuestring != NULL) { strcpy(script->data[r][c], col->valuestring); @@ -561,7 +560,7 @@ int simParseHttpCommandResult(SScript *script, char *command) { return 0; } -int simExecuteRestFulCommand(SScript *script, char *command) { +int32_t simExecuteRestFulCommand(SScript *script, char *command) { char buf[5000] = {0}; sprintf(buf, "%s 2>/dev/null", command); @@ -571,13 +570,13 @@ int simExecuteRestFulCommand(SScript *script, char *command) { return -1; } - int mallocSize = 2000; - int alreadyReadSize = 0; - char* content = malloc(mallocSize); + int32_t mallocSize = 2000; + int32_t alreadyReadSize = 0; + char * content = malloc(mallocSize); while (!feof(fp)) { - int availSize = mallocSize - alreadyReadSize; - int len = (int)fread(content + alreadyReadSize, 1, availSize, fp); + int32_t availSize = mallocSize - alreadyReadSize; + int32_t len = (int32_t)fread(content + alreadyReadSize, 1, availSize, fp); if (len >= availSize) { alreadyReadSize += len; mallocSize *= 2; @@ -595,10 +594,11 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) { sprintf(command, "curl 127.0.0.1:6041/rest/login/%s/%s", user, pass); bool success = false; - for (int attempt = 0; attempt < 10; ++attempt) { + for (int32_t attempt = 0; attempt < 10; ++attempt) { success = simExecuteRestFulCommand(script, command) == 0; if (!success) { - simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), attempt); + simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), + attempt); taosMsleep(1000); } else { simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); @@ -607,7 +607,8 @@ bool simCreateRestFulConnect(SScript *script, char *user, char *pass) { } if (!success) { - sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, taos_errstr(NULL)); + sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, + taos_errstr(NULL)); return false; } @@ -619,10 +620,11 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { simCloseTaosdConnect(script); void *taos = NULL; taosMsleep(2000); - for (int attempt = 0; attempt < 10; ++attempt) { + for (int32_t attempt = 0; attempt < 10; ++attempt) { taos = taos_connect(NULL, user, pass, NULL, tsDnodeShellPort); if (taos == NULL) { - simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), attempt); + simDebug("script:%s, user:%s connect taosd failed:%s, attempt:%d", script->fileName, user, taos_errstr(NULL), + attempt); taosMsleep(1000); } else { simDebug("script:%s, user:%s connect taosd successed, attempt:%d", script->fileName, user, attempt); @@ -631,7 +633,8 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { } if (taos == NULL) { - sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, taos_errstr(NULL)); + sprintf(script->error, "lineNum:%d. connect taosd failed:%s", script->lines[script->linePos].lineNum, + taos_errstr(NULL)); return false; } @@ -642,9 +645,9 @@ bool simCreateNativeConnect(SScript *script, char *user, char *pass) { } bool simCreateTaosdConnect(SScript *script, char *rest) { - char *user = TSDB_DEFAULT_USER; - char *token; - int tokenLen; + char * user = TSDB_DEFAULT_USER; + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); rest = paGetToken(rest, &token, &tokenLen); if (tokenLen != 0) { @@ -659,26 +662,27 @@ bool simCreateTaosdConnect(SScript *script, char *rest) { } bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { - char timeStr[30] = {0}; - time_t tt; + char timeStr[30] = {0}; + time_t tt; struct tm *tp; - SCmdLine *line = &script->lines[script->linePos]; - int ret = -1; + SCmdLine * line = &script->lines[script->linePos]; + int32_t ret = -1; - TAOS_RES* pSql = NULL; - - for (int attempt = 0; attempt < 10; ++attempt) { + TAOS_RES *pSql = NULL; + + for (int32_t attempt = 0; attempt < 10; ++attempt) { simLogSql(rest, false); pSql = taos_query(script->taos, rest); ret = taos_errno(pSql); - + if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); + simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret)); ret = 0; break; } else if (ret != 0) { - simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", - script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret), taos_errstr(pSql)); + simDebug("script:%s, taos:%p, %s failed, ret:%d:%s, error:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret), taos_errstr(pSql)); if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -689,7 +693,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { } else { break; } - + taos_free_result(pSql); } @@ -698,8 +702,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { return false; } - int numOfRows = 0; - int num_fields = taos_field_count(pSql); + int32_t numOfRows = 0; + int32_t num_fields = taos_field_count(pSql); if (num_fields != 0) { if (pSql == NULL) { simDebug("script:%s, taos:%p, %s failed, result is null", script->fileName, script->taos, rest); @@ -717,9 +721,9 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { while ((row = taos_fetch_row(pSql))) { if (numOfRows < MAX_QUERY_ROW_NUM) { TAOS_FIELD *fields = taos_fetch_fields(pSql); - int* length = taos_fetch_lengths(pSql); - - for (int i = 0; i < num_fields; i++) { + int32_t * length = taos_fetch_lengths(pSql); + + for (int32_t i = 0; i < num_fields; i++) { char *value = NULL; if (i < MAX_QUERY_COL_NUM) { value = script->data[numOfRows][i]; @@ -735,8 +739,7 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { switch (fields[i].type) { case TSDB_DATA_TYPE_BOOL: - sprintf(value, "%s", - ((((int)(*((char *)row[i]))) == 1) ? "1" : "0")); + sprintf(value, "%s", ((((int32_t)(*((char *)row[i]))) == 1) ? "1" : "0")); break; case TSDB_DATA_TYPE_TINYINT: sprintf(value, "%d", *((int8_t *)row[i])); @@ -779,9 +782,8 @@ bool simExecuteNativeSqlCommand(SScript *script, char *rest, bool isSlow) { tp = localtime(&tt); strftime(timeStr, 64, "%y-%m-%d %H:%M:%S", tp); - sprintf(value, "%s.%03d", timeStr, - (int)(*((int64_t *)row[i]) % 1000)); - + sprintf(value, "%s.%03d", timeStr, (int32_t)(*((int64_t *)row[i]) % 1000)); + break; default: break; @@ -814,17 +816,16 @@ bool simExecuteRestFulSqlCommand(SScript *script, char *rest) { char command[4096]; sprintf(command, "curl -H 'Authorization: Taosd %s' -d \"%s\" 127.0.0.1:6041/rest/sql", script->auth, rest); - int ret = -1; - for (int attempt = 0; attempt < 10; ++attempt) { + int32_t ret = -1; + for (int32_t attempt = 0; attempt < 10; ++attempt) { ret = simExecuteRestFulCommand(script, command); - if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || - ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { - simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); + if (ret == TSDB_CODE_MND_TABLE_ALREADY_EXIST || ret == TSDB_CODE_MND_DB_ALREADY_EXIST) { + simDebug("script:%s, taos:%p, %s success, ret:%d:%s", script->fileName, script->taos, rest, ret & 0XFFFF, + tstrerror(ret)); ret = 0; break; } else if (ret != 0) { - simDebug("script:%s, taos:%p, %s failed, ret:%d", - script->fileName, script->taos, rest, ret); + simDebug("script:%s, taos:%p, %s failed, ret:%d", script->fileName, script->taos, rest, ret); if (line->errorJump == SQL_JUMP_TRUE) { script->linePos = line->jump; @@ -854,8 +855,8 @@ bool simExecuteSqlImpCmd(SScript *script, char *rest, bool isSlow) { simDebug("script:%s, exec:%s", script->fileName, rest); strcpy(script->rows, "-1"); - for (int row = 0; row < MAX_QUERY_ROW_NUM; ++row) { - for (int col = 0; col < MAX_QUERY_COL_NUM; ++col) { + for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { + for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { strcpy(script->data[row][col], "null"); } } @@ -903,23 +904,23 @@ bool simExecuteSqlSlowCmd(SScript *script, char *rest) { bool simExecuteRestfulCmd(SScript *script, char *rest) { FILE *fp = NULL; - char filename[256]; - sprintf(filename, "%s/tmp.sql", tsScriptDir); + char filename[256]; + sprintf(filename, "%s/tmp.sql", tsScriptDir); fp = fopen(filename, "w"); if (fp == NULL) { fprintf(stderr, "ERROR: failed to open file: %s\n", filename); return false; } - char db[64] = {0}; - char tb[64] = {0}; - char gzip[32] = {0}; + char db[64] = {0}; + char tb[64] = {0}; + char gzip[32] = {0}; int32_t ts; int32_t times; sscanf(rest, "%s %s %d %d %s", db, tb, &ts, ×, gzip); - + fprintf(fp, "insert into %s.%s values ", db, tb); - for (int i = 0; i < times; ++i) { + for (int32_t i = 0; i < times; ++i) { fprintf(fp, "(%d000, %d)", ts + i, ts); } fprintf(fp, " \n"); @@ -951,8 +952,8 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { simDebug("script:%s, exec:%s", script->fileName, rest); strcpy(script->rows, "-1"); - for (int row = 0; row < MAX_QUERY_ROW_NUM; ++row) { - for (int col = 0; col < MAX_QUERY_COL_NUM; ++col) { + for (int32_t row = 0; row < MAX_QUERY_ROW_NUM; ++row) { + for (int32_t col = 0; col < MAX_QUERY_COL_NUM; ++col) { strcpy(script->data[row][col], "null"); } } @@ -981,27 +982,27 @@ bool simExecuteSqlErrorCmd(SScript *script, char *rest) { return true; } - int ret; - TAOS_RES* pSql = NULL; + int32_t ret; + TAOS_RES *pSql = NULL; if (simAsyncQuery) { char command[4096]; sprintf(command, "curl -H 'Authorization: Taosd %s' -d '%s' 127.0.0.1:6041/rest/sql", script->auth, rest); ret = simExecuteRestFulCommand(script, command); - } - else { + } else { pSql = taos_query(script->taos, rest); ret = taos_errno(pSql); taos_free_result(pSql); } if (ret != TSDB_CODE_SUCCESS) { - simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", - script->fileName, script->taos, rest, ret & 0XFFFF, tstrerror(ret)); + simDebug("script:%s, taos:%p, %s execute, expect failed, so success, ret:%d:%s", script->fileName, script->taos, + rest, ret & 0XFFFF, tstrerror(ret)); script->linePos++; return true; } - - sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, tstrerror(ret)); + + sprintf(script->error, "lineNum:%d. sql:%s expect failed, but success, ret:%d:%s", line->lineNum, rest, ret & 0XFFFF, + tstrerror(ret)); return false; } diff --git a/tests/tsim/src/simMain.c b/tests/tsim/src/simMain.c index ef1a488f60..33fd24dd58 100644 --- a/tests/tsim/src/simMain.c +++ b/tests/tsim/src/simMain.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tglobal.h" #include "sim.h" @@ -20,15 +21,15 @@ bool simAsyncQuery = false; -void simHandleSignal(int signo) { +void simHandleSignal(int32_t signo) { simSystemCleanUp(); exit(1); } -int main(int argc, char *argv[]) { +int32_t main(int32_t argc, char *argv[]) { char scriptFile[MAX_FILE_NAME_LEN] = "sim_main_test.sim"; - for (int i = 1; i < argc; ++i) { + for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0 && i < argc - 1) { tstrncpy(configDir, argv[++i], MAX_FILE_NAME_LEN); } else if (strcmp(argv[i], "-f") == 0 && i < argc - 1) { @@ -37,8 +38,7 @@ int main(int argc, char *argv[]) { simAsyncQuery = true; } else { printf("usage: %s [options] \n", argv[0]); - printf(" [-c config]: config directory, default is: %s\n", - configDir); + printf(" [-c config]: config directory, default is: %s\n", configDir); printf(" [-f script]: script filename\n"); exit(0); } diff --git a/tests/tsim/src/simParse.c b/tests/tsim/src/simParse.c index 2e6121304f..b909f5bd8f 100644 --- a/tests/tsim/src/simParse.c +++ b/tests/tsim/src/simParse.c @@ -57,6 +57,7 @@ * */ +#define _DEFAULT_SOURCE #include "os.h" #include "sim.h" #include "simParse.h" @@ -64,16 +65,16 @@ #undef TAOS_MEM_CHECK static SCommand *cmdHashList[MAX_NUM_CMD]; -static SCmdLine cmdLine[MAX_CMD_LINES]; -static char parseErr[MAX_ERROR_LEN]; -static char optionBuffer[MAX_OPTION_BUFFER]; -static int numOfLines, optionOffset; -static SLabel label, dest; -static SBlock block; +static SCmdLine cmdLine[MAX_CMD_LINES]; +static char parseErr[MAX_ERROR_LEN]; +static char optionBuffer[MAX_OPTION_BUFFER]; +static int32_t numOfLines, optionOffset; +static SLabel label, dest; +static SBlock block; -int simHashCmd(char *token, int tokenLen) { - int i; - int hash = 0; +int32_t simHashCmd(char *token, int32_t tokenLen) { + int32_t i; + int32_t hash = 0; for (i = 0; i < tokenLen; ++i) hash += token[i]; @@ -82,8 +83,8 @@ int simHashCmd(char *token, int tokenLen) { return hash; } -SCommand *simCheckCmd(char *token, int tokenLen) { - int hash; +SCommand *simCheckCmd(char *token, int32_t tokenLen) { + int32_t hash; SCommand *node; hash = simHashCmd(token, tokenLen); @@ -102,10 +103,10 @@ SCommand *simCheckCmd(char *token, int tokenLen) { } void simAddCmdIntoHash(SCommand *pCmd) { - int hash; + int32_t hash; SCommand *node; - hash = simHashCmd(pCmd->name, (int)strlen(pCmd->name)); + hash = simHashCmd(pCmd->name, (int32_t)strlen(pCmd->name)); node = cmdHashList[hash]; pCmd->next = node; cmdHashList[hash] = pCmd; @@ -122,7 +123,7 @@ void simResetParser() { } SScript *simBuildScriptObj(char *fileName) { - int i, destPos; + int32_t i, destPos; /* process labels */ @@ -176,11 +177,11 @@ SScript *simBuildScriptObj(char *fileName) { } SScript *simParseScript(char *fileName) { - FILE *fd; - int tokenLen, lineNum = 0; - char buffer[MAX_LINE_LEN], name[128], *token, *rest; + FILE * fd; + int32_t tokenLen, lineNum = 0; + char buffer[MAX_LINE_LEN], name[128], *token, *rest; SCommand *pCmd; - SScript *script; + SScript * script; if ((fileName[0] == '.') || (fileName[0] == '/')) { strcpy(name, fileName); @@ -199,12 +200,13 @@ SScript *simParseScript(char *fileName) { if (fgets(buffer, sizeof(buffer), fd) == NULL) continue; lineNum++; - int cmdlen = (int)strlen(buffer); - if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') + int32_t cmdlen = (int32_t)strlen(buffer); + if (buffer[cmdlen - 1] == '\r' || buffer[cmdlen - 1] == '\n') { buffer[cmdlen - 1] = 0; + } rest = buffer; - for (int i = 0; i < cmdlen; ++i) { + for (int32_t i = 0; i < cmdlen; ++i) { if (buffer[i] == '\r' || buffer[i] == '\n') { buffer[i] = ' '; } @@ -249,9 +251,9 @@ SScript *simParseScript(char *fileName) { return script; } -int simCheckExpression(char *exp) { - char *op1, *op2, *op, *rest; - int op1Len, op2Len, opLen; +int32_t simCheckExpression(char *exp) { + char * op1, *op2, *op, *rest; + int32_t op1Len, op2Len, opLen; rest = paGetToken(exp, &op1, &op1Len); if (op1Len == 0) { @@ -282,8 +284,7 @@ int simCheckExpression(char *exp) { return -1; } } else if (opLen == 2) { - if (op[1] != '=' || - (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) { + if (op[1] != '=' || (op[0] != '=' && op[0] != '<' && op[0] != '>' && op[0] != '!')) { sprintf(parseErr, "left side of assignment must be variable"); return -1; } @@ -294,10 +295,10 @@ int simCheckExpression(char *exp) { rest = paGetToken(rest, &op, &opLen); - if (opLen == 0) return (int)(rest - exp); + if (opLen == 0) return (int32_t)(rest - exp); /* if it is key word "then" */ - if (strncmp(op, "then", 4) == 0) return (int)(op - exp); + if (strncmp(op, "then", 4) == 0) return (int32_t)(op - exp); rest = paGetToken(rest, &op2, &op2Len); if (op2Len == 0) { @@ -310,16 +311,15 @@ int simCheckExpression(char *exp) { return -1; } - if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || - op[0] == '.') { - return (int)(rest - exp); + if (op[0] == '+' || op[0] == '-' || op[0] == '*' || op[0] == '/' || op[0] == '.') { + return (int32_t)(rest - exp); } return -1; } -bool simParseExpression(char *token, int lineNum) { - int expLen; +bool simParseExpression(char *token, int32_t lineNum) { + int32_t expLen; expLen = simCheckExpression(token); if (expLen <= 0) return -1; @@ -335,9 +335,9 @@ bool simParseExpression(char *token, int lineNum) { return true; } -bool simParseIfCmd(char *rest, SCommand *pCmd, int lineNum) { - char *ret; - int expLen; +bool simParseIfCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * ret; + int32_t expLen; expLen = simCheckExpression(rest); @@ -364,8 +364,8 @@ bool simParseIfCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseElifCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; expLen = simCheckExpression(rest); @@ -382,8 +382,7 @@ bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) { } cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = - &(cmdLine[numOfLines].jump); + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); block.numJump[block.top - 1]++; numOfLines++; @@ -402,7 +401,7 @@ bool simParseElifCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseElseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { if (block.top < 1) { sprintf(parseErr, "no matching if"); return false; @@ -414,8 +413,7 @@ bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) { } cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = - &(cmdLine[numOfLines].jump); + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); block.numJump[block.top - 1]++; numOfLines++; @@ -426,8 +424,8 @@ bool simParseElseCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) { - int i; +bool simParseEndiCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; if (block.top < 1) { sprintf(parseErr, "no matching if"); @@ -441,8 +439,9 @@ bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) { if (block.pos[block.top - 1]) *(block.pos[block.top - 1]) = numOfLines; - for (i = 0; i < block.numJump[block.top - 1]; ++i) + for (i = 0; i < block.numJump[block.top - 1]; ++i) { *(block.jump[block.top - 1][i]) = numOfLines; + } block.numJump[block.top - 1] = 0; block.top--; @@ -450,8 +449,8 @@ bool simParseEndiCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseWhileCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseWhileCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; expLen = simCheckExpression(rest); @@ -473,8 +472,8 @@ bool simParseWhileCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseEndwCmd(char *rest, SCommand *pCmd, int lineNum) { - int i; +bool simParseEndwCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; if (block.top < 1) { sprintf(parseErr, "no matching while"); @@ -493,17 +492,18 @@ bool simParseEndwCmd(char *rest, SCommand *pCmd, int lineNum) { *(block.pos[block.top - 1]) = numOfLines; - for (i = 0; i < block.numJump[block.top - 1]; ++i) + for (i = 0; i < block.numJump[block.top - 1]; ++i) { *(block.jump[block.top - 1][i]) = numOfLines; + } block.top--; return true; } -bool simParseSwitchCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseSwitchCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); if (tokenLen == 0) { @@ -524,9 +524,9 @@ bool simParseSwitchCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseCaseCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); if (tokenLen == 0) { @@ -544,16 +544,16 @@ bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) { return false; } - if (block.pos[block.top - 1] != NULL) + if (block.pos[block.top - 1] != NULL) { *(block.pos[block.top - 1]) = numOfLines; + } block.pos[block.top - 1] = &(cmdLine[numOfLines].jump); cmdLine[numOfLines].cmdno = SIM_CMD_TEST; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1], - block.sexpLen[block.top - 1]); + memcpy(optionBuffer + optionOffset, block.sexp[block.top - 1], block.sexpLen[block.top - 1]); optionOffset += block.sexpLen[block.top - 1]; *(optionBuffer + optionOffset++) = ' '; *(optionBuffer + optionOffset++) = '='; @@ -567,20 +567,18 @@ bool simParseCaseCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseBreakCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseBreakCmd(char *rest, SCommand *pCmd, int32_t lineNum) { if (block.top < 1) { sprintf(parseErr, "no blcok exists"); return false; } - if (block.type[block.top - 1] != BLOCK_SWITCH && - block.type[block.top - 1] != BLOCK_WHILE) { + if (block.type[block.top - 1] != BLOCK_SWITCH && block.type[block.top - 1] != BLOCK_WHILE) { sprintf(parseErr, "not in switch or while block"); return false; } - block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = - &(cmdLine[numOfLines].jump); + block.jump[block.top - 1][(uint8_t)block.numJump[block.top - 1]] = &(cmdLine[numOfLines].jump); block.numJump[block.top - 1]++; cmdLine[numOfLines].cmdno = SIM_CMD_GOTO; @@ -590,7 +588,7 @@ bool simParseBreakCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseDefaultCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseDefaultCmd(char *rest, SCommand *pCmd, int32_t lineNum) { if (block.top < 1) { sprintf(parseErr, "no matching switch"); return false; @@ -601,14 +599,15 @@ bool simParseDefaultCmd(char *rest, SCommand *pCmd, int lineNum) { return false; } - if (block.pos[block.top - 1] != NULL) + if (block.pos[block.top - 1] != NULL) { *(block.pos[block.top - 1]) = numOfLines; + } return true; } -bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) { - int i; +bool simParseEndsCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t i; if (block.top < 1) { sprintf(parseErr, "no matching switch"); @@ -620,8 +619,9 @@ bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) { return false; } - for (i = 0; i < block.numJump[block.top - 1]; ++i) + for (i = 0; i < block.numJump[block.top - 1]; ++i) { *(block.jump[block.top - 1][i]) = numOfLines; + } block.numJump[block.top - 1] = 0; block.top--; @@ -629,7 +629,7 @@ bool simParseEndsCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseContinueCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseContinueCmd(char *rest, SCommand *pCmd, int32_t lineNum) { if (block.top < 1) { sprintf(parseErr, "no matching while"); return false; @@ -648,14 +648,14 @@ bool simParseContinueCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParsePrintCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParsePrintCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; rest++; cmdLine[numOfLines].cmdno = SIM_CMD_PRINT; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int)strlen(rest); + expLen = (int32_t)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -665,8 +665,8 @@ bool simParsePrintCmd(char *rest, SCommand *pCmd, int lineNum) { } void simCheckSqlOption(char *rest) { - int valueLen; - char *value, *xpos; + int32_t valueLen; + char * value, *xpos; xpos = strstr(rest, " -x"); // need a blank if (xpos) { @@ -682,15 +682,15 @@ void simCheckSqlOption(char *rest) { } } -bool simParseSqlCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseSqlCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; rest++; simCheckSqlOption(rest); cmdLine[numOfLines].cmdno = SIM_CMD_SQL; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int)strlen(rest); + expLen = (int32_t)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -699,14 +699,14 @@ bool simParseSqlCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; rest++; cmdLine[numOfLines].cmdno = SIM_CMD_SQL_ERROR; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int)strlen(rest); + expLen = (int32_t)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -715,26 +715,26 @@ bool simParseSqlErrorCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseSqlSlowCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSqlCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_SQL_SLOW; return true; } -bool simParseRestfulCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseRestfulCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSqlCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_RESTFUL; return true; } -bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { - int expLen; +bool simParseSystemCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + int32_t expLen; rest++; cmdLine[numOfLines].cmdno = SIM_CMD_SYSTEM; cmdLine[numOfLines].lineNum = lineNum; cmdLine[numOfLines].optionOffset = optionOffset; - expLen = (int)strlen(rest); + expLen = (int32_t)strlen(rest); memcpy(optionBuffer + optionOffset, rest, expLen); optionOffset += expLen + 1; *(optionBuffer + optionOffset - 1) = 0; @@ -743,15 +743,15 @@ bool simParseSystemCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseSystemContentCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseSystemCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_SYSTEM_CONTENT; return true; } -bool simParseSleepCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseSleepCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; cmdLine[numOfLines].cmdno = SIM_CMD_SLEEP; cmdLine[numOfLines].lineNum = lineNum; @@ -768,9 +768,9 @@ bool simParseSleepCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseReturnCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseReturnCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; cmdLine[numOfLines].cmdno = SIM_CMD_RETURN; cmdLine[numOfLines].lineNum = lineNum; @@ -787,9 +787,9 @@ bool simParseReturnCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseGotoCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseGotoCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -810,9 +810,9 @@ bool simParseGotoCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseRunCmd(char *rest, SCommand *pCmd, int lineNum) { - char *token; - int tokenLen; +bool simParseRunCmd(char *rest, SCommand *pCmd, int32_t lineNum) { + char * token; + int32_t tokenLen; rest = paGetToken(rest, &token, &tokenLen); @@ -832,14 +832,14 @@ bool simParseRunCmd(char *rest, SCommand *pCmd, int lineNum) { return true; } -bool simParseRunBackCmd(char *rest, SCommand *pCmd, int lineNum) { +bool simParseRunBackCmd(char *rest, SCommand *pCmd, int32_t lineNum) { simParseRunCmd(rest, pCmd, lineNum); cmdLine[numOfLines - 1].cmdno = SIM_CMD_RUN_BACK; return true; } void simInitsimCmdList() { - int cmdno; + int32_t cmdno; memset(simCmdList, 0, SIM_CMD_END * sizeof(SCommand)); /* internal command */ diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 17df7f306a..71cdc74b3c 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "sim.h" #include "taos.h" @@ -24,11 +25,11 @@ SScript *simScriptList[MAX_MAIN_SCRIPT_NUM]; SCommand simCmdList[SIM_CMD_END]; -int simScriptPos = -1; -int simScriptSucced = 0; -int simDebugFlag = 135; -void simCloseTaosdConnect(SScript *script); -char simHostName[128]; +int32_t simScriptPos = -1; +int32_t simScriptSucced = 0; +int32_t simDebugFlag = 135; +void simCloseTaosdConnect(SScript *script); +char simHostName[128]; char *simParseArbitratorName(char *varName) { static char hostName[140]; @@ -39,8 +40,8 @@ char *simParseArbitratorName(char *varName) { char *simParseHostName(char *varName) { static char hostName[140]; - int index = atoi(varName + 8); - int port = 7100; + int32_t index = atoi(varName + 8); + int32_t port = 7100; switch (index) { case 1: port = 7100; @@ -70,9 +71,9 @@ char *simParseHostName(char *varName) { port = 7900; break; } - + sprintf(hostName, "'%s:%d'", simHostName, port); - //simInfo("hostName:%s", hostName); + // simInfo("hostName:%s", hostName); return hostName; } @@ -88,12 +89,19 @@ void simSystemCleanUp() {} void simFreeScript(SScript *script) { if (script->type == SIM_SCRIPT_TYPE_MAIN) { - for (int i = 0; i < script->bgScriptLen; ++i) { + simInfo("script:%s, background script num:%d, stop them", script->fileName, script->bgScriptLen); + + for (int32_t i = 0; i < script->bgScriptLen; ++i) { SScript *bgScript = script->bgScripts[i]; + simInfo("script:%s, set stop flag", script->fileName); bgScript->killed = true; + if (bgScript->bgPid) { + pthread_join(bgScript->bgPid, NULL); + } } } + simDebug("script:%s, is freed", script->fileName); taos_close(script->taos); taosTFree(script->lines); taosTFree(script->optionBuffer); @@ -103,25 +111,24 @@ void simFreeScript(SScript *script) { SScript *simProcessCallOver(SScript *script) { if (script->type == SIM_SCRIPT_TYPE_MAIN) { if (script->killed) { - simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX - "failed" FAILED_POSTFIX ", error:%s", - script->fileName, script->error); + simInfo("script:" FAILED_PREFIX "%s" FAILED_POSTFIX ", " FAILED_PREFIX "failed" FAILED_POSTFIX ", error:%s", + script->fileName, script->error); exit(-1); } else { - simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX - "success" SUCCESS_POSTFIX, - script->fileName); + simInfo("script:" SUCCESS_PREFIX "%s" SUCCESS_POSTFIX ", " SUCCESS_PREFIX "success" SUCCESS_POSTFIX, + script->fileName); simCloseTaosdConnect(script); simScriptSucced++; simScriptPos--; + + simFreeScript(script); if (simScriptPos == -1) { simInfo("----------------------------------------------------------------------"); simInfo("Simulation Test Done, " SUCCESS_PREFIX "%d" SUCCESS_POSTFIX " Passed:\n", simScriptSucced); exit(0); } - simFreeScript(script); - return simScriptList[simScriptPos]; + return NULL; } } else { simInfo("script:%s, is stopped by main script", script->fileName); @@ -143,11 +150,11 @@ void *simExecuteScript(void *inputScript) { if (script == NULL) break; } else { SCmdLine *line = &script->lines[script->linePos]; - char *option = script->optionBuffer + line->optionOffset; + char * option = script->optionBuffer + line->optionOffset; simDebug("script:%s, line:%d with option \"%s\"", script->fileName, line->lineNum, option); SCommand *cmd = &simCmdList[line->cmdno]; - int ret = (*(cmd->executeCmd))(script, option); + int32_t ret = (*(cmd->executeCmd))(script, option); if (!ret) { script->killed = true; } From fcb6b6a7c51e1f7af57f95f17da82959e7496019 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 8 Nov 2020 03:19:33 +0000 Subject: [PATCH 083/122] scrpits --- tests/script/jenkins/basic.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a5d04f79a6..1b2fe37c71 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -57,8 +57,8 @@ cd ../../../debug; make ./test.sh -f general/db/delete_reuse2.sim ./test.sh -f general/db/delete_reusevnode.sim ./test.sh -f general/db/delete_reusevnode2.sim -#./test.sh -f general/db/delete_writing1.sim -#./test.sh -f general/db/delete_writing2.sim +./test.sh -f general/db/delete_writing1.sim +./test.sh -f general/db/delete_writing2.sim ./test.sh -f general/db/delete.sim ./test.sh -f general/db/len.sim ./test.sh -f general/db/repeat.sim From 7a7242d8ae1ea88abf191db1dbd3ee2b76c96a78 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 8 Nov 2020 03:31:42 +0000 Subject: [PATCH 084/122] compile error in windows --- tests/tsim/src/simSystem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 71cdc74b3c..30d0d247b8 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -95,7 +95,7 @@ void simFreeScript(SScript *script) { SScript *bgScript = script->bgScripts[i]; simInfo("script:%s, set stop flag", script->fileName); bgScript->killed = true; - if (bgScript->bgPid) { + if (taosCheckPthreadValid(bgScript->bgPid)) { pthread_join(bgScript->bgPid, NULL); } } From a194ea05b35c86528c722d8628a37a9b97a5e00d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 8 Nov 2020 13:19:32 +0000 Subject: [PATCH 085/122] change wal cfg options --- src/inc/twal.h | 14 +++++++++----- src/mnode/src/mnodeSdb.c | 2 +- src/vnode/src/vnodeCfg.c | 3 +-- src/wal/src/walMgmt.c | 2 +- src/wal/src/walWrite.c | 9 ++++----- src/wal/test/waltest.c | 5 ----- 6 files changed, 16 insertions(+), 19 deletions(-) diff --git a/src/inc/twal.h b/src/inc/twal.h index 931cf5daba..05be90775f 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -25,6 +25,11 @@ typedef enum { TAOS_WAL_FSYNC = 2 } EWalType; +typedef enum { + TAOS_WAL_NOT_KEEP = 0, + TAOS_WAL_KEEP = 1 +} EWalKeep; + typedef struct { int8_t msgType; int8_t reserved[3]; @@ -36,11 +41,10 @@ typedef struct { } SWalHead; typedef struct { - int32_t vgId; - int32_t fsyncPeriod; // millisecond - int8_t walLevel; // wal level - int8_t wals; // number of WAL files; - int8_t keep; // keep the wal file when closed + int32_t vgId; + int32_t fsyncPeriod; // millisecond + EWalType walLevel; // wal level + EWalKeep keep; // keep the wal file when closed } SWalCfg; typedef void * twalh; // WAL HANDLE diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 8cd8de62ad..6b6a49db93 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -175,7 +175,7 @@ static void *sdbGetTableFromId(int32_t tableId) { } static int32_t sdbInitWal() { - SWalCfg walCfg = {.vgId = 1, .walLevel = 2, .wals = 2, .keep = 1, .fsyncPeriod = 0}; + SWalCfg walCfg = {.vgId = 1, .walLevel = TAOS_WAL_FSYNC, .keep = TAOS_WAL_KEEP, .fsyncPeriod = 0}; char temp[TSDB_FILENAME_LEN]; sprintf(temp, "%s/wal", tsMnodeDir); tsSdbObj.wal = walOpen(temp, &walCfg); diff --git a/src/vnode/src/vnodeCfg.c b/src/vnode/src/vnodeCfg.c index e8dd44b48f..e2e57a7566 100644 --- a/src/vnode/src/vnodeCfg.c +++ b/src/vnode/src/vnodeCfg.c @@ -39,8 +39,7 @@ static void vnodeLoadCfg(SVnodeObj *pVnode, SCreateVnodeMsg* vnodeMsg) { pVnode->tsdbCfg.compression = vnodeMsg->cfg.compression; pVnode->walCfg.walLevel = vnodeMsg->cfg.walLevel; pVnode->walCfg.fsyncPeriod = vnodeMsg->cfg.fsyncPeriod; - pVnode->walCfg.wals = vnodeMsg->cfg.wals; - pVnode->walCfg.keep = 0; + pVnode->walCfg.keep = TAOS_WAL_NOT_KEEP; pVnode->syncCfg.replica = vnodeMsg->cfg.replications; pVnode->syncCfg.quorum = vnodeMsg->cfg.quorum; diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 2ae342244d..1f6a8f5546 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -128,7 +128,7 @@ void walClose(void *handle) { taosClose(pWal->fd); - if (!pWal->keep) { + if (pWal->keep != TAOS_WAL_KEEP) { int64_t fileId = -1; while (walGetNextFile(pWal, &fileId) >= 0) { snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, fileId); diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 0d27ce1768..b4769fe57b 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -41,7 +41,7 @@ int32_t walRenew(void *handle) { wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name); } - if (pWal->keep) { + if (pWal->keep == TAOS_WAL_KEEP) { pWal->fileId = 0; } else { if (walGetNewFile(pWal, &pWal->fileId) != 0) pWal->fileId = 0; @@ -58,7 +58,7 @@ int32_t walRenew(void *handle) { wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name); } - if (!pWal->keep) { + if (pWal->keep != TAOS_WAL_KEEP) { // remove the oldest wal file int64_t oldFileId = -1; if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) { @@ -144,12 +144,12 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { continue; } - wDebug("vgId:%d, file:%s, restore success and keep it", pWal->vgId, walName); + wDebug("vgId:%d, file:%s, restore success", pWal->vgId, walName); count++; } - if (!pWal->keep) return TSDB_CODE_SUCCESS; + if (pWal->keep != TAOS_WAL_KEEP) return TSDB_CODE_SUCCESS; if (count == 0) { wDebug("vgId:%d, wal file not exist, renew it", pWal->vgId); @@ -173,7 +173,6 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { if (handle == NULL) return -1; SWal *pWal = handle; - // for keep if (*fileId == 0) *fileId = -1; pthread_mutex_lock(&(pWal->mutex)); diff --git a/src/wal/test/waltest.c b/src/wal/test/waltest.c index 14e439c072..7a473ed18c 100644 --- a/src/wal/test/waltest.c +++ b/src/wal/test/waltest.c @@ -37,7 +37,6 @@ int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { int main(int argc, char *argv[]) { char path[128] = "/home/jhtao/test/wal"; - int max = 3; int level = 2; int total = 5; int rows = 10000; @@ -47,8 +46,6 @@ int main(int argc, char *argv[]) { for (int i=1; i Date: Sun, 8 Nov 2020 13:26:54 +0000 Subject: [PATCH 086/122] change int to int32 --- src/cq/src/cqMain.c | 14 +++++++------- src/inc/tcq.h | 6 +++--- src/inc/twal.h | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 9efa517ac3..8b10860ef0 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -39,16 +39,16 @@ #define cTrace(...) { if (cqDebugFlag & DEBUG_TRACE) { taosPrintLog("CQ ", cqDebugFlag, __VA_ARGS__); }} typedef struct { - int vgId; + int32_t vgId; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; void *ahandle; - int num; // number of continuous streams + int32_t num; // number of continuous streams struct SCqObj *pHead; void *dbConn; - int master; + int32_t master; void *tmrCtrl; pthread_mutex_t mutex; } SCqContext; @@ -57,7 +57,7 @@ typedef struct SCqObj { tmr_h tmrId; uint64_t uid; int32_t tid; // table ID - int rowSize; // bytes of a row + int32_t rowSize; // bytes of a row char * sqlStr; // SQL string STSchema * pSchema; // pointer to schema array void * pStream; @@ -175,7 +175,7 @@ void cqStop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } -void *cqCreate(void *handle, uint64_t uid, int tid, char *sqlStr, STSchema *pSchema) { +void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) { SCqContext *pContext = handle; SCqObj *pObj = calloc(sizeof(SCqObj), 1); @@ -237,7 +237,7 @@ void cqDrop(void *handle) { pthread_mutex_unlock(&pContext->mutex); } -static void doCreateStream(void *param, TAOS_RES *result, int code) { +static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { SCqObj* pObj = (SCqObj*)param; SCqContext* pContext = pObj->pContext; SSqlObj* pSql = (SSqlObj*)result; @@ -288,7 +288,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { cDebug("vgId:%d, id:%d CQ:%s stream result is ready", pContext->vgId, pObj->tid, pObj->sqlStr); - int size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; + int32_t size = sizeof(SWalHead) + sizeof(SSubmitMsg) + sizeof(SSubmitBlk) + TD_DATA_ROW_HEAD_SIZE + pObj->rowSize; char *buffer = calloc(size, 1); SWalHead *pHead = (SWalHead *)buffer; diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 4a23695a1a..0f8a7a6a9e 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -24,7 +24,7 @@ extern "C" { typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); typedef struct { - int vgId; + int32_t vgId; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; @@ -42,12 +42,12 @@ void cqStart(void *handle); void cqStop(void *handle); // cqCreate is called by TSDB to start an instance of CQ -void *cqCreate(void *handle, uint64_t uid, int sid, char *sqlStr, STSchema *pSchema); +void *cqCreate(void *handle, uint64_t uid, int32_t sid, char *sqlStr, STSchema *pSchema); // cqDrop is called by TSDB to stop an instance of CQ, handle is the return value of cqCreate void cqDrop(void *handle); -extern int cqDebugFlag; +extern int32_t cqDebugFlag; #ifdef __cplusplus diff --git a/src/inc/twal.h b/src/inc/twal.h index 05be90775f..a9e0ddd6b7 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -32,7 +32,8 @@ typedef enum { typedef struct { int8_t msgType; - int8_t reserved[3]; + int8_t sver; + int8_t reserved[2]; int32_t len; uint64_t version; uint32_t signature; From 4f0ed0792185d9cb414d3539739212e0cef4e053 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 8 Nov 2020 13:56:41 +0000 Subject: [PATCH 087/122] TD-1919 change version type to uint64 --- src/inc/twal.h | 2 +- src/vnode/inc/vnodeInt.h | 4 ++-- src/vnode/src/vnodeMain.c | 3 +++ src/vnode/src/vnodeVersion.c | 8 ++++---- src/wal/src/walWrite.c | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/inc/twal.h b/src/inc/twal.h index a9e0ddd6b7..c32bb87021 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -63,7 +63,7 @@ int32_t walWrite(twalh, SWalHead *); void walFsync(twalh, bool forceFsync); int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); int32_t walGetWalFile(twalh, char *fileName, int64_t *fileId); -int64_t walGetVersion(twalh); +uint64_t walGetVersion(twalh); #ifdef __cplusplus } diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 89ec277006..317d4904cb 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -41,8 +41,8 @@ typedef struct { int8_t status; int8_t role; int8_t accessState; - int64_t version; // current version - int64_t fversion; // version on saved data file + uint64_t version; // current version + uint64_t fversion; // version on saved data file void *wqueue; void *rqueue; void *wal; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index f5b670f423..4cdf9f898a 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -227,6 +227,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { pVnode->vgId = vnode; pVnode->status = TAOS_VN_STATUS_INIT; + pVnode->fversion = 0; pVnode->version = 0; pVnode->tsdbCfg.tsdbId = pVnode->vgId; pVnode->rootDir = strdup(rootDir); @@ -288,6 +289,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { vnodeCleanUp(pVnode); return terrno; } else { + pVnode->fversion = 0; pVnode->version = 0; } } @@ -302,6 +304,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { walRestore(pVnode->wal, pVnode, vnodeProcessWrite); if (pVnode->version == 0) { + pVnode->fversion = 0; pVnode->version = walGetVersion(pVnode->wal); } diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c index 1d0695fb53..51f9ac8f3a 100644 --- a/src/vnode/src/vnodeVersion.c +++ b/src/vnode/src/vnodeVersion.c @@ -61,10 +61,10 @@ int32_t vnodeReadVersion(SVnodeObj *pVnode) { vError("vgId:%d, failed to read %s, version not found", pVnode->vgId, file); goto PARSE_VER_ERROR; } - pVnode->version = ver->valueint; + pVnode->version = (uint64_t)ver->valueint; terrno = TSDB_CODE_SUCCESS; - vInfo("vgId:%d, read %s successfully, version:%" PRId64, pVnode->vgId, file, pVnode->version); + vInfo("vgId:%d, read %s successfully, version:%" PRIu64, pVnode->vgId, file, pVnode->version); PARSE_VER_ERROR: if (content != NULL) free(content); @@ -89,7 +89,7 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) { char * content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); - len += snprintf(content + len, maxLen - len, " \"version\": %" PRId64 "\n", pVnode->fversion); + len += snprintf(content + len, maxLen - len, " \"version\": %" PRIu64 "\n", pVnode->fversion); len += snprintf(content + len, maxLen - len, "}\n"); fwrite(content, 1, len, fp); @@ -98,6 +98,6 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) { free(content); terrno = 0; - vInfo("vgId:%d, successed to write %s, version:%" PRId64, pVnode->vgId, file, pVnode->fversion); + vInfo("vgId:%d, successed to write %s, version:%" PRIu64, pVnode->vgId, file, pVnode->fversion); return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index b4769fe57b..ecd0fd0ca0 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -310,7 +310,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch return code; } -int64_t walGetVersion(twalh param) { +uint64_t walGetVersion(twalh param) { SWal *pWal = param; if (pWal == 0) return 0; From 32f104687556ef375d4f8c84f66ddf662e73433a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 8 Nov 2020 13:59:37 +0000 Subject: [PATCH 088/122] format --- src/inc/tcq.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 0f8a7a6a9e..7a0727f1b8 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -24,7 +24,7 @@ extern "C" { typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); typedef struct { - int32_t vgId; + int32_t vgId; char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; From 4a6a9ec77963fcb57daabb163bd9a104bf93eec1 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sun, 8 Nov 2020 23:31:30 +0800 Subject: [PATCH 089/122] [TD-1794] add test case --- tests/perftest-scripts/perftest-query.sh | 92 ++++++++++++++++++++++++ tests/pytest/fulltest.sh | 3 + tests/pytest/wal/addOldWalTest.py | 70 ++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100755 tests/perftest-scripts/perftest-query.sh create mode 100644 tests/pytest/wal/addOldWalTest.py diff --git a/tests/perftest-scripts/perftest-query.sh b/tests/perftest-scripts/perftest-query.sh new file mode 100755 index 0000000000..bb5d9e0a9d --- /dev/null +++ b/tests/perftest-scripts/perftest-query.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +today=`date +"%Y%m%d"` +WORK_DIR=/home/ubuntu/pxiao/ +PERFORMANCE_TEST_REPORT=$TDENGINE_DIR/tests/performance-test-report-$today.log + +# Coloured Echoes # +function red_echo { echo -e "\033[31m$@\033[0m"; } # +function green_echo { echo -e "\033[32m$@\033[0m"; } # +function yellow_echo { echo -e "\033[33m$@\033[0m"; } # +function white_echo { echo -e "\033[1;37m$@\033[0m"; } # +# Coloured Printfs # +function red_printf { printf "\033[31m$@\033[0m"; } # +function green_printf { printf "\033[32m$@\033[0m"; } # +function yellow_printf { printf "\033[33m$@\033[0m"; } # +function white_printf { printf "\033[1;37m$@\033[0m"; } # +# Debugging Outputs # +function white_brackets { local args="$@"; white_printf "["; printf "${args}"; white_printf "]"; } # +function echoInfo { local args="$@"; white_brackets $(green_printf "INFO") && echo " ${args}"; } # +function echoWarn { local args="$@"; echo "$(white_brackets "$(yellow_printf "WARN")" && echo " ${args}";)" 1>&2; } # +function echoError { local args="$@"; echo "$(white_brackets "$(red_printf "ERROR")" && echo " ${args}";)" 1>&2; } # + + +function stopTaosd { + echo "Stop taosd" + systemctl stop taosd + snap stop tdengine + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + while [ -n "$PID" ] + do + pkill -TERM -x taosd + sleep 1 + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` + done +} + +function buildTDengine { + echoInfo "Build TDengine" + cd $WORK_DIR/TDengine + + git remote update > /dev/null + REMOTE_COMMIT=`git rev-parse --short remotes/origin/develop` + LOCAL_COMMIT=`git rev-parse --short @` + + echo " LOCAL: $LOCAL_COMMIT" + echo "REMOTE: $REMOTE_COMMIT" + if [ "$LOCAL_COMMIT" == "$REMOTE_COMMIT" ]; then + echo "repo up-to-date" + else + echo "repo need to pull" + git pull > /dev/null + + LOCAL_COMMIT=`git rev-parse --short @` + cd debug + rm -rf * + cmake .. > /dev/null + make > /dev/null + make install + fi +} + +function runQueryPerfTest { + nohup $WORK_DIR/TDengine/debug/build/bin/taosd -c /etc/taodperf/ > /dev/null 2>&1 & + echoInfo "Run Performance Test" + cd $WORK_DIR/TDengine/tests/pytest + + python3 query/queryPerformance.py | tee -a $PERFORMANCE_TEST_REPORT +} + + +function sendReport { + echo "send report" + receiver="pxiao@taosdata.com" + mimebody="MIME-Version: 1.0\nContent-Type: text/html; charset=utf-8\n" + + cd $TDENGINE_DIR + + sed -i 's/\x1b\[[0-9;]*m//g' $PERFORMANCE_TEST_REPORT + BODY_CONTENT=`cat $PERFORMANCE_TEST_REPORT` + echo -e "to: ${receiver}\nsubject: Query Performace Report ${today}, commit ID: ${LOCAL_COMMIT}\n\n${today}:\n${BODY_CONTENT}" | \ + (cat - && uuencode $PERFORMANCE_TEST_REPORT performance-test-report-$today.log) | \ + ssmtp "${receiver}" && echo "Report Sent!" +} + + +stopTaosd +buildTDengine +runQueryPerfTest + +echoInfo "Send Report" +sendReport +echoInfo "End of Test" diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index b555a27c93..a48dbdc480 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -221,3 +221,6 @@ python3 ./test.py -f update/merge_commit_data2.py python3 ./test.py -f update/merge_commit_data2_update0.py python3 ./test.py -f update/merge_commit_last-0.py python3 ./test.py -f update/merge_commit_last.py + +# wal +python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file diff --git a/tests/pytest/wal/addOldWalTest.py b/tests/pytest/wal/addOldWalTest.py new file mode 100644 index 0000000000..2f4dcd5ce8 --- /dev/null +++ b/tests/pytest/wal/addOldWalTest.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def createOldDir(self): + oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old" + os.system("sudo mkdir -p %s" % oldDir) + + def createOldDirAndAddWal(self): + oldDir = tdDnodes.getDnodesRootDir() + "dnode1/data/vnode/vnode2/wal/old" + os.system("sudo echo 'test' >> %s/wal" % oldDir) + + + def run(self): + tdSql.prepare() + + tdSql.execute("create table t1(ts timestamp, a int)") + tdSql.execute("insert into t1 values(now, 1)") + + # create old dir only + self.createOldDir() + os.system("sudo kill -9 $(pgrep taosd)") + tdDnodes.start(1) + + tdSql.execute("use db") + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + # create old dir and add wal under old dir + self.createOldDir() + self.createOldDirAndAddWal() + os.system("sudo kill -9 $(pgrep taosd)") + tdDnodes.start(1) + + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 7c86618ce626c626a9e6db2e068af3e92baff477 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 9 Nov 2020 02:16:25 +0000 Subject: [PATCH 090/122] TD-1952 --- tests/tsim/src/simSystem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 30d0d247b8..8c3f3b8803 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -128,7 +128,7 @@ SScript *simProcessCallOver(SScript *script) { exit(0); } - return NULL; + return simScriptList[simScriptPos]; } } else { simInfo("script:%s, is stopped by main script", script->fileName); From 13e35660ef9cd5c010f5c8070b64959df010dcc8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 9 Nov 2020 02:16:59 +0000 Subject: [PATCH 091/122] change ulog to slog in taosTcpPool --- src/sync/src/taosTcpPool.c | 103 +++++++++++++++++++------------------ 1 file changed, 53 insertions(+), 50 deletions(-) diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index 6a210a136f..7397a629d3 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -13,18 +13,22 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "tulog.h" #include "tutil.h" #include "tsocket.h" #include "taoserror.h" #include "taosTcpPool.h" +#include "twal.h" +#include "tsync.h" +#include "syncInt.h" typedef struct SThreadObj { pthread_t thread; bool stop; - int pollFd; - int numOfFds; + int32_t pollFd; + int32_t numOfFds; struct SPoolObj *pPool; } SThreadObj; @@ -32,15 +36,15 @@ typedef struct SPoolObj { SPoolInfo info; SThreadObj **pThread; pthread_t thread; - int nextId; - int acceptFd; // FD for accept new connection + int32_t nextId; + int32_t acceptFd; // FD for accept new connection } SPoolObj; typedef struct { SThreadObj *pThread; - void *ahandle; - int fd; - int closedByApp; + void * ahandle; + int32_t fd; + int32_t closedByApp; } SConnObj; static void *taosAcceptPeerTcpConnection(void *argv); @@ -53,66 +57,66 @@ void *taosOpenTcpThreadPool(SPoolInfo *pInfo) { SPoolObj *pPool = calloc(sizeof(SPoolObj), 1); if (pPool == NULL) { - uError("TCP server, no enough memory"); + sError("failed to alloc pool for TCP server since no enough memory"); return NULL; } pPool->info = *pInfo; - pPool->pThread = (SThreadObj **)calloc(sizeof(SThreadObj *), pInfo->numOfThreads); + pPool->pThread = calloc(sizeof(SThreadObj *), pInfo->numOfThreads); if (pPool->pThread == NULL) { - uError("TCP server, no enough memory"); - free(pPool); + sError("failed to alloc pool thread for TCP server since no enough memory"); + tfree(pPool); return NULL; } pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port); if (pPool->acceptFd < 0) { - free(pPool->pThread); - free(pPool); - uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno)); + tfree(pPool->pThread); + tfree(pPool); + sError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno)); return NULL; } pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); if (pthread_create(&(pPool->thread), &thattr, (void *)taosAcceptPeerTcpConnection, pPool) != 0) { - uError("TCP server, failed to create accept thread, reason:%s", strerror(errno)); + sError("failed to create accept thread for TCP server since %s", strerror(errno)); close(pPool->acceptFd); - free(pPool->pThread); - free(pPool); + tfree(pPool->pThread); + tfree(pPool); return NULL; } pthread_attr_destroy(&thattr); - uDebug("%p TCP pool is created", pPool); + sDebug("%p TCP pool is created", pPool); return pPool; } void taosCloseTcpThreadPool(void *param) { - SPoolObj * pPool = (SPoolObj *)param; + SPoolObj * pPool = param; SThreadObj *pThread; shutdown(pPool->acceptFd, SHUT_RD); pthread_join(pPool->thread, NULL); - for (int i = 0; i < pPool->info.numOfThreads; ++i) { + for (int32_t i = 0; i < pPool->info.numOfThreads; ++i) { pThread = pPool->pThread[i]; if (pThread) taosStopPoolThread(pThread); } - uDebug("%p TCP pool is closed", pPool); + sDebug("%p TCP pool is closed", pPool); taosTFree(pPool->pThread); - free(pPool); + tfree(pPool); } -void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { +void *taosAllocateTcpConn(void *param, void *pPeer, int32_t connFd) { struct epoll_event event; - SPoolObj *pPool = (SPoolObj *)param; + SPoolObj *pPool = param; - SConnObj *pConn = (SConnObj *)calloc(sizeof(SConnObj), 1); + SConnObj *pConn = calloc(sizeof(SConnObj), 1); if (pConn == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; @@ -120,7 +124,7 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { SThreadObj *pThread = taosGetTcpThread(pPool); if (pThread == NULL) { - free(pConn); + tfree(pConn); return NULL; } @@ -133,13 +137,13 @@ void *taosAllocateTcpConn(void *param, void *pPeer, int connFd) { event.data.ptr = pConn; if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) { - uError("failed to add fd:%d(%s)", connFd, strerror(errno)); + sError("failed to add fd:%d since %s", connFd, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - free(pConn); + tfree(pConn); pConn = NULL; } else { pThread->numOfFds++; - uDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); + sDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds); } return pConn; @@ -149,7 +153,7 @@ void taosFreeTcpConn(void *param) { SConnObj * pConn = (SConnObj *)param; SThreadObj *pThread = pConn->pThread; - uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); + sDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd); pConn->closedByApp = 1; shutdown(pConn->fd, SHUT_WR); } @@ -164,9 +168,9 @@ static void taosProcessBrokenLink(SConnObj *pConn) { pThread->numOfFds--; epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL); - uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); + sDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds); taosClose(pConn->fd); - free(pConn); + tfree(pConn); } #define maxEvents 10 @@ -183,18 +187,18 @@ static void *taosProcessTcpData(void *param) { while (1) { if (pThread->stop) break; - int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); + int32_t fdNum = epoll_wait(pThread->pollFd, events, maxEvents, TAOS_EPOLL_WAIT_TIME); if (pThread->stop) { - uDebug("%p TCP epoll thread is exiting...", pThread); + sDebug("%p TCP epoll thread is exiting...", pThread); break; } if (fdNum < 0) { - uError("epoll_wait failed (%s)", strerror(errno)); + sError("epoll_wait failed since %s", strerror(errno)); continue; } - for (int i = 0; i < fdNum; ++i) { + for (int32_t i = 0; i < fdNum; ++i) { pConn = events[i].data.ptr; assert(pConn); @@ -219,17 +223,16 @@ static void *taosProcessTcpData(void *param) { continue; } } - } if (pThread->stop) break; } - uDebug("%p TCP epoll thread exits", pThread); + sDebug("%p TCP epoll thread exits", pThread); close(pThread->pollFd); - free(pThread); - free(buffer); + tfree(pThread); + tfree(buffer); return NULL; } @@ -242,18 +245,18 @@ static void *taosAcceptPeerTcpConnection(void *argv) { while (1) { struct sockaddr_in clientAddr; socklen_t addrlen = sizeof(clientAddr); - int connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); + int32_t connFd = accept(pPool->acceptFd, (struct sockaddr *)&clientAddr, &addrlen); if (connFd < 0) { if (errno == EINVAL) { - uDebug("%p TCP server accept is exiting...", pPool); + sDebug("%p TCP server accept is exiting...", pPool); break; } else { - uError("TCP accept failure, reason:%s", strerror(errno)); + sError("TCP accept failure since %s", strerror(errno)); continue; } } - // uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); + // sDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); taosKeepTcpAlive(connFd); (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr); } @@ -273,23 +276,23 @@ static SThreadObj *taosGetTcpThread(SPoolObj *pPool) { pThread->pPool = pPool; pThread->pollFd = epoll_create(10); // size does not matter if (pThread->pollFd < 0) { - free(pThread); + tfree(pThread); return NULL; } pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - int ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread); + int32_t ret = pthread_create(&(pThread->thread), &thattr, (void *)taosProcessTcpData, pThread); pthread_attr_destroy(&thattr); if (ret != 0) { close(pThread->pollFd); - free(pThread); + tfree(pThread); return NULL; } - uDebug("%p TCP epoll thread is created", pThread); + sDebug("%p TCP epoll thread is created", pThread); pPool->pThread[pPool->nextId] = pThread; pPool->nextId++; pPool->nextId = pPool->nextId % pPool->info.numOfThreads; @@ -314,12 +317,12 @@ static void taosStopPoolThread(SThreadObj *pThread) { eventfd_t fd = eventfd(1, 0); if (fd == -1) { // failed to create eventfd, call pthread_cancel instead, which may result in data corruption - uError("failed to create eventfd(%s)", strerror(errno)); + sError("failed to create eventfd since %s", strerror(errno)); pthread_cancel(pThread->thread); pThread->stop = true; } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption - uError("failed to call epoll_ctl(%s)", strerror(errno)); + sError("failed to call epoll_ctl since %s", strerror(errno)); pthread_cancel(pThread->thread); } From 2c02b8cd0accb8d15144150a6b5e483224e80abc Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 9 Nov 2020 11:42:41 +0800 Subject: [PATCH 092/122] [TD-1990]fix test case error --- tests/test-all.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index f4e992eb5a..214360f3b8 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -26,7 +26,12 @@ function runPyCaseOneByOne { while read -r line; do if [[ $line =~ ^python.* ]]; then if [[ $line != *sleep* ]]; then - case=`echo $line|awk '{print $NF}'` + + if [[ $line =~ '-r' ]];then + case=`echo $line|awk '{print $4}'` + else + case=`echo $line|awk '{print $NF}'` + fi start_time=`date +%s` $line > /dev/null 2>&1 && \ echo -e "${GREEN}$case success${NC}" | tee -a pytest-out.log || \ From 98e380be51fe8c5d471b84affe8a17bae419e14d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 9 Nov 2020 05:41:58 +0000 Subject: [PATCH 093/122] add log for wal --- src/common/src/tglobal.c | 2 ++ src/dnode/src/dnodeVWrite.c | 4 ++-- src/inc/taosdef.h | 2 ++ src/sync/src/syncMain.c | 4 ++-- src/sync/src/syncRestore.c | 2 +- src/sync/src/syncRetrieve.c | 4 ++-- src/vnode/src/vnodeMain.c | 5 ++++- src/vnode/src/vnodeVersion.c | 4 ++-- src/vnode/src/vnodeWrite.c | 14 +++++++++----- src/wal/src/walWrite.c | 10 +++++----- 10 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 75c8f93a82..3bc1e4d0cc 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -218,6 +218,8 @@ int32_t (*monitorStartSystemFp)() = NULL; void (*monitorStopSystemFp)() = NULL; void (*monitorExecuteSQLFp)(char *sql) = NULL; +char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; + static pthread_once_t tsInitGlobalCfgOnce = PTHREAD_ONCE_INIT; void taosSetAllDebugFlag() { diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 9b7497d3ec..e0345eb1f6 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -207,8 +207,8 @@ static void *dnodeProcessVWriteQueue(void *param) { bool forceFsync = false; for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); - dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%d version:%" PRIu64, pWrite->rpcAhandle, pWrite, - taosMsg[pWrite->pHead->msgType], qtype, pWrite->pHead->version); + dTrace("%p, msg:%p:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite->rpcAhandle, pWrite, + taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet); if (pWrite->code <= 0) pWrite->processedCount = 1; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 6ee2567322..fa8af2c67e 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -467,6 +467,8 @@ typedef enum { TSDB_CHECK_ITEM_MAX } ECheckItemType; +extern char *qtypeStr[]; + #ifdef __cplusplus } #endif diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index eee8af7bcd..3d860f8bbb 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1221,8 +1221,8 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle // always update version nodeVersion = pWalHead->version; - sDebug("vgId:%d, replica:%d nodeRole:%s qtype:%d ver:%" PRIu64, pNode->vgId, pNode->replica, syncRole[nodeRole], - qtype, pWalHead->version); + sDebug("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, + syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 9216567bc5..b35388db35 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -153,7 +153,7 @@ static int syncRestoreWal(SSyncPeer *pPeer) { ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); if (ret < 0) break; - sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version); + sDebug("%s, restore a record, qtype:wal hver:%" PRIu64, pPeer->id, pHead->version); (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL); } diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 0fe189db7a..d77587f227 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -256,7 +256,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, } (void)lseek(sfd, offset, SEEK_SET); - sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion); + sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion); while (1) { int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); @@ -325,7 +325,7 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { // if all data up to fversion is read out, it is over if (pPeer->sversion >= fversion && fversion > 0) { code = 0; - sDebug("%s, data up to fversion:%" PRId64 " has been read out, bytes:%d", pPeer->id, fversion, bytes); + sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes); break; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 4cdf9f898a..f90b9701fe 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -577,10 +577,12 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { if (status == TSDB_STATUS_COMMIT_START) { pVnode->fversion = pVnode->version; + vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); return walRenew(pVnode->wal); } if (status == TSDB_STATUS_COMMIT_OVER) { + vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); return vnodeSaveVersion(pVnode); } @@ -656,11 +658,12 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) { static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) { SVnodeObj *pVnode = ahandle; - vDebug("vgId:%d, data file is synced, fversion:%" PRId64, pVnode->vgId, fversion); pVnode->fversion = fversion; pVnode->version = fversion; vnodeSaveVersion(pVnode); + vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, + pVnode->version); return vnodeResetTsdb(pVnode); } diff --git a/src/vnode/src/vnodeVersion.c b/src/vnode/src/vnodeVersion.c index 51f9ac8f3a..8f6360b4f9 100644 --- a/src/vnode/src/vnodeVersion.c +++ b/src/vnode/src/vnodeVersion.c @@ -64,7 +64,7 @@ int32_t vnodeReadVersion(SVnodeObj *pVnode) { pVnode->version = (uint64_t)ver->valueint; terrno = TSDB_CODE_SUCCESS; - vInfo("vgId:%d, read %s successfully, version:%" PRIu64, pVnode->vgId, file, pVnode->version); + vInfo("vgId:%d, read %s successfully, fver:%" PRIu64, pVnode->vgId, file, pVnode->version); PARSE_VER_ERROR: if (content != NULL) free(content); @@ -98,6 +98,6 @@ int32_t vnodeSaveVersion(SVnodeObj *pVnode) { free(content); terrno = 0; - vInfo("vgId:%d, successed to write %s, version:%" PRIu64, pVnode->vgId, file, pVnode->fversion); + vInfo("vgId:%d, successed to write %s, fver:%" PRIu64, pVnode->vgId, file, pVnode->fversion); return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index f42b359b26..99594607ac 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -52,20 +52,24 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara SRspRet * pRspRet = rparam; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) { - vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[pHead->msgType]); + vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId, + taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version); return TSDB_CODE_VND_MSG_NOT_PROCESSED; } + vTrace("vgId:%d, msg:%s will be processed in vnode, qtype:%s hver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, + taosMsg[pHead->msgType], qtypeStr[qtype], pHead->version, pVnode->version); + if (pHead->version == 0) { // from client or CQ if (pVnode->status != TAOS_VN_STATUS_READY) { - vDebug("vgId:%d, msgType:%s not processed, vnode status is %d", pVnode->vgId, taosMsg[pHead->msgType], - pVnode->status); + vDebug("vgId:%d, msg:%s not processed since vstatus:%d, qtype:%s hver:%" PRIu64, pVnode->vgId, + taosMsg[pHead->msgType], pVnode->status, qtypeStr[qtype], pHead->version); return TSDB_CODE_APP_NOT_READY; // it may be in deleting or closing state } if (pVnode->role != TAOS_SYNC_ROLE_MASTER) { - vDebug("vgId:%d, msgType:%s not processed, replica:%d role:%s", pVnode->vgId, taosMsg[pHead->msgType], - pVnode->syncCfg.replica, syncRole[pVnode->role]); + vDebug("vgId:%d, msg:%s not processed since replica:%d role:%s, qtype:%s hver:%" PRIu64, pVnode->vgId, + taosMsg[pHead->msgType], pVnode->syncCfg.replica, syncRole[pVnode->role], qtypeStr[qtype], pHead->version); return TSDB_CODE_APP_NOT_READY; } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index ecd0fd0ca0..9681f4b898 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -99,8 +99,8 @@ int32_t walWrite(void *handle, SWalHead *pHead) { code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%s, failed to write since %s", pWal->vgId, pWal->name, strerror(errno)); } else { - wTrace("vgId:%d, fileId:%" PRId64 " fd:%d, write wal ver:%" PRId64 ", head ver:%" PRIu64 ", len:%d ", pWal->vgId, - pWal->fileId, pWal->fd, pWal->version, pHead->version, pHead->len); + wTrace("vgId:%d, write wal, fileId:%" PRId64 " fd:%d hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId, + pWal->fileId, pWal->fd, pHead->version, pWal->version, pHead->len); pWal->version = pHead->version; } @@ -261,7 +261,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch } if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { - wError("vgId:%d, file:%s, wal head cksum is messed up, ver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, + wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, pHead->version, pHead->len, offset); code = walSkipCorruptedRecord(pWal, pHead, fd, &offset); if (code != TSDB_CODE_SUCCESS) { @@ -297,8 +297,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch offset = offset + sizeof(SWalHead) + pHead->len; - wTrace("vgId:%d, fileId:%" PRId64 ", restore wal ver:%" PRIu64 ", head ver:%" PRIu64 " len:%d", pWal->vgId, fileId, - pWal->version, pHead->version, pHead->len); + wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, + fileId, pHead->version, pWal->version, pHead->len); pWal->version = pHead->version; (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); From 19e385c89a9ca5a4b48307a37698edf4b9f9d44d Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Mon, 9 Nov 2020 13:34:18 +0800 Subject: [PATCH 094/122] insert data for ningsi --- tests/pytest/insert/ningsiInsert.py | 88 +++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/pytest/insert/ningsiInsert.py diff --git a/tests/pytest/insert/ningsiInsert.py b/tests/pytest/insert/ningsiInsert.py new file mode 100644 index 0000000000..bcad2b03ed --- /dev/null +++ b/tests/pytest/insert/ningsiInsert.py @@ -0,0 +1,88 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import requests, json +import threading +import string +import random +import time + +class RestfulInsert: + def init(self): + self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} + self.url = "http://ningsi60:6041/rest/sql" + self.ts = 1104508800000 + self.numOfThreads = 10 + self.numOfTables = 3000 + self.dbName = 'netmonitortaos' + self.stbName = 'devinfomt' + self.prefix = 'dev' + + def get_random_string(self, length): + letters = string.ascii_lowercase + result_str = ''.join(random.choice(letters) for i in range(length)) + return result_str + + def createTables(self, threadID): + print("create table: thread %d started" % threadID) + tablesPerThread = int (self.numOfTables / self.numOfThreads) + for i in range(tablesPerThread): + data = "create table '%s'.dev_%d using '%s'.'%s' tags('%s', '%s')" % (self.dbName, i + threadID * tablesPerThread, self.dbName, self.stbName, self.get_random_string(25), self.get_random_string(25)) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) + + def insertData(self, threadID): + print("insert data: thread %d started" % threadID) + tablesPerThread = int (self.numOfTables / self.numOfThreads) + base_ts = self.ts + while True: + i = 0 + for i in range(tablesPerThread): + data = "insert into %s.dev_%d values(%d, '%s', '%s', %d, %d, %d)" % (self.dbName, i + threadID * tablesPerThread, base_ts, self.get_random_string(25), self.get_random_string(30), random.randint(1, 10000), random.randint(1, 10000), random.randint(1, 10000)) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) + + time.sleep(30) + base_ts = base_ts + 1 + + def run(self): + data = "create database if not exists %s keep 7300" % self.dbName + requests.post(self.url, data, headers = self.header) + + data = "create table '%s'.'%s' (timeid timestamp, devdesc binary(50), devname binary(50), cpu bigint, temp bigint, ram bigint) tags(devid binary(50), modelid binary(50))" % (self.dbName, self.stbName) + requests.post(self.url, data, headers = self.header) + + threads = [] + for i in range(self.numOfThreads): + thread = threading.Thread(target=self.createTables, args=(i,)) + thread.start() + threads.append(thread) + + for i in range(self.numOfThreads): + threads[i].join() + + threads = [] + for i in range(self.numOfThreads): + thread = threading.Thread(target=self.insertData, args=(i,)) + thread.start() + threads.append(thread) + + for i in range(self.numOfThreads): + threads[i].join() + +ri = RestfulInsert() +ri.init() +ri.run() From b29d96f1f1ad997a2b8a36e14d596d5436b45a32 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 9 Nov 2020 15:31:53 +0800 Subject: [PATCH 095/122] [TD-1890] add test cae for TD-1876 --- tests/pytest/query/bug1876.py | 58 +++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 tests/pytest/query/bug1876.py diff --git a/tests/pytest/query/bug1876.py b/tests/pytest/query/bug1876.py new file mode 100644 index 0000000000..a1cc4b6eb2 --- /dev/null +++ b/tests/pytest/query/bug1876.py @@ -0,0 +1,58 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + stable=0 + insertRows = 1000 + tbnum = 3 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(tbnum): + tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + stable=stable+1 + for i in range(tbnum): + tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + print("==========step2") + print("join query ") + tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;") + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 6995a3d8f9293086522ca952d0a9cd8b2f1a137b Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Mon, 9 Nov 2020 16:08:38 +0800 Subject: [PATCH 096/122] [TD-1988] add query_performance.py --- tests/pytest/query/query_performance.py | 218 ++++++++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 tests/pytest/query/query_performance.py diff --git a/tests/pytest/query/query_performance.py b/tests/pytest/query/query_performance.py new file mode 100644 index 0000000000..d70adce7a5 --- /dev/null +++ b/tests/pytest/query/query_performance.py @@ -0,0 +1,218 @@ +import time +import taos +import csv +import numpy as np +import random +import os +import requests +import json +import sys + +""" +需要第三方库： taos,requests,numpy +当前机器已经启动taosd服务 +使用方法见底部示例 +""" + + + +class Ding: + """ + 发送消息到钉钉, + urls: 钉钉群的token组成的list，可以发多个钉钉群，需要提前加白名单或其他放行策略 + at_mobiles： 需要@的人的手机号组成的list + msg： 要发送的str + """ + def __init__(self, url_list, at_mobiles): + self.urls = url_list + self.at_mobiles = at_mobiles + + def send_message(self, msg): + data1 = { + "msgtype": "text", + "text": { + "content": msg + }, + "at": { + "atMobiles": self.at_mobiles, + "isAtAll": False + } + } + + header = {'Content-Type': 'application/json; charset=utf-8'} + + for url in self.urls: + requests.post(url=url, data=json.dumps(data1), headers=header) + + + + +class TDConn: + def __init__(self, config:dict): + self.host = config['host'] + self.user = config['user'] + self.password = config['password'] + self.config = config['config'] + self.conn = None + self.cursor = None + + def connect(self): + conn = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config) + cursor = conn.cursor() + self.conn = conn + self.cursor = cursor + print('connect ...') + return self.cursor + + def close(self): + self.cursor.close() + self.conn.close() + print('close ... ') + + +class Tool: + """ + 可能有用 + """ + @staticmethod + def str_gen(num): + return ''.join(random.sample('abcdefghijklmnopqrstuvwxyz', num)) + + @staticmethod + def float_gen(n, m): + return random.uniform(n, m) + + @staticmethod + def int_gen(n, m): + return random.randint(n, m) + +class Demo: + def __init__(self, engine): + self.engine = engine['engine'](engine['config']) + self.cursor = self.engine.connect() + + + def date_gen(self, db, number_per_table, type_of_cols, num_of_cols_per_record, num_of_tables): + """ + :目前都是 taosdemo 的参数 + :return: + """ + sql = 'yes | sudo taosdemo -d {db} -n {number_per_table} -b {type_of_cols} -l {num_of_cols_per_record} ' \ + '-t {num_of_tables}'.format(db=db, number_per_table=number_per_table, type_of_cols=type_of_cols, + num_of_cols_per_record=num_of_cols_per_record, num_of_tables=num_of_tables) + os.system(sql) + print('insert data completed') + + + # def main(self, db, circle, csv_name, case_func, result_csv, nums, ding_flag): + def main(self, every_num_per_table, result_csv, all_result_csv, values): + db = values['db_name'] + number_per_table = every_num_per_table + type_of_cols = values['col_type'] + num_of_cols_per_record = values['col_num'] + num_of_tables = values['table_num'] + self.date_gen(db=db, number_per_table=number_per_table, type_of_cols=type_of_cols, + num_of_cols_per_record=num_of_cols_per_record, num_of_tables=num_of_tables) + + circle = values['circle'] + # print(every_num_per_table, result_csv, values) + csv_name = result_csv + case_func = values['sql_func'] + nums = num_of_tables * number_per_table + ding_flag = values['ding_flag'] + + _data = [] + f = open(csv_name,'w',encoding='utf-8') + f1 = open(all_result_csv,'a',encoding='utf-8') + csv_writer = csv.writer(f) + csv_writer1 = csv.writer(f1) + csv_writer.writerow(["number", "elapse", 'sql']) + self.cursor.execute('use {db};'.format(db=db)) + + + for i in range(circle): + self.cursor.execute('reset query cache;') + sql = case_func() + start = time.time() + self.cursor.execute(sql) + self.cursor.fetchall() + end = time.time() + _data.append(end-start) + elapse = '%.4f' %(end -start) + print(sql, i, elapse, '\n') + csv_writer.writerow([i+1, elapse, sql]) + + # time.sleep(1) + _list = [nums, np.mean(_data)] + _str = '总数据: %s 条 , table数: %s , 每个table数据数: %s , 数据类型: %s \n' % \ + (nums, num_of_tables, number_per_table, type_of_cols) + # print('avg : ', np.mean(_data), '\n') + _str += '平均值 : %.4f 秒\n' % np.mean(_data) + for each in (50, 80, 90, 95): + _list.append(np.percentile(_data,each)) + _str += ' %d 分位数 : %.4f 秒\n' % (each , np.percentile(_data,each)) + + if ding_flag: + ding = Ding(values['ding_config']['urls'], values['ding_config']['at_mobiles']) + ding.send_message(_str) + csv_writer1.writerow(_list) + f.close() + f1.close() + self.engine.close() + + +def run(engine, test_cases: dict, result_dir): + for each_case, values in test_cases.items(): + for every_num_per_table in values['number_per_table']: + result_csv = result_dir + '{case}_table{table_num}_{number_per_table}.csv'.\ + format(case=each_case, table_num=values['table_num'], number_per_table=every_num_per_table) + all_result_csv = result_dir + '{case_all}_result.csv'.format(case_all=each_case) + d = Demo(engine) + # print(each_case, result_csv) + d.main(every_num_per_table, result_csv, all_result_csv, values) + + + +if __name__ == '__main__': + """ + 测试用例在test_cases中添加。 + result_dir： 报告生成目录，会生成每次测试结果，和具体某一用例的统计结果.需注意目录权限需要执行用户可写。 + case1、case2 : 具体用例名称 + engine: 数据库引擎，目前只有taosd。使用时需开启taosd服务。 + table_num: 造数据时的table数目 + circle: 循环测试次数，求平均值 + number_per_table：需要传list，多个数值代表会按照list内的数值逐个测试 + col_num：table col的数目 + col_type： 表中数据类型 + db_name： 造数据的db名，默认用test + sql_func： 当前测试的sql方法，需要自己定义 + ding_flag： 如果需要钉钉发送数据，flag设置真值， + ding_config： 如ding_flag 设置为真值，此项才有意义。ding_flag为假时此项可以为空。urls传入一list，内容为要发送的群的token， + 需提前设置白名单，at_mobiles传入一list，内容为在群内需要@的人的手机号 + """ + engine_dict = { + 'taosd': {'engine': TDConn, 'config': + {'host': '127.0.0.1', 'user': 'root', 'password': 'taosdata', 'config':'/etc/taos'}} + } + + def case1(): + return 'select * from meters where f1 = {n};'.format(n=random.randint(1,30)) + + def case2(): + return 'select * from meters where f1 = %.4f;' %random.uniform(1,30) + + + result_dir = '/usr/local/demo/benchmarktestdata/' + test_cases = { + 'case1': {'engine':'taosd', 'table_num': 10, 'circle': 100, 'number_per_table':[10, 100], 'col_num': 5, + 'col_type': 'INT', 'db_name': 'test', 'sql_func': case1, 'ding_flag': True, + 'ding_config': + {'urls': [r'https://oapi.dingtalk.com/robot/send?access_token=58fe61c28ee7142a90d2b1e7b9a8b9d99f502f248739c12a7fb62579edc0cd93'], + 'at_mobiles':[17080138990,],}}, + 'case2': {'engine':'taosd', 'table_num': 10, 'circle': 50, 'number_per_table':[10, 100], 'col_num': 5, + 'col_type': 'FLOAT', 'db_name': 'test', 'sql_func': case2, 'ding_flag': False, + 'ding_config': None + } + } + + run(engine_dict['taosd'], test_cases, result_dir) From df45487aad396d13e48d450f0934b37ec3a07710 Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Mon, 9 Nov 2020 16:15:55 +0800 Subject: [PATCH 097/122] [TD-1988] add query_performance.py --- tests/pytest/query/query_performance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/query/query_performance.py b/tests/pytest/query/query_performance.py index d70adce7a5..9c9c7c7996 100644 --- a/tests/pytest/query/query_performance.py +++ b/tests/pytest/query/query_performance.py @@ -151,7 +151,8 @@ class Demo: for each in (50, 80, 90, 95): _list.append(np.percentile(_data,each)) _str += ' %d 分位数 : %.4f 秒\n' % (each , np.percentile(_data,each)) - + + print(_str) if ding_flag: ding = Ding(values['ding_config']['urls'], values['ding_config']['at_mobiles']) ding.send_message(_str) From 802c744a4ae30de1df32e6efaf1e9dafec334849 Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Mon, 9 Nov 2020 16:19:14 +0800 Subject: [PATCH 098/122] [TD-1988] delete personal message --- tests/pytest/query/query_performance.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/query_performance.py b/tests/pytest/query/query_performance.py index 9c9c7c7996..c31569ac13 100644 --- a/tests/pytest/query/query_performance.py +++ b/tests/pytest/query/query_performance.py @@ -208,8 +208,8 @@ if __name__ == '__main__': 'case1': {'engine':'taosd', 'table_num': 10, 'circle': 100, 'number_per_table':[10, 100], 'col_num': 5, 'col_type': 'INT', 'db_name': 'test', 'sql_func': case1, 'ding_flag': True, 'ding_config': - {'urls': [r'https://oapi.dingtalk.com/robot/send?access_token=58fe61c28ee7142a90d2b1e7b9a8b9d99f502f248739c12a7fb62579edc0cd93'], - 'at_mobiles':[17080138990,],}}, + {'urls': [r'https://oapi.dingtalk.com/robot/send?access_token=xxxxxxx0cd93'], + 'at_mobiles':[17000000000,],}}, 'case2': {'engine':'taosd', 'table_num': 10, 'circle': 50, 'number_per_table':[10, 100], 'col_num': 5, 'col_type': 'FLOAT', 'db_name': 'test', 'sql_func': case2, 'ding_flag': False, 'ding_config': None From a0876eb776d2f91d41018bca558576c43d23e6e8 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 9 Nov 2020 16:52:42 +0800 Subject: [PATCH 099/122] [TD-1883]: fix memory leak when resultSet pointer is null --- src/connector/jdbc/deploy-pom.xml | 2 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 3f6ebeff03..893290868d 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.6 + 2.0.9 jar JDBCDriver diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 0cd185de50..edc160e323 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -111,8 +111,8 @@ public class TSDBJNIConnector { * @throws SQLException */ public long executeQuery(String sql) throws SQLException { - // close previous result set if the user forgets to invoke the - // free method to close previous result set. + // close previous result set if the user forgets to invoke the + // free method to close previous result set. if (!this.isResultsetClosed) { freeResultSet(taosResultSetPointer); } @@ -122,23 +122,23 @@ public class TSDBJNIConnector { pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos); } catch (Exception e) { e.printStackTrace(); - this.freeResultSet(pSql); + this.freeResultSetImp(this.taos, pSql); throw new SQLException(TSDBConstants.WrapErrMsg("Unsupported encoding")); } - + int code = this.getErrCode(pSql); if (code != 0) { affectedRows = -1; String msg = this.getErrMsg(pSql); - - this.freeResultSet(pSql); + + this.freeResultSetImp(this.taos, pSql); throw new SQLException(TSDBConstants.WrapErrMsg(msg), "", code); } // Try retrieving result set for the executed SQL using the current connection pointer. taosResultSetPointer = this.getResultSetImp(this.taos, pSql); isResultsetClosed = (taosResultSetPointer == TSDBConstants.JNI_NULL_POINTER); - + return pSql; } @@ -171,11 +171,11 @@ public class TSDBJNIConnector { } private native long getResultSetImp(long connection, long pSql); - + public boolean isUpdateQuery(long pSql) { - return isUpdateQueryImp(this.taos, pSql) == 1? true:false; + return isUpdateQueryImp(this.taos, pSql) == 1 ? true : false; } - + private native long isUpdateQueryImp(long connection, long pSql); /** @@ -191,7 +191,7 @@ public class TSDBJNIConnector { res = this.freeResultSetImp(this.taos, result); taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; } - + isResultsetClosed = true; return res; } @@ -274,7 +274,7 @@ public class TSDBJNIConnector { * Consume a subscription */ long consume(long subscription) { - return this.consumeImp(subscription); + return this.consumeImp(subscription); } private native long consumeImp(long subscription); From 4c34bb53246718b66c4473a8443d4e8346141a92 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 10 Nov 2020 03:36:13 +0000 Subject: [PATCH 100/122] TD-1793 --- src/inc/mnode.h | 1 + src/inc/taoserror.h | 1 + src/kit/shell/inc/shell.h | 2 +- src/kit/shell/src/shellEngine.c | 7 ++++--- src/mnode/src/mnodeInt.c | 1 + src/mnode/src/mnodeTable.c | 19 +++++++++++++++---- 6 files changed, 23 insertions(+), 8 deletions(-) diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 128e4d35a4..bdc30b0c46 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -47,6 +47,7 @@ typedef struct SMnodeMsg { int8_t successed; int8_t expected; int8_t retry; + int32_t incomingTs; int32_t code; void * pObj; SRpcMsg rpcMsg; diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 310bd78c7e..01891234ef 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -181,6 +181,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB, 0, 0x0383, "Invalid da TAOS_DEFINE_ERROR(TSDB_CODE_MND_MONITOR_DB_FORBIDDEN, 0, 0x0384, "Cannot delete monitor database") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many databases for account") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced") // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed") diff --git a/src/kit/shell/inc/shell.h b/src/kit/shell/inc/shell.h index f508d18608..d65c943e28 100644 --- a/src/kit/shell/inc/shell.h +++ b/src/kit/shell/inc/shell.h @@ -60,7 +60,7 @@ typedef struct SShellArguments { extern void shellParseArgument(int argc, char* argv[], SShellArguments* arguments); extern TAOS* shellInit(SShellArguments* args); extern void* shellLoopQuery(void* arg); -extern void taos_error(TAOS_RES* tres); +extern void taos_error(TAOS_RES* tres, int64_t st); extern int regex_match(const char* s, const char* reg, int cflags); void shellReadCommand(TAOS* con, char command[]); int32_t shellRunCommand(TAOS* con, char* command); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index 9b166f9351..d32100d321 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -296,7 +296,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { TAOS_RES* pSql = taos_query_h(con, command, &result); if (taos_errno(pSql)) { - taos_error(pSql); + taos_error(pSql, st); return; } @@ -807,9 +807,10 @@ void write_history() { fclose(f); } -void taos_error(TAOS_RES *tres) { +void taos_error(TAOS_RES *tres, int64_t st) { + int64_t et = taosGetTimestampUs(); atomic_store_ptr(&result, 0); - fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres)); + fprintf(stderr, "\nDB error: %s (%.6fs)\n", taos_errstr(tres), (et - st) / 1E6); taos_free_result(tres); } diff --git a/src/mnode/src/mnodeInt.c b/src/mnode/src/mnodeInt.c index d3f2407386..8431baf4f4 100644 --- a/src/mnode/src/mnodeInt.c +++ b/src/mnode/src/mnodeInt.c @@ -40,6 +40,7 @@ void *mnodeCreateMsg(SRpcMsg *pRpcMsg) { pMsg->rpcMsg = *pRpcMsg; pMsg->rpcMsg.pCont = pMsg->pCont; + pMsg->incomingTs = taosGetTimestampSec(); memcpy(pMsg->pCont, pRpcMsg->pCont, pRpcMsg->contLen); return pMsg; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index d27688f8a9..879e753451 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -45,6 +45,10 @@ #include "mnodeRead.h" #include "mnodePeer.h" +#define ALTER_CTABLE_RETRY_TIMES 3 +#define CREATE_CTABLE_RETRY_TIMES 10 +#define CREATE_CTABLE_RETRY_SEC 10 + static void * tsChildTableSdb; static void * tsSuperTableSdb; static int32_t tsChildTableUpdateSize; @@ -2421,7 +2425,9 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { dnodeSendRpcMWriteRsp(mnodeMsg, code); } } else { - if (mnodeMsg->retry++ < 10) { + mnodeMsg->retry++; + int32_t sec = taosGetTimestampSec(); + if (mnodeMsg->retry < CREATE_CTABLE_RETRY_TIMES && ABS(sec - mnodeMsg->incomingTs) < CREATE_CTABLE_RETRY_SEC) { mDebug("app:%p:%p, table:%s, create table rsp received, need retry, times:%d vgId:%d sid:%d uid:%" PRIu64 " result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, pTable->vgId, pTable->tid, @@ -2429,13 +2435,18 @@ static void mnodeProcessCreateChildTableRsp(SRpcMsg *rpcMsg) { dnodeDelayReprocessMWriteMsg(mnodeMsg); } else { - mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 ", result:%s thandle:%p", + mError("app:%p:%p, table:%s, failed to create in dnode, vgId:%d sid:%d uid:%" PRIu64 + ", result:%s thandle:%p incomingTs:%d curTs:%d retryTimes:%d", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, pTable->vgId, pTable->tid, pTable->uid, - tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); + tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle, mnodeMsg->incomingTs, sec, mnodeMsg->retry); SSdbOper oper = {.type = SDB_OPER_GLOBAL, .table = tsChildTableSdb, .pObj = pTable}; sdbDeleteRow(&oper); + if (rpcMsg->code == TSDB_CODE_APP_NOT_READY) { + //Avoid retry again in client + rpcMsg->code = TSDB_CODE_MND_VGROUP_NOT_READY; + } dnodeSendRpcMWriteRsp(mnodeMsg, rpcMsg->code); } } @@ -2456,7 +2467,7 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg) { dnodeSendRpcMWriteRsp(mnodeMsg, TSDB_CODE_SUCCESS); } else { - if (mnodeMsg->retry++ < 3) { + if (mnodeMsg->retry++ < ALTER_CTABLE_RETRY_TIMES) { mDebug("app:%p:%p, table:%s, alter table rsp received, need retry, times:%d result:%s thandle:%p", mnodeMsg->rpcMsg.ahandle, mnodeMsg, pTable->info.tableId, mnodeMsg->retry, tstrerror(rpcMsg->code), mnodeMsg->rpcMsg.handle); From 97f34cc49f86f0de60d43dbe5b4ec746851e9d53 Mon Sep 17 00:00:00 2001 From: zFeng Date: Tue, 10 Nov 2020 12:58:04 +0800 Subject: [PATCH 101/122] Update administrator-ch.md --- documentation20/webdocs/markdowndocs/administrator-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 1e36e6c5e6..36466d2b7e 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -253,7 +253,7 @@ ALTER USER PASS <'password'>; 修改用户密码, 为避免被转换为小写，密码需要用单引号引用,单引号为英文半角 ``` -ALTER USER PRIVILEDGE ; +ALTER USER PRIVILEGE ; ``` 修改用户权限为：super/write/read，不需要添加单引号 From 95d3e55042917be968af88f6875efd24e509f4bd Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 10 Nov 2020 06:44:12 +0000 Subject: [PATCH 102/122] TD-1848 --- src/client/src/tscFunctionImpl.c | 2 +- src/client/src/tscLocalMerge.c | 62 +++++++-------- src/client/src/tscParseInsert.c | 10 +-- src/client/src/tscSQLParser.c | 4 +- src/client/src/tscServer.c | 8 +- src/client/src/tscSql.c | 6 +- src/client/src/tscStream.c | 4 +- src/client/src/tscSubquery.c | 28 +++---- src/client/src/tscUtil.c | 74 ++++++++--------- src/common/inc/tdataformat.h | 4 +- src/common/src/tdataformat.c | 10 +-- src/common/src/tvariant.c | 2 +- src/cq/src/cqMain.c | 2 +- src/dnode/src/dnodeMPeer.c | 2 +- src/dnode/src/dnodeMWrite.c | 2 +- src/kit/shell/src/shellDarwin.c | 4 +- src/kit/shell/src/shellEngine.c | 8 +- src/kit/shell/src/shellLinux.c | 6 +- src/kit/taosdemo/taosdemo.c | 2 +- src/mnode/src/mnodeAcct.c | 2 +- src/mnode/src/mnodeCluster.c | 2 +- src/mnode/src/mnodeDb.c | 6 +- src/mnode/src/mnodeDnode.c | 4 +- src/mnode/src/mnodeMnode.c | 6 +- src/mnode/src/mnodeProfile.c | 4 +- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeShow.c | 2 +- src/mnode/src/mnodeTable.c | 14 ++-- src/mnode/src/mnodeUser.c | 4 +- src/mnode/src/mnodeVgroup.c | 2 +- src/os/inc/os.h | 1 - src/os/inc/osAlloc.h | 43 ---------- src/os/inc/osMemory.h | 60 +++++++------- src/os/src/detail/osAlloc.c | 79 ------------------ src/os/src/detail/osMemory.c | 124 ++++++++++++++--------------- src/os/src/detail/osSysinfo.c | 14 ++-- src/plugins/http/src/httpContext.c | 2 +- src/plugins/http/src/httpSystem.c | 2 +- src/query/src/qExecutor.c | 98 +++++++++++------------ src/query/src/qExtbuffer.c | 16 ++-- src/query/src/qFill.c | 16 ++-- src/query/src/qParserImpl.c | 8 +- src/query/src/qPercentile.c | 6 +- src/query/src/qResultbuf.c | 12 +-- src/query/src/qTsbuf.c | 8 +- src/query/src/qUtil.c | 6 +- src/rpc/src/rpcCache.c | 6 +- src/rpc/src/rpcMain.c | 4 +- src/rpc/src/rpcTcp.c | 10 +-- src/rpc/src/rpcUdp.c | 4 +- src/sync/inc/taosTcpPool.h | 22 ++--- src/sync/src/syncMain.c | 10 +-- src/sync/src/syncRestore.c | 4 +- src/sync/src/taosTcpPool.c | 2 +- src/sync/src/tarbitrator.c | 4 +- src/tsdb/src/tsdbBuffer.c | 2 +- src/tsdb/src/tsdbFile.c | 6 +- src/tsdb/src/tsdbMain.c | 22 ++--- src/tsdb/src/tsdbMemTable.c | 8 +- src/tsdb/src/tsdbMeta.c | 20 ++--- src/tsdb/src/tsdbRead.c | 22 ++--- src/tsdb/tests/tsdbTests.cpp | 4 +- src/util/src/hash.c | 4 +- src/util/src/talgo.c | 2 +- src/util/src/tcache.c | 4 +- src/util/src/tconfig.c | 8 +- src/util/src/tkvstore.c | 10 +-- src/util/src/tlog.c | 6 +- src/util/src/tmempool.c | 6 +- src/util/src/tref.c | 4 +- src/util/src/tskiplist.c | 8 +- src/util/tests/skiplistTest.cpp | 10 +-- src/vnode/src/vnodeMain.c | 2 +- src/wal/src/walMgmt.c | 3 +- tests/tsim/src/simSystem.c | 6 +- 75 files changed, 441 insertions(+), 565 deletions(-) delete mode 100644 src/os/inc/osAlloc.h delete mode 100644 src/os/src/detail/osAlloc.c diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index affa4aee83..39cc753d40 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2117,7 +2117,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { } } - taosTFree(pData); + tfree(pData); } /* diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index f49c4da0cb..e18ad0b4b0 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -227,7 +227,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (ds == NULL) { tscError("%p failed to create merge structure", pSql); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - taosTFree(pReducer); + tfree(pReducer); return; } @@ -254,7 +254,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (ds->filePage.num == 0) { // no data in this flush, the index does not increase tscDebug("%p flush data is empty, ignore %d flush record", pSql, idx); - taosTFree(ds); + tfree(ds); continue; } @@ -264,7 +264,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd // no data actually, no need to merge result. if (idx == 0) { - taosTFree(pReducer); + tfree(pReducer); return; } @@ -272,7 +272,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd SCompareParam *param = malloc(sizeof(SCompareParam)); if (param == NULL) { - taosTFree(pReducer); + tfree(pReducer); return; } @@ -286,8 +286,8 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pRes->code = tLoserTreeCreate(&pReducer->pLoserTree, pReducer->numOfBuffer, param, treeComparator); if (pReducer->pLoserTree == NULL || pRes->code != 0) { - taosTFree(param); - taosTFree(pReducer); + tfree(param); + tfree(pReducer); return; } @@ -330,14 +330,14 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL || /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { - taosTFree(pReducer->pTempBuffer); - taosTFree(pReducer->discardData); - taosTFree(pReducer->pResultBuf); - taosTFree(pReducer->pFinalRes); - taosTFree(pReducer->prevRowOfInput); - taosTFree(pReducer->pLoserTree); - taosTFree(param); - taosTFree(pReducer); + tfree(pReducer->pTempBuffer); + tfree(pReducer->discardData); + tfree(pReducer->pResultBuf); + tfree(pReducer->pFinalRes); + tfree(pReducer->prevRowOfInput); + tfree(pReducer->pLoserTree); + tfree(param); + tfree(pReducer); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return; } @@ -495,33 +495,33 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; tVariantDestroy(&pCtx->tag); - taosTFree(pCtx->resultInfo); + tfree(pCtx->resultInfo); if (pCtx->tagInfo.pTagCtxList != NULL) { - taosTFree(pCtx->tagInfo.pTagCtxList); + tfree(pCtx->tagInfo.pTagCtxList); } } - taosTFree(pLocalReducer->pCtx); + tfree(pLocalReducer->pCtx); } - taosTFree(pLocalReducer->prevRowOfInput); + tfree(pLocalReducer->prevRowOfInput); - taosTFree(pLocalReducer->pTempBuffer); - taosTFree(pLocalReducer->pResultBuf); + tfree(pLocalReducer->pTempBuffer); + tfree(pLocalReducer->pResultBuf); if (pLocalReducer->pLoserTree) { - taosTFree(pLocalReducer->pLoserTree->param); - taosTFree(pLocalReducer->pLoserTree); + tfree(pLocalReducer->pLoserTree->param); + tfree(pLocalReducer->pLoserTree); } - taosTFree(pLocalReducer->pFinalRes); - taosTFree(pLocalReducer->discardData); + tfree(pLocalReducer->pFinalRes); + tfree(pLocalReducer->discardData); tscLocalReducerEnvDestroy(pLocalReducer->pExtMemBuffer, pLocalReducer->pDesc, pLocalReducer->resColModel, pLocalReducer->numOfVnode); for (int32_t i = 0; i < pLocalReducer->numOfBuffer; ++i) { - taosTFree(pLocalReducer->pLocalDataSrc[i]); + tfree(pLocalReducer->pLocalDataSrc[i]); } pLocalReducer->numOfBuffer = 0; @@ -588,7 +588,7 @@ static int32_t createOrderDescriptor(tOrderDescriptor **pOrderDesc, SSqlCmd *pCm } *pOrderDesc = tOrderDesCreate(orderColIndexList, numOfGroupByCols, pModel, pQueryInfo->order.order); - taosTFree(orderColIndexList); + tfree(orderColIndexList); if (*pOrderDesc == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -699,7 +699,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr if (createOrderDescriptor(pOrderDesc, pCmd, pModel) != TSDB_CODE_SUCCESS) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - taosTFree(pSchema); + tfree(pSchema); return pRes->code; } @@ -736,7 +736,7 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr } *pFinalModel = createColumnModel(pSchema, (int32_t)size, capacity); - taosTFree(pSchema); + tfree(pSchema); return TSDB_CODE_SUCCESS; } @@ -756,7 +756,7 @@ void tscLocalReducerEnvDestroy(tExtMemBuffer **pMemBuffer, tOrderDescriptor *pDe pMemBuffer[i] = destoryExtMemBuffer(pMemBuffer[i]); } - taosTFree(pMemBuffer); + tfree(pMemBuffer); } /** @@ -978,10 +978,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO pBeforeFillData->num = 0; for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - taosTFree(pResPages[i]); + tfree(pResPages[i]); } - taosTFree(pResPages); + tfree(pResPages); } static void savePreviousRow(SLocalReducer *pLocalReducer, tFilePage *tmpBuffer) { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index f960beb5d9..c9115a8324 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1406,7 +1406,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { assert(taos_errno(pSql) == code); taos_free_result(pSql); - taosTFree(pSupporter); + tfree(pSupporter); fclose(fp); pParentSql->res.code = code; @@ -1445,7 +1445,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { char *tokenBuf = calloc(1, 4096); - while ((readLen = taosGetline(&line, &n, fp)) != -1) { + while ((readLen = tgetline(&line, &n, fp)) != -1) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { line[--readLen] = 0; } @@ -1470,7 +1470,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { } } - taosTFree(tokenBuf); + tfree(tokenBuf); free(line); if (count > 0) { @@ -1483,7 +1483,7 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int code) { } else { taos_free_result(pSql); - taosTFree(pSupporter); + tfree(pSupporter); fclose(fp); pParentSql->fp = pParentSql->fetchFp; @@ -1513,7 +1513,7 @@ void tscProcessMultiVnodesImportFromFile(SSqlObj *pSql) { pSql->res.code = TAOS_SYSTEM_ERROR(errno); tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code)); - taosTFree(pSupporter) + tfree(pSupporter) tscQueueAsyncRes(pSql); return; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 18ffaa2522..579e5f5410 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3957,7 +3957,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { taosStringBuilderDestroy(&sb1); - taosTFree(segments); + tfree(segments); invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); return ret; @@ -3970,7 +3970,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, pQueryInfo->tagCond.tbnameCond.cond = strdup(str); taosStringBuilderDestroy(&sb1); - taosTFree(segments); + tfree(segments); return TSDB_CODE_SUCCESS; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 0cb5852867..e97e95a317 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -124,7 +124,7 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { pVgroupInfo->inUse = pEpSet->inUse; pVgroupInfo->numOfEps = pEpSet->numOfEps; for (int32_t i = 0; i < pVgroupInfo->numOfEps; i++) { - taosTFree(pVgroupInfo->epAddr[i].fqdn); + tfree(pVgroupInfo->epAddr[i].fqdn); pVgroupInfo->epAddr[i].fqdn = strndup(pEpSet->fqdn[i], tListLen(pEpSet->fqdn[i])); pVgroupInfo->epAddr[i].port = pEpSet->port[i]; } @@ -1549,7 +1549,7 @@ int tscBuildMultiMeterMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { memcpy(pInfoMsg->tableIds, tmpData, pCmd->payloadLen); } - taosTFree(tmpData); + tfree(tmpData); pCmd->payloadLen += sizeof(SMgmtHead) + sizeof(SMultiTableInfoMsg); pCmd->msgType = TSDB_MSG_TYPE_CM_TABLES_META; @@ -1956,7 +1956,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { pCmd->numOfCols = pQueryInfo->fieldsInfo.numOfOutput; tscFieldInfoUpdateOffset(pQueryInfo); - taosTFree(pTableMeta); + tfree(pTableMeta); return 0; } @@ -1981,7 +1981,7 @@ static void createHBObj(STscObj* pObj) { pSql->cmd.command = pQueryInfo->command; if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { - taosTFree(pSql); + tfree(pSql); return; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index bdc46c5446..5f8a2eb6b7 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -566,7 +566,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { pRes->rspType = 0; pSql->subState.numOfSub = 0; - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); assert(pSql->fp == NULL); @@ -894,7 +894,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { if (sqlLen > tsMaxSQLStringLen) { tscError("%p sql too long", pSql); pRes->code = TSDB_CODE_TSC_INVALID_SQL; - taosTFree(pSql); + tfree(pSql); return pRes->code; } @@ -903,7 +903,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscError("%p failed to malloc sql string buffer", pSql); tscDebug("%p Valid SQL result:%d, %s pObj:%p", pSql, pRes->code, taos_errstr(pSql), pObj); - taosTFree(pSql); + tfree(pSql); return pRes->code; } diff --git a/src/client/src/tscStream.c b/src/client/src/tscStream.c index 0f67911bbe..68c3bcae16 100644 --- a/src/client/src/tscStream.c +++ b/src/client/src/tscStream.c @@ -273,7 +273,7 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf taosCacheRelease(tscMetaCache, (void**)&(pTableMetaInfo->pTableMeta), false); tscFreeSqlResult(pSql); - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); pSql->subState.numOfSub = 0; pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList); tscSetNextLaunchTimer(pStream, pSql); @@ -617,6 +617,6 @@ void taos_close_stream(TAOS_STREAM *handle) { pStream->pSql = NULL; taos_free_result(pSql); - taosTFree(pStream); + tfree(pStream); } } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index b3d151ba27..eb32e2490a 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -255,7 +255,7 @@ static void tscDestroyJoinSupporter(SJoinSupporter* pSupporter) { pSupporter->pVgroupTables = NULL; } - taosTFree(pSupporter->pIdTagList); + tfree(pSupporter->pIdTagList); tscTagCondRelease(&pSupporter->tagCond); free(pSupporter); } @@ -308,7 +308,7 @@ static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { assert(taosArrayGetSize(pVgroupTables) > 0); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); - taosTFree(list); + tfree(list); } static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { @@ -335,7 +335,7 @@ static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupT taosArrayPush(pNew, &info); } - taosTFree(list); + tfree(list); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); return pNew; @@ -1624,8 +1624,8 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs) { SRetrieveSupport* pSupport = pSub->param; - taosTFree(pSupport->localBuffer); - taosTFree(pSupport); + tfree(pSupport->localBuffer); + tfree(pSupport); taos_free_result(pSub); } @@ -1666,7 +1666,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { if (ret != 0) { pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscQueueAsyncRes(pSql); - taosTFree(pMemoryBuf); + tfree(pMemoryBuf); return ret; } @@ -1675,7 +1675,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { tscDebug("%p retrieved query data from %d vnode(s)", pSql, pState->numOfSub); if (pSql->pSubs == NULL) { - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; tscLocalReducerEnvDestroy(pMemoryBuf, pDesc, pModel, pState->numOfSub); @@ -1700,7 +1700,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->localBuffer = (tFilePage *)calloc(1, nBufferSize + sizeof(tFilePage)); if (trs->localBuffer == NULL) { tscError("%p failed to malloc buffer for local buffer, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - taosTFree(trs); + tfree(trs); break; } @@ -1711,8 +1711,8 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); - taosTFree(trs->localBuffer); - taosTFree(trs); + tfree(trs->localBuffer); + tfree(trs); break; } @@ -1766,8 +1766,8 @@ static void tscFreeRetrieveSup(SSqlObj *pSql) { // SSqlObj *pParentSql = trsupport->pParentSql; // assert(pSql == pParentSql->pSubs[index]); - taosTFree(trsupport->localBuffer); - taosTFree(trsupport); + tfree(trsupport->localBuffer); + tfree(trsupport); } static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfRows); @@ -2167,7 +2167,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows) pParentObj->res.code = pSql->res.code; } - taosTFree(pSupporter); + tfree(pSupporter); if (atomic_sub_fetch_32(&pParentObj->subState.numOfRemain, 1) > 0) { return; @@ -2442,7 +2442,7 @@ TAOS_ROW doSetResultRowData(SSqlObj *pSql, bool finalResult) { assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker - taosTFree(pRes->tsrow); + tfree(pRes->tsrow); return pRes->tsrow; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 2aee90653d..bb9725a744 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -245,7 +245,7 @@ void tscClearInterpInfo(SQueryInfo* pQueryInfo) { } pQueryInfo->fillType = TSDB_FILL_NONE; - taosTFree(pQueryInfo->fillVal); + tfree(pQueryInfo->fillVal); } int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { @@ -259,7 +259,7 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { // not enough memory if (pRes->tsrow == NULL || (pRes->buffer == NULL && pRes->numOfCols > 0)) { - taosTFree(pRes->tsrow); + tfree(pRes->tsrow); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; return pRes->code; } @@ -271,24 +271,24 @@ int32_t tscCreateResPointerInfo(SSqlRes* pRes, SQueryInfo* pQueryInfo) { static void tscDestroyResPointerInfo(SSqlRes* pRes) { if (pRes->buffer != NULL) { // free all buffers containing the multibyte string for (int i = 0; i < pRes->numOfCols; i++) { - taosTFree(pRes->buffer[i]); + tfree(pRes->buffer[i]); } pRes->numOfCols = 0; } - taosTFree(pRes->pRsp); + tfree(pRes->pRsp); - taosTFree(pRes->tsrow); - taosTFree(pRes->length); - taosTFree(pRes->buffer); + tfree(pRes->tsrow); + tfree(pRes->length); + tfree(pRes->buffer); - taosTFree(pRes->pGroupRec); - taosTFree(pRes->pColumnIndex); + tfree(pRes->pGroupRec); + tfree(pRes->pColumnIndex); if (pRes->pArithSup != NULL) { - taosTFree(pRes->pArithSup->data); - taosTFree(pRes->pArithSup); + tfree(pRes->pArithSup->data); + tfree(pRes->pArithSup); } pRes->data = NULL; // pRes->data points to the buffer of pRsp, no need to free @@ -305,11 +305,11 @@ static void tscFreeQueryInfo(SSqlCmd* pCmd, bool removeFromCache) { freeQueryInfoImpl(pQueryInfo); clearAllTableMetaInfo(pQueryInfo, (const char*)addr, removeFromCache); - taosTFree(pQueryInfo); + tfree(pQueryInfo); } pCmd->numOfClause = 0; - taosTFree(pCmd->pQueryInfo); + tfree(pCmd->pQueryInfo); } void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { @@ -387,14 +387,14 @@ void tscFreeTableMetaHelper(void *pTableMeta) { assert(numOfEps >= 0 && numOfEps <= TSDB_MAX_REPLICA); for(int32_t i = 0; i < numOfEps; ++i) { - taosTFree(p->vgroupInfo.epAddr[i].fqdn); + tfree(p->vgroupInfo.epAddr[i].fqdn); } int32_t numOfEps1 = p->corVgroupInfo.numOfEps; assert(numOfEps1 >= 0 && numOfEps1 <= TSDB_MAX_REPLICA); for(int32_t i = 0; i < numOfEps1; ++i) { - taosTFree(p->corVgroupInfo.epAddr[i].fqdn); + tfree(p->corVgroupInfo.epAddr[i].fqdn); } } @@ -418,9 +418,9 @@ void tscFreeSqlObj(SSqlObj* pSql) { pSql->signature = NULL; pSql->fp = NULL; - taosTFree(pSql->sqlstr); + tfree(pSql->sqlstr); - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); pSql->subState.numOfSub = 0; pSql->self = 0; @@ -428,7 +428,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscResetSqlCmdObj(pCmd, false); memset(pCmd->payload, 0, (size_t)pCmd->allocSize); - taosTFree(pCmd->payload); + tfree(pCmd->payload); pCmd->allocSize = 0; tsem_destroy(&pSql->rspSem); @@ -440,15 +440,15 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) { return; } - taosTFree(pDataBlock->pData); - taosTFree(pDataBlock->params); + tfree(pDataBlock->pData); + tfree(pDataBlock->params); // free the refcount for metermeta if (pDataBlock->pTableMeta != NULL) { taosCacheRelease(tscMetaCache, (void**)&(pDataBlock->pTableMeta), false); } - taosTFree(pDataBlock); + tfree(pDataBlock); } SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes, @@ -723,7 +723,7 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { taosHashCleanup(pVnodeDataBlockHashList); tscDestroyBlockArrayList(pVnodeDataBlockList); - taosTFree(dataBuf->pData); + tfree(dataBuf->pData); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -783,7 +783,7 @@ void tscCloseTscObj(void *param) { pthread_mutex_destroy(&pObj->mutex); tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p); - taosTFree(pObj); + tfree(pObj); } bool tscIsInsertData(char* sqlstr) { @@ -947,12 +947,12 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { if (pInfo->pArithExprInfo != NULL) { tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL); - taosTFree(pInfo->pArithExprInfo); + tfree(pInfo->pArithExprInfo); } } taosArrayDestroy(pFieldInfo->internalField); - taosTFree(pFieldInfo->final); + tfree(pFieldInfo->final); memset(pFieldInfo, 0, sizeof(SFieldInfo)); } @@ -1068,7 +1068,7 @@ void* sqlExprDestroy(SSqlExpr* pExpr) { tVariantDestroy(&pExpr->param[i]); } - taosTFree(pExpr); + tfree(pExpr); return NULL; } @@ -1168,11 +1168,11 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { static void destroyFilterInfo(SColumnFilterInfo* pFilterInfo, int32_t numOfFilters) { for(int32_t i = 0; i < numOfFilters; ++i) { if (pFilterInfo[i].filterstr) { - taosTFree(pFilterInfo[i].pz); + tfree(pFilterInfo[i].pz); } } - taosTFree(pFilterInfo); + tfree(pFilterInfo); } SColumn* tscColumnClone(const SColumn* src) { @@ -1460,7 +1460,7 @@ void tscTagCondRelease(STagCond* pTagCond) { size_t s = taosArrayGetSize(pTagCond->pCond); for (int32_t i = 0; i < s; ++i) { SCond* p = taosArrayGet(pTagCond->pCond, i); - taosTFree(p->cond); + tfree(p->cond); } taosArrayDestroy(pTagCond->pCond); @@ -1652,7 +1652,7 @@ static void freeQueryInfoImpl(SQueryInfo* pQueryInfo) { pQueryInfo->tsBuf = tsBufDestroy(pQueryInfo->tsBuf); - taosTFree(pQueryInfo->fillVal); + tfree(pQueryInfo->fillVal); } void tscClearSubqueryInfo(SSqlCmd* pCmd) { @@ -1672,7 +1672,7 @@ void tscFreeVgroupTableInfo(SArray* pVgroupTables) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - taosTFree(pInfo->vgInfo.epAddr[j].fqdn); + tfree(pInfo->vgInfo.epAddr[j].fqdn); } taosArrayDestroy(pInfo->itemList); @@ -1689,7 +1689,7 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index); for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - taosTFree(pInfo->vgInfo.epAddr[j].fqdn); + tfree(pInfo->vgInfo.epAddr[j].fqdn); } taosArrayDestroy(pInfo->itemList); @@ -1737,7 +1737,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem free(pTableMetaInfo); } - taosTFree(pQueryInfo->pTableMetaInfo); + tfree(pQueryInfo->pTableMetaInfo); } STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, @@ -2336,7 +2336,7 @@ void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp) { pRes->numOfTotal = num; - taosTFree(pSql->pSubs); + tfree(pSql->pSubs); pSql->subState.numOfSub = 0; pSql->fp = fp; @@ -2461,11 +2461,11 @@ void* tscVgroupInfoClear(SVgroupsInfo *vgroupList) { SVgroupInfo* pVgroupInfo = &vgroupList->vgroups[i]; for(int32_t j = 0; j < pVgroupInfo->numOfEps; ++j) { - taosTFree(pVgroupInfo->epAddr[j].fqdn); + tfree(pVgroupInfo->epAddr[j].fqdn); } } - taosTFree(vgroupList); + tfree(vgroupList); return NULL; } @@ -2473,7 +2473,7 @@ void tscSVgroupInfoCopy(SVgroupInfo* dst, const SVgroupInfo* src) { dst->vgId = src->vgId; dst->numOfEps = src->numOfEps; for(int32_t i = 0; i < dst->numOfEps; ++i) { - taosTFree(dst->epAddr[i].fqdn); + tfree(dst->epAddr[i].fqdn); dst->epAddr[i].port = src->epAddr[i].port; dst->epAddr[i].fqdn = strdup(src->epAddr[i].fqdn); } diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 43968a6108..8d4949d9b4 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -80,7 +80,7 @@ typedef struct { #define schemaFLen(s) ((s)->flen) #define schemaVLen(s) ((s)->vlen) #define schemaColAt(s, i) ((s)->columns + i) -#define tdFreeSchema(s) taosTFree((s)) +#define tdFreeSchema(s) tfree((s)) STSchema *tdDupSchema(STSchema *pSchema); int tdEncodeSchema(void **buf, STSchema *pSchema); @@ -308,7 +308,7 @@ typedef struct { #define kvRowCpy(dst, r) memcpy((dst), (r), kvRowLen(r)) #define kvRowColVal(r, colIdx) POINTER_SHIFT(kvRowValues(r), (colIdx)->offset) #define kvRowColIdxAt(r, i) (kvRowColIdx(r) + (i)) -#define kvRowFree(r) taosTFree(r) +#define kvRowFree(r) tfree(r) #define kvRowEnd(r) POINTER_SHIFT(r, kvRowLen(r)) SKVRow tdKVRowDup(SKVRow row); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 40b524488a..f212054793 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -97,7 +97,7 @@ int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { void tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder) { if (pBuilder) { - taosTFree(pBuilder->columns); + tfree(pBuilder->columns); } } @@ -339,8 +339,8 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) { void tdFreeDataCols(SDataCols *pCols) { if (pCols) { - taosTFree(pCols->buf); - taosTFree(pCols->cols); + tfree(pCols->buf); + tfree(pCols->cols); free(pCols); } } @@ -669,8 +669,8 @@ int tdInitKVRowBuilder(SKVRowBuilder *pBuilder) { } void tdDestroyKVRowBuilder(SKVRowBuilder *pBuilder) { - taosTFree(pBuilder->pColIdx); - taosTFree(pBuilder->buf); + tfree(pBuilder->pColIdx); + tfree(pBuilder->buf); } void tdResetKVRowBuilder(SKVRowBuilder *pBuilder) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index ca1644c0a2..6dd0653822 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -125,7 +125,7 @@ void tVariantDestroy(tVariant *pVar) { if (pVar == NULL) return; if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR) { - taosTFree(pVar->pz); + tfree(pVar->pz); pVar->nLen = 0; } diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 8b10860ef0..1be7552a89 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -115,7 +115,7 @@ void cqClose(void *handle) { SCqObj *pTemp = pObj; pObj = pObj->next; tdFreeSchema(pTemp->pSchema); - taosTFree(pTemp->sqlStr); + tfree(pTemp->sqlStr); free(pTemp); } diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index b5ecc4930f..05b37bd338 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -84,7 +84,7 @@ void dnodeCleanupMPeer() { taosCloseQset(tsMPeerQset); tsMPeerQset = NULL; - taosTFree(tsMPeerWP.worker); + tfree(tsMPeerWP.worker); } int32_t dnodeAllocateMPeerQueue() { diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 3940a251d4..bde4b95bc6 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -86,7 +86,7 @@ void dnodeCleanupMWrite() { taosCloseQset(tsMWriteQset); tsMWriteQset = NULL; - taosTFree(tsMWriteWP.worker); + tfree(tsMWriteWP.worker); } int32_t dnodeAllocMWritequeue() { diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index ffe537dd91..995b56f341 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -229,8 +229,8 @@ void shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosTFree(cmd.buffer); - taosTFree(cmd.command); + tfree(cmd.buffer); + tfree(cmd.command); return; } else { updateBuffer(&cmd); diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index d32100d321..22f01ac142 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -193,7 +193,7 @@ int32_t shellRunCommand(TAOS* con, char* command) { history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE] == NULL || strcmp(command, history.hist[(history.hend + MAX_HISTORY_SIZE - 1) % MAX_HISTORY_SIZE]) != 0) { if (history.hist[history.hend] != NULL) { - taosTFree(history.hist[history.hend]); + tfree(history.hist[history.hend]); } history.hist[history.hend] = strdup(command); @@ -770,7 +770,7 @@ void read_history() { return; } - while ((read_size = taosGetline(&line, &line_size, f)) != -1) { + while ((read_size = tgetline(&line, &line_size, f)) != -1) { line[read_size - 1] = '\0'; history.hist[history.hend] = strdup(line); @@ -800,7 +800,7 @@ void write_history() { for (int i = history.hstart; i != history.hend;) { if (history.hist[i] != NULL) { fprintf(f, "%s\n", history.hist[i]); - taosTFree(history.hist[i]); + tfree(history.hist[i]); } i = (i + 1) % MAX_HISTORY_SIZE; } @@ -854,7 +854,7 @@ void source_file(TAOS *con, char *fptr) { return; } - while ((read_len = taosGetline(&line, &line_len, f)) != -1) { + while ((read_len = tgetline(&line, &line_len, f)) != -1) { if (read_len >= tsMaxSQLStringLen) continue; line[--read_len] = '\0'; diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 6f5ea33d79..57f0f65be4 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -232,8 +232,8 @@ void shellReadCommand(TAOS *con, char *command) { printf("\n"); if (isReadyGo(&cmd)) { sprintf(command, "%s%s", cmd.buffer, cmd.command); - taosTFree(cmd.buffer); - taosTFree(cmd.command); + tfree(cmd.buffer); + tfree(cmd.command); return; } else { updateBuffer(&cmd); @@ -351,7 +351,7 @@ void *shellLoopQuery(void *arg) { reset_terminal_mode(); } while (shellRunCommand(con, command) == 0); - taosTFree(command); + tfree(command); exitShell(); pthread_cleanup_pop(1); diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index a866a677ad..74dc239cb5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -955,7 +955,7 @@ void querySqlFile(TAOS* taos, char* sqlFile) double t = getCurrentTime(); - while ((read_len = taosGetline(&line, &line_len, fp)) != -1) { + while ((read_len = tgetline(&line, &line_len, fp)) != -1) { if (read_len >= MAX_SQL_SIZE) continue; line[--read_len] = '\0'; diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index e161940a2b..365cf656de 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -34,7 +34,7 @@ static int32_t mnodeCreateRootAcct(); static int32_t mnodeAcctActionDestroy(SSdbOper *pOper) { SAcctObj *pAcct = pOper->pObj; pthread_mutex_destroy(&pAcct->mutex); - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 35b6a67ab2..98587cf53d 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -33,7 +33,7 @@ static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void * static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeClusterActionDestroy(SSdbOper *pOper) { - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index fb4793c056..3b3a21aeb1 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -52,8 +52,8 @@ static int32_t mnodeProcessDropDbMsg(SMnodeMsg *pMsg); static void mnodeDestroyDb(SDbObj *pDb) { pthread_mutex_destroy(&pDb->mutex); - taosTFree(pDb->vgList); - taosTFree(pDb); + tfree(pDb->vgList); + tfree(pDb); } static int32_t mnodeDbActionDestroy(SSdbOper *pOper) { @@ -405,7 +405,7 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCreateDbMsg *pCreate, SMnodeMsg * code = mnodeCheckDbCfg(&pDb->cfg); if (code != TSDB_CODE_SUCCESS) { - taosTFree(pDb); + tfree(pDb); return code; } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index bf8647c193..37e00fc4e3 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -88,7 +88,7 @@ static char* offlineReason[] = { }; static int32_t mnodeDnodeActionDestroy(SSdbOper *pOper) { - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -655,7 +655,7 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) { int32_t code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { int dnodeId = pDnode->dnodeId; - taosTFree(pDnode); + tfree(pDnode); mError("failed to create dnode:%d, reason:%s", dnodeId, tstrerror(code)); } else { mLInfo("dnode:%d is created", pDnode->dnodeId); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 5a1825a2a3..092f246c13 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -59,7 +59,7 @@ static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, vo #endif static int32_t mnodeMnodeActionDestroy(SSdbOper *pOper) { - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -342,14 +342,14 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) { } if (code != TSDB_CODE_SUCCESS) { - taosTFree(pMnode); + tfree(pMnode); return; } code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("dnode:%d, failed to create mnode, ep:%s reason:%s", dnodeId, dnodeEp, tstrerror(code)); - taosTFree(pMnode); + tfree(pMnode); } } diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 9831a95c5f..12ac64854c 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -131,8 +131,8 @@ SConnObj *mnodeAccquireConn(int32_t connId, char *user, uint32_t ip, uint16_t po static void mnodeFreeConn(void *data) { SConnObj *pConn = data; - taosTFree(pConn->pQueries); - taosTFree(pConn->pStreams); + tfree(pConn->pQueries); + tfree(pConn->pStreams); mDebug("connId:%d, is destroyed", pConn->connId); } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 6b6a49db93..12d977f32b 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -984,7 +984,7 @@ void sdbCleanupWriteWorker() { } sdbFreeWritequeue(); - taosTFree(tsSdbPool.writeWorker); + tfree(tsSdbPool.writeWorker); mInfo("sdb write is closed"); } diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 2c9d404e1d..0d53fa9617 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -415,7 +415,7 @@ static void mnodeFreeShowObj(void *data) { sdbFreeIter(pShow->pIter); mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index); - taosTFree(pShow); + tfree(pShow); } static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove) { diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 879e753451..e729fb2254 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -93,10 +93,10 @@ static void mnodeProcessAlterTableRsp(SRpcMsg *rpcMsg); static int32_t mnodeFindSuperTableColumnIndex(SSTableObj *pStable, char *colName); static void mnodeDestroyChildTable(SCTableObj *pTable) { - taosTFree(pTable->info.tableId); - taosTFree(pTable->schema); - taosTFree(pTable->sql); - taosTFree(pTable); + tfree(pTable->info.tableId); + tfree(pTable->schema); + tfree(pTable->sql); + tfree(pTable); } static int32_t mnodeChildTableActionDestroy(SSdbOper *pOper) { @@ -425,9 +425,9 @@ static void mnodeDestroySuperTable(SSTableObj *pStable) { taosHashCleanup(pStable->vgHash); pStable->vgHash = NULL; } - taosTFree(pStable->info.tableId); - taosTFree(pStable->schema); - taosTFree(pStable); + tfree(pStable->info.tableId); + tfree(pStable->schema); + tfree(pStable); } static int32_t mnodeSuperTableActionDestroy(SSdbOper *pOper) { diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 8e9d2908c5..cd02d5a935 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -43,7 +43,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessAuthMsg(SMnodeMsg *pMsg); static int32_t mnodeUserActionDestroy(SSdbOper *pOper) { - taosTFree(pOper->pObj); + tfree(pOper->pObj); return TSDB_CODE_SUCCESS; } @@ -270,7 +270,7 @@ int32_t mnodeCreateUser(SAcctObj *pAcct, char *name, char *pass, void *pMsg) { code = sdbInsertRow(&oper); if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("user:%s, failed to create by %s, reason:%s", pUser->user, mnodeGetUserFromMsg(pMsg), tstrerror(code)); - taosTFree(pUser); + tfree(pUser); } else { mLInfo("user:%s, is created by %s", pUser->user, mnodeGetUserFromMsg(pMsg)); } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index cf40e59f68..9e398e94f1 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -69,7 +69,7 @@ static void mnodeDestroyVgroup(SVgObj *pVgroup) { pVgroup->idPool = NULL; } - taosTFree(pVgroup); + tfree(pVgroup); } static int32_t mnodeVgroupActionDestroy(SSdbOper *pOper) { diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 9720499004..9383ae48dc 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -53,7 +53,6 @@ extern "C" { #endif #include "osDef.h" -#include "osAlloc.h" #include "osAtomic.h" #include "osCommon.h" #include "osDir.h" diff --git a/src/os/inc/osAlloc.h b/src/os/inc/osAlloc.h deleted file mode 100644 index 2d97017480..0000000000 --- a/src/os/inc/osAlloc.h +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_OS_ALLOC_H -#define TDENGINE_OS_ALLOC_H - -#ifdef __cplusplus -extern "C" { -#endif - -#ifndef TAOS_OS_FUNC_ALLOC - #define tmalloc(size) malloc(size) - #define tcalloc(nmemb, size) calloc(nmemb, size) - #define trealloc(p, size) realloc(p, size) - #define tmemalign(alignment, size) malloc(size) - #define tfree(p) free(p) - #define tmemzero(p, size) memset(p, 0, size) -#else - void *tmalloc(int32_t size); - void *tcalloc(int32_t nmemb, int32_t size); - void *trealloc(void *p, int32_t size); - void *tmemalign(int32_t alignment, int32_t size); - void tfree(void *p); - void tmemzero(void *p, int32_t size); -#endif - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h index 37d9dc9828..438bb88474 100644 --- a/src/os/inc/osMemory.h +++ b/src/os/inc/osMemory.h @@ -31,6 +31,7 @@ typedef enum { void taosSetAllocMode(int mode, const char *path, bool autoDump); void taosDumpMemoryLeak(); +// used in tsdb module void * taosTMalloc(size_t size); void * taosTCalloc(size_t nmemb, size_t size); void * taosTRealloc(void *ptr, size_t size); @@ -38,7 +39,14 @@ void taosTZfree(void *ptr); size_t taosTSizeof(void *ptr); void taosTMemset(void *ptr, int c); -#define taosTFree(x) \ +// used in other module +#define tmalloc(size) malloc(size) +#define tcalloc(num, size) calloc(num, size) +#define trealloc(ptr, size) realloc(ptr, size) +#define tstrdup(str) taosStrdupImp(str) +#define tstrndup(str, size) taosStrndupImp(str, size) +#define tgetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream) +#define tfree(x) \ do { \ if (x) { \ free((void *)(x)); \ @@ -46,37 +54,29 @@ void taosTMemset(void *ptr, int c); } \ } while (0); -#define taosMalloc(size) malloc(size) -#define taosCalloc(num, size) calloc(num, size) -#define taosRealloc(ptr, size) realloc(ptr, size) -#define taosFree(ptr) free(ptr) -#define taosStrdup(str) taosStrdupImp(str) -#define taosStrndup(str, size) taosStrndupImp(str, size) -#define taosGetline(lineptr, n, stream) taosGetlineImp(lineptr, n, stream) - #ifdef TAOS_MEM_CHECK #ifdef TAOS_MEM_CHECK_TEST - void * taos_malloc(size_t size, const char *file, uint32_t line); - void * taos_calloc(size_t num, size_t size, const char *file, uint32_t line); - void * taos_realloc(void *ptr, size_t size, const char *file, uint32_t line); - void taos_free(void *ptr, const char *file, uint32_t line); - char * taos_strdup(const char *str, const char *file, uint32_t line); - char * taos_strndup(const char *str, size_t size, const char *file, uint32_t line); - ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); - #undef taosMalloc - #undef taosCalloc - #undef taosRealloc - #undef taosFree - #undef taosStrdup - #undef taosStrndup - #undef taosGetline - #define taosMalloc(size) taos_malloc(size, __FILE__, __LINE__) - #define taosCalloc(num, size) taos_calloc(num, size, __FILE__, __LINE__) - #define taosRealloc(ptr, size) taos_realloc(ptr, size, __FILE__, __LINE__) - #define taosFree(ptr) taos_free(ptr, __FILE__, __LINE__) - //#define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__) - //#define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) - //#define taosGetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) + void * taosMallocMem(size_t size, const char *file, uint32_t line); + void * taosCallocMem(size_t num, size_t size, const char *file, uint32_t line); + void * taosReallocMem(void *ptr, size_t size, const char *file, uint32_t line); + void taosFreeMem(void *ptr, const char *file, uint32_t line); + char * taosStrdupMem(const char *str, const char *file, uint32_t line); + char * taosStrndupMem(const char *str, size_t size, const char *file, uint32_t line); + ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char *file, uint32_t line); + #undef tmalloc + #undef tcalloc + #undef trealloc + #define tmalloc(size) taosMallocMem(size, __FILE__, __LINE__) + #define tcalloc(num, size) taosCallocMem(num, size, __FILE__, __LINE__) + #define trealloc(ptr, size) taosReallocMem(ptr, size, __FILE__, __LINE__) + #define tfree(ptr) taosFreeMem(ptr, __FILE__, __LINE__) + + // #undef tstrdup + // #undef tstrndup + // #undef tgetline + // #define taosStrdup(str) taos_strdup(str, __FILE__, __LINE__) + // #define taosStrndup(str, size) taos_strndup(str, size, __FILE__, __LINE__) + // #define tgetline(lineptr, n, stream) taos_getline(lineptr, n, stream, __FILE__, __LINE__) #endif #endif diff --git a/src/os/src/detail/osAlloc.c b/src/os/src/detail/osAlloc.c deleted file mode 100644 index 4ca35793e7..0000000000 --- a/src/os/src/detail/osAlloc.c +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tulog.h" -#include "osAlloc.h" - -#define TSDB_HAVE_MEMALIGN -#ifdef TAOS_OS_FUNC_ALLOC - -void *tmalloc(int32_t size) { - void *p = malloc(size); - if (p == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to malloc memory, size:%d reason:%s", size, strerror(errno)); - } - - return p; -} - -void *tcalloc(int32_t nmemb, int32_t size) { - void *p = calloc(nmemb, size); - if (p == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to calloc memory, nmemb:%d size:%d reason:%s", nmemb, size, strerror(errno)); - } - - return p; -} - -void *trealloc(void *p, int32_t size) { - p = realloc(p, size); - if (p == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to realloc memory, size:%d reason:%s", size, strerror(errno)); - } - - return p; -} - -void tfree(void *p) { free(p); } - -void tmemzero(void *p, int32_t size) { memset(p, 0, size); } - -#ifdef TSDB_HAVE_MEMALIGN - -void *tmemalign(int32_t alignment, int32_t size) { - void *p; - - int err = posix_memalign(&p, alignment, size); - if (err) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to memalign memory, alignment:%d size:%d reason:%s", alignment, size, strerror(err)); - p = NULL; - } - - return p; -} - -#else - -void *tmemalign(int32_t alignment, int32_t size) { return tmalloc(size); } - -#endif -#endif \ No newline at end of file diff --git a/src/os/src/detail/osMemory.c b/src/os/src/detail/osMemory.c index dfd320be89..53310d179c 100644 --- a/src/os/src/detail/osMemory.c +++ b/src/os/src/detail/osMemory.c @@ -28,7 +28,7 @@ static FILE* fpAllocLog = NULL; extern int32_t taosGetTimestampSec(); static int32_t startTime = INT32_MAX; -static bool random_alloc_fail(size_t size, const char* file, uint32_t line) { +static bool taosRandomAllocFail(size_t size, const char* file, uint32_t line) { if (taosGetTimestampSec() < startTime) { return false; } @@ -48,33 +48,33 @@ static bool random_alloc_fail(size_t size, const char* file, uint32_t line) { return true; } -static void* malloc_random(size_t size, const char* file, uint32_t line) { - return random_alloc_fail(size, file, line) ? NULL : malloc(size); +static void* taosRandmoMalloc(size_t size, const char* file, uint32_t line) { + return taosRandomAllocFail(size, file, line) ? NULL : malloc(size); } -static void* calloc_random(size_t num, size_t size, const char* file, uint32_t line) { - return random_alloc_fail(num * size, file, line) ? NULL : calloc(num, size); +static void* taosRandomCalloc(size_t num, size_t size, const char* file, uint32_t line) { + return taosRandomAllocFail(num * size, file, line) ? NULL : calloc(num, size); } -static void* realloc_random(void* ptr, size_t size, const char* file, uint32_t line) { - return random_alloc_fail(size, file, line) ? NULL : realloc(ptr, size); +static void* taosRandomRealloc(void* ptr, size_t size, const char* file, uint32_t line) { + return taosRandomAllocFail(size, file, line) ? NULL : realloc(ptr, size); } -static char* strdup_random(const char* str, const char* file, uint32_t line) { +static char* taosRandomStrdup(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - return random_alloc_fail(len + 1, file, line) ? NULL : taosStrdupImp(str); + return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrdupImp(str); } -static char* strndup_random(const char* str, size_t size, const char* file, uint32_t line) { +static char* taosRandomStrndup(const char* str, size_t size, const char* file, uint32_t line) { size_t len = strlen(str); if (len > size) { len = size; } - return random_alloc_fail(len + 1, file, line) ? NULL : taosStrndupImp(str, len); + return taosRandomAllocFail(len + 1, file, line) ? NULL : taosStrndupImp(str, len); } -static ssize_t getline_random(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { - return random_alloc_fail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream); +static ssize_t taosRandomGetline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { + return taosRandomAllocFail(*n, file, line) ? -1 : taosGetlineImp(lineptr, n, stream); } //////////////////////////////////////////////////////////////////////////////// @@ -96,7 +96,7 @@ typedef struct SMemBlock { static SMemBlock *blocks = NULL; static uintptr_t lock = 0; -static void add_mem_block(SMemBlock* blk) { +static void taosAddMemBlock(SMemBlock* blk) { blk->prev = NULL; while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); blk->next = blocks; @@ -107,7 +107,7 @@ static void add_mem_block(SMemBlock* blk) { atomic_store_ptr(&lock, 0); } -static void remove_mem_block(SMemBlock* blk) { +static void taosRemoveMemBlock(SMemBlock* blk) { while (atomic_val_compare_exchange_ptr(&lock, 0, 1) != 0); if (blocks == blk) { @@ -126,7 +126,7 @@ static void remove_mem_block(SMemBlock* blk) { blk->next = NULL; } -static void free_detect_leak(void* ptr, const char* file, uint32_t line) { +static void taosFreeDetectLeak(void* ptr, const char* file, uint32_t line) { if (ptr == NULL) { return; } @@ -140,11 +140,11 @@ static void free_detect_leak(void* ptr, const char* file, uint32_t line) { return; } - remove_mem_block(blk); + taosRemoveMemBlock(blk); free(blk); } -static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) { +static void* taosMallocDetectLeak(size_t size, const char* file, uint32_t line) { if (size == 0) { return NULL; } @@ -166,28 +166,28 @@ static void* malloc_detect_leak(size_t size, const char* file, uint32_t line) { blk->line = (uint16_t)line; blk->magic = MEMBLK_MAGIC; blk->size = size; - add_mem_block(blk); + taosAddMemBlock(blk); return blk->data; } -static void* calloc_detect_leak(size_t num, size_t size, const char* file, uint32_t line) { +static void* taosCallocDetectLeak(size_t num, size_t size, const char* file, uint32_t line) { size *= num; - void* p = malloc_detect_leak(size, file, line); + void* p = taosMallocDetectLeak(size, file, line); if (p != NULL) { memset(p, 0, size); } return p; } -static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint32_t line) { +static void* taosReallocDetectLeak(void* ptr, size_t size, const char* file, uint32_t line) { if (size == 0) { - free_detect_leak(ptr, file, line); + taosFreeDetectLeak(ptr, file, line); return NULL; } if (ptr == NULL) { - return malloc_detect_leak(size, file, line); + return taosMallocDetectLeak(size, file, line); } SMemBlock* blk = (SMemBlock *)((char*)ptr) - sizeof(SMemBlock); @@ -198,11 +198,11 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3 return realloc(ptr, size); } - remove_mem_block(blk); + taosRemoveMemBlock(blk); void* p = realloc(blk, size + sizeof(SMemBlock)); if (p == NULL) { - add_mem_block(blk); + taosAddMemBlock(blk); return NULL; } @@ -212,13 +212,13 @@ static void* realloc_detect_leak(void* ptr, size_t size, const char* file, uint3 blk = (SMemBlock*)p; blk->size = size; - add_mem_block(blk); + taosAddMemBlock(blk); return blk->data; } -static char* strdup_detect_leak(const char* str, const char* file, uint32_t line) { +static char* taosStrdupDetectLeak(const char* str, const char* file, uint32_t line) { size_t len = strlen(str); - char *p = malloc_detect_leak(len + 1, file, line); + char *p = taosMallocDetectLeak(len + 1, file, line); if (p != NULL) { memcpy(p, str, len); p[len] = 0; @@ -226,12 +226,12 @@ static char* strdup_detect_leak(const char* str, const char* file, uint32_t line return p; } -static char* strndup_detect_leak(const char* str, size_t size, const char* file, uint32_t line) { +static char* taosStrndupDetectLeak(const char* str, size_t size, const char* file, uint32_t line) { size_t len = strlen(str); if (len > size) { len = size; } - char *p = malloc_detect_leak(len + 1, file, line); + char *p = taosMallocDetectLeak(len + 1, file, line); if (p != NULL) { memcpy(p, str, len); p[len] = 0; @@ -239,13 +239,13 @@ static char* strndup_detect_leak(const char* str, size_t size, const char* file, return p; } -static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { +static ssize_t taosGetlineDetectLeak(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { char* buf = NULL; size_t bufSize = 0; ssize_t size = taosGetlineImp(&buf, &bufSize, stream); if (size != -1) { if (*n < size + 1) { - void* p = realloc_detect_leak(*lineptr, size + 1, file, line); + void* p = taosReallocDetectLeak(*lineptr, size + 1, file, line); if (p == NULL) { free(buf); return -1; @@ -260,7 +260,7 @@ static ssize_t getline_detect_leak(char **lineptr, size_t *n, FILE *stream, cons return size; } -static void dump_memory_leak() { +static void taosDumpMemoryLeakImp() { const char* hex = "0123456789ABCDEF"; const char* fmt = ":%d: addr=%p, size=%d, content(first 16 bytes)="; size_t numOfBlk = 0, totalSize = 0; @@ -299,7 +299,7 @@ static void dump_memory_leak() { fflush(fpAllocLog); } -static void dump_memory_leak_on_sig(int sig) { +static void taosDumpMemoryLeakOnSig(int sig) { fprintf(fpAllocLog, "signal %d received.\n", sig); // restore default signal handler @@ -307,55 +307,55 @@ static void dump_memory_leak_on_sig(int sig) { act.sa_handler = SIG_DFL; sigaction(sig, &act, NULL); - dump_memory_leak(); + taosDumpMemoryLeakImp(); } //////////////////////////////////////////////////////////////////////////////// // interface functions -void* taos_malloc(size_t size, const char* file, uint32_t line) { +void* taosMallocMem(size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return malloc(size); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return malloc_random(size, file, line); + return taosRandmoMalloc(size, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return malloc_detect_leak(size, file, line); + return taosMallocDetectLeak(size, file, line); } return malloc(size); } -void* taos_calloc(size_t num, size_t size, const char* file, uint32_t line) { +void* taosCallocMem(size_t num, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return calloc(num, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return calloc_random(num, size, file, line); + return taosRandomCalloc(num, size, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return calloc_detect_leak(num, size, file, line); + return taosCallocDetectLeak(num, size, file, line); } return calloc(num, size); } -void* taos_realloc(void* ptr, size_t size, const char* file, uint32_t line) { +void* taosReallocMem(void* ptr, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return realloc(ptr, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return realloc_random(ptr, size, file, line); + return taosRandomRealloc(ptr, size, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return realloc_detect_leak(ptr, size, file, line); + return taosReallocDetectLeak(ptr, size, file, line); } return realloc(ptr, size); } -void taos_free(void* ptr, const char* file, uint32_t line) { +void taosFreeMem(void* ptr, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return free(ptr); @@ -364,54 +364,54 @@ void taos_free(void* ptr, const char* file, uint32_t line) { return free(ptr); case TAOS_ALLOC_MODE_DETECT_LEAK: - return free_detect_leak(ptr, file, line); + return taosFreeDetectLeak(ptr, file, line); } return free(ptr); } -char* taos_strdup(const char* str, const char* file, uint32_t line) { +char* taosStrdupMem(const char* str, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return taosStrdupImp(str); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return strdup_random(str, file, line); + return taosRandomStrdup(str, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return strdup_detect_leak(str, file, line); + return taosStrdupDetectLeak(str, file, line); } return taosStrdupImp(str); } -char* taos_strndup(const char* str, size_t size, const char* file, uint32_t line) { +char* taosStrndupMem(const char* str, size_t size, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return taosStrndupImp(str, size); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return strndup_random(str, size, file, line); + return taosRandomStrndup(str, size, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return strndup_detect_leak(str, size, file, line); + return taosStrndupDetectLeak(str, size, file, line); } return taosStrndupImp(str, size); } -ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { +ssize_t taosGetlineMem(char **lineptr, size_t *n, FILE *stream, const char* file, uint32_t line) { switch (allocMode) { case TAOS_ALLOC_MODE_DEFAULT: return taosGetlineImp(lineptr, n, stream); case TAOS_ALLOC_MODE_RANDOM_FAIL: - return getline_random(lineptr, n, stream, file, line); + return taosRandomGetline(lineptr, n, stream, file, line); case TAOS_ALLOC_MODE_DETECT_LEAK: - return getline_detect_leak(lineptr, n, stream, file, line); + return taosGetlineDetectLeak(lineptr, n, stream, file, line); } return taosGetlineImp(lineptr, n, stream); } -static void close_alloc_log() { +static void taosCloseAllocLog() { if (fpAllocLog != NULL) { if (fpAllocLog != stdout) { fclose(fpAllocLog); @@ -432,7 +432,7 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) { if (path == NULL || path[0] == 0) { fpAllocLog = stdout; } else if ((fpAllocLog = fopen(path, "w")) != NULL) { - atexit(close_alloc_log); + atexit(taosCloseAllocLog); } else { printf("failed to open memory allocation log file '%s', errno=%d\n", path, errno); return; @@ -446,10 +446,10 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) { } if (autoDump && mode == TAOS_ALLOC_MODE_DETECT_LEAK) { - atexit(dump_memory_leak); + atexit(taosDumpMemoryLeakImp); struct sigaction act = {0}; - act.sa_handler = dump_memory_leak_on_sig; + act.sa_handler = taosDumpMemoryLeakOnSig; sigaction(SIGFPE, &act, NULL); sigaction(SIGSEGV, &act, NULL); sigaction(SIGILL, &act, NULL); @@ -457,8 +457,8 @@ void taosSetAllocMode(int mode, const char* path, bool autoDump) { } void taosDumpMemoryLeak() { - dump_memory_leak(); - close_alloc_log(); + taosDumpMemoryLeakImp(); + taosCloseAllocLog(); } #else // 'TAOS_MEM_CHECK' not defined diff --git a/src/os/src/detail/osSysinfo.c b/src/os/src/detail/osSysinfo.c index 8df671f9c8..b0ca6139ed 100644 --- a/src/os/src/detail/osSysinfo.c +++ b/src/os/src/detail/osSysinfo.c @@ -61,7 +61,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { size_t len; char * line = NULL; while (!feof(fp)) { - taosTFree(line); + tfree(line); len = 0; getline(&line, &len, fp); if (line == NULL) { @@ -83,7 +83,7 @@ bool taosGetProcMemory(float *memoryUsedMB) { sscanf(line, "%s %" PRId64, tmp, &memKB); *memoryUsedMB = (float)((double)memKB / 1024); - taosTFree(line); + tfree(line); fclose(fp); return true; } @@ -107,7 +107,7 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) { char cpu[10] = {0}; sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle); - taosTFree(line); + tfree(line); fclose(fp); return true; } @@ -136,7 +136,7 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) { } } - taosTFree(line); + tfree(line); fclose(fp); return true; } @@ -377,7 +377,7 @@ static bool taosGetCardInfo(int64_t *bytes) { *bytes += (rbytes + tbytes); } - taosTFree(line); + tfree(line); fclose(fp); return true; @@ -432,7 +432,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { int readIndex = 0; while (!feof(fp)) { - taosTFree(line); + tfree(line); len = 0; getline(&line, &len, fp); if (line == NULL) { @@ -450,7 +450,7 @@ static bool taosReadProcIO(int64_t *readbyte, int64_t *writebyte) { if (readIndex >= 2) break; } - taosTFree(line); + tfree(line); fclose(fp); if (readIndex < 2) { diff --git a/src/plugins/http/src/httpContext.c b/src/plugins/http/src/httpContext.c index ec60b984b2..22f464924e 100644 --- a/src/plugins/http/src/httpContext.c +++ b/src/plugins/http/src/httpContext.c @@ -63,7 +63,7 @@ static void httpDestroyContext(void *data) { pContext->parser = NULL; } - taosTFree(pContext); + tfree(pContext); } bool httpInitContexts() { diff --git a/src/plugins/http/src/httpSystem.c b/src/plugins/http/src/httpSystem.c index 8993b233dd..3b8858b62e 100644 --- a/src/plugins/http/src/httpSystem.c +++ b/src/plugins/http/src/httpSystem.c @@ -107,7 +107,7 @@ void httpCleanUpSystem() { httpCleanupResultQueue(); pthread_mutex_destroy(&tsHttpServer.serverMutex); - taosTFree(tsHttpServer.pThreads); + tfree(tsHttpServer.pThreads); tsHttpServer.pThreads = NULL; tsHttpServer.status = HTTP_SERVER_CLOSED; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 10417f03b8..17e39f031e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1019,7 +1019,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * bool hasTimeWindow = false; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { - taosTFree(sasArray); + tfree(sasArray); return; } @@ -1081,10 +1081,10 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * continue; } - taosTFree(sasArray[i].data); + tfree(sasArray[i].data); } - taosTFree(sasArray); + tfree(sasArray); } static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { @@ -1410,7 +1410,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - taosTFree(sasArray[i].data); + tfree(sasArray[i].data); } free(sasArray); @@ -1596,7 +1596,7 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx p->tagInfo.numOfTagCols = num; p->tagInfo.tagsLen = tagLen; } else { - taosTFree(pTagCtx); + tfree(pTagCtx); } } @@ -1706,7 +1706,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order return TSDB_CODE_SUCCESS; _clean: - taosTFree(pRuntimeEnv->pCtx); + tfree(pRuntimeEnv->pCtx); return TSDB_CODE_QRY_OUT_OF_MEMORY; } @@ -1744,10 +1744,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { } tVariantDestroy(&pCtx->tag); - taosTFree(pCtx->tagInfo.pTagCtxList); + tfree(pCtx->tagInfo.pTagCtxList); } - taosTFree(pRuntimeEnv->pCtx); + tfree(pRuntimeEnv->pCtx); } pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); @@ -1756,8 +1756,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { doFreeQueryHandle(pQInfo); pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); - taosTFree(pRuntimeEnv->keyBuf); - taosTFree(pRuntimeEnv->rowCellInfoOffset); + tfree(pRuntimeEnv->keyBuf); + tfree(pRuntimeEnv->rowCellInfoOffset); taosHashCleanup(pRuntimeEnv->pResultRowHashTable); pRuntimeEnv->pResultRowHashTable = NULL; @@ -2985,8 +2985,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); if (pTableList == NULL || posList == NULL) { - taosTFree(posList); - taosTFree(pTableList); + tfree(posList); + tfree(pTableList); qError("QInfo:%p failed alloc memory", pQInfo); longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -3010,12 +3010,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { // there is no data in current group if (numOfTables == 0) { - taosTFree(posList); - taosTFree(pTableList); + tfree(posList); + tfree(pTableList); return 0; } else if (numOfTables == 1) { // no need to merge results since only one table in each group - taosTFree(posList); - taosTFree(pTableList); + tfree(posList); + tfree(pTableList); SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; @@ -3045,9 +3045,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); - taosTFree(pTableList); - taosTFree(posList); - taosTFree(pTree); + tfree(pTableList); + tfree(posList); + tfree(pTree); longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -3118,9 +3118,9 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); - taosTFree(pTree); - taosTFree(pTableList); - taosTFree(posList); + tfree(pTree); + tfree(pTableList); + tfree(posList); return -1; } } @@ -3133,12 +3133,12 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt); - taosTFree(pTableList); - taosTFree(posList); - taosTFree(pTree); + tfree(pTableList); + tfree(posList); + tfree(pTree); -// taosTFree(pResultInfo); -// taosTFree(buf); +// tfree(pResultInfo); +// tfree(buf); return pQInfo->groupResInfo.numOfDataPages; } @@ -5880,13 +5880,13 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, return TSDB_CODE_SUCCESS; _cleanup: - taosTFree(*pExpr); + tfree(*pExpr); taosArrayDestroy(*pTableIdList); *pTableIdList = NULL; - taosTFree(*tbnameCond); - taosTFree(*groupbyCols); - taosTFree(*tagCols); - taosTFree(*tagCond); + tfree(*tbnameCond); + tfree(*groupbyCols); + tfree(*tagCols); + tfree(*tagCond); return code; } @@ -5937,7 +5937,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * code = buildAirthmeticExprFromMsg(&pExprs[i], pQueryMsg); if (code != TSDB_CODE_SUCCESS) { - taosTFree(pExprs); + tfree(pExprs); return code; } @@ -5976,7 +5976,7 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * int32_t param = (int32_t)pExprs[i].base.arg[0].argValue.i64; if (getResultDataInfo(type, bytes, pExprs[i].base.functionId, param, &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, 0, isSuperTable) != TSDB_CODE_SUCCESS) { - taosTFree(pExprs); + tfree(pExprs); return TSDB_CODE_QRY_INVALID_MSG; } @@ -6352,7 +6352,7 @@ _cleanup_query: free(pGroupbyExpr); } - taosTFree(pTagCols); + tfree(pTagCols); for (int32_t i = 0; i < numOfOutput; ++i) { SExprInfo* pExprInfo = &pExprs[i]; if (pExprInfo->pExpr != NULL) { @@ -6360,7 +6360,7 @@ _cleanup_query: } } - taosTFree(pExprs); + tfree(pExprs); _cleanup: freeQInfo(pQInfo); @@ -6479,19 +6479,19 @@ static void freeQInfo(SQInfo *pQInfo) { if (pQuery != NULL) { if (pQuery->sdata != NULL) { for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - taosTFree(pQuery->sdata[col]); + tfree(pQuery->sdata[col]); } - taosTFree(pQuery->sdata); + tfree(pQuery->sdata); } if (pQuery->fillVal != NULL) { - taosTFree(pQuery->fillVal); + tfree(pQuery->fillVal); } for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; if (pColFilter->numOfFilters > 0) { - taosTFree(pColFilter->pFilters); + tfree(pColFilter->pFilters); } } @@ -6504,31 +6504,31 @@ static void freeQInfo(SQInfo *pQInfo) { } } - taosTFree(pQuery->pSelectExpr); + tfree(pQuery->pSelectExpr); } if (pQuery->pGroupbyExpr != NULL) { taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); - taosTFree(pQuery->pGroupbyExpr); + tfree(pQuery->pGroupbyExpr); } - taosTFree(pQuery->tagColList); - taosTFree(pQuery->pFilterInfo); + tfree(pQuery->tagColList); + tfree(pQuery->pFilterInfo); if (pQuery->colList != NULL) { for (int32_t i = 0; i < pQuery->numOfCols; i++) { SColumnInfo *column = pQuery->colList + i; freeColumnFilterInfo(column->filters, column->numOfFilters); } - taosTFree(pQuery->colList); + tfree(pQuery->colList); } - taosTFree(pQuery); + tfree(pQuery); } doDestroyTableQueryInfo(&pQInfo->tableqinfoGroupInfo); - taosTFree(pQInfo->pBuf); + tfree(pQInfo->pBuf); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); @@ -6536,7 +6536,7 @@ static void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p QInfo is freed", pQInfo); - taosTFree(pQInfo); + tfree(pQInfo); } static size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows) { @@ -7231,7 +7231,7 @@ void qCleanupQueryMgmt(void* pQMgmt) { taosCacheCleanup(pqinfoPool); pthread_mutex_destroy(&pQueryMgmt->lock); - taosTFree(pQueryMgmt); + tfree(pQueryMgmt); qDebug("vgId:%d queryMgmt cleanup completed", vgId); } diff --git a/src/query/src/qExtbuffer.c b/src/query/src/qExtbuffer.c index 17be294531..fa3fe285a8 100644 --- a/src/query/src/qExtbuffer.c +++ b/src/query/src/qExtbuffer.c @@ -64,7 +64,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { // release flush out info link SExtFileInfo *pFileMeta = &pMemBuffer->fileMeta; if (pFileMeta->flushoutData.nAllocSize != 0 && pFileMeta->flushoutData.pFlushoutInfo != NULL) { - taosTFree(pFileMeta->flushoutData.pFlushoutInfo); + tfree(pFileMeta->flushoutData.pFlushoutInfo); } // release all in-memory buffer pages @@ -72,7 +72,7 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { while (pFilePages != NULL) { tFilePagesItem *pTmp = pFilePages; pFilePages = pFilePages->pNext; - taosTFree(pTmp); + tfree(pTmp); } // close temp file @@ -87,8 +87,8 @@ void* destoryExtMemBuffer(tExtMemBuffer *pMemBuffer) { destroyColumnModel(pMemBuffer->pColumnModel); - taosTFree(pMemBuffer->path); - taosTFree(pMemBuffer); + tfree(pMemBuffer->path); + tfree(pMemBuffer); return NULL; } @@ -275,7 +275,7 @@ int32_t tExtMemBufferFlush(tExtMemBuffer *pMemBuffer) { tFilePagesItem *ptmp = first; first = first->pNext; - taosTFree(ptmp); // release all data in memory buffer + tfree(ptmp); // release all data in memory buffer } fflush(pMemBuffer->file); // flush to disk @@ -300,7 +300,7 @@ void tExtMemBufferClear(tExtMemBuffer *pMemBuffer) { while (first != NULL) { tFilePagesItem *ptmp = first; first = first->pNext; - taosTFree(ptmp); + tfree(ptmp); } pMemBuffer->fileMeta.numOfElemsInFile = 0; @@ -802,7 +802,7 @@ void destroyColumnModel(SColumnModel *pModel) { return; } - taosTFree(pModel); + tfree(pModel); } static void printBinaryData(char *data, int32_t len) { @@ -1087,5 +1087,5 @@ void tOrderDescDestroy(tOrderDescriptor *pDesc) { } destroyColumnModel(pDesc->pColumnModel); - taosTFree(pDesc); + tfree(pDesc); } diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index 788779b2bb..33d627c839 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -96,18 +96,18 @@ void* taosDestroyFillInfo(SFillInfo* pFillInfo) { return NULL; } - taosTFree(pFillInfo->prevValues); - taosTFree(pFillInfo->nextValues); - taosTFree(pFillInfo->pTags); + tfree(pFillInfo->prevValues); + tfree(pFillInfo->nextValues); + tfree(pFillInfo->pTags); for(int32_t i = 0; i < pFillInfo->numOfCols; ++i) { - taosTFree(pFillInfo->pData[i]); + tfree(pFillInfo->pData[i]); } - taosTFree(pFillInfo->pData); - taosTFree(pFillInfo->pFillCol); + tfree(pFillInfo->pData); + tfree(pFillInfo->pFillCol); - taosTFree(pFillInfo); + tfree(pFillInfo); return NULL; } @@ -496,7 +496,7 @@ int32_t generateDataBlockImpl(SFillInfo* pFillInfo, tFilePage** data, int32_t nu pFillInfo->numOfRows = 0; /* the raw data block is exhausted, next value does not exists */ - taosTFree(*nextValues); + tfree(*nextValues); } pFillInfo->numOfTotal += pFillInfo->numOfCurrent; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 6e4b08316d..02a7012b0e 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -571,7 +571,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { doDestroyQuerySql(pQuerySql); } - taosTFree(pClause->pClause); + tfree(pClause->pClause); } SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName, @@ -641,12 +641,12 @@ void SQLInfoDestroy(SSqlInfo *pInfo) { tFieldListDestroy(pCreateTableInfo->colInfo.pTagColumns); tVariantListDestroy(pCreateTableInfo->usingInfo.pTagVals); - taosTFree(pInfo->pCreateTableInfo); + tfree(pInfo->pCreateTableInfo); } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { tVariantListDestroy(pInfo->pAlterInfo->varList); tFieldListDestroy(pInfo->pAlterInfo->pAddColumns); - taosTFree(pInfo->pAlterInfo); + tfree(pInfo->pAlterInfo); } else { if (pInfo->pDCLInfo != NULL && pInfo->pDCLInfo->nAlloc > 0) { free(pInfo->pDCLInfo->a); @@ -656,7 +656,7 @@ void SQLInfoDestroy(SSqlInfo *pInfo) { tVariantListDestroy(pInfo->pDCLInfo->dbOpt.keep); } - taosTFree(pInfo->pDCLInfo); + tfree(pInfo->pDCLInfo); } } diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index ab9ffb7bcb..3bdc0d477f 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -361,8 +361,8 @@ void tMemBucketDestroy(tMemBucket *pBucket) { } destroyResultBuf(pBucket->pBuffer); - taosTFree(pBucket->pSlots); - taosTFree(pBucket); + tfree(pBucket->pSlots); + tfree(pBucket); } void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) { @@ -680,7 +680,7 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) } double val = (1 - fraction) * td + fraction * nd; - taosTFree(buffer); + tfree(buffer); return val; } else { // incur a second round bucket split diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index edb2ca687f..2645cff678 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -267,7 +267,7 @@ static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { assert(d->pn == pn); d->pn = NULL; - taosTFree(pn); + tfree(pn); bufPage = flushPageToDisk(pResultBuf, d); } @@ -418,7 +418,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } unlink(pResultBuf->path); - taosTFree(pResultBuf->path); + tfree(pResultBuf->path); SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet); while(taosHashIterNext(iter)) { @@ -426,8 +426,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { size_t n = taosArrayGetSize(*p); for(int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); - taosTFree(pi->pData); - taosTFree(pi); + tfree(pi->pData); + tfree(pi); } taosArrayDestroy(*p); @@ -440,8 +440,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { taosHashCleanup(pResultBuf->groupSet); taosHashCleanup(pResultBuf->all); - taosTFree(pResultBuf->assistBuf); - taosTFree(pResultBuf); + tfree(pResultBuf->assistBuf); + tfree(pResultBuf); } SPageInfo* getLastPageInfo(SIDList pList) { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index dd5297bc78..8a5a87baab 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -142,11 +142,11 @@ void* tsBufDestroy(STSBuf* pTSBuf) { return NULL; } - taosTFree(pTSBuf->assistBuf); - taosTFree(pTSBuf->tsData.rawBuf); + tfree(pTSBuf->assistBuf); + tfree(pTSBuf->tsData.rawBuf); - taosTFree(pTSBuf->pData); - taosTFree(pTSBuf->block.payload); + tfree(pTSBuf->pData); + tfree(pTSBuf->block.payload); fclose(pTSBuf->f); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index dc968bad06..f057dc1a49 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -57,7 +57,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { return; } - taosTFree(pWindowResInfo->pResult); + tfree(pWindowResInfo->pResult); } void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) { @@ -358,11 +358,11 @@ void* destroyResultRowPool(SResultRowPool* p) { size_t size = taosArrayGetSize(p->pData); for(int32_t i = 0; i < size; ++i) { void** ptr = taosArrayGet(p->pData, i); - taosTFree(*ptr); + tfree(*ptr); } taosArrayDestroy(p->pData); - taosTFree(p); + tfree(p); return NULL; } diff --git a/src/rpc/src/rpcCache.c b/src/rpc/src/rpcCache.c index 46b0d4e3bb..09d8f3bff1 100644 --- a/src/rpc/src/rpcCache.c +++ b/src/rpc/src/rpcCache.c @@ -101,9 +101,9 @@ void rpcCloseConnCache(void *handle) { if (pCache->connHashMemPool) taosMemPoolCleanUp(pCache->connHashMemPool); - taosTFree(pCache->connHashList); - taosTFree(pCache->count); - taosTFree(pCache->lockedBy); + tfree(pCache->connHashList); + tfree(pCache->count); + tfree(pCache->lockedBy); pthread_mutex_unlock(&pCache->mutex); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 05275c28b0..f963eeb68d 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1620,10 +1620,10 @@ static void rpcDecRef(SRpcInfo *pRpc) taosTmrCleanUp(pRpc->tmrCtrl); taosIdPoolCleanUp(pRpc->idPool); - taosTFree(pRpc->connList); + tfree(pRpc->connList); pthread_mutex_destroy(&pRpc->mutex); tDebug("%s rpc resources are released", pRpc->label); - taosTFree(pRpc); + tfree(pRpc); atomic_sub_fetch_32(&tsRpcNum, 1); } diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index c6de398608..7b32d3416d 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -236,8 +236,8 @@ void taosCleanUpTcpServer(void *handle) { tDebug("%s TCP server is cleaned up", pServerObj->label); - taosTFree(pServerObj->pThreadObj); - taosTFree(pServerObj); + tfree(pServerObj->pThreadObj); + tfree(pServerObj); } static void *taosAcceptTcpConnection(void *arg) { @@ -535,7 +535,7 @@ static void *taosProcessTcpData(void *param) { pthread_mutex_destroy(&(pThreadObj->mutex)); tDebug("%s TCP thread exits ...", pThreadObj->label); - taosTFree(pThreadObj); + tfree(pThreadObj); return NULL; } @@ -556,7 +556,7 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { event.events = EPOLLIN | EPOLLRDHUP; event.data.ptr = pFdObj; if (epoll_ctl(pThreadObj->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) { - taosTFree(pFdObj); + tfree(pFdObj); terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } @@ -609,5 +609,5 @@ static void taosFreeFdObj(SFdObj *pFdObj) { tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); - taosTFree(pFdObj); + tfree(pFdObj); } diff --git a/src/rpc/src/rpcUdp.c b/src/rpc/src/rpcUdp.c index 9bac2b1c19..22301fcecc 100644 --- a/src/rpc/src/rpcUdp.c +++ b/src/rpc/src/rpcUdp.c @@ -147,7 +147,7 @@ void taosStopUdpConnection(void *handle) { if (taosCheckPthreadValid(pConn->thread)) { pthread_join(pConn->thread, NULL); } - taosTFree(pConn->buffer); + tfree(pConn->buffer); // tTrace("%s UDP thread is closed, index:%d", pConn->label, i); } @@ -166,7 +166,7 @@ void taosCleanUpUdpConnection(void *handle) { } tDebug("%s UDP is cleaned up", pSet->label); - taosTFree(pSet); + tfree(pSet); } void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { diff --git a/src/sync/inc/taosTcpPool.h b/src/sync/inc/taosTcpPool.h index 261d190ad3..41043b0cd4 100644 --- a/src/sync/inc/taosTcpPool.h +++ b/src/sync/inc/taosTcpPool.h @@ -20,23 +20,23 @@ extern "C" { #endif -typedef void* ttpool_h; -typedef void* tthread_h; +typedef void *ttpool_h; +typedef void *tthread_h; typedef struct { - int numOfThreads; + int32_t numOfThreads; uint32_t serverIp; int16_t port; - int bufferSize; - void (*processBrokenLink)(void *ahandle); - int (*processIncomingMsg)(void *ahandle, void *buffer); - void (*processIncomingConn)(int fd, uint32_t ip); + int32_t bufferSize; + void (*processBrokenLink)(void *ahandle); + int32_t (*processIncomingMsg)(void *ahandle, void *buffer); + void (*processIncomingConn)(int32_t fd, uint32_t ip); } SPoolInfo; -ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo); -void taosCloseTcpThreadPool(ttpool_h); -void *taosAllocateTcpConn(void *, void *ahandle, int connFd); -void taosFreeTcpConn(void *); +ttpool_h taosOpenTcpThreadPool(SPoolInfo *pInfo); +void taosCloseTcpThreadPool(ttpool_h); +void * taosAllocateTcpConn(void *, void *ahandle, int32_t connFd); +void taosFreeTcpConn(void *); #ifdef __cplusplus } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 3d860f8bbb..1d0dab7f46 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -437,9 +437,9 @@ static void syncFreeNode(void *param) { SSyncNode *pNode = param; pthread_mutex_destroy(&pNode->mutex); - taosTFree(pNode->pRecv); - taosTFree(pNode->pSyncFwds); - taosTFree(pNode); + tfree(pNode->pRecv); + tfree(pNode->pSyncFwds); + tfree(pNode); } void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } @@ -449,8 +449,8 @@ int syncDecPeerRef(SSyncPeer *pPeer) { taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); sDebug("%s, resource is freed", pPeer->id); - taosTFree(pPeer->watchFd); - taosTFree(pPeer); + tfree(pPeer->watchFd); + tfree(pPeer); return 0; } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index b35388db35..004dae1729 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -225,10 +225,10 @@ int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { static void syncCloseRecvBuffer(SSyncNode *pNode) { if (pNode->pRecv) { - taosTFree(pNode->pRecv->buffer); + tfree(pNode->pRecv->buffer); } - taosTFree(pNode->pRecv); + tfree(pNode->pRecv); } static int syncOpenRecvBuffer(SSyncNode *pNode) { diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index 7397a629d3..3024d7d4e3 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -108,7 +108,7 @@ void taosCloseTcpThreadPool(void *param) { sDebug("%p TCP pool is closed", pPool); - taosTFree(pPool->pThread); + tfree(pPool->pThread); tfree(pPool); } diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 360ea93f6c..34e936c9f3 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -131,7 +131,7 @@ static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) { snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); if (firstPkt.syncHead.vgId) { sDebug("%s, vgId in head is not zero, close the connection", pNode->id); - taosTFree(pNode); + tfree(pNode); taosCloseSocket(connFd); return; } @@ -147,7 +147,7 @@ static void arbProcessBrokenLink(void *param) { SNodeConn *pNode = param; sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno)); - taosTFree(pNode); + tfree(pNode); } static int arbProcessPeerMsg(void *param, void *buffer) { diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 2e097c6ff7..dcc9d4ca1b 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -157,4 +157,4 @@ _err: return NULL; } -static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { taosTFree(pBufBlock); } \ No newline at end of file +static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } \ No newline at end of file diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 3a5416dd30..1efde49ed0 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -65,7 +65,7 @@ _err: void tsdbFreeFileH(STsdbFileH *pFileH) { if (pFileH) { pthread_rwlock_destroy(&pFileH->fhlock); - taosTFree(pFileH->pFGroup); + tfree(pFileH->pFGroup); free(pFileH); } } @@ -199,7 +199,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { regfree(®ex1); regfree(®ex2); - taosTFree(tDataDir); + tfree(tDataDir); closedir(dir); return 0; @@ -209,7 +209,7 @@ _err: regfree(®ex1); regfree(®ex2); - taosTFree(tDataDir); + tfree(tDataDir); if (dir != NULL) closedir(dir); tsdbCloseFileH(pRepo); return -1; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 9a7c2db3d3..e2523d1145 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -228,7 +228,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ int prefixLen = (int)strlen(prefix); if (name[0] == 0) { // get the file from index or after, but not larger than eindex - taosTFree(sdup); + tfree(sdup); int fid = (*index) / TSDB_FILE_TYPE_MAX; if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { @@ -260,8 +260,8 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ fname = malloc(prefixLen + strlen(name) + 2); sprintf(fname, "%s/%s", prefix, name); if (access(fname, F_OK) != 0) { - taosFree(fname); - taosFree(sdup); + tfree(fname); + tfree(sdup); return 0; } if (*index == TSDB_META_FILE_INDEX) { // get meta file @@ -269,20 +269,20 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ } else { tsdbGetFileInfoImpl(fname, &magic, size); } - taosFree(fname); - taosFree(sdup); + tfree(fname); + tfree(sdup); return magic; } if (stat(fname, &fState) < 0) { - taosTFree(fname); + tfree(fname); return 0; } *size = fState.st_size; // magic = *size; - taosTFree(fname); + tfree(fname); return magic; } @@ -600,7 +600,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { return 0; _err: - taosTFree(fname); + tfree(fname); if (fd >= 0) close(fd); return -1; } @@ -637,13 +637,13 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { tsdbDecodeCfg(buf, pCfg); - taosTFree(fname); + tfree(fname); close(fd); return 0; _err: - taosTFree(fname); + tfree(fname); if (fd >= 0) close(fd); return -1; } @@ -718,7 +718,7 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { tsdbFreeMeta(pRepo->tsdbMeta); // tsdbFreeMemTable(pRepo->mem); // tsdbFreeMemTable(pRepo->imem); - taosTFree(pRepo->rootDir); + tfree(pRepo->rootDir); pthread_mutex_destroy(&pRepo->mutex); free(pRepo); } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 150bda3b80..081929a875 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -488,7 +488,7 @@ static void tsdbFreeMemTable(SMemTable* pMemTable) { tdListFree(pMemTable->extraBuffList); tdListFree(pMemTable->bufBlockList); tdListFree(pMemTable->actList); - taosTFree(pMemTable->tData); + tfree(pMemTable->tData); free(pMemTable); } } @@ -746,7 +746,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe goto _err; } - taosTFree(dataDir); + tfree(dataDir); tsdbCloseHelperFile(pHelper, 0, pGroup); pthread_rwlock_wrlock(&(pFileH->fhlock)); @@ -768,7 +768,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe return 0; _err: - taosTFree(dataDir); + tfree(dataDir); tsdbCloseHelperFile(pHelper, 1, NULL); return -1; } @@ -843,7 +843,7 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { pMemTable->tData = pTableData; taosWUnLockLatch(&(pMemTable->latch)); - taosTFree(tData); + tfree(tData); return 0; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 6811c976ca..25c815b74e 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -192,7 +192,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { return 0; _err: - taosTFree(tbname); + tfree(tbname); return -1; } @@ -462,7 +462,7 @@ void tsdbFreeMeta(STsdbMeta *pMeta) { if (pMeta) { taosHashCleanup(pMeta->uidMap); tdListFree(pMeta->superList); - taosTFree(pMeta->tables); + tfree(pMeta->tables); pthread_rwlock_destroy(&pMeta->rwLock); free(pMeta); } @@ -486,11 +486,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) { } tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo)); - taosTFree(fname); + tfree(fname); return 0; _err: - taosTFree(fname); + tfree(fname); return -1; } @@ -761,7 +761,7 @@ static void tsdbFreeTable(STable *pTable) { if (pTable->name != NULL) tsdbTrace("table %s tid %d uid %" PRIu64 " is freed", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); - taosTFree(TABLE_NAME(pTable)); + tfree(TABLE_NAME(pTable)); if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) { tdFreeSchema(pTable->schema[i]); @@ -775,7 +775,7 @@ static void tsdbFreeTable(STable *pTable) { kvRowFree(pTable->tagVal); tSkipListDestroy(pTable->pIndex); - taosTFree(pTable->sql); + tfree(pTable->sql); free(pTable); } } @@ -1066,9 +1066,9 @@ void tsdbClearTableCfg(STableCfg *config) { if (config->schema) tdFreeSchema(config->schema); if (config->tagSchema) tdFreeSchema(config->tagSchema); if (config->tagValues) kvRowFree(config->tagValues); - taosTFree(config->name); - taosTFree(config->sname); - taosTFree(config->sql); + tfree(config->name); + tfree(config->sname); + tfree(config->sql); free(config); } } @@ -1290,7 +1290,7 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { STable **tTables = pMeta->tables; pMeta->tables = tables; - taosTFree(tTables); + tfree(tTables); tsdbDebug("vgId:%d tsdb meta maxTables is adjusted as %d", REPO_ID(pRepo), maxTables); return 0; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 27f49b1902..021c10ab6a 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1544,15 +1544,15 @@ int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { } static void cleanBlockOrderSupporter(SBlockOrderSupporter* pSupporter, int32_t numOfTables) { - taosTFree(pSupporter->numOfBlocksPerTable); - taosTFree(pSupporter->blockIndexArray); + tfree(pSupporter->numOfBlocksPerTable); + tfree(pSupporter->blockIndexArray); for (int32_t i = 0; i < numOfTables; ++i) { STableBlockInfo* pBlockInfo = pSupporter->pDataBlockInfo[i]; - taosTFree(pBlockInfo); + tfree(pBlockInfo); } - taosTFree(pSupporter->pDataBlockInfo); + tfree(pSupporter->pDataBlockInfo); } static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* param) { @@ -1970,7 +1970,7 @@ static void destroyHelper(void* param) { tQueryInfo* pInfo = (tQueryInfo*)param; if (pInfo->optr != TSDB_RELATION_IN) { - taosTFree(pInfo->q); + tfree(pInfo->q); } free(param); @@ -2035,7 +2035,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { STsdbQueryHandle* pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qinfo, pQueryHandle->pMemRef); - taosTFree(cond.colList); + tfree(cond.colList); pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pQueryHandle->pTableCheckInfo, pSecQueryHandle->window.skey); if (pSecQueryHandle->pTableCheckInfo == NULL) { @@ -2750,7 +2750,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); destroyTableMemIterator(pTableCheckInfo); - taosTFree(pTableCheckInfo->pCompInfo); + tfree(pTableCheckInfo->pCompInfo); } taosArrayDestroy(pQueryHandle->pTableCheckInfo); } @@ -2759,14 +2759,14 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { size_t cols = taosArrayGetSize(pQueryHandle->pColumns); for (int32_t i = 0; i < cols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - taosTFree(pColInfo->pData); + tfree(pColInfo->pData); } taosArrayDestroy(pQueryHandle->pColumns); } taosArrayDestroy(pQueryHandle->defaultLoadColumn); - taosTFree(pQueryHandle->pDataBlockInfo); - taosTFree(pQueryHandle->statis); + tfree(pQueryHandle->pDataBlockInfo); + tfree(pQueryHandle->statis); // todo check error tsdbMayUnTakeMemSnapshot(pQueryHandle); @@ -2780,7 +2780,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); - taosTFree(pQueryHandle); + tfree(pQueryHandle); } void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index 605586515b..ef5ed6f044 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -80,7 +80,7 @@ static int insertData(SInsertInfo *pInfo) { pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); if (tsdbInsertData(pInfo->pRepo, pMsg, NULL) < 0) { - taosTFree(pMsg); + tfree(pMsg); return -1; } } @@ -88,7 +88,7 @@ static int insertData(SInsertInfo *pInfo) { double etime = getCurTime(); printf("Spent %f seconds to write %d records\n", etime - stime, pInfo->totalRows); - taosTFree(pMsg); + tfree(pMsg); return 0; } diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 22b5da1491..03a7342497 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -22,7 +22,7 @@ #define DO_FREE_HASH_NODE(_n) \ do { \ - taosTFree(_n); \ + tfree(_n); \ } while (0) #define FREE_HASH_NODE(_h, _n) \ @@ -522,7 +522,7 @@ void taosHashCleanup(SHashObj *pHashObj) { size_t memBlock = taosArrayGetSize(pHashObj->pMemBlock); for (int32_t i = 0; i < memBlock; ++i) { void *p = taosArrayGetP(pHashObj->pMemBlock, i); - taosTFree(p); + tfree(p); } taosArrayDestroy(pHashObj->pMemBlock); diff --git a/src/util/src/talgo.c b/src/util/src/talgo.c index 4b96e62e91..278683539e 100644 --- a/src/util/src/talgo.c +++ b/src/util/src/talgo.c @@ -153,7 +153,7 @@ static void tqsortImpl(void *src, int32_t start, int32_t end, size_t size, const void taosqsort(void *src, size_t numOfElem, size_t size, const void* param, __ext_compar_fn_t comparFn) { char *buf = calloc(1, size); // prepare the swap buffer tqsortImpl(src, 0, (int32_t)numOfElem - 1, (int32_t)size, param, comparFn, buf); - taosTFree(buf); + tfree(buf); } void * taosbsearch(const void *key, const void *base, size_t nmemb, size_t size, __compar_fn_t compar, int flags) { diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 5be7253f6d..9bdaf73700 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -228,7 +228,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v pCacheObj->freeFp(p->data); } - taosTFree(p); + tfree(p); } else { taosAddToTrashcan(pCacheObj, p); uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data); @@ -614,7 +614,7 @@ void doCleanupDataCache(SCacheObj *pCacheObj) { __cache_lock_destroy(pCacheObj); - taosTFree(pCacheObj->name); + tfree(pCacheObj->name); memset(pCacheObj, 0, sizeof(SCacheObj)); free(pCacheObj); } diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index f449bfb68b..e89dea5a24 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -288,7 +288,7 @@ void taosReadGlobalLogCfg() { option = value = NULL; olen = vlen = 0; - taosGetline(&line, &len, fp); + tgetline(&line, &len, fp); line[len - 1] = 0; paGetToken(line, &option, &olen); @@ -302,7 +302,7 @@ void taosReadGlobalLogCfg() { taosReadLogOption(option, value); } - taosTFree(line); + tfree(line); fclose(fp); } @@ -334,7 +334,7 @@ bool taosReadGlobalCfg() { option = value = NULL; olen = vlen = 0; - taosGetline(&line, &len, fp); + tgetline(&line, &len, fp); line[len - 1] = 0; paGetToken(line, &option, &olen); @@ -354,7 +354,7 @@ bool taosReadGlobalCfg() { fclose(fp); - taosTFree(line); + tfree(line); if (debugFlag & DEBUG_TRACE || debugFlag & DEBUG_DEBUG || debugFlag & DEBUG_DUMP) { taosSetAllDebugFlag(); diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 0806c29ff8..101dd4a6f4 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -475,9 +475,9 @@ _err: static void tdFreeKVStore(SKVStore *pStore) { if (pStore) { - taosTFree(pStore->fname); - taosTFree(pStore->fsnap); - taosTFree(pStore->fnew); + tfree(pStore->fname); + tfree(pStore->fsnap); + tfree(pStore->fnew); taosHashCleanup(pStore->map); free(pStore); } @@ -616,11 +616,11 @@ static int tdRestoreKVStore(SKVStore *pStore) { if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); taosHashDestroyIter(pIter); - taosTFree(buf); + tfree(buf); return 0; _err: taosHashDestroyIter(pIter); - taosTFree(buf); + tfree(buf); return -1; } diff --git a/src/util/src/tlog.c b/src/util/src/tlog.c index 0ad7917b39..4157d88c45 100644 --- a/src/util/src/tlog.c +++ b/src/util/src/tlog.c @@ -512,8 +512,8 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { return tLogBuff; _err: - taosTFree(LOG_BUF_BUFFER(tLogBuff)); - taosTFree(tLogBuff); + tfree(LOG_BUF_BUFFER(tLogBuff)); + tfree(tLogBuff); return NULL; } @@ -522,7 +522,7 @@ static void taosLogBuffDestroy(SLogBuff *tLogBuff) { tsem_destroy(&(tLogBuff->buffNotEmpty)); pthread_mutex_destroy(&(tLogBuff->buffMutex)); free(tLogBuff->buffer); - taosTFree(tLogBuff); + tfree(tLogBuff); } #endif diff --git a/src/util/src/tmempool.c b/src/util/src/tmempool.c index a3d1035500..678c965eb1 100644 --- a/src/util/src/tmempool.c +++ b/src/util/src/tmempool.c @@ -52,9 +52,9 @@ mpool_h taosMemPoolInit(int numOfBlock, int blockSize) { if (pool_p->pool == NULL || pool_p->freeList == NULL) { uError("failed to allocate memory\n"); - taosTFree(pool_p->freeList); - taosTFree(pool_p->pool); - taosTFree(pool_p); + tfree(pool_p->freeList); + tfree(pool_p->pool); + tfree(pool_p); return NULL; } diff --git a/src/util/src/tref.c b/src/util/src/tref.c index b998dfd43c..760d1c0eb4 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -484,8 +484,8 @@ static void taosDecRsetCount(SRefSet *pSet) { pSet->max = 0; pSet->fp = NULL; - taosTFree(pSet->nodeList); - taosTFree(pSet->lockedBy); + tfree(pSet->nodeList); + tfree(pSet->lockedBy); tsRefSetNum--; uTrace("rsetId:%d is cleaned, refSetNum:%d count:%d", pSet->rsetId, tsRefSetNum, pSet->count); diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index c8e0febd7a..45e3308bc8 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -27,7 +27,7 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData); static SSkipListNode * tSkipListNewNode(uint8_t level); -#define tSkipListFreeNode(n) taosTFree((n)) +#define tSkipListFreeNode(n) tfree((n)) static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); @@ -97,12 +97,12 @@ void tSkipListDestroy(SSkipList *pSkipList) { tSkipListUnlock(pSkipList); if (pSkipList->lock != NULL) { pthread_rwlock_destroy(pSkipList->lock); - taosTFree(pSkipList->lock); + tfree(pSkipList->lock); } tSkipListFreeNode(pSkipList->pHead); tSkipListFreeNode(pSkipList->pTail); - taosTFree(pSkipList); + tfree(pSkipList); } SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { @@ -265,7 +265,7 @@ void *tSkipListDestroyIter(SSkipListIterator *iter) { return NULL; } - taosTFree(iter); + tfree(iter); return NULL; } diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index cd1fd1f7e1..2203ae8e4f 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -59,7 +59,7 @@ void doubleSkipListTest() { } if (size > 0) { - taosTFree(pNodes); + tfree(pNodes); } } @@ -196,7 +196,7 @@ void stringKeySkiplistTest() { tSkipListRemoveNode(pSkipList, pres[0]); if (num > 0) { - taosTFree(pres); + tfree(pres); } } @@ -276,7 +276,7 @@ void skiplistPerformanceTest() { assert(SL_GET_SIZE(pSkipList) == size); tSkipListDestroy(pSkipList); - taosTFree(total); + tfree(total); } // todo not support duplicated key yet @@ -357,7 +357,7 @@ TEST(testCase, skiplist_test) { printf("-----%lf\n", pNodes[i]->key.dKey); } printf("the range query result size is: %d\n", size); - taosTFree(pNodes); + tfree(pNodes); SSkipListKey *pKeys = malloc(sizeof(SSkipListKey) * 20); for (int32_t i = 0; i < 8; i += 2) { @@ -371,7 +371,7 @@ TEST(testCase, skiplist_test) { for (int32_t i = 0; i < r; ++i) { // printf("%lf ", pNodes[i]->key.dKey); } - taosTFree(pNodes); + tfree(pNodes); free(pKeys);*/ } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index f90b9701fe..bd44ce8e1c 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -416,7 +416,7 @@ void vnodeRelease(void *pVnodeRaw) { pVnode->rqueue = NULL; } - taosTFree(pVnode->rootDir); + tfree(pVnode->rootDir); if (pVnode->dropped) { char rootDir[TSDB_FILENAME_LEN] = {0}; diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 1f6a8f5546..de666c85e8 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -28,14 +28,13 @@ typedef struct { pthread_mutex_t mutex; } SWalMgmt; -static SWalMgmt tsWal; +static SWalMgmt tsWal = {0}; static int32_t walCreateThread(); static void walStopThread(); static int32_t walInitObj(SWal *pWal); static void walFreeObj(void *pWal); int32_t walInit() { - tmemzero(&tsWal, sizeof(SWalMgmt)); tsWal.refId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); int32_t code = walCreateThread(); diff --git a/tests/tsim/src/simSystem.c b/tests/tsim/src/simSystem.c index 8c3f3b8803..693ade7b35 100644 --- a/tests/tsim/src/simSystem.c +++ b/tests/tsim/src/simSystem.c @@ -103,9 +103,9 @@ void simFreeScript(SScript *script) { simDebug("script:%s, is freed", script->fileName); taos_close(script->taos); - taosTFree(script->lines); - taosTFree(script->optionBuffer); - taosTFree(script); + tfree(script->lines); + tfree(script->optionBuffer); + tfree(script); } SScript *simProcessCallOver(SScript *script) { From 6854e550a290a0357851b1d6e57065b7a8861292 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 10 Nov 2020 06:48:31 +0000 Subject: [PATCH 103/122] TD-1793 --- src/mnode/src/mnodeTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index e729fb2254..2b5e6455c0 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -47,7 +47,7 @@ #define ALTER_CTABLE_RETRY_TIMES 3 #define CREATE_CTABLE_RETRY_TIMES 10 -#define CREATE_CTABLE_RETRY_SEC 10 +#define CREATE_CTABLE_RETRY_SEC 14 static void * tsChildTableSdb; static void * tsSuperTableSdb; From 42693517ae217811e2cf31ad5a5a12c039b9e3ec Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 10 Nov 2020 07:09:25 +0000 Subject: [PATCH 104/122] TD-1848 --- src/os/inc/osMemory.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/os/inc/osMemory.h b/src/os/inc/osMemory.h index 438bb88474..0616006650 100644 --- a/src/os/inc/osMemory.h +++ b/src/os/inc/osMemory.h @@ -66,6 +66,7 @@ void taosTMemset(void *ptr, int c); #undef tmalloc #undef tcalloc #undef trealloc + #undef tfree #define tmalloc(size) taosMallocMem(size, __FILE__, __LINE__) #define tcalloc(num, size) taosCallocMem(num, size, __FILE__, __LINE__) #define trealloc(ptr, size) taosReallocMem(ptr, size, __FILE__, __LINE__) From a5fdb787948a0c431a8d705c824e9083a3bcaeb5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 10 Nov 2020 08:03:02 +0000 Subject: [PATCH 105/122] change int to int32_t --- src/inc/tsync.h | 57 +++++----- src/sync/inc/syncInt.h | 46 ++++---- src/sync/src/syncMain.c | 206 ++++++++++++++++++------------------ src/sync/src/syncRestore.c | 26 ++--- src/sync/src/syncRetrieve.c | 60 +++++------ src/sync/src/tarbitrator.c | 22 ++-- 6 files changed, 208 insertions(+), 209 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 0861c94a76..d57433eba9 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -51,9 +51,9 @@ typedef struct { } SSyncCfg; typedef struct { - int selfIndex; - uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; - int role[TAOS_SYNC_MAX_REPLICA]; + int32_t selfIndex; + uint32_t nodeId[TAOS_SYNC_MAX_REPLICA]; + int32_t role[TAOS_SYNC_MAX_REPLICA]; } SNodesRole; /* @@ -83,25 +83,24 @@ typedef void (*FNotifyRole)(void *ahandle, int8_t role); typedef void (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds); // when data file is synced successfully, notity app -typedef int (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); +typedef int32_t (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); typedef struct { - int32_t vgId; // vgroup ID - uint64_t version; // initial version - SSyncCfg syncCfg; // configuration from mgmt - char path[128]; // path to the file - - void *ahandle; // handle provided by APP - FGetFileInfo getFileInfo; - FGetWalInfo getWalInfo; - FWriteToCache writeToCache; - FConfirmForward confirmForward; - FNotifyRole notifyRole; - FNotifyFlowCtrl notifyFlowCtrl; + int32_t vgId; // vgroup ID + uint64_t version; // initial version + SSyncCfg syncCfg; // configuration from mgmt + char path[128]; // path to the file + void * ahandle; // handle provided by APP + FGetFileInfo getFileInfo; + FGetWalInfo getWalInfo; + FWriteToCache writeToCache; + FConfirmForward confirmForward; + FNotifyRole notifyRole; + FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; } SSyncInfo; -typedef void* tsync_h; +typedef void *tsync_h; int32_t syncInit(); void syncCleanUp(); @@ -109,22 +108,22 @@ void syncCleanUp(); int64_t syncStart(const SSyncInfo *); void syncStop(int64_t rid); int32_t syncReconfig(int64_t rid, const SSyncCfg *); -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype); +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype); void syncConfirmForward(int64_t rid, uint64_t version, int32_t code); -void syncRecover(int64_t rid); // recover from other nodes: -int syncGetNodesRole(int64_t rid, SNodesRole *); +void syncRecover(int64_t rid); // recover from other nodes: +int32_t syncGetNodesRole(int64_t rid, SNodesRole *); -extern char *syncRole[]; +extern char *syncRole[]; //global configurable parameters -extern int tsMaxSyncNum; -extern int tsSyncTcpThreads; -extern int tsMaxWatchFiles; -extern int tsSyncTimer; -extern int tsMaxFwdInfo; -extern int sDebugFlag; -extern char tsArbitrator[]; -extern uint16_t tsSyncPort; +extern int32_t tsMaxSyncNum; +extern int32_t tsSyncTcpThreads; +extern int32_t tsMaxWatchFiles; +extern int32_t tsSyncTimer; +extern int32_t tsMaxFwdInfo; +extern int32_t sDebugFlag; +extern char tsArbitrator[]; +extern uint16_t tsSyncPort; #ifdef __cplusplus } diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 8808a82c46..240b401bda 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -89,11 +89,11 @@ typedef struct { #pragma pack(pop) typedef struct { - char *buffer; - int bufferSize; - char *offset; - int forwards; - int code; + char * buffer; + int32_t bufferSize; + char * offset; + int32_t forwards; + int32_t code; } SRecvBuffer; typedef struct { @@ -107,10 +107,10 @@ typedef struct { } SFwdInfo; typedef struct { - int first; - int last; - int fwds; // number of forwards - SFwdInfo fwdInfo[]; + int32_t first; + int32_t last; + int32_t fwds; // number of forwards + SFwdInfo fwdInfo[]; } SSyncFwds; typedef struct SsyncPeer { @@ -123,15 +123,15 @@ typedef struct SsyncPeer { int8_t sstatus; // sync status uint64_t version; uint64_t sversion; // track the peer version in retrieve process - int syncFd; - int peerFd; // forward FD - int numOfRetrieves; // number of retrieves tried - int fileChanged; // a flag to indicate file is changed during retrieving process + int32_t syncFd; + int32_t peerFd; // forward FD + int32_t numOfRetrieves; // number of retrieves tried + int32_t fileChanged; // a flag to indicate file is changed during retrieving process void * timer; void * pConn; - int notifyFd; - int watchNum; - int * watchFd; + int32_t notifyFd; + int32_t watchNum; + int32_t *watchFd; int8_t refCount; // reference count struct SSyncNode *pSyncNode; } SSyncPeer; @@ -161,16 +161,16 @@ typedef struct SSyncNode { } SSyncNode; // sync module global -extern int tsSyncNum; -extern char tsNodeFqdn[TSDB_FQDN_LEN]; +extern int32_t tsSyncNum; +extern char tsNodeFqdn[TSDB_FQDN_LEN]; void *syncRetrieveData(void *param); void *syncRestoreData(void *param); -int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); -void syncRestartConnection(SSyncPeer *pPeer); -void syncBroadcastStatus(SSyncNode *pNode); -void syncAddPeerRef(SSyncPeer *pPeer); -int syncDecPeerRef(SSyncPeer *pPeer); +int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead); +void syncRestartConnection(SSyncPeer *pPeer); +void syncBroadcastStatus(SSyncNode *pNode); +void syncAddPeerRef(SSyncPeer *pPeer); +int32_t syncDecPeerRef(SSyncPeer *pPeer); #ifdef __cplusplus } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 1d0dab7f46..9dcd0fd632 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -31,38 +31,38 @@ #include "syncInt.h" // global configurable -int tsMaxSyncNum = 2; -int tsSyncTcpThreads = 2; -int tsMaxWatchFiles = 500; -int tsMaxFwdInfo = 200; -int tsSyncTimer = 1; +int32_t tsMaxSyncNum = 2; +int32_t tsSyncTcpThreads = 2; +int32_t tsMaxWatchFiles = 500; +int32_t tsMaxFwdInfo = 200; +int32_t tsSyncTimer = 1; // module global, not configurable -int tsSyncNum; // number of sync in process in whole system -char tsNodeFqdn[TSDB_FQDN_LEN]; +int32_t tsSyncNum; // number of sync in process in whole system +char tsNodeFqdn[TSDB_FQDN_LEN]; static ttpool_h tsTcpPool; -static void * syncTmrCtrl = NULL; -static void * vgIdHash; -static int tsSyncRefId = -1; +static void * tsSyncTmrCtrl = NULL; +static void * tsVgIdHash; +static int32_t tsSyncRefId = -1; // local functions -static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); -static void syncRecoverFromMaster(SSyncPeer *pPeer); -static void syncCheckPeerConnection(void *param, void *tmrId); -static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack); -static void syncProcessBrokenLink(void *param); -static int syncProcessPeerMsg(void *param, void *buffer); -static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp); -static void syncRemovePeer(SSyncPeer *pPeer); -static void syncAddArbitrator(SSyncNode *pNode); -static void syncFreeNode(void *); -static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); -static void syncMonitorFwdInfos(void *param, void *tmrId); -static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); -static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); -static void syncRestartPeer(SSyncPeer *pPeer); -static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtyp); +static void syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer); +static void syncRecoverFromMaster(SSyncPeer *pPeer); +static void syncCheckPeerConnection(void *param, void *tmrId); +static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack); +static void syncProcessBrokenLink(void *param); +static int32_t syncProcessPeerMsg(void *param, void *buffer); +static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); +static void syncRemovePeer(SSyncPeer *pPeer); +static void syncAddArbitrator(SSyncNode *pNode); +static void syncFreeNode(void *); +static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode); +static void syncMonitorFwdInfos(void *param, void *tmrId); +static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code); +static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); +static void syncRestartPeer(SSyncPeer *pPeer); +static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp); static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo); char* syncRole[] = { @@ -90,21 +90,21 @@ int32_t syncInit() { return -1; } - syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); - if (syncTmrCtrl == NULL) { + tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC"); + if (tsSyncTmrCtrl == NULL) { sError("failed to init tmrCtrl"); taosCloseTcpThreadPool(tsTcpPool); tsTcpPool = NULL; return -1; } - vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); - if (vgIdHash == NULL) { - sError("failed to init vgIdHash"); - taosTmrCleanUp(syncTmrCtrl); + tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + if (tsVgIdHash == NULL) { + sError("failed to init tsVgIdHash"); + taosTmrCleanUp(tsSyncTmrCtrl); taosCloseTcpThreadPool(tsTcpPool); tsTcpPool = NULL; - syncTmrCtrl = NULL; + tsSyncTmrCtrl = NULL; return -1; } @@ -126,14 +126,14 @@ void syncCleanUp() { tsTcpPool = NULL; } - if (syncTmrCtrl) { - taosTmrCleanUp(syncTmrCtrl); - syncTmrCtrl = NULL; + if (tsSyncTmrCtrl) { + taosTmrCleanUp(tsSyncTmrCtrl); + tsSyncTmrCtrl = NULL; } - if (vgIdHash) { - taosHashCleanup(vgIdHash); - vgIdHash = NULL; + if (tsVgIdHash) { + taosHashCleanup(tsVgIdHash); + tsVgIdHash = NULL; } taosCloseRef(tsSyncRefId); @@ -176,7 +176,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { return -1; } - for (int i = 0; i < pCfg->replica; ++i) { + for (int32_t i = 0; i < pCfg->replica; ++i) { const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i; pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo); if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) { @@ -204,7 +204,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { return -1; } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl); if (pNode->pFwdTimer == NULL) { sError("vgId:%d, failed to allocate timer", pNode->vgId); syncStop(pNode->rid); @@ -212,7 +212,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { } syncAddArbitrator(pNode); - taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); + taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); if (pNode->notifyRole) { (*pNode->notifyRole)(pNode->ahandle, nodeRole); @@ -231,10 +231,10 @@ void syncStop(int64_t rid) { pthread_mutex_lock(&(pNode->mutex)); - if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t)); + if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t)); if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer); - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; if (pPeer) syncRemovePeer(pPeer); } @@ -249,7 +249,7 @@ void syncStop(int64_t rid) { } int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { - int i, j; + int32_t i, j; SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG; @@ -321,7 +321,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { return 0; } -int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int qtype) { +int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) { SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); if (pNode == NULL) return 0; @@ -348,8 +348,8 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) { pFwdRsp->version = version; pFwdRsp->code = code; - int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); - int retLen = write(pPeer->peerFd, msg, msgLen); + int32_t msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp); + int32_t retLen = write(pPeer->peerFd, msg, msgLen); if (retLen == msgLen) { sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version); @@ -377,7 +377,7 @@ void syncRecover(int64_t rid) { pthread_mutex_lock(&(pNode->mutex)); - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = (SSyncPeer *)pNode->peerInfo[i]; if (pPeer->peerFd >= 0) { syncRestartConnection(pPeer); @@ -389,12 +389,12 @@ void syncRecover(int64_t rid) { taosReleaseRef(tsSyncRefId, rid); } -int syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { +int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) { SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid); if (pNode == NULL) return -1; pNodesRole->selfIndex = pNode->selfIndex; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId; pNodesRole->role[i] = pNode->peerInfo[i]->role; } @@ -416,7 +416,7 @@ static void syncAddArbitrator(SSyncNode *pNode) { SNodeInfo nodeInfo; nodeInfo.nodeId = 0; - int ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); + int32_t ret = taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort); if (-1 == ret) { nodeInfo.nodePort = tsArbitratorPort; } @@ -444,7 +444,7 @@ static void syncFreeNode(void *param) { void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_8(&pPeer->refCount, 1); } -int syncDecPeerRef(SSyncPeer *pPeer) { +int32_t syncDecPeerRef(SSyncPeer *pPeer) { if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) { taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); @@ -495,12 +495,12 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->refCount = 1; sInfo("%s, it is configured", pPeer->id); - int ret = strcmp(pPeer->fqdn, tsNodeFqdn); + int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { int32_t checkMs = 100 + (pNode->vgId * 10) % 100; if (pNode->vgId > 1) checkMs = tsStatusInterval * 2000 + checkMs; sDebug("%s, start to check peer connection after %d ms", pPeer->id, checkMs); - taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer); } taosAcquireRef(tsSyncRefId, pNode->rid); @@ -510,7 +510,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { void syncBroadcastStatus(SSyncNode *pNode) { SSyncPeer *pPeer; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { if (i == pNode->selfIndex) continue; pPeer = pNode->peerInfo[i]; syncSendPeersStatusMsgToPeer(pPeer, 1); @@ -518,7 +518,7 @@ void syncBroadcastStatus(SSyncNode *pNode) { } static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pNode->peerInfo[i]->numOfRetrieves = 0; } @@ -529,13 +529,13 @@ static void syncResetFlowCtrl(SSyncNode *pNode) { static void syncChooseMaster(SSyncNode *pNode) { SSyncPeer *pPeer; - int onlineNum = 0; - int index = -1; - int replica = pNode->replica; + int32_t onlineNum = 0; + int32_t index = -1; + int32_t replica = pNode->replica; sDebug("vgId:%d, choose master", pNode->vgId); - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) { onlineNum++; } @@ -544,7 +544,7 @@ static void syncChooseMaster(SSyncNode *pNode) { if (onlineNum == pNode->replica) { // if all peers are online, peer with highest version shall be master index = 0; - for (int i = 1; i < pNode->replica; ++i) { + for (int32_t i = 1; i < pNode->replica; ++i) { if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version) { index = i; } @@ -560,7 +560,7 @@ static void syncChooseMaster(SSyncNode *pNode) { if (index < 0 && onlineNum > replica / 2.0) { // over half of nodes are online - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { // slave with highest version shall be master pPeer = pNode->peerInfo[i]; if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) { @@ -587,11 +587,11 @@ static void syncChooseMaster(SSyncNode *pNode) { } static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { - int onlineNum = 0; - int index = -1; - int replica = pNode->replica; + int32_t onlineNum = 0; + int32_t index = -1; + int32_t replica = pNode->replica; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) { onlineNum++; } @@ -612,7 +612,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { SSyncPeer *pTemp = pNode->peerInfo[i]; if (pTemp->role != TAOS_SYNC_ROLE_MASTER) continue; if (index < 0) { @@ -631,9 +631,9 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { return pMaster; } -static int syncValidateMaster(SSyncPeer *pPeer) { +static int32_t syncValidateMaster(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - int code = 0; + int32_t code = 0; if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { sDebug("%s, slave has higher version, restart all connections!!!", pPeer->id); @@ -641,7 +641,7 @@ static int syncValidateMaster(SSyncPeer *pPeer) { (*pNode->notifyRole)(pNode->ahandle, nodeRole); code = -1; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { if (i == pNode->selfIndex) continue; syncRestartPeer(pNode->peerInfo[i]); } @@ -683,7 +683,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t ne } } else { // master not there, if all peer's state and version are consistent, choose the master - int consistent = 0; + int32_t consistent = 0; if (peersStatus) { for (i = 0; i < pNode->replica; ++i) { SSyncPeer *pTemp = pNode->peerInfo[i]; @@ -721,9 +721,9 @@ static void syncRestartPeer(SSyncPeer *pPeer) { pPeer->sstatus = TAOS_SYNC_STATUS_INIT; - int ret = strcmp(pPeer->fqdn, tsNodeFqdn); + int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); } } @@ -757,7 +757,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { pthread_t thread; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - int ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer); + int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer); pthread_attr_destroy(&thattr); if (ret != 0) { @@ -802,7 +802,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { // Ensure the sync of mnode not interrupted if (pNode->vgId != 1 && tsSyncNum >= tsMaxSyncNum) { sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum); - taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -815,7 +815,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead); tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); firstPkt.port = tsSyncPort; - taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncNotStarted, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) { sError("%s, failed to send sync-req to peer", pPeer->id); @@ -836,7 +836,7 @@ static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) { if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) { // find the forwardInfo from first - for (int i = 0; i < pSyncFwds->fwds; ++i) { + for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { pFwdInfo = pSyncFwds->fwdInfo + (i + pSyncFwds->first) % tsMaxFwdInfo; if (pFwdRsp->version == pFwdInfo->version) break; } @@ -879,10 +879,10 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { } } -static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { +static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { if (pPeer->peerFd < 0) return -1; - int hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead)); + int32_t hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead)); if (hlen != sizeof(SSyncHead)) { sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen); return -1; @@ -894,7 +894,7 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return -1; } - int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); + int32_t bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); if (bytes != pHead->len) { sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len); return -1; @@ -903,7 +903,7 @@ static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return 0; } -static int syncProcessPeerMsg(void *param, void *buffer) { +static int32_t syncProcessPeerMsg(void *param, void *buffer) { SSyncPeer *pPeer = param; SSyncHead head; char * cont = buffer; @@ -911,7 +911,7 @@ static int syncProcessPeerMsg(void *param, void *buffer) { SSyncNode *pNode = pPeer->pSyncNode; pthread_mutex_lock(&(pNode->mutex)); - int code = syncReadPeerMsg(pPeer, &head, cont); + int32_t code = syncReadPeerMsg(pPeer, &head, cont); if (code == 0) { if (head.type == TAOS_SMSG_FORWARD) { @@ -948,12 +948,12 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack) { pPeersStatus->role = nodeRole; pPeersStatus->ack = ack; - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role; pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version; } - int retLen = write(pPeer->peerFd, msg, statusMsgLen); + int32_t retLen = write(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { sDebug("%s, status msg is sent, self:%s ver:%" PRIu64 ", ack:%d", pPeer->id, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack); @@ -975,10 +975,10 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { return; } - int connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); + int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (connFd < 0) { sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno)); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); return; } @@ -999,7 +999,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { } else { sDebug("try later"); close(connFd); - taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, syncTmrCtrl, &pPeer->timer); + taosTmrReset(syncCheckPeerConnection, tsSyncTimer * 1000, pPeer, tsSyncTmrCtrl, &pPeer->timer); } } @@ -1024,7 +1024,7 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); syncAddPeerRef(pPeer); - int ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer); + int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer); pthread_attr_destroy(&thattr); if (ret < 0) { @@ -1036,9 +1036,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { } } -static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { - char ipstr[24]; - int i; +static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { + char ipstr[24]; + int32_t i; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); @@ -1051,7 +1051,7 @@ static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) { } int32_t vgId = firstPkt.syncHead.vgId; - SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t)); + SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { sError("vgId:%d, vgId could not be found", vgId); taosCloseSocket(connFd); @@ -1137,8 +1137,8 @@ static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; - int fwds = pSyncFwds->fwds; - for (int i = 0; i < fwds; ++i) { + int32_t fwds = pSyncFwds->fwds; + for (int32_t i = 0; i < fwds; ++i) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first; if (pFwdInfo->confirmed == 0) break; @@ -1152,7 +1152,7 @@ static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode) { } static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) { - int confirm = 0; + int32_t confirm = 0; if (pFwdInfo->code == 0) pFwdInfo->code = code; if (code == 0) { @@ -1186,7 +1186,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { if (pSyncFwds->fwds > 0) { pthread_mutex_lock(&(pNode->mutex)); - for (int i = 0; i < pSyncFwds->fwds; ++i) { + for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; if (time - pFwdInfo->time < 2000) break; syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL); @@ -1196,23 +1196,23 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { pthread_mutex_unlock(&(pNode->mutex)); } - pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, syncTmrCtrl); + pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, (void *)pNode->rid, tsSyncTmrCtrl); } taosReleaseRef(tsSyncRefId, rid); } -static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int qtype) { +static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) { SSyncPeer *pPeer; SSyncHead *pSyncHead; SWalHead * pWalHead = data; - int fwdLen; + int32_t fwdLen; int32_t code = 0; if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pWalHead->version != nodeVersion + 1) { sError("vgId:%d, received ver:%" PRIu64 ", inconsistent with last ver:%" PRIu64 ", restart connection", pNode->vgId, pWalHead->version, nodeVersion); - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; syncRestartConnection(pPeer); } @@ -1238,7 +1238,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle pthread_mutex_lock(&(pNode->mutex)); - for (int i = 0; i < pNode->replica; ++i) { + for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; if (pPeer == NULL || pPeer->peerFd < 0) continue; if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; @@ -1248,7 +1248,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle code = 1; } - int retLen = write(pPeer->peerFd, pSyncHead, fwdLen); + int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); } else { diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 004dae1729..44aed220d7 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -48,12 +48,12 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex } } -static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { +static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info SFileAck fileAck; - int code = -1; + int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; uint32_t pindex = 0; // index in last restore bool fileChanged = false; @@ -62,7 +62,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { sinfo.index = 0; while (1) { // read file info - int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); + int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); if (ret < 0) break; // if no more file from master, break; @@ -104,7 +104,7 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { minfo.name[sizeof(minfo.name) - 1] = 0; snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name); - int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); + int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); if (dfd < 0) { sError("%s, failed to open file:%s", pPeer->id, name); break; @@ -132,9 +132,9 @@ static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { return code; } -static int syncRestoreWal(SSyncPeer *pPeer) { +static int32_t syncRestoreWal(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - int ret, code = -1; + int32_t ret, code = -1; void *buffer = calloc(1024000, 1); // size for one record if (buffer == NULL) return -1; @@ -175,10 +175,10 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { return offset; } -static int syncProcessBufferedFwd(SSyncPeer *pPeer) { +static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; - int forwards = 0; + int32_t forwards = 0; sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards); @@ -203,12 +203,12 @@ static int syncProcessBufferedFwd(SSyncPeer *pPeer) { return pRecv->code; } -int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { +int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { SSyncNode * pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; if (pRecv == NULL) return -1; - int len = pHead->len + sizeof(SWalHead); + int32_t len = pHead->len + sizeof(SWalHead); if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) { memcpy(pRecv->offset, pHead, len); @@ -231,7 +231,7 @@ static void syncCloseRecvBuffer(SSyncNode *pNode) { tfree(pNode->pRecv); } -static int syncOpenRecvBuffer(SSyncNode *pNode) { +static int32_t syncOpenRecvBuffer(SSyncNode *pNode) { syncCloseRecvBuffer(pNode); SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1); @@ -252,13 +252,13 @@ static int syncOpenRecvBuffer(SSyncNode *pNode) { return 0; } -static int syncRestoreDataStepByStep(SSyncPeer *pPeer) { +static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; sDebug("%s, start to restore file", pPeer->id); - int code = syncRestoreFile(pPeer, &fversion); + int32_t code = syncRestoreFile(pPeer, &fversion); if (code < 0) { sError("%s, failed to restore file", pPeer->id); return -1; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index d77587f227..82a4627ea5 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -27,7 +27,7 @@ #include "tsync.h" #include "syncInt.h" -static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { +static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { sDebug("%s, start to monitor:%s", pPeer->id, name); if (pPeer->notifyFd <= 0) { @@ -38,16 +38,16 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { return -1; } - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles); + if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles); if (pPeer->watchFd == NULL) { sError("%s, failed to allocate watchFd", pPeer->id); return -1; } - memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles); + memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles); } - int *wd = pPeer->watchFd + pPeer->watchNum; + int32_t *wd = pPeer->watchFd + pPeer->watchNum; if (*wd >= 0) { if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) { @@ -69,17 +69,17 @@ static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { return 0; } -static int syncAreFilesModified(SSyncPeer *pPeer) { +static int32_t syncAreFilesModified(SSyncPeer *pPeer) { if (pPeer->notifyFd <= 0) return 0; - char buf[2048]; - int len = read(pPeer->notifyFd, buf, sizeof(buf)); + char buf[2048]; + int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); if (len < 0 && errno != EAGAIN) { sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); return -1; } - int code = 0; + int32_t code = 0; if (len > 0) { const struct inotify_event *event; char *ptr; @@ -97,11 +97,11 @@ static int syncAreFilesModified(SSyncPeer *pPeer) { return code; } -static int syncRetrieveFile(SSyncPeer *pPeer) { +static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo fileInfo; SFileAck fileAck; - int code = -1; + int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; memset(&fileInfo, 0, sizeof(fileInfo)); @@ -146,7 +146,7 @@ static int syncRetrieveFile(SSyncPeer *pPeer) { } // send the file to peer - int sfd = open(name, O_RDONLY); + int32_t sfd = open(name, O_RDONLY); if (sfd < 0) break; ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); @@ -169,8 +169,8 @@ static int syncRetrieveFile(SSyncPeer *pPeer) { /* if only a partial record is read out, set the IN_MODIFY flag in event, so upper layer will reload the file to get a complete record */ -static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) { - int ret; +static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEvent) { + int32_t ret; ret = read(sfd, pHead, sizeof(SWalHead)); if (ret < 0) return -1; @@ -194,7 +194,7 @@ static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) { return sizeof(SWalHead) + pHead->len; } -static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { +static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) { pPeer->watchNum = 0; taosClose(pPeer->notifyFd); pPeer->notifyFd = inotify_init1(IN_NONBLOCK); @@ -203,14 +203,14 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { return -1; } - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int) * tsMaxWatchFiles); + if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles); if (pPeer->watchFd == NULL) { sError("%s, failed to allocate watchFd", pPeer->id); return -1; } - memset(pPeer->watchFd, -1, sizeof(int) * tsMaxWatchFiles); - int *wd = pPeer->watchFd; + memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles); + int32_t *wd = pPeer->watchFd; *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE); if (*wd == -1) { @@ -222,8 +222,8 @@ static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) { } static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { - char buf[2048]; - int len = read(pPeer->notifyFd, buf, sizeof(buf)); + char buf[2048]; + int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); if (len < 0 && errno != EAGAIN) { sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno)); return -1; @@ -243,11 +243,11 @@ static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { return 0; } -static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { +static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { SWalHead *pHead = malloc(640000); - int code = -1; + int32_t code = -1; int32_t bytes = 0; - int sfd; + int32_t sfd; sfd = open(name, O_RDONLY); if (sfd < 0) { @@ -259,7 +259,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion); while (1) { - int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); + int32_t wsize = syncReadOneWalRecord(sfd, pHead, pEvent); if (wsize < 0) break; if (wsize == 0) { code = 0; @@ -267,7 +267,7 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, } sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version); - int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); + int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); if (ret != wsize) break; pPeer->sversion = pHead->version; @@ -287,9 +287,9 @@ static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, return -1; } -static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { +static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { SSyncNode *pNode = pPeer->pSyncNode; - int code = -1; + int32_t code = -1; char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file if (syncAreFilesModified(pPeer) != 0) return -1; @@ -370,13 +370,13 @@ static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { return code; } -static int syncRetrieveWal(SSyncPeer *pPeer) { +static int32_t syncRetrieveWal(SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; char fname[TSDB_FILENAME_LEN * 3]; char wname[TSDB_FILENAME_LEN * 2]; int32_t size; struct stat fstat; - int code = -1; + int32_t code = -1; int64_t index = 0; while (1) { @@ -403,7 +403,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) { size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); - int sfd = open(fname, O_RDONLY); + int32_t sfd = open(fname, O_RDONLY); if (sfd < 0) break; code = taosSendFile(pPeer->syncFd, sfd, NULL, size); @@ -428,7 +428,7 @@ static int syncRetrieveWal(SSyncPeer *pPeer) { return code; } -static int syncRetrieveDataStepByStep(SSyncPeer *pPeer) { +static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFirstPkt firstPkt; diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 34e936c9f3..496bf07435 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -28,22 +28,22 @@ #include "syncInt.h" static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context); -static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp); +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp); static void arbProcessBrokenLink(void *param); -static int arbProcessPeerMsg(void *param, void *buffer); +static int32_t arbProcessPeerMsg(void *param, void *buffer); static tsem_t tsArbSem; static ttpool_h tsArbTcpPool; typedef struct { - char id[TSDB_EP_LEN + 24]; - int nodeFd; - void *pConn; + char id[TSDB_EP_LEN + 24]; + int32_t nodeFd; + void * pConn; } SNodeConn; -int main(int argc, char *argv[]) { +int32_t main(int32_t argc, char *argv[]) { char arbLogPath[TSDB_FILENAME_LEN + 16] = {0}; - for (int i = 1; i < argc; ++i) { + for (int32_t i = 1; i < argc; ++i) { if (strcmp(argv[i], "-p") == 0 && i < argc - 1) { tsArbitratorPort = atoi(argv[++i]); } else if (strcmp(argv[i], "-d") == 0 && i < argc - 1) { @@ -108,7 +108,7 @@ int main(int argc, char *argv[]) { return 0; } -static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp) { +static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { char ipstr[24]; tinet_ntoa(ipstr, sourceIp); sDebug("peer TCP connection from ip:%s", ipstr); @@ -150,13 +150,13 @@ static void arbProcessBrokenLink(void *param) { tfree(pNode); } -static int arbProcessPeerMsg(void *param, void *buffer) { +static int32_t arbProcessPeerMsg(void *param, void *buffer) { SNodeConn *pNode = param; SSyncHead head; - int bytes = 0; + int32_t bytes = 0; char * cont = (char *)buffer; - int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head)); + int32_t hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head)); if (hlen != sizeof(head)) { sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen); return -1; From 031ab3b73721d5b5dc3d788b2a01654c6b081505 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 10 Nov 2020 17:19:14 +0800 Subject: [PATCH 106/122] [2028] add C# connector for windows --- cmake/install.inc | 1 + src/connector/C#/TDengineDriver.cs | 154 +++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 src/connector/C#/TDengineDriver.cs diff --git a/cmake/install.inc b/cmake/install.inc index dfca758b93..5344163f23 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -13,6 +13,7 @@ ELSEIF (TD_WINDOWS) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C# DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) diff --git a/src/connector/C#/TDengineDriver.cs b/src/connector/C#/TDengineDriver.cs new file mode 100644 index 0000000000..b6f143e181 --- /dev/null +++ b/src/connector/C#/TDengineDriver.cs @@ -0,0 +1,154 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; + +namespace TDengineDriver +{ + enum TDengineDataType { + TSDB_DATA_TYPE_NULL = 0, // 1 bytes + TSDB_DATA_TYPE_BOOL = 1, // 1 bytes + TSDB_DATA_TYPE_TINYINT = 2, // 1 bytes + TSDB_DATA_TYPE_SMALLINT = 3, // 2 bytes + TSDB_DATA_TYPE_INT = 4, // 4 bytes + TSDB_DATA_TYPE_BIGINT = 5, // 8 bytes + TSDB_DATA_TYPE_FLOAT = 6, // 4 bytes + TSDB_DATA_TYPE_DOUBLE = 7, // 8 bytes + TSDB_DATA_TYPE_BINARY = 8, // string + TSDB_DATA_TYPE_TIMESTAMP = 9,// 8 bytes + TSDB_DATA_TYPE_NCHAR = 10 // unicode string + } + + enum TDengineInitOption + { + TSDB_OPTION_LOCALE = 0, + TSDB_OPTION_CHARSET = 1, + TSDB_OPTION_TIMEZONE = 2, + TDDB_OPTION_CONFIGDIR = 3, + TDDB_OPTION_SHELL_ACTIVITY_TIMER = 4 + } + + class TDengineMeta + { + public string name; + public short size; + public byte type; + public string TypeName() + { + switch ((TDengineDataType)type) + { + case TDengineDataType.TSDB_DATA_TYPE_BOOL: + return "BOOLEAN"; + case TDengineDataType.TSDB_DATA_TYPE_TINYINT: + return "BYTE"; + case TDengineDataType.TSDB_DATA_TYPE_SMALLINT: + return "SHORT"; + case TDengineDataType.TSDB_DATA_TYPE_INT: + return "INT"; + case TDengineDataType.TSDB_DATA_TYPE_BIGINT: + return "LONG"; + case TDengineDataType.TSDB_DATA_TYPE_FLOAT: + return "FLOAT"; + case TDengineDataType.TSDB_DATA_TYPE_DOUBLE: + return "DOUBLE"; + case TDengineDataType.TSDB_DATA_TYPE_BINARY: + return "STRING"; + case TDengineDataType.TSDB_DATA_TYPE_TIMESTAMP: + return "TIMESTAMP"; + case TDengineDataType.TSDB_DATA_TYPE_NCHAR: + return "NCHAR"; + default: + return "undefine"; + } + } + } + + class TDengine + { + public const int TSDB_CODE_SUCCESS = 0; + + [DllImport("taos.dll", EntryPoint = "taos_init", CallingConvention = CallingConvention.Cdecl)] + static extern public void Init(); + + [DllImport("taos.dll", EntryPoint = "taos_cleanup", CallingConvention = CallingConvention.Cdecl)] + static extern public void Cleanup(); + + [DllImport("taos.dll", EntryPoint = "taos_options", CallingConvention = CallingConvention.Cdecl)] + static extern public void Options(int option, string value); + + [DllImport("taos.dll", EntryPoint = "taos_connect", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Connect(string ip, string user, string password, string db, short port); + + [DllImport("taos.dll", EntryPoint = "taos_errstr", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_errstr(IntPtr res); + static public string Error(IntPtr res) + { + IntPtr errPtr = taos_errstr(res); + return Marshal.PtrToStringAnsi(errPtr); + } + + [DllImport("taos.dll", EntryPoint = "taos_errno", CallingConvention = CallingConvention.Cdecl)] + static extern public int ErrorNo(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_query", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr Query(IntPtr conn, string sqlstr); + + [DllImport("taos.dll", EntryPoint = "taos_affected_rows", CallingConvention = CallingConvention.Cdecl)] + static extern public int AffectRows(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_field_count", CallingConvention = CallingConvention.Cdecl)] + static extern public int FieldCount(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_fetch_fields", CallingConvention = CallingConvention.Cdecl)] + static extern private IntPtr taos_fetch_fields(IntPtr res); + static public List FetchFields(IntPtr res) + { + const int fieldSize = 68; + + List metas = new List(); + if (res == IntPtr.Zero) + { + return metas; + } + + int fieldCount = FieldCount(res); + IntPtr fieldsPtr = taos_fetch_fields(res); + + for (int i = 0; i < fieldCount; ++i) + { + int offset = i * fieldSize; + + TDengineMeta meta = new TDengineMeta(); + meta.name = Marshal.PtrToStringAnsi(fieldsPtr + offset); + meta.type = Marshal.ReadByte(fieldsPtr + offset + 65); + meta.size = Marshal.ReadInt16(fieldsPtr + offset + 66); + metas.Add(meta); + } + + return metas; + } + + [DllImport("taos.dll", EntryPoint = "taos_fetch_row", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FetchRows(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_free_result", CallingConvention = CallingConvention.Cdecl)] + static extern public IntPtr FreeResult(IntPtr res); + + [DllImport("taos.dll", EntryPoint = "taos_close", CallingConvention = CallingConvention.Cdecl)] + static extern public int Close(IntPtr taos); + } +} \ No newline at end of file From 86d78a2927528f04238bc8fd1f5c7734970cd0e1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 10 Nov 2020 09:39:50 +0000 Subject: [PATCH 107/122] TD-2044 --- src/plugins/http/inc/httpQueue.h | 4 ++- src/plugins/http/src/httpQueue.c | 22 ++++++++------ src/plugins/http/src/httpSql.c | 51 ++++++++++++++++++-------------- 3 files changed, 44 insertions(+), 33 deletions(-) diff --git a/src/plugins/http/inc/httpQueue.h b/src/plugins/http/inc/httpQueue.h index a4590719ff..1ffbd51481 100644 --- a/src/plugins/http/inc/httpQueue.h +++ b/src/plugins/http/inc/httpQueue.h @@ -22,9 +22,11 @@ extern "C" { #include +typedef void (*FHttpResultFp)(void *param, void *result, int32_t code, int32_t rows); + bool httpInitResultQueue(); void httpCleanupResultQueue(); -void httpDispatchToResultQueue(); +void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp); #ifdef __cplusplus } diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 0fb055972a..2722dba998 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -25,6 +25,7 @@ #include "httpResp.h" #include "httpAuth.h" #include "httpSession.h" +#include "httpQueue.h" typedef struct { pthread_t thread; @@ -37,26 +38,28 @@ typedef struct { } SHttpWorkerPool; typedef struct { - void *param; - void *result; - int32_t numOfRows; - void (*fp)(void *param, void *result, int32_t numOfRows); + void * param; + void * result; + int32_t code; + int32_t rows; + FHttpResultFp fp; } SHttpResult; static SHttpWorkerPool tsHttpPool; static taos_qset tsHttpQset; static taos_queue tsHttpQueue; -void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t numOfRows, void (*fp)(void *param, void *result, int32_t numOfRows)) { +void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int32_t rows, FHttpResultFp fp) { if (tsHttpQueue != NULL) { SHttpResult *pMsg = taosAllocateQitem(sizeof(SHttpResult)); pMsg->param = param; pMsg->result = result; - pMsg->numOfRows = numOfRows; + pMsg->code = code; + pMsg->rows = rows; pMsg->fp = fp; taosWriteQitem(tsHttpQueue, TAOS_QTYPE_RPC, pMsg); } else { - (*fp)(param, result, numOfRows); + (*fp)(param, result, code, rows); } } @@ -71,8 +74,9 @@ static void *httpProcessResultQueue(void *param) { break; } - httpTrace("context:%p, res:%p will be processed in result queue", pMsg->param, pMsg->result); - (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->numOfRows); + httpTrace("context:%p, res:%p will be processed in result queue, code:%d rows:%d", pMsg->param, pMsg->result, + pMsg->code, pMsg->rows); + (*pMsg->fp)(pMsg->param, pMsg->result, pMsg->code, pMsg->rows); taosFreeQitem(pMsg); } diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 70d644146c..20e52ff40a 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -31,7 +31,7 @@ void httpProcessMultiSql(HttpContext *pContext); void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); -void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) { +void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int code, int numOfRows) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -43,7 +43,7 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n bool isContinue = false; - if (numOfRows > 0) { + if (code == TSDB_CODE_SUCCESS && numOfRows > 0) { if (singleCmd->cmdReturnType == HTTP_CMD_RETURN_TYPE_WITH_RETURN && encode->buildQueryJsonFp) { isContinue = (encode->buildQueryJsonFp)(pContext, singleCmd, result, numOfRows); } @@ -58,9 +58,9 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n httpDebug("context:%p, fd:%d, user:%s, process pos:%d, stop retrieve, numOfRows:%d, sql:%s", pContext, pContext->fd, pContext->user, multiCmds->pos, numOfRows, sql); - if (numOfRows < 0) { + if (code < 0) { httpError("context:%p, fd:%d, user:%s, process pos:%d, retrieve failed code:%s, sql:%s", pContext, pContext->fd, - pContext->user, multiCmds->pos, tstrerror(numOfRows), sql); + pContext->user, multiCmds->pos, tstrerror(code), sql); } taos_free_result(result); @@ -74,14 +74,14 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int n } void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { - httpDispatchToResultQueue(param, result, numOfRows, httpProcessMultiSqlRetrieveCallBackImp); + int32_t code = taos_errno(result); + httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessMultiSqlRetrieveCallBackImp); } -void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { +void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code, int affectRowsInput) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; - code = taos_errno(result); HttpSqlCmds *multiCmds = pContext->multiCmds; HttpEncodeMethod *encode = pContext->encodeMethod; @@ -94,7 +94,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { return; } - if (code < 0) { + if (code != TSDB_CODE_SUCCESS) { if (encode->checkFinishedFp != NULL && !encode->checkFinishedFp(pContext, singleCmd, code)) { singleCmd->code = code; httpDebug("context:%p, fd:%d, user:%s, process pos jump to:%d, last code:%s, last sql:%s", pContext, pContext->fd, @@ -157,7 +157,9 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code) { } void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { - httpDispatchToResultQueue(param, result, unUsedCode, httpProcessMultiSqlCallBackImp); + int32_t code = taos_errno(result); + int32_t affectRows = taos_affected_rows(result); + httpDispatchToResultQueue(param, result, code, affectRows, httpProcessMultiSqlCallBackImp); } void httpProcessMultiSql(HttpContext *pContext) { @@ -204,7 +206,7 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) { void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); -void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int numOfRows) { +void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int code, int numOfRows) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -212,7 +214,7 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int bool isContinue = false; - if (numOfRows > 0) { + if (code == TSDB_CODE_SUCCESS && numOfRows > 0) { if (encode->buildQueryJsonFp) { isContinue = (encode->buildQueryJsonFp)(pContext, &pContext->singleCmd, result, numOfRows); } @@ -227,9 +229,9 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int httpDebug("context:%p, fd:%d, user:%s, stop retrieve, numOfRows:%d", pContext, pContext->fd, pContext->user, numOfRows); - if (numOfRows < 0) { + if (code < 0) { httpError("context:%p, fd:%d, user:%s, retrieve failed, code:%s", pContext, pContext->fd, pContext->user, - tstrerror(numOfRows)); + tstrerror(code)); } taos_free_result(result); @@ -243,29 +245,29 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int } void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { - httpDispatchToResultQueue(param, result, numOfRows, httpProcessSingleSqlRetrieveCallBackImp); + int32_t code = taos_errno(result); + httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessSingleSqlRetrieveCallBackImp); } -void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCode) { +void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int code, int affectRowsInput) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; - int32_t code = taos_errno(result); - HttpEncodeMethod *encode = pContext->encodeMethod; if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { httpError("context:%p, fd:%d, user:%s, query error, code:%s:inprogress, sqlObj:%p", pContext, pContext->fd, - pContext->user, tstrerror(code), (SSqlObj *)result); + pContext->user, tstrerror(code), result); return; } - if (code < 0) { + if (code != TSDB_CODE_SUCCESS) { SSqlObj *pObj = (SSqlObj *)result; if (code == TSDB_CODE_TSC_INVALID_SQL) { - httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, - pContext->fd, pContext->user, tstrerror(code), pObj, pObj->cmd.payload); - httpSendTaosdInvalidSqlErrorResp(pContext, pObj->cmd.payload); + terrno = code; + httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p, error:%s", pContext, pContext->fd, + pContext->user, tstrerror(code), pObj, taos_errstr(pObj)); + httpSendTaosdInvalidSqlErrorResp(pContext, taos_errstr(pObj)); } else { httpError("context:%p, fd:%d, user:%s, query error, code:%s, sqlObj:%p", pContext, pContext->fd, pContext->user, tstrerror(code), pObj); @@ -279,6 +281,7 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo if (isUpdate) { // not select or show commands int affectRows = taos_affected_rows(result); + assert(affectRows == affectRowsInput); httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd, pContext->user, affectRows, result); @@ -309,7 +312,9 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int unUsedCo } void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { - httpDispatchToResultQueue(param, result, unUsedCode, httpProcessSingleSqlCallBackImp); + int32_t code = taos_errno(result); + int32_t affectRows = taos_affected_rows(result); + httpDispatchToResultQueue(param, result, code, affectRows, httpProcessSingleSqlCallBackImp); } void httpProcessSingleSqlCmd(HttpContext *pContext) { From caa7f06d842da3504f61d864d75ad37e56e5206a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 10 Nov 2020 09:51:46 +0000 Subject: [PATCH 108/122] change int to int32_t --- src/plugins/http/src/httpQueue.c | 6 +++--- src/plugins/http/src/httpSql.c | 26 +++++++++++++------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/plugins/http/src/httpQueue.c b/src/plugins/http/src/httpQueue.c index 2722dba998..1c039abb4d 100644 --- a/src/plugins/http/src/httpQueue.c +++ b/src/plugins/http/src/httpQueue.c @@ -65,9 +65,9 @@ void httpDispatchToResultQueue(void *param, TAOS_RES *result, int32_t code, int3 static void *httpProcessResultQueue(void *param) { SHttpResult *pMsg; - int32_t type; - void *unUsed; - + int32_t type; + void * unUsed; + while (1) { if (taosReadQitemFromQset(tsHttpQset, &type, (void **)&pMsg, &unUsed) == 0) { httpDebug("qset:%p, http queue got no message from qset, exiting", tsHttpQset); diff --git a/src/plugins/http/src/httpSql.c b/src/plugins/http/src/httpSql.c index 20e52ff40a..564f555c40 100644 --- a/src/plugins/http/src/httpSql.c +++ b/src/plugins/http/src/httpSql.c @@ -29,9 +29,9 @@ void httpProcessMultiSql(HttpContext *pContext); -void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); +void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows); -void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int code, int numOfRows) { +void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -73,12 +73,12 @@ void httpProcessMultiSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int c } } -void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { +void httpProcessMultiSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) { int32_t code = taos_errno(result); httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessMultiSqlRetrieveCallBackImp); } -void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code, int affectRowsInput) { +void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -119,7 +119,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code, int bool isUpdate = tscIsUpdateQuery(result); if (isUpdate) { // not select or show commands - int affectRows = taos_affected_rows(result); + int32_t affectRows = taos_affected_rows(result); httpDebug("context:%p, fd:%d, user:%s, process pos:%d, affect rows:%d, sql:%s", pContext, pContext->fd, pContext->user, multiCmds->pos, affectRows, sql); @@ -156,7 +156,7 @@ void httpProcessMultiSqlCallBackImp(void *param, TAOS_RES *result, int code, int } } -void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { +void httpProcessMultiSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) { int32_t code = taos_errno(result); int32_t affectRows = taos_affected_rows(result); httpDispatchToResultQueue(param, result, code, affectRows, httpProcessMultiSqlCallBackImp); @@ -204,9 +204,9 @@ void httpProcessMultiSqlCmd(HttpContext *pContext) { httpProcessMultiSql(pContext); } -void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows); +void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows); -void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int code, int numOfRows) { +void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t numOfRows) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -244,12 +244,12 @@ void httpProcessSingleSqlRetrieveCallBackImp(void *param, TAOS_RES *result, int } } -void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int numOfRows) { +void httpProcessSingleSqlRetrieveCallBack(void *param, TAOS_RES *result, int32_t numOfRows) { int32_t code = taos_errno(result); httpDispatchToResultQueue(param, result, code, numOfRows, httpProcessSingleSqlRetrieveCallBackImp); } -void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int code, int affectRowsInput) { +void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int32_t code, int32_t affectRowsInput) { HttpContext *pContext = (HttpContext *)param; if (pContext == NULL) return; @@ -280,7 +280,7 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int code, in bool isUpdate = tscIsUpdateQuery(result); if (isUpdate) { // not select or show commands - int affectRows = taos_affected_rows(result); + int32_t affectRows = taos_affected_rows(result); assert(affectRows == affectRowsInput); httpDebug("context:%p, fd:%d, user:%s, affect rows:%d, stop query, sqlObj:%p", pContext, pContext->fd, @@ -311,7 +311,7 @@ void httpProcessSingleSqlCallBackImp(void *param, TAOS_RES *result, int code, in } } -void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int unUsedCode) { +void httpProcessSingleSqlCallBack(void *param, TAOS_RES *result, int32_t unUsedCode) { int32_t code = taos_errno(result); int32_t affectRows = taos_affected_rows(result); httpDispatchToResultQueue(param, result, code, affectRows, httpProcessSingleSqlCallBackImp); @@ -378,7 +378,7 @@ void httpExecCmd(HttpContext *pContext) { } } -void httpProcessRequestCb(void *param, TAOS_RES *result, int code) { +void httpProcessRequestCb(void *param, TAOS_RES *result, int32_t code) { HttpContext *pContext = param; taos_free_result(result); From cc74b689fc76d8dae9c3e82ef702fc10baf8209a Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 10 Nov 2020 13:00:31 +0000 Subject: [PATCH 109/122] add some notes on tqueue.c --- src/util/inc/tqueue.h | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/util/inc/tqueue.h b/src/util/inc/tqueue.h index 8493a64315..c3051464e5 100644 --- a/src/util/inc/tqueue.h +++ b/src/util/inc/tqueue.h @@ -20,6 +20,23 @@ extern "C" { #endif +/* + +This set of API for queue is designed specially for vnode/mnode. The main purpose is to +consume all the items instead of one item from a queue by one single read. Also, it can +combine multiple queues into a queue set, a consumer thread can consume a queue set via +a single API instead of looping every queue by itself. + +Notes: +1: taosOpenQueue/taosCloseQueue, taosOpenQset/taosCloseQset is NOT multi-thread safe +2: after taosCloseQueue/taosCloseQset is called, read/write operation APIs are not safe. +3: read/write operation APIs are multi-thread safe + +To remove the limitation and make this set of queue APIs multi-thread safe, REF(tref.c) +shall be used to set up the protection. + +*/ + typedef void* taos_queue; typedef void* taos_qset; typedef void* taos_qall; From c57777c4a12d65bb0b7c3e6bd2257ed4cf6c21c9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 10 Nov 2020 22:55:38 +0800 Subject: [PATCH 110/122] TD-2041 --- src/inc/taosdef.h | 2 + src/inc/taoserror.h | 3 +- src/sync/inc/syncInt.h | 2 + src/sync/src/syncMain.c | 8 +- src/sync/src/syncRetrieve.c | 2 +- src/sync/src/tarbitrator.c | 2 +- src/vnode/src/vnodeWrite.c | 5 ++ src/wal/inc/walInt.h | 2 +- tests/script/general/http/sync.sim | 124 +++++++++++++++++++++++++++++ tests/script/jenkins/basic.txt | 1 + 10 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 tests/script/general/http/sync.sim diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index fa8af2c67e..875092b88d 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -431,6 +431,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_PORT_HTTP 11 #define TSDB_PORT_ARBITRATOR 12 +#define TSDB_MAX_WAL_SIZE (1024*1024) + typedef enum { TAOS_QTYPE_RPC = 0, TAOS_QTYPE_FWD = 1, diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 01891234ef..1919747a0b 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -237,7 +237,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") -TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") @@ -261,6 +261,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sy // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, 0, 0x1001, "WAL file is corrupted") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, 0, 0x1002, "WAL size exceeds limit") // http TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_SERVER_OFFLINE, 0, 0x1100, "http server is not onlin") diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 240b401bda..93c6efc20a 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -35,6 +35,8 @@ extern "C" { #define TAOS_SMSG_SYNC_MUST 6 #define TAOS_SMSG_STATUS 7 +#define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) + #define nodeRole pNode->peerInfo[pNode->selfIndex]->role #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version #define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 9dcd0fd632..a68ebdec2a 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -79,7 +79,7 @@ int32_t syncInit() { info.numOfThreads = tsSyncTcpThreads; info.serverIp = 0; info.port = tsSyncPort; - info.bufferSize = 640000; + info.bufferSize = SYNC_MAX_SIZE; info.processBrokenLink = syncProcessBrokenLink; info.processIncomingMsg = syncProcessPeerMsg; info.processIncomingConn = syncProcessIncommingConnection; @@ -850,7 +850,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)cont; - sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version); + sDebug("%s, forward is received, hver:%" PRIu64 ", len:%d", pPeer->id, pHead->version, pHead->len); if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; @@ -859,7 +859,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); } else { - sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version); + sError("%s, forward discarded, hver:%" PRIu64, pPeer->id, pHead->version); } } } @@ -890,7 +890,7 @@ static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { // head.len = htonl(head.len); if (pHead->len < 0) { - sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len); + sError("%s, invalid pkt length, hlen:%d", pPeer->id, pHead->len); return -1; } diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 82a4627ea5..968f5becad 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -244,7 +244,7 @@ static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { } static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { - SWalHead *pHead = malloc(640000); + SWalHead *pHead = malloc(SYNC_MAX_SIZE); int32_t code = -1; int32_t bytes = 0; int32_t sfd; diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 496bf07435..ae8f2ea518 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -86,7 +86,7 @@ int32_t main(int32_t argc, char *argv[]) { info.numOfThreads = 1; info.serverIp = 0; info.port = tsArbitratorPort; - info.bufferSize = 640000; + info.bufferSize = SYNC_MAX_SIZE; info.processBrokenLink = arbProcessBrokenLink; info.processIncomingMsg = arbProcessPeerMsg; info.processIncomingConn = arbProcessIncommingConnection; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 99594607ac..e67c544fb2 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -217,6 +217,11 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar if (code != TSDB_CODE_SUCCESS) return code; } + if (pHead->len > TSDB_MAX_WAL_SIZE) { + vError("vgId:%d, wal len:%d exceeds limit, hver:%" PRIu64, pVnode->vgId, pHead->len, pHead->version); + return TSDB_CODE_WAL_SIZE_LIMIT; + } + int32_t size = sizeof(SVWriteMsg) + sizeof(SWalHead) + pHead->len; SVWriteMsg *pWrite = taosAllocateQitem(size); if (pWrite == NULL) { diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h index d1e9772259..36311c8f5d 100644 --- a/src/wal/inc/walInt.h +++ b/src/wal/inc/walInt.h @@ -34,7 +34,7 @@ extern int32_t wDebugFlag; #define WAL_PREFIX "wal" #define WAL_PREFIX_LEN 3 #define WAL_REFRESH_MS 1000 -#define WAL_MAX_SIZE (1024 * 1024) +#define WAL_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + 16) #define WAL_SIGNATURE ((uint32_t)(0xFAFBFDFE)) #define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12) #define WAL_FILE_LEN (TSDB_FILENAME_LEN + 32) diff --git a/tests/script/general/http/sync.sim b/tests/script/general/http/sync.sim new file mode 100644 index 0000000000..abaf22f919 --- /dev/null +++ b/tests/script/general/http/sync.sim @@ -0,0 +1,124 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c http -v 1 +system sh/cfg.sh -n dnode2 -c http -v 1 +system sh/cfg.sh -n dnode3 -c http -v 1 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 + +system sh/cfg.sh -n dnode1 -c replica -v 3 +system sh/cfg.sh -n dnode2 -c replica -v 3 +system sh/cfg.sh -n dnode3 -c replica -v 3 + +system sh/cfg.sh -n dnode1 -c maxSQLLength -v 940032 +system sh/cfg.sh -n dnode2 -c maxSQLLength -v 940032 +system sh/cfg.sh -n dnode3 -c maxSQLLength -v 940032 + +print ============== deploy + +system sh/exec.sh -n dnode1 -s start +sleep 5001 +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +print =============== step1 +$x = 0 +show1: + $x = $x + 1 + sleep 2000 + if $x == 5 then + return -1 + endi +sql show mnodes -x show1 +$mnode1Role = $data2_1 +print mnode1Role $mnode1Role +$mnode2Role = $data2_2 +print mnode2Role $mnode2Role +$mnode3Role = $data2_3 +print mnode3Role $mnode3Role + +if $mnode1Role != master then + goto show1 +endi +if $mnode2Role != slave then + goto show1 +endi +if $mnode3Role != slave then + goto show1 +endi + +print =============== step2 +sql create database d1 replica 3 +sql use d1 + +sql create table table_rest (ts timestamp, i int) +print sql length is 870KB +restful d1 table_rest 1591072800 30000 +restful d1 table_rest 1591172800 30000 +restful d1 table_rest 1591272800 30000 +restful d1 table_rest 1591372800 30000 +restful d1 table_rest 1591472800 30000 +restful d1 table_rest 1591572800 30000 +restful d1 table_rest 1591672800 30000 +restful d1 table_rest 1591772800 30000 +restful d1 table_rest 1591872800 30000 +restful d1 table_rest 1591972800 30000 + +sql select * from table_rest; +print rows: $rows +if $rows != 300000 then + return -1 +endi + +print =============== step3 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 5000 +sql select * from table_rest; +print rows: $rows +if $rows != 300000 then + return -1 +endi +system sh/exec.sh -n dnode1 -s start -x SIGINT +sleep 5000 + +print =============== step4 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +sleep 5000 +sql select * from table_rest; +print rows: $rows +if $rows != 300000 then + return -1 +endi +system sh/exec.sh -n dnode2 -s start -x SIGINT +sleep 5000 + +print =============== step5 +system sh/exec.sh -n dnode3 -s stop -x SIGINT +sleep 5000 +sql select * from table_rest; +print rows: $rows +if $rows != 300000 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 1b2fe37c71..b5462ba261 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -89,6 +89,7 @@ cd ../../../debug; make ./test.sh -f general/http/telegraf.sim ./test.sh -f general/http/grafana_bug.sim ./test.sh -f general/http/grafana.sim +./test.sh -f general/http/sync.sim ./test.sh -f general/import/basic.sim ./test.sh -f general/import/commit.sim From d5d1aa0ac249c0b49043b15b3561b080c92753d1 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 11 Nov 2020 10:51:45 +0800 Subject: [PATCH 111/122] [TD-1889]add case for td-1875 --- tests/pytest/query/bug1875.py | 60 +++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 tests/pytest/query/bug1875.py diff --git a/tests/pytest/query/bug1875.py b/tests/pytest/query/bug1875.py new file mode 100644 index 0000000000..12e22a7a25 --- /dev/null +++ b/tests/pytest/query/bug1875.py @@ -0,0 +1,60 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + stable=0 + insertRows = 1000 + tbnum = 3 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(tbnum): + tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + stable=stable+1 + for i in range(tbnum): + tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + print("==========step2") + print("join query ") + tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc limit 3;") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3;") + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 860d3cf8070ba1c10773804f82c7f1ec80056c42 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 11 Nov 2020 10:53:00 +0800 Subject: [PATCH 112/122] [TD-1888]add test case for td-1874 --- tests/pytest/query/bug1874.py | 59 +++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/pytest/query/bug1874.py diff --git a/tests/pytest/query/bug1874.py b/tests/pytest/query/bug1874.py new file mode 100644 index 0000000000..7177484870 --- /dev/null +++ b/tests/pytest/query/bug1874.py @@ -0,0 +1,59 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table join_mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + tdSql.execute("create table join_mt1 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) TAGS(t1 int, t2 binary(12))") + stable=0 + insertRows = 1000 + tbnum = 3 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(tbnum): + tdSql.execute("create table join_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + stable=stable+1 + for i in range(tbnum): + tdSql.execute("create table join_1_tb%d using join_mt%d tags(%d,'abc')" %(i,stable,i)) + for j in range(insertRows): + ret = tdSql.execute( + "insert into join_tb%d values (%d , %d,%d,%d,%d,%d,%d,%d, '%s','%s')" % + (i,t0+i,i%100,i%100,i%100,i%100,i%100,i%100,i%100,'binary'+str(i%100),'nchar'+str(i%100))) + print("==========step2") + print("join query ") + tdLog.info("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From d4ff94a2bd715f0a0ba3c82fb21a1c795e83db2e Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 11 Nov 2020 13:43:53 +0800 Subject: [PATCH 113/122] [TECO-3] : add hivemq extension documents. --- .../webdocs/markdowndocs/Documentation-ch.md | 3 ++- .../webdocs/markdowndocs/Evaluation-ch.md | 2 +- documentation20/webdocs/markdowndocs/insert-ch.md | 14 +++++++++++--- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md index 077a043138..370197901c 100644 --- a/documentation20/webdocs/markdowndocs/Documentation-ch.md +++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md @@ -34,7 +34,8 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 - [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入)：使用SQL insert命令向一张或多张表写入单条或多条记录 - [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入)：配置Telegraf, 不用任何代码，将采集数据直接写入 - [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入)：配置Prometheus, 不用任何代码，将数据直接写入 -- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入)：配置EMQ X，不用任何代码，就可将MQTT数据直接写入 +- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker)：配置EMQ X，不用任何代码，就可将 MQTT 数据直接写入 +- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker)：通过 HiveMQ Extension，不用任何代码，就可将 MQTT 数据直接写入 ## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries) diff --git a/documentation20/webdocs/markdowndocs/Evaluation-ch.md b/documentation20/webdocs/markdowndocs/Evaluation-ch.md index 9e7e0ec6aa..a92f97a8d9 100644 --- a/documentation20/webdocs/markdowndocs/Evaluation-ch.md +++ b/documentation20/webdocs/markdowndocs/Evaluation-ch.md @@ -10,7 +10,7 @@ TDengine的模块之一是时序数据库。但除此之外，为减少研发的 * __硬件或云服务成本降至1/5__：由于超强性能，计算资源不到通用大数据方案的1/5；通过列式存储和先进的压缩算法，存储空间不到通用数据库的1/10。 * __全栈时序数据处理引擎__：将数据库、消息队列、缓存、流式计算等功能融为一体，应用无需再集成Kafka/Redis/HBase/Spark/HDFS等软件，大幅降低应用开发和维护的复杂度成本。 * __强大的分析功能__：无论是十年前还是一秒钟前的数据，指定时间范围即可查询。数据可在时间轴上或多个设备上进行聚合。即席查询可通过Shell, Python, R, Matlab随时进行。 -* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。 +* __与第三方工具无缝连接__：不用一行代码，即可与Telegraf, Grafana, EMQ, HiveMQ, Prometheus, Matlab, R等集成。后续将支持OPC, Hadoop, Spark等, BI工具也将无缝连接。 * __零运维成本、零学习成本__：安装集群简单快捷，无需分库分表，实时备份。类似标准SQL，支持RESTful, 支持Python/Java/C/C++/C#/Go/Node.js, 与MySQL相似，零学习成本。 采用TDengine，可将典型的物联网、车联网、工业互联网大数据平台的总拥有成本大幅降低。但需要指出的是，因充分利用了物联网时序数据的特点，它无法用来处理网络爬虫、微博、微信、电商、ERP、CRM等通用型数据。 diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md index fa53cbd62b..7b3f537990 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/webdocs/markdowndocs/insert-ch.md @@ -1,6 +1,6 @@ # 高效写入数据 -TDengine支持多种接口写入数据，包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, CSV文件等，后续还将提供Kafka, OPC等接口。数据可以单条插入，也可以批量插入，可以插入一个数据采集点的数据，也可以同时插入多个数据采集点的数据。支持多线程插入，支持时间乱序数据插入，也支持历史数据插入。 +TDengine支持多种接口写入数据，包括SQL, Prometheus, Telegraf, EMQ MQTT Broker, HiveMQ Broker, CSV文件等，后续还将提供Kafka, OPC等接口。数据可以单条插入，也可以批量插入，可以插入一个数据采集点的数据，也可以同时插入多个数据采集点的数据。支持多线程插入，支持时间乱序数据插入，也支持历史数据插入。 ## SQL写入 @@ -218,7 +218,15 @@ use telegraf; select * from cpu; ``` -## EMQ X Broker直接写入 +## MQTT Broker直接写入 -MQTT是一流行的物联网数据传输协议，[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDEngine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 +MQTT是一流行的物联网数据传输协议，TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。 + +### EMQ Broker + +[EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 + +### HiveMQ Broker + +[HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。 From 2c922ffca96a424dae88c4013c8192447d940e72 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 11 Nov 2020 14:02:41 +0800 Subject: [PATCH 114/122] make mqtt broker and hivemq extension same level with other methods. --- documentation20/webdocs/markdowndocs/Documentation-ch.md | 4 ++-- documentation20/webdocs/markdowndocs/insert-ch.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Documentation-ch.md b/documentation20/webdocs/markdowndocs/Documentation-ch.md index 370197901c..f1f2d58f05 100644 --- a/documentation20/webdocs/markdowndocs/Documentation-ch.md +++ b/documentation20/webdocs/markdowndocs/Documentation-ch.md @@ -34,8 +34,8 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台，专 - [SQL写入](https://www.taosdata.com/cn/documentation20/insert/#SQL写入)：使用SQL insert命令向一张或多张表写入单条或多条记录 - [Telegraf写入](https://www.taosdata.com/cn/documentation20/insert/#Telegraf直接写入)：配置Telegraf, 不用任何代码，将采集数据直接写入 - [Prometheus写入](https://www.taosdata.com/cn/documentation20/insert/#Prometheus直接写入)：配置Prometheus, 不用任何代码，将数据直接写入 -- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker)：配置EMQ X，不用任何代码，就可将 MQTT 数据直接写入 -- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker)：通过 HiveMQ Extension，不用任何代码，就可将 MQTT 数据直接写入 +- [EMQ X Broker](https://www.taosdata.com/cn/documentation20/insert/#EMQ-X-Broker直接写入)：配置EMQ X，不用任何代码，就可将 MQTT 数据直接写入 +- [HiveMQ Broker](https://www.taosdata.com/cn/documentation20/insert/#HiveMQ-Broker直接写入)：通过 HiveMQ Extension，不用任何代码，就可将 MQTT 数据直接写入 ## [高效查询数据](https://www.taosdata.com/cn/documentation20/queries) diff --git a/documentation20/webdocs/markdowndocs/insert-ch.md b/documentation20/webdocs/markdowndocs/insert-ch.md index 7b3f537990..77ba596d4e 100644 --- a/documentation20/webdocs/markdowndocs/insert-ch.md +++ b/documentation20/webdocs/markdowndocs/insert-ch.md @@ -218,15 +218,15 @@ use telegraf; select * from cpu; ``` -## MQTT Broker直接写入 + MQTT是一流行的物联网数据传输协议，TDengine 可以很方便的接入 MQTT Broker 接受的数据并写入到 TDengine。 -### EMQ Broker +## EMQ Broker 直接写入 [EMQ](https://github.com/emqx/emqx)是一开源的MQTT Broker软件，无需任何代码，只需要在EMQ Dashboard里使用“规则”做简单配置，即可将MQTT的数据直接写入TDengine。EMQ X 支持通过 发送到 Web 服务 的方式保存数据到 TDengine，也在企业版上提供原生的 TDEngine 驱动实现直接保存。详细使用方法请参考 [EMQ 官方文档](https://docs.emqx.io/broker/latest/cn/rule/rule-example.html#%E4%BF%9D%E5%AD%98%E6%95%B0%E6%8D%AE%E5%88%B0-tdengine)。 -### HiveMQ Broker +## HiveMQ Broker 直接写入 [HiveMQ](https://www.hivemq.com/) 是一个提供免费个人版和企业版的 MQTT 代理，主要用于企业和新兴的机器到机器M2M通讯和内部传输，满足可伸缩性、易管理和安全特性。HiveMQ 提供了开源的插件开发包。可以通过 HiveMQ extension - TDengine 保存数据到 TDengine。详细使用方法请参考 [HiveMQ extension - TDengine 说明文档](https://github.com/huskar-t/hivemq-tdengine-extension/blob/b62a26ecc164a310104df57691691b237e091c89/README.md)。 From 23fba6341362be287520307e65e67ad402e79087 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 06:28:21 +0000 Subject: [PATCH 115/122] TD-2041 --- src/balance/src/balance.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 4c687cb134..6d4f08eb06 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -586,7 +586,7 @@ static void balanceProcessBalanceTimer(void *handle, void *tmrId) { if (!sdbIsMaster()) return; tsBalanceTimer = NULL; - tsAccessSquence ++; + if (handle == NULL) tsAccessSquence++; balanceCheckDnodeAccess(); bool updateSoon = false; From c7285a96f041dfde2967dacf1e8e83a013fc299f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 11 Nov 2020 06:29:33 +0000 Subject: [PATCH 116/122] implement TD-2061 --- src/util/src/tskiplist.c | 68 ++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 45e3308bc8..5ecc8ce119 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -24,8 +24,8 @@ static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, in static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); -static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData); +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode); +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData); static SSkipListNode * tSkipListNewNode(uint8_t level); #define tSkipListFreeNode(n) tfree((n)) @@ -108,17 +108,17 @@ void tSkipListDestroy(SSkipList *pSkipList) { SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { if (pSkipList == NULL || pData == NULL) return NULL; - SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; + SSkipListNode *backward[MAX_SKIP_LIST_LEVEL] = {0}; uint8_t dupMode = SL_DUP_MODE(pSkipList); SSkipListNode *pNode = NULL; tSkipListWLock(pSkipList); - bool hasDup = tSkipListGetPosToPut(pSkipList, forward, pData); + bool hasDup = tSkipListGetPosToPut(pSkipList, backward, pData); if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) { if (dupMode == SL_UPDATE_DUP_KEY) { - pNode = SL_NODE_GET_FORWARD_POINTER(forward[0], 0); + pNode = SL_NODE_GET_BACKWARD_POINTER(backward[0], 0); atomic_store_ptr(&(pNode->pData), pData); } } else { @@ -126,7 +126,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { if (pNode != NULL) { pNode->pData = pData; - tSkipListDoInsert(pSkipList, forward, pNode); + tSkipListDoInsert(pSkipList, backward, pNode); } } @@ -310,7 +310,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { } } -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **backward, SSkipListNode *pNode) { for (int32_t i = 0; i < pNode->level; ++i) { if (i >= pSkipList->level) { SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; @@ -318,14 +318,14 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSk SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; } else { - SSkipListNode *x = forward[i]; - SL_NODE_GET_BACKWARD_POINTER(pNode, i) = x; + SSkipListNode *x = backward[i]; + SL_NODE_GET_FORWARD_POINTER(pNode, i) = x; - SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(x, i); - SL_NODE_GET_BACKWARD_POINTER(next, i) = pNode; + SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(x, i); + SL_NODE_GET_FORWARD_POINTER(prev, i) = pNode; - SL_NODE_GET_FORWARD_POINTER(pNode, i) = next; - SL_NODE_GET_FORWARD_POINTER(x, i) = pNode; + SL_NODE_GET_BACKWARD_POINTER(x, i) = pNode; + SL_NODE_GET_BACKWARD_POINTER(pNode, i) = prev; } } @@ -371,57 +371,57 @@ static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { return 0; } -static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData) { +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **backward, void *pData) { int compare = 0; bool hasDupKey = false; char * pDataKey = pSkipList->keyFn(pData); if (pSkipList->size == 0) { for (int i = 0; i < pSkipList->level; i++) { - forward[i] = pSkipList->pHead; + backward[i] = pSkipList->pTail; } } else { char *pKey = NULL; - // Compare min key - pKey = SL_GET_MIN_KEY(pSkipList); - compare = pSkipList->comparFn(pDataKey, pKey); - if (compare <= 0) { - for (int i = 0; i < pSkipList->level; i++) { - forward[i] = pSkipList->pHead; - } - - return (compare == 0); - } - // Compare max key pKey = SL_GET_MAX_KEY(pSkipList); compare = pSkipList->comparFn(pDataKey, pKey); - if (compare > 0) { + if (compare >= 0) { for (int i = 0; i < pSkipList->level; i++) { - forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i); + backward[i] = pSkipList->pTail; } return (compare == 0); } - SSkipListNode *px = pSkipList->pHead; + // Compare min key + pKey = SL_GET_MIN_KEY(pSkipList); + compare = pSkipList->comparFn(pDataKey, pKey); + if (compare < 0) { + for (int i = 0; i < pSkipList->level; i++) { + backward[i] = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i); + } + + return (compare == 0); + } + + SSkipListNode *px = pSkipList->pTail; for (int i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(px, i); - while (p != pSkipList->pTail) { + SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(px, i); + while (p != pSkipList->pHead) { pKey = SL_GET_NODE_KEY(pSkipList, p); compare = pSkipList->comparFn(pKey, pDataKey); - if (compare >= 0) { + if (compare <= 0) { if (compare == 0 && !hasDupKey) hasDupKey = true; break; } else { px = p; - p = SL_NODE_GET_FORWARD_POINTER(px, i); + p = SL_NODE_GET_BACKWARD_POINTER(px, i); } } - forward[i] = px; + backward[i] = px; } } From 9730ad8eba05b20dfa21ad084e9bac4aa2bdf79d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 14:30:12 +0800 Subject: [PATCH 117/122] TD-2041 --- src/sync/src/syncMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index a68ebdec2a..ef1ada4c2e 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -894,6 +894,7 @@ static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) { return -1; } + assert(pHead->len <= TSDB_MAX_WAL_SIZE); int32_t bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len); if (bytes != pHead->len) { sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len); From 86625df80a7d613d7ba187cc44d7389a9547c514 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Wed, 11 Nov 2020 15:26:59 +0800 Subject: [PATCH 118/122] [none] --- cmake/install.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/install.inc b/cmake/install.inc index 5344163f23..9bbcc2cf40 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -13,7 +13,7 @@ ELSEIF (TD_WINDOWS) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/go DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/nodejs DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/python DESTINATION connector) - INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C# DESTINATION connector) + INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/src/connector/C\# DESTINATION connector) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/tests/examples DESTINATION .) INSTALL(DIRECTORY ${TD_COMMUNITY_DIR}/packaging/cfg DESTINATION .) INSTALL(FILES ${TD_COMMUNITY_DIR}/src/inc/taos.h DESTINATION include) From 13ed70f61f4367b33d23849015dec2c7b186b70b Mon Sep 17 00:00:00 2001 From: huili <52318143+plum-lihui@users.noreply.github.com> Date: Wed, 11 Nov 2020 16:35:42 +0800 Subject: [PATCH 119/122] Update architecture-ch.md --- documentation20/webdocs/markdowndocs/architecture-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index d4705ccb05..3721d3c2cd 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -221,7 +221,7 @@ TDengine采用时间驱动缓存管理策略（First-In-First-Out，FIFO）， TDengine通过查询函数向用户提供毫秒级的数据获取能力。直接将最近到达的数据保存在缓存中，可以更加快速地响应用户针对最近一条或一批数据的查询分析，整体上提供更快的数据库查询响应能力。从这个意义上来说，**可通过设置合适的配置参数将TDengine作为数据缓存来使用，而不需要再部署Redis或其他额外的缓存系统**，可有效地简化系统架构，降低运维的成本。需要注意的是，TDengine重启以后系统的缓存将被清空，之前缓存的数据均会被批量写入磁盘，缓存的数据将不会像专门的Key-value缓存系统再将之前缓存的数据重新加载到缓存中。 -每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当一半以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有一半内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。 +每个vnode有自己独立的内存，而且由多个固定大小的内存块组成，不同vnode之间完全隔离。数据写入时，类似于日志的写法，数据被顺序追加写入内存，但每个vnode维护有自己的skip list，便于迅速查找。当三分之一以上的内存块写满时，启动落盘操作，而且后续写的操作在新的内存块进行。这样，一个vnode里有三分之一内存块是保留有最近的数据的，以达到缓存、快速查找的目的。一个vnode的内存块的个数由配置参数blocks决定，内存块的大小由配置参数cache决定。 ### 持久化存储 TDengine采用数据驱动的方式让缓存中的数据写入硬盘进行持久化存储。当vnode中缓存的数据达到一定规模时，为了不阻塞后续数据的写入，TDengine也会拉起落盘线程将缓存的数据写入持久化存储。TDengine在数据落盘时会打开新的数据库日志文件，在落盘成功后则会删除老的数据库日志文件，避免日志文件无限制的增长。 From 6ea61b49167901f320be19d5f9ecfc952e042c63 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 08:43:50 +0000 Subject: [PATCH 120/122] revert codes --- src/balance/src/balance.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index 6d4f08eb06..4c687cb134 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -586,7 +586,7 @@ static void balanceProcessBalanceTimer(void *handle, void *tmrId) { if (!sdbIsMaster()) return; tsBalanceTimer = NULL; - if (handle == NULL) tsAccessSquence++; + tsAccessSquence ++; balanceCheckDnodeAccess(); bool updateSoon = false; From 101e0bb1d27c2ba5e3b09f984940cd8a3880f04b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 09:12:22 +0000 Subject: [PATCH 121/122] TD-1949 --- src/inc/twal.h | 1 + src/vnode/src/vnodeMain.c | 1 + src/wal/src/walMgmt.c | 2 +- src/wal/src/walWrite.c | 34 +++++---- tests/script/general/wal/kill.sim | 77 +++++++++++++++++++++ tests/script/general/{http => wal}/sync.sim | 0 6 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 tests/script/general/wal/kill.sim rename tests/script/general/{http => wal}/sync.sim (100%) diff --git a/src/inc/twal.h b/src/inc/twal.h index c32bb87021..b85377d8d4 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -59,6 +59,7 @@ int32_t walAlter(twalh pWal, SWalCfg *pCfg); void walStop(twalh); void walClose(twalh); int32_t walRenew(twalh); +void walRemoveOldFiles(twalh); int32_t walWrite(twalh, SWalHead *); void walFsync(twalh, bool forceFsync); int32_t walRestore(twalh, void *pVnode, FWalWrite writeFp); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index bd44ce8e1c..7592b4edc4 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -583,6 +583,7 @@ static int vnodeProcessTsdbStatus(void *arg, int status) { if (status == TSDB_STATUS_COMMIT_OVER) { vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); + walRemoveOldFiles(pVnode->wal); return vnodeSaveVersion(pVnode); } diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index de666c85e8..9ba0dfd124 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -135,7 +135,7 @@ void walClose(void *handle) { if (remove(pWal->name) < 0) { wError("vgId:%d, wal:%p file:%s, failed to remove", pWal->vgId, pWal, pWal->name); } else { - wDebug("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name); + wInfo("vgId:%d, wal:%p file:%s, it is removed", pWal->vgId, pWal, pWal->name); } } } else { diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 9681f4b898..d3a41ec6b2 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -58,24 +58,32 @@ int32_t walRenew(void *handle) { wDebug("vgId:%d, file:%s, it is created", pWal->vgId, pWal->name); } - if (pWal->keep != TAOS_WAL_KEEP) { - // remove the oldest wal file - int64_t oldFileId = -1; - if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) { - char walName[WAL_FILE_LEN] = {0}; - snprintf(walName, sizeof(walName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, oldFileId); + pthread_mutex_unlock(&pWal->mutex); - if (remove(walName) < 0) { - wError("vgId:%d, file:%s, failed to remove since %s", pWal->vgId, walName, strerror(errno)); - } else { - wDebug("vgId:%d, file:%s, it is removed", pWal->vgId, walName); - } + return code; +} + +void walRemoveOldFiles(void *handle) { + SWal *pWal = handle; + if (pWal == NULL) return; + if (pWal->keep == TAOS_WAL_KEEP) return; + + pthread_mutex_lock(&pWal->mutex); + + // remove the oldest wal file + int64_t oldFileId = -1; + if (walGetOldFile(pWal, pWal->fileId, WAL_FILE_NUM, &oldFileId) == 0) { + char walName[WAL_FILE_LEN] = {0}; + snprintf(walName, sizeof(walName), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, oldFileId); + + if (remove(walName) < 0) { + wError("vgId:%d, file:%s, failed to remove since %s", pWal->vgId, walName, strerror(errno)); + } else { + wInfo("vgId:%d, file:%s, it is removed", pWal->vgId, walName); } } pthread_mutex_unlock(&pWal->mutex); - - return code; } int32_t walWrite(void *handle, SWalHead *pHead) { diff --git a/tests/script/general/wal/kill.sim b/tests/script/general/wal/kill.sim new file mode 100644 index 0000000000..7f103874a5 --- /dev/null +++ b/tests/script/general/wal/kill.sim @@ -0,0 +1,77 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 + +print ============== deploy +system sh/exec.sh -n dnode1 -s start +sleep 3001 +sql connect + +sql create database d1 +sql use d1 + +sql create table t1 (ts timestamp, i int) +sql insert into t1 values(now, 1); + +print =============== step3 +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step4 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step5 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step6 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step7 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL +sleep 3000 + +print =============== step8 +system sh/exec.sh -n dnode1 -s start -x SIGKILL +sleep 3000 +sql select * from t1; +print rows: $rows +if $rows != 1 then + return -1 +endi +system sh/exec.sh -n dnode1 -s stop -x SIGKILL diff --git a/tests/script/general/http/sync.sim b/tests/script/general/wal/sync.sim similarity index 100% rename from tests/script/general/http/sync.sim rename to tests/script/general/wal/sync.sim From 862bdd5bd21b57955b1e31a7f97c1f25acbe9fbb Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 11 Nov 2020 09:14:44 +0000 Subject: [PATCH 122/122] scripts --- tests/script/jenkins/basic.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index b5462ba261..2a84172da9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -89,7 +89,6 @@ cd ../../../debug; make ./test.sh -f general/http/telegraf.sim ./test.sh -f general/http/grafana_bug.sim ./test.sh -f general/http/grafana.sim -./test.sh -f general/http/sync.sim ./test.sh -f general/import/basic.sim ./test.sh -f general/import/commit.sim @@ -237,6 +236,9 @@ cd ../../../debug; make ./test.sh -f general/vector/table_query.sim ./test.sh -f general/vector/table_time.sim +./test.sh -f general/wal/sync.sim +./test.sh -f general/wal/kill.sim + ./test.sh -f unique/account/account_create.sim ./test.sh -f unique/account/account_delete.sim ./test.sh -f unique/account/account_len.sim