From 09e91f9ad40417bfa0b69f3481328081204e51d7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Sep 2020 11:22:49 +0800 Subject: [PATCH 01/58] TD-1437 --- src/query/src/qAst.c | 6 +- src/tsdb/src/tsdbMemTable.c | 40 +- src/tsdb/src/tsdbMeta.c | 23 +- src/util/CMakeLists.txt | 2 +- src/util/inc/tskiplist.h | 51 +- src/util/src/tskiplist.c | 1016 +++++++++++++++++-------------- src/util/tests/skiplistTest.cpp | 4 +- 7 files changed, 602 insertions(+), 540 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 634f014d97..1e60c71afd 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -411,9 +411,9 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr } if (cond.start != NULL) { - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_ASC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_ASC); } else { - iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->keyInfo.type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*)(cond.end ? cond.end->v: NULL), pSkipList->type, TSDB_ORDER_DESC); } if (cond.start != NULL) { @@ -468,7 +468,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr tSkipListDestroyIter(iter); comp = true; - iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->keyInfo.type, TSDB_ORDER_DESC); + iter = tSkipListCreateIterFromVal(pSkipList, (char*) cond.start->v, pSkipList->type, TSDB_ORDER_DESC); while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index f6a7f1b35c..0b7cc4dca4 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -37,38 +37,25 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { STsdbCfg * pCfg = &pRepo->config; STsdbMeta * pMeta = pRepo->tsdbMeta; - int32_t level = 0; - int32_t headSize = 0; TSKEY key = dataRowKey(row); SMemTable * pMemTable = pRepo->mem; STableData *pTableData = NULL; - SSkipList * pSList = NULL; + // SSkipList * pSList = NULL; - if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && - pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { - pTableData = pMemTable->tData[TABLE_TID(pTable)]; - pSList = pTableData->pData; - } - - tSkipListNewNodeInfo(pSList, &level, &headSize); - - SSkipListNode *pNode = (SSkipListNode *)malloc(headSize + sizeof(SDataRow *)); - if (pNode == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } + // if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && + // pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { + // pTableData = pMemTable->tData[TABLE_TID(pTable)]; + // pSList = pTableData->pData; + // } void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); if (pRow == NULL) { tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s", REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), dataRowLen(row), tstrerror(terrno)); - free(pNode); return -1; } - pNode->level = level; dataRowCpy(pRow, row); - *(SDataRow *)SL_GET_NODE_DATA(pNode) = pRow; // Operations above may change pRepo->mem, retake those values ASSERT(pRepo->mem != NULL); @@ -77,7 +64,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (TABLE_TID(pTable) >= pMemTable->maxTables) { if (tsdbAdjustMemMaxTables(pMemTable, pMeta->maxTables) < 0) { tsdbFreeBytes(pRepo, pRow, dataRowLen(row)); - free(pNode); return -1; } } @@ -97,7 +83,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { " to table %s while create new table data object since %s", REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno)); tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); - free(pNode); return -1; } @@ -106,20 +91,19 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); - if (tSkipListPut(pTableData->pData, pNode) == NULL) { + int64_t oldSize = SL_GET_SIZE(pTableData->pData); + if (tSkipListPut(pTableData->pData, (void *)(&pRow), sizeof(void *)) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); - free(pNode); } else { + int64_t deltaSize = SL_GET_SIZE(pTableData->pData) - oldSize; if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; - pMemTable->numOfRows++; + pMemTable->numOfRows += deltaSize; if (pTableData->keyFirst > key) pTableData->keyFirst = key; if (pTableData->keyLast < key) pTableData->keyLast = key; - pTableData->numOfRows++; - - ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); + pTableData->numOfRows += deltaSize; } tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), @@ -439,7 +423,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->numOfRows = 0; pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 1, tsdbGetTsTupleKey); + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], /*SL_DISCARD_DUP_KEY*/ SL_APPEND_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index a84bb69777..dede109928 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -690,7 +690,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { } pTable->tagVal = NULL; STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -892,23 +892,8 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper pTable->pSuper = pSTable; - int32_t level = 0; - int32_t headSize = 0; + tSkipListPut(pSTable->pIndex, (void *)(&pTable), sizeof(STable *)); - tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize); - - // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the - // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function - SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *)); - if (pNode == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - pNode->level = level; - - memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *)); - - tSkipListPut(pSTable->pIndex, pNode); if (refSuper) T_REF_INC(pSTable); return 0; } @@ -1165,7 +1150,7 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { buf = tdDecodeSchema(buf, &(pTable->tagSchema)); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), 1, 0, 1, getTagIndexKey); + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbFreeTable(pTable); @@ -1191,7 +1176,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); } else { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (tSkipListGetSize(pTable->pIndex) + 1)); + tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_GET_SIZE(pTable->pIndex) + 1)); } else { tlen = sizeof(SListNode) + sizeof(SActObj); } diff --git a/src/util/CMakeLists.txt b/src/util/CMakeLists.txt index e63a085cc8..211c5ab320 100644 --- a/src/util/CMakeLists.txt +++ b/src/util/CMakeLists.txt @@ -7,7 +7,7 @@ TARGET_LINK_LIBRARIES(tutil pthread osdetail lz4) IF (TD_LINUX) TARGET_LINK_LIBRARIES(tutil m rt) - ADD_SUBDIRECTORY(tests) + # ADD_SUBDIRECTORY(tests) FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/) IF (ICONV_INCLUDE_EXIST) diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 4ba620dce0..f1e09501e9 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -27,6 +27,14 @@ extern "C" { #define MAX_SKIP_LIST_LEVEL 15 #define SKIP_LIST_RECORD_PERFORMANCE 0 +// For key property setting +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists +#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key +#define SL_UPDATA_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert +#define SL_APPEND_DUP_KEY (uint8_t)0x3 // Update duplicate key by append +// For thread safety setting +#define SL_THREAD_SAFE (uint8_t)0x4 + typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); @@ -41,6 +49,9 @@ typedef struct SSkipListNode { uint8_t level; } SSkipListNode; +#define SL_IS_THREAD_SAFE(flags) ((flags)&SL_THREAD_SAFE) +#define SL_DUP_MODE(flags) ((flags) & ((((uint8_t)1) << 2) - 1)) + #define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES) #define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)] @@ -54,6 +65,8 @@ typedef struct SSkipListNode { #define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0))) #define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) +#define SL_GET_SIZE(s) (s)->size +#define SL_GET_TSIZE(s) (s)->tsize /* * @version 0.3 @@ -113,13 +126,16 @@ typedef struct SSkipListKeyInfo { typedef struct SSkipList { __compar_fn_t comparFn; __sl_key_fn_t keyFn; - uint32_t size; - uint8_t maxLevel; - uint8_t level; - SSkipListKeyInfo keyInfo; pthread_rwlock_t *lock; - SSkipListNode * pHead; // point to the first element - SSkipListNode * pTail; // point to the last element + uint16_t len; + uint8_t maxLevel; + uint8_t flags; + uint8_t type; // static info above + uint8_t level; + uint32_t size; // not including duplicate keys + uint32_t tsize; // including duplicate keys + SSkipListNode * pHead; // point to the first element + SSkipListNode * pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE tSkipListState state; // skiplist state #endif @@ -145,8 +161,7 @@ typedef struct SSkipListIterator { * @param dupKey allow the duplicated key in the skip list * @return */ -SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t threadsafe, - uint8_t freeNode, __sl_key_fn_t fn); +SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn); /** * @@ -163,6 +178,17 @@ void *tSkipListDestroy(SSkipList *pSkipList); */ void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize); +/** + * put the data into the skiplist + * If failed, NULL will be returned, otherwise, the pNode will be returned. + * + * @param pSkipList + * @param pData + * @param dataLen + * @return + */ +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen); + /** * put the skip list node into the skip list. * If failed, NULL will be returned, otherwise, the pNode will be returned. @@ -171,7 +197,7 @@ void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSiz * @param pNode * @return */ -SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); +SSkipListNode *tSkipListPutNode(SSkipList *pSkipList, SSkipListNode *pNode); /** * get *all* nodes which key are equivalent to pKey @@ -182,13 +208,6 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode); */ SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); -/** - * get the size of skip list - * @param pSkipList - * @return - */ -size_t tSkipListGetSize(const SSkipList *pSkipList); - /** * display skip list of the given level, for debug purpose only * @param pSkipList diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 303c2440bf..73f6bf0502 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -13,13 +13,521 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "os.h" -#include "tulog.h" #include "tskiplist.h" -#include "tutil.h" +#include "os.h" #include "tcompare.h" +#include "tulog.h" +#include "tutil.h" -UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, int32_t level) { // record link count in each level +#define DO_MEMSET_PTR_AREA(n) \ + do { \ + int32_t _l = (n)->level; \ + memset(pNode, 0, SL_NODE_HEADER_SIZE(_l)); \ + (n)->level = _l; \ + } while (0) + +static int initForwardBackwardPtr(SSkipList *pSkipList); +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); +static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); +static void tSkipListCorrectLevel(SSkipList *pSkipList); +static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); + +static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); +static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); +static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); +static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); +static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); + +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn) { + SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); + if (pSkipList == NULL) { + return NULL; + } + + if (maxLevel > MAX_SKIP_LIST_LEVEL) { + maxLevel = MAX_SKIP_LIST_LEVEL; + } + + pSkipList->maxLevel = maxLevel; + pSkipList->type = keyType; + pSkipList->len = keyLen; + pSkipList->flags = flags; + pSkipList->keyFn = fn; + pSkipList->comparFn = getKeyComparFunc(keyType); + + // pSkipList->level = 1; // TODO: check if 1 is valid + if (initForwardBackwardPtr(pSkipList) < 0) { + taosTFree(pSkipList); + return NULL; + } + + if (SL_IS_THREAD_SAFE(flags)) { + pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t)); + + if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { + taosTFree(pSkipList->pHead); + taosTFree(pSkipList); + + return NULL; + } + } + + srand((uint32_t)time(NULL)); + +#if SKIP_LIST_RECORD_PERFORMANCE + pSkipList->state.nTotalMemSize += sizeof(SSkipList); +#endif + + return pSkipList; +} + +void *tSkipListDestroy(SSkipList *pSkipList) { + if (pSkipList == NULL) return NULL; + + tSkipListWLock(pSkipList); + + SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); + + while (pNode != pSkipList->pTail) { + SSkipListNode *pTemp = pNode; + pNode = SL_GET_FORWARD_POINTER(pNode, 0); + taosTFree(pTemp); + } + + tSkipListUnlock(pSkipList); + if (pSkipList->lock != NULL) { + pthread_rwlock_destroy(pSkipList->lock); + taosTFree(pSkipList->lock); + } + + taosTFree(pSkipList->pHead); + taosTFree(pSkipList); + return NULL; +} + +void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) { + if (pSkipList == NULL) { + *level = 1; + *headSize = SL_NODE_HEADER_SIZE(*level); + return; + } + + *level = getSkipListRandLevel(pSkipList); + *headSize = SL_NODE_HEADER_SIZE(*level); +} + +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen) { + if (pSkipList == NULL || pData == NULL) return NULL; + + tSkipListWLock(pSkipList); + + int32_t level = getSkipListRandLevel(pSkipList); + + SSkipListNode *pNode = (SSkipListNode *)malloc(SL_NODE_HEADER_SIZE(level) + dataLen); + if (pNode == NULL) { + tSkipListUnlock(pSkipList); + return NULL; + } + + pNode->level = level; + memcpy(SL_GET_NODE_DATA(pNode), pData, dataLen); + + if (tSkipListPutNodeImpl(pSkipList, pNode) == NULL) { + tSkipListUnlock(pSkipList); + taosTFree(pNode); + return NULL; + } + + tSkipListUnlock(pSkipList); + + return pNode; +} + +SSkipListNode *tSkipListPutNode(SSkipList *pSkipList, SSkipListNode *pNode) { + SSkipListNode *pRetNode = NULL; + + if (pSkipList == NULL || pNode == NULL) return NULL; + + tSkipListWLock(pSkipList); + pRetNode = tSkipListPutNodeImpl(pSkipList, pNode); + tSkipListUnlock(pSkipList); + + return pRetNode; +} + +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { + SArray *sa = taosArrayInit(1, POINTER_BYTES); + + tSkipListRLock(pSkipList); + + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + while (1) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + taosArrayPush(sa, &p); + pNode = p; + } + + tSkipListUnlock(pSkipList); + + return sa; +} + +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { + uint32_t count = 0; + + tSkipListWLock(pSkipList); + + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + while (1) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + + tSkipListRemoveNodeImpl(pSkipList, p); + + ++count; + } + + tSkipListCorrectLevel(pSkipList); + + tSkipListUnlock(pSkipList); + + return count; +} + +void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { + tSkipListWLock(pSkipList); + tSkipListRemoveNodeImpl(pSkipList, pNode); + tSkipListCorrectLevel(pSkipList); + tSkipListUnlock(pSkipList); +} + +SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) { + if (pSkipList == NULL) return NULL; + + return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC); +} + +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order) { + assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + assert(pSkipList != NULL); + + SSkipListIterator *iter = doCreateSkipListIterator(pSkipList, order); + if (val == NULL) { + return iter; + } + + tSkipListRLock(pSkipList); + + iter->cur = getPriorNode(pSkipList, val, order); + + tSkipListUnlock(pSkipList); + + return iter; +} + +bool tSkipListIterNext(SSkipListIterator *iter) { + if (iter->pSkipList == NULL) return false; + + SSkipList *pSkipList = iter->pSkipList; + uint8_t dupMod = SL_DUP_MODE(pSkipList->flags); + + tSkipListRLock(pSkipList); + + if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate + if (dupMod == SL_APPEND_DUP_KEY) { + if (iter->cur == pSkipList->pHead) { + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + iter->step++; + } else { + while (true) { + iter->step++; + SSkipListNode *pNode = iter->cur; + iter->cur = SL_GET_FORWARD_POINTER(pNode, 0); + + if (iter->cur == pSkipList->pTail) break; + if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { + continue; + } else { + break; + } + } + } + + if (iter->cur != pSkipList->pTail) { + while (true) { + SSkipListNode *pNode = SL_GET_FORWARD_POINTER(iter->cur, 0); + if (pNode == pSkipList->pTail) break; + if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { + iter->step++; + iter->cur = pNode; + } else { + break; + } + } + } + } else { + iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + iter->step++; + } + } else { // descending order iterate + if (dupMod == SL_APPEND_DUP_KEY) { + if (iter->cur == pSkipList->pTail) { + iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + iter->step++; + } else { + while (true) { + SSkipListNode *pNode = iter->cur; + iter->cur = SL_GET_BACKWARD_POINTER(pNode, 0); + + if (iter->cur == pSkipList->pHead) break; + if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { + iter->cur = pNode; + continue; + } else { + break; + } + } + } + } else { + iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + iter->step++; + } + } + + tSkipListUnlock(pSkipList); + + return (iter->order == TSDB_ORDER_ASC) ? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead); +} + +SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) { + if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) { + return NULL; + } else { + return iter->cur; + } +} + +void *tSkipListDestroyIter(SSkipListIterator *iter) { + if (iter == NULL) { + return NULL; + } + + taosTFree(iter); + return NULL; +} + +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { + if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) { + return; + } + + SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); + + int32_t id = 1; + char * prev = NULL; + + while (p != pSkipList->pTail) { + char *key = SL_GET_NODE_KEY(pSkipList, p); + if (prev != NULL) { + assert(pSkipList->comparFn(prev, key) < 0); + } + + switch (pSkipList->type) { + case TSDB_DATA_TYPE_INT: + fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_BIGINT: + fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key); + break; + case TSDB_DATA_TYPE_BINARY: + fprintf(stdout, "%d: %s \n", id++, key); + break; + case TSDB_DATA_TYPE_DOUBLE: + fprintf(stdout, "%d: %lf \n", id++, *(double *)key); + break; + default: + fprintf(stdout, "\n"); + } + + prev = SL_GET_NODE_KEY(pSkipList, p); + + p = SL_GET_FORWARD_POINTER(p, nlevel - 1); + } +} + +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode, bool hasDupKey) { + uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + + // FIXME: this may cause the level of skiplist change + if (dupMode == SL_UPDATA_DUP_KEY && hasDupKey) { + tSkipListRemoveNodeImpl(pSkipList, forward[0]); + tSkipListCorrectLevel(pSkipList); + } + + DO_MEMSET_PTR_AREA(pNode); + + for (int32_t i = 0; i < pNode->level; ++i) { + SSkipListNode *x = forward[i]; + SL_GET_BACKWARD_POINTER(pNode, i) = x; + + SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); + SL_GET_BACKWARD_POINTER(next, i) = pNode; + + SL_GET_FORWARD_POINTER(pNode, i) = next; + SL_GET_FORWARD_POINTER(x, i) = pNode; + } + + if (!(dupMode == SL_APPEND_DUP_KEY && hasDupKey)) { + pSkipList->size += 1; + } + pSkipList->tsize += 1; +} + +static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { + SSkipListIterator *iter = calloc(1, sizeof(SSkipListIterator)); + + iter->pSkipList = pSkipList; + iter->order = order; + if (order == TSDB_ORDER_ASC) { + iter->cur = pSkipList->pHead; + } else { + iter->cur = pSkipList->pTail; + } + + return iter; +} + +static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) { + if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + return pthread_rwlock_wrlock(pSkipList->lock); + } + return 0; +} + +static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) { + if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + return pthread_rwlock_rdlock(pSkipList->lock); + } + return 0; +} + +static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { + if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + return pthread_rwlock_unlock(pSkipList->lock); + } + return 0; +} + +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { + int compare = 1; + bool hasDupKey = false; + char * pNodeKey = SL_GET_NODE_KEY(pSkipList, pNode); + uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + + if (pSkipList->size == 0) { + for (int i = 0; i < pNode->level; i++) { + forward[i] = pSkipList->pHead; + } + } else { + char *pKey = NULL; + + // Compare min key + pKey = SL_GET_SL_MIN_KEY(pSkipList); + compare = pSkipList->comparFn(pNodeKey, pKey); + if ((dupMode == SL_APPEND_DUP_KEY && compare < 0) || (dupMode != SL_APPEND_DUP_KEY && compare <= 0)) { + for (int i = 0; i < pNode->level; i++) { + forward[i] = pSkipList->pHead; + } + return (compare == 0); + } + + // Compare max key + pKey = SL_GET_SL_MAX_KEY(pSkipList); + compare = pSkipList->comparFn(pNodeKey, pKey); + if ((dupMode == SL_DISCARD_DUP_KEY && compare > 0) || (dupMode != SL_DISCARD_DUP_KEY && compare >= 0)) { + for (int i = 0; i < pNode->level; i++) { + forward[i] = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); + } + + return (compare == 0); + } + + SSkipListNode *px = pSkipList->pHead; + for (int i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); + while (p != pSkipList->pTail) { + pKey = SL_GET_NODE_KEY(pSkipList, p); + + compare = pSkipList->comparFn(pKey, pNodeKey); + if (compare == 0 && hasDupKey == false) hasDupKey = true; + if ((dupMode == SL_APPEND_DUP_KEY && compare > 0) || (dupMode != SL_APPEND_DUP_KEY && compare >= 0)) { + break; + } else { + px = p; + p = SL_GET_FORWARD_POINTER(px, i); + } + } + + forward[i] = px; + } + } + + return hasDupKey; +} + +static bool tSkipListIsNodeDup(SSkipList *pSkipList, SSkipListNode *pNode) { + SSkipListNode *pPrevNode = SL_GET_BACKWARD_POINTER(pNode, 0); + SSkipListNode *pNextNode = SL_GET_FORWARD_POINTER(pNode, 0); + char * pNodeKey = SL_GET_NODE_KEY(pSkipList, pNode); + char * pPrevNodeKey = (pPrevNode == pSkipList->pHead) ? NULL : SL_GET_NODE_KEY(pSkipList, pPrevNode); + char * pNextNodeKey = (pNextNode == pSkipList->pTail) ? NULL : SL_GET_NODE_KEY(pSkipList, pNextNode); + + return ((pPrevNodeKey != NULL && pSkipList->comparFn(pNodeKey, pPrevNodeKey) == 0) || + (pNextNodeKey != NULL && pSkipList->comparFn(pNodeKey, pNextNodeKey) == 0)); +} + +static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { + int32_t level = pNode->level; + uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + + bool sizeReduce = !(dupMode == SL_APPEND_DUP_KEY && tSkipListIsNodeDup(pSkipList, pNode)); + + for (int32_t j = level - 1; j >= 0; --j) { + SSkipListNode *prev = SL_GET_BACKWARD_POINTER(pNode, j); + SSkipListNode *next = SL_GET_FORWARD_POINTER(pNode, j); + + SL_GET_FORWARD_POINTER(prev, j) = next; + SL_GET_BACKWARD_POINTER(next, j) = prev; + } + + taosTFree(pNode); + + if (sizeReduce) pSkipList->size--; + pSkipList->tsize--; +} + +// Function must be called after calling tSkipListRemoveNodeImpl() function +static void tSkipListCorrectLevel(SSkipList *pSkipList) { + while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { + pSkipList->level -= 1; + } +} + +UNUSED_FUNC static FORCE_INLINE void recordNodeEachLevel(SSkipList *pSkipList, + int32_t level) { // record link count in each level #if SKIP_LIST_RECORD_PERFORMANCE for (int32_t i = 0; i < level; ++i) { pSkipList->state.nLevelNodeCnt[i]++; @@ -47,40 +555,28 @@ static FORCE_INLINE int32_t getSkipListNodeRandomHeight(SSkipList *pSkipList) { } static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { - int32_t level = getSkipListNodeRandomHeight(pSkipList); + int32_t level = 0; if (pSkipList->size == 0) { level = 1; - pSkipList->level = 1; } else { + level = getSkipListNodeRandomHeight(pSkipList); if (level > pSkipList->level) { if (pSkipList->level < pSkipList->maxLevel) { - level = (++pSkipList->level); + level = pSkipList->level + 1; } else { level = pSkipList->level; } } } - + assert(level <= pSkipList->maxLevel); return level; } -#define DO_MEMSET_PTR_AREA(n) do {\ -int32_t _l = (n)->level;\ -memset(pNode, 0, SL_NODE_HEADER_SIZE(_l));\ -(n)->level = _l;\ -} while(0) - -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); -static SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode); -static SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode); -static SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); - - // when order is TSDB_ORDER_ASC, return the last node with key less than val // when order is TSDB_ORDER_DESC, return the first node with key large than val -static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_t order) { - __compar_fn_t comparFn = pSkipList->comparFn; +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order) { + __compar_fn_t comparFn = pSkipList->comparFn; SSkipListNode *pNode = NULL; if (order == TSDB_ORDER_ASC) { @@ -116,214 +612,49 @@ static SSkipListNode* getPriorNode(SSkipList* pSkipList, const char* val, int32_ return pNode; } - -static bool initForwardBackwardPtr(SSkipList* pSkipList) { +static int initForwardBackwardPtr(SSkipList *pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; - + // head info - pSkipList->pHead = (SSkipListNode *)calloc(1, SL_NODE_HEADER_SIZE(maxLevel) * 2); + pSkipList->pHead = (SSkipListNode *)malloc(SL_NODE_HEADER_SIZE(maxLevel) * 2); if (pSkipList->pHead == NULL) { - return false; + return -1; } - - pSkipList->pHead->level = pSkipList->maxLevel; - + + pSkipList->pHead->level = maxLevel; + // tail info - pSkipList->pTail = (SSkipListNode*) ((char*) pSkipList->pHead + SL_NODE_HEADER_SIZE(maxLevel)); - pSkipList->pTail->level = pSkipList->maxLevel; - + pSkipList->pTail = (SSkipListNode *)POINTER_SHIFT(pSkipList->pHead, SL_NODE_HEADER_SIZE(maxLevel)); + pSkipList->pTail->level = maxLevel; + for (uint32_t i = 0; i < maxLevel; ++i) { SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; } - - return true; + + return 0; } -SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint8_t keyLen, uint8_t dupKey, uint8_t lock, - uint8_t freeNode, __sl_key_fn_t fn) { - SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); - if (pSkipList == NULL) { - return NULL; - } - - if (maxLevel > MAX_SKIP_LIST_LEVEL) { - maxLevel = MAX_SKIP_LIST_LEVEL; - } - - pSkipList->keyInfo.type = keyType; - pSkipList->keyInfo.len = keyLen; - pSkipList->keyInfo.dupKey = dupKey; - pSkipList->keyInfo.freeNode = freeNode; - - pSkipList->keyFn = fn; - pSkipList->comparFn = getKeyComparFunc(keyType); - pSkipList->maxLevel = maxLevel; - pSkipList->level = 1; - - if (!initForwardBackwardPtr(pSkipList)) { - taosTFree(pSkipList); - return NULL; - } - - if (lock) { - pSkipList->lock = calloc(1, sizeof(pthread_rwlock_t)); - - if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { - taosTFree(pSkipList->pHead); - taosTFree(pSkipList); - - return NULL; - } - } - - srand((uint32_t)time(NULL)); - -#if SKIP_LIST_RECORD_PERFORMANCE - pSkipList->state.nTotalMemSize += sizeof(SSkipList); -#endif - - return pSkipList; -} - -void *tSkipListDestroy(SSkipList *pSkipList) { - if (pSkipList == NULL) { - return NULL; - } - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - if (pSkipList->keyInfo.freeNode) { - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); - - while (pNode != pSkipList->pTail) { - SSkipListNode *pTemp = pNode; - pNode = SL_GET_FORWARD_POINTER(pNode, 0); - taosTFree(pTemp); - } - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - pthread_rwlock_destroy(pSkipList->lock); - - taosTFree(pSkipList->lock); - } - - taosTFree(pSkipList->pHead); - taosTFree(pSkipList); - return NULL; -} - -void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) { - if (pSkipList == NULL) { - *level = 1; - *headSize = SL_NODE_HEADER_SIZE(*level); - return; - } - - *level = getSkipListRandLevel(pSkipList); - *headSize = SL_NODE_HEADER_SIZE(*level); -} - -SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { - if (pSkipList == NULL || pNode == NULL) { - return NULL; - } - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - // if the new key is greater than the maximum key of skip list, push back this node at the end of skip list - char *newDatakey = SL_GET_NODE_KEY(pSkipList, pNode); - if (pSkipList->size == 0 || pSkipList->comparFn(SL_GET_SL_MAX_KEY(pSkipList), newDatakey) < 0) { - return tSkipListPushBack(pSkipList, pNode); - } - - // if the new key is less than the minimum key of skip list, push front this node at the front of skip list - assert(pSkipList->size > 0); - char* minKey = SL_GET_SL_MIN_KEY(pSkipList); - if (pSkipList->comparFn(newDatakey, minKey) < 0) { - return tSkipListPushFront(pSkipList, pNode); - } - - // find the appropriated position to insert data - SSkipListNode *px = pSkipList->pHead; +static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { + SSkipListNode *pRetNode = NULL; SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; - int32_t ret = -1; - for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); - while (p != pSkipList->pTail) { - char *key = SL_GET_NODE_KEY(pSkipList, p); + int hasDupKey = tSkipListGetPosToPut(pSkipList, forward, pNode); + if (SL_DUP_MODE(pSkipList->flags) == SL_DISCARD_DUP_KEY && hasDupKey) { + pRetNode = NULL; + } else { + pRetNode = pNode; + tSkipListDoInsert(pSkipList, forward, pNode, hasDupKey); - // if the forward element is less than the specified key, forward one step - ret = pSkipList->comparFn(key, newDatakey); - if (ret < 0) { - px = p; - p = SL_GET_FORWARD_POINTER(px, i); - } else { - break; - } - } - - forward[i] = px; + if (pNode->level > pSkipList->level) pSkipList->level = pNode->level; } - // if the skip list does not allowed identical key inserted, the new data will be discarded. - if (pSkipList->keyInfo.dupKey == 0 && ret == 0) { - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return NULL; - } - - tSkipListDoInsert(pSkipList, forward, pNode); - return pNode; + return pRetNode; } - - -SArray* tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { - SArray* sa = taosArrayInit(1, POINTER_BYTES); - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); - while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); - if (p == pSkipList->pTail) { - break; - } - if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { - break; - } - taosArrayPush(sa, &p); - pNode = p; - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return sa; -} - - - -size_t tSkipListGetSize(const SSkipList* pSkipList) { - if (pSkipList == NULL) { - return 0; - } - - return pSkipList->size; -} +// static void tSkipListSeek(SSkipList *pSkipList, char *key, int order) { +// // TODO +// } // static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey, // int32_t cond, SSkipListNode ***pRes) { @@ -444,158 +775,7 @@ size_t tSkipListGetSize(const SSkipList* pSkipList) { // // return true; //} - -uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { - uint32_t count = 0; - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - SSkipListNode* pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); - while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); - if (p == pSkipList->pTail) { - break; - } - if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { - break; - } - - for (int32_t j = p->level - 1; j >= 0; --j) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(p, j); - SSkipListNode* next = SL_GET_FORWARD_POINTER(p, j); - SL_GET_FORWARD_POINTER(prev, j) = next; - SL_GET_BACKWARD_POINTER(next, j) = prev; - } - - if (pSkipList->keyInfo.freeNode) { - taosTFree(p); - } - - ++count; - } - - // compress the minimum level of skip list - while (pSkipList->level > 0) { - if (SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) != NULL) { - break; - } - pSkipList->level--; - } - - pSkipList->size -= count; - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return count; -} - -void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { - int32_t level = pNode->level; - - if (pSkipList->lock) { - pthread_rwlock_wrlock(pSkipList->lock); - } - - for (int32_t j = level - 1; j >= 0; --j) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pNode, j); - SSkipListNode* next = SL_GET_FORWARD_POINTER(pNode, j); - - SL_GET_FORWARD_POINTER(prev, j) = next; - SL_GET_BACKWARD_POINTER(next, j) = prev; - } - - if (pSkipList->keyInfo.freeNode) { - taosTFree(pNode); - } - - atomic_sub_fetch_32(&pSkipList->size, 1); - - // compress the minimum level of skip list - while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { - pSkipList->level -= 1; - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } -} - -SSkipListIterator* tSkipListCreateIter(SSkipList *pSkipList) { - if (pSkipList == NULL) { - return NULL; - } - - return doCreateSkipListIterator(pSkipList, TSDB_ORDER_ASC); -} - -SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order) { - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - assert(pSkipList != NULL); - - SSkipListIterator* iter = doCreateSkipListIterator(pSkipList, order); - if (val == NULL) { - return iter; - } - - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); - } - - iter->cur = getPriorNode(pSkipList, val, order); - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return iter; -} - -bool tSkipListIterNext(SSkipListIterator *iter) { - if (iter->pSkipList == NULL) { - return false; - } - - SSkipList *pSkipList = iter->pSkipList; - - if (pSkipList->lock) { - pthread_rwlock_rdlock(pSkipList->lock); - } - - if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); - } else { // descending order iterate - iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); - } - - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - iter->step += 1; - return (iter->order == TSDB_ORDER_ASC)? (iter->cur != pSkipList->pTail) : (iter->cur != pSkipList->pHead); -} - -SSkipListNode *tSkipListIterGet(SSkipListIterator *iter) { - if (iter == NULL || iter->cur == iter->pSkipList->pTail || iter->cur == iter->pSkipList->pHead) { - return NULL; - } else { - return iter->cur; - } -} - -void* tSkipListDestroyIter(SSkipListIterator* iter) { - if (iter == NULL) { - return NULL; - } - - taosTFree(iter); - return NULL; -} - +// // bool tSkipListRemove(SSkipList *pSkipList, SSkipListKey *pKey) { // SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; // __compar_fn_t filterComparFn = getComparFunc(pSkipList, pKey->nType); @@ -614,110 +794,4 @@ void* tSkipListDestroyIter(SSkipListIterator* iter) { // pthread_rwlock_unlock(&pSkipList->lock); // // return ret; -//} - -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { - if (pSkipList == NULL || pSkipList->level < nlevel || nlevel <= 0) { - return; - } - - SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); - - int32_t id = 1; - char* prev = NULL; - - while (p != pSkipList->pTail) { - char *key = SL_GET_NODE_KEY(pSkipList, p); - if (prev != NULL) { - assert(pSkipList->comparFn(prev, key) < 0); - } - - switch (pSkipList->keyInfo.type) { - case TSDB_DATA_TYPE_INT: - fprintf(stdout, "%d: %d\n", id++, *(int32_t *)key); - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_BIGINT: - fprintf(stdout, "%d: %" PRId64 " \n", id++, *(int64_t *)key); - break; - case TSDB_DATA_TYPE_BINARY: - fprintf(stdout, "%d: %s \n", id++, key); - break; - case TSDB_DATA_TYPE_DOUBLE: - fprintf(stdout, "%d: %lf \n", id++, *(double *)key); - break; - default: - fprintf(stdout, "\n"); - } - - prev = SL_GET_NODE_KEY(pSkipList, p); - - p = SL_GET_FORWARD_POINTER(p, nlevel - 1); - } -} - -void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { - DO_MEMSET_PTR_AREA(pNode); - - for (int32_t i = 0; i < pNode->level; ++i) { - SSkipListNode *x = forward[i]; - SL_GET_BACKWARD_POINTER(pNode, i) = x; - - SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); - SL_GET_BACKWARD_POINTER(next, i) = pNode; - - SL_GET_FORWARD_POINTER(pNode, i) = next; - SL_GET_FORWARD_POINTER(x, i) = pNode; - } - - atomic_add_fetch_32(&pSkipList->size, 1); - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } -} - -SSkipListNode* tSkipListPushFront(SSkipList* pSkipList, SSkipListNode *pNode) { - SSkipListNode* forward[MAX_SKIP_LIST_LEVEL] = {0}; - for(int32_t i = 0; i < pSkipList->level; ++i) { - forward[i] = pSkipList->pHead; - } - - tSkipListDoInsert(pSkipList, forward, pNode); - return pNode; -} - -SSkipListNode* tSkipListPushBack(SSkipList *pSkipList, SSkipListNode *pNode) { - // do clear pointer area - DO_MEMSET_PTR_AREA(pNode); - - for(int32_t i = 0; i < pNode->level; ++i) { - SSkipListNode* prev = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); - SL_GET_FORWARD_POINTER(prev, i) = pNode; - SL_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; - - SL_GET_BACKWARD_POINTER(pNode, i) = prev; - SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; - } - - atomic_add_fetch_32(&pSkipList->size, 1); - if (pSkipList->lock) { - pthread_rwlock_unlock(pSkipList->lock); - } - - return pNode; -} - -SSkipListIterator* doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { - SSkipListIterator* iter = calloc(1, sizeof(SSkipListIterator)); - - iter->pSkipList = pSkipList; - iter->order = order; - if(order == TSDB_ORDER_ASC) { - iter->cur = pSkipList->pHead; - } else { - iter->cur = pSkipList->pTail; - } - - return iter; -} \ No newline at end of file +//} \ No newline at end of file diff --git a/src/util/tests/skiplistTest.cpp b/src/util/tests/skiplistTest.cpp index 77174f69fd..cd1fd1f7e1 100644 --- a/src/util/tests/skiplistTest.cpp +++ b/src/util/tests/skiplistTest.cpp @@ -247,7 +247,7 @@ void skiplistPerformanceTest() { printf("total:%" PRIu64 " ms, avg:%f\n", e - s, (e - s) / (double)size); printf("max level of skiplist:%d, actually level:%d\n ", pSkipList->maxLevel, pSkipList->level); - assert(tSkipListGetSize(pSkipList) == size); + assert(SL_GET_SIZE(pSkipList) == size); // printf("the level of skiplist is:\n"); // @@ -273,7 +273,7 @@ void skiplistPerformanceTest() { int64_t et = taosGetTimestampMs(); printf("delete %d data from skiplist, elapased time:%" PRIu64 "ms\n", 10000, et - st); - assert(tSkipListGetSize(pSkipList) == size); + assert(SL_GET_SIZE(pSkipList) == size); tSkipListDestroy(pSkipList); taosTFree(total); From db37e3cd13cfec79de28496de03b6bc660388bd5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Sep 2020 15:56:40 +0800 Subject: [PATCH 02/58] TD-1438 --- src/common/inc/tdataformat.h | 2 -- src/common/src/tdataformat.c | 11 ++++++-- src/inc/tsdb.h | 1 + src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbMain.c | 5 ++++ src/tsdb/src/tsdbMemTable.c | 14 ++++++---- src/tsdb/src/tsdbRWHelper.c | 53 +++++++++++++++++++----------------- 7 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index cc4afeb3f8..aa39d5b0b2 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -255,8 +255,6 @@ void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows); // ----------------- K-V data row structure /* diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index e7f40442a0..0c446b6d4f 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,6 +18,9 @@ #include "tcoding.h" #include "wchar.h" +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows); + /** * Duplicate the schema and return a new object */ @@ -499,7 +502,9 @@ _err: return -1; } -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) { +// src2 data has more priority than src1 +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows) { tdResetDataCols(target); ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); @@ -509,7 +514,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; - if (key1 <= key2) { + if (key1 < key2) { for (int i = 0; i < src1->numOfCols; i++) { ASSERT(target->cols[i].type == src1->cols[i].type); if (src1->cols[i].len > 0) { @@ -520,7 +525,6 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter1)++; - if (key1 == key2) (*iter2)++; } else { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); @@ -532,6 +536,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter2)++; + if (key1 == key2) (*iter1)++; } } } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 85f9b3bdc7..fe03277763 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -65,6 +65,7 @@ typedef struct { int32_t maxRowsPerFileBlock; // maximum rows per file block int8_t precision; int8_t compression; + int8_t update; } STsdbCfg; // --------- TSDB REPOSITORY USAGE STATISTICS diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 256b8189f8..95b7010038 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -430,7 +430,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem) void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TSKEY* filterKeys, int nFilterKeys); + TSKEY* filterKeys, int nFilterKeys, bool keepDup); static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index a1e6376304..494f17807e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } } + // update check + if (pCfg->update != 0) pCfg->update = 1; + return 0; _err: @@ -923,6 +926,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); tlen += taosEncodeFixedI8(buf, pCfg->precision); tlen += taosEncodeFixedI8(buf, pCfg->compression); + tlen += taosEncodeFixedI8(buf, pCfg->update); return tlen; } @@ -939,6 +943,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); buf = taosDecodeFixedI8(buf, &(pCfg->precision)); buf = taosDecodeFixedI8(buf, &(pCfg->compression)); + buf = taosDecodeFixedI8(buf, &(pCfg->update)); return buf; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0b7cc4dca4..c58460e18f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -278,7 +278,7 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { } int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TSKEY *filterKeys, int nFilterKeys) { + TSKEY *filterKeys, int nFilterKeys, bool keepDup) { ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; STSchema *pSchema = NULL; @@ -319,6 +319,10 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (!keyFiltered) { if (numOfRows >= maxRowsToRead) break; + numOfRows++; + } + + if (!keyFiltered || keepDup) { if (pCols) { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); @@ -329,8 +333,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey tdAppendDataRowToDataCol(row, pSchema, pCols); } - numOfRows++; - } + } } while (tSkipListIterNext(pIter)); return numOfRows; @@ -422,8 +425,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->keyLast = 0; pTableData->numOfRows = 0; - pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], /*SL_DISCARD_DUP_KEY*/ SL_APPEND_DUP_KEY, tsdbGetTsTupleKey); + pTableData->pData = + tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], + pCfg->update ? SL_APPEND_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index bb0f00ef53..f0017d0a14 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -62,7 +62,7 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx); static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows); + TSKEY maxKey, int maxRows, int8_t update); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -1453,7 +1453,7 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, - pDataCols, NULL, 0); + pDataCols, NULL, 0, pCfg->update); ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { @@ -1474,7 +1474,8 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } else { ASSERT(!pHelper->hasOldLastBlock); tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0); + int rowsRead = + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update); ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; @@ -1516,17 +1517,17 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); int rows1 = defaultRowsInBlock - pCompBlock->numOfRows; - int rows2 = - tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows); - if (rows2 == 0) { // all data filtered out + int rows2 = tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, + pDataCols0->numOfRows, pCfg->update); + if (!pCfg->update && rows2 == 0) { // all data filtered out *(pCommitIter->pIter) = slIter; } else { if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows); + pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); + ASSERT(rowsRead == rows2 && rowsRead <= pDataCols->numOfRows && pDataCols->numOfRows > 0); if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; @@ -1535,9 +1536,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int round = 0; int dIter = 0; while (true) { - tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock); + int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, + defaultRowsInBlock, pCfg->update); if (rowsRead == 0) break; if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; @@ -1561,8 +1561,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (keyFirst < blkKeyFirst) { while (true) { tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0); + int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, + NULL, 0, pCfg->update); if (rowsRead == 0) break; ASSERT(rowsRead == pDataCols->numOfRows); @@ -1580,21 +1580,21 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, slIter = *(pCommitIter->pIter); int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows); + pDataCols0->numOfRows, pCfg->update); - if (rows2 == 0) { // all filtered out + if (!pCfg->update && rows2 == 0) { // all filtered out *(pCommitIter->pIter) = slIter; ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { - int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2; + int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0, pCfg->update) + rows2; if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { int rows = (rows1 >= rows3) ? rows3 : rows2; tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows); + pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); + ASSERT(rowsRead == rows && rowsRead <= pDataCols->numOfRows); if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; @@ -1606,12 +1606,11 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int round = 0; int dIter = 0; while (true) { - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); + int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, + defaultRowsInBlock, pCfg->update); if (rowsRead == 0) break; - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) - return -1; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (round == 0) { if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; } else { @@ -1633,7 +1632,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows) { + TSKEY maxKey, int maxRows, int8_t update) { int numOfRows = 0; TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; @@ -1649,14 +1648,17 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte if (key1 == INT64_MAX && key2 == INT64_MAX) break; - if (key1 <= key2) { + if ((key1 < key2) || ((!update) && (key1 == key2))) { for (int i = 0; i < pDataCols->numOfCols; i++) { dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } pTarget->numOfRows++; (*iter)++; - if (key1 == key2) tSkipListIterNext(pCommitIter->pIter); + + if ((!update) && (key1 == key2)) { + tSkipListIterNext(pCommitIter->pIter); + } } else { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); @@ -1665,6 +1667,7 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte tdAppendDataRowToDataCol(row, pSchema, pTarget); tSkipListIterNext(pCommitIter->pIter); + if (key1 == key2) (*iter)++; } numOfRows++; From 5023ce7d36e676270f8b871a389ba4b46b3d9d21 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Sep 2020 15:56:40 +0800 Subject: [PATCH 03/58] TD-1438 --- src/common/inc/tdataformat.h | 2 -- src/common/src/tdataformat.c | 11 ++++++-- src/inc/tsdb.h | 1 + src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbMain.c | 5 ++++ src/tsdb/src/tsdbMemTable.c | 14 ++++++---- src/tsdb/src/tsdbRWHelper.c | 53 +++++++++++++++++++----------------- 7 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index cc4afeb3f8..aa39d5b0b2 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -255,8 +255,6 @@ void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, - int limit2, int tRows); // ----------------- K-V data row structure /* diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index e7f40442a0..0c446b6d4f 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -18,6 +18,9 @@ #include "tcoding.h" #include "wchar.h" +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows); + /** * Duplicate the schema and return a new object */ @@ -499,7 +502,9 @@ _err: return -1; } -void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, int limit2, int tRows) { +// src2 data has more priority than src1 +static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2, + int limit2, int tRows) { tdResetDataCols(target); ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows); @@ -509,7 +514,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; - if (key1 <= key2) { + if (key1 < key2) { for (int i = 0; i < src1->numOfCols; i++) { ASSERT(target->cols[i].type == src1->cols[i].type); if (src1->cols[i].len > 0) { @@ -520,7 +525,6 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter1)++; - if (key1 == key2) (*iter2)++; } else { for (int i = 0; i < src2->numOfCols; i++) { ASSERT(target->cols[i].type == src2->cols[i].type); @@ -532,6 +536,7 @@ void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limi target->numOfRows++; (*iter2)++; + if (key1 == key2) (*iter1)++; } } } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 85f9b3bdc7..fe03277763 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -65,6 +65,7 @@ typedef struct { int32_t maxRowsPerFileBlock; // maximum rows per file block int8_t precision; int8_t compression; + int8_t update; } STsdbCfg; // --------- TSDB REPOSITORY USAGE STATISTICS diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 256b8189f8..95b7010038 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -430,7 +430,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem) void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TSKEY* filterKeys, int nFilterKeys); + TSKEY* filterKeys, int nFilterKeys, bool keepDup); static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index a1e6376304..494f17807e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -512,6 +512,9 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } } + // update check + if (pCfg->update != 0) pCfg->update = 1; + return 0; _err: @@ -923,6 +926,7 @@ static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); tlen += taosEncodeFixedI8(buf, pCfg->precision); tlen += taosEncodeFixedI8(buf, pCfg->compression); + tlen += taosEncodeFixedI8(buf, pCfg->update); return tlen; } @@ -939,6 +943,7 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); buf = taosDecodeFixedI8(buf, &(pCfg->precision)); buf = taosDecodeFixedI8(buf, &(pCfg->compression)); + buf = taosDecodeFixedI8(buf, &(pCfg->update)); return buf; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0b7cc4dca4..c58460e18f 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -278,7 +278,7 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { } int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TSKEY *filterKeys, int nFilterKeys) { + TSKEY *filterKeys, int nFilterKeys, bool keepDup) { ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); if (pIter == NULL) return 0; STSchema *pSchema = NULL; @@ -319,6 +319,10 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey if (!keyFiltered) { if (numOfRows >= maxRowsToRead) break; + numOfRows++; + } + + if (!keyFiltered || keepDup) { if (pCols) { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); @@ -329,8 +333,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey tdAppendDataRowToDataCol(row, pSchema, pCols); } - numOfRows++; - } + } } while (tSkipListIterNext(pIter)); return numOfRows; @@ -422,8 +425,9 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->keyLast = 0; pTableData->numOfRows = 0; - pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, - TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], /*SL_DISCARD_DUP_KEY*/ SL_APPEND_DUP_KEY, tsdbGetTsTupleKey); + pTableData->pData = + tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], + pCfg->update ? SL_APPEND_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index bb0f00ef53..f0017d0a14 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -62,7 +62,7 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx); static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows); + TSKEY maxKey, int maxRows, int8_t update); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -1453,7 +1453,7 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, - pDataCols, NULL, 0); + pDataCols, NULL, 0, pCfg->update); ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { @@ -1474,7 +1474,8 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } else { ASSERT(!pHelper->hasOldLastBlock); tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0); + int rowsRead = + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update); ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; @@ -1516,17 +1517,17 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); int rows1 = defaultRowsInBlock - pCompBlock->numOfRows; - int rows2 = - tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows); - if (rows2 == 0) { // all data filtered out + int rows2 = tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, + pDataCols0->numOfRows, pCfg->update); + if (!pCfg->update && rows2 == 0) { // all data filtered out *(pCommitIter->pIter) = slIter; } else { if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows2 && rowsRead == pDataCols->numOfRows); + pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); + ASSERT(rowsRead == rows2 && rowsRead <= pDataCols->numOfRows && pDataCols->numOfRows > 0); if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; tblkIdx++; @@ -1535,9 +1536,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int round = 0; int dIter = 0; while (true) { - tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, defaultRowsInBlock); + int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, + defaultRowsInBlock, pCfg->update); if (rowsRead == 0) break; if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; @@ -1561,8 +1561,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (keyFirst < blkKeyFirst) { while (true) { tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, NULL, 0); + int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, + NULL, 0, pCfg->update); if (rowsRead == 0) break; ASSERT(rowsRead == pDataCols->numOfRows); @@ -1580,21 +1580,21 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, slIter = *(pCommitIter->pIter); int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows); + pDataCols0->numOfRows, pCfg->update); - if (rows2 == 0) { // all filtered out + if (!pCfg->update && rows2 == 0) { // all filtered out *(pCommitIter->pIter) = slIter; ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { - int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0) + rows2; + int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0, pCfg->update) + rows2; if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { int rows = (rows1 >= rows3) ? rows3 : rows2; tdResetDataCols(pDataCols); int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows); - ASSERT(rowsRead == rows && rowsRead == pDataCols->numOfRows); + pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); + ASSERT(rowsRead == rows && rowsRead <= pDataCols->numOfRows); if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) return -1; if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; @@ -1606,12 +1606,11 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, int round = 0; int dIter = 0; while (true) { - int rowsRead = - tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock); + int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, + defaultRowsInBlock, pCfg->update); if (rowsRead == 0) break; - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) - return -1; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (round == 0) { if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; } else { @@ -1633,7 +1632,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, - TSKEY maxKey, int maxRows) { + TSKEY maxKey, int maxRows, int8_t update) { int numOfRows = 0; TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; @@ -1649,14 +1648,17 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte if (key1 == INT64_MAX && key2 == INT64_MAX) break; - if (key1 <= key2) { + if ((key1 < key2) || ((!update) && (key1 == key2))) { for (int i = 0; i < pDataCols->numOfCols; i++) { dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } pTarget->numOfRows++; (*iter)++; - if (key1 == key2) tSkipListIterNext(pCommitIter->pIter); + + if ((!update) && (key1 == key2)) { + tSkipListIterNext(pCommitIter->pIter); + } } else { if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); @@ -1665,6 +1667,7 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte tdAppendDataRowToDataCol(row, pSchema, pTarget); tSkipListIterNext(pCommitIter->pIter); + if (key1 == key2) (*iter)++; } numOfRows++; From aca8e3644b0825ac8c7244a91090fe1492f5df1c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 16 Sep 2020 17:26:08 +0800 Subject: [PATCH 04/58] TD-1439 --- src/tsdb/src/tsdbRead.c | 43 ++++++++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d829a85754..806f0ea828 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -432,7 +432,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { +static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -466,9 +466,15 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order TSKEY r2 = dataRowKey(rimem); if (r1 == r2) { // data ts are duplicated, ignore the data in mem - tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; - return rimem; + if (!update) { + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + tSkipListIterNext(pCheckInfo->iiter); + pCheckInfo->chosen = 0; + return rmem; + } } else { if (ASCENDING_TRAVERSE(order)) { if (r1 < r2) { @@ -522,6 +528,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { } static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { + STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = -1; @@ -535,7 +542,7 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { initTableMemIterator(pHandle, pCheckInfo); } - SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pHandle->order, pCfg->update); if (row == NULL) { return false; } @@ -741,11 +748,12 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){ SQueryFilePos* cur = &pQueryHandle->cur; + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); int32_t code = TSDB_CODE_SUCCESS; /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); assert(cur->pos >= 0 && cur->pos <= binfo.rows); @@ -1208,6 +1216,7 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) { SQueryFilePos* cur = &pQueryHandle->cur; SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; initTableMemIterator(pQueryHandle, pCheckInfo); @@ -1256,7 +1265,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } @@ -1286,7 +1295,22 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveToNextRowInMem(pCheckInfo); } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it - moveToNextRowInMem(pCheckInfo); + if (pCfg->update) { + copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row, numOfCols, pTable); + numOfRows += 1; + if (cur->win.skey == TSKEY_INITIAL_VAL) { + cur->win.skey = key; + } + + cur->win.ekey = key; + cur->lastKey = key + step; + cur->mixBlock = true; + + moveToNextRowInMem(pCheckInfo); + pos += step; + } else { + moveToNextRowInMem(pCheckInfo); + } } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { @@ -1765,13 +1789,14 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbQueryHandle* pQueryHandle) { int numOfRows = 0; int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); STable* pTable = pCheckInfo->pTableObj; do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update); if (row == NULL) { break; } From 3d2a77ea6d4c8e6c22e80dae08098bdf0d4fd8b3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 17 Sep 2020 11:13:06 +0800 Subject: [PATCH 05/58] fix skiplist crash bug --- src/util/src/tskiplist.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 73f6bf0502..c3e8ee41bd 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -466,7 +466,12 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, } SSkipListNode *px = pSkipList->pHead; - for (int i = pSkipList->level - 1; i >= 0; --i) { + for (int i = pNode->level - 1; i >= 0; --i) { + if (i >= pSkipList->level) { + forward[i] = pSkipList->pHead; + continue; + } + SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); while (p != pSkipList->pTail) { pKey = SL_GET_NODE_KEY(pSkipList, p); From 2b1dc53ae1af311ef5763848c3d877785bc30451 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 17 Sep 2020 12:13:19 +0800 Subject: [PATCH 06/58] fix update bug --- src/tsdb/src/tsdbRead.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 806f0ea828..4f902b624b 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -1321,7 +1321,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* assert(end != -1); if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it - moveToNextRowInMem(pCheckInfo); + if (!pCfg->update) { + moveToNextRowInMem(pCheckInfo); + } else { + end -= step; + } } int32_t qstart = 0, qend = 0; From 384b5104fab77cc74998fc105bdc931fa8e31fbd Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 18 Sep 2020 10:51:12 +0800 Subject: [PATCH 07/58] set update --- src/vnode/src/vnodeMain.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index a4e88fb946..9914ecc13b 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -135,6 +135,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.precision = pVnodeCfg->cfg.precision; tsdbCfg.compression = pVnodeCfg->cfg.compression; + tsdbCfg.update = 1; char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); From 4209f666b8f75ff54b11a60242f980be41c1cec8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 21 Sep 2020 12:20:33 +0800 Subject: [PATCH 08/58] [TD-1485] add update options --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 11 ++++++++++ src/inc/taosdef.h | 4 ++++ src/inc/taosmsg.h | 5 ++++- src/mnode/inc/mnodeDef.h | 3 ++- src/mnode/src/mnodeDb.c | 25 +++++++++++++++++++++- src/mnode/src/mnodeVgroup.c | 1 + src/vnode/src/vnodeMain.c | 5 +++-- tests/script/general/http/restful_full.sim | 2 +- 9 files changed, 51 insertions(+), 6 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 77e8b76456..b6a2146c13 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -83,6 +83,7 @@ extern int16_t tsWAL; extern int32_t tsFsyncPeriod; extern int32_t tsReplications; extern int32_t tsQuorum; +extern int32_t tsUpdate; // balance extern int32_t tsEnableBalance; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 7e46f58a93..f03b0f85b4 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -112,6 +112,7 @@ int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL; int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD; int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION; int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION; +int32_t tsUpdate = TSDB_DEFAULT_DB_UPDATE_OPTION; int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; @@ -774,6 +775,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "update"; + cfg.ptr = &tsUpdate; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = TSDB_MIN_DB_UPDATE; + cfg.maxValue = TSDB_MAX_DB_UPDATE; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "mqttHostName"; cfg.ptr = tsMqttHostName; cfg.valType = TAOS_CFG_VTYPE_STRING; diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 9a411d8f85..5d08d1db59 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -350,6 +350,10 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_WAL_LEVEL 2 #define TSDB_DEFAULT_WAL_LEVEL 1 +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 1 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 + #define TSDB_MIN_FSYNC_PERIOD 0 #define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond #define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index e49e2caca1..0c7334afcd 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -531,6 +531,8 @@ typedef struct { int8_t replications; int8_t quorum; int8_t ignoreExist; + int8_t update; + int8_t reserve[9]; } SCMCreateDbMsg, SCMAlterDbMsg; typedef struct { @@ -630,7 +632,8 @@ typedef struct { int8_t replications; int8_t wals; int8_t quorum; - int8_t reserved[16]; + int8_t update; + int8_t reserved[15]; } SMDVnodeCfg; typedef struct { diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index 682986b29f..43fe107666 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -171,7 +171,8 @@ typedef struct { int8_t walLevel; int8_t replications; int8_t quorum; - int8_t reserved[12]; + int8_t update; + int8_t reserved[11]; } SDbCfg; typedef struct SDbObj { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 54c049d242..d3bbaebd5d 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -319,6 +319,11 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { } #endif + if (pCfg->update < TSDB_MIN_DB_UPDATE || pCfg->update > TSDB_MAX_DB_UPDATE) { + mError("invalid db option update:%d valid range: [%d, %d]", pCfg->update, TSDB_MIN_DB_UPDATE, TSDB_MAX_DB_UPDATE); + return TSDB_CODE_MND_INVALID_DB_OPTION; + } + return TSDB_CODE_SUCCESS; } @@ -339,6 +344,7 @@ static void mnodeSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->walLevel < 0) pCfg->walLevel = tsWAL; if (pCfg->replications < 0) pCfg->replications = tsReplications; if (pCfg->quorum < 0) pCfg->quorum = tsQuorum; + if (pCfg->update < 0) pCfg->update = tsUpdate; } static int32_t mnodeCreateDbCb(SMnodeMsg *pMsg, int32_t code) { @@ -391,7 +397,8 @@ static int32_t mnodeCreateDb(SAcctObj *pAcct, SCMCreateDbMsg *pCreate, SMnodeMsg .compression = pCreate->compression, .walLevel = pCreate->walLevel, .replications = pCreate->replications, - .quorum = pCreate->quorum + .quorum = pCreate->quorum, + .update = pCreate->update }; mnodeSetDefaultDbCfg(&pDb->cfg); @@ -610,6 +617,12 @@ static int32_t mnodeGetDbMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; + strcpy(pSchema[cols].name, "update"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); @@ -745,6 +758,10 @@ static int32_t mnodeRetrieveDbs(SShowObj *pShow, char *data, int32_t rows, void STR_WITH_SIZE_TO_VARSTR(pWrite, prec, 2); cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int8_t *)pWrite = pDb->cfg.update; + cols++; + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; if (pDb->status == TSDB_DB_STATUS_READY) { const char *src = "ready"; @@ -842,6 +859,7 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { int8_t replications = pAlter->replications; int8_t quorum = pAlter->quorum; int8_t precision = pAlter->precision; + int8_t update = pAlter->update; terrno = TSDB_CODE_SUCCESS; @@ -944,6 +962,11 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SCMAlterDbMsg *pAlter) { newCfg.quorum = quorum; } + if (update >= 0 && update != pDb->cfg.update) { + mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update); + newCfg.update = update; + } + return newCfg; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index aa6631ff83..a336bb8781 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -837,6 +837,7 @@ static SMDCreateVnodeMsg *mnodeBuildVnodeMsg(SVgObj *pVgroup) { pCfg->replications = (int8_t) pVgroup->numOfVnodes; pCfg->wals = 3; pCfg->quorum = pDb->cfg.quorum; + pCfg->update = pDb->cfg.update; SMDVnodeDesc *pNodes = pVnode->nodes; for (int32_t j = 0; j < pVgroup->numOfVnodes; ++j) { diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 9914ecc13b..7815b32858 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -135,7 +135,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock; tsdbCfg.precision = pVnodeCfg->cfg.precision; tsdbCfg.compression = pVnodeCfg->cfg.compression; - tsdbCfg.update = 1; + tsdbCfg.update = pVnodeCfg->cfg.update; char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); @@ -144,7 +144,8 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { return TSDB_CODE_VND_INIT_FAILED; } - vInfo("vgId:%d, vnode is created, walLevel:%d fsyncPeriod:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel, pVnodeCfg->cfg.fsyncPeriod); + vInfo("vgId:%d, vnode is created, walLevel:%d fsyncPeriod:%d update:%d", pVnodeCfg->cfg.vgId, pVnodeCfg->cfg.walLevel, + pVnodeCfg->cfg.fsyncPeriod, pVnodeCfg->cfg.update); code = vnodeOpen(pVnodeCfg->cfg.vgId, rootDir); return code; diff --git a/tests/script/general/http/restful_full.sim b/tests/script/general/http/restful_full.sim index a02140a419..290787261f 100644 --- a/tests/script/general/http/restful_full.sim +++ b/tests/script/general/http/restful_full.sim @@ -81,7 +81,7 @@ print =============== step2 - no db #11 system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'show databases' 127.0.0.1:7111/rest/sql print 11-> $system_content -if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","status"],"data":[],"rows":0}@ then +if $system_content != @{"status":"succ","head":["name","created_time","ntables","vgroups","replica","quorum","days","keep1,keep2,keep(D)","cache(MB)","blocks","minrows","maxrows","wallevel","fsync","comp","precision","update","status"],"data":[],"rows":0}@ then return -1 endi From cf9ba60ef25215375255cf40264c09bac10a97e0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 21 Sep 2020 17:17:31 +0000 Subject: [PATCH 09/58] support option to db --- src/client/src/tscSQLParser.c | 1 + src/inc/ttokendef.h | 411 +-- src/query/inc/qSqlparser.h | 1 + src/query/inc/sql.y | 3 + src/query/src/qTokenizer.c | 3 +- src/query/src/sql.c | 3394 +++++++++-------------- tests/pytest/alter/db_update_options.py | 71 + 7 files changed, 1617 insertions(+), 2267 deletions(-) create mode 100644 tests/pytest/alter/db_update_options.py diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4bc793e79f..4364659ca6 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5183,6 +5183,7 @@ static void setCreateDBOption(SCMCreateDbMsg* pMsg, SCreateDBInfo* pCreateDb) { pMsg->replications = pCreateDb->replica; pMsg->quorum = pCreateDb->quorum; pMsg->ignoreExist = pCreateDb->ignoreExists; + pMsg->update = pCreateDb->update; } int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql) { diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index c5831a9b8a..41798133e1 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -16,212 +16,213 @@ #ifndef TDENGINE_TTOKENDEF_H #define TDENGINE_TTOKENDEF_H -#define TK_ID 1 -#define TK_BOOL 2 -#define TK_TINYINT 3 -#define TK_SMALLINT 4 -#define TK_INTEGER 5 -#define TK_BIGINT 6 -#define TK_FLOAT 7 -#define TK_DOUBLE 8 -#define TK_STRING 9 -#define TK_TIMESTAMP 10 -#define TK_BINARY 11 -#define TK_NCHAR 12 -#define TK_OR 13 -#define TK_AND 14 -#define TK_NOT 15 -#define TK_EQ 16 -#define TK_NE 17 -#define TK_ISNULL 18 -#define TK_NOTNULL 19 -#define TK_IS 20 -#define TK_LIKE 21 -#define TK_GLOB 22 -#define TK_BETWEEN 23 -#define TK_IN 24 -#define TK_GT 25 -#define TK_GE 26 -#define TK_LT 27 -#define TK_LE 28 -#define TK_BITAND 29 -#define TK_BITOR 30 -#define TK_LSHIFT 31 -#define TK_RSHIFT 32 -#define TK_PLUS 33 -#define TK_MINUS 34 -#define TK_DIVIDE 35 -#define TK_TIMES 36 -#define TK_STAR 37 -#define TK_SLASH 38 -#define TK_REM 39 -#define TK_CONCAT 40 -#define TK_UMINUS 41 -#define TK_UPLUS 42 -#define TK_BITNOT 43 -#define TK_SHOW 44 -#define TK_DATABASES 45 -#define TK_MNODES 46 -#define TK_DNODES 47 -#define TK_ACCOUNTS 48 -#define TK_USERS 49 -#define TK_MODULES 50 -#define TK_QUERIES 51 -#define TK_CONNECTIONS 52 -#define TK_STREAMS 53 -#define TK_VARIABLES 54 -#define TK_SCORES 55 -#define TK_GRANTS 56 -#define TK_VNODES 57 -#define TK_IPTOKEN 58 -#define TK_DOT 59 -#define TK_TABLES 60 -#define TK_STABLES 61 -#define TK_VGROUPS 62 -#define TK_DROP 63 -#define TK_TABLE 64 -#define TK_DATABASE 65 -#define TK_DNODE 66 -#define TK_USER 67 -#define TK_ACCOUNT 68 -#define TK_USE 69 -#define TK_DESCRIBE 70 -#define TK_ALTER 71 -#define TK_PASS 72 -#define TK_PRIVILEGE 73 -#define TK_LOCAL 74 -#define TK_IF 75 -#define TK_EXISTS 76 -#define TK_CREATE 77 -#define TK_PPS 78 -#define TK_TSERIES 79 -#define TK_DBS 80 -#define TK_STORAGE 81 -#define TK_QTIME 82 -#define TK_CONNS 83 -#define TK_STATE 84 -#define TK_KEEP 85 -#define TK_CACHE 86 -#define TK_REPLICA 87 -#define TK_QUORUM 88 -#define TK_DAYS 89 -#define TK_MINROWS 90 -#define TK_MAXROWS 91 -#define TK_BLOCKS 92 -#define TK_CTIME 93 -#define TK_WAL 94 -#define TK_FSYNC 95 -#define TK_COMP 96 -#define TK_PRECISION 97 -#define TK_LP 98 -#define TK_RP 99 -#define TK_TAGS 100 -#define TK_USING 101 -#define TK_AS 102 -#define TK_COMMA 103 -#define TK_NULL 104 -#define TK_SELECT 105 -#define TK_UNION 106 -#define TK_ALL 107 -#define TK_FROM 108 -#define TK_VARIABLE 109 -#define TK_INTERVAL 110 -#define TK_FILL 111 -#define TK_SLIDING 112 -#define TK_ORDER 113 -#define TK_BY 114 -#define TK_ASC 115 -#define TK_DESC 116 -#define TK_GROUP 117 -#define TK_HAVING 118 -#define TK_LIMIT 119 -#define TK_OFFSET 120 -#define TK_SLIMIT 121 -#define TK_SOFFSET 122 -#define TK_WHERE 123 -#define TK_NOW 124 -#define TK_RESET 125 -#define TK_QUERY 126 -#define TK_ADD 127 -#define TK_COLUMN 128 -#define TK_TAG 129 -#define TK_CHANGE 130 -#define TK_SET 131 -#define TK_KILL 132 -#define TK_CONNECTION 133 -#define TK_STREAM 134 -#define TK_COLON 135 -#define TK_ABORT 136 -#define TK_AFTER 137 -#define TK_ATTACH 138 -#define TK_BEFORE 139 -#define TK_BEGIN 140 -#define TK_CASCADE 141 -#define TK_CLUSTER 142 -#define TK_CONFLICT 143 -#define TK_COPY 144 -#define TK_DEFERRED 145 -#define TK_DELIMITERS 146 -#define TK_DETACH 147 -#define TK_EACH 148 -#define TK_END 149 -#define TK_EXPLAIN 150 -#define TK_FAIL 151 -#define TK_FOR 152 -#define TK_IGNORE 153 -#define TK_IMMEDIATE 154 -#define TK_INITIALLY 155 -#define TK_INSTEAD 156 -#define TK_MATCH 157 -#define TK_KEY 158 -#define TK_OF 159 -#define TK_RAISE 160 -#define TK_REPLACE 161 -#define TK_RESTRICT 162 -#define TK_ROW 163 -#define TK_STATEMENT 164 -#define TK_TRIGGER 165 -#define TK_VIEW 166 -#define TK_COUNT 167 -#define TK_SUM 168 -#define TK_AVG 169 -#define TK_MIN 170 -#define TK_MAX 171 -#define TK_FIRST 172 -#define TK_LAST 173 -#define TK_TOP 174 -#define TK_BOTTOM 175 -#define TK_STDDEV 176 -#define TK_PERCENTILE 177 -#define TK_APERCENTILE 178 -#define TK_LEASTSQUARES 179 -#define TK_HISTOGRAM 180 -#define TK_DIFF 181 -#define TK_SPREAD 182 -#define TK_TWA 183 -#define TK_INTERP 184 -#define TK_LAST_ROW 185 -#define TK_RATE 186 -#define TK_IRATE 187 -#define TK_SUM_RATE 188 -#define TK_SUM_IRATE 189 -#define TK_AVG_RATE 190 -#define TK_AVG_IRATE 191 -#define TK_TBID 192 -#define TK_SEMI 193 -#define TK_NONE 194 -#define TK_PREV 195 -#define TK_LINEAR 196 -#define TK_IMPORT 197 -#define TK_METRIC 198 -#define TK_TBNAME 199 -#define TK_JOIN 200 -#define TK_METRICS 201 -#define TK_STABLE 202 -#define TK_INSERT 203 -#define TK_INTO 204 -#define TK_VALUES 205 +#define TK_ID 1 +#define TK_BOOL 2 +#define TK_TINYINT 3 +#define TK_SMALLINT 4 +#define TK_INTEGER 5 +#define TK_BIGINT 6 +#define TK_FLOAT 7 +#define TK_DOUBLE 8 +#define TK_STRING 9 +#define TK_TIMESTAMP 10 +#define TK_BINARY 11 +#define TK_NCHAR 12 +#define TK_OR 13 +#define TK_AND 14 +#define TK_NOT 15 +#define TK_EQ 16 +#define TK_NE 17 +#define TK_ISNULL 18 +#define TK_NOTNULL 19 +#define TK_IS 20 +#define TK_LIKE 21 +#define TK_GLOB 22 +#define TK_BETWEEN 23 +#define TK_IN 24 +#define TK_GT 25 +#define TK_GE 26 +#define TK_LT 27 +#define TK_LE 28 +#define TK_BITAND 29 +#define TK_BITOR 30 +#define TK_LSHIFT 31 +#define TK_RSHIFT 32 +#define TK_PLUS 33 +#define TK_MINUS 34 +#define TK_DIVIDE 35 +#define TK_TIMES 36 +#define TK_STAR 37 +#define TK_SLASH 38 +#define TK_REM 39 +#define TK_CONCAT 40 +#define TK_UMINUS 41 +#define TK_UPLUS 42 +#define TK_BITNOT 43 +#define TK_SHOW 44 +#define TK_DATABASES 45 +#define TK_MNODES 46 +#define TK_DNODES 47 +#define TK_ACCOUNTS 48 +#define TK_USERS 49 +#define TK_MODULES 50 +#define TK_QUERIES 51 +#define TK_CONNECTIONS 52 +#define TK_STREAMS 53 +#define TK_VARIABLES 54 +#define TK_SCORES 55 +#define TK_GRANTS 56 +#define TK_VNODES 57 +#define TK_IPTOKEN 58 +#define TK_DOT 59 +#define TK_TABLES 60 +#define TK_STABLES 61 +#define TK_VGROUPS 62 +#define TK_DROP 63 +#define TK_TABLE 64 +#define TK_DATABASE 65 +#define TK_DNODE 66 +#define TK_USER 67 +#define TK_ACCOUNT 68 +#define TK_USE 69 +#define TK_DESCRIBE 70 +#define TK_ALTER 71 +#define TK_PASS 72 +#define TK_PRIVILEGE 73 +#define TK_LOCAL 74 +#define TK_IF 75 +#define TK_EXISTS 76 +#define TK_CREATE 77 +#define TK_PPS 78 +#define TK_TSERIES 79 +#define TK_DBS 80 +#define TK_STORAGE 81 +#define TK_QTIME 82 +#define TK_CONNS 83 +#define TK_STATE 84 +#define TK_KEEP 85 +#define TK_CACHE 86 +#define TK_REPLICA 87 +#define TK_QUORUM 88 +#define TK_DAYS 89 +#define TK_MINROWS 90 +#define TK_MAXROWS 91 +#define TK_BLOCKS 92 +#define TK_CTIME 93 +#define TK_WAL 94 +#define TK_FSYNC 95 +#define TK_COMP 96 +#define TK_PRECISION 97 +#define TK_UPDATE 98 +#define TK_LP 99 +#define TK_RP 100 +#define TK_TAGS 101 +#define TK_USING 102 +#define TK_AS 103 +#define TK_COMMA 104 +#define TK_NULL 105 +#define TK_SELECT 106 +#define TK_UNION 107 +#define TK_ALL 108 +#define TK_FROM 109 +#define TK_VARIABLE 110 +#define TK_INTERVAL 111 +#define TK_FILL 112 +#define TK_SLIDING 113 +#define TK_ORDER 114 +#define TK_BY 115 +#define TK_ASC 116 +#define TK_DESC 117 +#define TK_GROUP 118 +#define TK_HAVING 119 +#define TK_LIMIT 120 +#define TK_OFFSET 121 +#define TK_SLIMIT 122 +#define TK_SOFFSET 123 +#define TK_WHERE 124 +#define TK_NOW 125 +#define TK_RESET 126 +#define TK_QUERY 127 +#define TK_ADD 128 +#define TK_COLUMN 129 +#define TK_TAG 130 +#define TK_CHANGE 131 +#define TK_SET 132 +#define TK_KILL 133 +#define TK_CONNECTION 134 +#define TK_STREAM 135 +#define TK_COLON 136 +#define TK_ABORT 137 +#define TK_AFTER 138 +#define TK_ATTACH 139 +#define TK_BEFORE 140 +#define TK_BEGIN 141 +#define TK_CASCADE 142 +#define TK_CLUSTER 143 +#define TK_CONFLICT 144 +#define TK_COPY 145 +#define TK_DEFERRED 146 +#define TK_DELIMITERS 147 +#define TK_DETACH 148 +#define TK_EACH 149 +#define TK_END 150 +#define TK_EXPLAIN 151 +#define TK_FAIL 152 +#define TK_FOR 153 +#define TK_IGNORE 154 +#define TK_IMMEDIATE 155 +#define TK_INITIALLY 156 +#define TK_INSTEAD 157 +#define TK_MATCH 158 +#define TK_KEY 159 +#define TK_OF 160 +#define TK_RAISE 161 +#define TK_REPLACE 162 +#define TK_RESTRICT 163 +#define TK_ROW 164 +#define TK_STATEMENT 165 +#define TK_TRIGGER 166 +#define TK_VIEW 167 +#define TK_COUNT 168 +#define TK_SUM 169 +#define TK_AVG 170 +#define TK_MIN 171 +#define TK_MAX 172 +#define TK_FIRST 173 +#define TK_LAST 174 +#define TK_TOP 175 +#define TK_BOTTOM 176 +#define TK_STDDEV 177 +#define TK_PERCENTILE 178 +#define TK_APERCENTILE 179 +#define TK_LEASTSQUARES 180 +#define TK_HISTOGRAM 181 +#define TK_DIFF 182 +#define TK_SPREAD 183 +#define TK_TWA 184 +#define TK_INTERP 185 +#define TK_LAST_ROW 186 +#define TK_RATE 187 +#define TK_IRATE 188 +#define TK_SUM_RATE 189 +#define TK_SUM_IRATE 190 +#define TK_AVG_RATE 191 +#define TK_AVG_IRATE 192 +#define TK_TBID 193 +#define TK_SEMI 194 +#define TK_NONE 195 +#define TK_PREV 196 +#define TK_LINEAR 197 +#define TK_IMPORT 198 +#define TK_METRIC 199 +#define TK_TBNAME 200 +#define TK_JOIN 201 +#define TK_METRICS 202 +#define TK_STABLE 203 +#define TK_INSERT 204 +#define TK_INTO 205 +#define TK_VALUES 206 #define TK_SPACE 300 #define TK_COMMENT 301 diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index d6664577a3..01250f9233 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -129,6 +129,7 @@ typedef struct SCreateDBInfo { int32_t compressionLevel; SStrToken precision; bool ignoreExists; + int8_t update; tVariantList *keep; } SCreateDBInfo; diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 8a5b9d61e2..6fcf647f4f 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -229,6 +229,7 @@ wal(Y) ::= WAL INTEGER(X). { Y = X; } fsync(Y) ::= FSYNC INTEGER(X). { Y = X; } comp(Y) ::= COMP INTEGER(X). { Y = X; } prec(Y) ::= PRECISION STRING(X). { Y = X; } +update(Y) ::= UPDATE INTEGER(X). { Y = X; } %type db_optr {SCreateDBInfo} db_optr(Y) ::= . {setDefaultCreateDbOption(&Y);} @@ -246,6 +247,7 @@ db_optr(Y) ::= db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z db_optr(Y) ::= db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } db_optr(Y) ::= db_optr(Z) prec(X). { Y = Z; Y.precision = X; } db_optr(Y) ::= db_optr(Z) keep(X). { Y = Z; Y.keep = X; } +db_optr(Y) ::= db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } %type alter_db_optr {SCreateDBInfo} alter_db_optr(Y) ::= . { setDefaultCreateDbOption(&Y);} @@ -257,6 +259,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) blocks(X). { Y = Z; Y.numOfBlocks = s alter_db_optr(Y) ::= alter_db_optr(Z) comp(X). { Y = Z; Y.compressionLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) wal(X). { Y = Z; Y.walLevel = strtol(X.z, NULL, 10); } alter_db_optr(Y) ::= alter_db_optr(Z) fsync(X). { Y = Z; Y.fsyncPeriod = strtol(X.z, NULL, 10); } +alter_db_optr(Y) ::= alter_db_optr(Z) update(X). { Y = Z; Y.update = strtol(X.z, NULL, 10); } %type typename {TAOS_FIELD} typename(A) ::= ids(X). { diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 0c9f92786f..0c6ee25f13 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -155,6 +155,7 @@ static SKeyword keywordTable[] = { {"INSERT", TK_INSERT}, {"INTO", TK_INTO}, {"VALUES", TK_VALUES}, + {"UPDATE", TK_UPDATE}, {"RESET", TK_RESET}, {"QUERY", TK_QUERY}, {"ADD", TK_ADD}, @@ -661,4 +662,4 @@ bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, l void taosCleanupKeywordsTable() { taosHashCleanup(KeywordHashTable); -} \ No newline at end of file +} diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 3145f39a5e..be5bcaae82 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -1,30 +1,9 @@ -/* -** 2000-05-29 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Driver template for the LEMON parser generator. -** -** The "lemon" program processes an LALR(1) input grammar file, then uses -** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the -** interstitial "-" characters) contained in this template is changed into -** the value of the %name directive from the grammar. Otherwise, the content -** of this template is copied straight through into the generate parser -** source file. -** -** The following is the concatenation of all %include directives from the -** input grammar file: +/* Driver template for the LEMON parser generator. +** The author disclaims copyright to this source code. */ +/* First off, code is included that follows the "include" declaration +** in the input grammar file. */ #include -#include -/************ Begin %include sections from the grammar ************************/ #include #include @@ -37,119 +16,95 @@ #include "ttokendef.h" #include "tutil.h" #include "tvariant.h" -/**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ - -/* The next sections is a series of control #defines. -** various aspects of the generated parser. -** YYCODETYPE is the data type used to store the integer codes -** that represent terminal and non-terminal symbols. -** "unsigned char" is used if there are fewer than -** 256 symbols. Larger types otherwise. -** YYNOCODE is a number of type YYCODETYPE that is not used for -** any terminal or nonterminal symbol. -** YYFALLBACK If defined, this indicates that one or more tokens -** (also known as: "terminal symbols") have fall-back -** values which should be used if the original symbol -** would not parse. This permits keywords to sometimes -** be used as identifiers, for example. -** YYACTIONTYPE is the data type used for "action codes" - numbers -** that indicate what to do in response to the next -** token. -** ParseTOKENTYPE is the data type used for minor type for terminal -** symbols. Background: A "minor type" is a semantic -** value associated with a terminal or non-terminal -** symbols. For example, for an "ID" terminal symbol, -** the minor type might be the name of the identifier. -** Each non-terminal can have a different minor type. -** Terminal symbols all have the same minor type, though. -** This macros defines the minor type for terminal -** symbols. -** YYMINORTYPE is the data type used for all minor types. -** This is typically a union of many types, one of -** which is ParseTOKENTYPE. The entry in the union -** for terminal symbols is called "yy0". -** YYSTACKDEPTH is the maximum depth of the parser's stack. If -** zero the stack is dynamically sized using realloc() -** ParseARG_SDECL A static variable declaration for the %extra_argument -** ParseARG_PDECL A parameter declaration for the %extra_argument -** ParseARG_PARAM Code to pass %extra_argument as a subroutine parameter -** ParseARG_STORE Code to store %extra_argument into yypParser -** ParseARG_FETCH Code to extract %extra_argument from yypParser -** ParseCTX_* As ParseARG_ except for %extra_context -** YYERRORSYMBOL is the code number of the error symbol. If not -** defined, then do no error processing. -** YYNSTATE the combined number of states. -** YYNRULE the number of rules in the grammar -** YYNTOKEN Number of terminal symbols -** YY_MAX_SHIFT Maximum value for shift actions -** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions -** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_ERROR_ACTION The yy_action[] code for syntax error -** YY_ACCEPT_ACTION The yy_action[] code for accept -** YY_NO_ACTION The yy_action[] code for no-op -** YY_MIN_REDUCE Minimum value for reduce actions -** YY_MAX_REDUCE Maximum value for reduce actions +/* Next is all token values, in a form suitable for use by makeheaders. +** This section will be null unless lemon is run with the -m switch. +*/ +/* +** These constants (all generated automatically by the parser generator) +** specify the various kinds of tokens (terminals) that the parser +** understands. +** +** Each symbol here is a terminal symbol in the grammar. +*/ +/* Make sure the INTERFACE macro is defined. */ #ifndef INTERFACE # define INTERFACE 1 #endif -/************* Begin control #defines *****************************************/ +/* The next thing included is series of defines which control +** various aspects of the generated parser. +** YYCODETYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 terminals +** and nonterminals. "int" is used otherwise. +** YYNOCODE is a number of type YYCODETYPE which corresponds +** to no legal terminal or nonterminal number. This +** number is used to fill in empty slots of the hash +** table. +** YYFALLBACK If defined, this indicates that one or more tokens +** have fall-back values which should be used if the +** original value of the token will not parse. +** YYACTIONTYPE is the data type used for storing terminal +** and nonterminal numbers. "unsigned char" is +** used if there are fewer than 250 rules and +** states combined. "int" is used otherwise. +** ParseTOKENTYPE is the data type used for minor tokens given +** directly to the parser from the tokenizer. +** YYMINORTYPE is the data type used for all minor tokens. +** This is typically a union of many types, one of +** which is ParseTOKENTYPE. The entry in the union +** for base tokens is called "yy0". +** YYSTACKDEPTH is the maximum depth of the parser's stack. If +** zero the stack is dynamically sized using realloc() +** ParseARG_SDECL A static variable declaration for the %extra_argument +** ParseARG_PDECL A parameter declaration for the %extra_argument +** ParseARG_STORE Code to store %extra_argument into yypParser +** ParseARG_FETCH Code to extract %extra_argument from yypParser +** YYNSTATE the combined number of states. +** YYNRULE the number of rules in the grammar +** YYERRORSYMBOL is the code number of the error symbol. If not +** defined, then do no error processing. +*/ #define YYCODETYPE unsigned short int -#define YYNOCODE 270 +#define YYNOCODE 274 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; - int yy112; - SCreateDBInfo yy118; - tVariantList* yy156; - tSQLExprList* yy158; - tSQLExpr* yy190; - SSubclauseInfo* yy333; - SIntervalVal yy340; - TAOS_FIELD yy343; - int64_t yy369; - SCreateTableSQL* yy398; - SLimitVal yy414; - SQuerySQL* yy444; - SCreateAcctSQL yy479; - tVariant yy506; - tFieldList* yy511; + int yy46; + tSQLExpr* yy64; + tVariant yy134; + SCreateAcctSQL yy149; + int64_t yy207; + SLimitVal yy216; + TAOS_FIELD yy223; + SSubclauseInfo* yy231; + SCreateDBInfo yy268; + tSQLExprList* yy290; + SQuerySQL* yy414; + SCreateTableSQL* yy470; + tVariantList* yy498; + tFieldList* yy523; + SIntervalVal yy532; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif #define ParseARG_SDECL SSqlInfo* pInfo; #define ParseARG_PDECL ,SSqlInfo* pInfo -#define ParseARG_PARAM ,pInfo -#define ParseARG_FETCH SSqlInfo* pInfo=yypParser->pInfo; -#define ParseARG_STORE yypParser->pInfo=pInfo; -#define ParseCTX_SDECL -#define ParseCTX_PDECL -#define ParseCTX_PARAM -#define ParseCTX_FETCH -#define ParseCTX_STORE +#define ParseARG_FETCH SSqlInfo* pInfo = yypParser->pInfo +#define ParseARG_STORE yypParser->pInfo = pInfo +#define YYNSTATE 414 +#define YYNRULE 231 #define YYFALLBACK 1 -#define YYNSTATE 248 -#define YYNRULE 228 -#define YYNRULE_WITH_ACTION 228 -#define YYNTOKEN 206 -#define YY_MAX_SHIFT 247 -#define YY_MIN_SHIFTREDUCE 410 -#define YY_MAX_SHIFTREDUCE 637 -#define YY_ERROR_ACTION 638 -#define YY_ACCEPT_ACTION 639 -#define YY_NO_ACTION 640 -#define YY_MIN_REDUCE 641 -#define YY_MAX_REDUCE 868 -/************* End control #defines *******************************************/ -#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) +#define YY_NO_ACTION (YYNSTATE+YYNRULE+2) +#define YY_ACCEPT_ACTION (YYNSTATE+YYNRULE+1) +#define YY_ERROR_ACTION (YYNSTATE+YYNRULE) + +/* The yyzerominor constant is used to initialize instances of +** YYMINORTYPE objects to zero. */ +static const YYMINORTYPE yyzerominor = { 0 }; /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -172,35 +127,33 @@ typedef union { ** Suppose the action integer is N. Then the action is determined as ** follows ** -** 0 <= N <= YY_MAX_SHIFT Shift N. That is, push the lookahead +** 0 <= N < YYNSTATE Shift N. That is, push the lookahead ** token onto the stack and goto state N. ** -** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then -** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. +** YYNSTATE <= N < YYNSTATE+YYNRULE Reduce by rule N-YYNSTATE. ** -** N == YY_ERROR_ACTION A syntax error has occurred. +** N == YYNSTATE+YYNRULE A syntax error has occurred. ** -** N == YY_ACCEPT_ACTION The parser accepts its input. +** N == YYNSTATE+YYNRULE+1 The parser accepts its input. ** -** N == YY_NO_ACTION No such action. Denotes unused +** N == YYNSTATE+YYNRULE+2 No such action. Denotes unused ** slots in the yy_action[] table. ** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE -** ** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as either: +** Given state S and lookahead X, the action is computed as ** -** (A) N = yy_action[ yy_shift_ofst[S] + X ] -** (B) N = yy_default[S] +** yy_action[ yy_shift_ofst[S] + X ] ** -** The (A) formula is preferred. The B formula is used instead if -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. +** If the index value yy_shift_ofst[S]+X is out of range or if the value +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] +** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table +** and that yy_default[S] should be used instead. ** -** The formulas above are for computing the action when the lookahead is +** The formula above is for computing the action when the lookahead is ** a terminal symbol. If the lookahead is a non-terminal (as occurs after ** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array. +** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of +** YY_SHIFT_USE_DFLT. ** ** The following are the tables generated in this section: ** @@ -212,226 +165,244 @@ typedef union { ** yy_reduce_ofst[] For each state, the offset into yy_action for ** shifting non-terminals after a reduce. ** yy_default[] Default action for each state. -** -*********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (560) +*/ +#define YY_ACTTAB_COUNT (665) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 741, 451, 11, 739, 740, 639, 247, 451, 742, 452, - /* 10 */ 744, 745, 743, 35, 36, 452, 37, 38, 156, 245, - /* 20 */ 167, 29, 138, 137, 202, 41, 39, 43, 40, 106, - /* 30 */ 517, 162, 856, 34, 33, 782, 138, 32, 31, 30, - /* 40 */ 35, 36, 771, 37, 38, 161, 856, 167, 29, 771, - /* 50 */ 106, 202, 41, 39, 43, 40, 187, 159, 223, 222, - /* 60 */ 34, 33, 138, 157, 32, 31, 30, 35, 36, 451, - /* 70 */ 37, 38, 855, 142, 167, 29, 760, 452, 202, 41, - /* 80 */ 39, 43, 40, 199, 78, 60, 779, 34, 33, 234, - /* 90 */ 234, 32, 31, 30, 21, 41, 39, 43, 40, 32, - /* 100 */ 31, 30, 56, 34, 33, 852, 808, 32, 31, 30, - /* 110 */ 21, 21, 106, 411, 412, 413, 414, 415, 416, 417, - /* 120 */ 418, 419, 420, 421, 422, 246, 591, 171, 36, 757, - /* 130 */ 37, 38, 225, 50, 167, 29, 21, 62, 202, 41, - /* 140 */ 39, 43, 40, 172, 221, 757, 757, 34, 33, 27, - /* 150 */ 51, 32, 31, 30, 8, 37, 38, 63, 116, 167, - /* 160 */ 29, 101, 758, 202, 41, 39, 43, 40, 809, 226, - /* 170 */ 197, 757, 34, 33, 170, 851, 32, 31, 30, 16, - /* 180 */ 214, 240, 239, 213, 212, 211, 238, 210, 237, 236, - /* 190 */ 235, 209, 737, 760, 725, 726, 727, 728, 729, 730, - /* 200 */ 731, 732, 733, 734, 735, 736, 166, 604, 12, 241, - /* 210 */ 595, 17, 598, 190, 601, 559, 166, 604, 26, 103, - /* 220 */ 595, 597, 598, 600, 601, 34, 33, 151, 760, 32, - /* 230 */ 31, 30, 21, 90, 89, 145, 572, 573, 163, 164, - /* 240 */ 173, 150, 201, 76, 80, 85, 88, 79, 163, 164, - /* 250 */ 166, 604, 549, 82, 595, 106, 598, 100, 601, 244, - /* 260 */ 243, 97, 17, 16, 26, 240, 239, 756, 680, 26, - /* 270 */ 238, 129, 237, 236, 235, 119, 120, 70, 66, 69, - /* 280 */ 203, 165, 163, 164, 689, 533, 180, 129, 530, 186, - /* 290 */ 531, 850, 532, 184, 183, 596, 153, 599, 133, 131, - /* 300 */ 93, 92, 91, 42, 174, 546, 681, 220, 219, 129, - /* 310 */ 18, 61, 541, 42, 603, 593, 175, 176, 563, 189, - /* 320 */ 3, 47, 46, 537, 603, 538, 564, 623, 605, 602, - /* 330 */ 14, 13, 13, 154, 523, 75, 74, 522, 46, 602, - /* 340 */ 48, 22, 207, 535, 155, 536, 22, 42, 10, 9, - /* 350 */ 140, 594, 87, 86, 141, 143, 144, 148, 603, 149, - /* 360 */ 147, 136, 146, 139, 865, 759, 819, 818, 168, 607, - /* 370 */ 815, 814, 169, 602, 751, 224, 781, 773, 786, 788, - /* 380 */ 102, 801, 117, 800, 115, 118, 26, 534, 188, 691, - /* 390 */ 208, 134, 24, 217, 688, 218, 864, 72, 863, 861, - /* 400 */ 121, 95, 709, 25, 23, 135, 678, 81, 558, 676, - /* 410 */ 83, 191, 84, 674, 158, 673, 195, 177, 130, 671, - /* 420 */ 52, 670, 770, 669, 49, 44, 668, 107, 108, 200, - /* 430 */ 667, 194, 659, 132, 665, 663, 198, 196, 192, 661, - /* 440 */ 28, 57, 58, 802, 216, 77, 227, 228, 229, 230, - /* 450 */ 231, 232, 205, 233, 242, 53, 637, 178, 179, 636, - /* 460 */ 152, 64, 67, 182, 181, 672, 635, 628, 94, 96, - /* 470 */ 185, 666, 124, 55, 123, 710, 122, 125, 126, 128, - /* 480 */ 127, 1, 2, 189, 755, 543, 59, 560, 111, 109, - /* 490 */ 112, 110, 104, 113, 114, 160, 19, 193, 5, 565, - /* 500 */ 105, 6, 606, 4, 20, 15, 204, 7, 608, 65, - /* 510 */ 206, 492, 488, 486, 485, 484, 481, 455, 215, 68, - /* 520 */ 45, 71, 22, 519, 73, 518, 516, 54, 476, 474, - /* 530 */ 466, 472, 468, 470, 464, 462, 491, 490, 489, 487, - /* 540 */ 483, 482, 46, 453, 426, 424, 641, 640, 640, 640, - /* 550 */ 640, 640, 640, 640, 640, 640, 640, 640, 98, 99, + /* 0 */ 398, 76, 80, 85, 88, 79, 341, 173, 397, 263, + /* 10 */ 180, 82, 35, 36, 18, 37, 38, 184, 183, 167, + /* 20 */ 29, 87, 86, 202, 41, 39, 43, 40, 10, 9, + /* 30 */ 646, 248, 34, 33, 223, 222, 32, 31, 30, 35, + /* 40 */ 36, 8, 37, 38, 63, 116, 167, 29, 101, 21, + /* 50 */ 202, 41, 39, 43, 40, 245, 244, 97, 414, 34, + /* 60 */ 33, 75, 74, 32, 31, 30, 35, 36, 256, 37, + /* 70 */ 38, 401, 174, 167, 29, 220, 219, 202, 41, 39, + /* 80 */ 43, 40, 187, 226, 207, 337, 34, 33, 22, 398, + /* 90 */ 32, 31, 30, 36, 307, 37, 38, 397, 22, 167, + /* 100 */ 29, 190, 56, 202, 41, 39, 43, 40, 32, 31, + /* 110 */ 30, 361, 34, 33, 363, 362, 32, 31, 30, 360, + /* 120 */ 306, 358, 357, 359, 46, 356, 297, 400, 133, 131, + /* 130 */ 93, 92, 91, 413, 412, 411, 410, 409, 408, 407, + /* 140 */ 406, 405, 404, 403, 402, 247, 375, 138, 374, 106, + /* 150 */ 37, 38, 50, 99, 167, 29, 161, 279, 202, 41, + /* 160 */ 39, 43, 40, 106, 373, 165, 372, 34, 33, 51, + /* 170 */ 203, 32, 31, 30, 311, 98, 323, 322, 321, 320, + /* 180 */ 319, 318, 317, 316, 315, 314, 313, 312, 310, 16, + /* 190 */ 214, 241, 240, 213, 212, 211, 239, 210, 238, 237, + /* 200 */ 236, 209, 235, 166, 285, 3, 270, 294, 197, 289, + /* 210 */ 293, 288, 292, 166, 285, 47, 78, 294, 199, 289, + /* 220 */ 60, 288, 234, 166, 285, 12, 291, 294, 290, 289, + /* 230 */ 46, 288, 21, 255, 48, 163, 164, 34, 33, 257, + /* 240 */ 189, 32, 31, 30, 151, 163, 164, 296, 370, 201, + /* 250 */ 90, 89, 145, 284, 392, 163, 164, 13, 150, 369, + /* 260 */ 299, 41, 39, 43, 40, 368, 221, 138, 337, 34, + /* 270 */ 33, 100, 21, 32, 31, 30, 162, 279, 26, 16, + /* 280 */ 17, 241, 240, 295, 275, 274, 239, 26, 238, 237, + /* 290 */ 236, 280, 235, 377, 21, 13, 380, 398, 379, 266, + /* 300 */ 378, 42, 106, 14, 21, 397, 172, 186, 337, 267, + /* 310 */ 106, 42, 286, 46, 153, 119, 120, 70, 66, 69, + /* 320 */ 225, 42, 286, 62, 175, 176, 242, 287, 171, 265, + /* 330 */ 337, 354, 286, 103, 129, 27, 331, 287, 355, 129, + /* 340 */ 343, 129, 256, 17, 138, 170, 371, 287, 159, 258, + /* 350 */ 26, 338, 156, 246, 278, 367, 157, 366, 365, 61, + /* 360 */ 364, 353, 352, 351, 350, 371, 349, 271, 371, 348, + /* 370 */ 347, 346, 22, 54, 340, 342, 339, 73, 71, 45, + /* 380 */ 68, 330, 215, 329, 328, 327, 326, 325, 65, 206, + /* 390 */ 204, 7, 324, 15, 4, 298, 376, 277, 6, 5, + /* 400 */ 105, 260, 20, 19, 268, 193, 160, 264, 104, 185, + /* 410 */ 55, 253, 252, 251, 59, 182, 181, 250, 179, 189, + /* 420 */ 249, 178, 2, 1, 96, 393, 243, 95, 94, 386, + /* 430 */ 232, 123, 127, 128, 305, 228, 230, 126, 231, 233, + /* 440 */ 124, 125, 229, 152, 122, 53, 67, 205, 216, 344, + /* 450 */ 64, 26, 114, 227, 28, 113, 77, 192, 198, 194, + /* 460 */ 44, 196, 273, 52, 49, 195, 276, 58, 57, 399, + /* 470 */ 396, 647, 395, 132, 394, 391, 390, 112, 389, 111, + /* 480 */ 388, 110, 387, 109, 130, 108, 269, 188, 200, 177, + /* 490 */ 107, 647, 385, 308, 234, 115, 304, 384, 84, 254, + /* 500 */ 224, 647, 647, 158, 191, 647, 647, 647, 647, 272, + /* 510 */ 647, 83, 647, 169, 383, 303, 647, 647, 647, 302, + /* 520 */ 168, 647, 301, 647, 81, 382, 135, 647, 23, 25, + /* 530 */ 345, 121, 336, 335, 72, 334, 218, 647, 332, 217, + /* 540 */ 24, 381, 134, 208, 309, 118, 117, 102, 262, 261, + /* 550 */ 259, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 560 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 570 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 580 */ 647, 647, 647, 647, 647, 647, 647, 300, 647, 647, + /* 590 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 600 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 610 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 647, + /* 620 */ 647, 647, 647, 333, 647, 647, 647, 647, 647, 647, + /* 630 */ 647, 647, 647, 647, 647, 647, 647, 647, 647, 139, + /* 640 */ 647, 647, 647, 647, 647, 647, 146, 647, 647, 136, + /* 650 */ 147, 149, 148, 144, 143, 141, 140, 155, 154, 283, + /* 660 */ 282, 281, 142, 137, 11, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 225, 1, 259, 228, 229, 206, 207, 1, 233, 9, - /* 10 */ 235, 236, 237, 13, 14, 9, 16, 17, 208, 209, - /* 20 */ 20, 21, 259, 259, 24, 25, 26, 27, 28, 209, - /* 30 */ 5, 268, 269, 33, 34, 209, 259, 37, 38, 39, - /* 40 */ 13, 14, 243, 16, 17, 268, 269, 20, 21, 243, - /* 50 */ 209, 24, 25, 26, 27, 28, 257, 226, 33, 34, - /* 60 */ 33, 34, 259, 257, 37, 38, 39, 13, 14, 1, - /* 70 */ 16, 17, 269, 259, 20, 21, 245, 9, 24, 25, - /* 80 */ 26, 27, 28, 263, 72, 265, 260, 33, 34, 78, - /* 90 */ 78, 37, 38, 39, 209, 25, 26, 27, 28, 37, - /* 100 */ 38, 39, 102, 33, 34, 259, 265, 37, 38, 39, - /* 110 */ 209, 209, 209, 45, 46, 47, 48, 49, 50, 51, - /* 120 */ 52, 53, 54, 55, 56, 57, 99, 242, 14, 244, - /* 130 */ 16, 17, 209, 103, 20, 21, 209, 246, 24, 25, - /* 140 */ 26, 27, 28, 242, 242, 244, 244, 33, 34, 258, - /* 150 */ 120, 37, 38, 39, 98, 16, 17, 101, 102, 20, - /* 160 */ 21, 209, 239, 24, 25, 26, 27, 28, 265, 242, - /* 170 */ 267, 244, 33, 34, 226, 259, 37, 38, 39, 85, - /* 180 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, - /* 190 */ 96, 97, 225, 245, 227, 228, 229, 230, 231, 232, - /* 200 */ 233, 234, 235, 236, 237, 238, 1, 2, 44, 226, - /* 210 */ 5, 98, 7, 261, 9, 99, 1, 2, 105, 103, - /* 220 */ 5, 5, 7, 7, 9, 33, 34, 63, 245, 37, - /* 230 */ 38, 39, 209, 69, 70, 71, 115, 116, 33, 34, - /* 240 */ 63, 77, 37, 64, 65, 66, 67, 68, 33, 34, - /* 250 */ 1, 2, 37, 74, 5, 209, 7, 98, 9, 60, - /* 260 */ 61, 62, 98, 85, 105, 87, 88, 244, 213, 105, - /* 270 */ 92, 216, 94, 95, 96, 64, 65, 66, 67, 68, - /* 280 */ 15, 59, 33, 34, 213, 2, 126, 216, 5, 125, - /* 290 */ 7, 259, 9, 133, 134, 5, 132, 7, 64, 65, - /* 300 */ 66, 67, 68, 98, 127, 103, 213, 130, 131, 216, - /* 310 */ 108, 265, 99, 98, 109, 1, 33, 34, 99, 106, - /* 320 */ 98, 103, 103, 5, 109, 7, 99, 99, 99, 124, - /* 330 */ 103, 103, 103, 259, 99, 128, 129, 99, 103, 124, - /* 340 */ 122, 103, 99, 5, 259, 7, 103, 98, 128, 129, - /* 350 */ 259, 37, 72, 73, 259, 259, 259, 259, 109, 259, - /* 360 */ 259, 259, 259, 259, 245, 245, 240, 240, 240, 104, - /* 370 */ 240, 240, 240, 124, 241, 240, 209, 243, 209, 209, - /* 380 */ 209, 266, 209, 266, 247, 209, 105, 104, 243, 209, - /* 390 */ 209, 209, 209, 209, 209, 209, 209, 209, 209, 209, - /* 400 */ 209, 59, 209, 209, 209, 209, 209, 209, 109, 209, - /* 410 */ 209, 262, 209, 209, 262, 209, 262, 209, 209, 209, - /* 420 */ 119, 209, 256, 209, 121, 118, 209, 255, 254, 113, - /* 430 */ 209, 111, 209, 209, 209, 209, 117, 112, 110, 209, - /* 440 */ 123, 210, 210, 210, 75, 84, 83, 49, 80, 82, - /* 450 */ 53, 81, 210, 79, 75, 210, 5, 135, 5, 5, - /* 460 */ 210, 214, 214, 5, 135, 210, 5, 86, 211, 211, - /* 470 */ 126, 210, 218, 107, 222, 224, 223, 221, 219, 217, - /* 480 */ 220, 215, 212, 106, 243, 99, 103, 99, 251, 253, - /* 490 */ 250, 252, 98, 249, 248, 1, 103, 98, 114, 99, - /* 500 */ 98, 114, 99, 98, 103, 98, 100, 98, 104, 72, - /* 510 */ 100, 9, 5, 5, 5, 5, 5, 76, 15, 72, - /* 520 */ 16, 129, 103, 5, 129, 5, 99, 98, 5, 5, - /* 530 */ 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, - /* 540 */ 5, 5, 103, 76, 59, 58, 0, 270, 270, 270, - /* 550 */ 270, 270, 270, 270, 270, 270, 270, 270, 21, 21, - /* 560 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 570 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 580 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 590 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 600 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 610 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 620 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 630 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 640 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 650 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 660 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 670 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 680 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 690 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 700 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 710 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 720 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 730 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 740 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 750 */ 270, 270, 270, 270, 270, 270, 270, 270, 270, 270, - /* 760 */ 270, 270, 270, 270, 270, 270, + /* 0 */ 1, 64, 65, 66, 67, 68, 5, 63, 9, 104, + /* 10 */ 127, 74, 13, 14, 109, 16, 17, 134, 135, 20, + /* 20 */ 21, 72, 73, 24, 25, 26, 27, 28, 129, 130, + /* 30 */ 208, 209, 33, 34, 33, 34, 37, 38, 39, 13, + /* 40 */ 14, 99, 16, 17, 102, 103, 20, 21, 211, 211, + /* 50 */ 24, 25, 26, 27, 28, 60, 61, 62, 0, 33, + /* 60 */ 34, 129, 130, 37, 38, 39, 13, 14, 246, 16, + /* 70 */ 17, 58, 128, 20, 21, 131, 132, 24, 25, 26, + /* 80 */ 27, 28, 260, 245, 100, 247, 33, 34, 104, 1, + /* 90 */ 37, 38, 39, 14, 100, 16, 17, 9, 104, 20, + /* 100 */ 21, 264, 103, 24, 25, 26, 27, 28, 37, 38, + /* 110 */ 39, 227, 33, 34, 230, 231, 37, 38, 39, 235, + /* 120 */ 100, 237, 238, 239, 104, 241, 100, 59, 64, 65, + /* 130 */ 66, 67, 68, 45, 46, 47, 48, 49, 50, 51, + /* 140 */ 52, 53, 54, 55, 56, 57, 5, 262, 7, 211, + /* 150 */ 16, 17, 104, 21, 20, 21, 271, 272, 24, 25, + /* 160 */ 26, 27, 28, 211, 5, 59, 7, 33, 34, 121, + /* 170 */ 15, 37, 38, 39, 227, 21, 229, 230, 231, 232, + /* 180 */ 233, 234, 235, 236, 237, 238, 239, 240, 241, 85, + /* 190 */ 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, + /* 200 */ 96, 97, 98, 1, 2, 99, 268, 5, 270, 7, + /* 210 */ 5, 9, 7, 1, 2, 104, 72, 5, 266, 7, + /* 220 */ 268, 9, 78, 1, 2, 44, 5, 5, 7, 7, + /* 230 */ 104, 9, 211, 100, 123, 33, 34, 33, 34, 37, + /* 240 */ 107, 37, 38, 39, 63, 33, 34, 1, 5, 37, + /* 250 */ 69, 70, 71, 100, 76, 33, 34, 104, 77, 5, + /* 260 */ 105, 25, 26, 27, 28, 5, 245, 262, 247, 33, + /* 270 */ 34, 99, 211, 37, 38, 39, 271, 272, 106, 85, + /* 280 */ 99, 87, 88, 37, 116, 117, 92, 106, 94, 95, + /* 290 */ 96, 100, 98, 2, 211, 104, 5, 1, 7, 100, + /* 300 */ 9, 99, 211, 104, 211, 9, 245, 126, 247, 100, + /* 310 */ 211, 99, 110, 104, 133, 64, 65, 66, 67, 68, + /* 320 */ 211, 99, 110, 249, 33, 34, 228, 125, 245, 100, + /* 330 */ 247, 215, 110, 104, 218, 261, 215, 125, 215, 218, + /* 340 */ 247, 218, 246, 99, 262, 228, 248, 125, 228, 211, + /* 350 */ 106, 242, 210, 211, 272, 5, 260, 5, 5, 268, + /* 360 */ 5, 5, 5, 5, 5, 248, 5, 268, 248, 5, + /* 370 */ 5, 5, 104, 99, 5, 100, 5, 130, 130, 16, + /* 380 */ 72, 76, 15, 5, 5, 5, 5, 5, 72, 101, + /* 390 */ 101, 99, 9, 99, 99, 105, 105, 100, 115, 115, + /* 400 */ 99, 263, 104, 104, 100, 99, 1, 100, 99, 127, + /* 410 */ 108, 100, 86, 5, 104, 5, 136, 5, 5, 107, + /* 420 */ 5, 136, 214, 217, 213, 212, 75, 59, 213, 212, + /* 430 */ 81, 224, 222, 219, 246, 49, 82, 221, 53, 79, + /* 440 */ 220, 223, 80, 212, 225, 212, 216, 212, 75, 226, + /* 450 */ 216, 106, 251, 83, 124, 252, 84, 111, 118, 112, + /* 460 */ 119, 113, 212, 120, 122, 265, 269, 212, 212, 211, + /* 470 */ 211, 273, 211, 211, 211, 211, 211, 253, 211, 254, + /* 480 */ 211, 255, 211, 256, 211, 257, 110, 246, 114, 211, + /* 490 */ 258, 273, 211, 244, 78, 250, 259, 211, 211, 246, + /* 500 */ 243, 273, 273, 265, 265, 273, 273, 273, 273, 269, + /* 510 */ 273, 211, 273, 243, 211, 243, 273, 273, 273, 243, + /* 520 */ 243, 273, 243, 273, 211, 211, 211, 273, 211, 211, + /* 530 */ 211, 211, 211, 211, 211, 211, 211, 273, 211, 211, + /* 540 */ 211, 248, 211, 211, 211, 211, 211, 211, 211, 211, + /* 550 */ 211, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 560 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 570 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 580 */ 273, 273, 273, 273, 273, 273, 273, 243, 273, 273, + /* 590 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 600 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 610 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 273, + /* 620 */ 273, 273, 273, 248, 273, 273, 273, 273, 273, 273, + /* 630 */ 273, 273, 273, 273, 273, 273, 273, 273, 273, 262, + /* 640 */ 273, 273, 273, 273, 273, 273, 262, 273, 273, 262, + /* 650 */ 262, 262, 262, 262, 262, 262, 262, 262, 262, 262, + /* 660 */ 262, 262, 262, 262, 262, }; -#define YY_SHIFT_COUNT (247) -#define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (546) -static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 164, 94, 178, 205, 249, 6, 6, 6, 6, 6, - /* 10 */ 6, 0, 68, 249, 283, 283, 283, 113, 6, 6, - /* 20 */ 6, 6, 6, 12, 11, 11, 560, 215, 249, 249, - /* 30 */ 249, 249, 249, 249, 249, 249, 249, 249, 249, 249, - /* 40 */ 249, 249, 249, 249, 249, 283, 283, 25, 25, 25, - /* 50 */ 25, 25, 25, 56, 25, 159, 6, 6, 6, 6, - /* 60 */ 121, 121, 202, 6, 6, 6, 6, 6, 6, 6, - /* 70 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 80 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 90 */ 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, - /* 100 */ 281, 342, 342, 299, 299, 299, 342, 301, 303, 307, - /* 110 */ 316, 319, 325, 320, 328, 317, 281, 342, 342, 369, - /* 120 */ 369, 342, 361, 363, 398, 368, 367, 397, 370, 374, - /* 130 */ 342, 379, 342, 379, 560, 560, 27, 54, 54, 54, - /* 140 */ 114, 139, 70, 70, 70, 179, 192, 192, 192, 192, - /* 150 */ 211, 234, 177, 160, 62, 62, 199, 213, 116, 219, - /* 160 */ 227, 228, 229, 216, 290, 314, 222, 265, 218, 30, - /* 170 */ 235, 238, 243, 207, 220, 318, 338, 280, 451, 322, - /* 180 */ 453, 454, 329, 458, 461, 381, 344, 377, 386, 366, - /* 190 */ 383, 388, 394, 494, 399, 400, 402, 393, 384, 401, - /* 200 */ 387, 403, 405, 404, 407, 406, 409, 410, 437, 502, - /* 210 */ 507, 508, 509, 510, 511, 441, 503, 447, 504, 392, - /* 220 */ 395, 419, 518, 520, 427, 429, 419, 523, 524, 525, - /* 230 */ 526, 527, 528, 529, 530, 531, 532, 533, 534, 535, - /* 240 */ 536, 439, 467, 537, 538, 485, 487, 546, +#define YY_SHIFT_USE_DFLT (-118) +#define YY_SHIFT_COUNT (248) +#define YY_SHIFT_MIN (-117) +#define YY_SHIFT_MAX (416) +static const short yy_shift_ofst[] = { + /* 0 */ 181, 104, 194, 212, 222, 296, 296, 296, 296, 296, + /* 10 */ 296, -1, 88, 222, 291, 291, 291, 244, 296, 296, + /* 20 */ 296, 296, 296, 144, 416, 416, -118, 202, 222, 222, + /* 30 */ 222, 222, 222, 222, 222, 222, 222, 222, 222, 222, + /* 40 */ 222, 222, 222, 222, 222, 291, 291, 1, 1, 1, + /* 50 */ 1, 1, 1, -58, 1, 172, 296, 296, 296, 296, + /* 60 */ 168, 168, -95, 296, 296, 296, 296, 296, 296, 296, + /* 70 */ 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, + /* 80 */ 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, + /* 90 */ 296, 296, 296, 296, 296, 296, 296, 296, 296, 296, + /* 100 */ 345, 368, 368, 376, 376, 376, 368, 343, 342, 341, + /* 110 */ 374, 340, 348, 347, 346, 330, 345, 368, 368, 373, + /* 120 */ 373, 368, 372, 370, 386, 362, 354, 385, 349, 360, + /* 130 */ 368, 351, 368, 351, -118, -118, 26, 53, 53, 53, + /* 140 */ 79, 134, 236, 236, 236, -63, 204, 204, 204, 204, + /* 150 */ 251, 64, -56, -117, 71, 71, -5, 133, 229, 209, + /* 160 */ 199, 191, 153, 221, 205, 246, 106, 155, 111, 48, + /* 170 */ 20, -6, -16, -68, -101, 159, 141, -51, 415, 285, + /* 180 */ 413, 412, 280, 410, 408, 326, 282, 312, 311, 302, + /* 190 */ 310, 307, 309, 405, 306, 304, 301, 299, 284, 298, + /* 200 */ 283, 297, 295, 290, 294, 289, 292, 288, 316, 383, + /* 210 */ 382, 381, 380, 379, 378, 305, 367, 308, 363, 248, + /* 220 */ 247, 268, 371, 369, 275, 274, 268, 366, 365, 364, + /* 230 */ 361, 359, 358, 357, 356, 355, 353, 352, 350, 260, + /* 240 */ 254, 243, 126, 178, 154, 132, 68, 13, 58, }; +#define YY_REDUCE_USE_DFLT (-179) #define YY_REDUCE_COUNT (135) -#define YY_REDUCE_MIN (-257) -#define YY_REDUCE_MAX (270) +#define YY_REDUCE_MIN (-178) +#define YY_REDUCE_MAX (402) static const short yy_reduce_ofst[] = { - /* 0 */ -201, -33, -225, -237, -223, -97, -180, -115, -99, -98, - /* 10 */ -73, -174, -190, -197, -169, -52, -17, -194, -48, -159, - /* 20 */ 46, -77, 23, 55, 71, 93, -109, -257, -236, -186, - /* 30 */ -154, -84, 32, 74, 85, 91, 95, 96, 97, 98, - /* 40 */ 100, 101, 102, 103, 104, 119, 120, 126, 127, 128, - /* 50 */ 130, 131, 132, 133, 135, 134, 167, 169, 170, 171, - /* 60 */ 115, 117, 137, 173, 176, 180, 181, 182, 183, 184, - /* 70 */ 185, 186, 187, 188, 189, 190, 191, 193, 194, 195, - /* 80 */ 196, 197, 198, 200, 201, 203, 204, 206, 208, 209, - /* 90 */ 210, 212, 214, 217, 221, 223, 224, 225, 226, 230, - /* 100 */ 145, 231, 232, 149, 152, 154, 233, 166, 172, 174, - /* 110 */ 236, 239, 237, 240, 244, 246, 241, 242, 245, 247, - /* 120 */ 248, 250, 251, 253, 252, 254, 256, 259, 260, 262, - /* 130 */ 255, 257, 261, 258, 266, 270, + /* 0 */ -178, -53, -116, 5, -115, -62, -48, 83, 61, 21, + /* 10 */ -162, 138, 142, 82, 120, 117, 98, 96, -163, 99, + /* 20 */ 91, 109, 93, 123, 121, 116, 74, 402, 401, 400, + /* 30 */ 399, 398, 397, 396, 395, 394, 393, 392, 391, 390, + /* 40 */ 389, 388, 387, 384, 377, 375, 293, 344, 279, 277, + /* 50 */ 276, 272, 270, 249, 257, 253, 339, 338, 337, 336, + /* 60 */ 240, 197, 245, 335, 334, 333, 332, 331, 329, 328, + /* 70 */ 327, 325, 324, 323, 322, 321, 320, 319, 318, 317, + /* 80 */ 315, 314, 313, 303, 300, 287, 286, 281, 278, 273, + /* 90 */ 271, 269, 267, 265, 264, 263, 262, 261, 259, 258, + /* 100 */ 241, 256, 255, 239, 238, 200, 250, 237, 232, 228, + /* 110 */ 227, 226, 225, 224, 203, 201, 188, 235, 233, 234, + /* 120 */ 230, 231, 223, 219, 207, 220, 218, 216, 210, 214, + /* 130 */ 217, 215, 213, 211, 206, 208, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 638, 690, 679, 858, 858, 638, 638, 638, 638, 638, - /* 10 */ 638, 783, 656, 858, 638, 638, 638, 638, 638, 638, - /* 20 */ 638, 638, 638, 692, 692, 692, 778, 638, 638, 638, - /* 30 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 40 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 50 */ 638, 638, 638, 638, 638, 638, 638, 785, 787, 638, - /* 60 */ 805, 805, 776, 638, 638, 638, 638, 638, 638, 638, - /* 70 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 80 */ 638, 677, 638, 675, 638, 638, 638, 638, 638, 638, - /* 90 */ 638, 638, 638, 638, 638, 638, 638, 664, 638, 638, - /* 100 */ 638, 658, 658, 638, 638, 638, 658, 812, 816, 810, - /* 110 */ 798, 806, 797, 793, 792, 820, 638, 658, 658, 687, - /* 120 */ 687, 658, 708, 706, 704, 696, 702, 698, 700, 694, - /* 130 */ 658, 685, 658, 685, 724, 738, 638, 821, 857, 811, - /* 140 */ 847, 846, 853, 845, 844, 638, 840, 841, 843, 842, - /* 150 */ 638, 638, 638, 638, 849, 848, 638, 638, 638, 638, - /* 160 */ 638, 638, 638, 638, 638, 638, 823, 638, 817, 813, - /* 170 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 180 */ 638, 638, 638, 638, 638, 638, 638, 775, 638, 638, - /* 190 */ 784, 638, 638, 638, 638, 638, 638, 807, 638, 799, - /* 200 */ 638, 638, 638, 638, 638, 638, 638, 752, 638, 638, - /* 210 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 220 */ 638, 862, 638, 638, 638, 746, 860, 638, 638, 638, - /* 230 */ 638, 638, 638, 638, 638, 638, 638, 638, 638, 638, - /* 240 */ 638, 711, 638, 662, 660, 638, 654, 638, + /* 0 */ 645, 463, 452, 634, 634, 645, 645, 645, 645, 645, + /* 10 */ 645, 559, 429, 634, 645, 645, 645, 645, 645, 645, + /* 20 */ 645, 645, 645, 465, 465, 465, 554, 645, 645, 645, + /* 30 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 40 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 50 */ 645, 645, 645, 645, 645, 645, 645, 561, 563, 645, + /* 60 */ 581, 581, 552, 645, 645, 645, 645, 645, 645, 645, + /* 70 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 80 */ 645, 450, 645, 448, 645, 645, 645, 645, 645, 645, + /* 90 */ 645, 645, 645, 645, 645, 645, 645, 437, 645, 645, + /* 100 */ 645, 431, 431, 645, 645, 645, 431, 588, 592, 586, + /* 110 */ 574, 582, 573, 569, 568, 596, 645, 431, 431, 460, + /* 120 */ 460, 431, 481, 479, 477, 469, 475, 471, 473, 467, + /* 130 */ 431, 458, 431, 458, 498, 513, 645, 597, 633, 587, + /* 140 */ 623, 622, 629, 621, 620, 645, 616, 617, 619, 618, + /* 150 */ 645, 645, 645, 645, 625, 624, 645, 645, 645, 645, + /* 160 */ 645, 645, 645, 645, 645, 645, 599, 645, 593, 589, + /* 170 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 180 */ 645, 645, 645, 645, 645, 645, 645, 551, 645, 645, + /* 190 */ 560, 645, 645, 645, 645, 645, 645, 583, 645, 575, + /* 200 */ 645, 645, 645, 645, 645, 645, 645, 528, 645, 645, + /* 210 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 220 */ 645, 638, 645, 645, 645, 522, 636, 645, 645, 645, + /* 230 */ 645, 645, 645, 645, 645, 645, 645, 645, 645, 645, + /* 240 */ 645, 645, 484, 645, 435, 433, 645, 427, 645, 644, + /* 250 */ 643, 642, 635, 550, 549, 548, 547, 556, 558, 557, + /* 260 */ 555, 562, 564, 553, 567, 566, 571, 570, 572, 565, + /* 270 */ 585, 584, 577, 578, 580, 579, 576, 613, 631, 632, + /* 280 */ 630, 628, 627, 626, 612, 611, 610, 609, 608, 605, + /* 290 */ 607, 604, 606, 603, 602, 601, 600, 598, 615, 614, + /* 300 */ 595, 594, 591, 590, 546, 531, 530, 529, 527, 464, + /* 310 */ 512, 511, 510, 509, 508, 507, 506, 505, 504, 503, + /* 320 */ 502, 501, 500, 499, 496, 492, 490, 489, 488, 485, + /* 330 */ 459, 462, 461, 641, 640, 639, 637, 533, 534, 526, + /* 340 */ 525, 524, 523, 532, 483, 482, 480, 478, 470, 476, + /* 350 */ 472, 474, 468, 466, 454, 453, 521, 520, 519, 518, + /* 360 */ 517, 516, 515, 514, 497, 495, 494, 493, 491, 487, + /* 370 */ 486, 536, 545, 544, 543, 542, 541, 540, 539, 538, + /* 380 */ 537, 535, 451, 449, 447, 446, 445, 444, 443, 442, + /* 390 */ 441, 440, 457, 439, 432, 438, 436, 456, 455, 434, + /* 400 */ 430, 428, 426, 425, 424, 423, 422, 421, 420, 419, + /* 410 */ 418, 417, 416, 415, }; -/********** End of lemon-generated parsing tables *****************************/ -/* The next table maps tokens (terminal symbols) into fallback tokens. -** If a construct like the following: +/* The next table maps tokens into fallback tokens. If a construct +** like the following: ** ** %fallback ID X Y Z. ** @@ -439,10 +410,6 @@ static const YYACTIONTYPE yy_default[] = { ** and Z. Whenever one of the tokens X, Y, or Z is input to the parser ** but it does not parse, the type of the token is changed to ID and ** the parse is retried before an error is thrown. -** -** This feature can be used, for example, to cause some keywords in a language -** to revert to identifiers if they keyword does not apply in the context where -** it appears. */ #ifdef YYFALLBACK static const YYCODETYPE yyFallback[] = { @@ -544,6 +511,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* FSYNC => nothing */ 0, /* COMP => nothing */ 0, /* PRECISION => nothing */ + 0, /* UPDATE => nothing */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* TAGS => nothing */ @@ -666,13 +634,9 @@ static const YYCODETYPE yyFallback[] = { ** + The semantic value stored at this level of the stack. This is ** the information used by the action routines in the grammar. ** It is sometimes called the "minor" token. -** -** After the "shift" half of a SHIFTREDUCE action, the stateno field -** actually contains the reduce action for the second half of the -** SHIFTREDUCE. */ struct yyStackEntry { - YYACTIONTYPE stateno; /* The state-number, or reduce action in SHIFTREDUCE */ + YYACTIONTYPE stateno; /* The state-number */ YYCODETYPE major; /* The major token value. This is the code ** number for the token at this stack level */ YYMINORTYPE minor; /* The user-supplied minor token value. This @@ -683,22 +647,17 @@ typedef struct yyStackEntry yyStackEntry; /* The state of the parser is completely contained in an instance of ** the following structure */ struct yyParser { - yyStackEntry *yytos; /* Pointer to top element of the stack */ + int yyidx; /* Index of top element in stack */ #ifdef YYTRACKMAXSTACKDEPTH - int yyhwm; /* High-water mark of the stack */ + int yyidxMax; /* Maximum value of yyidx */ #endif -#ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ -#endif ParseARG_SDECL /* A place to hold %extra_argument */ - ParseCTX_SDECL /* A place to hold %extra_context */ #if YYSTACKDEPTH<=0 int yystksz; /* Current side of the stack */ yyStackEntry *yystack; /* The parser's stack */ - yyStackEntry yystk0; /* First stack entry */ #else yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ - yyStackEntry *yystackEnd; /* Last entry in the stack */ #endif }; typedef struct yyParser yyParser; @@ -735,282 +694,81 @@ void ParseTrace(FILE *TraceFILE, char *zTracePrompt){ } #endif /* NDEBUG */ -#if defined(YYCOVERAGE) || !defined(NDEBUG) +#ifndef NDEBUG /* For tracing shifts, the names of all terminals and nonterminals ** are required. The following table supplies these names */ static const char *const yyTokenName[] = { - /* 0 */ "$", - /* 1 */ "ID", - /* 2 */ "BOOL", - /* 3 */ "TINYINT", - /* 4 */ "SMALLINT", - /* 5 */ "INTEGER", - /* 6 */ "BIGINT", - /* 7 */ "FLOAT", - /* 8 */ "DOUBLE", - /* 9 */ "STRING", - /* 10 */ "TIMESTAMP", - /* 11 */ "BINARY", - /* 12 */ "NCHAR", - /* 13 */ "OR", - /* 14 */ "AND", - /* 15 */ "NOT", - /* 16 */ "EQ", - /* 17 */ "NE", - /* 18 */ "ISNULL", - /* 19 */ "NOTNULL", - /* 20 */ "IS", - /* 21 */ "LIKE", - /* 22 */ "GLOB", - /* 23 */ "BETWEEN", - /* 24 */ "IN", - /* 25 */ "GT", - /* 26 */ "GE", - /* 27 */ "LT", - /* 28 */ "LE", - /* 29 */ "BITAND", - /* 30 */ "BITOR", - /* 31 */ "LSHIFT", - /* 32 */ "RSHIFT", - /* 33 */ "PLUS", - /* 34 */ "MINUS", - /* 35 */ "DIVIDE", - /* 36 */ "TIMES", - /* 37 */ "STAR", - /* 38 */ "SLASH", - /* 39 */ "REM", - /* 40 */ "CONCAT", - /* 41 */ "UMINUS", - /* 42 */ "UPLUS", - /* 43 */ "BITNOT", - /* 44 */ "SHOW", - /* 45 */ "DATABASES", - /* 46 */ "MNODES", - /* 47 */ "DNODES", - /* 48 */ "ACCOUNTS", - /* 49 */ "USERS", - /* 50 */ "MODULES", - /* 51 */ "QUERIES", - /* 52 */ "CONNECTIONS", - /* 53 */ "STREAMS", - /* 54 */ "VARIABLES", - /* 55 */ "SCORES", - /* 56 */ "GRANTS", - /* 57 */ "VNODES", - /* 58 */ "IPTOKEN", - /* 59 */ "DOT", - /* 60 */ "TABLES", - /* 61 */ "STABLES", - /* 62 */ "VGROUPS", - /* 63 */ "DROP", - /* 64 */ "TABLE", - /* 65 */ "DATABASE", - /* 66 */ "DNODE", - /* 67 */ "USER", - /* 68 */ "ACCOUNT", - /* 69 */ "USE", - /* 70 */ "DESCRIBE", - /* 71 */ "ALTER", - /* 72 */ "PASS", - /* 73 */ "PRIVILEGE", - /* 74 */ "LOCAL", - /* 75 */ "IF", - /* 76 */ "EXISTS", - /* 77 */ "CREATE", - /* 78 */ "PPS", - /* 79 */ "TSERIES", - /* 80 */ "DBS", - /* 81 */ "STORAGE", - /* 82 */ "QTIME", - /* 83 */ "CONNS", - /* 84 */ "STATE", - /* 85 */ "KEEP", - /* 86 */ "CACHE", - /* 87 */ "REPLICA", - /* 88 */ "QUORUM", - /* 89 */ "DAYS", - /* 90 */ "MINROWS", - /* 91 */ "MAXROWS", - /* 92 */ "BLOCKS", - /* 93 */ "CTIME", - /* 94 */ "WAL", - /* 95 */ "FSYNC", - /* 96 */ "COMP", - /* 97 */ "PRECISION", - /* 98 */ "LP", - /* 99 */ "RP", - /* 100 */ "TAGS", - /* 101 */ "USING", - /* 102 */ "AS", - /* 103 */ "COMMA", - /* 104 */ "NULL", - /* 105 */ "SELECT", - /* 106 */ "UNION", - /* 107 */ "ALL", - /* 108 */ "FROM", - /* 109 */ "VARIABLE", - /* 110 */ "INTERVAL", - /* 111 */ "FILL", - /* 112 */ "SLIDING", - /* 113 */ "ORDER", - /* 114 */ "BY", - /* 115 */ "ASC", - /* 116 */ "DESC", - /* 117 */ "GROUP", - /* 118 */ "HAVING", - /* 119 */ "LIMIT", - /* 120 */ "OFFSET", - /* 121 */ "SLIMIT", - /* 122 */ "SOFFSET", - /* 123 */ "WHERE", - /* 124 */ "NOW", - /* 125 */ "RESET", - /* 126 */ "QUERY", - /* 127 */ "ADD", - /* 128 */ "COLUMN", - /* 129 */ "TAG", - /* 130 */ "CHANGE", - /* 131 */ "SET", - /* 132 */ "KILL", - /* 133 */ "CONNECTION", - /* 134 */ "STREAM", - /* 135 */ "COLON", - /* 136 */ "ABORT", - /* 137 */ "AFTER", - /* 138 */ "ATTACH", - /* 139 */ "BEFORE", - /* 140 */ "BEGIN", - /* 141 */ "CASCADE", - /* 142 */ "CLUSTER", - /* 143 */ "CONFLICT", - /* 144 */ "COPY", - /* 145 */ "DEFERRED", - /* 146 */ "DELIMITERS", - /* 147 */ "DETACH", - /* 148 */ "EACH", - /* 149 */ "END", - /* 150 */ "EXPLAIN", - /* 151 */ "FAIL", - /* 152 */ "FOR", - /* 153 */ "IGNORE", - /* 154 */ "IMMEDIATE", - /* 155 */ "INITIALLY", - /* 156 */ "INSTEAD", - /* 157 */ "MATCH", - /* 158 */ "KEY", - /* 159 */ "OF", - /* 160 */ "RAISE", - /* 161 */ "REPLACE", - /* 162 */ "RESTRICT", - /* 163 */ "ROW", - /* 164 */ "STATEMENT", - /* 165 */ "TRIGGER", - /* 166 */ "VIEW", - /* 167 */ "COUNT", - /* 168 */ "SUM", - /* 169 */ "AVG", - /* 170 */ "MIN", - /* 171 */ "MAX", - /* 172 */ "FIRST", - /* 173 */ "LAST", - /* 174 */ "TOP", - /* 175 */ "BOTTOM", - /* 176 */ "STDDEV", - /* 177 */ "PERCENTILE", - /* 178 */ "APERCENTILE", - /* 179 */ "LEASTSQUARES", - /* 180 */ "HISTOGRAM", - /* 181 */ "DIFF", - /* 182 */ "SPREAD", - /* 183 */ "TWA", - /* 184 */ "INTERP", - /* 185 */ "LAST_ROW", - /* 186 */ "RATE", - /* 187 */ "IRATE", - /* 188 */ "SUM_RATE", - /* 189 */ "SUM_IRATE", - /* 190 */ "AVG_RATE", - /* 191 */ "AVG_IRATE", - /* 192 */ "TBID", - /* 193 */ "SEMI", - /* 194 */ "NONE", - /* 195 */ "PREV", - /* 196 */ "LINEAR", - /* 197 */ "IMPORT", - /* 198 */ "METRIC", - /* 199 */ "TBNAME", - /* 200 */ "JOIN", - /* 201 */ "METRICS", - /* 202 */ "STABLE", - /* 203 */ "INSERT", - /* 204 */ "INTO", - /* 205 */ "VALUES", - /* 206 */ "program", - /* 207 */ "cmd", - /* 208 */ "dbPrefix", - /* 209 */ "ids", - /* 210 */ "cpxName", - /* 211 */ "ifexists", - /* 212 */ "alter_db_optr", - /* 213 */ "acct_optr", - /* 214 */ "ifnotexists", - /* 215 */ "db_optr", - /* 216 */ "pps", - /* 217 */ "tseries", - /* 218 */ "dbs", - /* 219 */ "streams", - /* 220 */ "storage", - /* 221 */ "qtime", - /* 222 */ "users", - /* 223 */ "conns", - /* 224 */ "state", - /* 225 */ "keep", - /* 226 */ "tagitemlist", - /* 227 */ "cache", - /* 228 */ "replica", - /* 229 */ "quorum", - /* 230 */ "days", - /* 231 */ "minrows", - /* 232 */ "maxrows", - /* 233 */ "blocks", - /* 234 */ "ctime", - /* 235 */ "wal", - /* 236 */ "fsync", - /* 237 */ "comp", - /* 238 */ "prec", - /* 239 */ "typename", - /* 240 */ "signed", - /* 241 */ "create_table_args", - /* 242 */ "columnlist", - /* 243 */ "select", - /* 244 */ "column", - /* 245 */ "tagitem", - /* 246 */ "selcollist", - /* 247 */ "from", - /* 248 */ "where_opt", - /* 249 */ "interval_opt", - /* 250 */ "fill_opt", - /* 251 */ "sliding_opt", - /* 252 */ "groupby_opt", - /* 253 */ "orderby_opt", - /* 254 */ "having_opt", - /* 255 */ "slimit_opt", - /* 256 */ "limit_opt", - /* 257 */ "union", - /* 258 */ "sclp", - /* 259 */ "expr", - /* 260 */ "as", - /* 261 */ "tablelist", - /* 262 */ "tmvar", - /* 263 */ "sortlist", - /* 264 */ "sortitem", - /* 265 */ "item", - /* 266 */ "sortorder", - /* 267 */ "grouplist", - /* 268 */ "exprlist", - /* 269 */ "expritem", + "$", "ID", "BOOL", "TINYINT", + "SMALLINT", "INTEGER", "BIGINT", "FLOAT", + "DOUBLE", "STRING", "TIMESTAMP", "BINARY", + "NCHAR", "OR", "AND", "NOT", + "EQ", "NE", "ISNULL", "NOTNULL", + "IS", "LIKE", "GLOB", "BETWEEN", + "IN", "GT", "GE", "LT", + "LE", "BITAND", "BITOR", "LSHIFT", + "RSHIFT", "PLUS", "MINUS", "DIVIDE", + "TIMES", "STAR", "SLASH", "REM", + "CONCAT", "UMINUS", "UPLUS", "BITNOT", + "SHOW", "DATABASES", "MNODES", "DNODES", + "ACCOUNTS", "USERS", "MODULES", "QUERIES", + "CONNECTIONS", "STREAMS", "VARIABLES", "SCORES", + "GRANTS", "VNODES", "IPTOKEN", "DOT", + "TABLES", "STABLES", "VGROUPS", "DROP", + "TABLE", "DATABASE", "DNODE", "USER", + "ACCOUNT", "USE", "DESCRIBE", "ALTER", + "PASS", "PRIVILEGE", "LOCAL", "IF", + "EXISTS", "CREATE", "PPS", "TSERIES", + "DBS", "STORAGE", "QTIME", "CONNS", + "STATE", "KEEP", "CACHE", "REPLICA", + "QUORUM", "DAYS", "MINROWS", "MAXROWS", + "BLOCKS", "CTIME", "WAL", "FSYNC", + "COMP", "PRECISION", "UPDATE", "LP", + "RP", "TAGS", "USING", "AS", + "COMMA", "NULL", "SELECT", "UNION", + "ALL", "FROM", "VARIABLE", "INTERVAL", + "FILL", "SLIDING", "ORDER", "BY", + "ASC", "DESC", "GROUP", "HAVING", + "LIMIT", "OFFSET", "SLIMIT", "SOFFSET", + "WHERE", "NOW", "RESET", "QUERY", + "ADD", "COLUMN", "TAG", "CHANGE", + "SET", "KILL", "CONNECTION", "STREAM", + "COLON", "ABORT", "AFTER", "ATTACH", + "BEFORE", "BEGIN", "CASCADE", "CLUSTER", + "CONFLICT", "COPY", "DEFERRED", "DELIMITERS", + "DETACH", "EACH", "END", "EXPLAIN", + "FAIL", "FOR", "IGNORE", "IMMEDIATE", + "INITIALLY", "INSTEAD", "MATCH", "KEY", + "OF", "RAISE", "REPLACE", "RESTRICT", + "ROW", "STATEMENT", "TRIGGER", "VIEW", + "COUNT", "SUM", "AVG", "MIN", + "MAX", "FIRST", "LAST", "TOP", + "BOTTOM", "STDDEV", "PERCENTILE", "APERCENTILE", + "LEASTSQUARES", "HISTOGRAM", "DIFF", "SPREAD", + "TWA", "INTERP", "LAST_ROW", "RATE", + "IRATE", "SUM_RATE", "SUM_IRATE", "AVG_RATE", + "AVG_IRATE", "TBID", "SEMI", "NONE", + "PREV", "LINEAR", "IMPORT", "METRIC", + "TBNAME", "JOIN", "METRICS", "STABLE", + "INSERT", "INTO", "VALUES", "error", + "program", "cmd", "dbPrefix", "ids", + "cpxName", "ifexists", "alter_db_optr", "acct_optr", + "ifnotexists", "db_optr", "pps", "tseries", + "dbs", "streams", "storage", "qtime", + "users", "conns", "state", "keep", + "tagitemlist", "cache", "replica", "quorum", + "days", "minrows", "maxrows", "blocks", + "ctime", "wal", "fsync", "comp", + "prec", "update", "typename", "signed", + "create_table_args", "columnlist", "select", "column", + "tagitem", "selcollist", "from", "where_opt", + "interval_opt", "fill_opt", "sliding_opt", "groupby_opt", + "orderby_opt", "having_opt", "slimit_opt", "limit_opt", + "union", "sclp", "expr", "as", + "tablelist", "tmvar", "sortlist", "sortitem", + "item", "sortorder", "grouplist", "exprlist", + "expritem", }; -#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ +#endif /* NDEBUG */ #ifndef NDEBUG /* For tracing reduce actions, the names of all rules are required. @@ -1099,226 +857,181 @@ static const char *const yyRuleName[] = { /* 80 */ "fsync ::= FSYNC INTEGER", /* 81 */ "comp ::= COMP INTEGER", /* 82 */ "prec ::= PRECISION STRING", - /* 83 */ "db_optr ::=", - /* 84 */ "db_optr ::= db_optr cache", - /* 85 */ "db_optr ::= db_optr replica", - /* 86 */ "db_optr ::= db_optr quorum", - /* 87 */ "db_optr ::= db_optr days", - /* 88 */ "db_optr ::= db_optr minrows", - /* 89 */ "db_optr ::= db_optr maxrows", - /* 90 */ "db_optr ::= db_optr blocks", - /* 91 */ "db_optr ::= db_optr ctime", - /* 92 */ "db_optr ::= db_optr wal", - /* 93 */ "db_optr ::= db_optr fsync", - /* 94 */ "db_optr ::= db_optr comp", - /* 95 */ "db_optr ::= db_optr prec", - /* 96 */ "db_optr ::= db_optr keep", - /* 97 */ "alter_db_optr ::=", - /* 98 */ "alter_db_optr ::= alter_db_optr replica", - /* 99 */ "alter_db_optr ::= alter_db_optr quorum", - /* 100 */ "alter_db_optr ::= alter_db_optr keep", - /* 101 */ "alter_db_optr ::= alter_db_optr blocks", - /* 102 */ "alter_db_optr ::= alter_db_optr comp", - /* 103 */ "alter_db_optr ::= alter_db_optr wal", - /* 104 */ "alter_db_optr ::= alter_db_optr fsync", - /* 105 */ "typename ::= ids", - /* 106 */ "typename ::= ids LP signed RP", - /* 107 */ "signed ::= INTEGER", - /* 108 */ "signed ::= PLUS INTEGER", - /* 109 */ "signed ::= MINUS INTEGER", - /* 110 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", - /* 111 */ "create_table_args ::= LP columnlist RP", - /* 112 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", - /* 113 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", - /* 114 */ "create_table_args ::= AS select", - /* 115 */ "columnlist ::= columnlist COMMA column", - /* 116 */ "columnlist ::= column", - /* 117 */ "column ::= ids typename", - /* 118 */ "tagitemlist ::= tagitemlist COMMA tagitem", - /* 119 */ "tagitemlist ::= tagitem", - /* 120 */ "tagitem ::= INTEGER", - /* 121 */ "tagitem ::= FLOAT", - /* 122 */ "tagitem ::= STRING", - /* 123 */ "tagitem ::= BOOL", - /* 124 */ "tagitem ::= NULL", - /* 125 */ "tagitem ::= MINUS INTEGER", - /* 126 */ "tagitem ::= MINUS FLOAT", - /* 127 */ "tagitem ::= PLUS INTEGER", - /* 128 */ "tagitem ::= PLUS FLOAT", - /* 129 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", - /* 130 */ "union ::= select", - /* 131 */ "union ::= LP union RP", - /* 132 */ "union ::= union UNION ALL select", - /* 133 */ "union ::= union UNION ALL LP select RP", - /* 134 */ "cmd ::= union", - /* 135 */ "select ::= SELECT selcollist", - /* 136 */ "sclp ::= selcollist COMMA", - /* 137 */ "sclp ::=", - /* 138 */ "selcollist ::= sclp expr as", - /* 139 */ "selcollist ::= sclp STAR", - /* 140 */ "as ::= AS ids", - /* 141 */ "as ::= ids", - /* 142 */ "as ::=", - /* 143 */ "from ::= FROM tablelist", - /* 144 */ "tablelist ::= ids cpxName", - /* 145 */ "tablelist ::= ids cpxName ids", - /* 146 */ "tablelist ::= tablelist COMMA ids cpxName", - /* 147 */ "tablelist ::= tablelist COMMA ids cpxName ids", - /* 148 */ "tmvar ::= VARIABLE", - /* 149 */ "interval_opt ::= INTERVAL LP tmvar RP", - /* 150 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", - /* 151 */ "interval_opt ::=", - /* 152 */ "fill_opt ::=", - /* 153 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", - /* 154 */ "fill_opt ::= FILL LP ID RP", - /* 155 */ "sliding_opt ::= SLIDING LP tmvar RP", - /* 156 */ "sliding_opt ::=", - /* 157 */ "orderby_opt ::=", - /* 158 */ "orderby_opt ::= ORDER BY sortlist", - /* 159 */ "sortlist ::= sortlist COMMA item sortorder", - /* 160 */ "sortlist ::= item sortorder", - /* 161 */ "item ::= ids cpxName", - /* 162 */ "sortorder ::= ASC", - /* 163 */ "sortorder ::= DESC", - /* 164 */ "sortorder ::=", - /* 165 */ "groupby_opt ::=", - /* 166 */ "groupby_opt ::= GROUP BY grouplist", - /* 167 */ "grouplist ::= grouplist COMMA item", - /* 168 */ "grouplist ::= item", - /* 169 */ "having_opt ::=", - /* 170 */ "having_opt ::= HAVING expr", - /* 171 */ "limit_opt ::=", - /* 172 */ "limit_opt ::= LIMIT signed", - /* 173 */ "limit_opt ::= LIMIT signed OFFSET signed", - /* 174 */ "limit_opt ::= LIMIT signed COMMA signed", - /* 175 */ "slimit_opt ::=", - /* 176 */ "slimit_opt ::= SLIMIT signed", - /* 177 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", - /* 178 */ "slimit_opt ::= SLIMIT signed COMMA signed", - /* 179 */ "where_opt ::=", - /* 180 */ "where_opt ::= WHERE expr", - /* 181 */ "expr ::= LP expr RP", - /* 182 */ "expr ::= ID", - /* 183 */ "expr ::= ID DOT ID", - /* 184 */ "expr ::= ID DOT STAR", - /* 185 */ "expr ::= INTEGER", - /* 186 */ "expr ::= MINUS INTEGER", - /* 187 */ "expr ::= PLUS INTEGER", - /* 188 */ "expr ::= FLOAT", - /* 189 */ "expr ::= MINUS FLOAT", - /* 190 */ "expr ::= PLUS FLOAT", - /* 191 */ "expr ::= STRING", - /* 192 */ "expr ::= NOW", - /* 193 */ "expr ::= VARIABLE", - /* 194 */ "expr ::= BOOL", - /* 195 */ "expr ::= ID LP exprlist RP", - /* 196 */ "expr ::= ID LP STAR RP", - /* 197 */ "expr ::= expr IS NULL", - /* 198 */ "expr ::= expr IS NOT NULL", - /* 199 */ "expr ::= expr LT expr", - /* 200 */ "expr ::= expr GT expr", - /* 201 */ "expr ::= expr LE expr", - /* 202 */ "expr ::= expr GE expr", - /* 203 */ "expr ::= expr NE expr", - /* 204 */ "expr ::= expr EQ expr", - /* 205 */ "expr ::= expr AND expr", - /* 206 */ "expr ::= expr OR expr", - /* 207 */ "expr ::= expr PLUS expr", - /* 208 */ "expr ::= expr MINUS expr", - /* 209 */ "expr ::= expr STAR expr", - /* 210 */ "expr ::= expr SLASH expr", - /* 211 */ "expr ::= expr REM expr", - /* 212 */ "expr ::= expr LIKE expr", - /* 213 */ "expr ::= expr IN LP exprlist RP", - /* 214 */ "exprlist ::= exprlist COMMA expritem", - /* 215 */ "exprlist ::= expritem", - /* 216 */ "expritem ::= expr", - /* 217 */ "expritem ::=", - /* 218 */ "cmd ::= RESET QUERY CACHE", - /* 219 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", - /* 220 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", - /* 221 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", - /* 222 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", - /* 223 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", - /* 224 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", - /* 225 */ "cmd ::= KILL CONNECTION INTEGER", - /* 226 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", - /* 227 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", + /* 83 */ "update ::= UPDATE INTEGER", + /* 84 */ "db_optr ::=", + /* 85 */ "db_optr ::= db_optr cache", + /* 86 */ "db_optr ::= db_optr replica", + /* 87 */ "db_optr ::= db_optr quorum", + /* 88 */ "db_optr ::= db_optr days", + /* 89 */ "db_optr ::= db_optr minrows", + /* 90 */ "db_optr ::= db_optr maxrows", + /* 91 */ "db_optr ::= db_optr blocks", + /* 92 */ "db_optr ::= db_optr ctime", + /* 93 */ "db_optr ::= db_optr wal", + /* 94 */ "db_optr ::= db_optr fsync", + /* 95 */ "db_optr ::= db_optr comp", + /* 96 */ "db_optr ::= db_optr prec", + /* 97 */ "db_optr ::= db_optr keep", + /* 98 */ "db_optr ::= db_optr update", + /* 99 */ "alter_db_optr ::=", + /* 100 */ "alter_db_optr ::= alter_db_optr replica", + /* 101 */ "alter_db_optr ::= alter_db_optr quorum", + /* 102 */ "alter_db_optr ::= alter_db_optr keep", + /* 103 */ "alter_db_optr ::= alter_db_optr blocks", + /* 104 */ "alter_db_optr ::= alter_db_optr comp", + /* 105 */ "alter_db_optr ::= alter_db_optr wal", + /* 106 */ "alter_db_optr ::= alter_db_optr fsync", + /* 107 */ "alter_db_optr ::= alter_db_optr update", + /* 108 */ "typename ::= ids", + /* 109 */ "typename ::= ids LP signed RP", + /* 110 */ "signed ::= INTEGER", + /* 111 */ "signed ::= PLUS INTEGER", + /* 112 */ "signed ::= MINUS INTEGER", + /* 113 */ "cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args", + /* 114 */ "create_table_args ::= LP columnlist RP", + /* 115 */ "create_table_args ::= LP columnlist RP TAGS LP columnlist RP", + /* 116 */ "create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP", + /* 117 */ "create_table_args ::= AS select", + /* 118 */ "columnlist ::= columnlist COMMA column", + /* 119 */ "columnlist ::= column", + /* 120 */ "column ::= ids typename", + /* 121 */ "tagitemlist ::= tagitemlist COMMA tagitem", + /* 122 */ "tagitemlist ::= tagitem", + /* 123 */ "tagitem ::= INTEGER", + /* 124 */ "tagitem ::= FLOAT", + /* 125 */ "tagitem ::= STRING", + /* 126 */ "tagitem ::= BOOL", + /* 127 */ "tagitem ::= NULL", + /* 128 */ "tagitem ::= MINUS INTEGER", + /* 129 */ "tagitem ::= MINUS FLOAT", + /* 130 */ "tagitem ::= PLUS INTEGER", + /* 131 */ "tagitem ::= PLUS FLOAT", + /* 132 */ "select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt", + /* 133 */ "union ::= select", + /* 134 */ "union ::= LP union RP", + /* 135 */ "union ::= union UNION ALL select", + /* 136 */ "union ::= union UNION ALL LP select RP", + /* 137 */ "cmd ::= union", + /* 138 */ "select ::= SELECT selcollist", + /* 139 */ "sclp ::= selcollist COMMA", + /* 140 */ "sclp ::=", + /* 141 */ "selcollist ::= sclp expr as", + /* 142 */ "selcollist ::= sclp STAR", + /* 143 */ "as ::= AS ids", + /* 144 */ "as ::= ids", + /* 145 */ "as ::=", + /* 146 */ "from ::= FROM tablelist", + /* 147 */ "tablelist ::= ids cpxName", + /* 148 */ "tablelist ::= ids cpxName ids", + /* 149 */ "tablelist ::= tablelist COMMA ids cpxName", + /* 150 */ "tablelist ::= tablelist COMMA ids cpxName ids", + /* 151 */ "tmvar ::= VARIABLE", + /* 152 */ "interval_opt ::= INTERVAL LP tmvar RP", + /* 153 */ "interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP", + /* 154 */ "interval_opt ::=", + /* 155 */ "fill_opt ::=", + /* 156 */ "fill_opt ::= FILL LP ID COMMA tagitemlist RP", + /* 157 */ "fill_opt ::= FILL LP ID RP", + /* 158 */ "sliding_opt ::= SLIDING LP tmvar RP", + /* 159 */ "sliding_opt ::=", + /* 160 */ "orderby_opt ::=", + /* 161 */ "orderby_opt ::= ORDER BY sortlist", + /* 162 */ "sortlist ::= sortlist COMMA item sortorder", + /* 163 */ "sortlist ::= item sortorder", + /* 164 */ "item ::= ids cpxName", + /* 165 */ "sortorder ::= ASC", + /* 166 */ "sortorder ::= DESC", + /* 167 */ "sortorder ::=", + /* 168 */ "groupby_opt ::=", + /* 169 */ "groupby_opt ::= GROUP BY grouplist", + /* 170 */ "grouplist ::= grouplist COMMA item", + /* 171 */ "grouplist ::= item", + /* 172 */ "having_opt ::=", + /* 173 */ "having_opt ::= HAVING expr", + /* 174 */ "limit_opt ::=", + /* 175 */ "limit_opt ::= LIMIT signed", + /* 176 */ "limit_opt ::= LIMIT signed OFFSET signed", + /* 177 */ "limit_opt ::= LIMIT signed COMMA signed", + /* 178 */ "slimit_opt ::=", + /* 179 */ "slimit_opt ::= SLIMIT signed", + /* 180 */ "slimit_opt ::= SLIMIT signed SOFFSET signed", + /* 181 */ "slimit_opt ::= SLIMIT signed COMMA signed", + /* 182 */ "where_opt ::=", + /* 183 */ "where_opt ::= WHERE expr", + /* 184 */ "expr ::= LP expr RP", + /* 185 */ "expr ::= ID", + /* 186 */ "expr ::= ID DOT ID", + /* 187 */ "expr ::= ID DOT STAR", + /* 188 */ "expr ::= INTEGER", + /* 189 */ "expr ::= MINUS INTEGER", + /* 190 */ "expr ::= PLUS INTEGER", + /* 191 */ "expr ::= FLOAT", + /* 192 */ "expr ::= MINUS FLOAT", + /* 193 */ "expr ::= PLUS FLOAT", + /* 194 */ "expr ::= STRING", + /* 195 */ "expr ::= NOW", + /* 196 */ "expr ::= VARIABLE", + /* 197 */ "expr ::= BOOL", + /* 198 */ "expr ::= ID LP exprlist RP", + /* 199 */ "expr ::= ID LP STAR RP", + /* 200 */ "expr ::= expr IS NULL", + /* 201 */ "expr ::= expr IS NOT NULL", + /* 202 */ "expr ::= expr LT expr", + /* 203 */ "expr ::= expr GT expr", + /* 204 */ "expr ::= expr LE expr", + /* 205 */ "expr ::= expr GE expr", + /* 206 */ "expr ::= expr NE expr", + /* 207 */ "expr ::= expr EQ expr", + /* 208 */ "expr ::= expr AND expr", + /* 209 */ "expr ::= expr OR expr", + /* 210 */ "expr ::= expr PLUS expr", + /* 211 */ "expr ::= expr MINUS expr", + /* 212 */ "expr ::= expr STAR expr", + /* 213 */ "expr ::= expr SLASH expr", + /* 214 */ "expr ::= expr REM expr", + /* 215 */ "expr ::= expr LIKE expr", + /* 216 */ "expr ::= expr IN LP exprlist RP", + /* 217 */ "exprlist ::= exprlist COMMA expritem", + /* 218 */ "exprlist ::= expritem", + /* 219 */ "expritem ::= expr", + /* 220 */ "expritem ::=", + /* 221 */ "cmd ::= RESET QUERY CACHE", + /* 222 */ "cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist", + /* 223 */ "cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids", + /* 224 */ "cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist", + /* 225 */ "cmd ::= ALTER TABLE ids cpxName DROP TAG ids", + /* 226 */ "cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids", + /* 227 */ "cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem", + /* 228 */ "cmd ::= KILL CONNECTION INTEGER", + /* 229 */ "cmd ::= KILL STREAM INTEGER COLON INTEGER", + /* 230 */ "cmd ::= KILL QUERY INTEGER COLON INTEGER", }; #endif /* NDEBUG */ #if YYSTACKDEPTH<=0 /* -** Try to increase the size of the parser stack. Return the number -** of errors. Return 0 on success. +** Try to increase the size of the parser stack. */ -static int yyGrowStack(yyParser *p){ +static void yyGrowStack(yyParser *p){ int newSize; - int idx; yyStackEntry *pNew; newSize = p->yystksz*2 + 100; - idx = p->yytos ? (int)(p->yytos - p->yystack) : 0; - if( p->yystack==&p->yystk0 ){ - pNew = malloc(newSize*sizeof(pNew[0])); - if( pNew ) pNew[0] = p->yystk0; - }else{ - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); - } + pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); if( pNew ){ p->yystack = pNew; - p->yytos = &p->yystack[idx]; + p->yystksz = newSize; #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", - yyTracePrompt, p->yystksz, newSize); + fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", + yyTracePrompt, p->yystksz); } #endif - p->yystksz = newSize; } - return pNew==0; } #endif -/* Datatype of the argument to the memory allocated passed as the -** second argument to ParseAlloc() below. This can be changed by -** putting an appropriate #define in the %include section of the input -** grammar. -*/ -#ifndef YYMALLOCARGTYPE -# define YYMALLOCARGTYPE size_t -#endif - -/* Initialize a new parser that has already been allocated. -*/ -void ParseInit(void *yypRawParser ParseCTX_PDECL){ - yyParser *yypParser = (yyParser*)yypRawParser; - ParseCTX_STORE -#ifdef YYTRACKMAXSTACKDEPTH - yypParser->yyhwm = 0; -#endif -#if YYSTACKDEPTH<=0 - yypParser->yytos = NULL; - yypParser->yystack = NULL; - yypParser->yystksz = 0; - if( yyGrowStack(yypParser) ){ - yypParser->yystack = &yypParser->yystk0; - yypParser->yystksz = 1; - } -#endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yypParser->yytos = yypParser->yystack; - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; -#if YYSTACKDEPTH>0 - yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; -#endif -} - -#ifndef Parse_ENGINEALWAYSONSTACK /* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like @@ -1331,32 +1044,34 @@ void ParseInit(void *yypRawParser ParseCTX_PDECL){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) ParseCTX_PDECL){ - yyParser *yypParser; - yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( yypParser ){ - ParseCTX_STORE - ParseInit(yypParser ParseCTX_PARAM); +void *ParseAlloc(void *(*mallocProc)(size_t)){ + yyParser *pParser; + pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + if( pParser ){ + pParser->yyidx = -1; +#ifdef YYTRACKMAXSTACKDEPTH + pParser->yyidxMax = 0; +#endif +#if YYSTACKDEPTH<=0 + pParser->yystack = NULL; + pParser->yystksz = 0; + yyGrowStack(pParser); +#endif } - return (void*)yypParser; + return pParser; } -#endif /* Parse_ENGINEALWAYSONSTACK */ - -/* The following function deletes the "minor type" or semantic value -** associated with a symbol. The symbol can be either a terminal -** or nonterminal. "yymajor" is the symbol code, and "yypminor" is -** a pointer to the value to be deleted. The code used to do the -** deletions is derived from the %destructor and/or %token_destructor -** directives of the input grammar. +/* The following function deletes the value associated with a +** symbol. The symbol can be either a terminal or nonterminal. +** "yymajor" is the symbol code, and "yypminor" is a pointer to +** the value. */ static void yy_destructor( yyParser *yypParser, /* The parser */ YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen @@ -1365,57 +1080,55 @@ static void yy_destructor( ** being destroyed before it is finished parsing. ** ** Note: during a reduce, the only symbols destroyed are those - ** which appear on the RHS of the rule, but which are *not* used + ** which appear on the RHS of the rule, but which are not used ** inside the C code. */ -/********* Begin destructor definitions ***************************************/ - case 225: /* keep */ - case 226: /* tagitemlist */ - case 250: /* fill_opt */ - case 252: /* groupby_opt */ - case 253: /* orderby_opt */ - case 263: /* sortlist */ - case 267: /* grouplist */ + case 227: /* keep */ + case 228: /* tagitemlist */ + case 253: /* fill_opt */ + case 255: /* groupby_opt */ + case 256: /* orderby_opt */ + case 266: /* sortlist */ + case 270: /* grouplist */ { -tVariantListDestroy((yypminor->yy156)); +tVariantListDestroy((yypminor->yy498)); } break; - case 242: /* columnlist */ + case 245: /* columnlist */ { -tFieldListDestroy((yypminor->yy511)); +tFieldListDestroy((yypminor->yy523)); } break; - case 243: /* select */ + case 246: /* select */ { -doDestroyQuerySql((yypminor->yy444)); +doDestroyQuerySql((yypminor->yy414)); } break; - case 246: /* selcollist */ - case 258: /* sclp */ - case 268: /* exprlist */ + case 249: /* selcollist */ + case 261: /* sclp */ + case 271: /* exprlist */ { -tSQLExprListDestroy((yypminor->yy158)); +tSQLExprListDestroy((yypminor->yy290)); } break; - case 248: /* where_opt */ - case 254: /* having_opt */ - case 259: /* expr */ - case 269: /* expritem */ + case 251: /* where_opt */ + case 257: /* having_opt */ + case 262: /* expr */ + case 272: /* expritem */ { -tSQLExprDestroy((yypminor->yy190)); +tSQLExprDestroy((yypminor->yy64)); } break; - case 257: /* union */ + case 260: /* union */ { -destroyAllSelectClause((yypminor->yy333)); +destroyAllSelectClause((yypminor->yy231)); } break; - case 264: /* sortitem */ + case 267: /* sortitem */ { -tVariantDestroy(&(yypminor->yy506)); +tVariantDestroy(&(yypminor->yy134)); } break; -/********* End destructor definitions *****************************************/ default: break; /* If no destructor action specified: do nothing */ } } @@ -1425,53 +1138,51 @@ tVariantDestroy(&(yypminor->yy506)); ** ** If there is a destructor routine associated with the token which ** is popped from the stack, then call it. +** +** Return the major token number for the symbol popped. */ -static void yy_pop_parser_stack(yyParser *pParser){ - yyStackEntry *yytos; - assert( pParser->yytos!=0 ); - assert( pParser->yytos > pParser->yystack ); - yytos = pParser->yytos--; +static int yy_pop_parser_stack(yyParser *pParser){ + YYCODETYPE yymajor; + yyStackEntry *yytos = &pParser->yystack[pParser->yyidx]; + + if( pParser->yyidx<0 ) return 0; #ifndef NDEBUG - if( yyTraceFILE ){ + if( yyTraceFILE && pParser->yyidx>=0 ){ fprintf(yyTraceFILE,"%sPopping %s\n", yyTracePrompt, yyTokenName[yytos->major]); } #endif - yy_destructor(pParser, yytos->major, &yytos->minor); + yymajor = yytos->major; + yy_destructor(pParser, yymajor, &yytos->minor); + pParser->yyidx--; + return yymajor; } -/* -** Clear all secondary memory allocations from the parser -*/ -void ParseFinalize(void *p){ - yyParser *pParser = (yyParser*)p; - while( pParser->yytos>pParser->yystack ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - if( pParser->yystack!=&pParser->yystk0 ) free(pParser->yystack); -#endif -} - -#ifndef Parse_ENGINEALWAYSONSTACK /* -** Deallocate and destroy a parser. Destructors are called for +** Deallocate and destroy a parser. Destructors are all called for ** all stack elements before shutting the parser down. ** -** If the YYPARSEFREENEVERNULL macro exists (for example because it -** is defined in a %include section of the input grammar) then it is -** assumed that the input pointer is never NULL. +** Inputs: +**
    +**
  • A pointer to the parser. This should be a pointer +** obtained from ParseAlloc. +**
  • A pointer to a function used to reclaim memory obtained +** from malloc. +**
*/ void ParseFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ -#ifndef YYPARSEFREENEVERNULL - if( p==0 ) return; + yyParser *pParser = (yyParser*)p; + if( pParser==0 ) return; + while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); +#if YYSTACKDEPTH<=0 + free(pParser->yystack); #endif - ParseFinalize(p); - (*freeProc)(p); + (*freeProc)((void*)pParser); } -#endif /* Parse_ENGINEALWAYSONSTACK */ /* ** Return the peak depth of the stack for a parser. @@ -1479,118 +1190,85 @@ void ParseFree( #ifdef YYTRACKMAXSTACKDEPTH int ParseStackPeak(void *p){ yyParser *pParser = (yyParser*)p; - return pParser->yyhwm; -} -#endif - -/* This array of booleans keeps track of the parser statement -** coverage. The element yycoverage[X][Y] is set when the parser -** is in state X and has a lookahead token Y. In a well-tested -** systems, every element of this matrix should end up being set. -*/ -#if defined(YYCOVERAGE) -static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; -#endif - -/* -** Write into out a description of every state/lookahead combination that -** -** (1) has not been used by the parser, and -** (2) is not a syntax error. -** -** Return the number of missed state/lookahead combinations. -*/ -#if defined(YYCOVERAGE) -int ParseCoverage(FILE *out){ - int stateno, iLookAhead, i; - int nMissed = 0; - for(stateno=0; statenoyyidxMax; } #endif /* ** Find the appropriate action for a parser given the terminal ** look-ahead token iLookAhead. +** +** If the look-ahead token is YYNOCODE, then check to see if the action is +** independent of the look-ahead. If it is, return the action, otherwise +** return YY_NO_ACTION. */ -static YYACTIONTYPE yy_find_shift_action( - YYCODETYPE iLookAhead, /* The look-ahead token */ - YYACTIONTYPE stateno /* Current state number */ +static int yy_find_shift_action( + yyParser *pParser, /* The parser */ + YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; - - if( stateno>YY_MAX_SHIFT ) return stateno; - assert( stateno <= YY_SHIFT_COUNT ); -#if defined(YYCOVERAGE) - yycoverage[stateno][iLookAhead] = 1; -#endif - do{ - i = yy_shift_ofst[stateno]; - assert( i>=0 ); - assert( i<=YY_ACTTAB_COUNT ); - assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); - assert( iLookAhead!=YYNOCODE ); - assert( iLookAhead < YYNTOKEN ); - i += iLookAhead; - assert( i<(int)YY_NLOOKAHEAD ); - if( yy_lookahead[i]!=iLookAhead ){ + int stateno = pParser->yystack[pParser->yyidx].stateno; + + if( stateno>YY_SHIFT_COUNT + || (i = yy_shift_ofst[stateno])==YY_SHIFT_USE_DFLT ){ + return yy_default[stateno]; + } + assert( iLookAhead!=YYNOCODE ); + i += iLookAhead; + if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ + if( iLookAhead>0 ){ #ifdef YYFALLBACK YYCODETYPE iFallback; /* Fallback token */ - assert( iLookAhead %s\n", yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); } #endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; + return yy_find_shift_action(pParser, iFallback); } #endif #ifdef YYWILDCARD { int j = i - iLookAhead + YYWILDCARD; - assert( j<(int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])) ); - if( yy_lookahead[j]==YYWILDCARD && iLookAhead>0 ){ + if( +#if YY_SHIFT_MIN+YYWILDCARD<0 + j>=0 && +#endif +#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT + j %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[YYWILDCARD]); } #endif /* NDEBUG */ return yy_action[j]; } } #endif /* YYWILDCARD */ - return yy_default[stateno]; - }else{ - assert( i>=0 && iyyidx--; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); } #endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser ** stack every overflows */ -/******** Begin %stack_overflow code ******************************************/ -/******** End %stack_overflow code ********************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument var */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument var */ } -/* -** Print tracing information for a SHIFT action -*/ -#ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ - if( yyTraceFILE ){ - if( yyNewStateyytos->major], - yyNewState); - }else{ - fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", - yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], - yyNewState - YY_MIN_REDUCE); - } - } -} -#else -# define yyTraceShift(X,Y,Z) -#endif - /* ** Perform a shift action. */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ - YYACTIONTYPE yyNewState, /* The new state to shift in */ - YYCODETYPE yyMajor, /* The major token to shift in */ - ParseTOKENTYPE yyMinor /* The minor token to shift in */ + int yyNewState, /* The new state to shift in */ + int yyMajor, /* The major token to shift in */ + YYMINORTYPE *yypMinor /* Pointer to the minor token to shift in */ ){ yyStackEntry *yytos; - yypParser->yytos++; + yypParser->yyidx++; #ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); + if( yypParser->yyidx>yypParser->yyidxMax ){ + yypParser->yyidxMax = yypParser->yyidx; } #endif #if YYSTACKDEPTH>0 - if( yypParser->yytos>yypParser->yystackEnd ){ - yypParser->yytos--; - yyStackOverflow(yypParser); + if( yypParser->yyidx>=YYSTACKDEPTH ){ + yyStackOverflow(yypParser, yypMinor); return; } #else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz] ){ - if( yyGrowStack(yypParser) ){ - yypParser->yytos--; - yyStackOverflow(yypParser); + if( yypParser->yyidx>=yypParser->yystksz ){ + yyGrowStack(yypParser); + if( yypParser->yyidx>=yypParser->yystksz ){ + yyStackOverflow(yypParser, yypMinor); return; } } #endif - if( yyNewState > YY_MAX_SHIFT ){ - yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + yytos = &yypParser->yystack[yypParser->yyidx]; + yytos->stateno = (YYACTIONTYPE)yyNewState; + yytos->major = (YYCODETYPE)yyMajor; + yytos->minor = *yypMinor; +#ifndef NDEBUG + if( yyTraceFILE && yypParser->yyidx>0 ){ + int i; + fprintf(yyTraceFILE,"%sShift %d\n",yyTracePrompt,yyNewState); + fprintf(yyTraceFILE,"%sStack:",yyTracePrompt); + for(i=1; i<=yypParser->yyidx; i++) + fprintf(yyTraceFILE," %s",yyTokenName[yypParser->yystack[i].major]); + fprintf(yyTraceFILE,"\n"); } - yytos = yypParser->yytos; - yytos->stateno = yyNewState; - yytos->major = yyMajor; - yytos->minor.yy0 = yyMinor; - yyTraceShift(yypParser, yyNewState, "Shift"); +#endif } -/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side -** of that rule */ -static const YYCODETYPE yyRuleInfoLhs[] = { - 206, /* (0) program ::= cmd */ - 207, /* (1) cmd ::= SHOW DATABASES */ - 207, /* (2) cmd ::= SHOW MNODES */ - 207, /* (3) cmd ::= SHOW DNODES */ - 207, /* (4) cmd ::= SHOW ACCOUNTS */ - 207, /* (5) cmd ::= SHOW USERS */ - 207, /* (6) cmd ::= SHOW MODULES */ - 207, /* (7) cmd ::= SHOW QUERIES */ - 207, /* (8) cmd ::= SHOW CONNECTIONS */ - 207, /* (9) cmd ::= SHOW STREAMS */ - 207, /* (10) cmd ::= SHOW VARIABLES */ - 207, /* (11) cmd ::= SHOW SCORES */ - 207, /* (12) cmd ::= SHOW GRANTS */ - 207, /* (13) cmd ::= SHOW VNODES */ - 207, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - 208, /* (15) dbPrefix ::= */ - 208, /* (16) dbPrefix ::= ids DOT */ - 210, /* (17) cpxName ::= */ - 210, /* (18) cpxName ::= DOT ids */ - 207, /* (19) cmd ::= SHOW dbPrefix TABLES */ - 207, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - 207, /* (21) cmd ::= SHOW dbPrefix STABLES */ - 207, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - 207, /* (23) cmd ::= SHOW dbPrefix VGROUPS */ - 207, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */ - 207, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */ - 207, /* (26) cmd ::= DROP DATABASE ifexists ids */ - 207, /* (27) cmd ::= DROP DNODE ids */ - 207, /* (28) cmd ::= DROP USER ids */ - 207, /* (29) cmd ::= DROP ACCOUNT ids */ - 207, /* (30) cmd ::= USE ids */ - 207, /* (31) cmd ::= DESCRIBE ids cpxName */ - 207, /* (32) cmd ::= ALTER USER ids PASS ids */ - 207, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */ - 207, /* (34) cmd ::= ALTER DNODE ids ids */ - 207, /* (35) cmd ::= ALTER DNODE ids ids ids */ - 207, /* (36) cmd ::= ALTER LOCAL ids */ - 207, /* (37) cmd ::= ALTER LOCAL ids ids */ - 207, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */ - 207, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */ - 207, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - 209, /* (41) ids ::= ID */ - 209, /* (42) ids ::= STRING */ - 211, /* (43) ifexists ::= IF EXISTS */ - 211, /* (44) ifexists ::= */ - 214, /* (45) ifnotexists ::= IF NOT EXISTS */ - 214, /* (46) ifnotexists ::= */ - 207, /* (47) cmd ::= CREATE DNODE ids */ - 207, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - 207, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - 207, /* (50) cmd ::= CREATE USER ids PASS ids */ - 216, /* (51) pps ::= */ - 216, /* (52) pps ::= PPS INTEGER */ - 217, /* (53) tseries ::= */ - 217, /* (54) tseries ::= TSERIES INTEGER */ - 218, /* (55) dbs ::= */ - 218, /* (56) dbs ::= DBS INTEGER */ - 219, /* (57) streams ::= */ - 219, /* (58) streams ::= STREAMS INTEGER */ - 220, /* (59) storage ::= */ - 220, /* (60) storage ::= STORAGE INTEGER */ - 221, /* (61) qtime ::= */ - 221, /* (62) qtime ::= QTIME INTEGER */ - 222, /* (63) users ::= */ - 222, /* (64) users ::= USERS INTEGER */ - 223, /* (65) conns ::= */ - 223, /* (66) conns ::= CONNS INTEGER */ - 224, /* (67) state ::= */ - 224, /* (68) state ::= STATE ids */ - 213, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - 225, /* (70) keep ::= KEEP tagitemlist */ - 227, /* (71) cache ::= CACHE INTEGER */ - 228, /* (72) replica ::= REPLICA INTEGER */ - 229, /* (73) quorum ::= QUORUM INTEGER */ - 230, /* (74) days ::= DAYS INTEGER */ - 231, /* (75) minrows ::= MINROWS INTEGER */ - 232, /* (76) maxrows ::= MAXROWS INTEGER */ - 233, /* (77) blocks ::= BLOCKS INTEGER */ - 234, /* (78) ctime ::= CTIME INTEGER */ - 235, /* (79) wal ::= WAL INTEGER */ - 236, /* (80) fsync ::= FSYNC INTEGER */ - 237, /* (81) comp ::= COMP INTEGER */ - 238, /* (82) prec ::= PRECISION STRING */ - 215, /* (83) db_optr ::= */ - 215, /* (84) db_optr ::= db_optr cache */ - 215, /* (85) db_optr ::= db_optr replica */ - 215, /* (86) db_optr ::= db_optr quorum */ - 215, /* (87) db_optr ::= db_optr days */ - 215, /* (88) db_optr ::= db_optr minrows */ - 215, /* (89) db_optr ::= db_optr maxrows */ - 215, /* (90) db_optr ::= db_optr blocks */ - 215, /* (91) db_optr ::= db_optr ctime */ - 215, /* (92) db_optr ::= db_optr wal */ - 215, /* (93) db_optr ::= db_optr fsync */ - 215, /* (94) db_optr ::= db_optr comp */ - 215, /* (95) db_optr ::= db_optr prec */ - 215, /* (96) db_optr ::= db_optr keep */ - 212, /* (97) alter_db_optr ::= */ - 212, /* (98) alter_db_optr ::= alter_db_optr replica */ - 212, /* (99) alter_db_optr ::= alter_db_optr quorum */ - 212, /* (100) alter_db_optr ::= alter_db_optr keep */ - 212, /* (101) alter_db_optr ::= alter_db_optr blocks */ - 212, /* (102) alter_db_optr ::= alter_db_optr comp */ - 212, /* (103) alter_db_optr ::= alter_db_optr wal */ - 212, /* (104) alter_db_optr ::= alter_db_optr fsync */ - 239, /* (105) typename ::= ids */ - 239, /* (106) typename ::= ids LP signed RP */ - 240, /* (107) signed ::= INTEGER */ - 240, /* (108) signed ::= PLUS INTEGER */ - 240, /* (109) signed ::= MINUS INTEGER */ - 207, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ - 241, /* (111) create_table_args ::= LP columnlist RP */ - 241, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ - 241, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ - 241, /* (114) create_table_args ::= AS select */ - 242, /* (115) columnlist ::= columnlist COMMA column */ - 242, /* (116) columnlist ::= column */ - 244, /* (117) column ::= ids typename */ - 226, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */ - 226, /* (119) tagitemlist ::= tagitem */ - 245, /* (120) tagitem ::= INTEGER */ - 245, /* (121) tagitem ::= FLOAT */ - 245, /* (122) tagitem ::= STRING */ - 245, /* (123) tagitem ::= BOOL */ - 245, /* (124) tagitem ::= NULL */ - 245, /* (125) tagitem ::= MINUS INTEGER */ - 245, /* (126) tagitem ::= MINUS FLOAT */ - 245, /* (127) tagitem ::= PLUS INTEGER */ - 245, /* (128) tagitem ::= PLUS FLOAT */ - 243, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - 257, /* (130) union ::= select */ - 257, /* (131) union ::= LP union RP */ - 257, /* (132) union ::= union UNION ALL select */ - 257, /* (133) union ::= union UNION ALL LP select RP */ - 207, /* (134) cmd ::= union */ - 243, /* (135) select ::= SELECT selcollist */ - 258, /* (136) sclp ::= selcollist COMMA */ - 258, /* (137) sclp ::= */ - 246, /* (138) selcollist ::= sclp expr as */ - 246, /* (139) selcollist ::= sclp STAR */ - 260, /* (140) as ::= AS ids */ - 260, /* (141) as ::= ids */ - 260, /* (142) as ::= */ - 247, /* (143) from ::= FROM tablelist */ - 261, /* (144) tablelist ::= ids cpxName */ - 261, /* (145) tablelist ::= ids cpxName ids */ - 261, /* (146) tablelist ::= tablelist COMMA ids cpxName */ - 261, /* (147) tablelist ::= tablelist COMMA ids cpxName ids */ - 262, /* (148) tmvar ::= VARIABLE */ - 249, /* (149) interval_opt ::= INTERVAL LP tmvar RP */ - 249, /* (150) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - 249, /* (151) interval_opt ::= */ - 250, /* (152) fill_opt ::= */ - 250, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - 250, /* (154) fill_opt ::= FILL LP ID RP */ - 251, /* (155) sliding_opt ::= SLIDING LP tmvar RP */ - 251, /* (156) sliding_opt ::= */ - 253, /* (157) orderby_opt ::= */ - 253, /* (158) orderby_opt ::= ORDER BY sortlist */ - 263, /* (159) sortlist ::= sortlist COMMA item sortorder */ - 263, /* (160) sortlist ::= item sortorder */ - 265, /* (161) item ::= ids cpxName */ - 266, /* (162) sortorder ::= ASC */ - 266, /* (163) sortorder ::= DESC */ - 266, /* (164) sortorder ::= */ - 252, /* (165) groupby_opt ::= */ - 252, /* (166) groupby_opt ::= GROUP BY grouplist */ - 267, /* (167) grouplist ::= grouplist COMMA item */ - 267, /* (168) grouplist ::= item */ - 254, /* (169) having_opt ::= */ - 254, /* (170) having_opt ::= HAVING expr */ - 256, /* (171) limit_opt ::= */ - 256, /* (172) limit_opt ::= LIMIT signed */ - 256, /* (173) limit_opt ::= LIMIT signed OFFSET signed */ - 256, /* (174) limit_opt ::= LIMIT signed COMMA signed */ - 255, /* (175) slimit_opt ::= */ - 255, /* (176) slimit_opt ::= SLIMIT signed */ - 255, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */ - 255, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */ - 248, /* (179) where_opt ::= */ - 248, /* (180) where_opt ::= WHERE expr */ - 259, /* (181) expr ::= LP expr RP */ - 259, /* (182) expr ::= ID */ - 259, /* (183) expr ::= ID DOT ID */ - 259, /* (184) expr ::= ID DOT STAR */ - 259, /* (185) expr ::= INTEGER */ - 259, /* (186) expr ::= MINUS INTEGER */ - 259, /* (187) expr ::= PLUS INTEGER */ - 259, /* (188) expr ::= FLOAT */ - 259, /* (189) expr ::= MINUS FLOAT */ - 259, /* (190) expr ::= PLUS FLOAT */ - 259, /* (191) expr ::= STRING */ - 259, /* (192) expr ::= NOW */ - 259, /* (193) expr ::= VARIABLE */ - 259, /* (194) expr ::= BOOL */ - 259, /* (195) expr ::= ID LP exprlist RP */ - 259, /* (196) expr ::= ID LP STAR RP */ - 259, /* (197) expr ::= expr IS NULL */ - 259, /* (198) expr ::= expr IS NOT NULL */ - 259, /* (199) expr ::= expr LT expr */ - 259, /* (200) expr ::= expr GT expr */ - 259, /* (201) expr ::= expr LE expr */ - 259, /* (202) expr ::= expr GE expr */ - 259, /* (203) expr ::= expr NE expr */ - 259, /* (204) expr ::= expr EQ expr */ - 259, /* (205) expr ::= expr AND expr */ - 259, /* (206) expr ::= expr OR expr */ - 259, /* (207) expr ::= expr PLUS expr */ - 259, /* (208) expr ::= expr MINUS expr */ - 259, /* (209) expr ::= expr STAR expr */ - 259, /* (210) expr ::= expr SLASH expr */ - 259, /* (211) expr ::= expr REM expr */ - 259, /* (212) expr ::= expr LIKE expr */ - 259, /* (213) expr ::= expr IN LP exprlist RP */ - 268, /* (214) exprlist ::= exprlist COMMA expritem */ - 268, /* (215) exprlist ::= expritem */ - 269, /* (216) expritem ::= expr */ - 269, /* (217) expritem ::= */ - 207, /* (218) cmd ::= RESET QUERY CACHE */ - 207, /* (219) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - 207, /* (220) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - 207, /* (221) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - 207, /* (222) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - 207, /* (223) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - 207, /* (224) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - 207, /* (225) cmd ::= KILL CONNECTION INTEGER */ - 207, /* (226) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - 207, /* (227) cmd ::= KILL QUERY INTEGER COLON INTEGER */ -}; - -/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number -** of symbols on the right-hand side of that rule. */ -static const signed char yyRuleInfoNRhs[] = { - -1, /* (0) program ::= cmd */ - -2, /* (1) cmd ::= SHOW DATABASES */ - -2, /* (2) cmd ::= SHOW MNODES */ - -2, /* (3) cmd ::= SHOW DNODES */ - -2, /* (4) cmd ::= SHOW ACCOUNTS */ - -2, /* (5) cmd ::= SHOW USERS */ - -2, /* (6) cmd ::= SHOW MODULES */ - -2, /* (7) cmd ::= SHOW QUERIES */ - -2, /* (8) cmd ::= SHOW CONNECTIONS */ - -2, /* (9) cmd ::= SHOW STREAMS */ - -2, /* (10) cmd ::= SHOW VARIABLES */ - -2, /* (11) cmd ::= SHOW SCORES */ - -2, /* (12) cmd ::= SHOW GRANTS */ - -2, /* (13) cmd ::= SHOW VNODES */ - -3, /* (14) cmd ::= SHOW VNODES IPTOKEN */ - 0, /* (15) dbPrefix ::= */ - -2, /* (16) dbPrefix ::= ids DOT */ - 0, /* (17) cpxName ::= */ - -2, /* (18) cpxName ::= DOT ids */ - -3, /* (19) cmd ::= SHOW dbPrefix TABLES */ - -5, /* (20) cmd ::= SHOW dbPrefix TABLES LIKE ids */ - -3, /* (21) cmd ::= SHOW dbPrefix STABLES */ - -5, /* (22) cmd ::= SHOW dbPrefix STABLES LIKE ids */ - -3, /* (23) cmd ::= SHOW dbPrefix VGROUPS */ - -4, /* (24) cmd ::= SHOW dbPrefix VGROUPS ids */ - -5, /* (25) cmd ::= DROP TABLE ifexists ids cpxName */ - -4, /* (26) cmd ::= DROP DATABASE ifexists ids */ - -3, /* (27) cmd ::= DROP DNODE ids */ - -3, /* (28) cmd ::= DROP USER ids */ - -3, /* (29) cmd ::= DROP ACCOUNT ids */ - -2, /* (30) cmd ::= USE ids */ - -3, /* (31) cmd ::= DESCRIBE ids cpxName */ - -5, /* (32) cmd ::= ALTER USER ids PASS ids */ - -5, /* (33) cmd ::= ALTER USER ids PRIVILEGE ids */ - -4, /* (34) cmd ::= ALTER DNODE ids ids */ - -5, /* (35) cmd ::= ALTER DNODE ids ids ids */ - -3, /* (36) cmd ::= ALTER LOCAL ids */ - -4, /* (37) cmd ::= ALTER LOCAL ids ids */ - -4, /* (38) cmd ::= ALTER DATABASE ids alter_db_optr */ - -4, /* (39) cmd ::= ALTER ACCOUNT ids acct_optr */ - -6, /* (40) cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ - -1, /* (41) ids ::= ID */ - -1, /* (42) ids ::= STRING */ - -2, /* (43) ifexists ::= IF EXISTS */ - 0, /* (44) ifexists ::= */ - -3, /* (45) ifnotexists ::= IF NOT EXISTS */ - 0, /* (46) ifnotexists ::= */ - -3, /* (47) cmd ::= CREATE DNODE ids */ - -6, /* (48) cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ - -5, /* (49) cmd ::= CREATE DATABASE ifnotexists ids db_optr */ - -5, /* (50) cmd ::= CREATE USER ids PASS ids */ - 0, /* (51) pps ::= */ - -2, /* (52) pps ::= PPS INTEGER */ - 0, /* (53) tseries ::= */ - -2, /* (54) tseries ::= TSERIES INTEGER */ - 0, /* (55) dbs ::= */ - -2, /* (56) dbs ::= DBS INTEGER */ - 0, /* (57) streams ::= */ - -2, /* (58) streams ::= STREAMS INTEGER */ - 0, /* (59) storage ::= */ - -2, /* (60) storage ::= STORAGE INTEGER */ - 0, /* (61) qtime ::= */ - -2, /* (62) qtime ::= QTIME INTEGER */ - 0, /* (63) users ::= */ - -2, /* (64) users ::= USERS INTEGER */ - 0, /* (65) conns ::= */ - -2, /* (66) conns ::= CONNS INTEGER */ - 0, /* (67) state ::= */ - -2, /* (68) state ::= STATE ids */ - -9, /* (69) acct_optr ::= pps tseries storage streams qtime dbs users conns state */ - -2, /* (70) keep ::= KEEP tagitemlist */ - -2, /* (71) cache ::= CACHE INTEGER */ - -2, /* (72) replica ::= REPLICA INTEGER */ - -2, /* (73) quorum ::= QUORUM INTEGER */ - -2, /* (74) days ::= DAYS INTEGER */ - -2, /* (75) minrows ::= MINROWS INTEGER */ - -2, /* (76) maxrows ::= MAXROWS INTEGER */ - -2, /* (77) blocks ::= BLOCKS INTEGER */ - -2, /* (78) ctime ::= CTIME INTEGER */ - -2, /* (79) wal ::= WAL INTEGER */ - -2, /* (80) fsync ::= FSYNC INTEGER */ - -2, /* (81) comp ::= COMP INTEGER */ - -2, /* (82) prec ::= PRECISION STRING */ - 0, /* (83) db_optr ::= */ - -2, /* (84) db_optr ::= db_optr cache */ - -2, /* (85) db_optr ::= db_optr replica */ - -2, /* (86) db_optr ::= db_optr quorum */ - -2, /* (87) db_optr ::= db_optr days */ - -2, /* (88) db_optr ::= db_optr minrows */ - -2, /* (89) db_optr ::= db_optr maxrows */ - -2, /* (90) db_optr ::= db_optr blocks */ - -2, /* (91) db_optr ::= db_optr ctime */ - -2, /* (92) db_optr ::= db_optr wal */ - -2, /* (93) db_optr ::= db_optr fsync */ - -2, /* (94) db_optr ::= db_optr comp */ - -2, /* (95) db_optr ::= db_optr prec */ - -2, /* (96) db_optr ::= db_optr keep */ - 0, /* (97) alter_db_optr ::= */ - -2, /* (98) alter_db_optr ::= alter_db_optr replica */ - -2, /* (99) alter_db_optr ::= alter_db_optr quorum */ - -2, /* (100) alter_db_optr ::= alter_db_optr keep */ - -2, /* (101) alter_db_optr ::= alter_db_optr blocks */ - -2, /* (102) alter_db_optr ::= alter_db_optr comp */ - -2, /* (103) alter_db_optr ::= alter_db_optr wal */ - -2, /* (104) alter_db_optr ::= alter_db_optr fsync */ - -1, /* (105) typename ::= ids */ - -4, /* (106) typename ::= ids LP signed RP */ - -1, /* (107) signed ::= INTEGER */ - -2, /* (108) signed ::= PLUS INTEGER */ - -2, /* (109) signed ::= MINUS INTEGER */ - -6, /* (110) cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ - -3, /* (111) create_table_args ::= LP columnlist RP */ - -7, /* (112) create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ - -7, /* (113) create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ - -2, /* (114) create_table_args ::= AS select */ - -3, /* (115) columnlist ::= columnlist COMMA column */ - -1, /* (116) columnlist ::= column */ - -2, /* (117) column ::= ids typename */ - -3, /* (118) tagitemlist ::= tagitemlist COMMA tagitem */ - -1, /* (119) tagitemlist ::= tagitem */ - -1, /* (120) tagitem ::= INTEGER */ - -1, /* (121) tagitem ::= FLOAT */ - -1, /* (122) tagitem ::= STRING */ - -1, /* (123) tagitem ::= BOOL */ - -1, /* (124) tagitem ::= NULL */ - -2, /* (125) tagitem ::= MINUS INTEGER */ - -2, /* (126) tagitem ::= MINUS FLOAT */ - -2, /* (127) tagitem ::= PLUS INTEGER */ - -2, /* (128) tagitem ::= PLUS FLOAT */ - -12, /* (129) select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ - -1, /* (130) union ::= select */ - -3, /* (131) union ::= LP union RP */ - -4, /* (132) union ::= union UNION ALL select */ - -6, /* (133) union ::= union UNION ALL LP select RP */ - -1, /* (134) cmd ::= union */ - -2, /* (135) select ::= SELECT selcollist */ - -2, /* (136) sclp ::= selcollist COMMA */ - 0, /* (137) sclp ::= */ - -3, /* (138) selcollist ::= sclp expr as */ - -2, /* (139) selcollist ::= sclp STAR */ - -2, /* (140) as ::= AS ids */ - -1, /* (141) as ::= ids */ - 0, /* (142) as ::= */ - -2, /* (143) from ::= FROM tablelist */ - -2, /* (144) tablelist ::= ids cpxName */ - -3, /* (145) tablelist ::= ids cpxName ids */ - -4, /* (146) tablelist ::= tablelist COMMA ids cpxName */ - -5, /* (147) tablelist ::= tablelist COMMA ids cpxName ids */ - -1, /* (148) tmvar ::= VARIABLE */ - -4, /* (149) interval_opt ::= INTERVAL LP tmvar RP */ - -6, /* (150) interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ - 0, /* (151) interval_opt ::= */ - 0, /* (152) fill_opt ::= */ - -6, /* (153) fill_opt ::= FILL LP ID COMMA tagitemlist RP */ - -4, /* (154) fill_opt ::= FILL LP ID RP */ - -4, /* (155) sliding_opt ::= SLIDING LP tmvar RP */ - 0, /* (156) sliding_opt ::= */ - 0, /* (157) orderby_opt ::= */ - -3, /* (158) orderby_opt ::= ORDER BY sortlist */ - -4, /* (159) sortlist ::= sortlist COMMA item sortorder */ - -2, /* (160) sortlist ::= item sortorder */ - -2, /* (161) item ::= ids cpxName */ - -1, /* (162) sortorder ::= ASC */ - -1, /* (163) sortorder ::= DESC */ - 0, /* (164) sortorder ::= */ - 0, /* (165) groupby_opt ::= */ - -3, /* (166) groupby_opt ::= GROUP BY grouplist */ - -3, /* (167) grouplist ::= grouplist COMMA item */ - -1, /* (168) grouplist ::= item */ - 0, /* (169) having_opt ::= */ - -2, /* (170) having_opt ::= HAVING expr */ - 0, /* (171) limit_opt ::= */ - -2, /* (172) limit_opt ::= LIMIT signed */ - -4, /* (173) limit_opt ::= LIMIT signed OFFSET signed */ - -4, /* (174) limit_opt ::= LIMIT signed COMMA signed */ - 0, /* (175) slimit_opt ::= */ - -2, /* (176) slimit_opt ::= SLIMIT signed */ - -4, /* (177) slimit_opt ::= SLIMIT signed SOFFSET signed */ - -4, /* (178) slimit_opt ::= SLIMIT signed COMMA signed */ - 0, /* (179) where_opt ::= */ - -2, /* (180) where_opt ::= WHERE expr */ - -3, /* (181) expr ::= LP expr RP */ - -1, /* (182) expr ::= ID */ - -3, /* (183) expr ::= ID DOT ID */ - -3, /* (184) expr ::= ID DOT STAR */ - -1, /* (185) expr ::= INTEGER */ - -2, /* (186) expr ::= MINUS INTEGER */ - -2, /* (187) expr ::= PLUS INTEGER */ - -1, /* (188) expr ::= FLOAT */ - -2, /* (189) expr ::= MINUS FLOAT */ - -2, /* (190) expr ::= PLUS FLOAT */ - -1, /* (191) expr ::= STRING */ - -1, /* (192) expr ::= NOW */ - -1, /* (193) expr ::= VARIABLE */ - -1, /* (194) expr ::= BOOL */ - -4, /* (195) expr ::= ID LP exprlist RP */ - -4, /* (196) expr ::= ID LP STAR RP */ - -3, /* (197) expr ::= expr IS NULL */ - -4, /* (198) expr ::= expr IS NOT NULL */ - -3, /* (199) expr ::= expr LT expr */ - -3, /* (200) expr ::= expr GT expr */ - -3, /* (201) expr ::= expr LE expr */ - -3, /* (202) expr ::= expr GE expr */ - -3, /* (203) expr ::= expr NE expr */ - -3, /* (204) expr ::= expr EQ expr */ - -3, /* (205) expr ::= expr AND expr */ - -3, /* (206) expr ::= expr OR expr */ - -3, /* (207) expr ::= expr PLUS expr */ - -3, /* (208) expr ::= expr MINUS expr */ - -3, /* (209) expr ::= expr STAR expr */ - -3, /* (210) expr ::= expr SLASH expr */ - -3, /* (211) expr ::= expr REM expr */ - -3, /* (212) expr ::= expr LIKE expr */ - -5, /* (213) expr ::= expr IN LP exprlist RP */ - -3, /* (214) exprlist ::= exprlist COMMA expritem */ - -1, /* (215) exprlist ::= expritem */ - -1, /* (216) expritem ::= expr */ - 0, /* (217) expritem ::= */ - -3, /* (218) cmd ::= RESET QUERY CACHE */ - -7, /* (219) cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ - -7, /* (220) cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ - -7, /* (221) cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ - -7, /* (222) cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ - -8, /* (223) cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ - -9, /* (224) cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ - -3, /* (225) cmd ::= KILL CONNECTION INTEGER */ - -5, /* (226) cmd ::= KILL STREAM INTEGER COLON INTEGER */ - -5, /* (227) cmd ::= KILL QUERY INTEGER COLON INTEGER */ +/* The following table contains information about every rule that +** is used during the reduce. +*/ +static const struct { + YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ + unsigned char nrhs; /* Number of right-hand side symbols in the rule */ +} yyRuleInfo[] = { + { 208, 1 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 2 }, + { 209, 3 }, + { 210, 0 }, + { 210, 2 }, + { 212, 0 }, + { 212, 2 }, + { 209, 3 }, + { 209, 5 }, + { 209, 3 }, + { 209, 5 }, + { 209, 3 }, + { 209, 4 }, + { 209, 5 }, + { 209, 4 }, + { 209, 3 }, + { 209, 3 }, + { 209, 3 }, + { 209, 2 }, + { 209, 3 }, + { 209, 5 }, + { 209, 5 }, + { 209, 4 }, + { 209, 5 }, + { 209, 3 }, + { 209, 4 }, + { 209, 4 }, + { 209, 4 }, + { 209, 6 }, + { 211, 1 }, + { 211, 1 }, + { 213, 2 }, + { 213, 0 }, + { 216, 3 }, + { 216, 0 }, + { 209, 3 }, + { 209, 6 }, + { 209, 5 }, + { 209, 5 }, + { 218, 0 }, + { 218, 2 }, + { 219, 0 }, + { 219, 2 }, + { 220, 0 }, + { 220, 2 }, + { 221, 0 }, + { 221, 2 }, + { 222, 0 }, + { 222, 2 }, + { 223, 0 }, + { 223, 2 }, + { 224, 0 }, + { 224, 2 }, + { 225, 0 }, + { 225, 2 }, + { 226, 0 }, + { 226, 2 }, + { 215, 9 }, + { 227, 2 }, + { 229, 2 }, + { 230, 2 }, + { 231, 2 }, + { 232, 2 }, + { 233, 2 }, + { 234, 2 }, + { 235, 2 }, + { 236, 2 }, + { 237, 2 }, + { 238, 2 }, + { 239, 2 }, + { 240, 2 }, + { 241, 2 }, + { 217, 0 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 217, 2 }, + { 214, 0 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 214, 2 }, + { 242, 1 }, + { 242, 4 }, + { 243, 1 }, + { 243, 2 }, + { 243, 2 }, + { 209, 6 }, + { 244, 3 }, + { 244, 7 }, + { 244, 7 }, + { 244, 2 }, + { 245, 3 }, + { 245, 1 }, + { 247, 2 }, + { 228, 3 }, + { 228, 1 }, + { 248, 1 }, + { 248, 1 }, + { 248, 1 }, + { 248, 1 }, + { 248, 1 }, + { 248, 2 }, + { 248, 2 }, + { 248, 2 }, + { 248, 2 }, + { 246, 12 }, + { 260, 1 }, + { 260, 3 }, + { 260, 4 }, + { 260, 6 }, + { 209, 1 }, + { 246, 2 }, + { 261, 2 }, + { 261, 0 }, + { 249, 3 }, + { 249, 2 }, + { 263, 2 }, + { 263, 1 }, + { 263, 0 }, + { 250, 2 }, + { 264, 2 }, + { 264, 3 }, + { 264, 4 }, + { 264, 5 }, + { 265, 1 }, + { 252, 4 }, + { 252, 6 }, + { 252, 0 }, + { 253, 0 }, + { 253, 6 }, + { 253, 4 }, + { 254, 4 }, + { 254, 0 }, + { 256, 0 }, + { 256, 3 }, + { 266, 4 }, + { 266, 2 }, + { 268, 2 }, + { 269, 1 }, + { 269, 1 }, + { 269, 0 }, + { 255, 0 }, + { 255, 3 }, + { 270, 3 }, + { 270, 1 }, + { 257, 0 }, + { 257, 2 }, + { 259, 0 }, + { 259, 2 }, + { 259, 4 }, + { 259, 4 }, + { 258, 0 }, + { 258, 2 }, + { 258, 4 }, + { 258, 4 }, + { 251, 0 }, + { 251, 2 }, + { 262, 3 }, + { 262, 1 }, + { 262, 3 }, + { 262, 3 }, + { 262, 1 }, + { 262, 2 }, + { 262, 2 }, + { 262, 1 }, + { 262, 2 }, + { 262, 2 }, + { 262, 1 }, + { 262, 1 }, + { 262, 1 }, + { 262, 1 }, + { 262, 4 }, + { 262, 4 }, + { 262, 3 }, + { 262, 4 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 3 }, + { 262, 5 }, + { 271, 3 }, + { 271, 1 }, + { 272, 1 }, + { 272, 0 }, + { 209, 3 }, + { 209, 7 }, + { 209, 7 }, + { 209, 7 }, + { 209, 7 }, + { 209, 8 }, + { 209, 9 }, + { 209, 3 }, + { 209, 5 }, + { 209, 5 }, }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -2169,76 +1602,43 @@ static void yy_accept(yyParser*); /* Forward Declaration */ /* ** Perform a reduce action and the shift that must immediately ** follow the reduce. -** -** The yyLookahead and yyLookaheadToken parameters provide reduce actions -** access to the lookahead token (if any). The yyLookahead will be YYNOCODE -** if the lookahead token has already been consumed. As this procedure is -** only called from one place, optimizing compilers will in-line it, which -** means that the extra parameters have no performance impact. */ -static YYACTIONTYPE yy_reduce( +static void yy_reduce( yyParser *yypParser, /* The parser */ - unsigned int yyruleno, /* Number of the rule by which to reduce */ - int yyLookahead, /* Lookahead token, or YYNOCODE if none */ - ParseTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ - ParseCTX_PDECL /* %extra_context */ + int yyruleno /* Number of the rule by which to reduce */ ){ int yygoto; /* The next state */ - YYACTIONTYPE yyact; /* The next action */ + int yyact; /* The next action */ + YYMINORTYPE yygotominor; /* The LHS of the rule reduced */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - ParseARG_FETCH - (void)yyLookahead; - (void)yyLookaheadToken; - yymsp = yypParser->yytos; + ParseARG_FETCH; + yymsp = &yypParser->yystack[yypParser->yyidx]; #ifndef NDEBUG - if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfoNRhs[yyruleno]; - if( yysize ){ - fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", - yyTracePrompt, - yyruleno, yyRuleName[yyruleno], - yyruleno=0 + && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ + fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt, + yyRuleName[yyruleno]); } #endif /* NDEBUG */ - /* Check that the stack is large enough to grow by a single entry - ** if the RHS of the rule is empty. This ensures that there is room - ** enough on the stack to push the LHS value */ - if( yyRuleInfoNRhs[yyruleno]==0 ){ -#ifdef YYTRACKMAXSTACKDEPTH - if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ - yypParser->yyhwm++; - assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack)); - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yytos>=yypParser->yystackEnd ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; - } -#else - if( yypParser->yytos>=&yypParser->yystack[yypParser->yystksz-1] ){ - if( yyGrowStack(yypParser) ){ - yyStackOverflow(yypParser); - /* The call to yyStackOverflow() above pops the stack until it is - ** empty, causing the main parser loop to exit. So the return value - ** is never used and does not matter. */ - return 0; - } - yymsp = yypParser->yytos; - } -#endif - } + /* Silence complaints from purify about yygotominor being uninitialized + ** in some cases when it is copied into the stack after the following + ** switch. yygotominor is uninitialized when a rule reduces that does + ** not set the value of its left-hand side nonterminal. Leaving the + ** value of the nonterminal uninitialized is utterly harmless as long + ** as the value is never used. So really the only thing this code + ** accomplishes is to quieten purify. + ** + ** 2007-01-16: The wireshark project (www.wireshark.org) reports that + ** without this code, their parser segfaults. I'm not sure what there + ** parser is doing to make this happen. This is the second bug report + ** from wireshark this week. Clearly they are stressing Lemon in ways + ** that it has not been previously stressed... (SQLite ticket #2172) + */ + /*memset(&yygotominor, 0, sizeof(yygotominor));*/ + yygotominor = yyzerominor; + switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -2249,8 +1649,6 @@ static YYACTIONTYPE yy_reduce( ** #line ** break; */ -/********** Begin reduce actions **********************************************/ - YYMINORTYPE yylhsminor; case 0: /* program ::= cmd */ {} break; @@ -2297,17 +1695,16 @@ static YYACTIONTYPE yy_reduce( { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &yymsp[0].minor.yy0, 0); } break; case 15: /* dbPrefix ::= */ -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.type = 0;} +{yygotominor.yy0.n = 0; yygotominor.yy0.type = 0;} break; case 16: /* dbPrefix ::= ids DOT */ -{yylhsminor.yy0 = yymsp[-1].minor.yy0; } - yymsp[-1].minor.yy0 = yylhsminor.yy0; +{yygotominor.yy0 = yymsp[-1].minor.yy0; } break; case 17: /* cpxName ::= */ -{yymsp[1].minor.yy0.n = 0; } +{yygotominor.yy0.n = 0; } break; case 18: /* cpxName ::= DOT ids */ -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n += 1; } +{yygotominor.yy0 = yymsp[0].minor.yy0; yygotominor.yy0.n += 1; } break; case 19: /* cmd ::= SHOW dbPrefix TABLES */ { @@ -2391,37 +1788,34 @@ static YYACTIONTYPE yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy118, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} break; case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy479);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} break; case 40: /* cmd ::= ALTER ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy479);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} break; case 41: /* ids ::= ID */ case 42: /* ids ::= STRING */ yytestcase(yyruleno==42); -{yylhsminor.yy0 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy0 = yylhsminor.yy0; +{yygotominor.yy0 = yymsp[0].minor.yy0; } break; case 43: /* ifexists ::= IF EXISTS */ -{yymsp[-1].minor.yy0.n = 1;} + case 45: /* ifnotexists ::= IF NOT EXISTS */ yytestcase(yyruleno==45); +{yygotominor.yy0.n = 1;} break; case 44: /* ifexists ::= */ case 46: /* ifnotexists ::= */ yytestcase(yyruleno==46); -{yymsp[1].minor.yy0.n = 0;} - break; - case 45: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy0.n = 1;} +{yygotominor.yy0.n = 0;} break; case 47: /* cmd ::= CREATE DNODE ids */ { setDCLSQLElems(pInfo, TSDB_SQL_CREATE_DNODE, 1, &yymsp[0].minor.yy0);} break; case 48: /* cmd ::= CREATE ACCOUNT ids PASS ids acct_optr */ -{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy479);} +{ setCreateAcctSQL(pInfo, TSDB_SQL_CREATE_ACCT, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy149);} break; case 49: /* cmd ::= CREATE DATABASE ifnotexists ids db_optr */ -{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy118, &yymsp[-2].minor.yy0);} +{ setCreateDBSQL(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &yymsp[-2].minor.yy0);} break; case 50: /* cmd ::= CREATE USER ids PASS ids */ { setCreateUserSQL(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);} @@ -2435,7 +1829,7 @@ static YYACTIONTYPE yy_reduce( case 63: /* users ::= */ yytestcase(yyruleno==63); case 65: /* conns ::= */ yytestcase(yyruleno==65); case 67: /* state ::= */ yytestcase(yyruleno==67); -{yymsp[1].minor.yy0.n = 0; } +{yygotominor.yy0.n = 0; } break; case 52: /* pps ::= PPS INTEGER */ case 54: /* tseries ::= TSERIES INTEGER */ yytestcase(yyruleno==54); @@ -2446,24 +1840,23 @@ static YYACTIONTYPE yy_reduce( case 64: /* users ::= USERS INTEGER */ yytestcase(yyruleno==64); case 66: /* conns ::= CONNS INTEGER */ yytestcase(yyruleno==66); case 68: /* state ::= STATE ids */ yytestcase(yyruleno==68); -{yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } +{yygotominor.yy0 = yymsp[0].minor.yy0; } break; case 69: /* acct_optr ::= pps tseries storage streams qtime dbs users conns state */ { - yylhsminor.yy479.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; - yylhsminor.yy479.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; - yylhsminor.yy479.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; - yylhsminor.yy479.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; - yylhsminor.yy479.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; - yylhsminor.yy479.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy479.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; - yylhsminor.yy479.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; - yylhsminor.yy479.stat = yymsp[0].minor.yy0; + yygotominor.yy149.maxUsers = (yymsp[-2].minor.yy0.n>0)?atoi(yymsp[-2].minor.yy0.z):-1; + yygotominor.yy149.maxDbs = (yymsp[-3].minor.yy0.n>0)?atoi(yymsp[-3].minor.yy0.z):-1; + yygotominor.yy149.maxTimeSeries = (yymsp[-7].minor.yy0.n>0)?atoi(yymsp[-7].minor.yy0.z):-1; + yygotominor.yy149.maxStreams = (yymsp[-5].minor.yy0.n>0)?atoi(yymsp[-5].minor.yy0.z):-1; + yygotominor.yy149.maxPointsPerSecond = (yymsp[-8].minor.yy0.n>0)?atoi(yymsp[-8].minor.yy0.z):-1; + yygotominor.yy149.maxStorage = (yymsp[-6].minor.yy0.n>0)?strtoll(yymsp[-6].minor.yy0.z, NULL, 10):-1; + yygotominor.yy149.maxQueryTime = (yymsp[-4].minor.yy0.n>0)?strtoll(yymsp[-4].minor.yy0.z, NULL, 10):-1; + yygotominor.yy149.maxConnections = (yymsp[-1].minor.yy0.n>0)?atoi(yymsp[-1].minor.yy0.z):-1; + yygotominor.yy149.stat = yymsp[0].minor.yy0; } - yymsp[-8].minor.yy479 = yylhsminor.yy479; break; case 70: /* keep ::= KEEP tagitemlist */ -{ yymsp[-1].minor.yy156 = yymsp[0].minor.yy156; } +{ yygotominor.yy498 = yymsp[0].minor.yy498; } break; case 71: /* cache ::= CACHE INTEGER */ case 72: /* replica ::= REPLICA INTEGER */ yytestcase(yyruleno==72); @@ -2477,540 +1870,461 @@ static YYACTIONTYPE yy_reduce( case 80: /* fsync ::= FSYNC INTEGER */ yytestcase(yyruleno==80); case 81: /* comp ::= COMP INTEGER */ yytestcase(yyruleno==81); case 82: /* prec ::= PRECISION STRING */ yytestcase(yyruleno==82); -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } + case 83: /* update ::= UPDATE INTEGER */ yytestcase(yyruleno==83); +{ yygotominor.yy0 = yymsp[0].minor.yy0; } break; - case 83: /* db_optr ::= */ -{setDefaultCreateDbOption(&yymsp[1].minor.yy118);} + case 84: /* db_optr ::= */ +{setDefaultCreateDbOption(&yygotominor.yy268);} break; - case 84: /* db_optr ::= db_optr cache */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 85: /* db_optr ::= db_optr cache */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.cacheBlockSize = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 85: /* db_optr ::= db_optr replica */ - case 98: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==98); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 86: /* db_optr ::= db_optr replica */ + case 100: /* alter_db_optr ::= alter_db_optr replica */ yytestcase(yyruleno==100); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.replica = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 86: /* db_optr ::= db_optr quorum */ - case 99: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==99); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 87: /* db_optr ::= db_optr quorum */ + case 101: /* alter_db_optr ::= alter_db_optr quorum */ yytestcase(yyruleno==101); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.quorum = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 87: /* db_optr ::= db_optr days */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 88: /* db_optr ::= db_optr days */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.daysPerFile = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 88: /* db_optr ::= db_optr minrows */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 89: /* db_optr ::= db_optr minrows */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.minRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } break; - case 89: /* db_optr ::= db_optr maxrows */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 90: /* db_optr ::= db_optr maxrows */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.maxRowsPerBlock = strtod(yymsp[0].minor.yy0.z, NULL); } break; - case 90: /* db_optr ::= db_optr blocks */ - case 101: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==101); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 91: /* db_optr ::= db_optr blocks */ + case 103: /* alter_db_optr ::= alter_db_optr blocks */ yytestcase(yyruleno==103); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.numOfBlocks = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 91: /* db_optr ::= db_optr ctime */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 92: /* db_optr ::= db_optr ctime */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.commitTime = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 92: /* db_optr ::= db_optr wal */ - case 103: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==103); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 93: /* db_optr ::= db_optr wal */ + case 105: /* alter_db_optr ::= alter_db_optr wal */ yytestcase(yyruleno==105); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.walLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 93: /* db_optr ::= db_optr fsync */ - case 104: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==104); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 94: /* db_optr ::= db_optr fsync */ + case 106: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==106); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 94: /* db_optr ::= db_optr comp */ - case 102: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==102); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 95: /* db_optr ::= db_optr comp */ + case 104: /* alter_db_optr ::= alter_db_optr comp */ yytestcase(yyruleno==104); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.compressionLevel = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 95: /* db_optr ::= db_optr prec */ -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.precision = yymsp[0].minor.yy0; } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 96: /* db_optr ::= db_optr prec */ +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.precision = yymsp[0].minor.yy0; } break; - case 96: /* db_optr ::= db_optr keep */ - case 100: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==100); -{ yylhsminor.yy118 = yymsp[-1].minor.yy118; yylhsminor.yy118.keep = yymsp[0].minor.yy156; } - yymsp[-1].minor.yy118 = yylhsminor.yy118; + case 97: /* db_optr ::= db_optr keep */ + case 102: /* alter_db_optr ::= alter_db_optr keep */ yytestcase(yyruleno==102); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.keep = yymsp[0].minor.yy498; } break; - case 97: /* alter_db_optr ::= */ -{ setDefaultCreateDbOption(&yymsp[1].minor.yy118);} + case 98: /* db_optr ::= db_optr update */ + case 107: /* alter_db_optr ::= alter_db_optr update */ yytestcase(yyruleno==107); +{ yygotominor.yy268 = yymsp[-1].minor.yy268; yygotominor.yy268.update = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 105: /* typename ::= ids */ + case 99: /* alter_db_optr ::= */ +{ setDefaultCreateDbOption(&yygotominor.yy268);} + break; + case 108: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; - tSQLSetColumnType (&yylhsminor.yy343, &yymsp[0].minor.yy0); + tSQLSetColumnType (&yygotominor.yy223, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy343 = yylhsminor.yy343; break; - case 106: /* typename ::= ids LP signed RP */ + case 109: /* typename ::= ids LP signed RP */ { - if (yymsp[-1].minor.yy369 <= 0) { + if (yymsp[-1].minor.yy207 <= 0) { yymsp[-3].minor.yy0.type = 0; - tSQLSetColumnType(&yylhsminor.yy343, &yymsp[-3].minor.yy0); + tSQLSetColumnType(&yygotominor.yy223, &yymsp[-3].minor.yy0); } else { - yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy369; // negative value of name length - tSQLSetColumnType(&yylhsminor.yy343, &yymsp[-3].minor.yy0); + yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy207; // negative value of name length + tSQLSetColumnType(&yygotominor.yy223, &yymsp[-3].minor.yy0); } } - yymsp[-3].minor.yy343 = yylhsminor.yy343; break; - case 107: /* signed ::= INTEGER */ -{ yylhsminor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[0].minor.yy369 = yylhsminor.yy369; + case 110: /* signed ::= INTEGER */ + case 111: /* signed ::= PLUS INTEGER */ yytestcase(yyruleno==111); +{ yygotominor.yy207 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 108: /* signed ::= PLUS INTEGER */ -{ yymsp[-1].minor.yy369 = strtol(yymsp[0].minor.yy0.z, NULL, 10); } + case 112: /* signed ::= MINUS INTEGER */ +{ yygotominor.yy207 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} break; - case 109: /* signed ::= MINUS INTEGER */ -{ yymsp[-1].minor.yy369 = -strtol(yymsp[0].minor.yy0.z, NULL, 10);} - break; - case 110: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ + case 113: /* cmd ::= CREATE TABLE ifnotexists ids cpxName create_table_args */ { yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; setCreatedTableName(pInfo, &yymsp[-2].minor.yy0, &yymsp[-3].minor.yy0); } break; - case 111: /* create_table_args ::= LP columnlist RP */ + case 114: /* create_table_args ::= LP columnlist RP */ { - yymsp[-2].minor.yy398 = tSetCreateSQLElems(yymsp[-1].minor.yy511, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); - setSQLInfo(pInfo, yymsp[-2].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); + yygotominor.yy470 = tSetCreateSQLElems(yymsp[-1].minor.yy523, NULL, NULL, NULL, NULL, TSQL_CREATE_TABLE); + setSQLInfo(pInfo, yygotominor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 112: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ + case 115: /* create_table_args ::= LP columnlist RP TAGS LP columnlist RP */ { - yymsp[-6].minor.yy398 = tSetCreateSQLElems(yymsp[-5].minor.yy511, yymsp[-1].minor.yy511, NULL, NULL, NULL, TSQL_CREATE_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); + yygotominor.yy470 = tSetCreateSQLElems(yymsp[-5].minor.yy523, yymsp[-1].minor.yy523, NULL, NULL, NULL, TSQL_CREATE_STABLE); + setSQLInfo(pInfo, yygotominor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 113: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ + case 116: /* create_table_args ::= USING ids cpxName TAGS LP tagitemlist RP */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; - yymsp[-6].minor.yy398 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy156, NULL, TSQL_CREATE_TABLE_FROM_STABLE); - setSQLInfo(pInfo, yymsp[-6].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); + yygotominor.yy470 = tSetCreateSQLElems(NULL, NULL, &yymsp[-5].minor.yy0, yymsp[-1].minor.yy498, NULL, TSQL_CREATE_TABLE_FROM_STABLE); + setSQLInfo(pInfo, yygotominor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 114: /* create_table_args ::= AS select */ + case 117: /* create_table_args ::= AS select */ { - yymsp[-1].minor.yy398 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy444, TSQL_CREATE_STREAM); - setSQLInfo(pInfo, yymsp[-1].minor.yy398, NULL, TSDB_SQL_CREATE_TABLE); + yygotominor.yy470 = tSetCreateSQLElems(NULL, NULL, NULL, NULL, yymsp[0].minor.yy414, TSQL_CREATE_STREAM); + setSQLInfo(pInfo, yygotominor.yy470, NULL, TSDB_SQL_CREATE_TABLE); } break; - case 115: /* columnlist ::= columnlist COMMA column */ -{yylhsminor.yy511 = tFieldListAppend(yymsp[-2].minor.yy511, &yymsp[0].minor.yy343); } - yymsp[-2].minor.yy511 = yylhsminor.yy511; + case 118: /* columnlist ::= columnlist COMMA column */ +{yygotominor.yy523 = tFieldListAppend(yymsp[-2].minor.yy523, &yymsp[0].minor.yy223); } break; - case 116: /* columnlist ::= column */ -{yylhsminor.yy511 = tFieldListAppend(NULL, &yymsp[0].minor.yy343);} - yymsp[0].minor.yy511 = yylhsminor.yy511; + case 119: /* columnlist ::= column */ +{yygotominor.yy523 = tFieldListAppend(NULL, &yymsp[0].minor.yy223);} break; - case 117: /* column ::= ids typename */ + case 120: /* column ::= ids typename */ { - tSQLSetColumnInfo(&yylhsminor.yy343, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy343); + tSQLSetColumnInfo(&yygotominor.yy223, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy223); } - yymsp[-1].minor.yy343 = yylhsminor.yy343; break; - case 118: /* tagitemlist ::= tagitemlist COMMA tagitem */ -{ yylhsminor.yy156 = tVariantListAppend(yymsp[-2].minor.yy156, &yymsp[0].minor.yy506, -1); } - yymsp[-2].minor.yy156 = yylhsminor.yy156; + case 121: /* tagitemlist ::= tagitemlist COMMA tagitem */ +{ yygotominor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); } break; - case 119: /* tagitemlist ::= tagitem */ -{ yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[0].minor.yy506, -1); } - yymsp[0].minor.yy156 = yylhsminor.yy156; + case 122: /* tagitemlist ::= tagitem */ +{ yygotominor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } break; - case 120: /* tagitem ::= INTEGER */ - case 121: /* tagitem ::= FLOAT */ yytestcase(yyruleno==121); - case 122: /* tagitem ::= STRING */ yytestcase(yyruleno==122); - case 123: /* tagitem ::= BOOL */ yytestcase(yyruleno==123); -{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yylhsminor.yy506, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy506 = yylhsminor.yy506; + case 123: /* tagitem ::= INTEGER */ + case 124: /* tagitem ::= FLOAT */ yytestcase(yyruleno==124); + case 125: /* tagitem ::= STRING */ yytestcase(yyruleno==125); + case 126: /* tagitem ::= BOOL */ yytestcase(yyruleno==126); +{toTSDBType(yymsp[0].minor.yy0.type); tVariantCreate(&yygotominor.yy134, &yymsp[0].minor.yy0); } break; - case 124: /* tagitem ::= NULL */ -{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yylhsminor.yy506, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy506 = yylhsminor.yy506; + case 127: /* tagitem ::= NULL */ +{ yymsp[0].minor.yy0.type = 0; tVariantCreate(&yygotominor.yy134, &yymsp[0].minor.yy0); } break; - case 125: /* tagitem ::= MINUS INTEGER */ - case 126: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==126); - case 127: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==127); - case 128: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==128); + case 128: /* tagitem ::= MINUS INTEGER */ + case 129: /* tagitem ::= MINUS FLOAT */ yytestcase(yyruleno==129); + case 130: /* tagitem ::= PLUS INTEGER */ yytestcase(yyruleno==130); + case 131: /* tagitem ::= PLUS FLOAT */ yytestcase(yyruleno==131); { yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = yymsp[0].minor.yy0.type; toTSDBType(yymsp[-1].minor.yy0.type); - tVariantCreate(&yylhsminor.yy506, &yymsp[-1].minor.yy0); + tVariantCreate(&yygotominor.yy134, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy506 = yylhsminor.yy506; break; - case 129: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ + case 132: /* select ::= SELECT selcollist from where_opt interval_opt fill_opt sliding_opt groupby_opt orderby_opt having_opt slimit_opt limit_opt */ { - yylhsminor.yy444 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy158, yymsp[-9].minor.yy156, yymsp[-8].minor.yy190, yymsp[-4].minor.yy156, yymsp[-3].minor.yy156, &yymsp[-7].minor.yy340, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy156, &yymsp[0].minor.yy414, &yymsp[-1].minor.yy414); + yygotominor.yy414 = tSetQuerySQLElems(&yymsp[-11].minor.yy0, yymsp[-10].minor.yy290, yymsp[-9].minor.yy498, yymsp[-8].minor.yy64, yymsp[-4].minor.yy498, yymsp[-3].minor.yy498, &yymsp[-7].minor.yy532, &yymsp[-5].minor.yy0, yymsp[-6].minor.yy498, &yymsp[0].minor.yy216, &yymsp[-1].minor.yy216); } - yymsp[-11].minor.yy444 = yylhsminor.yy444; break; - case 130: /* union ::= select */ -{ yylhsminor.yy333 = setSubclause(NULL, yymsp[0].minor.yy444); } - yymsp[0].minor.yy333 = yylhsminor.yy333; + case 133: /* union ::= select */ +{ yygotominor.yy231 = setSubclause(NULL, yymsp[0].minor.yy414); } break; - case 131: /* union ::= LP union RP */ -{ yymsp[-2].minor.yy333 = yymsp[-1].minor.yy333; } + case 134: /* union ::= LP union RP */ +{ yygotominor.yy231 = yymsp[-1].minor.yy231; } break; - case 132: /* union ::= union UNION ALL select */ -{ yylhsminor.yy333 = appendSelectClause(yymsp[-3].minor.yy333, yymsp[0].minor.yy444); } - yymsp[-3].minor.yy333 = yylhsminor.yy333; + case 135: /* union ::= union UNION ALL select */ +{ yygotominor.yy231 = appendSelectClause(yymsp[-3].minor.yy231, yymsp[0].minor.yy414); } break; - case 133: /* union ::= union UNION ALL LP select RP */ -{ yylhsminor.yy333 = appendSelectClause(yymsp[-5].minor.yy333, yymsp[-1].minor.yy444); } - yymsp[-5].minor.yy333 = yylhsminor.yy333; + case 136: /* union ::= union UNION ALL LP select RP */ +{ yygotominor.yy231 = appendSelectClause(yymsp[-5].minor.yy231, yymsp[-1].minor.yy414); } break; - case 134: /* cmd ::= union */ -{ setSQLInfo(pInfo, yymsp[0].minor.yy333, NULL, TSDB_SQL_SELECT); } + case 137: /* cmd ::= union */ +{ setSQLInfo(pInfo, yymsp[0].minor.yy231, NULL, TSDB_SQL_SELECT); } break; - case 135: /* select ::= SELECT selcollist */ + case 138: /* select ::= SELECT selcollist */ { - yylhsminor.yy444 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy158, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); + yygotominor.yy414 = tSetQuerySQLElems(&yymsp[-1].minor.yy0, yymsp[0].minor.yy290, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL); } - yymsp[-1].minor.yy444 = yylhsminor.yy444; break; - case 136: /* sclp ::= selcollist COMMA */ -{yylhsminor.yy158 = yymsp[-1].minor.yy158;} - yymsp[-1].minor.yy158 = yylhsminor.yy158; + case 139: /* sclp ::= selcollist COMMA */ +{yygotominor.yy290 = yymsp[-1].minor.yy290;} break; - case 137: /* sclp ::= */ -{yymsp[1].minor.yy158 = 0;} + case 140: /* sclp ::= */ +{yygotominor.yy290 = 0;} break; - case 138: /* selcollist ::= sclp expr as */ + case 141: /* selcollist ::= sclp expr as */ { - yylhsminor.yy158 = tSQLExprListAppend(yymsp[-2].minor.yy158, yymsp[-1].minor.yy190, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); + yygotominor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290, yymsp[-1].minor.yy64, yymsp[0].minor.yy0.n?&yymsp[0].minor.yy0:0); } - yymsp[-2].minor.yy158 = yylhsminor.yy158; break; - case 139: /* selcollist ::= sclp STAR */ + case 142: /* selcollist ::= sclp STAR */ { tSQLExpr *pNode = tSQLExprIdValueCreate(NULL, TK_ALL); - yylhsminor.yy158 = tSQLExprListAppend(yymsp[-1].minor.yy158, pNode, 0); + yygotominor.yy290 = tSQLExprListAppend(yymsp[-1].minor.yy290, pNode, 0); } - yymsp[-1].minor.yy158 = yylhsminor.yy158; break; - case 140: /* as ::= AS ids */ -{ yymsp[-1].minor.yy0 = yymsp[0].minor.yy0; } + case 143: /* as ::= AS ids */ + case 144: /* as ::= ids */ yytestcase(yyruleno==144); +{ yygotominor.yy0 = yymsp[0].minor.yy0; } break; - case 141: /* as ::= ids */ -{ yylhsminor.yy0 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy0 = yylhsminor.yy0; + case 145: /* as ::= */ +{ yygotominor.yy0.n = 0; } break; - case 142: /* as ::= */ -{ yymsp[1].minor.yy0.n = 0; } + case 146: /* from ::= FROM tablelist */ + case 161: /* orderby_opt ::= ORDER BY sortlist */ yytestcase(yyruleno==161); + case 169: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==169); +{yygotominor.yy498 = yymsp[0].minor.yy498;} break; - case 143: /* from ::= FROM tablelist */ -{yymsp[-1].minor.yy156 = yymsp[0].minor.yy156;} - break; - case 144: /* tablelist ::= ids cpxName */ + case 147: /* tablelist ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy156 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[-1].minor.yy0, -1); // table alias name + yygotominor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yygotominor.yy498, &yymsp[-1].minor.yy0, -1); // table alias name } - yymsp[-1].minor.yy156 = yylhsminor.yy156; break; - case 145: /* tablelist ::= ids cpxName ids */ + case 148: /* tablelist ::= ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy156 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[0].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yygotominor.yy498, &yymsp[0].minor.yy0, -1); } - yymsp[-2].minor.yy156 = yylhsminor.yy156; break; - case 146: /* tablelist ::= tablelist COMMA ids cpxName */ + case 149: /* tablelist ::= tablelist COMMA ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - yylhsminor.yy156 = tVariantListAppendToken(yymsp[-3].minor.yy156, &yymsp[-1].minor.yy0, -1); - yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[-1].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yygotominor.yy498, &yymsp[-1].minor.yy0, -1); } - yymsp[-3].minor.yy156 = yylhsminor.yy156; break; - case 147: /* tablelist ::= tablelist COMMA ids cpxName ids */ + case 150: /* tablelist ::= tablelist COMMA ids cpxName ids */ { toTSDBType(yymsp[-2].minor.yy0.type); toTSDBType(yymsp[0].minor.yy0.type); yymsp[-2].minor.yy0.n += yymsp[-1].minor.yy0.n; - yylhsminor.yy156 = tVariantListAppendToken(yymsp[-4].minor.yy156, &yymsp[-2].minor.yy0, -1); - yylhsminor.yy156 = tVariantListAppendToken(yylhsminor.yy156, &yymsp[0].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yymsp[-4].minor.yy498, &yymsp[-2].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(yygotominor.yy498, &yymsp[0].minor.yy0, -1); } - yymsp[-4].minor.yy156 = yylhsminor.yy156; break; - case 148: /* tmvar ::= VARIABLE */ -{yylhsminor.yy0 = yymsp[0].minor.yy0;} - yymsp[0].minor.yy0 = yylhsminor.yy0; + case 151: /* tmvar ::= VARIABLE */ +{yygotominor.yy0 = yymsp[0].minor.yy0;} break; - case 149: /* interval_opt ::= INTERVAL LP tmvar RP */ -{yymsp[-3].minor.yy340.interval = yymsp[-1].minor.yy0; yymsp[-3].minor.yy340.offset.n = 0; yymsp[-3].minor.yy340.offset.z = NULL; yymsp[-3].minor.yy340.offset.type = 0;} + case 152: /* interval_opt ::= INTERVAL LP tmvar RP */ +{yygotominor.yy532.interval = yymsp[-1].minor.yy0; yygotominor.yy532.offset.n = 0; yygotominor.yy532.offset.z = NULL; yygotominor.yy532.offset.type = 0;} break; - case 150: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ -{yymsp[-5].minor.yy340.interval = yymsp[-3].minor.yy0; yymsp[-5].minor.yy340.offset = yymsp[-1].minor.yy0;} + case 153: /* interval_opt ::= INTERVAL LP tmvar COMMA tmvar RP */ +{yygotominor.yy532.interval = yymsp[-3].minor.yy0; yygotominor.yy532.offset = yymsp[-1].minor.yy0;} break; - case 151: /* interval_opt ::= */ -{memset(&yymsp[1].minor.yy340, 0, sizeof(yymsp[1].minor.yy340));} + case 154: /* interval_opt ::= */ +{memset(&yygotominor.yy532, 0, sizeof(yygotominor.yy532));} break; - case 152: /* fill_opt ::= */ -{yymsp[1].minor.yy156 = 0; } + case 155: /* fill_opt ::= */ +{yygotominor.yy498 = 0; } break; - case 153: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ + case 156: /* fill_opt ::= FILL LP ID COMMA tagitemlist RP */ { tVariant A = {0}; toTSDBType(yymsp[-3].minor.yy0.type); tVariantCreate(&A, &yymsp[-3].minor.yy0); - tVariantListInsert(yymsp[-1].minor.yy156, &A, -1, 0); - yymsp[-5].minor.yy156 = yymsp[-1].minor.yy156; + tVariantListInsert(yymsp[-1].minor.yy498, &A, -1, 0); + yygotominor.yy498 = yymsp[-1].minor.yy498; } break; - case 154: /* fill_opt ::= FILL LP ID RP */ + case 157: /* fill_opt ::= FILL LP ID RP */ { toTSDBType(yymsp[-1].minor.yy0.type); - yymsp[-3].minor.yy156 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); + yygotominor.yy498 = tVariantListAppendToken(NULL, &yymsp[-1].minor.yy0, -1); } break; - case 155: /* sliding_opt ::= SLIDING LP tmvar RP */ -{yymsp[-3].minor.yy0 = yymsp[-1].minor.yy0; } + case 158: /* sliding_opt ::= SLIDING LP tmvar RP */ +{yygotominor.yy0 = yymsp[-1].minor.yy0; } break; - case 156: /* sliding_opt ::= */ -{yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = NULL; yymsp[1].minor.yy0.type = 0; } + case 159: /* sliding_opt ::= */ +{yygotominor.yy0.n = 0; yygotominor.yy0.z = NULL; yygotominor.yy0.type = 0; } break; - case 157: /* orderby_opt ::= */ - case 165: /* groupby_opt ::= */ yytestcase(yyruleno==165); -{yymsp[1].minor.yy156 = 0;} + case 160: /* orderby_opt ::= */ + case 168: /* groupby_opt ::= */ yytestcase(yyruleno==168); +{yygotominor.yy498 = 0;} break; - case 158: /* orderby_opt ::= ORDER BY sortlist */ - case 166: /* groupby_opt ::= GROUP BY grouplist */ yytestcase(yyruleno==166); -{yymsp[-2].minor.yy156 = yymsp[0].minor.yy156;} - break; - case 159: /* sortlist ::= sortlist COMMA item sortorder */ + case 162: /* sortlist ::= sortlist COMMA item sortorder */ { - yylhsminor.yy156 = tVariantListAppend(yymsp[-3].minor.yy156, &yymsp[-1].minor.yy506, yymsp[0].minor.yy112); + yygotominor.yy498 = tVariantListAppend(yymsp[-3].minor.yy498, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } - yymsp[-3].minor.yy156 = yylhsminor.yy156; break; - case 160: /* sortlist ::= item sortorder */ + case 163: /* sortlist ::= item sortorder */ { - yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[-1].minor.yy506, yymsp[0].minor.yy112); + yygotominor.yy498 = tVariantListAppend(NULL, &yymsp[-1].minor.yy134, yymsp[0].minor.yy46); } - yymsp[-1].minor.yy156 = yylhsminor.yy156; break; - case 161: /* item ::= ids cpxName */ + case 164: /* item ::= ids cpxName */ { toTSDBType(yymsp[-1].minor.yy0.type); yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; - tVariantCreate(&yylhsminor.yy506, &yymsp[-1].minor.yy0); + tVariantCreate(&yygotominor.yy134, &yymsp[-1].minor.yy0); } - yymsp[-1].minor.yy506 = yylhsminor.yy506; break; - case 162: /* sortorder ::= ASC */ -{yymsp[0].minor.yy112 = TSDB_ORDER_ASC; } + case 165: /* sortorder ::= ASC */ +{yygotominor.yy46 = TSDB_ORDER_ASC; } break; - case 163: /* sortorder ::= DESC */ -{yymsp[0].minor.yy112 = TSDB_ORDER_DESC;} + case 166: /* sortorder ::= DESC */ +{yygotominor.yy46 = TSDB_ORDER_DESC;} break; - case 164: /* sortorder ::= */ -{yymsp[1].minor.yy112 = TSDB_ORDER_ASC;} + case 167: /* sortorder ::= */ +{yygotominor.yy46 = TSDB_ORDER_ASC;} break; - case 167: /* grouplist ::= grouplist COMMA item */ + case 170: /* grouplist ::= grouplist COMMA item */ { - yylhsminor.yy156 = tVariantListAppend(yymsp[-2].minor.yy156, &yymsp[0].minor.yy506, -1); + yygotominor.yy498 = tVariantListAppend(yymsp[-2].minor.yy498, &yymsp[0].minor.yy134, -1); } - yymsp[-2].minor.yy156 = yylhsminor.yy156; break; - case 168: /* grouplist ::= item */ + case 171: /* grouplist ::= item */ { - yylhsminor.yy156 = tVariantListAppend(NULL, &yymsp[0].minor.yy506, -1); + yygotominor.yy498 = tVariantListAppend(NULL, &yymsp[0].minor.yy134, -1); } - yymsp[0].minor.yy156 = yylhsminor.yy156; break; - case 169: /* having_opt ::= */ - case 179: /* where_opt ::= */ yytestcase(yyruleno==179); - case 217: /* expritem ::= */ yytestcase(yyruleno==217); -{yymsp[1].minor.yy190 = 0;} + case 172: /* having_opt ::= */ + case 182: /* where_opt ::= */ yytestcase(yyruleno==182); + case 220: /* expritem ::= */ yytestcase(yyruleno==220); +{yygotominor.yy64 = 0;} break; - case 170: /* having_opt ::= HAVING expr */ - case 180: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==180); -{yymsp[-1].minor.yy190 = yymsp[0].minor.yy190;} + case 173: /* having_opt ::= HAVING expr */ + case 183: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==183); + case 219: /* expritem ::= expr */ yytestcase(yyruleno==219); +{yygotominor.yy64 = yymsp[0].minor.yy64;} break; - case 171: /* limit_opt ::= */ - case 175: /* slimit_opt ::= */ yytestcase(yyruleno==175); -{yymsp[1].minor.yy414.limit = -1; yymsp[1].minor.yy414.offset = 0;} + case 174: /* limit_opt ::= */ + case 178: /* slimit_opt ::= */ yytestcase(yyruleno==178); +{yygotominor.yy216.limit = -1; yygotominor.yy216.offset = 0;} break; - case 172: /* limit_opt ::= LIMIT signed */ - case 176: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==176); -{yymsp[-1].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-1].minor.yy414.offset = 0;} + case 175: /* limit_opt ::= LIMIT signed */ + case 179: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==179); +{yygotominor.yy216.limit = yymsp[0].minor.yy207; yygotominor.yy216.offset = 0;} break; - case 173: /* limit_opt ::= LIMIT signed OFFSET signed */ - case 177: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==177); -{yymsp[-3].minor.yy414.limit = yymsp[-2].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[0].minor.yy369;} + case 176: /* limit_opt ::= LIMIT signed OFFSET signed */ + case 180: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ yytestcase(yyruleno==180); +{yygotominor.yy216.limit = yymsp[-2].minor.yy207; yygotominor.yy216.offset = yymsp[0].minor.yy207;} break; - case 174: /* limit_opt ::= LIMIT signed COMMA signed */ - case 178: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==178); -{yymsp[-3].minor.yy414.limit = yymsp[0].minor.yy369; yymsp[-3].minor.yy414.offset = yymsp[-2].minor.yy369;} + case 177: /* limit_opt ::= LIMIT signed COMMA signed */ + case 181: /* slimit_opt ::= SLIMIT signed COMMA signed */ yytestcase(yyruleno==181); +{yygotominor.yy216.limit = yymsp[0].minor.yy207; yygotominor.yy216.offset = yymsp[-2].minor.yy207;} break; - case 181: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy190 = yymsp[-1].minor.yy190; } + case 184: /* expr ::= LP expr RP */ +{yygotominor.yy64 = yymsp[-1].minor.yy64; } break; - case 182: /* expr ::= ID */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 185: /* expr ::= ID */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} break; - case 183: /* expr ::= ID DOT ID */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 186: /* expr ::= ID DOT ID */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ID);} break; - case 184: /* expr ::= ID DOT STAR */ -{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 187: /* expr ::= ID DOT STAR */ +{yymsp[-2].minor.yy0.n += (1+yymsp[0].minor.yy0.n); yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[-2].minor.yy0, TK_ALL);} break; - case 185: /* expr ::= INTEGER */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 188: /* expr ::= INTEGER */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_INTEGER);} break; - case 186: /* expr ::= MINUS INTEGER */ - case 187: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==187); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} - yymsp[-1].minor.yy190 = yylhsminor.yy190; + case 189: /* expr ::= MINUS INTEGER */ + case 190: /* expr ::= PLUS INTEGER */ yytestcase(yyruleno==190); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_INTEGER; yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_INTEGER);} break; - case 188: /* expr ::= FLOAT */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 191: /* expr ::= FLOAT */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_FLOAT);} break; - case 189: /* expr ::= MINUS FLOAT */ - case 190: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==190); -{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} - yymsp[-1].minor.yy190 = yylhsminor.yy190; + case 192: /* expr ::= MINUS FLOAT */ + case 193: /* expr ::= PLUS FLOAT */ yytestcase(yyruleno==193); +{yymsp[-1].minor.yy0.n += yymsp[0].minor.yy0.n; yymsp[-1].minor.yy0.type = TK_FLOAT; yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[-1].minor.yy0, TK_FLOAT);} break; - case 191: /* expr ::= STRING */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 194: /* expr ::= STRING */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_STRING);} break; - case 192: /* expr ::= NOW */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 195: /* expr ::= NOW */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_NOW); } break; - case 193: /* expr ::= VARIABLE */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 196: /* expr ::= VARIABLE */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_VARIABLE);} break; - case 194: /* expr ::= BOOL */ -{yylhsminor.yy190 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} - yymsp[0].minor.yy190 = yylhsminor.yy190; + case 197: /* expr ::= BOOL */ +{yygotominor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_BOOL);} break; - case 195: /* expr ::= ID LP exprlist RP */ -{ yylhsminor.yy190 = tSQLExprCreateFunction(yymsp[-1].minor.yy158, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy190 = yylhsminor.yy190; + case 198: /* expr ::= ID LP exprlist RP */ +{ yygotominor.yy64 = tSQLExprCreateFunction(yymsp[-1].minor.yy290, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } break; - case 196: /* expr ::= ID LP STAR RP */ -{ yylhsminor.yy190 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } - yymsp[-3].minor.yy190 = yylhsminor.yy190; + case 199: /* expr ::= ID LP STAR RP */ +{ yygotominor.yy64 = tSQLExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); } break; - case 197: /* expr ::= expr IS NULL */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, NULL, TK_ISNULL);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 200: /* expr ::= expr IS NULL */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, NULL, TK_ISNULL);} break; - case 198: /* expr ::= expr IS NOT NULL */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-3].minor.yy190, NULL, TK_NOTNULL);} - yymsp[-3].minor.yy190 = yylhsminor.yy190; + case 201: /* expr ::= expr IS NOT NULL */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-3].minor.yy64, NULL, TK_NOTNULL);} break; - case 199: /* expr ::= expr LT expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LT);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 202: /* expr ::= expr LT expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LT);} break; - case 200: /* expr ::= expr GT expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_GT);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 203: /* expr ::= expr GT expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GT);} break; - case 201: /* expr ::= expr LE expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LE);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 204: /* expr ::= expr LE expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LE);} break; - case 202: /* expr ::= expr GE expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_GE);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 205: /* expr ::= expr GE expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_GE);} break; - case 203: /* expr ::= expr NE expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_NE);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 206: /* expr ::= expr NE expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_NE);} break; - case 204: /* expr ::= expr EQ expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_EQ);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 207: /* expr ::= expr EQ expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_EQ);} break; - case 205: /* expr ::= expr AND expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_AND);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 208: /* expr ::= expr AND expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_AND);} break; - case 206: /* expr ::= expr OR expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_OR); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 209: /* expr ::= expr OR expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_OR); } break; - case 207: /* expr ::= expr PLUS expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_PLUS); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 210: /* expr ::= expr PLUS expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_PLUS); } break; - case 208: /* expr ::= expr MINUS expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_MINUS); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 211: /* expr ::= expr MINUS expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_MINUS); } break; - case 209: /* expr ::= expr STAR expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_STAR); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 212: /* expr ::= expr STAR expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_STAR); } break; - case 210: /* expr ::= expr SLASH expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_DIVIDE);} - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 213: /* expr ::= expr SLASH expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_DIVIDE);} break; - case 211: /* expr ::= expr REM expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_REM); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 214: /* expr ::= expr REM expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_REM); } break; - case 212: /* expr ::= expr LIKE expr */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-2].minor.yy190, yymsp[0].minor.yy190, TK_LIKE); } - yymsp[-2].minor.yy190 = yylhsminor.yy190; + case 215: /* expr ::= expr LIKE expr */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-2].minor.yy64, yymsp[0].minor.yy64, TK_LIKE); } break; - case 213: /* expr ::= expr IN LP exprlist RP */ -{yylhsminor.yy190 = tSQLExprCreate(yymsp[-4].minor.yy190, (tSQLExpr*)yymsp[-1].minor.yy158, TK_IN); } - yymsp[-4].minor.yy190 = yylhsminor.yy190; + case 216: /* expr ::= expr IN LP exprlist RP */ +{yygotominor.yy64 = tSQLExprCreate(yymsp[-4].minor.yy64, (tSQLExpr*)yymsp[-1].minor.yy290, TK_IN); } break; - case 214: /* exprlist ::= exprlist COMMA expritem */ -{yylhsminor.yy158 = tSQLExprListAppend(yymsp[-2].minor.yy158,yymsp[0].minor.yy190,0);} - yymsp[-2].minor.yy158 = yylhsminor.yy158; + case 217: /* exprlist ::= exprlist COMMA expritem */ +{yygotominor.yy290 = tSQLExprListAppend(yymsp[-2].minor.yy290,yymsp[0].minor.yy64,0);} break; - case 215: /* exprlist ::= expritem */ -{yylhsminor.yy158 = tSQLExprListAppend(0,yymsp[0].minor.yy190,0);} - yymsp[0].minor.yy158 = yylhsminor.yy158; + case 218: /* exprlist ::= expritem */ +{yygotominor.yy290 = tSQLExprListAppend(0,yymsp[0].minor.yy64,0);} break; - case 216: /* expritem ::= expr */ -{yylhsminor.yy190 = yymsp[0].minor.yy190;} - yymsp[0].minor.yy190 = yylhsminor.yy190; - break; - case 218: /* cmd ::= RESET QUERY CACHE */ + case 221: /* cmd ::= RESET QUERY CACHE */ { setDCLSQLElems(pInfo, TSDB_SQL_RESET_CACHE, 0);} break; - case 219: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ + case 222: /* cmd ::= ALTER TABLE ids cpxName ADD COLUMN columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy511, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 220: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ + case 223: /* cmd ::= ALTER TABLE ids cpxName DROP COLUMN ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3021,14 +2335,14 @@ static YYACTIONTYPE yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 221: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ + case 224: /* cmd ::= ALTER TABLE ids cpxName ADD TAG columnlist */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; - SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy511, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); + SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-4].minor.yy0, yymsp[0].minor.yy523, NULL, TSDB_ALTER_TABLE_ADD_TAG_COLUMN); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 222: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ + case 225: /* cmd ::= ALTER TABLE ids cpxName DROP TAG ids */ { yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; @@ -3039,7 +2353,7 @@ static YYACTIONTYPE yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 223: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ + case 226: /* cmd ::= ALTER TABLE ids cpxName CHANGE TAG ids ids */ { yymsp[-5].minor.yy0.n += yymsp[-4].minor.yy0.n; @@ -3053,49 +2367,55 @@ static YYACTIONTYPE yy_reduce( setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 224: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ + case 227: /* cmd ::= ALTER TABLE ids cpxName SET TAG ids EQ tagitem */ { yymsp[-6].minor.yy0.n += yymsp[-5].minor.yy0.n; toTSDBType(yymsp[-2].minor.yy0.type); tVariantList* A = tVariantListAppendToken(NULL, &yymsp[-2].minor.yy0, -1); - A = tVariantListAppend(A, &yymsp[0].minor.yy506, -1); + A = tVariantListAppend(A, &yymsp[0].minor.yy134, -1); SAlterTableSQL* pAlterTable = tAlterTableSQLElems(&yymsp[-6].minor.yy0, NULL, A, TSDB_ALTER_TABLE_UPDATE_TAG_VAL); setSQLInfo(pInfo, pAlterTable, NULL, TSDB_SQL_ALTER_TABLE); } break; - case 225: /* cmd ::= KILL CONNECTION INTEGER */ + case 228: /* cmd ::= KILL CONNECTION INTEGER */ {setKillSQL(pInfo, TSDB_SQL_KILL_CONNECTION, &yymsp[0].minor.yy0);} break; - case 226: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ + case 229: /* cmd ::= KILL STREAM INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_STREAM, &yymsp[-2].minor.yy0);} break; - case 227: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ + case 230: /* cmd ::= KILL QUERY INTEGER COLON INTEGER */ {yymsp[-2].minor.yy0.n += (yymsp[-1].minor.yy0.n + yymsp[0].minor.yy0.n); setKillSQL(pInfo, TSDB_SQL_KILL_QUERY, &yymsp[-2].minor.yy0);} break; default: break; -/********** End reduce actions ************************************************/ }; - assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); - - /* It is not possible for a REDUCE to be followed by an error */ - assert( yyact!=YY_ERROR_ACTION ); - - yymsp += yysize+1; - yypParser->yytos = yymsp; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact, "... then shift"); - return yyact; + yygoto = yyRuleInfo[yyruleno].lhs; + yysize = yyRuleInfo[yyruleno].nrhs; + yypParser->yyidx -= yysize; + yyact = yy_find_reduce_action(yymsp[-yysize].stateno,(YYCODETYPE)yygoto); + if( yyact < YYNSTATE ){ +#ifdef NDEBUG + /* If we are not debugging and the reduce action popped at least + ** one element off the stack, then we can push the new element back + ** onto the stack here, and skip the stack overflow test in yy_shift(). + ** That gives a significant speed improvement. */ + if( yysize ){ + yypParser->yyidx++; + yymsp -= yysize-1; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yymsp->minor = yygotominor; + }else +#endif + { + yy_shift(yypParser,yyact,yygoto,&yygotominor); + } + }else{ + assert( yyact == YYNSTATE + YYNRULE + 1 ); + yy_accept(yypParser); + } } /* @@ -3105,20 +2425,16 @@ static YYACTIONTYPE yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); } #endif - while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ -/************ Begin %parse_failure code ***************************************/ -/************ End %parse_failure code *****************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } #endif /* YYNOERRORRECOVERY */ @@ -3128,12 +2444,10 @@ static void yy_parse_failed( static void yy_syntax_error( yyParser *yypParser, /* The parser */ int yymajor, /* The major type of the error token */ - ParseTOKENTYPE yyminor /* The minor type of the error token */ + YYMINORTYPE yyminor /* The minor type of the error token */ ){ - ParseARG_FETCH - ParseCTX_FETCH -#define TOKEN yyminor -/************ Begin %syntax_error code ****************************************/ + ParseARG_FETCH; +#define TOKEN (yyminor.yy0) pInfo->valid = false; int32_t outputBufLen = tListLen(pInfo->pzErrMsg); @@ -3156,9 +2470,7 @@ static void yy_syntax_error( } assert(len <= outputBufLen); -/************ End %syntax_error code ******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } /* @@ -3167,24 +2479,17 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - ParseARG_FETCH - ParseCTX_FETCH + ParseARG_FETCH; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); } #endif -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - assert( yypParser->yytos==yypParser->yystack ); + while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser accepts */ -/*********** Begin %parse_accept code *****************************************/ -/*********** End %parse_accept code *******************************************/ - ParseARG_STORE /* Suppress warning about unused %extra_argument variable */ - ParseCTX_STORE + ParseARG_STORE; /* Suppress warning about unused %extra_argument variable */ } /* The main parser program. @@ -3213,54 +2518,50 @@ void Parse( ParseARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - YYACTIONTYPE yyact; /* The parser action. */ -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) + int yyact; /* The parser action. */ int yyendofinput; /* True if we are at the end of input */ -#endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif - yyParser *yypParser = (yyParser*)yyp; /* The parser */ - ParseCTX_FETCH - ParseARG_STORE + yyParser *yypParser; /* The parser */ - assert( yypParser->yytos!=0 ); -#if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) - yyendofinput = (yymajor==0); + /* (re)initialize the parser, if necessary */ + yypParser = (yyParser*)yyp; + if( yypParser->yyidx<0 ){ +#if YYSTACKDEPTH<=0 + if( yypParser->yystksz <=0 ){ + /*memset(&yyminorunion, 0, sizeof(yyminorunion));*/ + yyminorunion = yyzerominor; + yyStackOverflow(yypParser, &yyminorunion); + return; + } #endif + yypParser->yyidx = 0; + yypParser->yyerrcnt = -1; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; + } + yyminorunion.yy0 = yyminor; + yyendofinput = (yymajor==0); + ParseARG_STORE; - yyact = yypParser->yytos->stateno; #ifndef NDEBUG if( yyTraceFILE ){ - if( yyact < YY_MIN_REDUCE ){ - fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact); - }else{ - fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", - yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); - } + fprintf(yyTraceFILE,"%sInput %s\n",yyTracePrompt,yyTokenName[yymajor]); } #endif do{ - assert( yyact==yypParser->yytos->stateno ); - yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); - if( yyact >= YY_MIN_REDUCE ){ - yyact = yy_reduce(yypParser,yyact-YY_MIN_REDUCE,yymajor, - yyminor ParseCTX_PARAM); - }else if( yyact <= YY_MAX_SHIFTREDUCE ){ - yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); -#ifndef YYNOERRORRECOVERY + yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); + if( yyactyyerrcnt--; -#endif - break; - }else if( yyact==YY_ACCEPT_ACTION ){ - yypParser->yytos--; - yy_accept(yypParser); - return; + yymajor = YYNOCODE; + }else if( yyact < YYNSTATE + YYNRULE ){ + yy_reduce(yypParser,yyact-YYNSTATE); }else{ assert( yyact == YY_ERROR_ACTION ); - yyminorunion.yy0 = yyminor; #ifdef YYERRORSYMBOL int yymx; #endif @@ -3290,9 +2591,9 @@ void Parse( ** */ if( yypParser->yyerrcnt<0 ){ - yy_syntax_error(yypParser,yymajor,yyminor); + yy_syntax_error(yypParser,yymajor,yyminorunion); } - yymx = yypParser->yytos->major; + yymx = yypParser->yystack[yypParser->yyidx].major; if( yymx==YYERRORSYMBOL || yyerrorhit ){ #ifndef NDEBUG if( yyTraceFILE ){ @@ -3300,31 +2601,30 @@ void Parse( yyTracePrompt,yyTokenName[yymajor]); } #endif - yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); + yy_destructor(yypParser, (YYCODETYPE)yymajor,&yyminorunion); yymajor = YYNOCODE; }else{ - while( yypParser->yytos >= yypParser->yystack - && (yyact = yy_find_reduce_action( - yypParser->yytos->stateno, - YYERRORSYMBOL)) > YY_MAX_SHIFTREDUCE + while( + yypParser->yyidx >= 0 && + yymx != YYERRORSYMBOL && + (yyact = yy_find_reduce_action( + yypParser->yystack[yypParser->yyidx].stateno, + YYERRORSYMBOL)) >= YYNSTATE ){ yy_pop_parser_stack(yypParser); } - if( yypParser->yytos < yypParser->yystack || yymajor==0 ){ + if( yypParser->yyidx < 0 || yymajor==0 ){ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ - yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); + YYMINORTYPE u2; + u2.YYERRSYMDT = 0; + yy_shift(yypParser,yyact,YYERRORSYMBOL,&u2); } } yypParser->yyerrcnt = 3; yyerrorhit = 1; - if( yymajor==YYNOCODE ) break; - yyact = yypParser->yytos->stateno; #elif defined(YYNOERRORRECOVERY) /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to ** do any kind of error recovery. Instead, simply invoke the syntax @@ -3333,9 +2633,10 @@ void Parse( ** Applications can set this macro (for example inside %include) if ** they intend to abandon the parse upon the first syntax error seen. */ - yy_syntax_error(yypParser,yymajor, yyminor); + yy_syntax_error(yypParser,yymajor,yyminorunion); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - break; + yymajor = YYNOCODE; + #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -3347,45 +2648,16 @@ void Parse( ** three input tokens have been successfully shifted. */ if( yypParser->yyerrcnt<=0 ){ - yy_syntax_error(yypParser,yymajor, yyminor); + yy_syntax_error(yypParser,yymajor,yyminorunion); } yypParser->yyerrcnt = 3; yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); if( yyendofinput ){ yy_parse_failed(yypParser); -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif } - break; + yymajor = YYNOCODE; #endif } - }while( yypParser->yytos>yypParser->yystack ); -#ifndef NDEBUG - if( yyTraceFILE ){ - yyStackEntry *i; - char cDiv = '['; - fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); - for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ - fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); - cDiv = ' '; - } - fprintf(yyTraceFILE,"]\n"); - } -#endif + }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); return; } - -/* -** Return the fallback token corresponding to canonical token iToken, or -** 0 if iToken has no fallback. -*/ -int ParseFallback(int iToken){ -#ifdef YYFALLBACK - assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); - return yyFallback[iToken]; -#else - (void)iToken; - return 0; -#endif -} diff --git a/tests/pytest/alter/db_update_options.py b/tests/pytest/alter/db_update_options.py new file mode 100644 index 0000000000..224e0f25b0 --- /dev/null +++ b/tests/pytest/alter/db_update_options.py @@ -0,0 +1,71 @@ + +# -*- coding: utf-8 -*- + +import random +import string +import subprocess +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + def run(self): + tdLog.debug("check database") + tdSql.prepare() + + # check default update value + sql = "create database if not exists db" + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + sql = "alter database db update 1" + + # check update value + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,1) + + + sql = "alter database db update 0" + tdSql.execute(sql) + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + sql = "alter database db update -1" + tdSql.error(sql) + + sql = "alter database db update 100" + tdSql.error(sql) + + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,0) + + tdSql.execute('drop database db') + tdSql.error('create database db update 100') + tdSql.error('create database db update -1') + + tdSql.execute('create database db update 1') + + tdSql.query('show databases') + tdSql.checkRows(1) + tdSql.checkData(0,16,1) + + tdSql.execute('drop database db') + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 4a83966f5c7cce0fcaf86dbe3e1410fdf8b6e338 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Sep 2020 10:02:23 +0800 Subject: [PATCH 10/58] TD-1194 --- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbMemTable.c | 10 +- src/tsdb/src/tsdbMeta.c | 6 +- src/tsdb/src/tsdbRead.c | 18 +- src/util/inc/tskiplist.h | 188 +++-------------- src/util/src/tskiplist.c | 405 ++++++++++++++---------------------- 6 files changed, 210 insertions(+), 419 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 95b7010038..b515752bc3 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -438,7 +438,7 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { SSkipListNode* node = tSkipListIterGet(pIter); if (node == NULL) return NULL; - return *(SDataRow *)SL_GET_NODE_DATA(node); + return (SDataRow)SL_GET_NODE_DATA(node); } static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index c58460e18f..9c76a09ca3 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -91,11 +91,11 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); - int64_t oldSize = SL_GET_SIZE(pTableData->pData); - if (tSkipListPut(pTableData->pData, (void *)(&pRow), sizeof(void *)) == NULL) { + int64_t oldSize = SL_SIZE(pTableData->pData); + if (tSkipListPut(pTableData->pData, pRow) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); } else { - int64_t deltaSize = SL_GET_SIZE(pTableData->pData) - oldSize; + int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize; if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; @@ -427,7 +427,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], - pCfg->update ? SL_APPEND_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); + pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -447,7 +447,7 @@ static void tsdbFreeTableData(STableData *pTableData) { } } -static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(*(SDataRow *)data); } +static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); } static void *tsdbCommitData(void *arg) { STsdbRepo * pRepo = (STsdbRepo *)arg; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 76f9d51b87..391cb55985 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -643,7 +643,7 @@ static void tsdbOrgMeta(void *pHandle) { } static char *getTagIndexKey(const void *pData) { - STable *pTable = *(STable **)pData; + STable *pTable = (STable *)pData; STSchema *pSchema = tsdbGetTableTagSchema(pTable); STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); @@ -900,7 +900,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable, bool refSuper pTable->pSuper = pSTable; - tSkipListPut(pSTable->pIndex, (void *)(&pTable), sizeof(STable *)); + tSkipListPut(pSTable->pIndex, (void *)pTable); if (refSuper) T_REF_INC(pSTable); return 0; @@ -1182,7 +1182,7 @@ static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { tlen = sizeof(SListNode) + sizeof(SActObj) + sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM); } else { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_GET_SIZE(pTable->pIndex) + 1)); + tlen = (int)((sizeof(SListNode) + sizeof(SActObj)) * (SL_SIZE(pTable->pIndex) + 1)); } else { tlen = sizeof(SListNode) + sizeof(SActObj); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 320f1c3765..f390208af1 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -386,7 +386,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); assert(node != NULL); - SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); + SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", @@ -408,7 +408,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); assert(node != NULL); - SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); + SDataRow row = (SDataRow)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %p", @@ -438,14 +438,14 @@ static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); if (node != NULL) { - rmem = *(SDataRow *)SL_GET_NODE_DATA(node); + rmem = (SDataRow)SL_GET_NODE_DATA(node); } } if (pCheckInfo->iiter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter); if (node != NULL) { - rimem = *(SDataRow *)SL_GET_NODE_DATA(node); + rimem = (SDataRow)SL_GET_NODE_DATA(node); } } @@ -1358,8 +1358,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((dataRowKey(*(SDataRow *)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || + ((dataRowKey((SDataRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; @@ -1864,9 +1864,9 @@ static int32_t getAllTableList(STable* pSuperTable, SArray* list) { while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + STable* pTable = (STable*) SL_GET_NODE_DATA((SSkipListNode*) pNode); - STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = pTable, .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(list, &info); } @@ -2434,7 +2434,7 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC static bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; - STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); + STable* pTable = (STable*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); char* val = NULL; diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index f1e09501e9..1eccac6646 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -30,43 +30,27 @@ extern "C" { // For key property setting #define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists #define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key -#define SL_UPDATA_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert -#define SL_APPEND_DUP_KEY (uint8_t)0x3 // Update duplicate key by append +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert // For thread safety setting #define SL_THREAD_SAFE (uint8_t)0x4 typedef char *SSkipListKey; typedef char *(*__sl_key_fn_t)(const void *); -/** - * the skip list node is located in a consecutive memory area, - * the format of skip list node is as follows: - * +------------+-----------------------+------------------------+-----+------+ - * | node level | forward pointer array | backward pointer array | key | data | - * +------------+-----------------------+------------------------+-----+------+ - */ typedef struct SSkipListNode { - uint8_t level; + uint8_t level; + uint8_t flags; + void * pData; + struct SSkipListNode *forwards[]; } SSkipListNode; -#define SL_IS_THREAD_SAFE(flags) ((flags)&SL_THREAD_SAFE) -#define SL_DUP_MODE(flags) ((flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_NODE_DELETED_FLAG (uint8_t)0x1 -#define SL_NODE_HEADER_SIZE(_l) (sizeof(SSkipListNode) + ((_l) << 1u) * POINTER_BYTES) - -#define SL_GET_FORWARD_POINTER(n, _l) ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode)))[(_l)] -#define SL_GET_BACKWARD_POINTER(n, _l) \ - ((SSkipListNode **)((char *)(n) + sizeof(SSkipListNode) + ((n)->level) * POINTER_BYTES))[(_l)] - -#define SL_GET_NODE_DATA(n) ((char *)(n) + SL_NODE_HEADER_SIZE((n)->level)) -#define SL_GET_NODE_KEY(s, n) ((s)->keyFn(SL_GET_NODE_DATA(n))) - -#define SL_GET_SL_MIN_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_FORWARD_POINTER((s)->pHead, 0))) -#define SL_GET_SL_MAX_KEY(s) (SL_GET_NODE_KEY((s), SL_GET_BACKWARD_POINTER((s)->pTail, 0))) - -#define SL_GET_NODE_LEVEL(n) *(uint8_t *)((n)) -#define SL_GET_SIZE(s) (s)->size -#define SL_GET_TSIZE(s) (s)->tsize +#define SL_GET_NODE_DATA(n) (n)->pData +#define SL_IS_NODE_DELETED(n) ((n)->flags & SL_NODE_DELETED_FLAG) +#define SL_SET_NODE_DELETED(n) (n)->flags |= SL_NODE_DELETED_FLAG +#define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] +#define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] /* * @version 0.3 @@ -116,13 +100,6 @@ typedef struct tSkipListState { uint64_t nTotalElapsedTimeForInsert; } tSkipListState; -typedef struct SSkipListKeyInfo { - uint8_t dupKey : 2; // if allow duplicated key in the skip list - uint8_t type : 4; // key type - uint8_t freeNode:2; // free node when destroy the skiplist - uint8_t len; // maximum key length, used in case of string key -} SSkipListKeyInfo; - typedef struct SSkipList { __compar_fn_t comparFn; __sl_key_fn_t keyFn; @@ -130,10 +107,10 @@ typedef struct SSkipList { uint16_t len; uint8_t maxLevel; uint8_t flags; - uint8_t type; // static info above + uint8_t type; // static info above uint8_t level; - uint32_t size; // not including duplicate keys - uint32_t tsize; // including duplicate keys + uint32_t size; // semantic meaning of size + uint32_t tsize; // # of all skiplist nodes in this SL SSkipListNode * pHead; // point to the first element SSkipListNode * pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE @@ -141,130 +118,33 @@ typedef struct SSkipList { #endif } SSkipList; -/* - * iterate the skiplist - * this will cause the multi-thread problem, when the skiplist is destroyed, the iterate may - * continue iterating the skiplist, so add the reference count for skiplist - * TODO add the ref for skip list when one iterator is created - */ typedef struct SSkipListIterator { SSkipList * pSkipList; SSkipListNode *cur; - int32_t step; // the number of nodes that have been checked already - int32_t order; // order of the iterator + int32_t step; // the number of nodes that have been checked already + int32_t order; // order of the iterator } SSkipListIterator; -/** - * - * @param nMaxLevel maximum skip list level - * @param keyType type of key - * @param dupKey allow the duplicated key in the skip list - * @return - */ -SSkipList *tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn); +#define SL_IS_THREAD_SAFE(s) (((s)->flags) & SL_THREAD_SAFE) +#define SL_DUP_MODE(s) (((s)->flags) & ((((uint8_t)1) << 2) - 1)) +#define SL_GET_NODE_KEY(s, n) ((s)->keyFn((n)->pData)) +#define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) +#define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) +#define SL_SIZE(s) (s)->size +#define SL_TSIZE(s) (s)->tsize -/** - * - * @param pSkipList - * @return NULL will always be returned - */ -void *tSkipListDestroy(SSkipList *pSkipList); - -/** - * - * @param pSkipList - * @param level - * @param headSize - */ -void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize); - -/** - * put the data into the skiplist - * If failed, NULL will be returned, otherwise, the pNode will be returned. - * - * @param pSkipList - * @param pData - * @param dataLen - * @return - */ -SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen); - -/** - * put the skip list node into the skip list. - * If failed, NULL will be returned, otherwise, the pNode will be returned. - * - * @param pSkipList - * @param pNode - * @return - */ -SSkipListNode *tSkipListPutNode(SSkipList *pSkipList, SSkipListNode *pNode); - -/** - * get *all* nodes which key are equivalent to pKey - * - * @param pSkipList - * @param pKey - * @return - */ -SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); - -/** - * display skip list of the given level, for debug purpose only - * @param pSkipList - * @param nlevel - */ -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); - -/** - * create skiplist iterator - * @param pSkipList - * @return - */ +SSkipList * tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn); +void tSkipListDestroy(SSkipList *pSkipList); +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); +SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); - -/** - * create skip list iterator from the given node and specified the order - * @param pSkipList - * @param pNode start position, instead of the first node in skip list - * @param order traverse order of the iterator - * @return - */ -SSkipListIterator *tSkipListCreateIterFromVal(SSkipList* pSkipList, const char* val, int32_t type, int32_t order); - -/** - * forward the skip list iterator - * @param iter - * @return - */ -bool tSkipListIterNext(SSkipListIterator *iter); - -/** - * get the element of skip list node - * @param iter - * @return - */ -SSkipListNode *tSkipListIterGet(SSkipListIterator *iter); - -/** - * destroy the skip list node - * @param iter - * @return - */ -void *tSkipListDestroyIter(SSkipListIterator *iter); - -/* - * remove nodes of the pKey value. - * If more than one node has the same value, all will be removed - * - * @Return - * the count of removed nodes - */ -uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); - -/* - * remove the specified node in parameters - */ -void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); +SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); +bool tSkipListIterNext(SSkipListIterator *iter); +SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); +void * tSkipListDestroyIter(SSkipListIterator *iter); +uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); +void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); #ifdef __cplusplus } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index c3e8ee41bd..3da137140f 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -19,30 +19,26 @@ #include "tulog.h" #include "tutil.h" -#define DO_MEMSET_PTR_AREA(n) \ - do { \ - int32_t _l = (n)->level; \ - memset(pNode, 0, SL_NODE_HEADER_SIZE(_l)); \ - (n)->level = _l; \ - } while (0) +#define DO_MEMSET_PTR_AREA(n) memset((n)->forwards, 0, ((n)->level * 2)) -static int initForwardBackwardPtr(SSkipList *pSkipList); -static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); -static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); -static void tSkipListCorrectLevel(SSkipList *pSkipList); +static int initForwardBackwardPtr(SSkipList *pSkipList); +static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); +static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); +static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode); +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData); +static SSkipListNode * tSkipListNewNode(uint8_t level); +#define tSkipListFreeNode(n) taosTFree((n)) static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); -static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn) { SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); - if (pSkipList == NULL) { - return NULL; - } + if (pSkipList == NULL) return NULL; if (maxLevel > MAX_SKIP_LIST_LEVEL) { maxLevel = MAX_SKIP_LIST_LEVEL; @@ -55,19 +51,20 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, u pSkipList->keyFn = fn; pSkipList->comparFn = getKeyComparFunc(keyType); - // pSkipList->level = 1; // TODO: check if 1 is valid if (initForwardBackwardPtr(pSkipList) < 0) { - taosTFree(pSkipList); + tSkipListDestroy(pSkipList); return NULL; } - if (SL_IS_THREAD_SAFE(flags)) { + if (SL_IS_THREAD_SAFE(pSkipList)) { pSkipList->lock = (pthread_rwlock_t *)calloc(1, sizeof(pthread_rwlock_t)); + if (pSkipList->lock == NULL) { + tSkipListDestroy(pSkipList); + return NULL; + } if (pthread_rwlock_init(pSkipList->lock, NULL) != 0) { - taosTFree(pSkipList->pHead); - taosTFree(pSkipList); - + tSkipListDestroy(pSkipList); return NULL; } } @@ -81,17 +78,17 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, u return pSkipList; } -void *tSkipListDestroy(SSkipList *pSkipList) { - if (pSkipList == NULL) return NULL; +void tSkipListDestroy(SSkipList *pSkipList) { + if (pSkipList == NULL) return; tSkipListWLock(pSkipList); - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(pSkipList->pHead, 0); + SSkipListNode *pNode = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, 0); while (pNode != pSkipList->pTail) { SSkipListNode *pTemp = pNode; - pNode = SL_GET_FORWARD_POINTER(pNode, 0); - taosTFree(pTemp); + pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0); + tSkipListFreeNode(pTemp); } tSkipListUnlock(pSkipList); @@ -100,42 +97,35 @@ void *tSkipListDestroy(SSkipList *pSkipList) { taosTFree(pSkipList->lock); } - taosTFree(pSkipList->pHead); + tSkipListFreeNode(pSkipList->pHead); + tSkipListFreeNode(pSkipList->pTail); taosTFree(pSkipList); - return NULL; } -void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) { - if (pSkipList == NULL) { - *level = 1; - *headSize = SL_NODE_HEADER_SIZE(*level); - return; - } - - *level = getSkipListRandLevel(pSkipList); - *headSize = SL_NODE_HEADER_SIZE(*level); -} - -SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen) { +SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { if (pSkipList == NULL || pData == NULL) return NULL; + SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; + uint8_t dupMode = SL_DUP_MODE(pSkipList); + SSkipListNode *pNode = NULL; + tSkipListWLock(pSkipList); - int32_t level = getSkipListRandLevel(pSkipList); + bool hasDup = tSkipListGetPosToPut(pSkipList, forward, pData); - SSkipListNode *pNode = (SSkipListNode *)malloc(SL_NODE_HEADER_SIZE(level) + dataLen); - if (pNode == NULL) { - tSkipListUnlock(pSkipList); - return NULL; - } + if (hasDup && (dupMode == SL_DISCARD_DUP_KEY || dupMode == SL_UPDATE_DUP_KEY)) { + if (dupMode == SL_UPDATE_DUP_KEY) { + pNode = SL_NODE_GET_FORWARD_POINTER(forward[0], 0); + atomic_store_ptr(&(pNode->pData), pData); + pNode->flags &= (~(SL_NODE_DELETED_FLAG)); + } + } else { + pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList)); + if (pNode != NULL) { + pNode->pData = pData; - pNode->level = level; - memcpy(SL_GET_NODE_DATA(pNode), pData, dataLen); - - if (tSkipListPutNodeImpl(pSkipList, pNode) == NULL) { - tSkipListUnlock(pSkipList); - taosTFree(pNode); - return NULL; + tSkipListDoInsert(pSkipList, forward, pNode); + } } tSkipListUnlock(pSkipList); @@ -143,41 +133,6 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData, int dataLen) { return pNode; } -SSkipListNode *tSkipListPutNode(SSkipList *pSkipList, SSkipListNode *pNode) { - SSkipListNode *pRetNode = NULL; - - if (pSkipList == NULL || pNode == NULL) return NULL; - - tSkipListWLock(pSkipList); - pRetNode = tSkipListPutNodeImpl(pSkipList, pNode); - tSkipListUnlock(pSkipList); - - return pRetNode; -} - -SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { - SArray *sa = taosArrayInit(1, POINTER_BYTES); - - tSkipListRLock(pSkipList); - - SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); - while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); - if (p == pSkipList->pTail) { - break; - } - if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { - break; - } - taosArrayPush(sa, &p); - pNode = p; - } - - tSkipListUnlock(pSkipList); - - return sa; -} - uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { uint32_t count = 0; @@ -185,7 +140,7 @@ uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); while (1) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, 0); + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); if (p == pSkipList->pTail) { break; } @@ -205,6 +160,31 @@ uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { return count; } +SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { + SArray *sa = taosArrayInit(1, POINTER_BYTES); + + tSkipListRLock(pSkipList); + + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + while (1) { + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); + if (p == pSkipList->pTail) { + break; + } + if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { + break; + } + if (!SL_IS_NODE_DELETED(p)) { + taosArrayPush(sa, &p); + } + pNode = p; + } + + tSkipListUnlock(pSkipList); + + return sa; +} + void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode) { tSkipListWLock(pSkipList); tSkipListRemoveNodeImpl(pSkipList, pNode); @@ -240,68 +220,22 @@ bool tSkipListIterNext(SSkipListIterator *iter) { if (iter->pSkipList == NULL) return false; SSkipList *pSkipList = iter->pSkipList; - uint8_t dupMod = SL_DUP_MODE(pSkipList->flags); tSkipListRLock(pSkipList); - if (iter->order == TSDB_ORDER_ASC) { // ascending order iterate - if (dupMod == SL_APPEND_DUP_KEY) { - if (iter->cur == pSkipList->pHead) { - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); - iter->step++; - } else { - while (true) { - iter->step++; - SSkipListNode *pNode = iter->cur; - iter->cur = SL_GET_FORWARD_POINTER(pNode, 0); - - if (iter->cur == pSkipList->pTail) break; - if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { - continue; - } else { - break; - } - } - } - - if (iter->cur != pSkipList->pTail) { - while (true) { - SSkipListNode *pNode = SL_GET_FORWARD_POINTER(iter->cur, 0); - if (pNode == pSkipList->pTail) break; - if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { - iter->step++; - iter->cur = pNode; - } else { - break; - } - } - } - } else { - iter->cur = SL_GET_FORWARD_POINTER(iter->cur, 0); + if (iter->order == TSDB_ORDER_ASC) { + while (true) { + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); iter->step++; + if (iter->cur == pSkipList->pTail) break; + if (!SL_IS_NODE_DELETED(iter->cur)) break; } - } else { // descending order iterate - if (dupMod == SL_APPEND_DUP_KEY) { - if (iter->cur == pSkipList->pTail) { - iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); - iter->step++; - } else { - while (true) { - SSkipListNode *pNode = iter->cur; - iter->cur = SL_GET_BACKWARD_POINTER(pNode, 0); - - if (iter->cur == pSkipList->pHead) break; - if (pSkipList->comparFn(SL_GET_NODE_KEY(pSkipList, pNode), SL_GET_NODE_KEY(pSkipList, iter->cur)) == 0) { - iter->cur = pNode; - continue; - } else { - break; - } - } - } - } else { - iter->cur = SL_GET_BACKWARD_POINTER(iter->cur, 0); + } else { + while (true) { + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); iter->step++; + if (iter->cur == pSkipList->pHead) break; + if (!SL_IS_NODE_DELETED(iter->cur)) break; } } @@ -332,7 +266,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { return; } - SSkipListNode *p = SL_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, nlevel - 1); int32_t id = 1; char * prev = NULL; @@ -364,35 +298,34 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { prev = SL_GET_NODE_KEY(pSkipList, p); - p = SL_GET_FORWARD_POINTER(p, nlevel - 1); + p = SL_NODE_GET_FORWARD_POINTER(p, nlevel - 1); } } -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode, bool hasDupKey) { - uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); - - // FIXME: this may cause the level of skiplist change - if (dupMode == SL_UPDATA_DUP_KEY && hasDupKey) { - tSkipListRemoveNodeImpl(pSkipList, forward[0]); - tSkipListCorrectLevel(pSkipList); - } - +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { DO_MEMSET_PTR_AREA(pNode); for (int32_t i = 0; i < pNode->level; ++i) { - SSkipListNode *x = forward[i]; - SL_GET_BACKWARD_POINTER(pNode, i) = x; + if (i >= pSkipList->level) { + SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; + SL_NODE_GET_BACKWARD_POINTER(pNode, i) = pSkipList->pHead; + SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pNode; + SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pNode; + } else { + SSkipListNode *x = forward[i]; + SL_NODE_GET_BACKWARD_POINTER(pNode, i) = x; - SSkipListNode *next = SL_GET_FORWARD_POINTER(x, i); - SL_GET_BACKWARD_POINTER(next, i) = pNode; + SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(x, i); + SL_NODE_GET_BACKWARD_POINTER(next, i) = pNode; - SL_GET_FORWARD_POINTER(pNode, i) = next; - SL_GET_FORWARD_POINTER(x, i) = pNode; + SL_NODE_GET_FORWARD_POINTER(pNode, i) = next; + SL_NODE_GET_FORWARD_POINTER(x, i) = pNode; + } } - if (!(dupMode == SL_APPEND_DUP_KEY && hasDupKey)) { - pSkipList->size += 1; - } + if (pSkipList->level < pNode->level) pSkipList->level = pNode->level; + + pSkipList->size += 1; pSkipList->tsize += 1; } @@ -411,78 +344,73 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t } static FORCE_INLINE int tSkipListWLock(SSkipList *pSkipList) { - if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + if (pSkipList->lock) { return pthread_rwlock_wrlock(pSkipList->lock); } return 0; } static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList) { - if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + if (pSkipList->lock) { return pthread_rwlock_rdlock(pSkipList->lock); } return 0; } static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { - if (SL_IS_THREAD_SAFE(pSkipList->flags)) { + if (pSkipList->lock) { return pthread_rwlock_unlock(pSkipList->lock); } return 0; } -static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { +static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData) { int compare = 1; bool hasDupKey = false; - char * pNodeKey = SL_GET_NODE_KEY(pSkipList, pNode); - uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + char * pDataKey = pSkipList->keyFn(pData); if (pSkipList->size == 0) { - for (int i = 0; i < pNode->level; i++) { + for (int i = 0; i < pSkipList->level; i++) { forward[i] = pSkipList->pHead; } } else { char *pKey = NULL; // Compare min key - pKey = SL_GET_SL_MIN_KEY(pSkipList); - compare = pSkipList->comparFn(pNodeKey, pKey); - if ((dupMode == SL_APPEND_DUP_KEY && compare < 0) || (dupMode != SL_APPEND_DUP_KEY && compare <= 0)) { - for (int i = 0; i < pNode->level; i++) { + pKey = SL_GET_MIN_KEY(pSkipList); + compare = pSkipList->comparFn(pDataKey, pKey); + if (compare <= 0) { + for (int i = 0; i < pSkipList->level; i++) { forward[i] = pSkipList->pHead; } + return (compare == 0); } // Compare max key - pKey = SL_GET_SL_MAX_KEY(pSkipList); - compare = pSkipList->comparFn(pNodeKey, pKey); - if ((dupMode == SL_DISCARD_DUP_KEY && compare > 0) || (dupMode != SL_DISCARD_DUP_KEY && compare >= 0)) { - for (int i = 0; i < pNode->level; i++) { - forward[i] = SL_GET_BACKWARD_POINTER(pSkipList->pTail, i); + pKey = SL_GET_MAX_KEY(pSkipList); + compare = pSkipList->comparFn(pDataKey, pKey); + if (compare > 0) { + for (int i = 0; i < pSkipList->level; i++) { + forward[i] = SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i); } return (compare == 0); } SSkipListNode *px = pSkipList->pHead; - for (int i = pNode->level - 1; i >= 0; --i) { - if (i >= pSkipList->level) { - forward[i] = pSkipList->pHead; - continue; - } - - SSkipListNode *p = SL_GET_FORWARD_POINTER(px, i); + for (int i = pSkipList->level - 1; i >= 0; --i) { + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(px, i); while (p != pSkipList->pTail) { pKey = SL_GET_NODE_KEY(pSkipList, p); - compare = pSkipList->comparFn(pKey, pNodeKey); - if (compare == 0 && hasDupKey == false) hasDupKey = true; - if ((dupMode == SL_APPEND_DUP_KEY && compare > 0) || (dupMode != SL_APPEND_DUP_KEY && compare >= 0)) { + compare = pSkipList->comparFn(pKey, pDataKey); + if (compare >= 0) { + if (compare == 0 && !hasDupKey) hasDupKey = true; break; } else { px = p; - p = SL_GET_FORWARD_POINTER(px, i); + p = SL_NODE_GET_FORWARD_POINTER(px, i); } } @@ -493,40 +421,35 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, return hasDupKey; } -static bool tSkipListIsNodeDup(SSkipList *pSkipList, SSkipListNode *pNode) { - SSkipListNode *pPrevNode = SL_GET_BACKWARD_POINTER(pNode, 0); - SSkipListNode *pNextNode = SL_GET_FORWARD_POINTER(pNode, 0); - char * pNodeKey = SL_GET_NODE_KEY(pSkipList, pNode); - char * pPrevNodeKey = (pPrevNode == pSkipList->pHead) ? NULL : SL_GET_NODE_KEY(pSkipList, pPrevNode); - char * pNextNodeKey = (pNextNode == pSkipList->pTail) ? NULL : SL_GET_NODE_KEY(pSkipList, pNextNode); - - return ((pPrevNodeKey != NULL && pSkipList->comparFn(pNodeKey, pPrevNodeKey) == 0) || - (pNextNodeKey != NULL && pSkipList->comparFn(pNodeKey, pNextNodeKey) == 0)); -} - static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { int32_t level = pNode->level; - uint8_t dupMode = SL_DUP_MODE(pSkipList->flags); + uint8_t dupMode = SL_DUP_MODE(pSkipList); - bool sizeReduce = !(dupMode == SL_APPEND_DUP_KEY && tSkipListIsNodeDup(pSkipList, pNode)); + if (dupMode == SL_UPDATE_DUP_KEY) { + if (SL_IS_NODE_DELETED(pNode)) { + return; + } else { + SL_SET_NODE_DELETED(pNode); + pSkipList->size--; + } + } else { + for (int32_t j = level - 1; j >= 0; --j) { + SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j); + SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j); - for (int32_t j = level - 1; j >= 0; --j) { - SSkipListNode *prev = SL_GET_BACKWARD_POINTER(pNode, j); - SSkipListNode *next = SL_GET_FORWARD_POINTER(pNode, j); + SL_NODE_GET_FORWARD_POINTER(prev, j) = next; + SL_NODE_GET_BACKWARD_POINTER(next, j) = prev; + } - SL_GET_FORWARD_POINTER(prev, j) = next; - SL_GET_BACKWARD_POINTER(next, j) = prev; + tSkipListFreeNode(pNode); + pSkipList->size--; + pSkipList->tsize--; } - - taosTFree(pNode); - - if (sizeReduce) pSkipList->size--; - pSkipList->tsize--; } // Function must be called after calling tSkipListRemoveNodeImpl() function static void tSkipListCorrectLevel(SSkipList *pSkipList) { - while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { + while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == pSkipList->pTail) { pSkipList->level -= 1; } } @@ -587,12 +510,12 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ if (order == TSDB_ORDER_ASC) { pNode = pSkipList->pHead; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_FORWARD_POINTER(pNode, i); + SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, i); while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); if (comparFn(key, val) < 0) { pNode = p; - p = SL_GET_FORWARD_POINTER(p, i); + p = SL_NODE_GET_FORWARD_POINTER(p, i); } else { break; } @@ -601,12 +524,12 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ } else { pNode = pSkipList->pTail; for (int32_t i = pSkipList->level - 1; i >= 0; --i) { - SSkipListNode *p = SL_GET_BACKWARD_POINTER(pNode, i); + SSkipListNode *p = SL_NODE_GET_BACKWARD_POINTER(pNode, i); while (p != pSkipList->pHead) { char *key = SL_GET_NODE_KEY(pSkipList, p); if (comparFn(key, val) > 0) { pNode = p; - p = SL_GET_BACKWARD_POINTER(p, i); + p = SL_NODE_GET_BACKWARD_POINTER(p, i); } else { break; } @@ -621,46 +544,34 @@ static int initForwardBackwardPtr(SSkipList *pSkipList) { uint32_t maxLevel = pSkipList->maxLevel; // head info - pSkipList->pHead = (SSkipListNode *)malloc(SL_NODE_HEADER_SIZE(maxLevel) * 2); - if (pSkipList->pHead == NULL) { + pSkipList->pHead = tSkipListNewNode(maxLevel); + if (pSkipList->pHead == NULL) return -1; + + // tail info + pSkipList->pTail = tSkipListNewNode(maxLevel); + if (pSkipList->pTail == NULL) { + tSkipListFreeNode(pSkipList->pHead); return -1; } - pSkipList->pHead->level = maxLevel; - - // tail info - pSkipList->pTail = (SSkipListNode *)POINTER_SHIFT(pSkipList->pHead, SL_NODE_HEADER_SIZE(maxLevel)); - pSkipList->pTail->level = maxLevel; - for (uint32_t i = 0; i < maxLevel; ++i) { - SL_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; - SL_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; + SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, i) = pSkipList->pTail; + SL_NODE_GET_BACKWARD_POINTER(pSkipList->pTail, i) = pSkipList->pHead; } return 0; } -static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { - SSkipListNode *pRetNode = NULL; - SSkipListNode *forward[MAX_SKIP_LIST_LEVEL] = {0}; +static SSkipListNode *tSkipListNewNode(uint8_t level) { + int32_t tsize = sizeof(SSkipListNode) + sizeof(SSkipListNode *) * level * 2; - int hasDupKey = tSkipListGetPosToPut(pSkipList, forward, pNode); - if (SL_DUP_MODE(pSkipList->flags) == SL_DISCARD_DUP_KEY && hasDupKey) { - pRetNode = NULL; - } else { - pRetNode = pNode; - tSkipListDoInsert(pSkipList, forward, pNode, hasDupKey); + SSkipListNode *pNode = (SSkipListNode *)calloc(1, tsize); + if (pNode == NULL) return NULL; - if (pNode->level > pSkipList->level) pSkipList->level = pNode->level; - } - - return pRetNode; + pNode->level = level; + return pNode; } -// static void tSkipListSeek(SSkipList *pSkipList, char *key, int order) { -// // TODO -// } - // static int32_t tSkipListEndParQuery(SSkipList *pSkipList, SSkipListNode *pStartNode, SSkipListKey *pEndKey, // int32_t cond, SSkipListNode ***pRes) { // pthread_rwlock_rdlock(&pSkipList->lock); @@ -774,7 +685,7 @@ static FORCE_INLINE SSkipListNode *tSkipListPutNodeImpl(SSkipList *pSkipList, SS // } // // // compress the minimum level of skip list -// while (pSkipList->level > 0 && SL_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { +// while (pSkipList->level > 0 && SL_NODE_GET_FORWARD_POINTER(pSkipList->pHead, pSkipList->level - 1) == NULL) { // pSkipList->level -= 1; // } // From 5ea611f2c8284c22f5895cdb3ae1cc69a53a505d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Sep 2020 13:44:22 +0800 Subject: [PATCH 11/58] TD-1194 --- src/tsdb/src/tsdbMemTable.c | 7 ------ src/util/inc/tskiplist.h | 2 ++ src/util/src/tskiplist.c | 43 ++++++++++++++++++++++++++++--------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 9c76a09ca3..e139b29d5e 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -40,13 +40,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { TSKEY key = dataRowKey(row); SMemTable * pMemTable = pRepo->mem; STableData *pTableData = NULL; - // SSkipList * pSList = NULL; - - // if (pMemTable != NULL && TABLE_TID(pTable) < pMemTable->maxTables && pMemTable->tData[TABLE_TID(pTable)] != NULL && - // pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { - // pTableData = pMemTable->tData[TABLE_TID(pTable)]; - // pSList = pTableData->pData; - // } void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); if (pRow == NULL) { diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 1eccac6646..030a9d69c1 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -145,6 +145,8 @@ SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); void * tSkipListDestroyIter(SSkipListIterator *iter); uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); +SSkipListKey tSkipListGetMinKey(SSkipList *pSkipList); +SSkipListKey tSkipListGetMaxKey(SSkipList *pSkipList); #ifdef __cplusplus } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 3da137140f..603a9759b8 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -19,8 +19,6 @@ #include "tulog.h" #include "tutil.h" -#define DO_MEMSET_PTR_AREA(n) memset((n)->forwards, 0, ((n)->level * 2)) - static int initForwardBackwardPtr(SSkipList *pSkipList); static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); @@ -117,7 +115,10 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { if (dupMode == SL_UPDATE_DUP_KEY) { pNode = SL_NODE_GET_FORWARD_POINTER(forward[0], 0); atomic_store_ptr(&(pNode->pData), pData); - pNode->flags &= (~(SL_NODE_DELETED_FLAG)); + if (SL_IS_NODE_DELETED(pNode)) { + pNode->flags &= (~(SL_NODE_DELETED_FLAG)); + pSkipList->size++; + } } } else { pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList)); @@ -136,6 +137,8 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { uint32_t count = 0; + if (SL_DUP_MODE(pSkipList) == SL_DISCARD_DUP_KEY) return 0; + tSkipListWLock(pSkipList); SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); @@ -199,8 +202,8 @@ SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList) { } SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order) { - assert(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); - assert(pSkipList != NULL); + ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + ASSERT(pSkipList != NULL); SSkipListIterator *iter = doCreateSkipListIterator(pSkipList, order); if (val == NULL) { @@ -274,7 +277,7 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { while (p != pSkipList->pTail) { char *key = SL_GET_NODE_KEY(pSkipList, p); if (prev != NULL) { - assert(pSkipList->comparFn(prev, key) < 0); + ASSERT(pSkipList->comparFn(prev, key) < 0); } switch (pSkipList->type) { @@ -302,9 +305,29 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { } } -static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { - DO_MEMSET_PTR_AREA(pNode); +SSkipListKey tSkipListGetMinKey(SSkipList *pSkipList) { + if (pSkipList == NULL || SL_SIZE(pSkipList) == 0) return NULL; + SSkipListNode *pNode = pSkipList->pHead; + while ((pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0)) != pSkipList->pTail) { + if (!SL_IS_NODE_DELETED(pNode)) return pSkipList->keyFn(pNode->pData); + } + + return NULL; +} + +SSkipListKey tSkipListGetMaxKey(SSkipList *pSkipList) { + if (pSkipList == NULL || SL_SIZE(pSkipList) == 0) return NULL; + + SSkipListNode *pNode = pSkipList->pTail; + while ((pNode = SL_NODE_GET_BACKWARD_POINTER(pNode, 0)) != pSkipList->pHead) { + if (!SL_IS_NODE_DELETED(pNode)) return pSkipList->keyFn(pNode->pData); + } + + return NULL; +} + +static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { for (int32_t i = 0; i < pNode->level; ++i) { if (i >= pSkipList->level) { SL_NODE_GET_FORWARD_POINTER(pNode, i) = pSkipList->pTail; @@ -365,7 +388,7 @@ static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList) { } static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, void *pData) { - int compare = 1; + int compare = 0; bool hasDupKey = false; char * pDataKey = pSkipList->keyFn(pData); @@ -497,7 +520,7 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { } } - assert(level <= pSkipList->maxLevel); + ASSERT(level <= pSkipList->maxLevel); return level; } From e118dc87a590f6e0167b75494a606bbe13adee7f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 22 Sep 2020 14:48:36 +0800 Subject: [PATCH 12/58] TD-1194 --- src/query/src/qAst.c | 20 ++++++++++---------- src/tsdb/src/tsdbMeta.c | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index ef98dbeda7..ca33f6a067 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -432,7 +432,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr break; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } } else if (optr == TSDB_RELATION_GREATER || optr == TSDB_RELATION_GREATER_EQUAL) { // greater equal @@ -450,7 +450,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_GREATER) { continue; } else { - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); comp = false; } @@ -465,7 +465,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr continue; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } @@ -476,11 +476,11 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr while(tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); comp = comp && (pQueryInfo->compare(SL_GET_NODE_KEY(pSkipList, pNode), cond.start->v) == 0); - if (comp) { + if (comp) { continue; } - STableKeyInfo info = {.pTable = *(void**)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } @@ -504,7 +504,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr if (ret == 0 && optr == TSDB_RELATION_LESS) { continue; } else { - STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); comp = false; // no need to compare anymore } @@ -518,7 +518,7 @@ static void tQueryIndexColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SArr bool isnull = isNull(SL_GET_NODE_KEY(pSkipList, pNode), pQueryInfo->sch.type); if ((pQueryInfo->optr == TSDB_RELATION_ISNULL && isnull) || (pQueryInfo->optr == TSDB_RELATION_NOTNULL && (!isnull))) { - STableKeyInfo info = {.pTable = *(void **)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void *)SL_GET_NODE_DATA(pNode), .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(result, &info); } } @@ -684,7 +684,7 @@ static void tSQLBinaryTraverseOnSkipList(tExprNode *pExpr, SArray *pResult, SSki while (tSkipListIterNext(iter)) { SSkipListNode *pNode = tSkipListIterGet(iter); if (filterItem(pExpr, pNode, param)) { - taosArrayPush(pResult, SL_GET_NODE_DATA(pNode)); + taosArrayPush(pResult, &(SL_GET_NODE_DATA(pNode))); } } tSkipListDestroyIter(iter); @@ -699,7 +699,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, SSkipListNode *pNode = tSkipListIterGet(iter); char * pData = SL_GET_NODE_DATA(pNode); - tstr *name = (tstr*) tsdbGetTableName(*(void**) pData); + tstr *name = (tstr*) tsdbGetTableName((void*) pData); // todo speed up by using hash if (pQueryInfo->sch.colId == TSDB_TBNAME_COLUMN_INDEX) { @@ -713,7 +713,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, } if (addToResult) { - STableKeyInfo info = {.pTable = *(void**)pData, .lastKey = TSKEY_INITIAL_VAL}; + STableKeyInfo info = {.pTable = (void*)pData, .lastKey = TSKEY_INITIAL_VAL}; taosArrayPush(res, &info); } } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 391cb55985..1914ff08ed 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -925,7 +925,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { SSkipListNode *pNode = taosArrayGetP(res, i); // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); - if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need + if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need tSkipListRemoveNode(pSTable->pIndex, pNode); } } @@ -1229,7 +1229,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable) { } while (tSkipListIterNext(pIter)) { - STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE); pBuf = tsdbInsertTableAct(pRepo, TSDB_DROP_META, pBuf, tTable); } @@ -1254,7 +1254,7 @@ static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable) { tsdbWLockRepoMeta(pRepo); while (tSkipListIterNext(pIter)) { - STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + STable *tTable = (STable *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); tsdbRemoveTableFromMeta(pRepo, tTable, false, false); } From bbf367772976725144942c9b7367abe644c098af Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 27 Sep 2020 10:27:53 +0800 Subject: [PATCH 13/58] TD-1548 --- src/common/inc/tdataformat.h | 74 +++++-- src/common/src/tdataformat.c | 69 ++++--- src/inc/taosdef.h | 1 + src/tsdb/inc/tsdbMain.h | 22 +- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbMemTable.c | 192 ++++++++++++----- src/tsdb/src/tsdbMeta.c | 11 +- src/tsdb/src/tsdbRWHelper.c | 387 ++++++++++++++++++++--------------- src/util/inc/tskiplist.h | 22 +- src/util/src/tskiplist.c | 87 +++----- 10 files changed, 516 insertions(+), 351 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index aa39d5b0b2..8124075e6c 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -119,6 +119,33 @@ void tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version); int tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId, int16_t bytes); STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); +// ----------------- Semantic timestamp key definition +typedef uint64_t TKEY; + +#define TKEY_INVALID UINT64_MAX +#define TKEY_NULL TKEY_INVALID +#define TKEY_NEGATIVE_FLAG (((TKEY)1) << (sizeof(TKEY) * 8 - 1)) +#define TKEY_DELETE_FLAG (((TKEY)1) << (sizeof(TKEY) * 8 - 2)) +#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) + +#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) +#define TKEY_IS_DELETED(tkey) (((tkey)&TKEY_DELETE_FLAG) != 0) +#define tdSetTKEYDeleted(tkey) ((tkey) | TKEY_DELETE_FLAG) +#define tdGetTKEY(key) (((TKEY)ABS(key)) | (TKEY_NEGATIVE_FLAG & (TKEY)(key))) +#define tdGetKey(tkey) (((TSKEY)((tkey)&TKEY_VALUE_FILTER)) * (TKEY_IS_NEGATIVE(tkey) ? -1 : 1)) + +static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) { + TSKEY key1 = tdGetKey(*(TKEY *)tkey1); + TSKEY key2 = tdGetKey(*(TKEY *)tkey2); + + if (key1 < key2) { + return -1; + } else if (key1 > key2) { + return 1; + } else { + return 0; + } +} // ----------------- Data row structure /* A data row, the format is like below: @@ -129,6 +156,8 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder); * +----------+----------+---------------------------------+---------------------------------+ * | len | sversion | First part | Second part | * +----------+----------+---------------------------------+---------------------------------+ + * + * NOTE: timestamp in this row structure is TKEY instead of TSKEY */ typedef void *SDataRow; @@ -137,11 +166,13 @@ typedef void *SDataRow; #define dataRowLen(r) (*(uint16_t *)(r)) #define dataRowVersion(r) *(int16_t *)POINTER_SHIFT(r, sizeof(int16_t)) #define dataRowTuple(r) POINTER_SHIFT(r, TD_DATA_ROW_HEAD_SIZE) -#define dataRowKey(r) (*(TSKEY *)(dataRowTuple(r))) +#define dataRowTKey(r) (*(TKEY *)(dataRowTuple(r))) +#define dataRowKey(r) tdGetKey(dataRowTKey(r)) #define dataRowSetLen(r, l) (dataRowLen(r) = (l)) #define dataRowSetVersion(r, v) (dataRowVersion(r) = (v)) #define dataRowCpy(dst, r) memcpy((dst), (r), dataRowLen(r)) #define dataRowMaxBytesFromSchema(s) (schemaTLen(s) + TD_DATA_ROW_HEAD_SIZE) +#define dataRowDeleted(r) TKEY_IS_DELETED(dataRowTKey(r)) SDataRow tdNewDataRowFromSchema(STSchema *pSchema); void tdFreeDataRow(SDataRow row); @@ -154,16 +185,18 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i int32_t toffset = offset + TD_DATA_ROW_HEAD_SIZE; char * ptr = (char *)POINTER_SHIFT(row, dataRowLen(row)); - switch (type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); - memcpy(ptr, value, varDataTLen(value)); - dataRowLen(row) += varDataTLen(value); - break; - default: + if (IS_VAR_DATA_TYPE(type)) { + *(VarDataOffsetT *)POINTER_SHIFT(row, toffset) = dataRowLen(row); + memcpy(ptr, value, varDataTLen(value)); + dataRowLen(row) += varDataTLen(value); + } else { + if (offset == 0) { + ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); + TKEY tvalue = tdGetTKEY(*(TSKEY *)value); + memcpy(POINTER_SHIFT(row, toffset), &tvalue, TYPE_BYTES[type]); + } else { memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); - break; + } } return 0; @@ -171,12 +204,10 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i // NOTE: offset here including the header size static FORCE_INLINE void *tdGetRowDataOfCol(SDataRow row, int8_t type, int32_t offset) { - switch (type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); - default: - return POINTER_SHIFT(row, offset); + if (IS_VAR_DATA_TYPE(type)) { + return POINTER_SHIFT(row, *(VarDataOffsetT *)POINTER_SHIFT(row, offset)); + } else { + return POINTER_SHIFT(row, offset); } } @@ -243,9 +274,14 @@ typedef struct { } SDataCols; #define keyCol(pCols) (&((pCols)->cols[0])) // Key column -#define dataColsKeyAt(pCols, idx) ((TSKEY *)(keyCol(pCols)->pData))[(idx)] -#define dataColsKeyFirst(pCols) dataColsKeyAt(pCols, 0) -#define dataColsKeyLast(pCols) ((pCols->numOfRows == 0) ? 0 : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) +#define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] +#define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) +#define dataColsTKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0) +#define dataColsKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TSDB_DATA_BIGINT_NULL : dataColsKeyAt(pCols, 0) +#define dataColsTKeyLast(pCols) \ + (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) +#define dataColsKeyLast(pCols) \ + (((pCols)->numOfRows == 0) ? TSDB_DATA_BIGINT_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 0c446b6d4f..b45cbd796d 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -423,30 +423,41 @@ void tdResetDataCols(SDataCols *pCols) { } void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) { - ASSERT(dataColsKeyLast(pCols) < dataRowKey(row)); + ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row)); int rcol = 0; int dcol = 0; - while (dcol < pCols->numOfCols) { - SDataCol *pDataCol = &(pCols->cols[dcol]); - if (rcol >= schemaNCols(pSchema)) { - dataColSetNullAt(pDataCol, pCols->numOfRows); - dcol++; - continue; + if (dataRowDeleted(row)) { + for (; dcol < pCols->numOfCols; dcol++) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (dcol == 0) { + dataColAppendVal(pDataCol, dataRowTuple(row), pCols->numOfRows, pCols->maxPoints); + } else { + dataColSetNullAt(pDataCol, pCols->numOfRows); + } } + } else { + while (dcol < pCols->numOfCols) { + SDataCol *pDataCol = &(pCols->cols[dcol]); + if (rcol >= schemaNCols(pSchema)) { + dataColSetNullAt(pDataCol, pCols->numOfRows); + dcol++; + continue; + } - STColumn *pRowCol = schemaColAt(pSchema, rcol); - if (pRowCol->colId == pDataCol->colId) { - void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset+TD_DATA_ROW_HEAD_SIZE); - dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); - dcol++; - rcol++; - } else if (pRowCol->colId < pDataCol->colId) { - rcol++; - } else { - dataColSetNullAt(pDataCol, pCols->numOfRows); - dcol++; + STColumn *pRowCol = schemaColAt(pSchema, rcol); + if (pRowCol->colId == pDataCol->colId) { + void *value = tdGetRowDataOfCol(row, pRowCol->type, pRowCol->offset + TD_DATA_ROW_HEAD_SIZE); + dataColAppendVal(pDataCol, value, pCols->numOfRows, pCols->maxPoints); + dcol++; + rcol++; + } else if (pRowCol->colId < pDataCol->colId) { + rcol++; + } else { + dataColSetNullAt(pDataCol, pCols->numOfRows); + dcol++; + } } } pCols->numOfRows++; @@ -511,8 +522,12 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i while (target->numOfRows < tRows) { if (*iter1 >= limit1 && *iter2 >= limit2) break; - TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : ((TSKEY *)(src1->cols[0].pData))[*iter1]; - TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : ((TSKEY *)(src2->cols[0].pData))[*iter2]; + TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1); + TKEY tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1); + TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2); + TKEY tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2); + + ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1))); if (key1 < key2) { for (int i = 0; i < src1->numOfCols; i++) { @@ -525,12 +540,14 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i target->numOfRows++; (*iter1)++; - } else { - for (int i = 0; i < src2->numOfCols; i++) { - ASSERT(target->cols[i].type == src2->cols[i].type); - if (src2->cols[i].len > 0) { - dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, - target->maxPoints); + } else if (key1 >= key2) { + if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) { + for (int i = 0; i < src2->numOfCols; i++) { + ASSERT(target->cols[i].type == src2->cols[i].type); + if (src2->cols[i].len > 0) { + dataColAppendVal(&(target->cols[i]), tdGetColDataOfRow(src2->cols + i, *iter2), target->numOfRows, + target->maxPoints); + } } } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index e9aa6a7050..a6cf76c473 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -75,6 +75,7 @@ extern const int32_t TYPE_BYTES[11]; #define TSDB_DATA_SMALLINT_NULL 0x8000 #define TSDB_DATA_INT_NULL 0x80000000 #define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL #define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN #define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index b515752bc3..7d40d7f00a 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -320,6 +320,15 @@ typedef struct { void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; +typedef struct { + int rowsInserted; + int rowsUpdated; + int rowsDeleteSucceed; + int rowsDeleteFailed; + int nOperations; + TSKEY keyFirst; + TSKEY keyLast; +} SMergeInfo; // ------------------ tsdbScan.c typedef struct { SFileGroup fGroup; @@ -422,7 +431,7 @@ void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); // ------------------ tsdbMemTable.c -int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); +int tsdbUpdateRowInMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); @@ -430,7 +439,7 @@ void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem) void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, - TSKEY* filterKeys, int nFilterKeys, bool keepDup); + TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { if (pIter == NULL) return NULL; @@ -443,11 +452,18 @@ static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) { static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) { SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) return -1; + if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL; return dataRowKey(row); } +static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) { + SDataRow row = tsdbNextIterRow(pIter); + if (row == NULL) return TKEY_NULL; + + return dataRowTKey(row); +} + static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { ASSERT(pRepo != NULL); if (pRepo->mem == NULL) return NULL; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 494f17807e..a0ff1a6b95 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -765,7 +765,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY return -1; } - if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1; + if (tsdbUpdateRowInMem(pRepo, row, pTable) < 0) return -1; (*affectedrows)++; points++; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index e139b29d5e..cd688d46a7 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -32,14 +32,31 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo); static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables); static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row); // ---------------- INTERNAL FUNCTIONS ---------------- -int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { +int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { STsdbCfg * pCfg = &pRepo->config; STsdbMeta * pMeta = pRepo->tsdbMeta; + TKEY tkey = dataRowTKey(row); TSKEY key = dataRowKey(row); SMemTable * pMemTable = pRepo->mem; STableData *pTableData = NULL; + bool isRowDelete = TKEY_IS_DELETED(tkey); + + if (isRowDelete) { + if (!pCfg->update) { + tsdbWarn("vgId:%d vnode is not allowed to update but try to delete a data row", REPO_ID(pRepo)); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + return -1; + } + + if (key > TABLE_LASTKEY(pTable)) { + tsdbTrace("vgId:%d skip to delete row key %" PRId64 " which is larger than table lastKey %" PRId64, + REPO_ID(pRepo), key, TABLE_LASTKEY(pTable)); + return 0; + } + } void *pRow = tsdbAllocBytes(pRepo, dataRowLen(row)); if (pRow == NULL) { @@ -88,8 +105,10 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (tSkipListPut(pTableData->pData, pRow) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); } else { + // TODO: may need to refact here int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize; - if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; + if ((!isRowDelete) && (TABLE_LASTKEY(pTable) < key)) TABLE_LASTKEY(pTable) = key; + if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; pMemTable->numOfRows += deltaSize; @@ -99,8 +118,9 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { pTableData->numOfRows += deltaSize; } - tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key); + tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), + isRowDelete ? "deleted from" : "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), + key); return 0; } @@ -270,66 +290,120 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { return 0; } +/** + * This is an important function to load data or try to load data from memory skiplist iterator. + * + * This function load memory data until: + * 1. iterator ends + * 2. data key exceeds maxKey + * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead + * 4. operations in pCols not exceeds its max capacity if pCols is given + * + * The function try to move as mush as possible. + */ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, - TSKEY *filterKeys, int nFilterKeys, bool keepDup) { - ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0); + TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { + ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL); if (pIter == NULL) return 0; STSchema *pSchema = NULL; - int numOfRows = 0; - TSKEY keyNext = 0; + TSKEY rowKey = 0; + TSKEY fKey = 0; + bool isRowDel = false; int filterIter = 0; + SDataRow row = NULL; - if (nFilterKeys != 0) { // for filter purpose - ASSERT(filterKeys != NULL); - keyNext = tsdbNextIterKey(pIter); - if (keyNext < 0 || keyNext > maxKey) return numOfRows; - void *ptr = taosbsearch((void *)(&keyNext), (void *)filterKeys, nFilterKeys, sizeof(TSKEY), compTSKEY, TD_GE); - filterIter = (ptr == NULL) ? nFilterKeys : (int)((POINTER_DISTANCE(ptr, filterKeys) / sizeof(TSKEY))); + memset(pMergeInfo, 0, sizeof(*pMergeInfo)); + pMergeInfo->keyFirst = INT64_MAX; + pMergeInfo->keyLast = INT64_MIN; + + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); } - do { - SDataRow row = tsdbNextIterRow(pIter); - if (row == NULL) break; + if (nFilterKeys == 0 || filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } - keyNext = dataRowKey(row); - if (keyNext > maxKey) break; + while (true) { + if (fKey == INT64_MAX && rowKey == INT64_MAX) break; - bool keyFiltered = false; - if (nFilterKeys != 0) { - while (true) { - if (filterIter >= nFilterKeys) break; - if (keyNext == filterKeys[filterIter]) { - keyFiltered = true; - filterIter++; - break; - } else if (keyNext < filterKeys[filterIter]) { - break; + if (fKey < rowKey) { + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); + + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + } else if (fKey > rowKey) { + if (isRowDel) { + pMergeInfo->rowsDeleteFailed++; + } else { + if (pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed >= maxRowsToRead) break; + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsInserted++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } + + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } + } else { + if (isRowDel) { + ASSERT(!keepDup); + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsDeleteSucceed++; + pMergeInfo->nOperations++; + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); + } else { + if (keepDup) { + if (pCols && pMergeInfo->nOperations >= pCols->maxPoints) break; + pMergeInfo->rowsUpdated++; + pMergeInfo->nOperations++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, rowKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, rowKey); + tsdbAppendTableRowToCols(pTable, pCols, &pSchema, row); } else { - filterIter++; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, fKey); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, fKey); } } - } - if (!keyFiltered) { - if (numOfRows >= maxRowsToRead) break; - numOfRows++; - } - - if (!keyFiltered || keepDup) { - if (pCols) { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); - if (pSchema == NULL) { - ASSERT(0); - } - } - - tdAppendDataRowToDataCol(row, pSchema, pCols); + tSkipListIterNext(pIter); + row = tsdbNextIterRow(pIter); + if (row == NULL || dataRowKey(row) > maxKey) { + rowKey = INT64_MAX; + } else { + rowKey = dataRowKey(row); + isRowDel = dataRowDeleted(row); } - } - } while (tSkipListIterNext(pIter)); - return numOfRows; + filterIter++; + if (filterIter >= nFilterKeys) { + fKey = INT64_MAX; + } else { + fKey = tdGetKey(filterKeys[filterIter]); + } + } + } + + return 0; } // ---------------- LOCAL FUNCTIONS ---------------- @@ -420,7 +494,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], - pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); + tkeyComparFn, pCfg->update ? SL_UPDATE_DUP_KEY : SL_DISCARD_DUP_KEY, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -562,7 +636,7 @@ static void tsdbEndCommit(STsdbRepo *pRepo) { static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { for (int i = 0; i < nIters; i++) { TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter); - if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; + if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1; } return 0; } @@ -758,5 +832,21 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) { taosTFree(tData); + return 0; +} + +static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SDataRow row) { + if (pCols) { + if (*ppSchema == NULL || schemaVersion(*ppSchema) != dataRowVersion(row)) { + *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, dataRowVersion(row)); + if (*ppSchema == NULL) { + ASSERT(false); + return -1; + } + } + + tdAppendDataRowToDataCol(row, *ppSchema, pCols); + } + return 0; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 1914ff08ed..53a1b3f5bf 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -86,7 +86,8 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { if (pTable != NULL) { tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); - return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + goto _err; } if (pCfg->type == TSDB_CHILD_TABLE) { @@ -700,7 +701,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { } pTable->tagVal = NULL; STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), SL_ALLOW_DUP_KEY, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -745,7 +746,7 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { T_REF_INC(pTable); - tsdbTrace("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), + tsdbDebug("table %s tid %d uid %" PRIu64 " is created", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable)); return pTable; @@ -1155,8 +1156,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { buf = tdDecodeSchema(buf, &(pTable->tagSchema)); STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); - pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), SL_ALLOW_DUP_KEY, getTagIndexKey); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), (uint8_t)(colBytes(pCol)), NULL, + SL_ALLOW_DUP_KEY, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbFreeTable(pTable); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 6c6a738899..c6d4ae7a5e 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -27,6 +27,7 @@ #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM)) #define TSDB_KEY_COL_OFFSET 0 #define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock)) +#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last) static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock, @@ -34,7 +35,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pD static int compareKeyBlock(const void *arg1, const void *arg2); static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize); static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo); static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); static void tsdbResetHelperFileImpl(SRWHelper *pHelper); static int tsdbInitHelperFile(SRWHelper *pHelper); @@ -61,8 +62,10 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pComp static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock); static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx); -static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update); +static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps); +static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -279,7 +282,7 @@ int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols while (true) { ASSERT(blkIdx <= (int)pIdx->numOfBlocks); TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - if (keyFirst < 0 || keyFirst > maxKey) break; // iter over + if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break; // iter over if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) { if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1; @@ -925,7 +928,7 @@ _err: return -1; } -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded) { +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) { ASSERT(pCompBlock->numOfSubBlocks == 0); SCompIdx *pIdx = &(pHelper->curCompIdx); @@ -958,9 +961,9 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pSCompBlock->numOfSubBlocks++; ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); pSCompBlock->len += sizeof(SCompBlock); - pSCompBlock->numOfRows += rowsAdded; - pSCompBlock->keyFirst = MIN(pSCompBlock->keyFirst, pCompBlock->keyFirst); - pSCompBlock->keyLast = MAX(pSCompBlock->keyLast, pCompBlock->keyLast); + pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; + pSCompBlock->keyFirst = pMergeInfo->keyFirst; + pSCompBlock->keyLast = pMergeInfo->keyLast; pIdx->len += sizeof(SCompBlock); } else { // Need to create two sub-blocks void *ptr = NULL; @@ -989,11 +992,11 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId ((SCompBlock *)ptr)[1] = *pCompBlock; pSCompBlock->numOfSubBlocks = 2; - pSCompBlock->numOfRows += rowsAdded; + pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo); pSCompBlock->len = sizeof(SCompBlock) * 2; - pSCompBlock->keyFirst = MIN(((SCompBlock *)ptr)[0].keyFirst, ((SCompBlock *)ptr)[1].keyFirst); - pSCompBlock->keyLast = MAX(((SCompBlock *)ptr)[0].keyLast, ((SCompBlock *)ptr)[1].keyLast); + pSCompBlock->keyFirst = pMergeInfo->keyFirst; + pSCompBlock->keyLast = pMergeInfo->keyLast; pIdx->len += (sizeof(SCompBlock) * 2); } @@ -1047,6 +1050,45 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int return 0; } +static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { + SCompIdx *pCompIdx = &(pHelper->curCompIdx); + + ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks); + + SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx); + SCompBlock compBlock = *pCompBlock; + ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS); + + if (pCompIdx->numOfBlocks == 1) { + memset(pCompIdx, 0, sizeof(*pCompIdx)); + } else { + int tsize = 0; + + if (compBlock.numOfSubBlocks > 1) { + tsize = pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks); + + ASSERT(tsize > 0); + memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset), + POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks), + tsize); + + pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks; + } + + tsize = pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo); + ASSERT(tsize > 0); + memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize); + + pCompIdx->len -= sizeof(SCompBlock); + + pCompIdx->numOfBlocks--; + pCompIdx->hasLast = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last; + pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast; + } + + return 0; +} + static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { pHelper->idxH.numOfIdx = 0; pHelper->idxH.curIdx = 0; @@ -1439,12 +1481,14 @@ static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { } static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SCompBlock compBlock = {0}; + STsdbCfg * pCfg = &(pHelper->pRepo->config); + STable * pTable = pCommitIter->pTable; + SCompIdx * pIdx = &(pHelper->curCompIdx); + TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); + int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; + SCompBlock compBlock = {0}; + SMergeInfo mergeInfo = {0}; + SMergeInfo *pMergeInfo = &mergeInfo; ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey); if (pIdx->hasLast) { // append to with last block @@ -1452,39 +1496,47 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, - pDataCols, NULL, 0, pCfg->update); - ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); - if (rowsRead + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && - pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, rowsRead) < 0) return -1; - } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols, + NULL, 0, pCfg->update, pMergeInfo); - if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1; - ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1; - if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; + if (pDataCols->numOfRows > 0) { + ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); + + if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && + pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { + if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; + if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1; + } else { + if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows); + + if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1; + ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows); + + if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1; + } + + if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; } - - if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; } else { ASSERT(!pHelper->hasOldLastBlock); tdResetDataCols(pDataCols); - int rowsRead = - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update); - ASSERT(rowsRead > 0 && rowsRead == pDataCols->numOfRows); + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); - if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; + if (pDataCols->numOfRows > 0) { + ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols))); + if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; + if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1; + } } #ifndef NDEBUG TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter); - ASSERT(keyNext < 0 || keyNext > pIdx->maxKey); + ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey); #endif return 0; @@ -1492,13 +1544,15 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey, int *blkIdx) { - STsdbCfg * pCfg = &(pHelper->pRepo->config); - STable * pTable = pCommitIter->pTable; - SCompIdx * pIdx = &(pHelper->curCompIdx); - SCompBlock compBlock = {0}; - TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); - int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; - SDataCols *pDataCols0 = pHelper->pDataCols[0]; + STsdbCfg * pCfg = &(pHelper->pRepo->config); + STable * pTable = pCommitIter->pTable; + SCompIdx * pIdx = &(pHelper->curCompIdx); + SCompBlock compBlock = {0}; + TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter); + int defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5; + SDataCols * pDataCols0 = pHelper->pDataCols[0]; + SMergeInfo mergeInfo = {0}; + SMergeInfo *pMergeInfo = &mergeInfo; SSkipListIterator slIter = {0}; @@ -1509,120 +1563,82 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pCompBlock != NULL); int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); - if (pCompBlock->last) { - ASSERT(pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && tblkIdx == pIdx->numOfBlocks - 1); - int16_t colId = 0; - slIter = *(pCommitIter->pIter); - if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; - ASSERT(pDataCols0->numOfRows == pCompBlock->numOfRows); + ASSERT((!TSDB_IS_LAST_BLOCK(pCompBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); - int rows1 = defaultRowsInBlock - pCompBlock->numOfRows; - int rows2 = tsdbLoadDataFromCache(pTable, &slIter, maxKey, rows1, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows, pCfg->update); - if (!pCfg->update && rows2 == 0) { // all data filtered out - *(pCommitIter->pIter) = slIter; - } else { - if (pCompBlock->numOfRows + rows2 < pCfg->minRowsPerFileBlock && - pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, rows1, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); - ASSERT(rowsRead == rows2 && rowsRead <= pDataCols->numOfRows && pDataCols->numOfRows > 0); - if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; - tblkIdx++; - } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - int round = 0; - int dIter = 0; - while (true) { - int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, maxKey, - defaultRowsInBlock, pCfg->update); - if (rowsRead == 0) break; + if ((!TSDB_IS_LAST_BLOCK(pCompBlock)) && keyFirst < pCompBlock->keyFirst) { + // Loop to write data until pCompBlock->keyFirst-1 + while (true) { + tdResetDataCols(pDataCols); + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, pCompBlock->keyLast - 1, defaultRowsInBlock, pDataCols, NULL, 0, + pCfg->update, pMergeInfo); + ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); + if (pDataCols->numOfRows == 0) break; - if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - if (round == 0) { - if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } else { - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } - - tblkIdx++; - round++; - } - } - if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; + if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; + tblkIdx++; } + ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL || + tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { - TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (pCompBlock[1].keyFirst - 1); - TSKEY blkKeyFirst = pCompBlock->keyFirst; - TSKEY blkKeyLast = pCompBlock->keyLast; + int16_t colId = 0; + if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; - if (keyFirst < blkKeyFirst) { - while (true) { - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, blkKeyFirst - 1, defaultRowsInBlock, pDataCols, - NULL, 0, pCfg->update); - if (rowsRead == 0) break; + TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1); - ASSERT(rowsRead == pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - tblkIdx++; + slIter = *(pCommitIter->pIter); + tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows, + pCfg->update, pMergeInfo); + + if (pMergeInfo->nOperations == 0) { + // Do nothing + ASSERT(pMergeInfo->rowsDeleteFailed > 0); + *(pCommitIter->pIter) = slIter; + tblkIdx++; + } else if (pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { + // Delete the block and do some stuff + ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN); + if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1; + *pCommitIter->pIter = slIter; + if (pCompBlock->last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + } else if (tsdbCheckAddSubBlockCond(pHelper, pCompBlock, pMergeInfo, pDataCols->maxPoints)) { + // Append as a sub-block of the searched block + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData, + pDataCols0->numOfRows, pCfg->update, pMergeInfo); + ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0); + if (tsdbWriteBlockToFile(pHelper, pCompBlock->last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, + &compBlock, pCompBlock->last, false) < 0) { + return -1; } - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) { + return -1; + } + tblkIdx++; } else { - ASSERT(keyFirst <= blkKeyLast); - int16_t colId = 0; - if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; + // load the block data, merge with the memory data + if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + int round = 0; + int dIter = 0; + while (true) { + tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock, + pCfg->update); - slIter = *(pCommitIter->pIter); - int rows1 = (pCfg->maxRowsPerFileBlock - pCompBlock->numOfRows); - int rows2 = tsdbLoadDataFromCache(pTable, &slIter, blkKeyLast, INT_MAX, NULL, pDataCols0->cols[0].pData, - pDataCols0->numOfRows, pCfg->update); - - if (!pCfg->update && rows2 == 0) { // all filtered out - *(pCommitIter->pIter) = slIter; - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); - } else { - int rows3 = tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, NULL, 0, pCfg->update) + rows2; - - if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && rows1 >= rows2) { - int rows = (rows1 >= rows3) ? rows3 : rows2; - tdResetDataCols(pDataCols); - int rowsRead = tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, rows, pDataCols, - pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update); - ASSERT(rowsRead == rows && rowsRead <= pDataCols->numOfRows); - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, false) < 0) - return -1; - if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, rowsRead) < 0) return -1; - tblkIdx++; - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + if (pDataCols->numOfRows == 0) break; + if (tblkIdx == pIdx->numOfBlocks - 1) { + if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; } else { - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; - int round = 0; - int dIter = 0; - while (true) { - int rowsRead = tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, - defaultRowsInBlock, pCfg->update); - if (rowsRead == 0) break; - - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - if (round == 0) { - if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } else { - if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; - } - - round++; - tblkIdx++; - } - ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) < 0 || - tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; } + + if (round == 0) { + if (pCompBlock->last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; + } else { + if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; + } + + round++; + tblkIdx++; } } } @@ -1631,9 +1647,8 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, return 0; } -static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, +static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget, TSKEY maxKey, int maxRows, int8_t update) { - int numOfRows = 0; TSKEY key1 = INT64_MAX; TSKEY key2 = INT64_MAX; STSchema *pSchema = NULL; @@ -1643,39 +1658,60 @@ static int tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIte while (true) { key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter); + bool isRowDel = false; SDataRow row = tsdbNextIterRow(pCommitIter->pIter); - key2 = (row == NULL || dataRowKey(row) > maxKey) ? INT64_MAX : dataRowKey(row); + if (row == NULL || dataRowKey(row) > maxKey) { + key2 = INT64_MAX; + } else { + key2 = dataRowKey(row); + isRowDel = dataRowDeleted(row); + } if (key1 == INT64_MAX && key2 == INT64_MAX) break; - if ((key1 < key2) || ((!update) && (key1 == key2))) { + if (key1 < key2) { for (int i = 0; i < pDataCols->numOfCols; i++) { dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, pTarget->maxPoints); } + pTarget->numOfRows++; (*iter)++; + } else if (key1 > key2) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } - if ((!update) && (key1 == key2)) { - tSkipListIterNext(pCommitIter->pIter); - } - } else { - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); - ASSERT(pSchema != NULL); + tdAppendDataRowToDataCol(row, pSchema, pTarget); } - tdAppendDataRowToDataCol(row, pSchema, pTarget); tSkipListIterNext(pCommitIter->pIter); - if (key1 == key2) (*iter)++; + } else { + if (update) { + if (!isRowDel) { + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row)); + ASSERT(pSchema != NULL); + } + + tdAppendDataRowToDataCol(row, pSchema, pTarget); + } + } else { + ASSERT(!isRowDel); + + for (int i = 0; i < pDataCols->numOfCols; i++) { + dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows, + pTarget->maxPoints); + } + + pTarget->numOfRows++; + } + (*iter)++; + tSkipListIterNext(pCommitIter->pIter); } - - numOfRows++; - if (numOfRows >= maxRows) break; - ASSERT(numOfRows == pTarget->numOfRows && numOfRows <= pTarget->maxPoints); } - - return numOfRows; } static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) { @@ -1698,3 +1734,20 @@ static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, return 0; } + +static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) { + STsdbCfg *pCfg = &(pHelper->pRepo->config); + int mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed; + + ASSERT(mergeRows > 0); + + if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) { + if (pCompBlock->last) { + if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true; + } else { + if (mergeRows < pCfg->maxRowsPerFileBlock) return true; + } + } + + return false; +} \ No newline at end of file diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 030a9d69c1..579483dbff 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -39,16 +39,11 @@ typedef char *(*__sl_key_fn_t)(const void *); typedef struct SSkipListNode { uint8_t level; - uint8_t flags; void * pData; struct SSkipListNode *forwards[]; } SSkipListNode; -#define SL_NODE_DELETED_FLAG (uint8_t)0x1 - #define SL_GET_NODE_DATA(n) (n)->pData -#define SL_IS_NODE_DELETED(n) ((n)->flags & SL_NODE_DELETED_FLAG) -#define SL_SET_NODE_DELETED(n) (n)->flags |= SL_NODE_DELETED_FLAG #define SL_NODE_GET_FORWARD_POINTER(n, l) (n)->forwards[(l)] #define SL_NODE_GET_BACKWARD_POINTER(n, l) (n)->forwards[(n)->level + (l)] @@ -109,8 +104,7 @@ typedef struct SSkipList { uint8_t flags; uint8_t type; // static info above uint8_t level; - uint32_t size; // semantic meaning of size - uint32_t tsize; // # of all skiplist nodes in this SL + uint32_t size; SSkipListNode * pHead; // point to the first element SSkipListNode * pTail; // point to the last element #if SKIP_LIST_RECORD_PERFORMANCE @@ -131,13 +125,13 @@ typedef struct SSkipListIterator { #define SL_GET_MIN_KEY(s) SL_GET_NODE_KEY(s, SL_NODE_GET_FORWARD_POINTER((s)->pHead, 0)) #define SL_GET_MAX_KEY(s) SL_GET_NODE_KEY((s), SL_NODE_GET_BACKWARD_POINTER((s)->pTail, 0)) #define SL_SIZE(s) (s)->size -#define SL_TSIZE(s) (s)->tsize -SSkipList * tSkipListCreate(uint8_t nMaxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn); -void tSkipListDestroy(SSkipList *pSkipList); -SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData); -SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); -void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, + __sl_key_fn_t fn); +void tSkipListDestroy(SSkipList *pSkipList); +SSkipListNode * tSkipListPut(SSkipList *pSkipList, void *pData); +SArray * tSkipListGet(SSkipList *pSkipList, SSkipListKey pKey); +void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel); SSkipListIterator *tSkipListCreateIter(SSkipList *pSkipList); SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char *val, int32_t type, int32_t order); bool tSkipListIterNext(SSkipListIterator *iter); @@ -145,8 +139,6 @@ SSkipListNode * tSkipListIterGet(SSkipListIterator *iter); void * tSkipListDestroyIter(SSkipListIterator *iter); uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key); void tSkipListRemoveNode(SSkipList *pSkipList, SSkipListNode *pNode); -SSkipListKey tSkipListGetMinKey(SSkipList *pSkipList); -SSkipListKey tSkipListGetMaxKey(SSkipList *pSkipList); #ifdef __cplusplus } diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 603a9759b8..ea166e4c25 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -34,7 +34,8 @@ static FORCE_INLINE int tSkipListRLock(SSkipList *pSkipList); static FORCE_INLINE int tSkipListUnlock(SSkipList *pSkipList); static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList); -SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, uint8_t flags, __sl_key_fn_t fn) { +SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, __compar_fn_t comparFn, uint8_t flags, + __sl_key_fn_t fn) { SSkipList *pSkipList = (SSkipList *)calloc(1, sizeof(SSkipList)); if (pSkipList == NULL) return NULL; @@ -47,7 +48,11 @@ SSkipList *tSkipListCreate(uint8_t maxLevel, uint8_t keyType, uint16_t keyLen, u pSkipList->len = keyLen; pSkipList->flags = flags; pSkipList->keyFn = fn; - pSkipList->comparFn = getKeyComparFunc(keyType); + if (comparFn == NULL) { + pSkipList->comparFn = getKeyComparFunc(keyType); + } else { + pSkipList->comparFn = comparFn; + } if (initForwardBackwardPtr(pSkipList) < 0) { tSkipListDestroy(pSkipList); @@ -115,10 +120,6 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { if (dupMode == SL_UPDATE_DUP_KEY) { pNode = SL_NODE_GET_FORWARD_POINTER(forward[0], 0); atomic_store_ptr(&(pNode->pData), pData); - if (SL_IS_NODE_DELETED(pNode)) { - pNode->flags &= (~(SL_NODE_DELETED_FLAG)); - pSkipList->size++; - } } } else { pNode = tSkipListNewNode(getSkipListRandLevel(pSkipList)); @@ -137,8 +138,6 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, void *pData) { uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { uint32_t count = 0; - if (SL_DUP_MODE(pSkipList) == SL_DISCARD_DUP_KEY) return 0; - tSkipListWLock(pSkipList); SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); @@ -177,9 +176,7 @@ SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { if (pSkipList->comparFn(key, SL_GET_NODE_KEY(pSkipList, p)) != 0) { break; } - if (!SL_IS_NODE_DELETED(p)) { - taosArrayPush(sa, &p); - } + taosArrayPush(sa, &p); pNode = p; } @@ -227,19 +224,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) { tSkipListRLock(pSkipList); if (iter->order == TSDB_ORDER_ASC) { - while (true) { - iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); - iter->step++; - if (iter->cur == pSkipList->pTail) break; - if (!SL_IS_NODE_DELETED(iter->cur)) break; - } + if (iter->cur == pSkipList->pTail) return false; + iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); + iter->step++; } else { - while (true) { - iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); - iter->step++; - if (iter->cur == pSkipList->pHead) break; - if (!SL_IS_NODE_DELETED(iter->cur)) break; - } + if (iter->cur == pSkipList->pHead) return false; + iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); + iter->step++; } tSkipListUnlock(pSkipList); @@ -305,28 +296,6 @@ void tSkipListPrint(SSkipList *pSkipList, int16_t nlevel) { } } -SSkipListKey tSkipListGetMinKey(SSkipList *pSkipList) { - if (pSkipList == NULL || SL_SIZE(pSkipList) == 0) return NULL; - - SSkipListNode *pNode = pSkipList->pHead; - while ((pNode = SL_NODE_GET_FORWARD_POINTER(pNode, 0)) != pSkipList->pTail) { - if (!SL_IS_NODE_DELETED(pNode)) return pSkipList->keyFn(pNode->pData); - } - - return NULL; -} - -SSkipListKey tSkipListGetMaxKey(SSkipList *pSkipList) { - if (pSkipList == NULL || SL_SIZE(pSkipList) == 0) return NULL; - - SSkipListNode *pNode = pSkipList->pTail; - while ((pNode = SL_NODE_GET_BACKWARD_POINTER(pNode, 0)) != pSkipList->pHead) { - if (!SL_IS_NODE_DELETED(pNode)) return pSkipList->keyFn(pNode->pData); - } - - return NULL; -} - static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSkipListNode *pNode) { for (int32_t i = 0; i < pNode->level; ++i) { if (i >= pSkipList->level) { @@ -349,7 +318,6 @@ static void tSkipListDoInsert(SSkipList *pSkipList, SSkipListNode **forward, SSk if (pSkipList->level < pNode->level) pSkipList->level = pNode->level; pSkipList->size += 1; - pSkipList->tsize += 1; } static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order) { @@ -447,27 +415,18 @@ static bool tSkipListGetPosToPut(SSkipList *pSkipList, SSkipListNode **forward, static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode) { int32_t level = pNode->level; uint8_t dupMode = SL_DUP_MODE(pSkipList); + ASSERT(dupMode != SL_DISCARD_DUP_KEY && dupMode != SL_UPDATE_DUP_KEY); - if (dupMode == SL_UPDATE_DUP_KEY) { - if (SL_IS_NODE_DELETED(pNode)) { - return; - } else { - SL_SET_NODE_DELETED(pNode); - pSkipList->size--; - } - } else { - for (int32_t j = level - 1; j >= 0; --j) { - SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j); - SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j); + for (int32_t j = level - 1; j >= 0; --j) { + SSkipListNode *prev = SL_NODE_GET_BACKWARD_POINTER(pNode, j); + SSkipListNode *next = SL_NODE_GET_FORWARD_POINTER(pNode, j); - SL_NODE_GET_FORWARD_POINTER(prev, j) = next; - SL_NODE_GET_BACKWARD_POINTER(next, j) = prev; - } - - tSkipListFreeNode(pNode); - pSkipList->size--; - pSkipList->tsize--; + SL_NODE_GET_FORWARD_POINTER(prev, j) = next; + SL_NODE_GET_BACKWARD_POINTER(next, j) = prev; } + + tSkipListFreeNode(pNode); + pSkipList->size--; } // Function must be called after calling tSkipListRemoveNodeImpl() function From 47ead850543d2483160a887761a1830450510197 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 27 Sep 2020 10:47:52 +0800 Subject: [PATCH 14/58] TD-1548 --- src/tsdb/src/tsdbRWHelper.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index c6d4ae7a5e..f56f12e48a 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1065,7 +1065,7 @@ static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { int tsize = 0; if (compBlock.numOfSubBlocks > 1) { - tsize = pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks); + tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks)); ASSERT(tsize > 0); memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset), @@ -1075,14 +1075,14 @@ static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) { pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks; } - tsize = pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo); + tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo)); ASSERT(tsize > 0); memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize); pCompIdx->len -= sizeof(SCompBlock); pCompIdx->numOfBlocks--; - pCompIdx->hasLast = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last; + pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last); pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast; } From 1f942936444c16982965e3595be3c0b2be7ffcb0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 27 Sep 2020 13:27:28 +0800 Subject: [PATCH 15/58] fix coredump --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index f56f12e48a..2fff744b5c 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1592,7 +1592,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (pMergeInfo->nOperations == 0) { // Do nothing - ASSERT(pMergeInfo->rowsDeleteFailed > 0); + ASSERT(pMergeInfo->rowsDeleteFailed >= 0); *(pCommitIter->pIter) = slIter; tblkIdx++; } else if (pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { From f879d6b0f601ddc8365479de054a86b338cf0db1 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 28 Oct 2020 18:19:19 +0800 Subject: [PATCH 16/58] merge develop --- src/util/src/tskiplist.c | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index 9f00480555..c8e0febd7a 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -20,7 +20,7 @@ #include "tutil.h" static int initForwardBackwardPtr(SSkipList *pSkipList); -static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order); +static SSkipListNode * getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur); static void tSkipListRemoveNodeImpl(SSkipList *pSkipList, SSkipListNode *pNode); static void tSkipListCorrectLevel(SSkipList *pSkipList); static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t order); @@ -140,7 +140,7 @@ uint32_t tSkipListRemove(SSkipList *pSkipList, SSkipListKey key) { tSkipListWLock(pSkipList); - SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); while (1) { SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); if (p == pSkipList->pTail) { @@ -167,7 +167,7 @@ SArray *tSkipListGet(SSkipList *pSkipList, SSkipListKey key) { tSkipListRLock(pSkipList); - SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC); + SSkipListNode *pNode = getPriorNode(pSkipList, key, TSDB_ORDER_ASC, NULL); while (1) { SSkipListNode *p = SL_NODE_GET_FORWARD_POINTER(pNode, 0); if (p == pSkipList->pTail) { @@ -209,7 +209,7 @@ SSkipListIterator *tSkipListCreateIterFromVal(SSkipList *pSkipList, const char * tSkipListRLock(pSkipList); - iter->cur = getPriorNode(pSkipList, val, order); + iter->cur = getPriorNode(pSkipList, val, order, &(iter->next)); tSkipListUnlock(pSkipList); @@ -226,10 +226,24 @@ bool tSkipListIterNext(SSkipListIterator *iter) { if (iter->order == TSDB_ORDER_ASC) { if (iter->cur == pSkipList->pTail) return false; iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); + + // a new node is inserted into between iter->cur and iter->next, ignore it + if (iter->cur != iter->next && (iter->next != NULL)) { + iter->cur = iter->next; + } + + iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); iter->step++; } else { if (iter->cur == pSkipList->pHead) return false; iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); + + // a new node is inserted into between iter->cur and iter->next, ignore it + if (iter->cur != iter->next && (iter->next != NULL)) { + iter->cur = iter->next; + } + + iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); iter->step++; } @@ -327,8 +341,10 @@ static SSkipListIterator *doCreateSkipListIterator(SSkipList *pSkipList, int32_t iter->order = order; if (order == TSDB_ORDER_ASC) { iter->cur = pSkipList->pHead; + iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); } else { iter->cur = pSkipList->pTail; + iter->next = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); } return iter; @@ -485,9 +501,12 @@ static FORCE_INLINE int32_t getSkipListRandLevel(SSkipList *pSkipList) { // when order is TSDB_ORDER_ASC, return the last node with key less than val // when order is TSDB_ORDER_DESC, return the first node with key large than val -static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order) { +static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_t order, SSkipListNode **pCur) { __compar_fn_t comparFn = pSkipList->comparFn; SSkipListNode *pNode = NULL; + if (pCur != NULL) { + *pCur = NULL; + } if (order == TSDB_ORDER_ASC) { pNode = pSkipList->pHead; @@ -499,6 +518,9 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ pNode = p; p = SL_NODE_GET_FORWARD_POINTER(p, i); } else { + if (pCur != NULL) { + *pCur = p; + } break; } } @@ -513,6 +535,9 @@ static SSkipListNode *getPriorNode(SSkipList *pSkipList, const char *val, int32_ pNode = p; p = SL_NODE_GET_BACKWARD_POINTER(p, i); } else { + if (pCur != NULL) { + *pCur = p; + } break; } } From 57e93db1499ad83d10614f5ad9182d4a8e68ddf8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 29 Oct 2020 15:55:59 +0800 Subject: [PATCH 17/58] fix some bug --- src/common/inc/tdataformat.h | 32 +++++---------- src/common/src/tdataformat.c | 80 ++++++++---------------------------- src/tsdb/src/tsdbMemTable.c | 15 +++++-- src/tsdb/src/tsdbRWHelper.c | 27 ++++++------ src/util/inc/tskiplist.h | 6 +-- 5 files changed, 58 insertions(+), 102 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 8124075e6c..f6683e4843 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -193,7 +193,7 @@ static FORCE_INLINE int tdAppendColVal(SDataRow row, void *value, int8_t type, i if (offset == 0) { ASSERT(type == TSDB_DATA_TYPE_TIMESTAMP); TKEY tvalue = tdGetTKEY(*(TSKEY *)value); - memcpy(POINTER_SHIFT(row, toffset), &tvalue, TYPE_BYTES[type]); + memcpy(POINTER_SHIFT(row, toffset), (void *)(&tvalue), TYPE_BYTES[type]); } else { memcpy(POINTER_SHIFT(row, toffset), value, TYPE_BYTES[type]); } @@ -227,7 +227,6 @@ static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; } void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints); void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints); -void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows); void dataColSetOffset(SDataCol *pCol, int nEle); bool isNEleNull(SDataCol *pCol, int nEle); @@ -235,28 +234,20 @@ void dataColSetNEleNull(SDataCol *pCol, int nEle, int maxPoints); // Get the data pointer from a column-wised data static FORCE_INLINE void *tdGetColDataOfRow(SDataCol *pCol, int row) { - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); - break; - - default: - return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); - break; + if (IS_VAR_DATA_TYPE(pCol->type)) { + return POINTER_SHIFT(pCol->pData, pCol->dataOff[row]); + } else { + return POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * row); } } static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) { ASSERT(rows > 0); - switch (pDataCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); - break; - default: - return TYPE_BYTES[pDataCol->type] * rows; + if (IS_VAR_DATA_TYPE(pDataCol->type)) { + return pDataCol->dataOff[rows - 1] + varDataTLen(tdGetColDataOfRow(pDataCol, rows - 1)); + } else { + return TYPE_BYTES[pDataCol->type] * rows; } } @@ -277,11 +268,11 @@ typedef struct { #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) #define dataColsTKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0) -#define dataColsKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TSDB_DATA_BIGINT_NULL : dataColsKeyAt(pCols, 0) +#define dataColsKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0) #define dataColsTKeyLast(pCols) \ (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) #define dataColsKeyLast(pCols) \ - (((pCols)->numOfRows == 0) ? TSDB_DATA_BIGINT_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) + (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, (pCols)->numOfRows - 1)) SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows); void tdResetDataCols(SDataCols *pCols); @@ -289,7 +280,6 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema); SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData); void tdFreeDataCols(SDataCols *pCols); void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols); -void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop); //!!!! int tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge); // ----------------- K-V data row structure diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 4596adaf14..49d21d9275 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -205,7 +205,7 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE; pDataCol->len = 0; - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pDataCol->type)) { pDataCol->dataOff = (VarDataOffsetT *)(*pBuf); pDataCol->pData = POINTER_SHIFT(*pBuf, sizeof(VarDataOffsetT) * maxPoints); pDataCol->spaceSize = pDataCol->bytes * maxPoints; @@ -218,60 +218,29 @@ void dataColInit(SDataCol *pDataCol, STColumn *pCol, void **pBuf, int maxPoints) } } +// value from timestamp should be TKEY here instead of TSKEY void dataColAppendVal(SDataCol *pCol, void *value, int numOfRows, int maxPoints) { ASSERT(pCol != NULL && value != NULL); - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - // set offset - pCol->dataOff[numOfRows] = pCol->len; - // Copy data - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); - // Update the length - pCol->len += varDataTLen(value); - break; - default: - ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); - pCol->len += pCol->bytes; - break; - } -} - -void dataColPopPoints(SDataCol *pCol, int pointsToPop, int numOfRows) { - int pointsLeft = numOfRows - pointsToPop; - - ASSERT(pointsLeft > 0); - - if (pCol->type == TSDB_DATA_TYPE_BINARY || pCol->type == TSDB_DATA_TYPE_NCHAR) { - ASSERT(pCol->len > 0); - VarDataOffsetT toffset = pCol->dataOff[pointsToPop]; - pCol->len = pCol->len - toffset; - ASSERT(pCol->len > 0); - memmove(pCol->pData, POINTER_SHIFT(pCol->pData, toffset), pCol->len); - dataColSetOffset(pCol, pointsLeft); + if (IS_VAR_DATA_TYPE(pCol->type)) { + // set offset + pCol->dataOff[numOfRows] = pCol->len; + // Copy data + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, varDataTLen(value)); + // Update the length + pCol->len += varDataTLen(value); } else { ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows); - pCol->len = TYPE_BYTES[pCol->type] * pointsLeft; - memmove(pCol->pData, POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * pointsToPop), pCol->len); + memcpy(POINTER_SHIFT(pCol->pData, pCol->len), value, pCol->bytes); + pCol->len += pCol->bytes; } } bool isNEleNull(SDataCol *pCol, int nEle) { - switch (pCol->type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - for (int i = 0; i < nEle; i++) { - if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; - } - return true; - default: - for (int i = 0; i < nEle; i++) { - if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; - } - return true; + for (int i = 0; i < nEle; i++) { + if (!isNull(tdGetColDataOfRow(pCol, i), pCol->type)) return false; } + return true; } void dataColSetNullAt(SDataCol *pCol, int index) { @@ -393,7 +362,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->cols[i].spaceSize = pDataCols->cols[i].spaceSize; pRet->cols[i].pData = (void *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].pData) - (char *)(pDataCols->buf))); - if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { ASSERT(pDataCols->cols[i].dataOff != NULL); pRet->cols[i].dataOff = (int32_t *)((char *)pRet->buf + ((char *)(pDataCols->cols[i].dataOff) - (char *)(pDataCols->buf))); @@ -403,7 +372,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) { pRet->cols[i].len = pDataCols->cols[i].len; if (pDataCols->cols[i].len > 0) { memcpy(pRet->cols[i].pData, pDataCols->cols[i].pData, pDataCols->cols[i].len); - if (pRet->cols[i].type == TSDB_DATA_TYPE_BINARY || pRet->cols[i].type == TSDB_DATA_TYPE_NCHAR) { + if (IS_VAR_DATA_TYPE(pRet->cols[i].type)) { memcpy(pRet->cols[i].dataOff, pDataCols->cols[i].dataOff, sizeof(VarDataOffsetT) * pDataCols->maxPoints); } } @@ -463,21 +432,6 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) pCols->numOfRows++; } -// Pop pointsToPop points from the SDataCols -void tdPopDataColsPoints(SDataCols *pCols, int pointsToPop) { - int pointsLeft = pCols->numOfRows - pointsToPop; - if (pointsLeft <= 0) { - tdResetDataCols(pCols); - return; - } - - for (int iCol = 0; iCol < pCols->numOfCols; iCol++) { - SDataCol *pCol = pCols->cols + iCol; - dataColPopPoints(pCol, pointsToPop, pCols->numOfRows); - } - pCols->numOfRows = pointsLeft; -} - int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); @@ -549,9 +503,9 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i target->maxPoints); } } + target->numOfRows++; } - target->numOfRows++; (*iter2)++; if (key1 == key2) (*iter1)++; } diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 7adf16443e..2bdb95163d 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -105,8 +105,14 @@ int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { if (tSkipListPut(pTableData->pData, pRow) == NULL) { tsdbFreeBytes(pRepo, (void *)pRow, dataRowLen(row)); } else { - // TODO: may need to refact here int64_t deltaSize = SL_SIZE(pTableData->pData) - oldSize; + if (isRowDelete) { + if (TABLE_LASTKEY(pTable) == key) { + // TODO: need to update table last key here (may from file) + } + } else { + if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; + } if ((!isRowDelete) && (TABLE_LASTKEY(pTable) < key)) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; @@ -301,7 +307,7 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) { * 3. rowsIncreased = rowsInserted - rowsDeleteSucceed >= maxRowsToRead * 4. operations in pCols not exceeds its max capacity if pCols is given * - * The function try to move as mush as possible. + * The function tries to procceed AS MUSH AS POSSIBLE. */ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) { @@ -321,12 +327,13 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) { rowKey = INT64_MAX; + isRowDel = false; } else { rowKey = dataRowKey(row); isRowDel = dataRowDeleted(row); } - if (nFilterKeys == 0 || filterIter >= nFilterKeys) { + if (filterIter >= nFilterKeys) { fKey = INT64_MAX; } else { fKey = tdGetKey(filterKeys[filterIter]); @@ -362,6 +369,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) { rowKey = INT64_MAX; + isRowDel = false; } else { rowKey = dataRowKey(row); isRowDel = dataRowDeleted(row); @@ -391,6 +399,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) { rowKey = INT64_MAX; + isRowDel = false; } else { rowKey = dataRowKey(row); isRowDel = dataRowDeleted(row); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 2fff744b5c..eb1193110d 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1507,6 +1507,8 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock && pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) { if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1; + pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst); + pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast); if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1; } else { if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; @@ -1553,6 +1555,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols * pDataCols0 = pHelper->pDataCols[0]; SMergeInfo mergeInfo = {0}; SMergeInfo *pMergeInfo = &mergeInfo; + SCompBlock oBlock = {0}; SSkipListIterator slIter = {0}; @@ -1562,14 +1565,14 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE); ASSERT(pCompBlock != NULL); int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock)); + oBlock = *pCompBlock; - ASSERT((!TSDB_IS_LAST_BLOCK(pCompBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); + ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1)); - if ((!TSDB_IS_LAST_BLOCK(pCompBlock)) && keyFirst < pCompBlock->keyFirst) { - // Loop to write data until pCompBlock->keyFirst-1 + if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { while (true) { tdResetDataCols(pDataCols); - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, pCompBlock->keyLast - 1, defaultRowsInBlock, pDataCols, NULL, 0, + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); if (pDataCols->numOfRows == 0) break; @@ -1582,7 +1585,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast)); } else { int16_t colId = 0; - if (tsdbLoadBlockDataCols(pHelper, pCompBlock, NULL, &colId, 1) < 0) return -1; + if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1; TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1); @@ -1595,19 +1598,19 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pMergeInfo->rowsDeleteFailed >= 0); *(pCommitIter->pIter) = slIter; tblkIdx++; - } else if (pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { + } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) { // Delete the block and do some stuff ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN); if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1; *pCommitIter->pIter = slIter; - if (pCompBlock->last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; - } else if (tsdbCheckAddSubBlockCond(pHelper, pCompBlock, pMergeInfo, pDataCols->maxPoints)) { + if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) { // Append as a sub-block of the searched block tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData, pDataCols0->numOfRows, pCfg->update, pMergeInfo); ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0); - if (tsdbWriteBlockToFile(pHelper, pCompBlock->last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, - &compBlock, pCompBlock->last, false) < 0) { + if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols, + &compBlock, oBlock.last, false) < 0) { return -1; } if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) { @@ -1616,7 +1619,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, tblkIdx++; } else { // load the block data, merge with the memory data - if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1; + if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1; int round = 0; int dIter = 0; while (true) { @@ -1631,7 +1634,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } if (round == 0) { - if (pCompBlock->last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; + if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; } else { if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1; diff --git a/src/util/inc/tskiplist.h b/src/util/inc/tskiplist.h index 1c5b02a502..65b6482520 100644 --- a/src/util/inc/tskiplist.h +++ b/src/util/inc/tskiplist.h @@ -28,9 +28,9 @@ extern "C" { #define SKIP_LIST_RECORD_PERFORMANCE 0 // For key property setting -#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists -#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key -#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert +#define SL_ALLOW_DUP_KEY (uint8_t)0x0 // Allow duplicate key exists (for tag index usage) +#define SL_DISCARD_DUP_KEY (uint8_t)0x1 // Discard duplicate key (for data update=0 case) +#define SL_UPDATE_DUP_KEY (uint8_t)0x2 // Update duplicate key by remove/insert (for data update=1 case) // For thread safety setting #define SL_THREAD_SAFE (uint8_t)0x4 From e4868728968595f5236a4c07b2150d509ed6de31 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 30 Oct 2020 10:43:40 +0800 Subject: [PATCH 18/58] initialize database update option --- src/query/src/qParserImpl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 7e8128f200..6e4b08316d 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -872,5 +872,6 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->quorum = -1; pDBInfo->keep = NULL; + pDBInfo->update = -1; memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } From 3060fafd9f7b254b995a994b04de2a4ccddddee3 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Fri, 30 Oct 2020 04:42:03 +0000 Subject: [PATCH 19/58] Now locking create table operation, resolving TD-1471 --- tests/pytest/crash_gen/crash_gen.py | 109 +++++++++++++++++++++------- tests/pytest/crash_gen/misc.py | 3 + 2 files changed, 84 insertions(+), 28 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen.py b/tests/pytest/crash_gen/crash_gen.py index 8d2b0080bc..9bc701aa8b 100755 --- a/tests/pytest/crash_gen/crash_gen.py +++ b/tests/pytest/crash_gen/crash_gen.py @@ -243,7 +243,7 @@ class WorkerThread: class ThreadCoordinator: - WORKER_THREAD_TIMEOUT = 180 # one minute + WORKER_THREAD_TIMEOUT = 120 # Normal: 120 def __init__(self, pool: ThreadPool, dbManager: DbManager): self._curStep = -1 # first step is 0 @@ -1177,6 +1177,8 @@ class Task(): instead. But a task is always associated with a DB ''' taskSn = 100 + _lock = threading.Lock() + _tableLocks: Dict[str, threading.Lock] = {} @classmethod def allocTaskNum(cls): @@ -1198,6 +1200,8 @@ class Task(): self._execStats = execStats self._db = db # A task is always associated/for a specific DB + + def isSuccess(self): return self._err is None @@ -1351,6 +1355,24 @@ class Task(): def getQueryResult(self, wt: WorkerThread): # execute an SQL on the worker thread return wt.getQueryResult() + def lockTable(self, ftName): # full table name + # print(" <<" + ftName + '_', end="", flush=True) + with Task._lock: + if not ftName in Task._tableLocks: + Task._tableLocks[ftName] = threading.Lock() + + Task._tableLocks[ftName].acquire() + + def unlockTable(self, ftName): + # print('_' + ftName + ">> ", end="", flush=True) + with Task._lock: + if not ftName in self._tableLocks: + raise RuntimeError("Corrupt state, no such lock") + lock = Task._tableLocks[ftName] + if not lock.locked(): + raise RuntimeError("Corrupte state, already unlocked") + lock.release() + class ExecutionStats: def __init__(self): @@ -1461,7 +1483,7 @@ class StateTransitionTask(Task): _baseTableNumber = None - _endState = None + _endState = None # TODO: no longter used? @classmethod def getInfo(cls): # each sub class should supply their own information @@ -1486,7 +1508,7 @@ class StateTransitionTask(Task): @classmethod def getRegTableName(cls, i): - if ( StateTransitionTask._baseTableNumber is None): + if ( StateTransitionTask._baseTableNumber is None): # Set it one time StateTransitionTask._baseTableNumber = Dice.throw( 999) if gConfig.dynamic_db_table_names else 0 return "reg_table_{}".format(StateTransitionTask._baseTableNumber + i) @@ -1583,14 +1605,23 @@ class TdSuperTable: def hasRegTables(self, dbc: DbConn, dbName: str): return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0 - def ensureTable(self, dbc: DbConn, dbName: str, regTableName: str): + def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str): sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName) if dbc.query(sql) >= 1 : # reg table exists already return - sql = "CREATE TABLE {}.{} USING {}.{} tags ({})".format( - dbName, regTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) - ) - dbc.execute(sql) + + # acquire a lock first, so as to be able to *verify*. More details in TD-1471 + fullTableName = dbName + '.' + regTableName + task.lockTable(fullTableName) + Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table + print("(" + fullTableName[-3:] + ")", end="", flush=True) + try: + sql = "CREATE TABLE {} USING {}.{} tags ({})".format( + fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) + ) + dbc.execute(sql) + finally: + task.unlockTable(fullTableName) # no matter what def _getTagStrForSql(self, dbc, dbName: str) : tags = self._getTags(dbc, dbName) @@ -1862,7 +1893,12 @@ class TaskAddData(StateTransitionTask): sTable = db.getFixedSuperTable() regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) - sTable.ensureTable(wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists + + + fullTableName = db.getName() + '.' + regTableName + # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" + sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists + # self._unlockTable(fullTableName) for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table nextInt = db.getNextInt() @@ -1872,27 +1908,29 @@ class TaskAddData(StateTransitionTask): self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) self.fAddLogReady.flush() os.fsync(self.fAddLogReady) - sql = "insert into {}.{} values ('{}', {});".format( # removed: tags ('{}', {}) - db.getName(), - regTableName, - # ds.getFixedSuperTableName(), - # ds.getNextBinary(), ds.getNextFloat(), - nextTick, nextInt) - dbc.execute(sql) - # Successfully wrote the data into the DB, let's record it - # somehow - te.recordDataMark(nextInt) - if gConfig.record_ops: - self.fAddLogDone.write( - "Wrote {} to {}\n".format( - nextInt, regTableName)) - self.fAddLogDone.flush() - os.fsync(self.fAddLogDone) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + if gConfig.verify_data: + self.lockTable(fullTableName) + # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written + + try: + sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {}) + fullTableName, + # ds.getFixedSuperTableName(), + # ds.getNextBinary(), ds.getNextFloat(), + nextTick, nextInt) + dbc.execute(sql) + except: # Any exception at all + if gConfig.verify_data: + self.unlockTable(fullTableName) + raise # Now read it back and verify, we might encounter an error if table is dropped if gConfig.verify_data: # only if command line asks for it try: - readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts= '{}'". + readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". format(db.getName(), regTableName, nextTick)) if readBack != nextInt : raise taos.error.ProgrammingError( @@ -1905,8 +1943,23 @@ class TaskAddData(StateTransitionTask): "Failed to read back same data for tick: {}, wrote: {}, read: {}" .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), errno) - # Re-throw no matter what - raise + elif errno in [0x218, 0x362]: # table doesn't exist + # do nothing + dummy = 0 + else: + # Re-throw otherwise + raise + finally: + self.unlockTable(fullTableName) # Unlock the table no matter what + + # Successfully wrote the data into the DB, let's record it somehow + te.recordDataMark(nextInt) + + if gConfig.record_ops: + self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) + self.fAddLogDone.flush() + os.fsync(self.fAddLogDone) + self.activeTable.discard(i) # not raising an error, unlike remove diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index 34a33c6af6..4dc053dae3 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -166,6 +166,8 @@ class Progress: SERVICE_RECONNECT_START = 4 SERVICE_RECONNECT_SUCCESS = 5 SERVICE_RECONNECT_FAILURE = 6 + CREATE_TABLE_ATTEMPT = 7 + tokens = { STEP_BOUNDARY: '.', BEGIN_THREAD_STEP: '[', @@ -174,6 +176,7 @@ class Progress: SERVICE_RECONNECT_START: '', SERVICE_RECONNECT_FAILURE: '.xr>', + CREATE_TABLE_ATTEMPT: '_c', } @classmethod From 45fcefe80c094471f85cfa15b3ebba5dd0246569 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 30 Oct 2020 13:27:51 +0800 Subject: [PATCH 20/58] refactor part of code --- src/common/inc/tdataformat.h | 4 ++-- src/tsdb/src/tsdbMemTable.c | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index f6683e4843..27052f08db 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -124,8 +124,8 @@ typedef uint64_t TKEY; #define TKEY_INVALID UINT64_MAX #define TKEY_NULL TKEY_INVALID -#define TKEY_NEGATIVE_FLAG (((TKEY)1) << (sizeof(TKEY) * 8 - 1)) -#define TKEY_DELETE_FLAG (((TKEY)1) << (sizeof(TKEY) * 8 - 2)) +#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63) +#define TKEY_DELETE_FLAG (((TKEY)1) << 62) #define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG | TKEY_DELETE_FLAG)) #define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 2bdb95163d..19f59f3080 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -113,7 +113,6 @@ int tsdbUpdateRowInMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { } else { if (TABLE_LASTKEY(pTable) < key) TABLE_LASTKEY(pTable) = key; } - if ((!isRowDelete) && (TABLE_LASTKEY(pTable) < key)) TABLE_LASTKEY(pTable) = key; if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; if (pMemTable->keyLast < key) pMemTable->keyLast = key; From fb76d7378207041c7f7dcd20ef495e898b006ad7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 3 Nov 2020 14:13:28 +0800 Subject: [PATCH 21/58] [TD-1557] add test cases for update --- tests/pytest/update/allow_update.py | 67 +++++++ tests/pytest/update/append_commit_data.py | 33 ++++ tests/pytest/update/append_commit_last.py | 42 +++++ tests/pytest/update/merge_commit_data.py | 206 ++++++++++++++++++++++ tests/pytest/update/merge_commit_data2.py | 153 ++++++++++++++++ tests/pytest/update/merge_commit_last.py | 187 ++++++++++++++++++++ 6 files changed, 688 insertions(+) create mode 100644 tests/pytest/update/allow_update.py create mode 100644 tests/pytest/update/append_commit_data.py create mode 100644 tests/pytest/update/append_commit_last.py create mode 100644 tests/pytest/update/merge_commit_data.py create mode 100644 tests/pytest/update/merge_commit_data2.py create mode 100644 tests/pytest/update/merge_commit_last.py diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py new file mode 100644 index 0000000000..cc6fab5fc5 --- /dev/null +++ b/tests/pytest/update/allow_update.py @@ -0,0 +1,67 @@ +# For step 1, we set all the values as 1 +# for step 2, we set all the values as 2 +# and so on +# check query result here means not + +# clear env set up + +# create database db update 1 + +# create table t (ts timestamp, a int) + +# set an init time (such as $t0 = 1604295582000) + +# Step 1 +insert into t values ($t0, 1); +insert into t values ($t0 - 3, 1); +insert into t values ($t0 + 3, 1); + +check query result + +# Step 2 +insert into t values ($t0, 2); +insert into t values ($t0 - 3, 2); +insert into t values ($t0 + 3, 2); + +check query result + +# Step 3 +insert into t values ($t0 - 4, 3); +insert into t values ($t0 - 2, 3); +insert into t values ($t0 + 2, 3); +insert into t values ($t0 + 4, 3); + +check query result + +# Step 4 +insert into t values ($t0 - 4, 4); +insert into t values ($t0 - 2, 4); +insert into t values ($t0 + 2, 4); +insert into t values ($t0 + 4, 4); + +check query result + +# Step 4 +insert into t values ($t0 - 1, 5); +insert into t values ($t0 + 1, 5); + +check query result + +# Step 4 +insert into t values ($t0 - 4, 6); +insert into t values ($t0 - 3, 6); +insert into t values ($t0 - 2, 6); +insert into t values ($t0 - 1, 6); +insert into t values ($t0, 6); +insert into t values ($t0 + 1, 6); +insert into t values ($t0 + 2, 6); +insert into t values ($t0 + 3, 6); +insert into t values ($t0 + 4, 6); + +check query result + +# Step 4 + +restart to commit + +check query result \ No newline at end of file diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py new file mode 100644 index 0000000000..c907acb932 --- /dev/null +++ b/tests/pytest/update/append_commit_data.py @@ -0,0 +1,33 @@ +# clear env set up + +create database db update 1 +$t0 = 1604298064000 + +## Step 1 +create table t1 (ts timestamp, a int); + +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($k = 1; $k <= 100; $k++) { + for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $k * 200 $i, 1); + } + + restart to commit + check query result +} + +## Step 2 +create table t2 (ts timestamp, a int); + +for ($i = 0; $i < 20000; $i++) { + insert into t2 values ($t0 + $i, 1); +} + +restart to commit +check query result \ No newline at end of file diff --git a/tests/pytest/update/append_commit_last.py b/tests/pytest/update/append_commit_last.py new file mode 100644 index 0000000000..9b31adccec --- /dev/null +++ b/tests/pytest/update/append_commit_last.py @@ -0,0 +1,42 @@ +# clear env set up + +create database db update 1; + +## Step 1 +$loops = 1000 +#t0 = 1604298064000 + +create table t1 (ts timestamp, a int); +for ($i = 0; $i < $loops; $i++) { + insert into t1 values ($t0 + $i, 1); + restart to commit + check query result +} + +## Step 2 +create table t2 (ts timestamp, a int); + +insert into t2 ($t0, 1); +restart to commit +check query result + +for ($i = 1; $i <= 150; $i++) { + insert into t2 values ($t0 + $i, 1); +} +restart to commit +check query result + +## Step 3 +create table t3 (ts timestamp, a int); + +insert into t3 ($t0, 1); +restart to commit +check query result + +for ($i = 0; $i < 8; $i++) { + for ($j = 1; $j <= 10; $j++) { + insert into t3 values ($t0 + $i * 10 + $j , 1); + } +} +restart to commit +check query result \ No newline at end of file diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py new file mode 100644 index 0000000000..e9e2596821 --- /dev/null +++ b/tests/pytest/update/merge_commit_data.py @@ -0,0 +1,206 @@ +# clear env set up + +$t0 = 1603152000000 + +create database db update 1 days 30; + +## Step 1. UPDATE THE WHOLE DATA BLOCK REPEATEDLY +create table t1 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($k = 0; $k < 10; $k++) { + for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 1); + } + + check query result + restart to commit + check query result +} + +## Step 2. PREPEND DATA +create table t2 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t2 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = -100; $i < 0; $i++) { + insert into t2 values ($t0 + $i, 1); +} + +check query result +restart to commit +check query result + +## Step 3. PREPEND MASSIVE DATA +create table t3 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t3 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = -6000; $i < 0; $i++) { + insert into t3 values ($t0 + $i, 1); +} + +check query result +restart to commit +check query result + +## Step 4. APPEND DATA +create table t4 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t4 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 100; $i++) { + insert into t4 values ($t0 + 200 + $i, 1); +} + +check query result +restart to commit +check query result + +## Step 5. APPEND MASSIVE DATA +create table t5 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t5 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 6000; $i++) { + insert into t5 values ($t0 + 200 + $i, 1); +} + +check query result +restart to commit +check query result + +## Step 6. UPDATE BLOCK IN TWO STEP +create table t6 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t6 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 100; $i++) { + insert into t6 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +for ($i = 100; $i < 200; $i++) { + insert into t6 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 7. UPDATE LAST HALF AND INSERT LITTLE DATA +create table t7 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t7 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 100; $i < 300; $i++) { + insert into t7 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 8. UPDATE LAST HALF AND INSERT MASSIVE DATA +create table t8 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t8 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 100; $i < 6000; $i++) { + insert into t8 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 9. UPDATE FIRST HALF AND PREPEND LITTLE DATA +create table t9 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t9 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = -100; $i < 100; $i++) { + insert into t9 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 10. UPDATE FIRST HALF AND PREPEND MASSIVE DATA +create table t10 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t10 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = -6000; $i < 100; $i++) { + insert into t10 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## Step 11. UPDATE FIRST HALF AND APPEND MASSIVE DATA +create table t11 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t11 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 100; $i++) { + insert into t11 values ($t0 + $i, 2); +} + +for ($i = 200; $i < 6000; $i++) { + insert into t11 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result \ No newline at end of file diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py new file mode 100644 index 0000000000..e9dd084eb0 --- /dev/null +++ b/tests/pytest/update/merge_commit_data2.py @@ -0,0 +1,153 @@ +# clear env set up + +$t0 = 1603152000000 + +create database db update 1 days 30; + +## Step 1. UPDATE TWO WHOLE DATA BLOCK REPEATEDLY +create table t1 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 1); +} + +restart to commit +check query result + +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + 5000 + $i, 1); +} + +check query result +restart to commit +check query result + +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + $i, 2); +} +for ($i = 0; $i < 200; $i++) { + insert into t1 values ($t0 + 5000 + $i, 1); +} +check query result +restart to commit +check query result + +## Step 2. INSERT IN MIDDLE LITTLE DATA REPEATEDLY +create table t2 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t2 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t2 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($k = 0; $k < 10; $k++) { + for ($i = 0; $i < 10; $i++) { + insert into t2 values ($t0 + 200 + $k * 10 + $i, 1); + } + check query result + restart to commit + check query result +} + +## Step 3. INSERT IN MIDDLE AND UPDATE TWO BLOCKS +create table t3 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t3 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t3 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = 0; $i < 5200; $i++) { + insert into t3 values ($t0 + $i, 2); +} +check query result +restart to commit +check query result + +## Step 4. INSERT IN MIDDLE AND UPDATE FIRST HALF OF TWO BLOCKS +create table t4 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t4 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t4 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = 0; $i < 100; $i++) { + insert into t4 values ($t0 + $i, 2); +} +for ($i = 200; $i < 5000; $i++) { + insert into t4 values ($t0 + $i, 2); +} +for ($i = 0; $i < 100; $i++) { + insert into t4 values ($t0 + 5000 + $i, 2); +} +check query result +restart to commit +check query result + +## Step 5. INSERT IN MIDDLE AND UPDATE last HALF OF TWO BLOCKS +create table t5 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t5 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t5 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = 100; $i < 200; $i++) { + insert into t5 values ($t0 + $i, 2); +} +for ($i = 200; $i < 5000; $i++) { + insert into t5 values ($t0 + $i, 2); +} +for ($i = 100; $i < 200; $i++) { + insert into t5 values ($t0 + 5000 + $i, 2); +} +check query result +restart to commit +check query result + +## Step 6. INSERT MASSIVE DATA AND UPDATE ALL BLOCKS +create table t6 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t6 values ($t0 + $i, 1); +} +restart to commit +for ($i = 0; $i < 200; $i++) { + insert into t6 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = -1000; $i < 10000; $i++) { + insert into t6 values ($t0 + $i, 2); +} +check query result +restart to commit +check query result + +## Step 7. SPLIT DATA BLOCK +create table t7 (ts timestamp, a int); +for ($i = 0; $i < 200; $i++) { + insert into t7 values ($t0 + $i, 1); +} +for ($i = 0; $i < 200; $i++) { + insert into t7 values ($t0 + 5000 + $i, 1); +} +restart to commit + +for ($i = -1000; $i < 10000; $i++) { + insert into t7 values ($t0 + $i, 2); +} +check query result +restart to commit +check query result \ No newline at end of file diff --git a/tests/pytest/update/merge_commit_last.py b/tests/pytest/update/merge_commit_last.py new file mode 100644 index 0000000000..9d832d8984 --- /dev/null +++ b/tests/pytest/update/merge_commit_last.py @@ -0,0 +1,187 @@ +# clear env set up + +$t0 = 1603152000000 + +create database db update 1 days 30; + +## STEP 1: UPDATE THE LAST RECORD REPEATEDLY +create table t1 (ts timestamp, a int); + +for ($i = 0; $i < 100; $i++) { + insert into t1 values ($t0, $i); + + restart to commit + check query result +} + +## STEP 2: UPDATE THE WHOLE LAST BLOCK +create table t2 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t2 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 0; $i < 50; $i++) { + insert into t2 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 3: UPDATE PART OF THE LAST BLOCK +create table t3 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t3 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 0; $i < 25; $i++) { + insert into t3 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +for ($i = 25; $i < 50; $i++) { + insert into t3 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 4: UPDATE AND INSERT APPEND AT END OF DATA +create table t4 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t4 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 0; $i < 25; $i++) { + insert into t4 values ($t0 + $i, 2); +} + +for ($i = 50; $i < 60; $i++) { + insert into t4 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 5: UPDATE AND INSERT PREPEND SOME DATA +create table t5 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t5 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = -10; $i < 0; $i++) { + insert into t4 values ($t0 + $i, 2); +} + +for ($i = 0; $i < 25; $i++) { + insert into t5 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +for ($i = -10; $i < 0; $i++) { + insert into t4 values ($t0 + $i, 3); +} + +for ($i = 25; $i < 50; $i++) { + insert into t5 values ($t0 + $i, 3); +} + +check query result +restart to commit +check query result + +## STEP 6: INSERT AHEAD A LOT OF DATA +create table t6 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t6 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = -1000; $i < 0; $i++) { + insert into t6 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 7: INSERT AHEAD A LOT AND UPDATE +create table t7 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t7 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = -1000; $i < 25; $i++) { + insert into t7 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 8: INSERT AHEAD A LOT AND UPDATE +create table t8 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t8 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 25; $i < 6000; $i++) { + insert into t8 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 9: UPDATE ONLY MIDDLE +create table t9 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t9 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = 20; $i < 30; $i++) { + insert into t9 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result + +## STEP 10: A LOT OF DATA COVER THE WHOLE BLOCK +create table t10 (ts timestamp, a int); +for ($i = 0; $i < 50; $i++) { + insert into t10 values ($t0 + $i, 1); +} +restart to commit +check query result + +for ($i = -4000; $i < 4000; $i++) { + insert into t10 values ($t0 + $i, 2); +} + +check query result +restart to commit +check query result \ No newline at end of file From 19ae9714bc46d01312a3a2030d3fa7af65d5ad41 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 14:48:20 +0800 Subject: [PATCH 22/58] add for update test --- tests/pytest/update/append_commit_data.py | 105 +++++++++++++++++----- 1 file changed, 81 insertions(+), 24 deletions(-) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index c907acb932..e0cda0cd90 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -1,33 +1,90 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -create database db update 1 -$t0 = 1604298064000 +# -*- coding: utf-8 -*- -## Step 1 -create table t1 (ts timestamp, a int); +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 1); -} -restart to commit -check query result +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) -for ($k = 1; $k <= 100; $k++) { - for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $k * 200 $i, 1); - } + def run(self): + print("==========step1") + print("create table && insert data") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db update 1' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') - restart to commit - check query result -} + insertRows = 200 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0+i)) + print("==========step2") + print("restart to commit ") + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows) + for k in range(0,100): + tdLog.info("insert %d rows" % (insertRows)) + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into db.t1 values(%d,1)' % + (t0+k*200+i) + ) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows+200*k) + print("==========step2") + print("insert into another table ") + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 20000 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d, 1)' % + (t0+i)) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) -## Step 2 -create table t2 (ts timestamp, a int); + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) -for ($i = 0; $i < 20000; $i++) { - insert into t2 values ($t0 + $i, 1); -} -restart to commit -check query result \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From fb7b1a7fd9c0e94377b34d7acb66c877de204f0e Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 14:54:52 +0800 Subject: [PATCH 23/58] test for update --- tests/pytest/update/append_commit_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index e0cda0cd90..9c23609414 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -63,7 +63,7 @@ class TDTestCase: tdDnodes.start(1) time.sleep(5) tdSql.query("select * from db.t1") - tdSql.checkRows(insertRows+200*k) + tdSql.checkRows(insertRows+200*(k+1)) print("==========step2") print("insert into another table ") s = 'use db' From 3d51199f5d77a53089cd11648f31e2ee1caf3bb6 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 3 Nov 2020 15:25:59 +0800 Subject: [PATCH 24/58] [add cases for update feature] --- tests/pytest/update/allow_update-0.py | 170 ++++++++++++++++++ tests/pytest/update/allow_update.py | 237 ++++++++++++++++++-------- 2 files changed, 340 insertions(+), 67 deletions(-) create mode 100644 tests/pytest/update/allow_update-0.py diff --git a/tests/pytest/update/allow_update-0.py b/tests/pytest/update/allow_update-0.py new file mode 100644 index 0000000000..61295ec4b7 --- /dev/null +++ b/tests/pytest/update/allow_update-0.py @@ -0,0 +1,170 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numOfRecords = 10 + self.ts = 1604295582000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.execute("use udb") + + def run(self): + tdSql.prepare() + startTs = self.ts + + print("==============step1") + tdSql.execute("create database udb update 0") + tdSql.execute("use udb") + tdSql.execute("create table t (ts timestamp, a int)") + tdSql.execute("insert into t values (%d, 1)" % (startTs)) + tdSql.execute("insert into t values (%d, 1)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 1)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step2") + tdSql.execute("insert into t values (%d, 2)" % (startTs)) + tdSql.execute("insert into t values (%d, 2)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 2)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step3") + tdSql.execute("insert into t values (%d, 3)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 3)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 3) + + print("==============step4") + tdSql.execute("insert into t values (%d, 4)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 4)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 1) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 1) + tdSql.checkData(6, 0, 3) + + print("==============step5") + tdSql.execute("insert into t values (%d, 5)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 5)" % (startTs + 1)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + print("==============step6") + tdSql.execute("insert into t values (%d, 6)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 1) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 3) + tdSql.checkData(7, 0, 1) + tdSql.checkData(8, 0, 3) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py index cc6fab5fc5..2879fc6f79 100644 --- a/tests/pytest/update/allow_update.py +++ b/tests/pytest/update/allow_update.py @@ -1,67 +1,170 @@ -# For step 1, we set all the values as 1 -# for step 2, we set all the values as 2 -# and so on -# check query result here means not - -# clear env set up - -# create database db update 1 - -# create table t (ts timestamp, a int) - -# set an init time (such as $t0 = 1604295582000) - -# Step 1 -insert into t values ($t0, 1); -insert into t values ($t0 - 3, 1); -insert into t values ($t0 + 3, 1); - -check query result - -# Step 2 -insert into t values ($t0, 2); -insert into t values ($t0 - 3, 2); -insert into t values ($t0 + 3, 2); - -check query result - -# Step 3 -insert into t values ($t0 - 4, 3); -insert into t values ($t0 - 2, 3); -insert into t values ($t0 + 2, 3); -insert into t values ($t0 + 4, 3); - -check query result - -# Step 4 -insert into t values ($t0 - 4, 4); -insert into t values ($t0 - 2, 4); -insert into t values ($t0 + 2, 4); -insert into t values ($t0 + 4, 4); - -check query result - -# Step 4 -insert into t values ($t0 - 1, 5); -insert into t values ($t0 + 1, 5); - -check query result - -# Step 4 -insert into t values ($t0 - 4, 6); -insert into t values ($t0 - 3, 6); -insert into t values ($t0 - 2, 6); -insert into t values ($t0 - 1, 6); -insert into t values ($t0, 6); -insert into t values ($t0 + 1, 6); -insert into t values ($t0 + 2, 6); -insert into t values ($t0 + 3, 6); -insert into t values ($t0 + 4, 6); - -check query result - -# Step 4 - -restart to commit - -check query result \ No newline at end of file +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.numOfRecords = 10 + self.ts = 1604295582000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.execute("use udb") + + def run(self): + tdSql.prepare() + startTs = self.ts + + print("==============step1") + tdSql.execute("create database udb update 1") + tdSql.execute("use udb") + tdSql.execute("create table t (ts timestamp, a int)") + tdSql.execute("insert into t values (%d, 1)" % (startTs)) + tdSql.execute("insert into t values (%d, 1)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 1)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + + print("==============step2") + tdSql.execute("insert into t values (%d, 2)" % (startTs)) + tdSql.execute("insert into t values (%d, 2)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 2)" % (startTs + 3)) + + tdSql.query("select * from t") + tdSql.checkRows(3) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 2) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 2) + + print("==============step3") + tdSql.execute("insert into t values (%d, 3)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 3)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 3)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 3) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 3) + tdSql.checkData(3, 0, 2) + tdSql.checkData(4, 0, 3) + tdSql.checkData(5, 0, 2) + tdSql.checkData(6, 0, 3) + + print("==============step4") + tdSql.execute("insert into t values (%d, 4)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 4)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 4)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(7) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 4) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 4) + tdSql.checkData(3, 0, 2) + tdSql.checkData(4, 0, 4) + tdSql.checkData(5, 0, 2) + tdSql.checkData(6, 0, 4) + + print("==============step5") + tdSql.execute("insert into t values (%d, 5)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 5)" % (startTs + 1)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 4) + tdSql.checkData(1, 0, 2) + tdSql.checkData(2, 0, 4) + tdSql.checkData(3, 0, 5) + tdSql.checkData(4, 0, 2) + tdSql.checkData(5, 0, 5) + tdSql.checkData(6, 0, 4) + tdSql.checkData(7, 0, 2) + tdSql.checkData(8, 0, 4) + + print("==============step6") + tdSql.execute("insert into t values (%d, 6)" % (startTs - 4)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs - 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 1)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 2)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 3)) + tdSql.execute("insert into t values (%d, 6)" % (startTs + 4)) + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 6) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 6) + tdSql.checkData(3, 0, 6) + tdSql.checkData(4, 0, 6) + tdSql.checkData(5, 0, 6) + tdSql.checkData(6, 0, 6) + tdSql.checkData(7, 0, 6) + tdSql.checkData(8, 0, 6) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from t") + tdSql.checkRows(9) + + tdSql.query("select a from t") + tdSql.checkData(0, 0, 6) + tdSql.checkData(1, 0, 6) + tdSql.checkData(2, 0, 6) + tdSql.checkData(3, 0, 6) + tdSql.checkData(4, 0, 6) + tdSql.checkData(5, 0, 6) + tdSql.checkData(6, 0, 6) + tdSql.checkData(7, 0, 6) + tdSql.checkData(8, 0, 6) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From c208bf23caf0939aa9c7aa9079bb3fcc6789f9b2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 Nov 2020 15:38:35 +0800 Subject: [PATCH 25/58] fix TD-1900 --- src/common/inc/tdataformat.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 27052f08db..43968a6108 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -267,8 +267,8 @@ typedef struct { #define keyCol(pCols) (&((pCols)->cols[0])) // Key column #define dataColsTKeyAt(pCols, idx) ((TKEY *)(keyCol(pCols)->pData))[(idx)] #define dataColsKeyAt(pCols, idx) tdGetKey(dataColsTKeyAt(pCols, idx)) -#define dataColsTKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0) -#define dataColsKeyFirst(pCols) ((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0) +#define dataColsTKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, 0)) +#define dataColsKeyFirst(pCols) (((pCols)->numOfRows == 0) ? TSDB_DATA_TIMESTAMP_NULL : dataColsKeyAt(pCols, 0)) #define dataColsTKeyLast(pCols) \ (((pCols)->numOfRows == 0) ? TKEY_INVALID : dataColsTKeyAt(pCols, (pCols)->numOfRows - 1)) #define dataColsKeyLast(pCols) \ From d76703f84d45c9c03a8a9591369669eee95a23fe Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 15:46:47 +0800 Subject: [PATCH 26/58] add test case for update --- tests/pytest/update/append_commit_data.py | 2 +- tests/pytest/update/merge_commit_data.py | 585 ++++++++++++++-------- 2 files changed, 389 insertions(+), 198 deletions(-) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index 9c23609414..e0cda0cd90 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -63,7 +63,7 @@ class TDTestCase: tdDnodes.start(1) time.sleep(5) tdSql.query("select * from db.t1") - tdSql.checkRows(insertRows+200*(k+1)) + tdSql.checkRows(insertRows+200*k) print("==========step2") print("insert into another table ") s = 'use db' diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index e9e2596821..3cc3ba828b 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -1,206 +1,397 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -$t0 = 1603152000000 +# -*- coding: utf-8 -*- -create database db update 1 days 30; +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * -## Step 1. UPDATE THE WHOLE DATA BLOCK REPEATEDLY -create table t1 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 1); -} -restart to commit -check query result +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) -for ($k = 0; $k < 10; $k++) { - for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 1); - } + def run(self): + print("==========step1") + print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db update 1 days 30' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') - check query result - restart to commit - check query result -} + insertRows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0 + i)) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) -## Step 2. PREPEND DATA -create table t2 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t2 values ($t0 + $i, 1); -} + for k in range(0,10): + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into t1 values(%d,1)' % + (t0+i) + ) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + print("==========step2") + print("PREPEND DATA ") + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + for i in range(-100,0): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + print("==========step3") + print("PREPEND MASSIVE DATA ") + ret = tdSql.execute('create table t3 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + for i in range(-6000,0): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + print("==========step4") + print("APPEND DATA") + ret = tdSql.execute('create table t4 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + print("==========step5") + print("APPEND DATA") + ret = tdSql.execute('create table t5 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + for i in range(0,6000): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + print("==========step6") + print("UPDATE BLOCK IN TWO STEP") + ret = tdSql.execute('create table t6 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t6 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows+100) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'300') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows+100) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'300') + for i in range(0,200): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows+200) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows+200) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'400') + print("==========step7") + print("UPDATE LAST HALF AND INSERT LITTLE DATA") + ret = tdSql.execute('create table t7 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t7 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + for i in range(100,300): + ret = tdSql.execute( + 'insert into t7 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows+200) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'500') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows+100) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'500') + print("==========step8") + print("UPDATE LAST HALF AND INSERT MASSIVE DATA") + ret = tdSql.execute('create table t8 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t8 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + for i in range(6000): + ret = tdSql.execute( + 'insert into t8 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'12000') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'12000') + print("==========step9") + print("UPDATE FIRST HALF AND PREPEND LITTLE DATA") + ret = tdSql.execute('create table t9 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t9 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + for i in range(-100,100): + ret = tdSql.execute( + 'insert into t9 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'500') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'500') + print("==========step10") + print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA") + ret = tdSql.execute('create table t10 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t10 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + for i in range(-6000,100): + ret = tdSql.execute( + 'insert into t10 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12300') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12300') + print("==========step11") + print("UPDATE FIRST HALF AND APPEND MASSIVE DATA") + ret = tdSql.execute('create table t11 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t11 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + for i in range(100): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + for i in range(200,6000): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11500') + tdDnodes.stop(1) + time.sleep(5) + tdDnodes.start(1) + time.sleep(5) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11500') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) -restart to commit -check query result -for ($i = -100; $i < 0; $i++) { - insert into t2 values ($t0 + $i, 1); -} - -check query result -restart to commit -check query result - -## Step 3. PREPEND MASSIVE DATA -create table t3 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t3 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = -6000; $i < 0; $i++) { - insert into t3 values ($t0 + $i, 1); -} - -check query result -restart to commit -check query result - -## Step 4. APPEND DATA -create table t4 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t4 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 0; $i < 100; $i++) { - insert into t4 values ($t0 + 200 + $i, 1); -} - -check query result -restart to commit -check query result - -## Step 5. APPEND MASSIVE DATA -create table t5 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t5 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 0; $i < 6000; $i++) { - insert into t5 values ($t0 + 200 + $i, 1); -} - -check query result -restart to commit -check query result - -## Step 6. UPDATE BLOCK IN TWO STEP -create table t6 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t6 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 0; $i < 100; $i++) { - insert into t6 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -for ($i = 100; $i < 200; $i++) { - insert into t6 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 7. UPDATE LAST HALF AND INSERT LITTLE DATA -create table t7 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t7 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 100; $i < 300; $i++) { - insert into t7 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 8. UPDATE LAST HALF AND INSERT MASSIVE DATA -create table t8 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t8 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 100; $i < 6000; $i++) { - insert into t8 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 9. UPDATE FIRST HALF AND PREPEND LITTLE DATA -create table t9 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t9 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = -100; $i < 100; $i++) { - insert into t9 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 10. UPDATE FIRST HALF AND PREPEND MASSIVE DATA -create table t10 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t10 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = -6000; $i < 100; $i++) { - insert into t10 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result - -## Step 11. UPDATE FIRST HALF AND APPEND MASSIVE DATA -create table t11 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t11 values ($t0 + $i, 1); -} - -restart to commit -check query result - -for ($i = 0; $i < 100; $i++) { - insert into t11 values ($t0 + $i, 2); -} - -for ($i = 200; $i < 6000; $i++) { - insert into t11 values ($t0 + $i, 2); -} - -check query result -restart to commit -check query result \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 6e4702f0168841da8ecded9bc3f3a5dd9a11c62c Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 16:08:27 +0800 Subject: [PATCH 27/58] add test case for update --- tests/pytest/update/append_commit_data.py | 6 --- tests/pytest/update/merge_commit_data.py | 45 ----------------------- 2 files changed, 51 deletions(-) diff --git a/tests/pytest/update/append_commit_data.py b/tests/pytest/update/append_commit_data.py index e0cda0cd90..3169b748e0 100644 --- a/tests/pytest/update/append_commit_data.py +++ b/tests/pytest/update/append_commit_data.py @@ -46,9 +46,7 @@ class TDTestCase: print("==========step2") print("restart to commit ") tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from db.t1") tdSql.checkRows(insertRows) for k in range(0,100): @@ -59,9 +57,7 @@ class TDTestCase: (t0+k*200+i) ) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from db.t1") tdSql.checkRows(insertRows+200*k) print("==========step2") @@ -75,9 +71,7 @@ class TDTestCase: 'insert into t2 values (%d, 1)' % (t0+i)) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t2") tdSql.checkRows(insertRows) diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index 3cc3ba828b..d2b425b955 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -44,9 +44,7 @@ class TDTestCase: 'insert into t1 values (%d , 1)' % (t0 + i)) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t1") tdSql.checkRows(insertRows) @@ -75,9 +73,6 @@ class TDTestCase: tdSql.query("select * from t2") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) - tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t2") tdSql.checkRows(insertRows) for i in range(-100,0): @@ -87,9 +82,7 @@ class TDTestCase: tdSql.query("select * from t2") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t2") tdSql.checkRows(insertRows+100) print("==========step3") @@ -103,9 +96,7 @@ class TDTestCase: tdSql.query("select * from t3") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t3") tdSql.checkRows(insertRows) for i in range(-6000,0): @@ -115,9 +106,7 @@ class TDTestCase: tdSql.query("select * from t3") tdSql.checkRows(insertRows+6000) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t3") tdSql.checkRows(insertRows+6000) print("==========step4") @@ -131,9 +120,7 @@ class TDTestCase: tdSql.query("select * from t4") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t4") tdSql.checkRows(insertRows) for i in range(0,100): @@ -143,9 +130,7 @@ class TDTestCase: tdSql.query("select * from t4") tdSql.checkRows(insertRows+100) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t4") tdSql.checkRows(insertRows+100) print("==========step5") @@ -159,9 +144,7 @@ class TDTestCase: tdSql.query("select * from t5") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t5") tdSql.checkRows(insertRows) for i in range(0,6000): @@ -171,9 +154,7 @@ class TDTestCase: tdSql.query("select * from t5") tdSql.checkRows(insertRows+6000) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t5") tdSql.checkRows(insertRows+6000) print("==========step6") @@ -187,9 +168,7 @@ class TDTestCase: tdSql.query("select * from t6") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t6") tdSql.checkRows(insertRows) for i in range(0,100): @@ -201,9 +180,7 @@ class TDTestCase: tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'300') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t6") tdSql.checkRows(insertRows+100) tdSql.query("select sum(a) from t6") @@ -217,9 +194,7 @@ class TDTestCase: tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'400') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t6") tdSql.checkRows(insertRows+200) tdSql.query("select sum(a) from t6") @@ -235,9 +210,7 @@ class TDTestCase: tdSql.query("select * from t7") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t7") tdSql.checkRows(insertRows) for i in range(100,300): @@ -249,9 +222,7 @@ class TDTestCase: tdSql.query("select sum(a) from t7") tdSql.checkData(0,0,'500') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t7") tdSql.checkRows(insertRows+100) tdSql.query("select sum(a) from t7") @@ -267,9 +238,7 @@ class TDTestCase: tdSql.query("select * from t8") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t8") tdSql.checkRows(insertRows) for i in range(6000): @@ -281,9 +250,7 @@ class TDTestCase: tdSql.query("select sum(a) from t8") tdSql.checkData(0,0,'12000') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t8") tdSql.checkRows(6000) tdSql.query("select sum(a) from t8") @@ -299,9 +266,7 @@ class TDTestCase: tdSql.query("select * from t9") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t9") tdSql.checkRows(insertRows) for i in range(-100,100): @@ -313,9 +278,7 @@ class TDTestCase: tdSql.query("select sum(a) from t9") tdSql.checkData(0,0,'500') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t9") tdSql.checkRows(300) tdSql.query("select sum(a) from t9") @@ -331,9 +294,7 @@ class TDTestCase: tdSql.query("select * from t10") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t10") tdSql.checkRows(insertRows) for i in range(-6000,100): @@ -345,9 +306,7 @@ class TDTestCase: tdSql.query("select sum(a) from t10") tdSql.checkData(0,0,'12300') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t10") tdSql.checkRows(6200) tdSql.query("select sum(a) from t10") @@ -363,9 +322,7 @@ class TDTestCase: tdSql.query("select * from t11") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t11") tdSql.checkRows(insertRows) for i in range(100): @@ -381,9 +338,7 @@ class TDTestCase: tdSql.query("select sum(a) from t11") tdSql.checkData(0,0,'11500') tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t11") tdSql.checkRows(6000) tdSql.query("select sum(a) from t11") From e84adb0c53ff57806b2064adb106985ce2a60baf Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 16:13:55 +0800 Subject: [PATCH 28/58] add test case for update --- tests/pytest/update/merge_commit_data.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index d2b425b955..9b3fea1865 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -57,9 +57,7 @@ class TDTestCase: tdSql.query("select * from t1") tdSql.checkRows(insertRows) tdDnodes.stop(1) - time.sleep(5) tdDnodes.start(1) - time.sleep(5) tdSql.query("select * from t1") tdSql.checkRows(insertRows) print("==========step2") @@ -73,6 +71,7 @@ class TDTestCase: tdSql.query("select * from t2") tdSql.checkRows(insertRows) tdDnodes.stop(1) + tdDnodes.start(1) tdSql.query("select * from t2") tdSql.checkRows(insertRows) for i in range(-100,0): From 97ba24b9365e515d70a3dcc2770853c2a93c61d1 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 3 Nov 2020 16:34:56 +0800 Subject: [PATCH 29/58] [add test case] --- tests/pytest/update/allow_update.py | 71 ++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 2 deletions(-) diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py index 2879fc6f79..cb77ea6a31 100644 --- a/tests/pytest/update/allow_update.py +++ b/tests/pytest/update/allow_update.py @@ -36,10 +36,11 @@ class TDTestCase: tdSql.prepare() startTs = self.ts - print("==============step1") tdSql.execute("create database udb update 1") tdSql.execute("use udb") tdSql.execute("create table t (ts timestamp, a int)") + + print("==============step1") tdSql.execute("insert into t values (%d, 1)" % (startTs)) tdSql.execute("insert into t values (%d, 1)" % (startTs - 3)) tdSql.execute("insert into t values (%d, 1)" % (startTs + 3)) @@ -159,7 +160,73 @@ class TDTestCase: tdSql.checkData(5, 0, 6) tdSql.checkData(6, 0, 6) tdSql.checkData(7, 0, 6) - tdSql.checkData(8, 0, 6) + tdSql.checkData(8, 0, 6) + + tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)") + + print("==============step1") + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs)) + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3)) + tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3)) + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, 1) + tdSql.checkData(1, 0, 1) + tdSql.checkData(2, 0, 1) + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(2, 1, None) + tdSql.checkData(0, 2, 'c-3') + tdSql.checkData(1, 2, 'c+0') + tdSql.checkData(2, 2, 'c+3') + tdSql.checkData(0, 3, None) + tdSql.checkData(1, 3, None) + tdSql.checkData(2, 3, None) + + print("==============step2") + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs)) + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3)) + tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3)) + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(0, 1, 2.0) + tdSql.checkData(1, 1, 2.0) + tdSql.checkData(2, 1, 2.0) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(0, 3, 1) + tdSql.checkData(1, 3, 1) + tdSql.checkData(2, 3, 0) + + # restart taosd to commit, and check + self.restartTaosd(); + + tdSql.query("select * from subt") + tdSql.checkRows(3) + + tdSql.query("select a,b,c,d from subt") + tdSql.checkData(0, 0, None) + tdSql.checkData(1, 0, None) + tdSql.checkData(2, 0, None) + tdSql.checkData(0, 1, 2.0) + tdSql.checkData(1, 1, 2.0) + tdSql.checkData(2, 1, 2.0) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(2, 2, None) + tdSql.checkData(0, 3, 1) + tdSql.checkData(1, 3, 1) + tdSql.checkData(2, 3, 0) def stop(self): tdSql.close() From 8640626418aeaba7603d6bb727c8220b0e2268ee Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 3 Nov 2020 16:35:12 +0800 Subject: [PATCH 30/58] add test case for update --- tests/pytest/update/merge_commit_data.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/pytest/update/merge_commit_data.py b/tests/pytest/update/merge_commit_data.py index 9b3fea1865..4fb6765361 100644 --- a/tests/pytest/update/merge_commit_data.py +++ b/tests/pytest/update/merge_commit_data.py @@ -79,7 +79,7 @@ class TDTestCase: 'insert into t2 values (%d , 1)' % (t0+i)) tdSql.query("select * from t2") - tdSql.checkRows(insertRows) + tdSql.checkRows(insertRows+100) tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t2") @@ -175,13 +175,13 @@ class TDTestCase: 'insert into t6 values (%d , 2)' % (t0+i)) tdSql.query("select * from t6") - tdSql.checkRows(insertRows+100) + tdSql.checkRows(insertRows) tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'300') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t6") - tdSql.checkRows(insertRows+100) + tdSql.checkRows(insertRows) tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'300') for i in range(0,200): @@ -189,13 +189,13 @@ class TDTestCase: 'insert into t6 values (%d , 2)' % (t0+i)) tdSql.query("select * from t6") - tdSql.checkRows(insertRows+200) + tdSql.checkRows(insertRows) tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'400') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t6") - tdSql.checkRows(insertRows+200) + tdSql.checkRows(insertRows) tdSql.query("select sum(a) from t6") tdSql.checkData(0,0,'400') print("==========step7") @@ -217,13 +217,13 @@ class TDTestCase: 'insert into t7 values (%d , 2)' % (t0+i)) tdSql.query("select * from t7") - tdSql.checkRows(insertRows+200) + tdSql.checkRows(300) tdSql.query("select sum(a) from t7") tdSql.checkData(0,0,'500') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t7") - tdSql.checkRows(insertRows+100) + tdSql.checkRows(300) tdSql.query("select sum(a) from t7") tdSql.checkData(0,0,'500') print("==========step8") @@ -335,13 +335,13 @@ class TDTestCase: tdSql.query("select * from t11") tdSql.checkRows(6000) tdSql.query("select sum(a) from t11") - tdSql.checkData(0,0,'11500') + tdSql.checkData(0,0,'11900') tdDnodes.stop(1) tdDnodes.start(1) tdSql.query("select * from t11") tdSql.checkRows(6000) tdSql.query("select sum(a) from t11") - tdSql.checkData(0,0,'11500') + tdSql.checkData(0,0,'11900') def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 9d18f77db53cfcf180b274f32537b76bc8ee9566 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Tue, 3 Nov 2020 17:31:39 +0800 Subject: [PATCH 31/58] [NONE] --- tests/pytest/update/allow_update.py | 33 +++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/tests/pytest/update/allow_update.py b/tests/pytest/update/allow_update.py index cb77ea6a31..5871197f02 100644 --- a/tests/pytest/update/allow_update.py +++ b/tests/pytest/update/allow_update.py @@ -164,7 +164,7 @@ class TDTestCase: tdSql.execute("create table subt (ts timestamp, a int, b float, c binary(16), d bool)") - print("==============step1") + print("==============step7") tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+0')" % (startTs)) tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c-3')" % (startTs - 3)) tdSql.execute("insert into subt (ts, a, c) values (%d, 1, 'c+3')" % (startTs + 3)) @@ -186,7 +186,7 @@ class TDTestCase: tdSql.checkData(1, 3, None) tdSql.checkData(2, 3, None) - print("==============step2") + print("==============step8") tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs)) tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, true)" % (startTs - 3)) tdSql.execute("insert into subt (ts, b, d) values (%d, 2.0, false)" % (startTs + 3)) @@ -227,6 +227,35 @@ class TDTestCase: tdSql.checkData(0, 3, 1) tdSql.checkData(1, 3, 1) tdSql.checkData(2, 3, 0) + + + + tdSql.execute("create table ct (ts timestamp, a int, b float, c binary(128))") + + print("==============step9") + insertRows = 20000 + for i in range(0, insertRows): + tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i, i)) + + tdSql.query("select * from ct") + tdSql.checkRows(insertRows) + + for i in range(0, insertRows): + tdSql.execute("insert into ct values (%d , %d, %d, 'aabbccddeeffgghhiijjkkllmmoonn112233445566778899xxyyzz')" % (startTs + i, i+insertRows, i+insertRows)) + + tdSql.query("select * from ct") + tdSql.checkRows(insertRows) + + tdSql.query("select a,b from ct limit 3") + tdSql.checkData(0, 0, insertRows+0) + tdSql.checkData(1, 0, insertRows+1) + tdSql.checkData(2, 0, insertRows+2) + + tdSql.checkData(0, 1, insertRows+0) + tdSql.checkData(1, 1, insertRows+1) + tdSql.checkData(2, 1, insertRows+2) + + def stop(self): tdSql.close() From 4746305dbdbbd926590ba470526098916bdb6e00 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 3 Nov 2020 17:35:02 +0800 Subject: [PATCH 32/58] fix TD-1902 --- src/tsdb/src/tsdbRWHelper.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index eb1193110d..c3f92e2bd1 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1572,7 +1572,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { while (true) { tdResetDataCols(pDataCols); - tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst, defaultRowsInBlock, pDataCols, NULL, 0, + tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); if (pDataCols->numOfRows == 0) break; @@ -1714,6 +1714,8 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt (*iter)++; tSkipListIterNext(pCommitIter->pIter); } + + if (pTarget->numOfRows >= maxRows) break; } } From 9c289dd7626e6be6d6f666acb7baad81f2b66707 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 3 Nov 2020 18:37:03 +0800 Subject: [PATCH 33/58] add update test case --- tests/pytest/update/append_commit_last.py | 109 ++++-- tests/pytest/update/merge_commit_last.py | 430 ++++++++++++++-------- 2 files changed, 358 insertions(+), 181 deletions(-) diff --git a/tests/pytest/update/append_commit_last.py b/tests/pytest/update/append_commit_last.py index 9b31adccec..013983f940 100644 --- a/tests/pytest/update/append_commit_last.py +++ b/tests/pytest/update/append_commit_last.py @@ -1,42 +1,85 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -create database db update 1; +# -*- coding: utf-8 -*- -## Step 1 -$loops = 1000 -#t0 = 1604298064000 +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * -create table t1 (ts timestamp, a int); -for ($i = 0; $i < $loops; $i++) { - insert into t1 values ($t0 + $i, 1); - restart to commit - check query result -} -## Step 2 -create table t2 (ts timestamp, a int); +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) -insert into t2 ($t0, 1); -restart to commit -check query result + self.ts = 1604298064000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use udb") -for ($i = 1; $i <= 150; $i++) { - insert into t2 values ($t0 + $i, 1); -} -restart to commit -check query result + def run(self): + tdSql.prepare() -## Step 3 -create table t3 (ts timestamp, a int); + print("==============step1") + tdSql.execute("create database udb update 1") + tdSql.execute("use udb") + tdSql.execute("create table t1 (ts timestamp, a int)") -insert into t3 ($t0, 1); -restart to commit -check query result + for i in range(10): + tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(i + 1) -for ($i = 0; $i < 8; $i++) { - for ($j = 1; $j <= 10; $j++) { - insert into t3 values ($t0 + $i * 10 + $j , 1); - } -} -restart to commit -check query result \ No newline at end of file + + print("==============step2") + tdSql.execute("create table t2 (ts timestamp, a int)") + tdSql.execute("insert into t2 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(1) + + for i in range(1, 151): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(151) + + + print("==============step3") + tdSql.execute("create table t3 (ts timestamp, a int)") + tdSql.execute("insert into t3 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(1) + + for i in range(8): + for j in range(1, 11): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j)) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(81) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_last.py b/tests/pytest/update/merge_commit_last.py index 9d832d8984..183cca0a1e 100644 --- a/tests/pytest/update/merge_commit_last.py +++ b/tests/pytest/update/merge_commit_last.py @@ -1,187 +1,321 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -$t0 = 1603152000000 +# -*- coding: utf-8 -*- -create database db update 1 days 30; +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * -## STEP 1: UPDATE THE LAST RECORD REPEATEDLY -create table t1 (ts timestamp, a int); -for ($i = 0; $i < 100; $i++) { - insert into t1 values ($t0, $i); +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) - restart to commit - check query result -} + self.ts = 1603152000000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use udb") -## STEP 2: UPDATE THE WHOLE LAST BLOCK -create table t2 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t2 values ($t0 + $i, 1); -} -restart to commit -check query result + def run(self): + tdSql.prepare() -for ($i = 0; $i < 50; $i++) { - insert into t2 values ($t0 + $i, 2); -} + tdSql.execute("create database udb update 1 days 30") + tdSql.execute("use udb") -check query result -restart to commit -check query result + print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY") + tdSql.execute("create table t1 (ts timestamp, a int)") -## STEP 3: UPDATE PART OF THE LAST BLOCK -create table t3 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t3 values ($t0 + $i, 1); -} -restart to commit -check query result + for i in range(5): + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, i) -for ($i = 0; $i < 25; $i++) { - insert into t3 values ($t0 + $i, 2); -} + print("==============step 2: UPDATE THE WHOLE LAST BLOCK") + tdSql.execute("create table t2 (ts timestamp, a int)") -check query result -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) -for ($i = 25; $i < 50; $i++) { - insert into t3 values ($t0 + $i, 2); -} + for i in range(50): + tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i)) + tdSql.query("select * from t2") + for i in range(50): + tdSql.checkData(i, 1, 2) -check query result -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 2) -## STEP 4: UPDATE AND INSERT APPEND AT END OF DATA -create table t4 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t4 values ($t0 + $i, 1); -} -restart to commit -check query result + print("==============step 3: UPDATE PART OF THE LAST BLOCK") + tdSql.execute("create table t3 (ts timestamp, a int)") -for ($i = 0; $i < 25; $i++) { - insert into t4 values ($t0 + $i, 2); -} + for i in range(50): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) -for ($i = 50; $i < 60; $i++) { - insert into t4 values ($t0 + $i, 2); -} + for i in range(25): + tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t3") + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) -check query result -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + + print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA") + tdSql.execute("create table t4 (ts timestamp, a int)") -## STEP 5: UPDATE AND INSERT PREPEND SOME DATA -create table t5 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t5 values ($t0 + $i, 1); -} -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) -for ($i = -10; $i < 0; $i++) { - insert into t4 values ($t0 + $i, 2); -} + for i in range(25): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + for i in range(50, 60): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t4") + tdSql.checkRows(60) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + for i in range(50, 60): + tdSql.checkData(i, 1, 2) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(60) + for i in range(25): + tdSql.checkData(i, 1, 2) + for i in range(25, 50): + tdSql.checkData(i, 1, 1) + for i in range(50, 60): + tdSql.checkData(i, 1, 2) -for ($i = 0; $i < 25; $i++) { - insert into t5 values ($t0 + $i, 2); -} + print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA") + tdSql.execute("create table t5 (ts timestamp, a int)") -check query result -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(50) + for i in range(50): + tdSql.checkData(i, 1, 1) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + for i in range(25): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 95) -for ($i = -10; $i < 0; $i++) { - insert into t4 values ($t0 + $i, 3); -} + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 95) -for ($i = 25; $i < 50; $i++) { - insert into t5 values ($t0 + $i, 3); -} + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + for i in range(25, 50): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) -check query result -restart to commit -check query result + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 155) -## STEP 6: INSERT AHEAD A LOT OF DATA -create table t6 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t6 values ($t0 + $i, 1); -} -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 155) + -for ($i = -1000; $i < 0; $i++) { - insert into t6 values ($t0 + $i, 2); -} + print("==============step 6: INSERT AHEAD A LOT OF DATA") + tdSql.execute("create table t6 (ts timestamp, a int)") -check query result -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i)) -## STEP 7: INSERT AHEAD A LOT AND UPDATE -create table t7 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t7 values ($t0 + $i, 1); -} -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 0): + tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) -for ($i = -1000; $i < 25; $i++) { - insert into t7 values ($t0 + $i, 2); -} + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) -check query result -restart to commit -check query result + print("==============step 7: INSERT AHEAD A LOT AND UPDATE") + tdSql.execute("create table t7 (ts timestamp, a int)") -## STEP 8: INSERT AHEAD A LOT AND UPDATE -create table t8 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t8 values ($t0 + $i, 1); -} -restart to commit -check query result + for i in range(50): + tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i)) -for ($i = 25; $i < 6000; $i++) { - insert into t8 values ($t0 + $i, 2); -} + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 25): + tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2075) -check query result -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2075) -## STEP 9: UPDATE ONLY MIDDLE -create table t9 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t9 values ($t0 + $i, 1); -} -restart to commit -check query result + print("==============step 8: INSERT AFTER A LOT AND UPDATE") + tdSql.execute("create table t8 (ts timestamp, a int)") -for ($i = 20; $i < 30; $i++) { - insert into t9 values ($t0 + $i, 2); -} + for i in range(50): + tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i)) -check query result -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 50) + + for i in range(25, 6000): + tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11975) -## STEP 10: A LOT OF DATA COVER THE WHOLE BLOCK -create table t10 (ts timestamp, a int); -for ($i = 0; $i < 50; $i++) { - insert into t10 values ($t0 + $i, 1); -} -restart to commit -check query result + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11975) -for ($i = -4000; $i < 4000; $i++) { - insert into t10 values ($t0 + $i, 2); -} + print("==============step 9: UPDATE ONLY MIDDLE") + tdSql.execute("create table t9 (ts timestamp, a int)") -check query result -restart to commit -check query result \ No newline at end of file + for i in range(50): + tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + for i in range(20, 30): + tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 60) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 60) + + print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK") + tdSql.execute("create table t10 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 50) + + for i in range(-4000, 4000): + tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 16000) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 16000) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a8e272d249518da1c897ac05aabeec286681254c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 Nov 2020 05:30:42 +0000 Subject: [PATCH 34/58] fix TD-1934 --- src/common/src/tdataformat.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 49d21d9275..40b524488a 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -434,12 +434,12 @@ void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols) int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge) { ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows); - ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); ASSERT(target->numOfCols == source->numOfCols); SDataCols *pTarget = NULL; if (dataColsKeyLast(target) < dataColsKeyFirst(source)) { // No overlap + ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints); for (int i = 0; i < rowsToMerge; i++) { for (int j = 0; j < source->numOfCols; j++) { if (source->cols[j].len > 0) { @@ -509,6 +509,8 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i (*iter2)++; if (key1 == key2) (*iter1)++; } + + ASSERT(target->numOfRows <= target->maxPoints); } } From 43701df54a7186a28fd0547af76a66ca839d4841 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 4 Nov 2020 15:28:32 +0800 Subject: [PATCH 35/58] add same test cases with update flag 0 --- tests/pytest/update/append_commit_last-0.py | 90 ++++++ tests/pytest/update/merge_commit_last-0.py | 309 ++++++++++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 tests/pytest/update/append_commit_last-0.py create mode 100644 tests/pytest/update/merge_commit_last-0.py diff --git a/tests/pytest/update/append_commit_last-0.py b/tests/pytest/update/append_commit_last-0.py new file mode 100644 index 0000000000..c884207f2b --- /dev/null +++ b/tests/pytest/update/append_commit_last-0.py @@ -0,0 +1,90 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1604298064000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use db") + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table t1 (ts timestamp, a int)") + + for i in range(10): + tdSql.execute("insert into t1 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(i + 1) + tdSql.query("select sum(a) from t1") + tdSql.checkData(0, 0, i + 1) + + print("==============step2") + tdSql.execute("create table t2 (ts timestamp, a int)") + tdSql.execute("insert into t2 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + for i in range(1, 151): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(151) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 151) + + + print("==============step3") + tdSql.execute("create table t3 (ts timestamp, a int)") + tdSql.execute("insert into t3 values(%d, 1)" % self.ts) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 1) + + for i in range(8): + for j in range(1, 11): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i * 10 + j)) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(81) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 81) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_last-0.py b/tests/pytest/update/merge_commit_last-0.py new file mode 100644 index 0000000000..8a247f3809 --- /dev/null +++ b/tests/pytest/update/merge_commit_last-0.py @@ -0,0 +1,309 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1603152000000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use db") + + def run(self): + tdSql.prepare() + + print("==============step 1: UPDATE THE LAST RECORD REPEATEDLY") + tdSql.execute("create table t1 (ts timestamp, a int)") + + for i in range(5): + tdSql.execute("insert into t1 values(%d, %d)" % (self.ts, i)) + self.restartTaosd() + tdSql.query("select * from t1") + tdSql.checkRows(1) + tdSql.checkData(0, 1, 0) + + print("==============step 2: UPDATE THE WHOLE LAST BLOCK") + tdSql.execute("create table t2 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t2 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + for i in range(50): + tdSql.execute("insert into t2 values(%d, 2)" % (self.ts + i)) + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t2") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t2") + tdSql.checkData(0, 0, 50) + + print("==============step 3: UPDATE PART OF THE LAST BLOCK") + tdSql.execute("create table t3 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t3 values(%d, 1)" % (self.ts + i)) + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + for i in range(25): + tdSql.execute("insert into t3 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t3") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t3") + tdSql.checkData(0, 0, 50) + + print("==============step 4: UPDATE AND INSERT APPEND AT END OF DATA") + tdSql.execute("create table t4 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t4 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 50) + + for i in range(25): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + for i in range(50, 60): + tdSql.execute("insert into t4 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t4") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t4") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t4") + tdSql.checkData(0, 0, 70) + + print("==============step 5: UPDATE AND INSERT PREPEND SOME DATA") + tdSql.execute("create table t5 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t5 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 50) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + for i in range(25): + tdSql.execute("insert into t5 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + for i in range(-10, 0): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + for i in range(25, 50): + tdSql.execute("insert into t5 values(%d, 3)" % (self.ts + i)) + + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + self.restartTaosd() + tdSql.query("select * from t5") + tdSql.checkRows(60) + tdSql.query("select sum(a) from t5") + tdSql.checkData(0, 0, 70) + + + print("==============step 6: INSERT AHEAD A LOT OF DATA") + tdSql.execute("create table t6 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t6 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 0): + tdSql.execute("insert into t6 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) + + self.restartTaosd() + tdSql.query("select * from t6") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0, 0, 2050) + + print("==============step 7: INSERT AHEAD A LOT AND UPDATE") + tdSql.execute("create table t7 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t7 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 50) + + for i in range(-1000, 25): + tdSql.execute("insert into t7 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2050) + + self.restartTaosd() + tdSql.query("select * from t7") + tdSql.checkRows(1050) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0, 0, 2050) + + print("==============step 8: INSERT AFTER A LOT AND UPDATE") + tdSql.execute("create table t8 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t8 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 50) + + for i in range(25, 6000): + tdSql.execute("insert into t8 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11950) + + self.restartTaosd() + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0, 0, 11950) + + print("==============step 9: UPDATE ONLY MIDDLE") + tdSql.execute("create table t9 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t9 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + for i in range(20, 30): + tdSql.execute("insert into t9 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + self.restartTaosd() + tdSql.query("select * from t9") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0, 0, 50) + + print("==============step 10: A LOT OF DATA COVER THE WHOLE BLOCK") + tdSql.execute("create table t10 (ts timestamp, a int)") + + for i in range(50): + tdSql.execute("insert into t10 values(%d, 1)" % (self.ts + i)) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(50) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 50) + + for i in range(-4000, 4000): + tdSql.execute("insert into t10 values(%d, 2)" % (self.ts + i)) + + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 15950) + + self.restartTaosd() + tdSql.query("select * from t10") + tdSql.checkRows(8000) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0, 0, 15950) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From f605be1018cdd3d3e961afdb80be270f363933c8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 4 Nov 2020 07:31:21 +0000 Subject: [PATCH 36/58] fix TD-1931 --- src/tsdb/src/tsdbMemTable.c | 1 + src/tsdb/src/tsdbRWHelper.c | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 19f59f3080..d68a97eb13 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -322,6 +322,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey memset(pMergeInfo, 0, sizeof(*pMergeInfo)); pMergeInfo->keyFirst = INT64_MAX; pMergeInfo->keyLast = INT64_MIN; + if (pCols) tdResetDataCols(pCols); row = tsdbNextIterRow(pIter); if (row == NULL || dataRowKey(row) > maxKey) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index e02a7bbfe5..6cbd119961 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1495,7 +1495,6 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, ASSERT(pIdx->len > 0); SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1); ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock); - tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols, NULL, 0, pCfg->update, pMergeInfo); @@ -1525,7 +1524,6 @@ static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, } } else { ASSERT(!pHelper->hasOldLastBlock); - tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); @@ -1571,7 +1569,6 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) { while (true) { - tdResetDataCols(pDataCols); tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo); ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows); From 7f88bc2c8e0c345b14a794bc639a37b69a443d5b Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 4 Nov 2020 15:34:31 +0800 Subject: [PATCH 37/58] add updatetest.sh file --- tests/pytest/updatetest.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 tests/pytest/updatetest.sh diff --git a/tests/pytest/updatetest.sh b/tests/pytest/updatetest.sh new file mode 100644 index 0000000000..ade1180553 --- /dev/null +++ b/tests/pytest/updatetest.sh @@ -0,0 +1,10 @@ +# update +python3 ./test.py -f update/allow_update.py +python3 ./test.py -f update/allow_update-0.py +python3 ./test.py -f update/append_commit_data.py +python3 ./test.py -f update/append_commit_last-0.py +python3 ./test.py -f update/append_commit_last.py +python3 ./test.py -f update/merge_commit_data.py +python3 ./test.py -f update/merge_commit_data2.py +python3 ./test.py -f update/merge_commit_last-0.py +python3 ./test.py -f update/merge_commit_last.py \ No newline at end of file From 13dda6b431d4bb5f509dcc2566fe881484bd0fb4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 Nov 2020 09:17:27 +0000 Subject: [PATCH 38/58] TD-1932 --- src/mnode/src/mnodeDb.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 812b1c609f..fb4793c056 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -969,8 +969,13 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { } if (update >= 0 && update != pDb->cfg.update) { +#if 0 mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update); newCfg.update = update; +#else + mError("db:%s, can't alter update option", pDb->name); + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; +#endif } return newCfg; From b636d216b9dbed55649b0d078888e8c31b394a2a Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Wed, 4 Nov 2020 18:07:43 +0800 Subject: [PATCH 39/58] add testcase for update --- tests/pytest/query/kill_query.py | 70 +++ tests/pytest/update/merge_commit_data2.py | 467 +++++++++++++----- .../update/merge_commit_data2_update0.py | 384 ++++++++++++++ 3 files changed, 787 insertions(+), 134 deletions(-) create mode 100644 tests/pytest/query/kill_query.py create mode 100644 tests/pytest/update/merge_commit_data2_update0.py diff --git a/tests/pytest/query/kill_query.py b/tests/pytest/query/kill_query.py new file mode 100644 index 0000000000..211e49c9ee --- /dev/null +++ b/tests/pytest/query/kill_query.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +from util.dnodes import tdDnodes + + +class TDTestCase: + """ + kill query + """ + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create database if not exists demo;"); + tdSql.execute("use demo;") + tdSql.execute("CREATE TABLE IF NOT EXISTS demo1 (ts TIMESTAMP, f1 int);") + ts = 1300000000000 + + for i in range(1000): + tdSql.execute(" insert into demo1 values({ts}, {data});".format(ts=ts, data=i)) + ts += 1 + + print("==============insert into test1 and test2 form test file") + + + + print("==============step2") + while True: + tdSql.query('select * from demo1;') + print('==============', len(tdSql.queryResult)) + with open(ordered_csv) as f1: + num1 = len(f1.readlines()) + tdSql.checkRows(num1) + + + tdSql.query('select * from test2;') + with open(disordered_csv) as f2: + num2 = len(f2.readlines()) + tdSql.checkRows(num2) + print("=============execute select count(*) from xxx") + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data2.py b/tests/pytest/update/merge_commit_data2.py index e9dd084eb0..3f0fc718ad 100644 --- a/tests/pytest/update/merge_commit_data2.py +++ b/tests/pytest/update/merge_commit_data2.py @@ -1,153 +1,352 @@ -# clear env set up +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### -$t0 = 1603152000000 +# -*- coding: utf-8 -*- -create database db update 1 days 30; +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time -## Step 1. UPDATE TWO WHOLE DATA BLOCK REPEATEDLY -create table t1 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 1); -} -restart to commit -check query result +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def restart_taosd(self,db): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use %s;" % db) -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + 5000 + $i, 1); -} + def date_to_timestamp_microseconds(self, date): + datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f") + obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0) + return obj_stamp -check query result -restart to commit -check query result + def timestamp_microseconds_to_date(self, timestamp): + d = datetime.datetime.fromtimestamp(timestamp/1000) + str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f") + return str1 -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + $i, 2); -} -for ($i = 0; $i < 200; $i++) { - insert into t1 values ($t0 + 5000 + $i, 1); -} -check query result -restart to commit -check query result -## Step 2. INSERT IN MIDDLE LITTLE DATA REPEATEDLY -create table t2 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t2 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t2 values ($t0 + 5000 + $i, 1); -} -restart to commit -for ($k = 0; $k < 10; $k++) { - for ($i = 0; $i < 10; $i++) { - insert into t2 values ($t0 + 200 + $k * 10 + $i, 1); - } - check query result - restart to commit - check query result -} + def run(self): + print("==========step1") + print("create table && insert data") + sql = 'reset query cache' + tdSql.execute(sql) + sql = 'drop database if exists db' + tdSql.execute(sql) + sql = 'create database db update 1 days 30;' + tdSql.execute(sql) + sql = 'use db;' + tdSql.execute(sql) + tdSql.execute('create table t1 (ts timestamp, a int)') -## Step 3. INSERT IN MIDDLE AND UPDATE TWO BLOCKS -create table t3 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t3 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t3 values ($t0 + 5000 + $i, 1); -} -restart to commit -for ($i = 0; $i < 5200; $i++) { - insert into t3 values ($t0 + $i, 2); -} -check query result -restart to commit -check query result + print("==================================1 start") + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i)) + print("==========step2") + print("restart to commit ") + self.restart_taosd('db') -## Step 4. INSERT IN MIDDLE AND UPDATE FIRST HALF OF TWO BLOCKS -create table t4 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t4 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t4 values ($t0 + 5000 + $i, 1); -} -restart to commit + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) -for ($i = 0; $i < 100; $i++) { - insert into t4 values ($t0 + $i, 2); -} -for ($i = 200; $i < 5000; $i++) { - insert into t4 values ($t0 + $i, 2); -} -for ($i = 0; $i < 100; $i++) { - insert into t4 values ($t0 + 5000 + $i, 2); -} -check query result -restart to commit -check query result + print("==========step3") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + print('check query result before restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) -## Step 5. INSERT IN MIDDLE AND UPDATE last HALF OF TWO BLOCKS -create table t5 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t5 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t5 values ($t0 + 5000 + $i, 1); -} -restart to commit + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) -for ($i = 100; $i < 200; $i++) { - insert into t5 values ($t0 + $i, 2); -} -for ($i = 200; $i < 5000; $i++) { - insert into t5 values ($t0 + $i, 2); -} -for ($i = 100; $i < 200; $i++) { - insert into t5 values ($t0 + 5000 + $i, 2); -} -check query result -restart to commit -check query result + print("==========step4") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 2)' %(t0+i)) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) -## Step 6. INSERT MASSIVE DATA AND UPDATE ALL BLOCKS -create table t6 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t6 values ($t0 + $i, 1); -} -restart to commit -for ($i = 0; $i < 200; $i++) { - insert into t6 values ($t0 + 5000 + $i, 1); -} -restart to commit + print('check query result before restart') + tdSql.query('select * from db.t1;') + print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 2) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) -for ($i = -1000; $i < 10000; $i++) { - insert into t6 values ($t0 + $i, 2); -} -check query result -restart to commit -check query result + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t1;') + # print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 2) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) -## Step 7. SPLIT DATA BLOCK -create table t7 (ts timestamp, a int); -for ($i = 0; $i < 200; $i++) { - insert into t7 values ($t0 + $i, 1); -} -for ($i = 0; $i < 200; $i++) { - insert into t7 values ($t0 + 5000 + $i, 1); -} -restart to commit + print("==================================2 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t2 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') -for ($i = -1000; $i < 10000; $i++) { - insert into t7 values ($t0 + $i, 2); -} -check query result -restart to commit -check query result \ No newline at end of file + for k in range(10): + for i in range(10): + tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + + + print("==================================3 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t3 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(5200): + tdSql.execute('insert into t3 values (%d , 2)' %(t0+i)) + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t3;') + for i in range(5200): + tdSql.checkData(i, 1, 2) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t3;') + for i in range(5200): + tdSql.checkData(i, 1, 2) + + print("==================================4 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t4 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t4;') + for i in range(100): + tdSql.checkData(i, 1, 2) + for i in range(100, 200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t4;') + for i in range(100): + tdSql.checkData(i, 1, 2) + for i in range(100, 200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================5 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t5 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t5;') + for i in range(100): + tdSql.checkData(i, 1, 1) + for i in range(100, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5100): + tdSql.checkData(i, 1, 1) + for i in range(5100, 5200): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t5;') + for i in range(100): + tdSql.checkData(i, 1, 1) + for i in range(100, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5100): + tdSql.checkData(i, 1, 1) + for i in range(5100, 5200): + tdSql.checkData(i, 1, 2) + + print("==================================6 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t6 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t6 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + + print("==================================7 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t7 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i)) + + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t7 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(11000): + tdSql.checkData(i, 1, 2) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data2_update0.py b/tests/pytest/update/merge_commit_data2_update0.py new file mode 100644 index 0000000000..def50e0466 --- /dev/null +++ b/tests/pytest/update/merge_commit_data2_update0.py @@ -0,0 +1,384 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import time + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def restart_taosd(self,db): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use %s;" % db) + + def date_to_timestamp_microseconds(self, date): + datetime_obj = datetime.strptime(date, "%Y-%m-%d %H:%M:%S.%f") + obj_stamp = int(time.mktime(datetime_obj.timetuple()) * 1000.0 + datetime_obj.microsecond / 1000.0) + return obj_stamp + + def timestamp_microseconds_to_date(self, timestamp): + d = datetime.datetime.fromtimestamp(timestamp/1000) + str1 = d.strftime("%Y-%m-%d %H:%M:%S.%f") + return str1 + + + + def run(self): + print("==========step1") + print("create table && insert data") + sql = 'reset query cache' + tdSql.execute(sql) + sql = 'drop database if exists db' + tdSql.execute(sql) + sql = 'create database db update 0 days 30;' + tdSql.execute(sql) + sql = 'use db;' + tdSql.execute(sql) + tdSql.execute('create table t1 (ts timestamp, a int)') + + + print("==================================1 start") + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i)) + print("==========step2") + print("restart to commit ") + self.restart_taosd('db') + + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + + print("==========step3") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + print('check query result before restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t1;') + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print("==========step4") + print('insert data') + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 2)' %(t0+i)) + for i in range(insert_rows): + tdSql.execute('insert into t1 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t1;') + print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t1;') + # print(tdSql.queryResult) + for i in range(insert_rows): + tdSql.checkData(i, 1, 1) + for i in range(insert_rows, insert_rows*2): + tdSql.checkData(i, 1, 1) + + print("==================================2 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t2 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + for i in range(insert_rows): + tdSql.execute('insert into t2 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for k in range(10): + for i in range(10): + tdSql.execute('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + # print('insert into t2 values (%d , 1)' %(t0 + 200 + k * 10 + i)) + + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t2;') + for i in range(insert_rows*2+100): + tdSql.checkData(i, 1, 1) + + + print("==================================3 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t3 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t3 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(5200): + tdSql.execute('insert into t3 values (%d , 2)' %(t0+i)) + + print("==========step2") + print('check query result before restart') + tdSql.query('select * from db.t3;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + # print(tdSql.queryResult) + print('restart to commit') + self.restart_taosd('db') + print('check query result after restart') + tdSql.query('select * from db.t3;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================4 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t4 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t4 values (%d , 2)' %(t0+i)) + + for i in range(100): + tdSql.execute('insert into t4 values (%d , 1)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t4;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t4;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + # + print("==================================5 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t5 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t5 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(200, 5000): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i)) + + for i in range(100, 200): + tdSql.execute('insert into t5 values (%d , 2)' %(t0+i+5000)) + + print('check query result before restart') + tdSql.query('select * from db.t5;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t5;') + for i in range(200): + tdSql.checkData(i, 1, 1) + for i in range(200, 5000): + tdSql.checkData(i, 1, 2) + for i in range(5000, 5200): + tdSql.checkData(i, 1, 1) + + print("==================================6 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t6 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(insert_rows): + tdSql.execute('insert into t6 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t6 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t6;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + + print("==================================7 start") + print("==========step1") + print("create table && insert data") + tdSql.execute('create table t7 (ts timestamp, a int)') + insert_rows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % insert_rows) + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i)) + + for i in range(insert_rows): + tdSql.execute('insert into t7 values (%d , 1)' %(t0+i+5000)) + print('restart to commit') + self.restart_taosd('db') + + for i in range(-1000, 10000): + tdSql.execute('insert into t7 values (%d , 2)' %(t0+i)) + + print('check query result before restart') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + print('check query result after restart') + self.restart_taosd('db') + tdSql.query('select * from db.t7;') + tdSql.checkRows(11000) + for i in range(1000): + tdSql.checkData(i, 1, 2) + for i in range(1000,1200): + tdSql.checkData(i, 1, 1) + for i in range(1200,6000): + tdSql.checkData(i, 1, 2) + for i in range(6000,6200): + tdSql.checkData(i, 1, 1) + for i in range(6200, 11000): + tdSql.checkData(i, 1, 2) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 1b4674bb063c6944e44d85507db4032a4faa7dc5 Mon Sep 17 00:00:00 2001 From: wangyazhou1313 Date: Wed, 4 Nov 2020 18:40:19 +0800 Subject: [PATCH 40/58] remove kill query.py --- tests/pytest/query/kill_query.py | 70 -------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 tests/pytest/query/kill_query.py diff --git a/tests/pytest/query/kill_query.py b/tests/pytest/query/kill_query.py deleted file mode 100644 index 211e49c9ee..0000000000 --- a/tests/pytest/query/kill_query.py +++ /dev/null @@ -1,70 +0,0 @@ -################################################################### -# Copyright (c) 2016 by TAOS Technologies, Inc. -# All rights reserved. -# -# This file is proprietary and confidential to TAOS Technologies. -# No part of this file may be reproduced, stored, transmitted, -# disclosed or used in any form or by any means other than as -# expressly provided by the written permission from Jianhui Tao -# -################################################################### - -# -*- coding: utf-8 -*- - -import sys -import taos -from util.log import tdLog -from util.cases import tdCases -from util.sql import tdSql -from util.dnodes import tdDnodes - - -class TDTestCase: - """ - kill query - """ - - def init(self, conn, logSql): - tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor(), logSql) - - def run(self): - tdSql.prepare() - - print("==============step1") - tdSql.execute("create database if not exists demo;"); - tdSql.execute("use demo;") - tdSql.execute("CREATE TABLE IF NOT EXISTS demo1 (ts TIMESTAMP, f1 int);") - ts = 1300000000000 - - for i in range(1000): - tdSql.execute(" insert into demo1 values({ts}, {data});".format(ts=ts, data=i)) - ts += 1 - - print("==============insert into test1 and test2 form test file") - - - - print("==============step2") - while True: - tdSql.query('select * from demo1;') - print('==============', len(tdSql.queryResult)) - with open(ordered_csv) as f1: - num1 = len(f1.readlines()) - tdSql.checkRows(num1) - - - tdSql.query('select * from test2;') - with open(disordered_csv) as f2: - num2 = len(f2.readlines()) - tdSql.checkRows(num2) - print("=============execute select count(*) from xxx") - - - def stop(self): - tdSql.close() - tdLog.success("%s successfully executed" % __file__) - - -tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) From 6f0f199a758a40fef8f5f5909d78a90bb96cb452 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 4 Nov 2020 18:58:57 +0800 Subject: [PATCH 41/58] add test case for update --- tests/pytest/update/append_commit_data-0.py | 84 +++++ tests/pytest/update/merge_commit_data-0.py | 351 ++++++++++++++++++++ 2 files changed, 435 insertions(+) create mode 100644 tests/pytest/update/append_commit_data-0.py create mode 100644 tests/pytest/update/merge_commit_data-0.py diff --git a/tests/pytest/update/append_commit_data-0.py b/tests/pytest/update/append_commit_data-0.py new file mode 100644 index 0000000000..b844a50a08 --- /dev/null +++ b/tests/pytest/update/append_commit_data-0.py @@ -0,0 +1,84 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + print("==========step1") + print("create table && insert data") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') + + insertRows = 200 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0+i)) + print("==========step2") + print("restart to commit ") + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows) + for k in range(0,100): + tdLog.info("insert %d rows" % (insertRows)) + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into db.t1 values(%d,1)' % + (t0+k*200+i) + ) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from db.t1") + tdSql.checkRows(insertRows+200*k) + print("==========step2") + print("insert into another table ") + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 20000 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d, 1)' % + (t0+i)) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/update/merge_commit_data-0.py b/tests/pytest/update/merge_commit_data-0.py new file mode 100644 index 0000000000..14d435f7f2 --- /dev/null +++ b/tests/pytest/update/merge_commit_data-0.py @@ -0,0 +1,351 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + print("==========step1") + print("UPDATE THE WHOLE DATA BLOCK REPEATEDLY") + s = 'reset query cache' + tdSql.execute(s) + s = 'drop database if exists db' + tdSql.execute(s) + s = 'create database db days 30' + tdSql.execute(s) + s = 'use db' + tdSql.execute(s) + ret = tdSql.execute('create table t1 (ts timestamp, a int)') + + insertRows = 200 + t0 = 1603152000000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t1 values (%d , 1)' % + (t0 + i)) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + + for k in range(0,10): + for i in range (0,insertRows): + ret = tdSql.execute( + 'insert into t1 values(%d,1)' % + (t0+i) + ) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t1") + tdSql.checkRows(insertRows) + print("==========step2") + print("PREPEND DATA ") + ret = tdSql.execute('create table t2 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows) + for i in range(-100,0): + ret = tdSql.execute( + 'insert into t2 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t2") + tdSql.checkRows(insertRows+100) + print("==========step3") + print("PREPEND MASSIVE DATA ") + ret = tdSql.execute('create table t3 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows) + for i in range(-6000,0): + ret = tdSql.execute( + 'insert into t3 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t3") + tdSql.checkRows(insertRows+6000) + print("==========step4") + print("APPEND DATA") + ret = tdSql.execute('create table t4 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t4 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t4") + tdSql.checkRows(insertRows+100) + print("==========step5") + print("APPEND DATA") + ret = tdSql.execute('create table t5 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows) + for i in range(0,6000): + ret = tdSql.execute( + 'insert into t5 values (%d , 1)' % + (t0+200+i)) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t5") + tdSql.checkRows(insertRows+6000) + print("==========step6") + print("UPDATE BLOCK IN TWO STEP") + ret = tdSql.execute('create table t6 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t6 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + for i in range(0,100): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + for i in range(0,200): + ret = tdSql.execute( + 'insert into t6 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t6") + tdSql.checkRows(insertRows) + tdSql.query("select sum(a) from t6") + tdSql.checkData(0,0,'200') + print("==========step7") + print("UPDATE LAST HALF AND INSERT LITTLE DATA") + ret = tdSql.execute('create table t7 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t7 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t7") + tdSql.checkRows(insertRows) + for i in range(100,300): + ret = tdSql.execute( + 'insert into t7 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t7") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t7") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t7") + tdSql.checkData(0,0,'400') + print("==========step8") + print("UPDATE LAST HALF AND INSERT MASSIVE DATA") + ret = tdSql.execute('create table t8 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t8 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t8") + tdSql.checkRows(insertRows) + for i in range(6000): + ret = tdSql.execute( + 'insert into t8 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'11800') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t8") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t8") + tdSql.checkData(0,0,'11800') + print("==========step9") + print("UPDATE FIRST HALF AND PREPEND LITTLE DATA") + ret = tdSql.execute('create table t9 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t9 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t9") + tdSql.checkRows(insertRows) + for i in range(-100,100): + ret = tdSql.execute( + 'insert into t9 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'400') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t9") + tdSql.checkRows(300) + tdSql.query("select sum(a) from t9") + tdSql.checkData(0,0,'400') + print("==========step10") + print("UPDATE FIRST HALF AND PREPEND MASSIVE DATA") + ret = tdSql.execute('create table t10 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t10 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t10") + tdSql.checkRows(insertRows) + for i in range(-6000,100): + ret = tdSql.execute( + 'insert into t10 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12200') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t10") + tdSql.checkRows(6200) + tdSql.query("select sum(a) from t10") + tdSql.checkData(0,0,'12200') + print("==========step11") + print("UPDATE FIRST HALF AND APPEND MASSIVE DATA") + ret = tdSql.execute('create table t11 (ts timestamp, a int)') + insertRows = 200 + for i in range(0, insertRows): + ret = tdSql.execute( + 'insert into t11 values (%d , 1)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t11") + tdSql.checkRows(insertRows) + for i in range(100): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + for i in range(200,6000): + ret = tdSql.execute( + 'insert into t11 values (%d , 2)' % + (t0+i)) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11800') + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select * from t11") + tdSql.checkRows(6000) + tdSql.query("select sum(a) from t11") + tdSql.checkData(0,0,'11800') + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From b43c5ba92630ccfdc1930b8d0fbeb29886fe4d01 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 4 Nov 2020 21:15:47 +0000 Subject: [PATCH 42/58] Refactoring of crash_gen tool --- .../{crash_gen.py => crash_gen_main.py} | 81 ++++++++++++------ tests/pytest/crash_gen/db.py | 6 ++ tests/pytest/crash_gen/misc.py | 8 +- tests/pytest/crash_gen/service_manager.py | 85 +++++++++++++------ tests/pytest/crash_gen_bootstrap.py | 2 +- 5 files changed, 129 insertions(+), 53 deletions(-) rename tests/pytest/crash_gen/{crash_gen.py => crash_gen_main.py} (97%) diff --git a/tests/pytest/crash_gen/crash_gen.py b/tests/pytest/crash_gen/crash_gen_main.py similarity index 97% rename from tests/pytest/crash_gen/crash_gen.py rename to tests/pytest/crash_gen/crash_gen_main.py index 9bc701aa8b..fee51b98c0 100755 --- a/tests/pytest/crash_gen/crash_gen.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -38,9 +38,9 @@ import resource from guppy import hpy import gc -from .service_manager import ServiceManager, TdeInstance -from .misc import Logging, Status, CrashGenError, Dice, Helper, Progress -from .db import DbConn, MyTDSql, DbConnNative, DbManager +from crash_gen.service_manager import ServiceManager, TdeInstance +from crash_gen.misc import Logging, Status, CrashGenError, Dice, Helper, Progress +from crash_gen.db import DbConn, MyTDSql, DbConnNative, DbManager import taos import requests @@ -435,7 +435,7 @@ class ThreadCoordinator: Logging.debug("\r\n\n--> Main thread ready to finish up...") Logging.debug("Main thread joining all threads") self._pool.joinAll() # Get all threads to finish - Logging.info("\nAll worker threads finished") + Logging.info(". . . All worker threads finished") # No CR/LF before self._execStats.endExec() def cleanup(self): # free resources @@ -1072,17 +1072,18 @@ class Database: t3 = datetime.datetime(2012, 1, 1) # default "keep" is 10 years t4 = datetime.datetime.fromtimestamp( t3.timestamp() + elSec2) # see explanation above - Logging.info("Setting up TICKS to start from: {}".format(t4)) + Logging.debug("Setting up TICKS to start from: {}".format(t4)) return t4 @classmethod def getNextTick(cls): with cls._clsLock: # prevent duplicate tick - if cls._lastLaggingTick==0: + if cls._lastLaggingTick==0 or cls._lastTick==0 : # not initialized # 10k at 1/20 chance, should be enough to avoid overlaps - cls._lastLaggingTick = cls.setupLastTick() + datetime.timedelta(0, -10000) - if cls._lastTick==0: # should be quite a bit into the future - cls._lastTick = cls.setupLastTick() + tick = cls.setupLastTick() + cls._lastTick = tick + cls._lastLaggingTick = tick + datetime.timedelta(0, -10000) + # if : # should be quite a bit into the future if Dice.throw(20) == 0: # 1 in 20 chance, return lagging tick cls._lastLaggingTick += datetime.timedelta(0, 1) # Go back in time 100 seconds @@ -1322,7 +1323,7 @@ class Task(): self._err = err self._aborted = True except Exception as e: - self.logInfo("Non-TAOS exception encountered") + Logging.info("Non-TAOS exception encountered with: {}".format(self.__class__.__name__)) self._err = e self._aborted = True traceback.print_exc() @@ -1566,8 +1567,11 @@ class TaskCreateSuperTable(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place + sTable.create(wt.getDbConn(), self._db.getName(), - {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'}) + {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'}, + dropIfExists = True + ) # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) # No need to create the regular tables, INSERT will do that # automatically @@ -1580,14 +1584,41 @@ class TdSuperTable: def getName(self): return self._stName + def drop(self, dbc, dbName, skipCheck = False): + if self.exists(dbc, dbName) : # if myself exists + fullTableName = dbName + '.' + self._stName + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: + if not skipCheck: + raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) + + def exists(self, dbc, dbName): + dbc.execute("USE " + dbName) + return dbc.existsSuperTable(self._stName) + # TODO: odd semantic, create() method is usually static? - def create(self, dbc, dbName, cols: dict, tags: dict): + def create(self, dbc, dbName, cols: dict, tags: dict, + dropIfExists = False + ): + '''Creating a super table''' - sql = "CREATE TABLE {}.{} ({}) TAGS ({})".format( - dbName, - self._stName, - ",".join(['%s %s'%(k,v) for (k,v) in cols.items()]), - ",".join(['%s %s'%(k,v) for (k,v) in tags.items()]) + dbc.execute("USE " + dbName) + fullTableName = dbName + '.' + self._stName + if dbc.existsSuperTable(self._stName): + if dropIfExists: + dbc.execute("DROP TABLE {}".format(fullTableName)) + else: # error + raise CrashGenError("Cannot create super table, already exists: {}".format(self._stName)) + + # Now let's create + sql = "CREATE TABLE {} ({})".format( + fullTableName, + ",".join(['%s %s'%(k,v) for (k,v) in cols.items()])) + if tags is None : + sql += " TAGS (dummy int) " + else: + sql += " TAGS ({})".format( + ",".join(['%s %s'%(k,v) for (k,v) in tags.items()]) ) dbc.execute(sql) @@ -1611,17 +1642,19 @@ class TdSuperTable: return # acquire a lock first, so as to be able to *verify*. More details in TD-1471 - fullTableName = dbName + '.' + regTableName - task.lockTable(fullTableName) + fullTableName = dbName + '.' + regTableName + if task is not None: # optional lock + task.lockTable(fullTableName) Progress.emit(Progress.CREATE_TABLE_ATTEMPT) # ATTEMPT to create a new table - print("(" + fullTableName[-3:] + ")", end="", flush=True) + # print("(" + fullTableName[-3:] + ")", end="", flush=True) try: sql = "CREATE TABLE {} USING {}.{} tags ({})".format( fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) ) dbc.execute(sql) finally: - task.unlockTable(fullTableName) # no matter what + if task is not None: + task.unlockTable(fullTableName) # no matter what def _getTagStrForSql(self, dbc, dbName: str) : tags = self._getTags(dbc, dbName) @@ -1840,7 +1873,7 @@ class TaskRestartService(StateTransitionTask): with self._classLock: if self._isRunning: - print("Skipping restart task, another running already") + Logging.info("Skipping restart task, another running already") return self._isRunning = True @@ -1999,7 +2032,7 @@ class ThreadStacks: # stack info for all threads class ClientManager: def __init__(self): - print("Starting service manager") + Logging.info("Starting service manager") # signal.signal(signal.SIGTERM, self.sigIntHandler) # signal.signal(signal.SIGINT, self.sigIntHandler) @@ -2101,7 +2134,7 @@ class ClientManager: thPool = ThreadPool(gConfig.num_threads, gConfig.max_steps) self.tc = ThreadCoordinator(thPool, dbManager) - print("Starting client instance to: {}".format(tInst)) + Logging.info("Starting client instance: {}".format(tInst)) self.tc.run() # print("exec stats: {}".format(self.tc.getExecStats())) # print("TC failed = {}".format(self.tc.isFailed())) diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index 43c855647c..2a4b362f82 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -95,6 +95,11 @@ class DbConn: # print("dbs = {}, str = {}, ret2={}, type2={}".format(dbs, dbName,ret2, type(dbName))) return dbName in dbs # TODO: super weird type mangling seen, once here + def existsSuperTable(self, stName): + self.query("show stables") + sts = [v[0] for v in self.getQueryResult()] + return stName in sts + def hasTables(self): return self.query("show tables") > 0 @@ -240,6 +245,7 @@ class MyTDSql: def _execInternal(self, sql): startTime = time.time() + # Logging.debug("Executing SQL: " + sql) ret = self._cursor.execute(sql) # print("\nSQL success: {}".format(sql)) queryTime = time.time() - startTime diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index 4dc053dae3..2d2ce99d95 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -27,7 +27,7 @@ class LoggingFilter(logging.Filter): class MyLoggingAdapter(logging.LoggerAdapter): def process(self, msg, kwargs): - return "[{}] {}".format(threading.get_ident() % 10000, msg), kwargs + return "[{:04d}] {}".format(threading.get_ident() % 10000, msg), kwargs # return '[%s] %s' % (self.extra['connid'], msg), kwargs @@ -51,7 +51,7 @@ class Logging: _logger.addHandler(ch) # Logging adapter, to be used as a logger - print("setting logger variable") + # print("setting logger variable") # global logger cls.logger = MyLoggingAdapter(_logger, []) @@ -166,7 +166,8 @@ class Progress: SERVICE_RECONNECT_START = 4 SERVICE_RECONNECT_SUCCESS = 5 SERVICE_RECONNECT_FAILURE = 6 - CREATE_TABLE_ATTEMPT = 7 + SERVICE_START_NAP = 7 + CREATE_TABLE_ATTEMPT = 8 tokens = { STEP_BOUNDARY: '.', @@ -176,6 +177,7 @@ class Progress: SERVICE_RECONNECT_START: '', SERVICE_RECONNECT_FAILURE: '.xr>', + SERVICE_START_NAP: '_zz', CREATE_TABLE_ATTEMPT: '_c', } diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index 196e9d944a..d249abc439 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -47,6 +47,17 @@ class TdeInstance(): .format(selfPath, projPath)) return buildPath + @classmethod + def prepareGcovEnv(cls, env): + # Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html + bPath = cls._getBuildPath() # build PATH + numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3 + numSegments = numSegments - 1 # DEBUG only + env['GCOV_PREFIX'] = bPath + '/svc_gcov' + env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings + Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format( + numSegments, env['GCOV_PREFIX'] )) + def __init__(self, subdir='test', tInstNum=0, port=6030, fepPort=6030): self._buildDir = self._getBuildPath() self._subdir = '/' + subdir # TODO: tolerate "/" @@ -217,6 +228,11 @@ class TdeSubProcess: # raise CrashGenError("Empty instance not allowed in TdeSubProcess") # self._tInst = tInst # Default create at ServiceManagerThread + def __repr__(self): + if self.subProcess is None: + return '[TdeSubProc: Empty]' + return '[TdeSubProc: pid = {}]'.format(self.getPid()) + def getStdOut(self): return self.subProcess.stdout @@ -235,17 +251,30 @@ class TdeSubProcess: # Sanity check if self.subProcess: # already there raise RuntimeError("Corrupt process state") - + + # Prepare environment variables for coverage information + # Ref: https://stackoverflow.com/questions/2231227/python-subprocess-popen-with-a-modified-environment + myEnv = os.environ.copy() + TdeInstance.prepareGcovEnv(myEnv) + + # print(myEnv) + # print(myEnv.items()) + # print("Starting TDengine via Shell: {}".format(cmdLineStr)) + + useShell = True self.subProcess = subprocess.Popen( - cmdLine, - shell=False, + ' '.join(cmdLine) if useShell else cmdLine, + shell=useShell, # svcCmdSingle, shell=True, # capture core dump? stdout=subprocess.PIPE, stderr=subprocess.PIPE, # bufsize=1, # not supported in binary mode - close_fds=ON_POSIX + close_fds=ON_POSIX, + env=myEnv ) # had text=True, which interferred with reading EOF + STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process? + def stop(self): """ Stop a sub process, and try to return a meaningful return code. @@ -267,7 +296,7 @@ class TdeSubProcess: SIGUSR2 12 """ if not self.subProcess: - print("Sub process already stopped") + Logging.error("Sub process already stopped") return # -1 retCode = self.subProcess.poll() # ret -N means killed with signal N, otherwise it's from exit(N) @@ -278,20 +307,25 @@ class TdeSubProcess: return retCode # process still alive, let's interrupt it - print("Terminate running process, send SIG_INT and wait...") - # sub process should end, then IPC queue should end, causing IO thread to end - # sig = signal.SIGINT - sig = signal.SIGKILL - self.subProcess.send_signal(sig) # SIGNINT or SIGKILL + Logging.info("Terminate running process, send SIG_{} and wait...".format(self.STOP_SIGNAL)) + # sub process should end, then IPC queue should end, causing IO thread to end + topSubProc = psutil.Process(self.subProcess.pid) + for child in topSubProc.children(recursive=True): # or parent.children() for recursive=False + child.send_signal(self.STOP_SIGNAL) + time.sleep(0.2) # 200 ms + # topSubProc.send_signal(sig) # now kill the main sub process (likely the Shell) + + self.subProcess.send_signal(self.STOP_SIGNAL) # main sub process (likely the Shell) self.subProcess.wait(20) retCode = self.subProcess.returncode # should always be there # May throw subprocess.TimeoutExpired exception above, therefore # The process is guranteed to have ended by now self.subProcess = None if retCode != 0: # != (- signal.SIGINT): - Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format(sig, retCode)) + Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format( + self.STOP_SIGNAL, retCode)) else: - Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(sig)) + Logging.info("TSP.stop(): sub proc successfully terminated with SIG {}".format(self.STOP_SIGNAL)) return - retCode class ServiceManager: @@ -439,7 +473,7 @@ class ServiceManager: time.sleep(self.PAUSE_BETWEEN_IPC_CHECK) # pause, before next round # raise CrashGenError("dummy") - print("Service Manager Thread (with subprocess) ended, main thread exiting...") + Logging.info("Service Manager Thread (with subprocess) ended, main thread exiting...") def _getFirstInstance(self): return self._tInsts[0] @@ -452,7 +486,7 @@ class ServiceManager: # Find if there's already a taosd service, and then kill it for proc in psutil.process_iter(): if proc.name() == 'taosd': - print("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt") + Logging.info("Killing an existing TAOSD process in 2 seconds... press CTRL-C to interrupt") time.sleep(2.0) proc.kill() # print("Process: {}".format(proc.name())) @@ -559,7 +593,8 @@ class ServiceManagerThread: for i in range(0, 100): time.sleep(1.0) # self.procIpcBatch() # don't pump message during start up - print("_zz_", end="", flush=True) + Progress.emit(Progress.SERVICE_START_NAP) + # print("_zz_", end="", flush=True) if self._status.isRunning(): Logging.info("[] TDengine service READY to process requests") Logging.info("[] TAOS service started: {}".format(self)) @@ -595,12 +630,12 @@ class ServiceManagerThread: def stop(self): # can be called from both main thread or signal handler - print("Terminating TDengine service running as the sub process...") + Logging.info("Terminating TDengine service running as the sub process...") if self.getStatus().isStopped(): - print("Service already stopped") + Logging.info("Service already stopped") return if self.getStatus().isStopping(): - print("Service is already being stopped") + Logging.info("Service is already being stopped") return # Linux will send Control-C generated SIGINT to the TDengine process # already, ref: @@ -616,10 +651,10 @@ class ServiceManagerThread: if retCode == signal.SIGSEGV : # SGV Logging.error("[[--ERROR--]]: TDengine service SEGV fault (check core file!)") except subprocess.TimeoutExpired as err: - print("Time out waiting for TDengine service process to exit") + Logging.info("Time out waiting for TDengine service process to exit") else: if self._tdeSubProcess.isRunning(): # still running, should now never happen - print("FAILED to stop sub process, it is still running... pid = {}".format( + Logging.error("FAILED to stop sub process, it is still running... pid = {}".format( self._tdeSubProcess.getPid())) else: self._tdeSubProcess = None # not running any more @@ -683,9 +718,9 @@ class ServiceManagerThread: return # we are done with THIS BATCH else: # got line, printing out if forceOutput: - Logging.info(line) + Logging.info('[TAOSD] ' + line) else: - Logging.debug(line) + Logging.debug('[TAOSD] ' + line) print(">", end="", flush=True) _ProgressBars = ["--", "//", "||", "\\\\"] @@ -728,11 +763,11 @@ class ServiceManagerThread: # queue.put(line) # meaning sub process must have died - Logging.info("\nEnd of stream detected for TDengine STDOUT: {}".format(self)) + Logging.info("EOF for TDengine STDOUT: {}".format(self)) out.close() def svcErrorReader(self, err: IO, queue): for line in iter(err.readline, b''): - print("\nTDengine Service (taosd) ERROR (from stderr): {}".format(line)) - Logging.info("\nEnd of stream detected for TDengine STDERR: {}".format(self)) + Logging.info("TDengine STDERR: {}".format(line)) + Logging.info("EOF for TDengine STDERR: {}".format(self)) err.close() \ No newline at end of file diff --git a/tests/pytest/crash_gen_bootstrap.py b/tests/pytest/crash_gen_bootstrap.py index a3417d21a8..fd12284b9d 100644 --- a/tests/pytest/crash_gen_bootstrap.py +++ b/tests/pytest/crash_gen_bootstrap.py @@ -11,7 +11,7 @@ ################################################################### import sys -from crash_gen.crash_gen import MainExec +from crash_gen.crash_gen_main import MainExec if __name__ == "__main__": From faa8ff7cc24740859f231d1a7ba29f594d7abc14 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 5 Nov 2020 03:43:26 +0000 Subject: [PATCH 43/58] [TD-1937]:client crash on percentile --- src/client/src/tscLocalMerge.c | 3 +++ src/query/src/qHistogram.c | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index c67edf5b5a..f49c4da0cb 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -97,6 +97,9 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->param[2].i64Key = pQueryInfo->order.order; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; pCtx->param[1].i64Key = pQueryInfo->order.orderColId; + } else if (functionId == TSDB_FUNC_APERCT) { + pCtx->param[0].i64Key = pExpr->param[0].i64Key; + pCtx->param[0].nType = pExpr->param[0].nType; } pCtx->interBufBytes = pExpr->interBytes; diff --git a/src/query/src/qHistogram.c b/src/query/src/qHistogram.c index 703ee2c521..35e5906d1f 100644 --- a/src/query/src/qHistogram.c +++ b/src/query/src/qHistogram.c @@ -168,7 +168,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { (*pHisto)->numOfEntries += 1; } } else { /* insert a new slot */ - if ((*pHisto)->numOfElems > 1 && idx < (*pHisto)->numOfEntries) { + if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) { if (idx > 0) { assert((*pHisto)->elems[idx - 1].val <= val); } @@ -661,4 +661,4 @@ SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2 free(pHistoBins); return pResHistogram; -} \ No newline at end of file +} From fdaf1e470ce98293ceec682b6c251343066c1003 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 5 Nov 2020 17:18:01 +0800 Subject: [PATCH 44/58] [TD-1954]: a inport/export tool for logical backup --- packaging/deb/makedeb.sh | 1 + packaging/rpm/tdengine.spec | 2 ++ packaging/tools/makeclient.sh | 2 +- packaging/tools/makepkg.sh | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index a7bb22f345..edc7de9692 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -48,6 +48,7 @@ cp ${compile_dir}/../packaging/deb/taosd ${pkg_dir}${install_home_pat cp ${compile_dir}/../packaging/tools/post.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/tools/preun.sh ${pkg_dir}${install_home_path}/script cp ${compile_dir}/build/bin/taosdemo ${pkg_dir}${install_home_path}/bin +cp ${compile_dir}/build/bin/taosdump ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taosd ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/bin/taos ${pkg_dir}${install_home_path}/bin cp ${compile_dir}/build/lib/${libfile} ${pkg_dir}${install_home_path}/driver diff --git a/packaging/rpm/tdengine.spec b/packaging/rpm/tdengine.spec index 54c3c9b279..afec1eaf9a 100644 --- a/packaging/rpm/tdengine.spec +++ b/packaging/rpm/tdengine.spec @@ -58,6 +58,7 @@ cp %{_compiledir}/../packaging/tools/preun.sh %{buildroot}%{homepath}/scri cp %{_compiledir}/build/bin/taos %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosd %{buildroot}%{homepath}/bin cp %{_compiledir}/build/bin/taosdemo %{buildroot}%{homepath}/bin +cp %{_compiledir}/build/bin/taosdump %{buildroot}%{homepath}/bin cp %{_compiledir}/build/lib/${libfile} %{buildroot}%{homepath}/driver cp %{_compiledir}/../src/inc/taos.h %{buildroot}%{homepath}/include cp %{_compiledir}/../src/inc/taoserror.h %{buildroot}%{homepath}/include @@ -134,6 +135,7 @@ if [ $1 -eq 0 ];then ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${cfg_link_dir}/* || : ${csudo} rm -f ${inc_link_dir}/taos.h || : ${csudo} rm -f ${inc_link_dir}/taoserror.h || : diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index 69fa53c087..83a9cb1ced 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -45,7 +45,7 @@ if [ "$osType" != "Darwin" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taos ${script_dir}/remove_client.sh" else - bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${script_dir}/remove_client.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" else diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index a6d868ed1d..00a92cb063 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -36,7 +36,7 @@ if [ "$pagMode" == "lite" ]; then strip ${build_dir}/bin/taos bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${script_dir}/remove.sh" else - bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" + bin_files="${build_dir}/bin/taosd ${build_dir}/bin/taos ${build_dir}/bin/taosdump ${build_dir}/bin/taosdemo ${build_dir}/bin/tarbitrator ${script_dir}/remove.sh ${script_dir}/set_core.sh" fi lib_files="${build_dir}/lib/libtaos.so.${version}" From a9593a44727b775bcb9f50a9bde188659a6610d2 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 5 Nov 2020 18:07:26 +0800 Subject: [PATCH 45/58] [TD-1939] add test case for TD-1939 --- tests/pytest/query/queryLike.py | 45 +++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/pytest/query/queryLike.py diff --git a/tests/pytest/query/queryLike.py b/tests/pytest/query/queryLike.py new file mode 100644 index 0000000000..aa943c4f1f --- /dev/null +++ b/tests/pytest/query/queryLike.py @@ -0,0 +1,45 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.prepare() + + tdSql.execute("create table cars(ts timestamp, c nchar(2)) tags(t1 nchar(2))") + tdSql.execute("insert into car0 using cars tags('aa') values(now, 'bb');") + tdSql.query("select count(*) from cars where t1 like '%50 90 30 04 00 00%'") + tdSql.checkRows(0) + + tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))") + tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');") + tdSql.query("select count(*) from test_cars where t1 like '%50 90 30 04 00 00%'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 5ba5336756c6130ee9df1494b648517e96dd90fb Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 5 Nov 2020 18:38:23 +0800 Subject: [PATCH 46/58] update Jenkinksfile --- Jenkinsfile | 1 + tests/pytest/query/queryLike.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 8bf7e435fd..834321240f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { cd debug cmake .. > /dev/null make > /dev/null + make install > /dev/null cd ${WKC}/tests #./test-all.sh smoke ./test-all.sh pytest diff --git a/tests/pytest/query/queryLike.py b/tests/pytest/query/queryLike.py index aa943c4f1f..3c3b030f8f 100644 --- a/tests/pytest/query/queryLike.py +++ b/tests/pytest/query/queryLike.py @@ -33,7 +33,7 @@ class TDTestCase: tdSql.execute("create table test_cars(ts timestamp, c nchar(2)) tags(t1 nchar(20))") tdSql.execute("insert into car1 using test_cars tags('150 90 30 04 00 002') values(now, 'bb');") - tdSql.query("select count(*) from test_cars where t1 like '%50 90 30 04 00 00%'") + tdSql.query("select * from test_cars where t1 like '%50 90 30 04 00 00%'") tdSql.checkRows(1) def stop(self): From 6ca2968862967cf55357e604cb7295c8d2809d48 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Thu, 5 Nov 2020 10:58:28 +0000 Subject: [PATCH 47/58] TD-1944 --- src/rpc/src/rpcMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index d01c34c810..d89d3e275d 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1622,7 +1622,7 @@ static void rpcDecRef(SRpcInfo *pRpc) int count = atomic_sub_fetch_32(&tsRpcNum, 1); if (count == 0) { - taosCloseRef(tsRpcRefId); + // taosCloseRef(tsRpcRefId); // tsRpcInit = PTHREAD_ONCE_INIT; // windows compliling error } } From 16219bc630431f55e5565b9d6147592d244a4573 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Nov 2020 19:15:20 +0800 Subject: [PATCH 48/58] [TD-225] --- src/client/src/tscSubquery.c | 4 ++++ src/util/src/tcompare.c | 9 ++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 9905de279e..fd56001174 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -853,11 +853,15 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow } if (taosArrayGetSize(s1) == 0 || taosArrayGetSize(s2) == 0) { // no results,return. + assert(pParentSql->fp != tscJoinQueryCallback); + tscDebug("%p tag intersect does not generated qualified tables for join, free all sub SqlObj and quit", pParentSql); freeJoinSubqueryObj(pParentSql); // set no result command pParentSql->cmd.command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; + assert(pParentSql->fp != tscJoinQueryCallback); + (*pParentSql->fp)(pParentSql->param, pParentSql, 0); } else { // proceed to for ts_comp query diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index ba711ced8f..1090ab8101 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -367,7 +367,14 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_BOOL: DEFAULT_COMP(GET_INT8_VAL(f1), GET_INT8_VAL(f2)); case TSDB_DATA_TYPE_NCHAR: { - int32_t ret = wcsncmp((wchar_t*) f1, (wchar_t*) f2, size/TSDB_NCHAR_SIZE); + tstr* t1 = (tstr*) f1; + tstr* t2 = (tstr*) f2; + + if (t1->len != t2->len) { + return t1->len > t2->len? 1:-1; + } + + int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE); if (ret == 0) { return ret; } From ea374b6f868ef2cfe6efa51238bafadb03042359 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 5 Nov 2020 11:34:40 +0000 Subject: [PATCH 49/58] TD-1915 --- src/dnode/src/dnodeVRead.c | 48 +++++++------------ src/inc/vnode.h | 17 ++++--- src/vnode/src/vnodeMain.c | 15 ------ src/vnode/src/vnodeRead.c | 97 +++++++++++++++++++++++++------------- src/vnode/src/vnodeWrite.c | 2 +- 5 files changed, 94 insertions(+), 85 deletions(-) diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index f571cfda9f..34df11adcc 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -92,33 +92,23 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) { pHead->vgId = htonl(pHead->vgId); pHead->contLen = htonl(pHead->contLen); - taos_queue queue = vnodeAcquireRqueue(pHead->vgId); - - if (queue == NULL) { - leftLen -= pHead->contLen; - pCont -= pHead->contLen; - continue; + void *pVnode = vnodeAcquire(pHead->vgId); + if (pVnode != NULL) { + int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg); + if (code == TSDB_CODE_SUCCESS) queuedMsgNum++; + vnodeRelease(pVnode); } - // put message into queue - SVReadMsg *pRead = taosAllocateQitem(sizeof(SVReadMsg)); - pRead->rpcMsg = *pMsg; - pRead->pCont = pCont; - pRead->contLen = pHead->contLen; - - // next vnode leftLen -= pHead->contLen; pCont -= pHead->contLen; - queuedMsgNum++; - - taosWriteQitem(queue, TAOS_QTYPE_RPC, pRead); } if (queuedMsgNum == 0) { SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID}; rpcSendResponse(&rpcRsp); - rpcFreeCont(pMsg->pCont); } + + rpcFreeCont(pMsg->pCont); } void *dnodeAllocVReadQueue(void *pVnode) { @@ -162,50 +152,48 @@ void dnodeFreeVReadQueue(void *rqueue) { void dnodeSendRpcVReadRsp(void *pVnode, SVReadMsg *pRead, int32_t code) { SRpcMsg rpcRsp = { - .handle = pRead->rpcMsg.handle, + .handle = pRead->rpcHandle, .pCont = pRead->rspRet.rsp, .contLen = pRead->rspRet.len, .code = code, }; rpcSendResponse(&rpcRsp); - rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); } void dnodeDispatchNonRspMsg(void *pVnode, SVReadMsg *pRead, int32_t code) { - rpcFreeCont(pRead->rpcMsg.pCont); vnodeRelease(pVnode); } static void *dnodeProcessReadQueue(void *param) { - SVReadMsg *pReadMsg; + SVReadMsg *pRead; int32_t qtype; void * pVnode; while (1) { - if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pReadMsg, &pVnode) == 0) { + if (taosReadQitemFromQset(tsVReadQset, &qtype, (void **)&pRead, &pVnode) == 0) { dDebug("qset:%p dnode vread got no message from qset, exiting", tsVReadQset); break; } - dDebug("%p, msg:%s will be processed in vread queue, qtype:%d, msg:%p", pReadMsg->rpcMsg.ahandle, - taosMsg[pReadMsg->rpcMsg.msgType], qtype, pReadMsg); + dDebug("%p, msg:%p:%s will be processed in vread queue, qtype:%d", pRead->rpcAhandle, pRead, + taosMsg[pRead->msgType], qtype); - int32_t code = vnodeProcessRead(pVnode, pReadMsg); + int32_t code = vnodeProcessRead(pVnode, pRead); if (qtype == TAOS_QTYPE_RPC && code != TSDB_CODE_QRY_NOT_READY) { - dnodeSendRpcVReadRsp(pVnode, pReadMsg, code); + dnodeSendRpcVReadRsp(pVnode, pRead, code); } else { if (code == TSDB_CODE_QRY_HAS_RSP) { - dnodeSendRpcVReadRsp(pVnode, pReadMsg, pReadMsg->rpcMsg.code); + dnodeSendRpcVReadRsp(pVnode, pRead, pRead->code); } else { // code == TSDB_CODE_QRY_NOT_READY, do not return msg to client - assert(pReadMsg->rpcMsg.handle == NULL || (pReadMsg->rpcMsg.handle != NULL && pReadMsg->rpcMsg.msgType == 5)); - dnodeDispatchNonRspMsg(pVnode, pReadMsg, code); + assert(pRead->rpcHandle == NULL || (pRead->rpcHandle != NULL && pRead->msgType == 5)); + dnodeDispatchNonRspMsg(pVnode, pRead, code); } } - taosFreeQitem(pReadMsg); + taosFreeQitem(pRead); } return NULL; diff --git a/src/inc/vnode.h b/src/inc/vnode.h index e77bec926a..018e96e193 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -37,10 +37,15 @@ typedef struct { } SRspRet; typedef struct { - SRspRet rspRet; - void * pCont; + int32_t code; int32_t contLen; - SRpcMsg rpcMsg; + void * rpcHandle; + void * rpcAhandle; + void * qhandle; + int8_t qtype; + int8_t msgType; + SRspRet rspRet; + char pCont[]; } SVReadMsg; typedef struct { @@ -62,13 +67,11 @@ int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg); int32_t vnodeClose(int32_t vgId); void* vnodeAcquire(int32_t vgId); // add refcount -void* vnodeAcquireRqueue(int32_t vgId); // add refCount, get read queue void vnodeRelease(void *pVnode); // dec refCount void* vnodeGetWal(void *pVnode); int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam); int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam); -int32_t vnodeCheckWrite(void *pVnode); int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes); void vnodeBuildStatusMsg(void *param); void vnodeConfirmForward(void *param, uint64_t version, int32_t code); @@ -77,8 +80,8 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes); int32_t vnodeInitResources(); void vnodeCleanupResources(); -int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pReadMsg); -int32_t vnodeCheckRead(void *pVnode); +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam); +int32_t vnodeProcessRead(void *pVnode, SVReadMsg *pRead); #ifdef __cplusplus } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 2dcdba5d7c..128da72623 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -468,21 +468,6 @@ void *vnodeAcquire(int32_t vgId) { return *ppVnode; } -void *vnodeAcquireRqueue(int32_t vgId) { - SVnodeObj *pVnode = vnodeAcquire(vgId); - if (pVnode == NULL) return NULL; - - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) { - terrno = code; - vInfo("vgId:%d, can not provide read service, status is %s", vgId, vnodeStatus[pVnode->status]); - vnodeRelease(pVnode); - return NULL; - } - - return pVnode->rqueue; -} - void *vnodeGetWal(void *pVnode) { return ((SVnodeObj *)pVnode)->wal; } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index faa35b0e02..fb984f6750 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -46,7 +46,7 @@ void vnodeInitReadFp(void) { // int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) { SVnodeObj *pVnode = (SVnodeObj *)param; - int msgType = pReadMsg->rpcMsg.msgType; + int32_t msgType = pReadMsg->msgType; if (vnodeProcessReadMsgFp[msgType] == NULL) { vDebug("vgId:%d, msgType:%s not processed, no handle", pVnode->vgId, taosMsg[msgType]); @@ -56,7 +56,7 @@ int32_t vnodeProcessRead(void *param, SVReadMsg *pReadMsg) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pReadMsg); } -int32_t vnodeCheckRead(void *param) { +static int32_t vnodeCheckRead(void *param) { SVnodeObj *pVnode = param; if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, vnode status is %s, recCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], @@ -78,24 +78,58 @@ int32_t vnodeCheckRead(void *param) { return TSDB_CODE_SUCCESS; } -static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { - int32_t code = vnodeCheckRead(pVnode); - if (code != TSDB_CODE_SUCCESS) return code; - SVReadMsg *pRead = (SVReadMsg *)taosAllocateQitem(sizeof(SVReadMsg)); - pRead->rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; - pRead->pCont = qhandle; - pRead->contLen = 0; - pRead->rpcMsg.ahandle = ahandle; +int32_t vnodeWriteToRQueue(void *vparam, void *pCont, int32_t contLen, int8_t qtype, void *rparam) { + SVnodeObj *pVnode = vparam; + + if (qtype == TAOS_QTYPE_RPC || qtype == TAOS_QTYPE_QUERY) { + int32_t code = vnodeCheckRead(pVnode); + if (code != TSDB_CODE_SUCCESS) return code; + } + + int32_t size = sizeof(SVReadMsg) + contLen; + SVReadMsg *pRead = taosAllocateQitem(size); + if (pRead == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + if (rparam != NULL) { + SRpcMsg *pRpcMsg = rparam; + pRead->rpcHandle = pRpcMsg->handle; + pRead->rpcAhandle = pRpcMsg->ahandle; + pRead->msgType = pRpcMsg->msgType; + pRead->code = pRpcMsg->code; + } + + if (contLen != 0) { + pRead->contLen = contLen; + memcpy(pRead->pCont, pCont, contLen); + } else { + pRead->qhandle = pCont; + } + + pRead->qtype = qtype; atomic_add_fetch_32(&pVnode->refCount, 1); + vTrace("vgId:%d, get vnode rqueue, refCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); - vDebug("QInfo:%p add to vread queue for exec query, msg:%p", *qhandle, pRead); - taosWriteQitem(pVnode->rqueue, TAOS_QTYPE_QUERY, pRead); - + taosWriteQitem(pVnode->rqueue, qtype, pRead); return TSDB_CODE_SUCCESS; } +static int32_t vnodePutItemIntoReadQueue(SVnodeObj *pVnode, void **qhandle, void *ahandle) { + SRpcMsg rpcMsg = {0}; + rpcMsg.msgType = TSDB_MSG_TYPE_QUERY; + rpcMsg.ahandle = ahandle; + + int32_t code = vnodeWriteToRQueue(pVnode, qhandle, 0, TAOS_QTYPE_QUERY, &rpcMsg); + if (code == TSDB_CODE_SUCCESS) { + vDebug("QInfo:%p add to vread queue for exec query", *qhandle); + } + + return code; +} + /** * * @param pRet response message object @@ -155,18 +189,18 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { memset(pRet, 0, sizeof(SRspRet)); // qHandle needs to be freed correctly - if (pReadMsg->rpcMsg.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (pReadMsg->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { SRetrieveTableMsg *killQueryMsg = (SRetrieveTableMsg *)pReadMsg->pCont; killQueryMsg->free = htons(killQueryMsg->free); killQueryMsg->qhandle = htobe64(killQueryMsg->qhandle); - vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcMsg.handle); - assert(pReadMsg->rpcMsg.contLen > 0 && killQueryMsg->free == 1); + vWarn("QInfo:%p connection %p broken, kill query", (void *)killQueryMsg->qhandle, pReadMsg->rpcHandle); + assert(pReadMsg->contLen > 0 && killQueryMsg->free == 1); void **qhandle = qAcquireQInfo(pVnode->qMgmt, (uint64_t)killQueryMsg->qhandle); if (qhandle == NULL || *qhandle == NULL) { vWarn("QInfo:%p invalid qhandle, no matched query handle, conn:%p", (void *)killQueryMsg->qhandle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); } else { assert(*qhandle == (void *)killQueryMsg->qhandle); @@ -208,9 +242,9 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } if (handle != NULL && - vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { vError("vgId:%d, QInfo:%p, query discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); pRsp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); return pRsp->code; @@ -221,7 +255,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { if (handle != NULL) { vDebug("vgId:%d, QInfo:%p, dnode query msg disposed, create qhandle and returns to app", vgId, *handle); - code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcMsg.ahandle); + code = vnodePutItemIntoReadQueue(pVnode, handle, pReadMsg->rpcHandle); if (code != TSDB_CODE_SUCCESS) { pRsp->code = code; qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -230,7 +264,7 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } } else { assert(pCont != NULL); - void **qhandle = (void **)pCont; + void **qhandle = (void **)pReadMsg->qhandle; vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); @@ -242,14 +276,14 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { // build query rsp, the retrieve request has reached here already if (buildRes) { // update the connection info according to the retrieve connection - pReadMsg->rpcMsg.handle = qGetResultRetrieveMsg(*qhandle); - assert(pReadMsg->rpcMsg.handle != NULL); + pReadMsg->rpcHandle = qGetResultRetrieveMsg(*qhandle); + assert(pReadMsg->rpcHandle != NULL); vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, - pReadMsg->rpcMsg.handle); + pReadMsg->rpcHandle); // set the real rsp error code - pReadMsg->rpcMsg.code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcMsg.ahandle); + pReadMsg->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pReadMsg->rpcHandle); // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client code = TSDB_CODE_QRY_HAS_RSP; @@ -283,7 +317,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { pRetrieve->qhandle = htobe64(pRetrieve->qhandle); vDebug("vgId:%d, QInfo:%p, retrieve msg is disposed, free:%d, conn:%p", pVnode->vgId, (void *)pRetrieve->qhandle, - pRetrieve->free, pReadMsg->rpcMsg.handle); + pRetrieve->free, pReadMsg->rpcHandle); memset(pRet, 0, sizeof(SRspRet)); @@ -314,9 +348,8 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } // register the qhandle to connect to quit query immediate if connection is broken - if (vnodeNotifyCurrentQhandle(pReadMsg->rpcMsg.handle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { - vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, - pReadMsg->rpcMsg.handle); + if (vnodeNotifyCurrentQhandle(pReadMsg->rpcHandle, *handle, pVnode->vgId) != TSDB_CODE_SUCCESS) { + vError("vgId:%d, QInfo:%p, retrieve discarded since link is broken, %p", pVnode->vgId, *handle, pReadMsg->rpcHandle); code = TSDB_CODE_RPC_NETWORK_UNAVAIL; qKillQuery(*handle); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, true); @@ -326,7 +359,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { bool freeHandle = true; bool buildRes = false; - code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcMsg.handle); + code = qRetrieveQueryResultInfo(*handle, &buildRes, pReadMsg->rpcHandle); if (code != TSDB_CODE_SUCCESS) { // TODO handle malloc failure pRet->rsp = (SRetrieveTableRsp *)rpcMallocCont(sizeof(SRetrieveTableRsp)); @@ -337,7 +370,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { assert(buildRes == true); #if _NON_BLOCKING_RETRIEVE if (!buildRes) { - assert(pReadMsg->rpcMsg.handle != NULL); + assert(pReadMsg->rpcHandle != NULL); qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); return TSDB_CODE_QRY_NOT_READY; @@ -345,7 +378,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { #endif // ahandle is the sqlObj pointer - code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcMsg.ahandle); + code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pReadMsg->rpcHandle); } // If qhandle is not added into vread queue, the query should be completed already or paused with error. diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 80c68b0917..3caee2fb0c 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -97,7 +97,7 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara return syncCode; } -int32_t vnodeCheckWrite(void *param) { +static int32_t vnodeCheckWrite(void *param) { SVnodeObj *pVnode = param; if (!(pVnode->accessState & TSDB_VN_WRITE_ACCCESS)) { vDebug("vgId:%d, no write auth, recCount:%d pVnode:%p", pVnode->vgId, pVnode->refCount, pVnode); From b6d40d3ebff37a5d50cb24e0595d7c4ef8254c1d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Nov 2020 22:27:01 +0800 Subject: [PATCH 50/58] [TD-1941]: fix client crash in join query while connection broken. --- src/client/src/tscAsync.c | 16 +++++++++++++++- src/client/src/tscParseInsert.c | 2 +- src/client/src/tscSubquery.c | 2 +- src/vnode/src/vnodeRead.c | 2 +- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index e70cd11fbe..99c03c6580 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -428,6 +428,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { } else { assert(code == TSDB_CODE_SUCCESS); } + // param already freed by other routine and pSql in tscCache when ctrl + c if (atomic_load_ptr(&pSql->param) == NULL) { return; @@ -441,6 +442,20 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { assert(pParObj->signature == pParObj && trs->subqueryIndex == pTableMetaInfo->vgroupIndex && pTableMetaInfo->vgroupIndex >= 0 && pTableMetaInfo->vgroupList != NULL); + // tscProcessSql can add error into async res + tscProcessSql(pSql); + return; + } else if (TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY)) { + tscDebug("%p update table meta in local cache, continue to process sql and send corresponding tid_tag query", pSql); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + code = tscGetTableMeta(pSql, pTableMetaInfo); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + return; + } else { + assert(code == TSDB_CODE_SUCCESS); + } + + assert((tscGetNumOfTags(pTableMetaInfo->pTableMeta) != 0)); // tscProcessSql can add error into async res tscProcessSql(pSql); return; @@ -465,7 +480,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscResetSqlCmdObj(pCmd, false); code = tsParseSql(pSql, true); - if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { return; } else if (code != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 5e65126ef6..f960beb5d9 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1309,7 +1309,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) { if ((!pCmd->parseFinished) && (!initial)) { tscDebug("%p resume to parse sql: %s", pSql, pCmd->curSql); } - + ret = tscAllocPayload(&pSql->cmd, TSDB_DEFAULT_PAYLOAD_SIZE); if (TSDB_CODE_SUCCESS != ret) { return ret; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index fd56001174..4b6dde902c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -523,7 +523,7 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { assert(pSqlObj->subState.numOfRemain > 0); if (atomic_sub_fetch_32(&pSqlObj->subState.numOfRemain, 1) <= 0) { - tscError("%p all subquery return and query failed, global code:%d", pSqlObj, pSqlObj->res.code); + tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code)); freeJoinSubqueryObj(pSqlObj); } } diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index faa35b0e02..dd38a2d9c5 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -298,7 +298,7 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pReadMsg) { } if (code != TSDB_CODE_SUCCESS) { - vDebug("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle); + vError("vgId:%d, invalid handle in retrieving result, code:0x%08x, QInfo:%p", pVnode->vgId, code, (void *)pRetrieve->qhandle); vnodeBuildNoResultQueryRsp(pRet); return code; } From bd87ba990ca74daccaa2a3ae21b607e70e7e7e28 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 5 Nov 2020 23:08:42 +0800 Subject: [PATCH 51/58] [TD-225] fix memory leak. --- src/query/src/qAst.c | 1 - src/query/src/qExecutor.c | 3 ++- src/query/src/qFill.c | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 847a8b8eb6..bc2866fb6f 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -32,7 +32,6 @@ #include "tstoken.h" #include "ttokendef.h" #include "tulog.h" -#include "tutil.h" /* * diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b365dc4283..b992a037ed 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1609,7 +1609,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); - pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);//calloc(1, sizeof(SResultRow)); + pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool); if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { goto _clean; @@ -1745,6 +1745,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); taosTFree(pRuntimeEnv->keyBuf); + taosTFree(pRuntimeEnv->rowCellInfoOffset); taosHashCleanup(pRuntimeEnv->pResultRowHashTable); pRuntimeEnv->pResultRowHashTable = NULL; diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index a219bd6abd..788779b2bb 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -170,7 +170,7 @@ int64_t getFilledNumOfRes(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows int32_t numOfRows = taosNumOfRemainRows(pFillInfo); TSKEY ekey1 = ekey; - if (pFillInfo->order != TSDB_ORDER_ASC) { + if (!FILL_IS_ASC_FILL(pFillInfo)) { pFillInfo->endKey = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision); } From 0d57114b776dd077999dbf10129f86ecd2ab6bb0 Mon Sep 17 00:00:00 2001 From: Steven Li Date: Thu, 5 Nov 2020 22:23:14 +0000 Subject: [PATCH 52/58] Minor refactoring of crash_gen tool, now doing multi-record insertion some times --- tests/pytest/crash_gen/crash_gen_main.py | 167 ++++++++++++----------- 1 file changed, 91 insertions(+), 76 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index fee51b98c0..8a074500e1 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -388,9 +388,9 @@ class ThreadCoordinator: self._syncAtBarrier() # For now just cross the barrier Progress.emit(Progress.END_THREAD_STEP) except threading.BrokenBarrierError as err: - Logging.info("Main loop aborted, caused by worker thread time-out") + Logging.info("Main loop aborted, caused by worker thread(s) time-out") self._execStats.registerFailure("Aborted due to worker thread timeout") - print("\n\nWorker Thread time-out detected, important thread info:") + print("\n\nWorker Thread time-out detected, TAOS related threads are:") ts = ThreadStacks() ts.print(filterInternal=True) workerTimeout = True @@ -1242,6 +1242,7 @@ class Task(): 0x0B, # Unable to establish connection, more details in TD-1648 0x200, # invalid SQL， TODO: re-examine with TD-934 0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776 + 0x213, # "Disconnected from service", result of "kill connection ???" 0x217, # "db not selected", client side defined error code # 0x218, # "Table does not exist" client side defined error code 0x360, # Table already exists @@ -1911,13 +1912,88 @@ class TaskAddData(StateTransitionTask): def canBeginFrom(cls, state: AnyState): return state.canAddData() + def _addDataInBatch(self, db, dbc, regTableName, te: TaskExecutor): + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + fullTableName = db.getName() + '.' + regTableName + + sql = "insert into {} values ".format(fullTableName) + for j in range(numRecords): # number of records per table + nextInt = db.getNextInt() + nextTick = db.getNextTick() + sql += "('{}', {});".format(nextTick, nextInt) + dbc.execute(sql) + + def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + + for j in range(numRecords): # number of records per table + nextInt = db.getNextInt() + nextTick = db.getNextTick() + if gConfig.record_ops: + self.prepToRecordOps() + self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) + self.fAddLogReady.flush() + os.fsync(self.fAddLogReady) + + # TODO: too ugly trying to lock the table reliably, refactor... + fullTableName = db.getName() + '.' + regTableName + if gConfig.verify_data: + self.lockTable(fullTableName) + # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written + + try: + sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {}) + fullTableName, + # ds.getFixedSuperTableName(), + # ds.getNextBinary(), ds.getNextFloat(), + nextTick, nextInt) + dbc.execute(sql) + except: # Any exception at all + if gConfig.verify_data: + self.unlockTable(fullTableName) + raise + + # Now read it back and verify, we might encounter an error if table is dropped + if gConfig.verify_data: # only if command line asks for it + try: + readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". + format(db.getName(), regTableName, nextTick)) + if readBack != nextInt : + raise taos.error.ProgrammingError( + "Failed to read back same data, wrote: {}, read: {}" + .format(nextInt, readBack), 0x999) + except taos.error.ProgrammingError as err: + errno = Helper.convertErrno(err.errno) + if errno in [0x991, 0x992] : # not a single result + raise taos.error.ProgrammingError( + "Failed to read back same data for tick: {}, wrote: {}, read: {}" + .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), + errno) + elif errno in [0x218, 0x362]: # table doesn't exist + # do nothing + dummy = 0 + else: + # Re-throw otherwise + raise + finally: + self.unlockTable(fullTableName) # Unlock the table no matter what + + # Successfully wrote the data into the DB, let's record it somehow + te.recordDataMark(nextInt) + + if gConfig.record_ops: + self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) + self.fAddLogDone.flush() + os.fsync(self.fAddLogDone) + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): # ds = self._dbManager # Quite DANGEROUS here, may result in multi-thread client access db = self._db dbc = wt.getDbConn() - tblSeq = list(range( - self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES)) - random.shuffle(tblSeq) + numTables = self.LARGE_NUMBER_OF_TABLES if gConfig.larger_data else self.SMALL_NUMBER_OF_TABLES + numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS + tblSeq = list(range(numTables )) + random.shuffle(tblSeq) # now we have random sequence for i in tblSeq: if (i in self.activeTable): # wow already active print("x", end="", flush=True) # concurrent insertion @@ -1925,82 +2001,20 @@ class TaskAddData(StateTransitionTask): self.activeTable.add(i) # marking it active sTable = db.getFixedSuperTable() - regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) - - + regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) fullTableName = db.getName() + '.' + regTableName # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists # self._unlockTable(fullTableName) - for j in range(self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS): # number of records per table - nextInt = db.getNextInt() - nextTick = db.getNextTick() - if gConfig.record_ops: - self.prepToRecordOps() - self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) - self.fAddLogReady.flush() - os.fsync(self.fAddLogReady) - - # TODO: too ugly trying to lock the table reliably, refactor... - fullTableName = db.getName() + '.' + regTableName - if gConfig.verify_data: - self.lockTable(fullTableName) - # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written - - try: - sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {}) - fullTableName, - # ds.getFixedSuperTableName(), - # ds.getNextBinary(), ds.getNextFloat(), - nextTick, nextInt) - dbc.execute(sql) - except: # Any exception at all - if gConfig.verify_data: - self.unlockTable(fullTableName) - raise - - # Now read it back and verify, we might encounter an error if table is dropped - if gConfig.verify_data: # only if command line asks for it - try: - readBack = dbc.queryScalar("SELECT speed from {}.{} WHERE ts='{}'". - format(db.getName(), regTableName, nextTick)) - if readBack != nextInt : - raise taos.error.ProgrammingError( - "Failed to read back same data, wrote: {}, read: {}" - .format(nextInt, readBack), 0x999) - except taos.error.ProgrammingError as err: - errno = Helper.convertErrno(err.errno) - if errno in [0x991, 0x992] : # not a single result - raise taos.error.ProgrammingError( - "Failed to read back same data for tick: {}, wrote: {}, read: {}" - .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), - errno) - elif errno in [0x218, 0x362]: # table doesn't exist - # do nothing - dummy = 0 - else: - # Re-throw otherwise - raise - finally: - self.unlockTable(fullTableName) # Unlock the table no matter what - - # Successfully wrote the data into the DB, let's record it somehow - te.recordDataMark(nextInt) - - if gConfig.record_ops: - self.fAddLogDone.write("Wrote {} to {}\n".format(nextInt, regTableName)) - self.fAddLogDone.flush() - os.fsync(self.fAddLogDone) - - + if Dice.throw(1) == 0: # 1 in 2 chance + self._addData(db, dbc, regTableName, te) + else: + self._addDataInBatch(db, dbc, regTableName, te) self.activeTable.discard(i) # not raising an error, unlike remove - - - class ThreadStacks: # stack info for all threads def __init__(self): self._allStacks = {} @@ -2022,13 +2036,14 @@ class ThreadStacks: # stack info for all threads '__init__']: # the thread that extracted the stack continue # ignore # Now print - print("\n<----- Thread Info for ID: {}".format(thNid)) + print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid)) + stackFrame = 0 for frame in stack: # print(frame) - print("File {filename}, line {lineno}, in {name}".format( - filename=frame.filename, lineno=frame.lineno, name=frame.name)) + print("[{sf}] File {filename}, line {lineno}, in {name}".format( + sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name)) print(" {}".format(frame.line)) - print("-----> End of Thread Info\n") + print("-----> End of Thread Info ----->\n") class ClientManager: def __init__(self): From d553f4083a5e46936f426cc4decc4a42dc34ed92 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 6 Nov 2020 09:30:40 +0800 Subject: [PATCH 53/58] [TD-1961] modify return when call taos_query() --- tests/examples/c/asyncdemo.c | 41 +++++++++++++++++++++++++++++------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/tests/examples/c/asyncdemo.c b/tests/examples/c/asyncdemo.c index 1e523bd7fe..225c4f7541 100644 --- a/tests/examples/c/asyncdemo.c +++ b/tests/examples/c/asyncdemo.c @@ -46,6 +46,34 @@ void taos_insert_call_back(void *param, TAOS_RES *tres, int code); void taos_select_call_back(void *param, TAOS_RES *tres, int code); void taos_error(TAOS *taos); +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + int main(int argc, char *argv[]) { TAOS *taos; @@ -78,16 +106,14 @@ int main(int argc, char *argv[]) printf("success to connect to server\n"); - sprintf(sql, "drop database %s", db); - taos_query(taos, sql); + sprintf(sql, "drop database if exists %s", db); + queryDB(taos, sql); sprintf(sql, "create database %s", db); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); sprintf(sql, "use %s", db); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); strcpy(prefix, "asytbl_"); for (i = 0; i < numOfTables; ++i) { @@ -95,8 +121,7 @@ int main(int argc, char *argv[]) tableList[i].taos = taos; sprintf(tableList[i].name, "%s%d", prefix, i); sprintf(sql, "create table %s%d (ts timestamp, volume bigint)", prefix, i); - if (taos_query(taos, sql) != 0) - taos_error(taos); + queryDB(taos, sql); } gettimeofday(&systemTime, NULL); From 73b23e8b8d0506e77799e676fe3adeeaa0c7a794 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 6 Nov 2020 09:48:57 +0800 Subject: [PATCH 54/58] [TD-1961] modify return when call taos_query() --- tests/examples/c/demo.c | 59 ++++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/tests/examples/c/demo.c b/tests/examples/c/demo.c index 8f8a66a325..59b9c74827 100644 --- a/tests/examples/c/demo.c +++ b/tests/examples/c/demo.c @@ -22,6 +22,34 @@ #include #include // TAOS header file +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < 5; i++) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + int main(int argc, char *argv[]) { TAOS * taos; char qstr[1024]; @@ -44,22 +72,26 @@ int main(int argc, char *argv[]) { printf("success to connect to server\n"); - taos_query(taos, "drop database demo"); + //taos_query(taos, "drop database demo"); + queryDB(taos, "drop database if exists demo"); - result = taos_query(taos, "create database demo"); - if (result == NULL) { - printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/); - exit(1); - } + //result = taos_query(taos, "create database demo"); + //if (result == NULL) { + // printf("failed to create database, reason:%s\n", "null result"/*taos_errstr(taos)*/); + // exit(1); + //} + queryDB(taos, "create database demo"); printf("success to create database\n"); - taos_query(taos, "use demo"); + //taos_query(taos, "use demo"); + queryDB(taos, "use demo"); // create table - if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) { - printf("failed to create table, reason:%s\n", taos_errstr(result)); - exit(1); - } + //if (taos_query(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))") == 0) { + // printf("failed to create table, reason:%s\n", taos_errstr(result)); + // exit(1); + //} + queryDB(taos, "create table m1 (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10))"); printf("success to create table\n"); // sleep for one second to make sure table is created on data node @@ -80,8 +112,10 @@ int main(int argc, char *argv[]) { printf("insert row: %i\n", i); } else { printf("failed to insert row: %i, reason:%s\n", i, "null result"/*taos_errstr(result)*/); + taos_free_result(result); exit(1); } + taos_free_result(result); //sleep(1); } @@ -91,7 +125,8 @@ int main(int argc, char *argv[]) { sprintf(qstr, "SELECT * FROM m1"); result = taos_query(taos, qstr); if (result == NULL || taos_errno(result) != 0) { - printf("failed to select, reason:%s\n", taos_errstr(result)); + printf("failed to select, reason:%s\n", taos_errstr(result)); + taos_free_result(result); exit(1); } From 86041893003f5b68f444ce2393e55c4ca3af7249 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 6 Nov 2020 10:05:07 +0800 Subject: [PATCH 55/58] [TD-1961] modify return when call taos_query() --- tests/examples/c/apitest.c | 76 +++++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 25 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index 759e16d1de..a2754913b3 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -9,26 +9,40 @@ static void prepare_data(TAOS* taos) { - taos_query(taos, "drop database if exists test;"); + TAOS_RES *result; + result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); usleep(100000); - taos_query(taos, "create database test;"); + result = taos_query(taos, "create database test;"); + taos_free_result(result); usleep(100000); taos_select_db(taos, "test"); - taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + result = taos_query(taos, "create table meters(ts timestamp, a int) tags(area int);"); + taos_free_result(result); - taos_query(taos, "create table t0 using meters tags(0);"); - taos_query(taos, "create table t1 using meters tags(1);"); - taos_query(taos, "create table t2 using meters tags(2);"); - taos_query(taos, "create table t3 using meters tags(3);"); - taos_query(taos, "create table t4 using meters tags(4);"); - taos_query(taos, "create table t5 using meters tags(5);"); - taos_query(taos, "create table t6 using meters tags(6);"); - taos_query(taos, "create table t7 using meters tags(7);"); - taos_query(taos, "create table t8 using meters tags(8);"); - taos_query(taos, "create table t9 using meters tags(9);"); + result = taos_query(taos, "create table t0 using meters tags(0);"); + taos_free_result(result); + result = taos_query(taos, "create table t1 using meters tags(1);"); + taos_free_result(result); + result = taos_query(taos, "create table t2 using meters tags(2);"); + taos_free_result(result); + result = taos_query(taos, "create table t3 using meters tags(3);"); + taos_free_result(result); + result = taos_query(taos, "create table t4 using meters tags(4);"); + taos_free_result(result); + result = taos_query(taos, "create table t5 using meters tags(5);"); + taos_free_result(result); + result = taos_query(taos, "create table t6 using meters tags(6);"); + taos_free_result(result); + result = taos_query(taos, "create table t7 using meters tags(7);"); + taos_free_result(result); + result = taos_query(taos, "create table t8 using meters tags(8);"); + taos_free_result(result); + result = taos_query(taos, "create table t9 using meters tags(9);"); + taos_free_result(result); - TAOS_RES* res = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" + result = taos_query(taos, "insert into t0 values('2020-01-01 00:00:00.000', 0)" " ('2020-01-01 00:01:00.000', 0)" " ('2020-01-01 00:02:00.000', 0)" " t1 values('2020-01-01 00:00:00.000', 0)" @@ -46,10 +60,11 @@ static void prepare_data(TAOS* taos) { " t7 values('2020-01-01 00:01:02.000', 0)" " t8 values('2020-01-01 00:01:02.000', 0)" " t9 values('2020-01-01 00:01:02.000', 0)"); - int affected = taos_affected_rows(res); + int affected = taos_affected_rows(result); if (affected != 18) { printf("\033[31m%d rows affected by last insert statement, but it should be 18\033[0m\n", affected); } + taos_free_result(result); // super tables subscription usleep(1000000); } @@ -153,23 +168,30 @@ static void verify_subscribe(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); - taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + TAOS_RES *result; + result = taos_query(taos, "insert into t0 values('2020-01-01 00:02:00.001', 0);"); + taos_free_result(result); + result = taos_query(taos, "insert into t8 values('2020-01-01 00:01:03.000', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + result = taos_query(taos, "insert into t2 values('2020-01-01 00:01:02.001', 0);"); + taos_free_result(result); + result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.001', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 2); - taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + result = taos_query(taos, "insert into t1 values('2020-01-01 00:03:00.002', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); // keep progress information and restart subscription taos_unsubscribe(tsub, 1); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.000', 0);"); + taos_free_result(result); tsub = taos_subscribe(taos, 1, "test", "select * from meters;", NULL, NULL, 0); res = taos_consume(tsub); check_row_count(__LINE__, res, 24); @@ -196,7 +218,8 @@ static void verify_subscribe(TAOS* taos) { res = taos_consume(tsub); check_row_count(__LINE__, res, 0); - taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:04:00.001', 0);"); + taos_free_result(result); res = taos_consume(tsub); check_row_count(__LINE__, res, 1); @@ -205,7 +228,8 @@ static void verify_subscribe(TAOS* taos) { int blockFetch = 0; tsub = taos_subscribe(taos, 1, "test", "select * from meters;", subscribe_callback, &blockFetch, 1000); usleep(2000000); - taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); + result = taos_query(taos, "insert into t0 values('2020-01-01 00:05:00.001', 0);"); + taos_free_result(result); usleep(2000000); taos_unsubscribe(tsub, 0); } @@ -213,8 +237,9 @@ static void verify_subscribe(TAOS* taos) { void verify_prepare(TAOS* taos) { TAOS_RES* result = taos_query(taos, "drop database if exists test;"); + taos_free_result(result); usleep(100000); - taos_query(taos, "create database test;"); + result = taos_query(taos, "create database test;"); int code = taos_errno(result); if (code != 0) { @@ -429,7 +454,8 @@ void verify_stream(TAOS* taos) { NULL); printf("waiting for stream data\n"); usleep(100000); - taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); + TAOS_RES* result = taos_query(taos, "insert into t0 values(now, 0)(now+5s,1)(now+10s, 2);"); + taos_free_result(result); usleep(200000000); taos_close_stream(strm); } From c034940cc84ebfa74bb3391afeebd002376b6459 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 6 Nov 2020 10:07:51 +0800 Subject: [PATCH 56/58] [TD-1961] modify return when call taos_query() --- tests/examples/c/apitest.c | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index a2754913b3..be60a88ad7 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -150,6 +150,7 @@ static void verify_query(TAOS* taos) { res = taos_query(taos, "select * from meters"); taos_stop_query(res); + taos_free_result(res); } From 2b07f83bbea9ca9610f21973bb7ddc45ed7fb4e4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 Nov 2020 10:27:19 +0800 Subject: [PATCH 57/58] [TD-225] --- src/query/inc/tsqlfunction.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 3bd4aad276..84ca78d822 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -149,7 +149,7 @@ typedef struct SResultRowCellInfo { int8_t hasResult; // result generated, not NULL value bool initialized; // output buffer has been initialized bool complete; // query has completed - uint16_t numOfRes; // num of output result in current buffer + uint32_t numOfRes; // num of output result in current buffer } SResultRowCellInfo; #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) From 1fed1db16a9dcce63f62139c8b9cf636cfaae710 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 6 Nov 2020 11:02:33 +0800 Subject: [PATCH 58/58] [TD-225] --- src/client/src/tscSubquery.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 4b6dde902c..b3d151ba27 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -565,7 +565,7 @@ int32_t tagValCompar(const void* p1, const void* p2) { return (tag1->len > tag2->len)? 1: -1; } - return strncmp(tag1->data, tag2->data, tag1->len); + return memcmp(tag1->data, tag2->data, tag1->len); } void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables) {