From ec8ac48a606e311c3c030a408510e74df30cd073 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 13 Nov 2020 12:44:53 +0000 Subject: [PATCH 001/188] return value is pointer --- src/sync/src/syncMain.c | 2 +- src/util/tests/trefTest.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 8d90315c48..4b51c770e7 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1097,7 +1097,7 @@ static void syncProcessBrokenLink(void *param) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return; + if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&(pNode->mutex)); sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno)); diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c index 6887b24abd..454860410b 100644 --- a/src/util/tests/trefTest.c +++ b/src/util/tests/trefTest.c @@ -77,8 +77,8 @@ void *acquireRelease(void *param) { printf("a"); id = random() % pSpace->refNum; - code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); - if (code >= 0) { + void *p = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); + if (p) { usleep(id % 5 + 1); taosReleaseRef(pSpace->rsetId, pSpace->p[id]); } From 4c11e46075d65d1bc29a116ed9359dd36c80d421 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 18 Nov 2020 17:40:00 +0800 Subject: [PATCH 002/188] change --- .../com/taosdata/jdbc/rs/RestfulDriver.java | 3 +- .../taosdata/jdbc/rs/RestfulDriverTest.java | 35 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index c267f660de..a07d6eb2b6 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -86,6 +86,7 @@ public class RestfulDriver extends AbstractTaosDriver { @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { - return null; + //TODO SQLFeatureNotSupportedException + throw new SQLFeatureNotSupportedException(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index a91d1c2d6b..768eceab8e 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -31,10 +31,43 @@ public class RestfulDriverTest { } @Test - public void testAcceptUrl() throws SQLException { + public void connect() { + + } + + @Test + public void acceptsURL() throws SQLException { Driver driver = new RestfulDriver(); boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://master:6041"); Assert.assertTrue(isAccept); + isAccept = driver.acceptsURL("jdbc:TAOS://master:6041"); + Assert.assertFalse(isAccept); } + @Test + public void getPropertyInfo() throws SQLException { + Driver driver = new RestfulDriver(); + final String url = ""; + DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, null); + } + + @Test + public void getMajorVersion() { + Assert.assertEquals(2, new RestfulDriver().getMajorVersion()); + } + + @Test + public void getMinorVersion() { + Assert.assertEquals(0, new RestfulDriver().getMinorVersion()); + } + + @Test + public void jdbcCompliant() { + Assert.assertFalse(new RestfulDriver().jdbcCompliant()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getParentLogger() throws SQLFeatureNotSupportedException { + new RestfulDriver().getParentLogger(); + } } From a8b87ce0cd79c498fc370f7c4656f335115256fd Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 20 Nov 2020 01:07:33 +0000 Subject: [PATCH 003/188] change applications --- src/query/src/qResultbuf.c | 8 +- src/util/inc/hash.h | 53 ++---- src/util/src/hash.c | 332 ++++++++++++++++++------------------- src/util/src/tkvstore.c | 19 +-- src/vnode/src/vnodeMain.c | 28 ++-- 5 files changed, 191 insertions(+), 249 deletions(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 2645cff678..e9ed8dc1c3 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -420,9 +420,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { unlink(pResultBuf->path); tfree(pResultBuf->path); - SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet); - while(taosHashIterNext(iter)) { - SArray** p = (SArray**) taosHashIterGet(iter); + SArray** p = taosHashIterate(pResultBuf->groupSet, NULL); + while(p) { size_t n = taosArrayGetSize(*p); for(int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); @@ -431,10 +430,9 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } taosArrayDestroy(*p); + p = taosHashIterate(pResultBuf->groupSet, p); } - taosHashDestroyIter(iter); - tdListFree(pResultBuf->lruList); taosArrayDestroy(pResultBuf->emptyDummyIdList); taosHashCleanup(pResultBuf->groupSet); diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 42bc136584..aaff48f4f8 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -31,16 +31,17 @@ extern "C" { typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { -// char *key; struct SHashNode *next; uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key -// char *data; + uint32_t dataLen; // length of data + int8_t count; // reference count + char data[]; } SHashNode; -#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode)) -#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen) - +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode)); typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, @@ -65,15 +66,6 @@ typedef struct SHashObj { SArray *pMemBlock; // memory block allocated for SHashEntry } SHashObj; -typedef struct SHashMutableIterator { - SHashObj *pHashObj; - int32_t entryIndex; - SHashNode *pCur; - SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current - size_t numOfChecked; // already check number of elements in hash table - size_t numOfEntries; // number of entries while the iterator is created -} SHashMutableIterator; - /** * init the hash table * @@ -142,33 +134,9 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi */ void taosHashCleanup(SHashObj *pHashObj); -/** - * - * @param pHashObj - * @return - */ -SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj); - -/** - * - * @param iter - * @return - */ -bool taosHashIterNext(SHashMutableIterator *iter); - -/** - * - * @param iter - * @return - */ -void *taosHashIterGet(SHashMutableIterator *iter); - -/** - * - * @param iter - * @return - */ -void* taosHashDestroyIter(SHashMutableIterator* iter); +/* +void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *); +*/ /** * @@ -179,6 +147,9 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); size_t taosHashGetMemSize(const SHashObj *pHashObj); +void *taosHashIterate(SHashObj *pHashObj, void *p); +void taosHashCancelIterate(SHashObj *pHashObj, void *p); + #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 03a7342497..cd9a9f7d0a 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -114,15 +114,25 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); + + pNode->count--; if (prev != NULL) { prev->next = pNewNode; } else { pe->next = pNewNode; } - pNewNode->next = pNode->next; + if (pNode->count <= 0) { + pNewNode->next = pNode->next; + DO_FREE_HASH_NODE(pNode); + } else { + pNewNode->next = pNode; + pe->num++; + atomic_add_fetch_64(&pHashObj->size, 1); + } + return pNewNode; } @@ -139,7 +149,6 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); * @param pIter * @return */ -static SHashNode *getNextHashNode(SHashMutableIterator *pIter); SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (capacity == 0 || fn == NULL) { @@ -244,8 +253,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } else { // not support the update operation, return error if (pHashObj->enableUpdate) { - doUpdateHashNode(pe, prev, pNode, pNewNode); - DO_FREE_HASH_NODE(pNode); + doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); } else { DO_FREE_HASH_NODE(pNewNode); } @@ -335,22 +343,10 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - // no data, return directly - if (pe->num == 0) { - __rd_unlock(&pHashObj->lock, pHashObj->type); - return -1; - } - if (pHashObj->type == HASH_ENTRY_LOCK) { taosWLockLatch(&pe->latch); } - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - // double check after locked if (pe->num == 0) { assert(pe->next == NULL); @@ -360,37 +356,36 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe return -1; } + int code = -1; SHashNode *pNode = pe->next; - SHashNode *pRes = NULL; + SHashNode *prevNode = NULL; - // remove it - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { - pe->num -= 1; - pRes = pNode; - pe->next = pNode->next; - } else { - while (pNode->next != NULL) { - if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) { - assert((pNode->next)->hashVal == hashVal); - break; + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) + break; + + prevNode = pNode; + pNode = pNode->next; + } + + if (pNode) { + code = 0; // it is found + + pNode->count--; + if (pNode->count <= 0) { + if (prevNode) { + prevNode->next = pNode->next; + } else { + pe->next = pNode->next; } + + if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - pNode = pNode->next; + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pNode); } - - - if (pNode->next != NULL) { - pe->num -= 1; - pRes = pNode->next; - pNode->next = pNode->next->next; - } - } - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } + } if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); @@ -398,17 +393,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe __rd_unlock(&pHashObj->lock, pHashObj->type); - if (data != NULL && pRes != NULL) { - memcpy(data, GET_HASH_NODE_DATA(pRes), dsize); - } - - if (pRes != NULL) { - atomic_sub_fetch_64(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pRes); - return 0; - } else { - return -1; - } + return code; } int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { @@ -531,98 +516,6 @@ void taosHashCleanup(SHashObj *pHashObj) { free(pHashObj); } -SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { - SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator)); - if (pIter == NULL) { - return NULL; - } - - pIter->pHashObj = pHashObj; - - // keep it in local variable, in case the resize operation expand the size - pIter->numOfEntries = pHashObj->capacity; - return pIter; -} - -bool taosHashIterNext(SHashMutableIterator *pIter) { - if (pIter == NULL) { - return false; - } - - size_t size = taosHashGetSize(pIter->pHashObj); - if (size == 0) { - return false; - } - - // check the first one - if (pIter->numOfChecked == 0) { - assert(pIter->pCur == NULL && pIter->pNext == NULL); - - while (1) { - SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - - pIter->entryIndex++; - continue; - } - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pEntry->latch); - } - - pIter->pCur = pEntry->next; - - if (pIter->pCur->next) { - pIter->pNext = pIter->pCur->next; - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - } else { - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - - pIter->pNext = getNextHashNode(pIter); - } - - break; - } - - pIter->numOfChecked++; - return true; - } else { - assert(pIter->pCur != NULL); - if (pIter->pNext) { - pIter->pCur = pIter->pNext; - } else { // no more data in the hash list - return false; - } - - pIter->numOfChecked++; - - if (pIter->pCur->next) { - pIter->pNext = pIter->pCur->next; - } else { - pIter->pNext = getNextHashNode(pIter); - } - - return true; - } -} - -void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); } - -void *taosHashDestroyIter(SHashMutableIterator *iter) { - if (iter == NULL) { - return NULL; - } - - free(iter); - return NULL; -} - // for profile only int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { if (pHashObj == NULL || pHashObj->size == 0) { @@ -759,6 +652,8 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; + pNewNode->dataLen = dsize; + pNewNode->count = 1; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -775,35 +670,6 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { pEntry->num += 1; } -SHashNode *getNextHashNode(SHashMutableIterator *pIter) { - assert(pIter != NULL); - - pIter->entryIndex++; - SHashNode *p = NULL; - - while (pIter->entryIndex < pIter->numOfEntries) { - SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry->num == 0) { - pIter->entryIndex++; - continue; - } - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pEntry->latch); - } - - p = pEntry->next; - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - - return p; - } - - return NULL; -} - size_t taosHashGetMemSize(const SHashObj *pHashObj) { if (pHashObj == NULL) { return 0; @@ -811,3 +677,119 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } + +// release the pNode, return next pNode, and lock the current entry +static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { + + SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); + SHashNode *prevNode = NULL; + + *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[*slot]; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + SHashNode *pNode = pe->next; + + while (pNode) { + if (pNode == pOld) + break; + + prevNode = pNode; + pNode = pNode->next; + } + + if (pNode) { + pNode = pNode->next; + pOld->count--; + if (pOld->count <=0) { + if (prevNode) { + prevNode->next = pOld->next; + } else { + pe->next = pOld->next; + } + + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pOld); + } + } else { + uError("pNode:%p data:%p is not there!!!", pNode, p); + } + + return pNode; +} + +void *taosHashIterate(SHashObj *pHashObj, void *p) { + if (pHashObj == NULL) return NULL; + + int slot = 0; + char *data = NULL; + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + SHashNode *pNode = NULL; + if (p) { + pNode = taosHashReleaseNode(pHashObj, p, &slot); + if (pNode == NULL) { + SHashEntry *pe = pHashObj->hashList[slot]; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + slot = slot + 1; + } + } + + if (pNode == NULL) { + for (; slot < pHashObj->capacity; ++slot) { + SHashEntry *pe = pHashObj->hashList[slot]; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + pNode = pe->next; + if (pNode) break; + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + } + } + + if (pNode) { + SHashEntry *pe = pHashObj->hashList[slot]; + pNode->count++; + data = GET_HASH_NODE_DATA(pNode); + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; + +} + +void taosHashCancelIterate(SHashObj *pHashObj, void *p) { + if (pHashObj == NULL || p == NULL) return; + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int slot; + taosHashReleaseNode(pHashObj, p, &slot); + + SHashEntry *pe = pHashObj->hashList[slot]; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); +} diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 31641ac9a7..2b1d13c78b 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -529,7 +529,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { void * buf = NULL; int64_t maxBufSize = 0; SKVRecord rInfo = {0}; - SHashMutableIterator *pIter = NULL; + SKVRecord *pRecord = NULL; ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR)); ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE); @@ -582,16 +582,8 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } - pIter = taosHashCreateIter(pStore->map); - if (pIter == NULL) { - uError("failed to create hash iter while opening KV store %s", pStore->fname); - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - while (taosHashIterNext(pIter)) { - SKVRecord *pRecord = taosHashIterGet(pIter); - + pRecord = taosHashIterate(pStore->map, NULL); + while (pRecord) { if (lseek(pStore->fd, (off_t)(pRecord->offset + sizeof(SKVRecord)), SEEK_SET) < 0) { uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); @@ -613,16 +605,17 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } } + + pRecord = taosHashIterate(pStore->map, pRecord); } if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); - taosHashDestroyIter(pIter); tfree(buf); return 0; _err: - taosHashDestroyIter(pIter); + taosHashCancelIterate(pStore->map, pRecord); tfree(buf); return -1; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 7447acc488..e3eb35ea32 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -517,11 +517,10 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { } int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); - while (taosHashIterNext(pIter)) { - SVnodeObj **pVnode = taosHashIterGet(pIter); - if (pVnode == NULL) continue; - if (*pVnode == NULL) continue; + void *pIter = taosHashIterate(tsVnodesHash, NULL); + while (pIter) { + SVnodeObj **pVnode = pIter; + if (*pVnode) { (*numOfVnodes)++; if (*numOfVnodes >= TSDB_MAX_VNODES) { @@ -530,25 +529,24 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { } else { vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId; } - } - taosHashDestroyIter(pIter); + } + + pIter = taosHashIterate(tsVnodesHash, pIter); + } return TSDB_CODE_SUCCESS; } void vnodeBuildStatusMsg(void *param) { SStatusMsg *pStatus = param; - SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); - - while (taosHashIterNext(pIter)) { - SVnodeObj **pVnode = taosHashIterGet(pIter); - if (pVnode == NULL) continue; - if (*pVnode == NULL) continue; + void *pIter = taosHashIterate(tsVnodesHash, NULL); + while (pIter) { + SVnodeObj **pVnode = pIter; + if (*pVnode) { vnodeBuildVloadMsg(*pVnode, pStatus); + } } - - taosHashDestroyIter(pIter); } void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { From 3e43a6533189e060a5e7f848959fd0bda55685da Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 Nov 2020 20:45:06 +0800 Subject: [PATCH 004/188] TD-1983 change hash in mnode --- src/balance/src/balance.c | 40 ++++++++++++------------------ src/mnode/inc/mnodeAcct.h | 1 + src/mnode/inc/mnodeDb.h | 1 + src/mnode/inc/mnodeDnode.h | 1 + src/mnode/inc/mnodeMnode.h | 1 + src/mnode/inc/mnodeSdb.h | 6 ++--- src/mnode/inc/mnodeShow.h | 2 ++ src/mnode/inc/mnodeTable.h | 2 ++ src/mnode/inc/mnodeUser.h | 1 + src/mnode/inc/mnodeVgroup.h | 1 + src/mnode/src/mnodeAcct.c | 6 +++-- src/mnode/src/mnodeCluster.c | 8 +++++- src/mnode/src/mnodeDb.c | 12 +++++---- src/mnode/src/mnodeDnode.c | 23 +++++++++--------- src/mnode/src/mnodeMnode.c | 9 ++++--- src/mnode/src/mnodeProfile.c | 24 +++++++++--------- src/mnode/src/mnodeSdb.c | 36 ++++++++++++--------------- src/mnode/src/mnodeShow.c | 9 ++++++- src/mnode/src/mnodeTable.c | 47 ++++++++++++++++++------------------ src/mnode/src/mnodeUser.c | 9 ++++--- src/mnode/src/mnodeVgroup.c | 15 +++++------- src/vnode/src/vnodeMain.c | 7 +++--- tests/script/basicSuite.sim | 1 - 23 files changed, 139 insertions(+), 123 deletions(-) diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index df78f4fe27..b172867929 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -162,7 +162,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); if (mnodeGetOnlineDnodesNum() == 0) { return TSDB_CODE_MND_NOT_READY; @@ -377,15 +376,13 @@ static bool balanceMonitorBalance() { srcScore, pDestDnode->score, destScore); balanceAddVnode(pVgroup, pSrcDnode, pDestDnode); mnodeDecVgroupRef(pVgroup); - sdbFreeIter(pIter); + mnodeCancelGetNextVgroup(pIter); return true; } } mnodeDecVgroupRef(pVgroup); } - - sdbFreeIter(pIter); } return false; @@ -413,8 +410,6 @@ void balanceReset() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - tsAccessSquence = 0; } @@ -443,12 +438,11 @@ static int32_t balanceMonitorVgroups() { mnodeDecVgroupRef(pVgroup); if (code == TSDB_CODE_SUCCESS) { + mnodeCancelGetNextVgroup(pIter); break; } } - sdbFreeIter(pIter); - return hasUpdatingVgroup; } @@ -465,11 +459,12 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) { hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup); mnodeDecVgroupRef(pVgroup); - if (hasThisDnode) break; + if (hasThisDnode) { + mnodeCancelGetNextVgroup(pIter); + break; + } } - sdbFreeIter(pIter); - if (!hasThisDnode) { mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId); mnodeDropDnode(pDnode, NULL); @@ -499,20 +494,18 @@ static bool balanceMontiorDropping() { pDnode->status = TAOS_DN_STATUS_DROPPING; mnodeUpdateDnode(pDnode); mnodeDecDnodeRef(pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return true; } if (pDnode->status == TAOS_DN_STATUS_DROPPING) { bool ret = balanceMonitorDnodeDropping(pDnode); mnodeDecDnodeRef(pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return ret; } } - sdbFreeIter(pIter); - return false; } @@ -556,8 +549,6 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) { } mnodeDecVgroupRef(pVgroup); } - - sdbFreeIter(pIter); } static void balanceCheckDnodeAccess() { @@ -578,8 +569,6 @@ static void balanceCheckDnodeAccess() { } mnodeDecDnodeRef(pDnode); } - - sdbFreeIter(pIter); } static void balanceProcessBalanceTimer(void *handle, void *tmrId) { @@ -630,6 +619,7 @@ void balanceAsyncNotify() { int32_t balanceInit() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode); pthread_mutex_init(&tsBalanceMutex, NULL); balanceInitDnodeList(); @@ -667,8 +657,6 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) { mnodeDecDnodeRef(pTempDnode); } - sdbFreeIter(pIter); - if (pDnode->openVnodes > totalFreeVnodes) { mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes); return TSDB_CODE_MND_NO_ENOUGH_DNODES; @@ -780,8 +768,12 @@ void balanceAccquireDnodeList() { SDnodeObj *pDnode = NULL; int32_t dnodeIndex = 0; - while (1) { - if (dnodeIndex >= dnodesNum) break; + while (1) { + if (dnodeIndex >= dnodesNum) { + mnodeCancelGetNextDnode(pIter); + break; + } + pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { @@ -802,8 +794,6 @@ void balanceAccquireDnodeList() { dnodeIndex++; } - sdbFreeIter(pIter); - tsBalanceDnodeListSize = dnodeIndex; } diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 744a62f948..595dcca413 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -27,6 +27,7 @@ void mnodeCleanupAccts(); void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo); void * mnodeGetAcct(char *acctName); void * mnodeGetNextAcct(void *pIter, SAcctObj **pAcct); +void mnodeCancelGetNextAcct(void *pIter); void mnodeIncAcctRef(SAcctObj *pAcct); void mnodeDecAcctRef(SAcctObj *pAcct); void mnodeAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index 7cbd08ed92..9354b923d7 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -34,6 +34,7 @@ int64_t mnodeGetDbNum(); SDbObj *mnodeGetDb(char *db); SDbObj *mnodeGetDbByTableId(char *db); void * mnodeGetNextDb(void *pIter, SDbObj **pDb); +void mnodeCancelGetNextDb(void *pIter); void mnodeIncDbRef(SDbObj *pDb); void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index b6ddb7a9bf..8bc29ef9ef 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -65,6 +65,7 @@ int32_t mnodeGetDnodesNum(); int32_t mnodeGetOnlinDnodesCpuCoreNum(); int32_t mnodeGetOnlineDnodesNum(); void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode); +void mnodeCancelGetNextDnode(void *pIter); void mnodeIncDnodeRef(SDnodeObj *pDnode); void mnodeDecDnodeRef(SDnodeObj *pDnode); void * mnodeGetDnode(int32_t dnodeId); diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index a28a03ea40..10cbcebe22 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -38,6 +38,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId); void * mnodeGetMnode(int32_t mnodeId); int32_t mnodeGetMnodesNum(); void * mnodeGetNextMnode(void *pIter, struct SMnodeObj **pMnode); +void mnodeCancelGetNextMnode(void *pIter); void mnodeIncMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode); diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 29d8cf1207..81629ede78 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -92,9 +92,9 @@ int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); -void *sdbGetRow(void *pTable, void *key); -void *sdbFetchRow(void *pTable, void *pIter, void **ppRow); -void sdbFreeIter(void *pIter); +void * sdbGetRow(void *pTable, void *key); +void * sdbFetchRow(void *pTable, void *pIter, void **ppRow); +void sdbFreeIter(void *pTable, void *pIter); void sdbIncRef(void *pTable, void *pRow); void sdbDecRef(void *pTable, void *pRow); int64_t sdbGetNumOfRows(void *pTable); diff --git a/src/mnode/inc/mnodeShow.h b/src/mnode/inc/mnodeShow.h index da66e71678..f985fe792d 100644 --- a/src/mnode/inc/mnodeShow.h +++ b/src/mnode/inc/mnodeShow.h @@ -26,8 +26,10 @@ void mnodeCleanUpShow(); typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); +typedef void (*SShowFreeIterFp)(void *pIter); void mnodeAddShowMetaHandle(uint8_t showType, SShowMetaFp fp); void mnodeAddShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp); +void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp); void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index 7c0077aa60..bf04f26a90 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -31,6 +31,8 @@ void mnodeIncTableRef(void *pTable); void mnodeDecTableRef(void *pTable); void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable); void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable); +void mnodeCancelGetNextChildTable(void *pIter); +void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 073460f9d3..156bc7aaeb 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -25,6 +25,7 @@ int32_t mnodeInitUsers(); void mnodeCleanupUsers(); SUserObj *mnodeGetUser(char *name); void * mnodeGetNextUser(void *pIter, SUserObj **pUser); +void mnodeCancelGetNextUser(void *pIter); void mnodeIncUserRef(SUserObj *pUser); void mnodeDecUserRef(SUserObj *pUser); SUserObj *mnodeGetUserFromConn(void *pConn); diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 0e6d9dfde4..ee9ec7ae93 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -34,6 +34,7 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); +void mnodeCancelGetNextVgroup(void *pIter); void mnodeUpdateVgroup(SVgObj *pVgroup); void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload); void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 9fff2f0229..3da889b284 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -144,7 +144,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) { pAcctInfo->numOfTimeSeries += pAcct->acctInfo.numOfTimeSeries; mnodeDecAcctRef(pAcct); } - sdbFreeIter(pIter); SVgObj *pVgroup = NULL; pIter = NULL; @@ -158,7 +157,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) { pAcctInfo->totalPoints += pVgroup->pointsWritten; mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); } void *mnodeGetAcct(char *name) { @@ -169,6 +167,10 @@ void *mnodeGetNextAcct(void *pIter, SAcctObj **pAcct) { return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); } +void mnodeCancelGetNextAcct(void *pIter) { + sdbFreeIter(tsAcctSdb, pIter); +} + void mnodeIncAcctRef(SAcctObj *pAcct) { sdbIncRef(tsAcctSdb, pAcct); } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 5be67e4ad9..839407c5b0 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -31,6 +31,7 @@ static int32_t mnodeCreateCluster(); static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static void mnodeCancelGetNextCluster(void *pIter); static int32_t mnodeClusterActionDestroy(SSdbRow *pRow) { tfree(pRow->pObj); @@ -108,6 +109,7 @@ int32_t mnodeInitCluster() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster); mDebug("table:%s, hash is created", desc.name); return TSDB_CODE_SUCCESS; @@ -122,6 +124,10 @@ void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) { return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster); } +void mnodeCancelGetNextCluster(void *pIter) { + sdbFreeIter(tsClusterSdb, pIter); +} + void mnodeIncClusterRef(SClusterObj *pCluster) { sdbIncRef(tsClusterSdb, pCluster); } @@ -167,7 +173,7 @@ void mnodeUpdateClusterId() { } mnodeDecClusterRef(pCluster); - sdbFreeIter(pIter); + mnodeCancelGetNextCluster(pIter); } static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index d121208447..77f3b93eb1 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -171,6 +171,7 @@ int32_t mnodeInitDbs() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mnodeProcessDropDbMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DB, mnodeGetDbMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mnodeRetrieveDbs); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb); mDebug("table:dbs table is created"); return 0; @@ -180,6 +181,10 @@ void *mnodeGetNextDb(void *pIter, SDbObj **pDb) { return sdbFetchRow(tsDbSdb, pIter, (void **)pDb); } +void mnodeCancelGetNextDb(void *pIter) { + sdbFreeIter(tsDbSdb, pIter); +} + SDbObj *mnodeGetDb(char *db) { return (SDbObj *)sdbGetRow(tsDbSdb, db); } @@ -986,8 +991,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { SDbObj *pDb = pMsg->pDb; void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; + SVgObj *pVgroup = NULL; + while (1) { pIter = mnodeGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; if (pVgroup->pDb == pDb) { @@ -995,7 +1000,6 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { } mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); mDebug("db:%s, all vgroups is altered", pDb->name); mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); @@ -1146,7 +1150,5 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mnodeDecDbRef(pDb); } - sdbFreeIter(pIter); - mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index f76533c760..0e9b56e459 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -206,6 +206,7 @@ int32_t mnodeInitDnodes() { mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mnodeRetrieveDnodes); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DNODE, mnodeCancelGetNextDnode); mDebug("table:dnodes table is created"); return 0; @@ -223,6 +224,10 @@ void *mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode) { return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode); } +void mnodeCancelGetNextDnode(void *pIter) { + sdbFreeIter(tsDnodeSdb, pIter); +} + int32_t mnodeGetDnodesNum() { return sdbGetNumOfRows(tsDnodeSdb); } @@ -241,8 +246,6 @@ int32_t mnodeGetOnlinDnodesCpuCoreNum() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - if (cpuCores < 2) cpuCores = 2; return cpuCores; } @@ -259,8 +262,6 @@ int32_t mnodeGetOnlineDnodesNum() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - return onlineDnodes; } @@ -276,13 +277,12 @@ void *mnodeGetDnodeByEp(char *ep) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; if (strcmp(ep, pDnode->dnodeEp) == 0) { - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return pDnode; } mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); return NULL; } @@ -464,7 +464,10 @@ static void mnodeUpdateDnodeEps() { while (1) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; - if (dnodesNum >= totalDnodes) break; + if (dnodesNum >= totalDnodes) { + mnodeCancelGetNextDnode(pIter); + break; + } SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum]; dnodesNum++; @@ -474,7 +477,6 @@ static void mnodeUpdateDnodeEps() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); pthread_mutex_unlock(&tsDnodeEpsMutex); } @@ -1100,7 +1102,7 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pDnode = mnodeGetDnodeByEp(pShow->payload); } else { void *pIter = mnodeGetNextDnode(NULL, (SDnodeObj **)&pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); } if (pDnode != NULL) { @@ -1148,7 +1150,6 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); } else { numOfRows = 0; } @@ -1217,8 +1218,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - if (pSelDnode == NULL) { mError("failed to alloc vnode to vgroup"); return TSDB_CODE_MND_NO_ENOUGH_DNODES; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 205bfda4b9..8e3ad0a248 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -123,7 +123,7 @@ static int32_t mnodeMnodeActionRestored() { pMnode->role = TAOS_SYNC_ROLE_MASTER; mnodeDecMnodeRef(pMnode); } - sdbFreeIter(pIter); + mnodeCancelGetNextMnode(pIter); } mnodeUpdateMnodeEpSet(); @@ -161,6 +161,7 @@ int32_t mnodeInitMnodes() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mnodeGetMnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mnodeRetrieveMnodes); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_MNODE, mnodeCancelGetNextMnode); mDebug("table:mnodes table is created"); return TSDB_CODE_SUCCESS; @@ -192,6 +193,10 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) { return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); } +void mnodeCancelGetNextMnode(void *pIter) { + sdbFreeIter(tsMnodeSdb, pIter); +} + void mnodeUpdateMnodeEpSet() { mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum()); @@ -239,8 +244,6 @@ void mnodeUpdateMnodeEpSet() { tsMnodeEpSetForShell.numOfEps = index; tsMnodeEpSetForPeer.numOfEps = index; - sdbFreeIter(pIter); - mnodeMnodeUnLock(); } diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 12ac64854c..8467adfc17 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -42,6 +42,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static void mnodeCancelGetNextConn(void *pIter); static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mnodeFreeConn(void *data); @@ -52,10 +53,13 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg); int32_t mnodeInitProfile() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_QUERIES, mnodeGetQueryMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_QUERIES, mnodeRetrieveQueries); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_QUERIES, mnodeCancelGetNextConn); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONNS, mnodeGetConnsMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONNS, mnodeRetrieveConns); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CONNS, mnodeCancelGetNextConn); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMS, mnodeGetStreamMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMS, mnodeRetrieveStreams); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMS, mnodeCancelGetNextConn); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_QUERY, mnodeProcessKillQueryMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg); @@ -137,21 +141,15 @@ static void mnodeFreeConn(void *data) { mDebug("connId:%d, is destroyed", pConn->connId); } -static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) { +static void *mnodeGetNextConn(void *pIter, SConnObj **pConn) { *pConn = NULL; - if (pIter == NULL) { - pIter = taosHashCreateIter(tsMnodeConnCache->pHashTable); - } + pIter = taosHashIterate(tsMnodeConnCache->pHashTable, pIter); + if (pIter == NULL) return NULL; - if (!taosHashIterNext(pIter)) { - taosHashDestroyIter(pIter); - return NULL; - } - - SCacheDataNode **pNode = taosHashIterGet(pIter); + SCacheDataNode **pNode = pIter; if (pNode == NULL || *pNode == NULL) { - taosHashDestroyIter(pIter); + taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter); return NULL; } @@ -159,6 +157,10 @@ static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) { return pIter; } +static void mnodeCancelGetNextConn(void *pIter) { + taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter); +} + static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SUserObj *pUser = mnodeGetUserFromConn(pConn); if (pUser == NULL) return 0; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 1728bb7e3b..575845b4f2 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -325,7 +325,6 @@ void sdbUpdateSync(void *pMnodes) { mnodeDecDnodeRef(pDnode); mnodeDecMnodeRef(pMnode); } - sdbFreeIter(pIter); syncCfg.replica = index; mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); } else { @@ -756,24 +755,17 @@ int32_t sdbUpdateRow(SSdbRow *pRow) { } } -void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) { +void *sdbFetchRow(void *tparam, void *pIter, void **ppRow) { SSdbTable *pTable = tparam; *ppRow = NULL; if (pTable == NULL) return NULL; - SHashMutableIterator *pIter = pNode; - if (pIter == NULL) { - pIter = taosHashCreateIter(pTable->iHandle); - } + pIter = taosHashIterate(pTable->iHandle, pIter); + if (pIter == NULL) return NULL; - if (!taosHashIterNext(pIter)) { - taosHashDestroyIter(pIter); - return NULL; - } - - void **ppMetaRow = taosHashIterGet(pIter); + void **ppMetaRow = pIter; if (ppMetaRow == NULL) { - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->iHandle, pIter); return NULL; } @@ -783,10 +775,11 @@ void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) { return pIter; } -void sdbFreeIter(void *pIter) { - if (pIter != NULL) { - taosHashDestroyIter(pIter); - } +void sdbFreeIter(void *tparam, void *pIter) { + SSdbTable *pTable = tparam; + if (pTable == NULL || pIter == NULL) return; + + taosHashCancelIterate(pTable->iHandle, pIter); } void *sdbOpenTable(SSdbTableDesc *pDesc) { @@ -827,9 +820,10 @@ void sdbCloseTable(void *handle) { tsSdbMgmt.numOfTables--; tsSdbMgmt.tableList[pTable->id] = NULL; - SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle); - while (taosHashIterNext(pIter)) { - void **ppRow = taosHashIterGet(pIter); + void *pIter = taosHashIterate(pTable->iHandle, NULL); + while (pIter) { + void **ppRow = pIter; + pIter = taosHashIterate(pTable->iHandle, pIter); if (ppRow == NULL) continue; SSdbRow row = { @@ -840,7 +834,7 @@ void sdbCloseTable(void *handle) { (*pTable->fpDestroy)(&row); } - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->iHandle, pIter); taosHashCleanup(pTable->iHandle); pthread_mutex_destroy(&pTable->mutex); diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 0d53fa9617..2621441cb5 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -57,6 +57,7 @@ static void *tsMnodeShowCache = NULL; static int32_t tsShowObjIndex = 0; static SShowMetaFp tsMnodeShowMetaFp[TSDB_MGMT_TABLE_MAX] = {0}; static SShowRetrieveFp tsMnodeShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0}; +static SShowFreeIterFp tsMnodeShowFreeIterFp[TSDB_MGMT_TABLE_MAX] = {0}; int32_t mnodeInitShow() { mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_SHOW, mnodeProcessShowMsg); @@ -85,6 +86,10 @@ void mnodeAddShowRetrieveHandle(uint8_t msgType, SShowRetrieveFp fp) { tsMnodeShowRetrieveFp[msgType] = fp; } +void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp) { + tsMnodeShowFreeIterFp[msgType] = fp; +} + static char *mnodeGetShowType(int32_t showType) { switch (showType) { case TSDB_MGMT_TABLE_ACCT: return "show accounts"; @@ -412,7 +417,9 @@ static void* mnodePutShowObj(SShowObj *pShow) { static void mnodeFreeShowObj(void *data) { SShowObj *pShow = *(SShowObj **)data; - sdbFreeIter(pShow->pIter); + if (tsMnodeShowFreeIterFp[pShow->type] != NULL && pShow->pIter != NULL) { + (*tsMnodeShowFreeIterFp[pShow->type])(pShow->pIter); + } mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index); tfree(pShow); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 4d5f0808f5..8b07f73c65 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -342,8 +342,7 @@ static int32_t mnodeChildTableActionRestored() { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - + mnodeCancelGetNextChildTable(pIter); return 0; } @@ -602,10 +601,13 @@ int32_t mnodeInitTables() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mnodeGetShowTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mnodeRetrieveShowTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_TABLE, mnodeCancelGetNextChildTable); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mnodeGetShowSuperTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mnodeRetrieveShowSuperTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_METRIC, mnodeCancelGetNextSuperTable); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeGetStreamTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeRetrieveStreamTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeCancelGetNextChildTable); return TSDB_CODE_SUCCESS; } @@ -626,14 +628,12 @@ static void *mnodeGetSuperTableByUid(uint64_t uid) { pIter = mnodeGetNextSuperTable(pIter, &pStable); if (pStable == NULL) break; if (pStable->uid == uid) { - sdbFreeIter(pIter); + mnodeCancelGetNextSuperTable(pIter); return pStable; } mnodeDecTableRef(pStable); } - sdbFreeIter(pIter); - return NULL; } @@ -655,10 +655,18 @@ void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) { return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable); } +void mnodeCancelGetNextChildTable(void *pIter) { + sdbFreeIter(tsChildTableSdb, pIter); +} + void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) { return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable); } +void mnodeCancelGetNextSuperTable(void *pIter) { + sdbFreeIter(tsSuperTableSdb, pIter); +} + void mnodeIncTableRef(void *p1) { STableObj *pTable = (STableObj *)p1; if (pTable->type == TSDB_SUPER_TABLE) { @@ -914,10 +922,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { - SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash); - while (taosHashIterNext(pIter)) { - int32_t *pVgId = taosHashIterGet(pIter); + int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL); + while (pVgId) { SVgObj *pVgroup = mnodeGetVgroup(*pVgId); + pVgId = taosHashIterate(pStable->vgHash, pVgId); if (pVgroup == NULL) break; SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg)); @@ -933,7 +941,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { dnodeSendMsgToDnode(&epSet, &rpcMsg); mnodeDecVgroupRef(pVgroup); } - taosHashDestroyIter(pIter); + + taosHashCancelIterate(pStable->vgHash, pVgId); mnodeDropAllChildTablesInStable(pStable); } @@ -1430,8 +1439,6 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } @@ -1523,11 +1530,11 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { } else { SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; - SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash); - int32_t vgSize = 0; - while (taosHashIterNext(pIter)) { - int32_t *pVgId = taosHashIterGet(pIter); - SVgObj * pVgroup = mnodeGetVgroup(*pVgId); + int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL); + int32_t vgSize = 0; + while (pVgId) { + SVgObj *pVgroup = mnodeGetVgroup(*pVgId); + pVgId = taosHashIterate(pTable->vgHash, pVgId); if (pVgroup == NULL) continue; pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId); @@ -1547,7 +1554,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { mnodeDecVgroupRef(pVgroup); } - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->vgHash, pVgId); mnodeDecTableRef(pTable); pVgroupMsg->numOfVgroups = htonl(vgSize); @@ -2230,8 +2237,6 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId); } @@ -2263,8 +2268,6 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); } @@ -2293,8 +2296,6 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index dc76d92eb8..aee167631f 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -123,7 +123,6 @@ static void mnodePrintUserAuth() { } fflush(fp); - sdbFreeIter(pIter); fclose(fp); } @@ -177,6 +176,8 @@ int32_t mnodeInitUsers() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mnodeProcessDropUserMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_USER, mnodeGetUserMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mnodeRetrieveUsers); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_USER, mnodeCancelGetNextUser); + mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mnodeProcessAuthMsg); mDebug("table:%s, hash is created", desc.name); @@ -196,6 +197,10 @@ void *mnodeGetNextUser(void *pIter, SUserObj **pUser) { return sdbFetchRow(tsUserSdb, pIter, (void **)pUser); } +void mnodeCancelGetNextUser(void *pIter) { + sdbFreeIter(tsUserSdb, pIter); +} + void mnodeIncUserRef(SUserObj *pUser) { return sdbIncRef(tsUserSdb, pUser); } @@ -574,8 +579,6 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { mnodeDecUserRef(pUser); } - sdbFreeIter(pIter); - mDebug("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers); } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index f9a49e5ec2..a8343083aa 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -230,6 +230,7 @@ int32_t mnodeInitVgroups() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mnodeGetVgroupMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mnodeRetrieveVgroups); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_VGROUP, mnodeCancelGetNextVgroup); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp); @@ -304,7 +305,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); + mnodeCancelGetNextVgroup(pIter); } void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) { @@ -491,6 +492,10 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) { return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); } +void mnodeCancelGetNextVgroup(void *pIter) { + sdbFreeIter(tsVgroupSdb, pIter); +} + static int32_t mnodeCreateVgroupFp(SMnodeMsg *pMsg) { SVgObj *pVgroup = pMsg->pVgroup; SDbObj *pDb = pMsg->pDb; @@ -1095,8 +1100,6 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("dnode:%d, all vgroups:%d is dropped from sdb", pDropDnode->dnodeId, numOfVgroups); } @@ -1118,8 +1121,6 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name); } #endif @@ -1147,8 +1148,6 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups); } @@ -1170,7 +1169,5 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) { numOfVgroups++; } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups); } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 339391369f..74df989090 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -542,10 +542,11 @@ void vnodeBuildStatusMsg(void *param) { void *pIter = taosHashIterate(tsVnodesHash, NULL); while (pIter) { - SVnodeObj **pVnode = pIter; - if (*pVnode) { - vnodeBuildVloadMsg(*pVnode, pStatus); + SVnodeObj **pVnode = pIter; + if (*pVnode) { + vnodeBuildVloadMsg(*pVnode, pStatus); } + pIter = taosHashIterate(tsVnodesHash, pIter); } } diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 61a9d68d26..5e22e02297 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -14,7 +14,6 @@ run general/table/vgroup.sim run general/user/authority.sim run general/vector/metrics_mix.sim run general/vector/table_field.sim -run general/user/authority.sim run general/tag/set.sim run general/table/delete_writing.sim run general/stable/disk.sim From 58bbdc329348a3c2fd81e48e186dd04043b2ccce Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 22 Nov 2020 17:19:34 +0800 Subject: [PATCH 005/188] TD-2166 add refcount while perform sync --- src/inc/tsync.h | 15 ++-- src/mnode/src/mnodeSdb.c | 15 ++-- src/sync/inc/syncInt.h | 1 - src/sync/src/syncMain.c | 23 +++--- src/sync/src/syncRestore.c | 14 ++-- src/sync/src/syncRetrieve.c | 10 +-- src/sync/test/syncServer.c | 13 ++- src/vnode/src/vnodeMain.c | 157 +++++++++++++++++++++++++----------- 8 files changed, 153 insertions(+), 95 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 967b254992..10b2e3f66a 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -64,33 +64,32 @@ typedef struct { if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated. */ -typedef uint32_t (*FGetFileInfo)(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); +typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); // get the wal file from index or after // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file -typedef int32_t (*FGetWalInfo)(void *ahandle, char *fileName, int64_t *fileId); +typedef int32_t (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); // when a forward pkt is received, call this to handle data -typedef int32_t (*FWriteToCache)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); +typedef int32_t (*FWriteToCache)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); // when forward is confirmed by peer, master call this API to notify app -typedef void (*FConfirmForward)(void *ahandle, void *mhandle, int32_t code); +typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); // when role is changed, call this to notify app -typedef void (*FNotifyRole)(void *ahandle, int8_t role); +typedef void (*FNotifyRole)(int32_t vgId, int8_t role); // if a number of retrieving data failed, call this to start flow control -typedef void (*FNotifyFlowCtrl)(void *ahandle, int32_t mseconds); +typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t mseconds); // when data file is synced successfully, notity app -typedef int32_t (*FNotifyFileSynced)(void *ahandle, uint64_t fversion); +typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version SSyncCfg syncCfg; // configuration from mgmt char path[128]; // path to the file - void * ahandle; // handle provided by APP FGetFileInfo getFileInfo; FGetWalInfo getWalInfo; FWriteToCache writeToCache; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 1728bb7e3b..e5df1d732a 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -107,7 +107,7 @@ static taos_queue tsSdbWQueue; static SSdbWorkerPool tsSdbPool; static int32_t sdbProcessWrite(void *pRow, void *pHead, int32_t qtype, void *unused); -static int32_t sdbWriteWalToQueue(void *vparam, void *pHead, int32_t qtype, void *rparam); +static int32_t sdbWriteWalToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam); static int32_t sdbWriteRowToQueue(SSdbRow *pRow, int32_t action); static void sdbFreeFromQueue(SSdbRow *pRow); static void * sdbWorkerFp(void *pWorker); @@ -228,16 +228,16 @@ void sdbUpdateMnodeRoles() { mnodeUpdateMnodeEpSet(); } -static uint32_t sdbGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { +static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { sdbUpdateMnodeRoles(); return 0; } -static int32_t sdbGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { +static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { return walGetWalFile(tsSdbMgmt.wal, fileName, fileId); } -static void sdbNotifyRole(void *ahandle, int8_t role) { +static void sdbNotifyRole(int32_t vgId, int8_t role) { sdbInfo("vgId:1, mnode role changed from %s to %s", syncRole[tsSdbMgmt.role], syncRole[role]); if (role == TAOS_SYNC_ROLE_MASTER && tsSdbMgmt.role != TAOS_SYNC_ROLE_MASTER) { @@ -264,7 +264,7 @@ static void sdbHandleFailedConfirm(SSdbRow *pRow) { } FORCE_INLINE -static void sdbConfirmForward(void *ahandle, void *wparam, int32_t code) { +static void sdbConfirmForward(int32_t vgId, void *wparam, int32_t code) { if (wparam == NULL) return; SSdbRow *pRow = wparam; SMnodeMsg * pMsg = pRow->pMsg; @@ -370,7 +370,6 @@ void sdbUpdateSync(void *pMnodes) { syncInfo.version = sdbGetVersion(); syncInfo.syncCfg = syncCfg; sprintf(syncInfo.path, "%s", tsMnodeDir); - syncInfo.ahandle = NULL; syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getFileInfo = sdbGetFileInfo; syncInfo.writeToCache = sdbWriteWalToQueue; @@ -967,7 +966,7 @@ static void sdbFreeFromQueue(SSdbRow *pRow) { taosFreeQitem(pRow); } -static int32_t sdbWriteWalToQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { +static int32_t sdbWriteWalToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { SWalHead *pHead = wparam; int32_t size = sizeof(SSdbRow) + sizeof(SWalHead) + pHead->len; @@ -1039,7 +1038,7 @@ static void *sdbWorkerFp(void *pWorker) { taosGetQitem(tsSdbWQall, &qtype, (void **)&pRow); if (qtype == TAOS_QTYPE_RPC) { - sdbConfirmForward(NULL, pRow, pRow->code); + sdbConfirmForward(1, pRow, pRow->code); } else { if (qtype == TAOS_QTYPE_FWD) { syncConfirmForward(tsSdbMgmt.sync, pRow->pHead->version, pRow->code); diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 7d846ebc80..f1371bdc59 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -153,7 +153,6 @@ typedef struct SSyncNode { int8_t selfIndex; uint32_t vgId; int64_t rid; - void *ahandle; SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator SSyncPeer *pMaster; SRecvBuffer *pRecv; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 843de9461f..de0fc8fadc 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -181,7 +181,6 @@ int64_t syncStart(const SSyncInfo *pInfo) { tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path)); pthread_mutex_init(&pNode->mutex, NULL); - pNode->ahandle = pInfo->ahandle; pNode->getFileInfo = pInfo->getFileInfo; pNode->getWalInfo = pInfo->getWalInfo; pNode->writeToCache = pInfo->writeToCache; @@ -255,7 +254,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); if (pNode->notifyRole) { - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } return pNode->rid; @@ -348,7 +347,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { if (pNewCfg->replica <= 1) { sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId); nodeRole = TAOS_SYNC_ROLE_MASTER; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } pthread_mutex_unlock(&(pNode->mutex)); @@ -412,7 +411,7 @@ void syncRecover(int64_t rid) { // if take this node to unsync state, the whole system may not work nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeVersion = 0; pthread_mutex_lock(&(pNode->mutex)); @@ -568,7 +567,7 @@ static void syncResetFlowCtrl(SSyncNode *pNode) { } if (pNode->notifyFlowCtrl) { - (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + (*pNode->notifyFlowCtrl)(pNode->vgId, 0); } } @@ -631,7 +630,7 @@ static void syncChooseMaster(SSyncNode *pNode) { } #endif syncResetFlowCtrl(pNode); - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } else { pPeer = pNode->peerInfo[index]; sInfo("%s, it shall work as master", pPeer->id); @@ -662,7 +661,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (onlineNum <= replica * 0.5) { if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) { nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica); } } else { @@ -675,7 +674,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) { if (masterIndex == pNode->selfIndex) { sError("%s, peer is master, work as slave instead", pTemp->id); nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } } } @@ -692,7 +691,7 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) { sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version); nodeRole = TAOS_SYNC_ROLE_UNSYNCED; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); code = -1; for (int32_t index = 0; index < pNode->replica; ++index) { @@ -729,7 +728,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } else { sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); nodeRole = TAOS_SYNC_ROLE_SLAVE; - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); } } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) { sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion); @@ -913,7 +912,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { // nodeVersion = pHead->version; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL); + (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); } else { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); @@ -1228,7 +1227,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code if (confirm && pFwdInfo->confirmed == 0) { sTrace("vgId:%d, forward is confirmed, hver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code); - (*pNode->confirmForward)(pNode->ahandle, pFwdInfo->mhandle, pFwdInfo->code); + (*pNode->confirmForward)(pNode->vgId, pFwdInfo->mhandle, pFwdInfo->code); pFwdInfo->confirmed = 1; } } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index cc2315fb15..b31ec5f7c7 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -38,7 +38,7 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex while (1) { name[0] = 0; - magic = (*pNode->getFileInfo)(pNode->ahandle, name, &index, eindex, &size, &fversion); + magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion); if (magic == 0) break; snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name); @@ -84,7 +84,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // check the file info sinfo = minfo; sDebug("%s, get file info:%s", pPeer->id, minfo.name); - sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, + sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); // if file not there or magic is not the same, file shall be synced @@ -164,7 +164,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } lastVer = pHead->version; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL, NULL); + (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL); } if (code < 0) { @@ -179,7 +179,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) { SSyncNode *pNode = pPeer->pSyncNode; SWalHead * pHead = (SWalHead *)offset; - (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD, NULL); + (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL); offset += pHead->len + sizeof(SWalHead); return offset; @@ -276,7 +276,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { // if code > 0, data file is changed, notify app, and pass the version if (code > 0 && pNode->notifyFileSynced) { - if ((*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0) { + if ((*pNode->notifyFileSynced)(pNode->vgId, fversion) < 0) { sError("%s, app not in ready state", pPeer->id); return -1; } @@ -307,7 +307,7 @@ void *syncRestoreData(void *param) { taosBlockSIGPIPE(); __sync_fetch_and_add(&tsSyncNum, 1); - (*pNode->notifyRole)(pNode->ahandle, TAOS_SYNC_ROLE_SYNCING); + (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING); if (syncOpenRecvBuffer(pNode) < 0) { sError("%s, failed to allocate recv buffer, restart connection", pPeer->id); @@ -324,7 +324,7 @@ void *syncRestoreData(void *param) { } } - (*pNode->notifyRole)(pNode->ahandle, nodeRole); + (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; taosClose(pPeer->syncFd); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 82f40854e8..7c1286f419 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -108,7 +108,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { while (1) { // retrieve file info fileInfo.name[0] = 0; - fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, + fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); // fileInfo.size = htonl(size); @@ -354,7 +354,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) index++; wname[0] = 0; - code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); + code = (*pNode->getWalInfo)(pNode->vgId, wname, &index); if (code < 0) break; if (wname[0] == 0) { code = 0; @@ -382,7 +382,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { while (1) { // retrieve wal info wname[0] = 0; - code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index); + code = (*pNode->getWalInfo)(pNode->vgId, wname, &index); if (code < 0) break; // error if (wname[0] == 0) { // no wal file sDebug("%s, no wal file", pPeer->id); @@ -487,10 +487,10 @@ void *syncRetrieveData(void *param) { // if file is changed 3 times continuously, start flow control pPeer->numOfRetrieves++; if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl) - (*pNode->notifyFlowCtrl)(pNode->ahandle, 4 << (pPeer->numOfRetrieves - 2)); + (*pNode->notifyFlowCtrl)(pNode->vgId, 4 << (pPeer->numOfRetrieves - 2)); } else { pPeer->numOfRetrieves = 0; - if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->ahandle, 0); + if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0); } pPeer->fileChanged = 0; diff --git a/src/sync/test/syncServer.c b/src/sync/test/syncServer.c index 9dd3feb461..5a64a0a36a 100644 --- a/src/sync/test/syncServer.c +++ b/src/sync/test/syncServer.c @@ -70,7 +70,7 @@ int writeIntoWal(SWalHead *pHead) { return 0; } -void confirmForward(void *ahandle, void *mhandle, int32_t code) { +void confirmForward(int32_t vgId, void *mhandle, int32_t code) { SRpcMsg * pMsg = (SRpcMsg *)mhandle; SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead)); @@ -227,7 +227,7 @@ void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); } -uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { +uint32_t getFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { uint32_t magic; struct stat fstat; char aname[280]; @@ -254,7 +254,7 @@ uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex return magic; } -int getWalInfo(void *ahandle, char *name, int64_t *index) { +int getWalInfo(int32_t vgId, char *name, int64_t *index) { struct stat fstat; char aname[280]; @@ -272,7 +272,7 @@ int getWalInfo(void *ahandle, char *name, int64_t *index) { return 1; } -int writeToCache(void *ahandle, void *data, int type) { +int writeToCache(int32_t vgId, void *data, int type) { SWalHead *pHead = data; uDebug("pkt from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type); @@ -285,9 +285,9 @@ int writeToCache(void *ahandle, void *data, int type) { return 0; } -void confirmFwd(void *ahandle, int64_t version) { return; } +void confirmFwd(int32_t vgId, int64_t version) { return; } -void notifyRole(void *ahandle, int8_t r) { +void notifyRole(int32_t vgId, int8_t r) { role = r; printf("current role:%s\n", syncRole[role]); } @@ -296,7 +296,6 @@ void initSync() { pCfg->replica = 1; pCfg->quorum = 1; syncInfo.vgId = 1; - syncInfo.ahandle = &syncInfo; syncInfo.getFileInfo = getFileInfo; syncInfo.getWalInfo = getWalInfo; syncInfo.writeToCache = writeToCache; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 3f20efb150..50744d59bc 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -30,19 +30,21 @@ static SHashObj*tsVnodesHash; static void vnodeCleanUp(SVnodeObj *pVnode); -static int vnodeProcessTsdbStatus(void *arg, int status, int eno); -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); -static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId); -static void vnodeNotifyRole(void *ahandle, int8_t role); -static void vnodeCtrlFlow(void *handle, int32_t mseconds); -static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion); +static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); +static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); +static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId); +static void vnodeNotifyRole(int32_t vgId, int8_t role); +static void vnodeCtrlFlow(int32_t vgId, int32_t mseconds); +static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); +static void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); +static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); #ifndef _SYNC int64_t syncStart(const SSyncInfo *info) { return NULL; } -int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int qtype) { return 0; } +int32_t syncForwardToPeer(int64_t rid, void *pHead, void *mhandle, int32_t qtype) { return 0; } void syncStop(int64_t rid) {} -int32_t syncReconfig(int64_t rid, const SSyncCfg * cfg) { return 0; } -int syncGetNodesRole(int64_t rid, SNodesRole * cfg) { return 0; } +int32_t syncReconfig(int64_t rid, const SSyncCfg *cfg) { return 0; } +int32_t syncGetNodesRole(int64_t rid, SNodesRole *cfg) { return 0; } void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {} #endif @@ -55,7 +57,7 @@ char* vnodeStatus[] = { }; int32_t vnodeInitResources() { - int code = syncInit(); + int32_t code = syncInit(); if (code != 0) return code; vnodeInitWriteFp(); @@ -325,16 +327,29 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { walRemoveAllOldFiles(pVnode->wal); walRenew(pVnode->wal); + pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); + if (pVnode->qMgmt == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } + + pVnode->events = NULL; + pVnode->status = TAOS_VN_STATUS_READY; + + vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); + tsdbIncCommitRef(pVnode->vgId); + + taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); + SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; syncInfo.version = pVnode->version; syncInfo.syncCfg = pVnode->syncCfg; sprintf(syncInfo.path, "%s", rootDir); - syncInfo.ahandle = pVnode; syncInfo.getWalInfo = vnodeGetWalInfo; syncInfo.getFileInfo = vnodeGetFileInfo; - syncInfo.writeToCache = vnodeWriteToWQueue; - syncInfo.confirmForward = dnodeSendRpcVWriteRsp; + syncInfo.writeToCache = vnodeWriteToCache; + syncInfo.confirmForward = vnodeConfirmForard; syncInfo.notifyRole = vnodeNotifyRole; syncInfo.notifyFlowCtrl = vnodeCtrlFlow; syncInfo.notifyFileSynced = vnodeNotifyFileSynced; @@ -346,23 +361,11 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { if (pVnode->sync <= 0) { vError("vgId:%d, failed to open sync module, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica, tstrerror(terrno)); + vnodeRelease(pVnode); vnodeCleanUp(pVnode); return terrno; } -#endif - - pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); - if (pVnode->qMgmt == NULL) { - vnodeCleanUp(pVnode); - return terrno; - } - - pVnode->events = NULL; - pVnode->status = TAOS_VN_STATUS_READY; - vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); - - tsdbIncCommitRef(pVnode->vgId); - taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); +#endif return TSDB_CODE_SUCCESS; } @@ -389,7 +392,7 @@ void vnodeRelease(void *vparam) { assert(refCount >= 0); if (refCount > 0) { - if (pVnode->status == TAOS_VN_STATUS_RESET && refCount == 2) { + if (pVnode->status == TAOS_VN_STATUS_RESET && refCount <= 3) { tsem_post(&pVnode->sem); } return; @@ -571,7 +574,7 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { if (pVnode->status != TAOS_VN_STATUS_INIT) { // it may be in updateing or reset state, then it shall wait - int i = 0; + int32_t i = 0; while (atomic_val_compare_exchange_8(&pVnode->status, TAOS_VN_STATUS_READY, TAOS_VN_STATUS_CLOSING) != TAOS_VN_STATUS_READY) { if (++i % 1000 == 0) { @@ -595,7 +598,7 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { } // TODO: this is a simple implement -static int vnodeProcessTsdbStatus(void *arg, int status, int eno) { +static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { SVnodeObj *pVnode = arg; if (eno != TSDB_CODE_SUCCESS) { @@ -625,20 +628,41 @@ static int vnodeProcessTsdbStatus(void *arg, int status, int eno) { return 0; } -static uint32_t vnodeGetFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int64_t *size, +static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { - SVnodeObj *pVnode = ahandle; + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while get file info", vgId); + return 0; + } + *fversion = pVnode->fversion; - return tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); + uint32_t ret = tsdbGetFileInfo(pVnode->tsdb, name, index, eindex, size); + + vnodeRelease(pVnode); + return ret; } -static int vnodeGetWalInfo(void *ahandle, char *fileName, int64_t *fileId) { - SVnodeObj *pVnode = ahandle; - return walGetWalFile(pVnode->wal, fileName, fileId); +static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while get wal info", vgId); + return -1; + } + + int32_t code = walGetWalFile(pVnode->wal, fileName, fileId); + + vnodeRelease(pVnode); + return code; } -static void vnodeNotifyRole(void *ahandle, int8_t role) { - SVnodeObj *pVnode = ahandle; +static void vnodeNotifyRole(int32_t vgId, int8_t role) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while notify role", vgId); + return; + } + vInfo("vgId:%d, sync role changed from %s to %s", pVnode->vgId, syncRole[pVnode->role], syncRole[role]); pVnode->role = role; dnodeSendStatusMsgToMnode(); @@ -648,17 +672,26 @@ static void vnodeNotifyRole(void *ahandle, int8_t role) { } else { cqStop(pVnode->cq); } + + vnodeRelease(pVnode); } -static void vnodeCtrlFlow(void *ahandle, int32_t mseconds) { - SVnodeObj *pVnode = ahandle; +static void vnodeCtrlFlow(int32_t vgId, int32_t mseconds) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while ctrl flow", vgId); + return; + } + if (pVnode->delayMs != mseconds) { pVnode->delayMs = mseconds; vDebug("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds); } + + vnodeRelease(pVnode); } -static int vnodeResetTsdb(SVnodeObj *pVnode) { +static int32_t vnodeResetTsdb(SVnodeObj *pVnode) { char rootDir[128] = "\0"; sprintf(rootDir, "%s/tsdb", pVnode->rootDir); @@ -672,7 +705,7 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) { // acquire vnode int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); - if (refCount > 2) { + if (refCount > 3) { tsem_wait(&pVnode->sem); } @@ -692,14 +725,44 @@ static int vnodeResetTsdb(SVnodeObj *pVnode) { return 0; } -static int vnodeNotifyFileSynced(void *ahandle, uint64_t fversion) { - SVnodeObj *pVnode = ahandle; +static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while notify file synced", vgId); + return 0; + } pVnode->fversion = fversion; pVnode->version = fversion; vnodeSaveVersion(pVnode); - vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, - pVnode->version); - return vnodeResetTsdb(pVnode); + vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion); + int32_t code = vnodeResetTsdb(pVnode); + + vnodeRelease(pVnode); + return code; +} + +void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) { + void *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while confirm forward", vgId); + return; + } + + dnodeSendRpcVWriteRsp(pVnode, wparam, code); + vnodeRelease(pVnode); +} + +static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while write to cache", vgId); + return TSDB_CODE_VND_INVALID_VGROUP_ID; + } + + int32_t code = vnodeWriteToWQueue(pVnode, wparam, qtype, rparam); + + vnodeRelease(pVnode); + return code; } From 212fe11d41db46fffbb306f7e794e3ca91bc0be5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 22 Nov 2020 17:46:23 +0800 Subject: [PATCH 006/188] TD-2166 --- src/vnode/src/vnodeMain.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 50744d59bc..75eacc8b49 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -63,7 +63,7 @@ int32_t vnodeInitResources() { vnodeInitWriteFp(); vnodeInitReadFp(); - tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + tsVnodesHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (tsVnodesHash == NULL) { vError("failed to init vnode list"); return TSDB_CODE_VND_OUT_OF_MEMORY; @@ -175,8 +175,8 @@ int32_t vnodeDrop(int32_t vgId) { return TSDB_CODE_SUCCESS; } -int32_t vnodeAlter(void *param, SCreateVnodeMsg *pVnodeCfg) { - SVnodeObj *pVnode = param; +int32_t vnodeAlter(void *vparam, SCreateVnodeMsg *pVnodeCfg) { + SVnodeObj *pVnode = vparam; // vnode in non-ready state and still needs to return success instead of TSDB_CODE_VND_INVALID_STATUS // cfgVersion can be corrected by status msg @@ -334,7 +334,6 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } pVnode->events = NULL; - pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); tsdbIncCommitRef(pVnode->vgId); @@ -367,6 +366,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } #endif + pVnode->status = TAOS_VN_STATUS_READY; return TSDB_CODE_SUCCESS; } @@ -611,17 +611,18 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { if (status == TSDB_STATUS_COMMIT_START) { pVnode->fversion = pVnode->version; vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); - if (pVnode->status == TAOS_VN_STATUS_INIT) { - return 0; - } else { + if (pVnode->status != TAOS_VN_STATUS_INIT) { return walRenew(pVnode->wal); } + return 0; } if (status == TSDB_STATUS_COMMIT_OVER) { vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); pVnode->isFull = 0; - walRemoveOneOldFile(pVnode->wal); + if (pVnode->status != TAOS_VN_STATUS_INIT) { + walRemoveOneOldFile(pVnode->wal); + } return vnodeSaveVersion(pVnode); } @@ -701,7 +702,7 @@ static int32_t vnodeResetTsdb(SVnodeObj *pVnode) { void *tsdb = pVnode->tsdb; pVnode->tsdb = NULL; - + // acquire vnode int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); From 4f1666e79f4fdb08b14d1ba3f6b6e27940fdd0ec Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sun, 22 Nov 2020 18:52:41 +0800 Subject: [PATCH 007/188] [TD-1880] add test case --- tests/pytest/cluster/clusterSetup.py | 27 ++++++++++++++++++---- tests/pytest/query/queryJoin.py | 34 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/tests/pytest/cluster/clusterSetup.py b/tests/pytest/cluster/clusterSetup.py index 36af8ac42e..dbda5657b6 100644 --- a/tests/pytest/cluster/clusterSetup.py +++ b/tests/pytest/cluster/clusterSetup.py @@ -31,6 +31,23 @@ class Node: self.homeDir = homeDir self.conn = Connection("{}@{}".format(username, hostName), connect_kwargs={"password": "{}".format(password)}) + def buildTaosd(self): + try: + self.conn.cd("/root/TDinternal/community") + self.conn.run("git checkout develop") + self.conn.run("git pull") + self.conn.cd("/root/TDinternal") + self.conn.run("git checkout develop") + self.conn.run("git pull") + self.conn.cd("/root/TDinternal/debug") + self.conn.run("cmake ..") + self.conn.run("make") + self.conn.run("make install") + except Exception as e: + print("Build Taosd error for node %d " % self.index) + logging.exception(e) + pass + def startTaosd(self): try: self.conn.run("sudo systemctl start taosd") @@ -50,7 +67,7 @@ class Node: self.conn.run("sudo systemctl restart taosd") except Exception as e: print("Stop Taosd error for node %d " % self.index) - logging.exception(e) + logging.exception(e) def removeTaosd(self): try: @@ -105,9 +122,11 @@ class Node: class Nodes: def __init__(self): - self.node1 = Node(1, 'ubuntu', '192.168.1.52', 'node1', 'tbase125!', '/home/ubuntu') - self.node2 = Node(2, 'ubuntu', '192.168.1.53', 'node2', 'tbase125!', '/home/ubuntu') - self.node3 = Node(3, 'ubuntu', '192.168.1.54', 'node3', 'tbase125!', '/home/ubuntu') + self.node1 = Node(1, 'root', '52.151.60.239', 'node1', 'r', '/root/') + self.node2 = Node(2, 'root', '52.183.32.246', 'node1', 'r', '/root/') + self.node3 = Node(3, 'root', '51.143.46.79', 'node1', 'r', '/root/') + self.node4 = Node(4, 'root', '52.183.2.76', 'node1', 'r', '/root/') + self.node5 = Node(5, 'root', '13.66.225.87', 'node1', 'r', '/root/') def stopAllTaosd(self): self.node1.stopTaosd() diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index 5ad49a265e..57b7e8868d 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -141,6 +141,40 @@ class TDTestCase: tdSql.query("select * from meters1, meters3 where meters1.ts = meters3.ts and meters1.tag1 = meters3.tag1") tdSql.checkRows(0) + tdSql.execute("create table join_mt0(ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) tags(t1 int, t2 binary(12))") + tdSql.execute("create table join_mt1(ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool, c8 binary(10), c9 nchar(9)) tags(t1 int, t2 binary(12), t3 int)") + + ts = 1538548685000 + for i in range(3): + tdSql.execute("create table join_tb%d using join_mt0 tags(%d, 'abc')" % (i, i)) + sql = "insert into join_tb%d values" % i + for j in range(500): + val = j % 100 + sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + j, val, val * 1.0, val, val, val, val * 1.0, val % 2, val, val) + tdSql.execute(sql) + sql = "insert into join_tb%d values" % i + for j in range(500, 1000): + val = j % 100 + sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + 500 + j, val, val * 1.0, val, val, val, val * 1.0, val % 2, val, val) + tdSql.execute(sql) + + for i in range(3): + tdSql.execute("create table join_1_tb%d using join_mt1 tags(%d, 'abc%d', %d)" % (i, i, i, i)) + sql = "insert into join_1_tb%d values" % i + for j in range(500): + val = j % 100 + sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + j, val, val * 1.0, val, val, val, val * 1.0, val % 2, val, val) + tdSql.execute(sql) + sql = "insert into join_1_tb%d values" % i + for j in range(500, 1000): + val = j % 100 + sql += "(%d, %d, %f, %d, %d, %d, %f, %d, 'binary%d', 'nchar%d')" % (ts + 500 + j, val, val * 1.0, val, val, val, val * 1.0, val % 2, val, val) + tdSql.execute(sql) + + tdSql.error("select count(join_mt0.c1), sum(join_mt1.c2), first(join_mt0.c5), last(join_mt1.c7) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1 order by join_mt0.ts desc") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3") + tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") def stop(self): tdSql.close() From 38af2df40bc68cde730e32c6e550c2209edab656 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 22 Nov 2020 21:06:35 +0800 Subject: [PATCH 008/188] TD-2166 --- src/dnode/src/dnodeEps.c | 2 +- src/inc/tsync.h | 2 +- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeTable.c | 2 +- src/sync/inc/syncInt.h | 10 +++++----- src/sync/src/syncMain.c | 4 ++-- src/vnode/src/vnodeRead.c | 3 +-- src/vnode/src/vnodeWrite.c | 8 ++++---- 8 files changed, 16 insertions(+), 17 deletions(-) diff --git a/src/dnode/src/dnodeEps.c b/src/dnode/src/dnodeEps.c index 9c90c39181..83f294e05e 100644 --- a/src/dnode/src/dnodeEps.c +++ b/src/dnode/src/dnodeEps.c @@ -33,7 +33,7 @@ static void dnodePrintEps(SDnodeEps *eps); int32_t dnodeInitEps() { pthread_mutex_init(&tsEpsMutex, NULL); - tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + tsEpsHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); dnodeResetEps(NULL); int32_t ret = dnodeReadEps(); if (ret == 0) { diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 10b2e3f66a..77a3b36e73 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -89,7 +89,7 @@ typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version SSyncCfg syncCfg; // configuration from mgmt - char path[128]; // path to the file + char path[TSDB_FILENAME_LEN]; // path to the file FGetFileInfo getFileInfo; FGetWalInfo getWalInfo; FWriteToCache writeToCache; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index e5df1d732a..26a881d477 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -812,7 +812,7 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) { hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); } - pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, true); + pTable->iHandle = taosHashInit(pTable->hashSessions, hashFp, true, HASH_ENTRY_LOCK); tsSdbMgmt.numOfTables++; tsSdbMgmt.tableList[pTable->id] = pTable; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 4d5f0808f5..c4601658b3 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -394,7 +394,7 @@ static void mnodeAddTableIntoStable(SSTableObj *pStable, SCTableObj *pCtable) { atomic_add_fetch_32(&pStable->numOfTables, 1); if (pStable->vgHash == NULL) { - pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + pStable->vgHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } if (pStable->vgHash != NULL) { diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index f1371bdc59..0790134997 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -153,12 +153,12 @@ typedef struct SSyncNode { int8_t selfIndex; uint32_t vgId; int64_t rid; - SSyncPeer *peerInfo[TAOS_SYNC_MAX_REPLICA+1]; // extra one for arbitrator - SSyncPeer *pMaster; + SSyncPeer * peerInfo[TAOS_SYNC_MAX_REPLICA + 1]; // extra one for arbitrator + SSyncPeer * pMaster; SRecvBuffer *pRecv; - SSyncFwds *pSyncFwds; // saved forward info if quorum >1 - void *pFwdTimer; - void *pRoleTimer; + SSyncFwds * pSyncFwds; // saved forward info if quorum >1 + void * pFwdTimer; + void * pRoleTimer; FGetFileInfo getFileInfo; FGetWalInfo getWalInfo; FWriteToCache writeToCache; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index de0fc8fadc..2402d8ebfa 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -100,7 +100,7 @@ uint16_t syncGenTranId() { } int32_t syncInit() { - SPoolInfo info; + SPoolInfo info = {0}; info.numOfThreads = tsSyncTcpThreads; info.serverIp = 0; @@ -124,7 +124,7 @@ int32_t syncInit() { return -1; } - tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, true); + tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (tsVgIdHash == NULL) { sError("failed to init tsVgIdHash"); taosTmrCleanUp(tsSyncTmrCtrl); diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index e10d62a0c9..5ef79cfbf0 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -53,8 +53,7 @@ int32_t vnodeProcessRead(void *vparam, SVReadMsg *pRead) { return (*vnodeProcessReadMsgFp[msgType])(pVnode, pRead); } -static int32_t vnodeCheckRead(void *vparam) { - SVnodeObj *pVnode = vparam; +static int32_t vnodeCheckRead(SVnodeObj *pVnode) { if (pVnode->status != TAOS_VN_STATUS_READY) { vDebug("vgId:%d, vnode status is %s, refCount:%d pVnode:%p", pVnode->vgId, vnodeStatus[pVnode->status], pVnode->refCount, pVnode); diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 2d2be602ed..a0227d84ba 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -48,10 +48,10 @@ void vnodeInitWriteFp(void) { } int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { - int32_t code = 0; - SVnodeObj * pVnode = vparam; - SWalHead * pHead = wparam; - SRspRet * pRspRet = rparam; + int32_t code = 0; + SVnodeObj *pVnode = vparam; + SWalHead * pHead = wparam; + SRspRet * pRspRet = rparam; if (vnodeProcessWriteMsgFp[pHead->msgType] == NULL) { vError("vgId:%d, msg:%s not processed since no handle, qtype:%s hver:%" PRIu64, pVnode->vgId, From 94e662c9b0245774aa46ffc85f20890328b4626b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sun, 22 Nov 2020 22:51:57 +0800 Subject: [PATCH 009/188] minor changes --- src/mnode/src/mnodeTable.c | 2 +- src/sync/src/syncMain.c | 40 +++++++++++++++++++------------------- src/vnode/src/vnodeMain.c | 4 ++-- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index c4601658b3..d0a5402986 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -413,7 +413,7 @@ static void mnodeRemoveTableFromStable(SSTableObj *pStable, SCTableObj *pCtable) SVgObj *pVgroup = mnodeGetVgroup(pCtable->vgId); if (pVgroup == NULL) { - taosHashRemove(pStable->vgHash, (char *)&pCtable->vgId, sizeof(pCtable->vgId)); + taosHashRemove(pStable->vgHash, &pCtable->vgId, sizeof(pCtable->vgId)); mDebug("table:%s, vgId:%d is remove from stable vgList, sizeOfVgList:%d", pStable->info.tableId, pCtable->vgId, (int32_t)taosHashGetSize(pStable->vgHash)); } diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 2402d8ebfa..41d68b3149 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -201,10 +201,10 @@ int64_t syncStart(const SSyncInfo *pInfo) { return -1; } - for (int32_t i = 0; i < pCfg->replica; ++i) { - const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i; - pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo); - if (pNode->peerInfo[i] == NULL) { + for (int32_t index = 0; index < pCfg->replica; ++index) { + const SNodeInfo *pNodeInfo = pCfg->nodeInfo + index; + pNode->peerInfo[index] = syncAddPeer(pNode, pNodeInfo); + if (pNode->peerInfo[index] == NULL) { sError("vgId:%d, node:%d fqdn:%s port:%u is not configured, stop taosd", pNode->vgId, pNodeInfo->nodeId, pNodeInfo->nodeFqdn, pNodeInfo->nodePort); syncStop(pNode->rid); @@ -212,7 +212,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { } if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) { - pNode->selfIndex = i; + pNode->selfIndex = index; } } @@ -251,7 +251,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { } syncAddArbitrator(pNode); - taosHashPut(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *)); + taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *)); if (pNode->notifyRole) { (*pNode->notifyRole)(pNode->vgId, nodeRole); @@ -268,14 +268,14 @@ void syncStop(int64_t rid) { sInfo("vgId:%d, cleanup sync", pNode->vgId); - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); - if (tsVgIdHash) taosHashRemove(tsVgIdHash, (const char *)&pNode->vgId, sizeof(int32_t)); + if (tsVgIdHash) taosHashRemove(tsVgIdHash, &pNode->vgId, sizeof(int32_t)); if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer); if (pNode->pRoleTimer) taosTmrStop(pNode->pRoleTimer); - for (int32_t i = 0; i < pNode->replica; ++i) { - pPeer = pNode->peerInfo[i]; + for (int32_t index = 0; index < pNode->replica; ++index) { + pPeer = pNode->peerInfo[index]; if (pPeer) syncRemovePeer(pPeer); } @@ -297,7 +297,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica, pNode->replica); - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); for (i = 0; i < pNode->replica; ++i) { for (j = 0; j < pNewCfg->replica; ++j) { @@ -414,7 +414,7 @@ void syncRecover(int64_t rid) { (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeVersion = 0; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; @@ -831,7 +831,7 @@ static void syncNotStarted(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); pPeer->timer = NULL; sInfo("%s, sync connection is still not up, restart", pPeer->id); syncRestartConnection(pPeer); @@ -842,7 +842,7 @@ static void syncTryRecoverFromMaster(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); syncRecoverFromMaster(pPeer); pthread_mutex_unlock(&(pNode->mutex)); } @@ -968,7 +968,7 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) { char * cont = buffer; SSyncNode *pNode = pPeer->pSyncNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); int32_t code = syncReadPeerMsg(pPeer, &head, cont); @@ -1066,7 +1066,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); sDebug("%s, check peer connection", pPeer->id); syncSetupPeerConnection(pPeer); @@ -1118,7 +1118,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { } SSyncNode *pNode = *ppNode; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); SSyncPeer *pPeer; for (i = 0; i < pNode->replica; ++i) { @@ -1156,7 +1156,7 @@ static void syncProcessBrokenLink(void *param) { SSyncNode *pNode = pPeer->pSyncNode; if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); sDebug("%s, TCP link is broken since %s", pPeer->id, strerror(errno)); pPeer->peerFd = -1; @@ -1262,7 +1262,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { int64_t time = taosGetTimestampMs(); if (pSyncFwds->fwds > 0) { - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pSyncFwds->fwds; ++i) { SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first + i) % tsMaxFwdInfo; if (ABS(time - pFwdInfo->time) < 2000) break; @@ -1320,7 +1320,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle pSyncHead->len = sizeof(SWalHead) + pWalHead->len; fwdLen = pSyncHead->len + sizeof(SSyncHead); // include the WAL and SYNC head - pthread_mutex_lock(&(pNode->mutex)); + pthread_mutex_lock(&pNode->mutex); for (int32_t i = 0; i < pNode->replica; ++i) { pPeer = pNode->peerInfo[i]; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 75eacc8b49..b94fea52bd 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -338,7 +338,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); tsdbIncCommitRef(pVnode->vgId); - taosHashPut(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t), (char *)(&pVnode), sizeof(SVnodeObj *)); + taosHashPut(tsVnodesHash, &pVnode->vgId, sizeof(int32_t), &pVnode, sizeof(SVnodeObj *)); SSyncInfo syncInfo; syncInfo.vgId = pVnode->vgId; @@ -570,7 +570,7 @@ void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { static void vnodeCleanUp(SVnodeObj *pVnode) { // remove from hash, so new messages wont be consumed - taosHashRemove(tsVnodesHash, (const char *)&pVnode->vgId, sizeof(int32_t)); + taosHashRemove(tsVnodesHash, &pVnode->vgId, sizeof(int32_t)); if (pVnode->status != TAOS_VN_STATUS_INIT) { // it may be in updateing or reset state, then it shall wait From d0e85228401fe967cb99dc06acc8a65bef573615 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 11:12:58 +0800 Subject: [PATCH 010/188] [TD-2175]: fix the error in result field name and nchar column length --- src/client/src/tscSql.c | 11 ++++++++++- src/query/inc/sql.y | 8 ++++---- src/query/src/sql.c | 11 +++++------ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 020305a0a8..de01776472 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -420,7 +420,16 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { SInternalField* pField = tscFieldInfoGetInternalField(pFieldInfo, i); if (pField->visible) { - f[j++] = pField->field; + f[j] = pField->field; + + // revise the length for binary and nchar fields + if (f[j].type == TSDB_DATA_TYPE_BINARY) { + f[j].bytes -= VARSTR_HEADER_SIZE; + } else if (f[j].type == TSDB_DATA_TYPE_NCHAR) { + f[j].bytes = (f[j].bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; + } + + j += 1; } } diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 09b1e1592a..e2b3bb6cbf 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -556,11 +556,11 @@ having_opt(A) ::= HAVING expr(X). {A = X;} //limit-offset subclause %type limit_opt {SLimitVal} limit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X). {printf("aa1, %d\n", X); A.limit = X; A.offset = 0;} +limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;} limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y). - {printf("aa2\n, %d\n", X); A.limit = X; A.offset = Y;} + { A.limit = X; A.offset = Y;} limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y). - {printf("aa3\n, %d\n", X); A.limit = Y; A.offset = X;} + { A.limit = Y; A.offset = X;} %type slimit_opt {SLimitVal} slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;} @@ -581,7 +581,7 @@ where_opt(A) ::= WHERE expr(X). {A = X;} %type expr {tSQLExpr*} %destructor expr {tSQLExprDestroy($$);} -expr(A) ::= LP expr(X) RP. {A = X; } +expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->token.z = X.z; A->token.n = (Z.z - X.z + 1);} expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);} expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} diff --git a/src/query/src/sql.c b/src/query/src/sql.c index da2c56ee9e..d4e2b7f5f4 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -2628,16 +2628,14 @@ static void yy_reduce( {yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;} break; case 177: /* limit_opt ::= LIMIT signed */ + case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181); {yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} break; case 178: /* limit_opt ::= LIMIT signed OFFSET signed */ -{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} +{ yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} break; case 179: /* limit_opt ::= LIMIT signed COMMA signed */ -{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} - break; - case 181: /* slimit_opt ::= SLIMIT signed */ -{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} +{ yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} break; case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ {yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} @@ -2646,7 +2644,8 @@ static void yy_reduce( {yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} break; case 186: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; } +{yylhsminor.yy64 = yymsp[-1].minor.yy64; yylhsminor.yy64->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy64->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; case 187: /* expr ::= ID */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} From 98ba80a993556d12fcc8cffa26d70c711445bc23 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 11:13:46 +0800 Subject: [PATCH 011/188] [TD-225] refactor codes. --- src/query/inc/qUtil.h | 2 ++ src/query/src/qExecutor.c | 54 ++++++++++++++++++++++++-------------- src/query/src/qResultbuf.c | 2 +- src/query/src/qUtil.c | 51 +++++++++++++++++------------------ 4 files changed, 63 insertions(+), 46 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 4cd0e60ebe..b2c4b6e2ec 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -80,5 +80,7 @@ void* destroyResultRowPool(SResultRowPool* p); int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); +uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv); + #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d0874c36bc..c44d65e373 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -186,7 +186,7 @@ static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); -static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); +static void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); @@ -283,8 +283,8 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } } -static UNUSED_FUNC int32_t getGroupResultId(int32_t groupIndex) { - int32_t base = 20000000; +static int32_t getMergeResultGroupId(int32_t groupIndex) { + int32_t base = 50000000; return base + (groupIndex * 10000); } @@ -1115,7 +1115,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - // not assign result buffer yet, add new result buffer + // not assign result buffer yet, add new result buffer, TODO remove it char* d = pData; int16_t len = bytes; if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) { @@ -1128,7 +1128,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - uint64_t uid = groupIndex; // uid is always set to be 0. + uint64_t uid = groupIndex; SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); if (pResultRow == NULL) { return -1; @@ -1714,7 +1714,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order // if it is group by normal column, do not set output buffer, the output buffer is pResult // fixed output query/multi-output query for normal table if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); } if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { @@ -2936,10 +2936,24 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { assert(size == pGroupResInfo->numOfDataPages); bool done = false; + + //TODO add API for release none-dirty pages +// SPageInfo* prev = NULL; + for (int32_t j = pGroupResInfo->pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); + // release previous buffer pages +// if (prev == NULL) { +// prev = pi; +// } else { +// if (prev->pageId != pi->pageId) { +// releaseResBufPageInfo(pResultBuf, prev); +// prev = pi; +// } +// } + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num); int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId); @@ -3058,7 +3072,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); + pQInfo->groupResInfo.groupId = getMergeResultGroupId(pQInfo->groupIndex); // todo add windowRes iterator int64_t lastTimestamp = -1; @@ -3339,12 +3353,12 @@ int32_t initResultRow(SResultRow *pResultRow) { return TSDB_CODE_SUCCESS; } -void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { +void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t groupIndex = 0; - int32_t uid = 0; - SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); + int32_t tid = 0; + int64_t uid = getResultInfoUId(pRuntimeEnv); + SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -3427,7 +3441,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= pQuery->rec.rows; pQuery->rec.rows = 0; - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); // clear the buffer full flag if exists CLEAR_QUERY_STATUS(pQuery, QUERY_RESBUF_FULL); @@ -3792,7 +3806,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - uint64_t uid = 0; // uid is always set to be 0 + uint64_t uid = getResultInfoUId(pRuntimeEnv); SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); if (pResultRow == NULL) { @@ -4629,10 +4643,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo int32_t ps = DEFAULT_PAGE_SIZE; int32_t rowsize = 0; getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - int32_t TWOMB = 1024*1024*2; + int32_t TENMB = 1024*1024*10; if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) { - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4660,7 +4674,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { int32_t numOfResultRows = getInitialPageNum(pQInfo); getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4930,7 +4944,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); if (isPointInterpoQuery(pQuery)) { - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); while (pQInfo->groupIndex < numOfGroups) { @@ -5096,7 +5110,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { return; } - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); SArray *group = GET_TABLEGROUP(pQInfo, 0); @@ -5456,7 +5470,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); } // skip blocks without load the actual data block from file if no filter condition present @@ -5486,7 +5500,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) qDebug("QInfo:%p skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->limit.offset, pQuery->current->lastKey, pQuery->current->win.ekey); - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); } limitResults(pRuntimeEnv); diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index c5ba551f20..67cb7af074 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -407,7 +407,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f", + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb", pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, pResultBuf->fileSize/1024.0); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index c5317226c7..650b886c36 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -20,6 +20,18 @@ #include "qExecutor.h" #include "qUtil.h" +static int32_t getResultRowKeyInfo(SResultRow* pResult, int16_t type, char** key, int16_t* bytes) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + *key = varDataVal(pResult->key); + *bytes = varDataLen(pResult->key); + } else { + *key = (char*) &pResult->win.skey; + *bytes = tDataTypeDesc[type].nSize; + } + + return 0; +} + int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; @@ -94,12 +106,8 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); assert(num >= 0 && num <= numOfClosed); - int16_t type = pWindowResInfo->type; - STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0. - int64_t uid = id->uid; - if (pRuntimeEnv->groupbyNormalCol) { - uid = 0; - } + int16_t type = pWindowResInfo->type; + int64_t uid = getResultInfoUId(pRuntimeEnv); char *key = NULL; int16_t bytes = -1; @@ -107,16 +115,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t i = 0; i < num; ++i) { SResultRow *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table - - // todo refactor - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - key = varDataVal(pResult->key); - bytes = varDataLen(pResult->key); - } else { - key = (char*) &pResult->win.skey; - bytes = tDataTypeDesc[pWindowResInfo->type].nSize; - } - + getResultRowKeyInfo(pResult, type, &key, &bytes); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); } else { @@ -141,16 +140,9 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SResultRow *pResult = pWindowResInfo->pResult[k]; - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - key = varDataVal(pResult->key); - bytes = varDataLen(pResult->key); - } else { - key = (char*) &pResult->win.skey; - bytes = tDataTypeDesc[pWindowResInfo->type].nSize; - } - + getResultRowKeyInfo(pResult, type, &key, &bytes); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); + int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); assert(p != NULL); @@ -379,3 +371,12 @@ void* destroyResultRowPool(SResultRowPool* p) { tfree(p); return NULL; } + +uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { + if (!pRuntimeEnv->stableQuery) { + return 0; // for simple table query, the uid is always set to be 0; + } + + STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current); + return id->uid; +} \ No newline at end of file From 227823762cac5dbfa333308d023c6c0a80463e5c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 11:14:27 +0800 Subject: [PATCH 012/188] [TD-2175] add test cases. --- src/client/tests/resultFieldTest.cpp | 138 +++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 src/client/tests/resultFieldTest.cpp diff --git a/src/client/tests/resultFieldTest.cpp b/src/client/tests/resultFieldTest.cpp new file mode 100644 index 0000000000..c917f0ebaf --- /dev/null +++ b/src/client/tests/resultFieldTest.cpp @@ -0,0 +1,138 @@ +#include +#include + +#include "taos.h" + +namespace { +static int64_t start_ts = 1433955661000; +} +/* test parse time function */ +TEST(testCase, result_field_test) { + taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + taos_init(); + + TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); + if (conn == NULL) { + printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); + exit(-1); + } + + TAOS_RES* res = taos_query(conn, "create database if not exists test"); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + res = taos_query(conn, "use test"); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + res = taos_query(conn, "create table if not exists t1(ts timestamp, k int, a binary(11), b nchar(4))"); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + char sql[512] = {0}; + sprintf(sql, "insert into t1 values(%ld, 99, 'abc', 'test')", start_ts); + + res = taos_query(conn, sql); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + res = taos_query(conn, "select count(*), spread(ts)/(1000 * 3600 * 24), first(a), last(b) from t1"); + ASSERT_EQ(taos_num_fields(res), 4); + + TAOS_FIELD* fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_BIGINT); + ASSERT_STREQ(fields[0].name, "count(*)"); + + ASSERT_EQ(fields[1].bytes, 8); + ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STREQ(fields[1].name, "spread(ts)/(1000 * 3600 * 24)"); + + ASSERT_EQ(fields[2].bytes, 11); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); + ASSERT_STREQ(fields[2].name, "first(a)"); + + ASSERT_EQ(fields[3].bytes, 4); + ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[3].name, "last(b)"); + + taos_free_result(res); + + res = taos_query(conn, "select last_row(*) from t1"); + ASSERT_EQ(taos_num_fields(res), 4); + + fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[0].name, "last_row(ts)"); + + ASSERT_EQ(fields[1].bytes, 4); + ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[1].name, "last_row(k)"); + + ASSERT_EQ(fields[2].bytes, 11); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); + ASSERT_STREQ(fields[2].name, "last_row(a)"); + + ASSERT_EQ(fields[3].bytes, 4); + ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[3].name, "last_row(b)"); + + taos_free_result(res); + res = taos_query(conn, "select first(*), last(*) from t1"); + ASSERT_EQ(taos_num_fields(res), 8); + + fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[0].name, "first(ts)"); + + ASSERT_EQ(fields[1].bytes, 4); + ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[1].name, "first(k)"); + + ASSERT_EQ(fields[2].bytes, 11); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); + ASSERT_STREQ(fields[2].name, "first(a)"); + + ASSERT_EQ(fields[3].bytes, 4); + ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[3].name, "first(b)"); + + taos_free_result(res); + + res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1"); + ASSERT_EQ(taos_num_fields(res), 7); + + fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[0].name, "first(ts)"); + + ASSERT_EQ(fields[1].bytes, 11); + ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_BINARY); + ASSERT_STREQ(fields[1].name, "first(a)"); + + ASSERT_EQ(fields[2].bytes, 4); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[2].name, "first(k)"); + + ASSERT_EQ(fields[3].bytes, 4); + ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[3].name, "first(k)"); + + ASSERT_EQ(fields[4].bytes, 4); + ASSERT_EQ(fields[4].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[4].name, "first(b)"); + + ASSERT_EQ(fields[5].bytes, 4); + ASSERT_EQ(fields[5].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[5].name, "first(b)"); + + ASSERT_EQ(fields[6].bytes, 8); + ASSERT_EQ(fields[6].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[6].name, "first(ts)"); + + taos_free_result(res); + taos_close(conn); +} From 6a837620f0539d30ebb6affe5a7fa89f3848db93 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 11:15:59 +0800 Subject: [PATCH 013/188] [TD-225] update the sim script. --- .../parser/col_arithmetic_operation.sim | 100 +++----- .../general/parser/col_arithmetic_query.sim | 240 ++++++++++++++++++ 2 files changed, 269 insertions(+), 71 deletions(-) create mode 100644 tests/script/general/parser/col_arithmetic_query.sim diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index d5ba57e6c7..c05229ab65 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -5,6 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start sleep 500 sql connect +#========================================= setup environment ================================ $dbPrefix = ca_db $tbPrefix = ca_tb @@ -52,44 +53,39 @@ while $i < $halfTbNum $i = $i + 1 endw -##### select from table -$tb = $tbPrefix . 0 -## TBASE-344 -sql select c1*2 from $tb -if $rows != $rowNum then - return -1 -endi -if $data00 != 0.000000000 then - return -1 -endi -if $data10 != 2.000000000 then - return -1 -endi -if $data20 != 4.000000000 then - return -1 -endi -if $data90 != 18.000000000 then - return -1 -endi +#=================================== above are setup test environment ============================= +run general/parser/col_arithmetic_query.sim -sql select c4*1+1/2 from $tb -if $rows != $rowNum then - return -1 -endi -if $data00 != 0.500000000 then - return -1 -endi -if $data10 != 1.500000000 then - return -1 -endi -if $data90 != 9.500000000 then - return -1 -endi +#======================================= all in files query ======================================= +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 3000 +system sh/exec.sh -n dnode1 -s start -#### illegal operations +print ================== server restart completed +sql connect +sleep 500 + +run general/parser/col_arithmetic_query.sim + +# ================================================================================================ + +print ====================> crash +# sql select spread(ts )/(1000*3600*24) from ca_stb0 interval(1y) + + +sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y) +sql_error select first(ts) - last(ts) from stb interval(1y) +sql_error select top(c1, 2) - last(c1) from stb; +sql_error select stddev(c1) - last(c1) from stb; +sql_error select diff(c1) - last(c1) from stb; +sql_error select first(c7) - last(c7) from stb; +sql_error select first(c8) - last(c8) from stb; +sql_error select first(c9) - last(c9) from stb; sql_error select max(c2*2) from $tb sql_error select max(c1-c2) from $tb +#========================================regression test cases==================================== print =====================> td-1764 sql select sum(c1)/count(*), sum(c1) as b, count(*) as b from $stb interval(1y) if $rows != 1 then @@ -108,42 +104,4 @@ if $data02 != 225000 then return -1 endi -sql select first(c1) - last(c1), first(c1) as b, last(c1) as b, min(c1) - max(c1), spread(c1) from ca_stb0 interval(1y) -if $rows != 1 then - return -1 -endi - -if $data00 != @18-01-01 00:00:00.000@ then - return -1 -endi - -if $data01 != -9.000000000 then - return -1 -endi - -if $data02 != 0 then - return -1 -endi - -if $data03 != 9 then - return -1 -endi - -if $data04 != -9.000000000 then - return -1 -endi - -if $data05 != 9.000000000 then - return -1 -endi - -sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y) -sql_error select first(ts) - last(ts) from stb interval(1y) -sql_error select top(c1, 2) - last(c1) from stb; -sql_error select stddev(c1) - last(c1) from stb; -sql_error select diff(c1) - last(c1) from stb; -sql_error select first(c7) - last(c7) from stb; -sql_error select first(c8) - last(c8) from stb; -sql_error select first(c9) - last(c9) from stb; - system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim new file mode 100644 index 0000000000..95258168ef --- /dev/null +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -0,0 +1,240 @@ +# ======================================= query test cases ======================================== +# select from table + +$tb = $tbPrefix . 0 +## TBASE-344 +sql select c1*2 from $tb +if $rows != $rowNum then + return -1 +endi +if $data00 != 0.000000000 then + return -1 +endi +if $data10 != 2.000000000 then + return -1 +endi +if $data20 != 4.000000000 then + return -1 +endi +if $data90 != 18.000000000 then + return -1 +endi + +# asc/desc order [d.2] +sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc; +sql select (c1 * 2) % 7.9 from $tb order by ts desc; + +sql select c1 * c2 /4 from $tb where ts < and ts > + +# no result return [d.3] +sql select c1 * 91- 7 from $tb where ts<1537146000000 +if $rows != 0 then + return -1 +endi + +# no result return [d.3] +sql select c2 - c2 from $tb where ts>xxx +if $rows != 0 then + return -1 +endi + +# single row result aggregation [d.4] +# not available + +# error cases +sql_error select first(c1,c2) - last(c1,c2) from $tb + +# multi row result aggregation [d.4] +sql select top(c1, 1) - bottom(c1, 1) from $tb +sql select top(c1, 99) - bottom(c1, 99) from $tb + +# all data types [d.6] +sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $tb + + + +# error case, ts/bool/binary/nchar not support arithmetic expression +sql_error select ts+ts from $tb +sql_error select ts+22 from $tb +sql_error select c7*12 from $tb +sql_error select c8/55 from $tb +sql_error select c9+c8 from $tb +sql_error select c7-c8, c9-c8 from $tb +sql_error select ts-c9 from $tb +sql_error select c8+c7, c9+c9+c8+c7/c6 from $tb +sql_error select c1/0, c2/0, c3/0.0 from $tb +sql_error select c1/0.00000000001 from $tb + +# arithmetic expression in join [d.7] + + +# arithmetic expression in union [d.8] + + +# arithmetic expression in group by [d.9] +# in group by tag +# not support for normal table +sql_error select c5*99 from $tb group by t1 + +# in group by column +sql_error select c6-(c6+c3)*12 from $tb group by c3; + + +# limit offset [d.10] +sql select c6 - c6 + 12 from $tb limit 12 offset 99; +sql select c4 / 99.123 from $tb limit 1 offset 9999; + +# slimit/soffset not support for normal table query. [d.11] +sql_error select sum(c1) from $tb slimit 1 soffset 19; + +# fill [d.12] +sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); + +# constant column. [d.13] + + +# column value filter [d.14] + + +# tag filter(not support for normal table). [d.15] +sql_error select c2+99 from $tb where t1=12; + +# multi-field output [d.16] +sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb + +sql select c4*1+1/2 from $tb +if $rows != $rowNum then + return -1 +endi +if $data00 != 0.500000000 then + return -1 +endi +if $data10 != 1.500000000 then + return -1 +endi +if $data90 != 9.500000000 then + return -1 +endi + +# interval query [d.17] +sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s) +sql_error select c7-c9 from $tb interval(2y) + +# aggregation query [d.18] +# see test cases below + +# first/last query [d.19] +# see test cases below + +# multiple retrieve [d.20] +sql select c2-c2 from $tb; + +#======================================= aggregation function arithmetic query cases ================ +# asc/desc order [d.2] +sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc; +sql select (count(c1) * 2) % 7.9 from $stb order by ts desc; + +# all possible function in the arithmetic expressioin +sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), xxxx, from $stb where ts < and ts > + +# no result return [d.3] +sql select first(c1) * 91 - 7, last(c3) from $stb where ts < 1537146000000 +if $rows != 0 then + return -1 +endi + +# no result return [d.3] +sql select sum(c2) - avg(c2) from $tb where ts>xxx +if $rows != 0 then + return -1 +endi + +# single row result aggregation [d.4] +sql select + +# error cases +sql_error select first(c1, c2) - last(c1, c2) from $tb + +# multi row result aggregation [d.4] +sql select top(c1, 1) - bottom(c1, 1) from $tb +sql select top(c1, 99) - bottom(c1, 99) from $tb + +# all data types [d.6] +sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $tb + +# error case, ts/bool/binary/nchar not support arithmetic expression +sql_error select ts+ts from $tb +sql_error select ts+22 from $tb +sql_error select c7*12 from $tb +sql_error select c8/55 from $tb +sql_error select c9+c8 from $tb + +# arithmetic expression in join [d.7] + + +# arithmetic expression in union [d.8] + + +# arithmetic expression in group by [d.9] +# in group by tag +# not support for normal table +sql_error select c5*99 from $tb group by t1 + +# in group by column +sql_error select c6-c6+c3*12 from $tb group by c3; + +sql select first(c6) - last(c6) *12 / count(*) from $tb group by c3; + +# limit offset [d.10] +sql select c6-c6+12 from $tb limit 12 offset 99; +sql select c4/99.123 from $tb limit 1 offset 9999; + +# slimit/soffset not suport for normal table query. [d.11] +sql_error select sum(c1) from $tb slimit 1 soffset 19; + +# fill [d.12] +sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); + +# constant column. [d.13] + + +# column value filter [d.14] + + +# tag filter(not support for normal table). [d.15] +sql_error select c2+99 from $tb where t1=12; + +# multi-field output [d.16] +sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb + +sql select c4*1+1/2 from $tb +if $rows != $rowNum then + return -1 +endi +if $data00 != 0.500000000 then + return -1 +endi +if $data10 != 1.500000000 then + return -1 +endi +if $data90 != 9.500000000 then + return -1 +endi + +# interval query [d.17] +sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s) +sql_error select c7-c9 from $tb interval(2y) + +# aggregation query [d.18] +# see test cases below + +# first/last query [d.19] +# see test cases below + +# multiple retrieve [d.20] +sql select c2-c2 from $tb; + + +sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb + + From b6540b4417fa61c934b86a4ce983835774d07ba3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 23 Nov 2020 11:35:38 +0800 Subject: [PATCH 014/188] TD-2153 --- src/mnode/src/mnodeSdb.c | 2 +- src/sync/inc/syncInt.h | 14 +++++++------- src/sync/src/syncMain.c | 22 ++++++++++------------ src/sync/src/syncRetrieve.c | 2 +- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 26a881d477..36e1a1eb2b 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -211,7 +211,7 @@ void sdbUpdateMnodeRoles() { if (tsSdbMgmt.sync <= 0) return; SNodesRole roles = {0}; - syncGetNodesRole(tsSdbMgmt.sync, &roles); + if (syncGetNodesRole(tsSdbMgmt.sync, &roles) != 0) return; sdbInfo("vgId:1, update mnodes role, replica:%d", tsSdbMgmt.cfg.replica); for (int32_t i = 0; i < tsSdbMgmt.cfg.replica; ++i) { diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 0790134997..aacd72e930 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -159,14 +159,14 @@ typedef struct SSyncNode { SSyncFwds * pSyncFwds; // saved forward info if quorum >1 void * pFwdTimer; void * pRoleTimer; - FGetFileInfo getFileInfo; - FGetWalInfo getWalInfo; - FWriteToCache writeToCache; - FConfirmForward confirmForward; - FNotifyRole notifyRole; - FNotifyFlowCtrl notifyFlowCtrl; + FGetFileInfo getFileInfo; + FGetWalInfo getWalInfo; + FWriteToCache writeToCache; + FConfirmForward confirmForward; + FNotifyRole notifyRole; + FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; - pthread_mutex_t mutex; + pthread_mutex_t mutex; } SSyncNode; // sync module global diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 41d68b3149..dd759ed9d4 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -552,18 +552,16 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { } void syncBroadcastStatus(SSyncNode *pNode) { - SSyncPeer *pPeer; - - for (int32_t i = 0; i < pNode->replica; ++i) { - if (i == pNode->selfIndex) continue; - pPeer = pNode->peerInfo[i]; + for (int32_t index = 0; index < pNode->replica; ++index) { + if (index == pNode->selfIndex) continue; + SSyncPeer *pPeer = pNode->peerInfo[index]; syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_BROADCAST, syncGenTranId()); } } static void syncResetFlowCtrl(SSyncNode *pNode) { - for (int32_t i = 0; i < pNode->replica; ++i) { - pNode->peerInfo[i]->numOfRetrieves = 0; + for (int32_t index = 0; index < pNode->replica; ++index) { + pNode->peerInfo[index]->numOfRetrieves = 0; } if (pNode->notifyFlowCtrl) { @@ -1171,7 +1169,7 @@ static void syncProcessBrokenLink(void *param) { static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) { SSyncFwds *pSyncFwds = pNode->pSyncFwds; - uint64_t time = taosGetTimestampMs(); + int64_t time = taosGetTimestampMs(); if (pSyncFwds->fwds >= tsMaxFwdInfo) { pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo; @@ -1289,7 +1287,6 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle int32_t fwdLen; int32_t code = 0; - if (pWalHead->version > nodeVersion + 1) { sError("vgId:%d, hver:%" PRIu64 ", inconsistent with sver:%" PRIu64, pNode->vgId, pWalHead->version, nodeVersion); if (nodeRole == TAOS_SYNC_ROLE_SLAVE) { @@ -1305,14 +1302,15 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle // always update version nodeVersion = pWalHead->version; - sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, - syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; // only pkt from RPC or CQ can be forwarded if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; + sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, + syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); + // a hacker way to improve the performance pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); pSyncHead->type = TAOS_SMSG_FORWARD; @@ -1332,7 +1330,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle code = 1; } - int32_t retLen = write(pPeer->peerFd, pSyncHead, fwdLen); + int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { sTrace("%s, forward is sent, hver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); } else { diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 7c1286f419..b6dacaa262 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -438,7 +438,7 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); firstPkt.port = tsSyncPort; - if (write(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) { + if (taosWriteMsg(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) { sError("%s, failed to send syncCmd", pPeer->id); return -1; } From 5894dded1ddf64561f6e991b8f92469bde27650a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 12:47:34 +0800 Subject: [PATCH 015/188] [TD-2169]: fix memory leak in handing the group by binary|nchar type data. --- src/query/inc/qUtil.h | 2 +- src/query/src/qUtil.c | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b2c4b6e2ec..62bf245a2b 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -26,7 +26,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery); -void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow); +void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type); void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 650b886c36..4b8d7ea930 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -68,6 +68,12 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->pResult == NULL); return; } + + if (pWindowResInfo->type == TSDB_DATA_TYPE_BINARY || pWindowResInfo->type == TSDB_DATA_TYPE_NCHAR) { + for(int32_t i = 0; i < pWindowResInfo->size; ++i) { + tfree(pWindowResInfo->pResult[i]->key); + } + } tfree(pWindowResInfo->pResult); } @@ -81,7 +87,7 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pWindowRes = pWindowResInfo->pResult[i]; - clearResultRow(pRuntimeEnv, pWindowRes); + clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); int32_t groupIndex = 0; int64_t uid = 0; @@ -133,7 +139,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // move the unclosed window in the front of the window list for (int32_t k = remain; k < pWindowResInfo->size; ++k) { SResultRow *pWindowRes = pWindowResInfo->pResult[k]; - clearResultRow(pRuntimeEnv, pWindowRes); + clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); } pWindowResInfo->size = remain; @@ -229,7 +235,7 @@ void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { getResultRow(pWindowResInfo, slot)->closed = true; } -void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { +void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) { if (pWindowRes == NULL) { return; } @@ -253,7 +259,12 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { pWindowRes->pageId = -1; pWindowRes->rowId = -1; pWindowRes->closed = false; - pWindowRes->win = TSWINDOW_INITIALIZER; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + tfree(pWindowRes->key); + } else { + pWindowRes->win = TSWINDOW_INITIALIZER; + } } /** From f55fce821764f91f36e2fb8c23706bead039f8bf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 14:01:10 +0800 Subject: [PATCH 016/188] [TD-2163]: add the version check for query message. --- src/client/src/tscServer.c | 3 ++- src/inc/taosmsg.h | 2 ++ src/query/src/qExecutor.c | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cbc5604a27..eaa00f290b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -669,6 +669,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; + tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); int32_t numOfTags = (int32_t)taosArrayGetSize(pTableMetaInfo->tagColList); @@ -693,8 +694,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit; pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->queryType = htonl(pQueryInfo->type); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 681fa44929..b77db69c46 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -464,6 +464,8 @@ typedef struct STimeWindow { typedef struct { SMsgHead head; + char version[TSDB_VERSION_LEN]; + STimeWindow window; int32_t numOfTables; int16_t order; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c44d65e373..630a469dd7 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5825,6 +5825,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { int32_t code = TSDB_CODE_SUCCESS; + if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { + return TSDB_CODE_QRY_INVALID_MSG; + } + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); From 7df48991217b8f487cfdc30eade6a1413e919b48 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 16:27:04 +0800 Subject: [PATCH 017/188] [TD-2180]: fix coverity issues. --- src/client/src/tscPrepare.c | 72 +++++++++---------- src/client/src/tscSQLParser.c | 13 +--- src/client/src/tscServer.c | 6 +- src/client/src/tscUtil.c | 7 +- src/common/src/tvariant.c | 4 +- src/query/src/qExecutor.c | 2 +- src/query/src/qParserImpl.c | 3 +- src/query/src/qResultbuf.c | 7 +- src/query/src/qTsbuf.c | 4 +- .../parser/col_arithmetic_operation.sim | 1 - 10 files changed, 59 insertions(+), 60 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 68f2ecbf0e..30e0729427 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -258,11 +258,7 @@ static char* normalStmtBuildSql(STscStmt* stmt) { static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { if (bind->is_null != NULL && *(bind->is_null)) { - if (param->type == TSDB_DATA_TYPE_BINARY || param->type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(data + param->offset, param->type); - } else { - setNull(data + param->offset, param->type, param->bytes); - } + setNull(data + param->offset, param->type, param->bytes); return TSDB_CODE_SUCCESS; } @@ -312,13 +308,13 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } memcpy(data + param->offset, &u.v1, sizeof(u.v1)); return TSDB_CODE_SUCCESS; @@ -336,40 +332,40 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_INT: { int32_t v = *(int32_t*)bind->buffer; u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)bind->buffer; u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { int64_t v; - int n,r; + int n, r; r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { + if (r == 1 && n == strlen((const char*)bind->buffer)) { u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } memcpy(data + param->offset, &u.v1, sizeof(u.v1)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_SMALLINT: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: @@ -383,34 +379,34 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)bind->buffer; u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { int64_t v; - int n,r; + int n, r; r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { + if (r == 1 && n == strlen((const char*)bind->buffer)) { u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } memcpy(data + param->offset, &u.v2, sizeof(u.v2)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_INT: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: @@ -514,17 +510,17 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } memcpy(data + param->offset, &u.v8, sizeof(u.v8)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_DOUBLE: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: @@ -556,15 +552,15 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; - } break; + } + } memcpy(data + param->offset, &u.f8, sizeof(u.f8)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_TIMESTAMP: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_TIMESTAMP: { @@ -589,10 +585,10 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { default: { return TSDB_CODE_TSC_INVALID_VALUE; } break; - } break; + }; memcpy(data + param->offset, &u.v8, sizeof(u.v8)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_BINARY: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_BINARY: { @@ -602,7 +598,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { short size = (short)*bind->length; STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -614,9 +610,9 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { case TSDB_DATA_TYPE_NCHAR: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } - } break; + } case TSDB_DATA_TYPE_NCHAR: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_NCHAR: { @@ -626,7 +622,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { } varDataSetLen(data + param->offset, output); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -638,12 +634,12 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { case TSDB_DATA_TYPE_BINARY: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } - } break; + } default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 15d2647c51..f859d2969e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -80,7 +80,6 @@ static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprLis static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); -static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd); @@ -2951,14 +2950,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) return TSDB_CODE_SUCCESS; } -void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) { - if (QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - tscFieldInfoUpdateOffset(pQueryInfo); - } else { - tscFieldInfoUpdateOffset(pQueryInfo); - } -} - static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { if (pColumn == NULL) { return NULL; @@ -3537,7 +3528,7 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer if (i == 0) { id = p1->uid; - } else if (id != p1->uid){ + } else if (id != p1->uid) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -6531,7 +6522,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return code; } - setColumnOffsetValueInResultset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); /* * fill options are set at the end position, when all columns are set properly diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index eaa00f290b..15859674ab 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -62,7 +62,7 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn); + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); pEpSet->port[i] = pVgroupInfo->epAddr[i].port; if (!hasFqdn) { @@ -2031,7 +2031,8 @@ static void createHBObj(STscObj* pObj) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); if (pQueryInfo == NULL) { - pSql->res.code = terrno; + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); return; } @@ -2039,6 +2040,7 @@ static void createHBObj(STscObj* pObj) { pSql->cmd.command = pQueryInfo->command; if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tfree(pSql); return; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7a82bcaaab..a75f6d8296 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -451,14 +451,16 @@ void tscFreeRegisteredSqlObj(void *pSql) { STscObj* pTscObj = (*p)->pTscObj; assert((*p)->self != 0 && (*p)->self == (p)); + + SSqlObj* ptr = *p; tscFreeSqlObj(*p); int32_t ref = T_REF_DEC(pTscObj); assert(ref >= 0); - tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref); + tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", ptr, pTscObj, ref); if (ref == 0) { - tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj); + tscDebug("%p all sqlObj freed, free tscObj:%p", ptr, pTscObj); taosRemoveRef(tscRefId, pTscObj->rid); } } @@ -644,6 +646,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff dataBuf->pData = calloc(1, dataBuf->nAllocSize); if (dataBuf->pData == NULL) { tscError("failed to allocated memory, reason:%s", strerror(errno)); + tfree(dataBuf); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 6dd0653822..fc00f50a7a 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -171,7 +171,9 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } } - pDst->nLen = tDataTypeDesc[pDst->nType].nSize; + if (pDst->nType != TSDB_DATA_TYPE_ARRAY) { + pDst->nLen = tDataTypeDesc[pDst->nType].nSize; + } } int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 630a469dd7..3fcf44f057 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6814,7 +6814,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { - size_t sz = read(fd, data, (uint32_t)s); + size_t sz = read(fd, data, (uint32_t) s); if(sz < s) { // todo handle error assert(0); } diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 0853565fc6..5cd9d3c77a 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -187,7 +187,8 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { pExpr->token.type = pLeft->token.type; } - if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { + if ((pLeft != NULL && pRight != NULL) && + (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) { /* * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond * Otherwise, the time precision is adaptive, determined by the time precision from databases. diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 67cb7af074..dfa8bcd95b 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -119,8 +119,11 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { pg->info.offset = allocatePositionInFile(pResultBuf, size); pResultBuf->nextPos += size; - fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - /*int32_t ret =*/ fwrite(t, 1, size, pResultBuf->file); + int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + assert(ret == 0); + + ret = fwrite(t, 1, size, pResultBuf->file); + assert(ret == size); if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { pResultBuf->fileSize = pg->info.offset + pg->info.length; diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 8a5a87baab..d0c59fe5ef 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -341,8 +341,10 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { pBlock->tag.pz = tp; sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + UNUSED(sz); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + UNUSED(sz); } sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index c05229ab65..64f41ae11f 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -73,7 +73,6 @@ run general/parser/col_arithmetic_query.sim print ====================> crash # sql select spread(ts )/(1000*3600*24) from ca_stb0 interval(1y) - sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y) sql_error select first(ts) - last(ts) from stb interval(1y) sql_error select top(c1, 2) - last(c1) from stb; From 0a90950bdfaddda5133ecd33596e6b7440105b26 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 23 Nov 2020 10:08:15 +0000 Subject: [PATCH 018/188] TD-2157 --- src/sync/inc/syncInt.h | 1 + src/sync/src/syncMain.c | 78 ++++++++++++++++++++++++------------- src/sync/src/syncRestore.c | 6 ++- src/sync/src/syncRetrieve.c | 6 ++- src/util/src/tsocket.c | 4 +- 5 files changed, 61 insertions(+), 34 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index aacd72e930..7dbb1985d4 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -172,6 +172,7 @@ typedef struct SSyncNode { // sync module global extern int32_t tsSyncNum; extern char tsNodeFqdn[TSDB_FQDN_LEN]; +extern char * syncStatus[]; void *syncRetrieveData(void *param); void *syncRestoreData(void *param); diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index dd759ed9d4..a68f0b7bbb 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -73,6 +73,14 @@ char* syncRole[] = { "master" }; +char *syncStatus[] = { + "init", + "start", + "file", + "cache", + "invalid" +}; + typedef enum { SYNC_STATUS_BROADCAST, SYNC_STATUS_BROADCAST_RSP, @@ -282,7 +290,7 @@ void syncStop(int64_t rid) { pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]; if (pPeer) syncRemovePeer(pPeer); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); taosReleaseRef(tsSyncRefId, rid); taosRemoveRef(tsSyncRefId, rid); @@ -350,7 +358,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) { (*pNode->notifyRole)(pNode->vgId, nodeRole); } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum); syncBroadcastStatus(pNode); @@ -423,7 +431,7 @@ void syncRecover(int64_t rid) { } } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); taosReleaseRef(tsSyncRefId, rid); } @@ -498,6 +506,8 @@ int32_t syncDecPeerRef(SSyncPeer *pPeer) { } static void syncClosePeerConn(SSyncPeer *pPeer) { + sDebug("%s, pfd:%d sfd:%d will be closed", pPeer->id, pPeer->peerFd, pPeer->syncFd); + taosTmrStopA(&pPeer->timer); taosClose(pPeer->syncFd); if (pPeer->peerFd >= 0) { @@ -751,7 +761,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new sDebug("vgId:%d, choose master", pNode->vgId); syncChooseMaster(pNode); } else { - sDebug("vgId:%d, version inconsistent, cannot choose master", pNode->vgId); + sDebug("vgId:%d, cannot choose master since roles inconsistent", pNode->vgId); } } @@ -770,11 +780,12 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new } static void syncRestartPeer(SSyncPeer *pPeer) { - sDebug("%s, restart peer connection", pPeer->id); + sDebug("%s, restart peer connection, last sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); syncClosePeerConn(pPeer); pPeer->sstatus = TAOS_SYNC_STATUS_INIT; + sDebug("%s, peer conn is restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) { @@ -803,7 +814,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { } if (pPeer->sstatus != TAOS_SYNC_STATUS_INIT) { - sDebug("%s, sync is already started", pPeer->id); + sDebug("%s, sync is already started for sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); return; // already started } @@ -821,7 +832,7 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) { syncDecPeerRef(pPeer); } else { pPeer->sstatus = TAOS_SYNC_STATUS_START; - sDebug("%s, thread is created to retrieve data", pPeer->id); + sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); } } @@ -831,9 +842,10 @@ static void syncNotStarted(void *param, void *tmrId) { pthread_mutex_lock(&pNode->mutex); pPeer->timer = NULL; - sInfo("%s, sync connection is still not up, restart", pPeer->id); + pPeer->sstatus = TAOS_SYNC_STATUS_INIT; + sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); syncRestartConnection(pPeer); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } static void syncTryRecoverFromMaster(void *param, void *tmrId) { @@ -842,14 +854,14 @@ static void syncTryRecoverFromMaster(void *param, void *tmrId) { pthread_mutex_lock(&pNode->mutex); syncRecoverFromMaster(pPeer); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } static void syncRecoverFromMaster(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { - sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus); + sDebug("%s, sync is already started since sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); return; } @@ -877,7 +889,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) { sError("%s, failed to send sync-req to peer", pPeer->id); } else { nodeSStatus = TAOS_SYNC_STATUS_START; - sInfo("%s, sync-req is sent", pPeer->id); + sInfo("%s, sync-req is sent to peer, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); } } @@ -915,7 +927,8 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) { if (nodeSStatus != TAOS_SYNC_STATUS_INIT) { syncSaveIntoBuffer(pPeer, pHead); } else { - sError("%s, forward discarded, hver:%" PRIu64, pPeer->id, pHead->version); + sError("%s, forward discarded since sstatus:%s, hver:%" PRIu64, pPeer->id, syncStatus[nodeSStatus], + pHead->version); } } } @@ -924,8 +937,10 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; - sDebug("%s, status msg is received, self:%s sver:%" PRIu64 " peer:%s sver:%" PRIu64 ", ack:%d tranId:%u type:%s", pPeer->id, - syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type]); + sDebug("%s, status msg is received, self:%s sstatus:%s sver:%" PRIu64 ", peer:%s sver:%" PRIu64 + ", ack:%d tranId:%u type:%s pfd:%d", + pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeersStatus->role], + pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type], pPeer->peerFd); pPeer->version = pPeersStatus->version; syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role); @@ -982,7 +997,7 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) { } } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); return code; } @@ -1014,8 +1029,11 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { - sDebug("%s, status msg is sent, self:%s sver:%" PRIu64 ", ack:%d tranId:%u type:%s", pPeer->id, syncRole[pPeersStatus->role], - pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type]); + sDebug("%s, status msg is sent, self:%s sstatus:%s sver:%" PRIu64 ", peer:%s sstatus:%s sver:%" PRIu64 + ", ack:%d tranId:%u type:%s pfd:%d", + pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], + syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId, + statusType[pPeersStatus->type], pPeer->peerFd); } else { sDebug("%s, failed to send status msg, restart", pPeer->id); syncRestartConnection(pPeer); @@ -1048,7 +1066,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) { firstPkt.sourceId = pNode->vgId; // tell arbitrator its vgId if (taosWriteMsg(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) { - sDebug("%s, connection to peer server is setup", pPeer->id); + sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d", pPeer->id, connFd, pPeer->syncFd); pPeer->peerFd = connFd; pPeer->role = TAOS_SYNC_ROLE_UNSYNCED; pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); @@ -1069,7 +1087,7 @@ static void syncCheckPeerConnection(void *param, void *tmrId) { sDebug("%s, check peer connection", pPeer->id); syncSetupPeerConnection(pPeer); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } static void syncCreateRestoreDataThread(SSyncPeer *pPeer) { @@ -1135,7 +1153,8 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pPeer->syncFd = connFd; syncCreateRestoreDataThread(pPeer); } else { - sDebug("%s, TCP connection is already up, close one", pPeer->id); + sDebug("%s, TCP connection is already up(pfd:%d), close one, new pfd:%d sfd:%d", pPeer->id, pPeer->peerFd, connFd, + pPeer->syncFd); syncClosePeerConn(pPeer); pPeer->peerFd = connFd; pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); @@ -1145,7 +1164,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { } } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } static void syncProcessBrokenLink(void *param) { @@ -1156,14 +1175,14 @@ static void syncProcessBrokenLink(void *param) { if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&pNode->mutex); - sDebug("%s, TCP link is broken since %s", pPeer->id, strerror(errno)); + sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd); pPeer->peerFd = -1; if (syncDecPeerRef(pPeer) != 0) { syncRestartConnection(pPeer); } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); taosReleaseRef(tsSyncRefId, pNode->rid); } @@ -1239,10 +1258,13 @@ static void syncMonitorNodeRole(void *param, void *tmrId) { if (index == pNode->selfIndex) continue; SSyncPeer *pPeer = pNode->peerInfo[index]; - if (pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue; - if (pPeer->sstatus > TAOS_SYNC_STATUS_INIT || nodeSStatus > TAOS_SYNC_STATUS_INIT) continue; + if (/*pPeer->role > TAOS_SYNC_ROLE_UNSYNCED && */ nodeRole > TAOS_SYNC_ROLE_UNSYNCED) continue; + if (/*pPeer->sstatus > TAOS_SYNC_STATUS_INIT || */ nodeSStatus > TAOS_SYNC_STATUS_INIT) continue; + sDebug("%s, check roles since self:%s sstatus:%s, peer:%s sstatus:%s", pPeer->id, syncRole[pPeer->role], + syncStatus[pPeer->sstatus], syncRole[nodeRole], syncStatus[nodeSStatus]); syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_CHECK_ROLE, syncGenTranId()); + break; } pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); @@ -1271,7 +1293,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { } syncRemoveConfirmedFwdInfo(pNode); - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); } pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl); @@ -1339,7 +1361,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle } } - pthread_mutex_unlock(&(pNode->mutex)); + pthread_mutex_unlock(&pNode->mutex); return code; } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index b31ec5f7c7..466c39060a 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -267,7 +267,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; - sDebug("%s, start to restore file", pPeer->id); + sDebug("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); int32_t code = syncRestoreFile(pPeer, &fversion); if (code < 0) { sError("%s, failed to restore file", pPeer->id); @@ -291,7 +291,7 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { } nodeSStatus = TAOS_SYNC_STATUS_CACHE; - sDebug("%s, start to insert buffered points", pPeer->id); + sDebug("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); if (syncProcessBufferedFwd(pPeer) < 0) { sError("%s, failed to insert buffered points", pPeer->id); return -1; @@ -327,6 +327,8 @@ void *syncRestoreData(void *param) { (*pNode->notifyRole)(pNode->vgId, nodeRole); nodeSStatus = TAOS_SYNC_STATUS_INIT; + sInfo("%s, sync over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + taosClose(pPeer->syncFd); syncCloseRecvBuffer(pNode); __sync_fetch_and_sub(&tsSyncNum, 1); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index b6dacaa262..04d4ce32cb 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -318,6 +318,7 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) if (((event & IN_MODIFY) == 0) || once) { if (fversion == 0) { pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt + sDebug("%s, fversion is 0 then set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); fversion = nodeVersion; // must read data to fversion } } @@ -416,8 +417,9 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { } if (code == 0) { - sInfo("%s, wal retrieve is finished", pPeer->id); pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; + sInfo("%s, wal retrieve is finished, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + SWalHead walHead; memset(&walHead, 0, sizeof(walHead)); code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)); @@ -445,7 +447,7 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { pPeer->sversion = 0; pPeer->sstatus = TAOS_SYNC_STATUS_FILE; - sInfo("%s, start to retrieve file", pPeer->id); + sInfo("%s, start to retrieve file, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); if (syncRetrieveFile(pPeer) < 0) { sError("%s, failed to retrieve file", pPeer->id); return -1; diff --git a/src/util/src/tsocket.c b/src/util/src/tsocket.c index 1be79b7bbd..2543c81708 100644 --- a/src/util/src/tsocket.c +++ b/src/util/src/tsocket.c @@ -107,7 +107,7 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) { while (nleft > 0) { nwritten = (int32_t)taosWriteSocket(fd, (char *)ptr, (size_t)nleft); if (nwritten <= 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) + if (errno == EINTR /* || errno == EAGAIN || errno == EWOULDBLOCK */) continue; else return -1; @@ -133,7 +133,7 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) { if (nread == 0) { break; } else if (nread < 0) { - if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) { + if (errno == EINTR/* || errno == EAGAIN || errno == EWOULDBLOCK*/) { continue; } else { return -1; From 2783c56f8ab3b33d7858a67ddab01062adf726cb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 18:45:09 +0800 Subject: [PATCH 019/188] [TD-225]update script. --- .../parser/col_arithmetic_operation.sim | 68 ++++++-- .../general/parser/col_arithmetic_query.sim | 147 ++++++++++++++---- 2 files changed, 175 insertions(+), 40 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index 64f41ae11f..7adae8ef81 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -29,12 +29,41 @@ sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 $i = 0 $ts = $ts0 $halfTbNum = $tbNum / 2 -while $i < $halfTbNum - $tbId = $i + $halfTbNum - $tb = $tbPrefix . $i - $tb1 = $tbPrefix . $tbId - sql create table $tb using $stb tags( $i ) - sql create table $tb1 using $stb tags( $tbId ) +#while $i < $halfTbNum + $t1 = $i + 1 + $t2 = $i + 2 + $t3 = $i + 3 + $t4 = $i + 4 + + $t5 = $i + $halfTbNum + $t6 = $t5 + 1 + $t7 = $t6 + 1 + $t8 = $t7 + 1 + $t9 = $t8 + 1 + + $tb0 = $tbPrefix . $i + $tb1 = $tbPrefix . $t1 + $tb2 = $tbPrefix . $t2 + $tb3 = $tbPrefix . $t3 + $tb4 = $tbPrefix . $t4 + + $tb5 = $tbPrefix . $t5 + $tb6 = $tbPrefix . $t6 + $tb7 = $tbPrefix . $t7 + $tb8 = $tbPrefix . $t8 + $tb9 = $tbPrefix . $t9 + + sql create table $tb0 using $stb tags( $i ) + sql create table $tb1 using $stb tags( $t1 ) + sql create table $tb2 using $stb tags( $t2 ) + sql create table $tb3 using $stb tags( $t3 ) + sql create table $tb4 using $stb tags( $t4 ) + + sql create table $tb5 using $stb tags( $t5 ) + sql create table $tb6 using $stb tags( $t6 ) + sql create table $tb7 using $stb tags( $t7 ) + sql create table $tb8 using $stb tags( $t8 ) + sql create table $tb9 using $stb tags( $t9 ) $x = 0 while $x < $rowNum @@ -47,11 +76,28 @@ while $i < $halfTbNum $binary = $binary . ' $nchar = 'nchar . $c $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) + sql insert into $tb0 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb2 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb3 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb4 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $x = $x + 1 - endw - $i = $i + 1 -endw + endw + #$i = $i + 1 + + $x = 0 + while $x < $rowNum + $xs = $x * $delta + $ts = $ts0 + $xs + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + + sql insert into $tb5 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb6 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb7 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb8 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb9 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) + $x = $x + 1 + endw + +#endw #=================================== above are setup test environment ============================= run general/parser/col_arithmetic_query.sim @@ -64,7 +110,7 @@ system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 500c run general/parser/col_arithmetic_query.sim diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 95258168ef..61a48e4a80 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -1,6 +1,17 @@ # ======================================= query test cases ======================================== # select from table +$dbPrefix = ca_db +$tbPrefix = ca_tb +$stbPrefix = ca_stb +$rowNum = 10000 + +$i = 0 +$db = $dbPrefix . $i + +$db = $dbPrefix . $i +sql use $db + $tb = $tbPrefix . 0 ## TBASE-344 sql select c1*2 from $tb @@ -20,38 +31,100 @@ if $data90 != 18.000000000 then return -1 endi -# asc/desc order [d.2] +# asc/desc order [d.2] ====================================================== sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc; +if $rows != 10000 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +if $data01 != NaN then + return -1 +endi + +if $data10 != 0.666666667 then + return -1 +endi + +if $data11 != 1.000000000 then + return -1 +endi + +if $data90 != 6.000000000 then + return -1 +endi + +if $data91 != 1.000000000 then + return -1 +endi + sql select (c1 * 2) % 7.9 from $tb order by ts desc; +if $rows != 10000 then + return -1 +endi -sql select c1 * c2 /4 from $tb where ts < and ts > +if $data00 != 0.100000000 then + return -1 +endi -# no result return [d.3] -sql select c1 * 91- 7 from $tb where ts<1537146000000 +if $data10 != 2.100000000 then + return -1 +endi + +if $data90 != 6.000000000 then + return -1 +endi + +# [d.3] +sql select c1 * c2 /4 from $tb where ts < 1537166000000 and ts > 1537156000000 +if $rows != 17 then + return -1 +endi + +if $data00 != 12.250000000 then + return -1 +endi + +if $data10 != 16.000000000 then + return -1 +endi + +if $data21 != 20.250000000 then + return -1 +endi + +if $data31 != 0.000000000 then + return -1 +endi + + +# no result return [d.3] ============================================================== +sql select c1 * 91- 7 from $tb where ts < 1537146000000 if $rows != 0 then return -1 endi # no result return [d.3] -sql select c2 - c2 from $tb where ts>xxx +sql select c2 - c2 from $tb where ts > '2018-09-17 12:50:00.000' and ts<'2018-09-17 13:00:00.000' if $rows != 0 then return -1 endi -# single row result aggregation [d.4] +# single row result aggregation [d.4] ================================================= # not available # error cases -sql_error select first(c1,c2) - last(c1,c2) from $tb +# not available # multi row result aggregation [d.4] sql select top(c1, 1) - bottom(c1, 1) from $tb sql select top(c1, 99) - bottom(c1, 99) from $tb -# all data types [d.6] -sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $tb - - +# all data types [d.6] ================================================================ +sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22, c2-c2 from $tb # error case, ts/bool/binary/nchar not support arithmetic expression sql_error select ts+ts from $tb @@ -65,41 +138,57 @@ sql_error select c8+c7, c9+c9+c8+c7/c6 from $tb sql_error select c1/0, c2/0, c3/0.0 from $tb sql_error select c1/0.00000000001 from $tb -# arithmetic expression in join [d.7] +# arithmetic expression in join [d.7]================================================== -# arithmetic expression in union [d.8] +# arithmetic expression in union [d.8]================================================= -# arithmetic expression in group by [d.9] -# in group by tag -# not support for normal table +# arithmetic expression in group by [d.9]============================================== +# in group by tag, not support for normal table sql_error select c5*99 from $tb group by t1 # in group by column sql_error select c6-(c6+c3)*12 from $tb group by c3; -# limit offset [d.10] -sql select c6 - c6 + 12 from $tb limit 12 offset 99; -sql select c4 / 99.123 from $tb limit 1 offset 9999; +# limit offset [d.10]================================================================== +sql select c6 * c1 + 12 from $tb limit 12 offset 99; +if $rows != 12 then + return -1 +endi -# slimit/soffset not support for normal table query. [d.11] +if $data00 != 93.000000000 then + return -1 +endi + +if $data90 != 12.000000000 then + return -1 +endi + +sql select c4 / 99.123 from $tb limit 10 offset 9999; +if $rows != 1 then + return -1 +endi + +if $data00 != + +# slimit/soffset not support for normal table query. [d.11]============================ sql_error select sum(c1) from $tb slimit 1 soffset 19; -# fill [d.12] +# fill [d.12]========================================================================== sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); -# constant column. [d.13] +# constant column. [d.13]============================================================== -# column value filter [d.14] +# column value filter [d.14]=========================================================== -# tag filter(not support for normal table). [d.15] +# tag filter(not support for normal table). [d.15]===================================== sql_error select c2+99 from $tb where t1=12; -# multi-field output [d.16] +# multi-field output [d.16]============================================================ sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb sql select c4*1+1/2 from $tb @@ -116,17 +205,17 @@ if $data90 != 9.500000000 then return -1 endi -# interval query [d.17] +# interval query [d.17]================================================================== sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s) sql_error select c7-c9 from $tb interval(2y) -# aggregation query [d.18] +# aggregation query [d.18]=============================================================== # see test cases below -# first/last query [d.19] +# first/last query [d.19]=============================================================== # see test cases below -# multiple retrieve [d.20] +# multiple retrieve [d.20]=============================================================== sql select c2-c2 from $tb; #======================================= aggregation function arithmetic query cases ================ From dd9228e06733c5a775e953e01b1bd12e1711a5e4 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 18:51:13 +0800 Subject: [PATCH 020/188] [TD-1859]: restful implementation for taos-jdbc --- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 9 +- .../taosdata/jdbc/rs/RestfulConnection.java | 41 ++++--- .../com/taosdata/jdbc/rs/RestfulDriver.java | 2 +- .../taosdata/jdbc/rs/RestfulStatement.java | 71 ++++++++++-- .../jdbc/utils/SqlSyntaxValidator.java | 29 ++++- .../taosdata/jdbc/rs/RestfulDriverTest.java | 23 ---- .../com/taosdata/jdbc/rs/RestfulJDBCTest.java | 108 ++++++++++++++++++ .../jdbc/utils/SqlSyntaxValidatorTest.java | 29 +++++ 9 files changed, 263 insertions(+), 51 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 51db837c7b..46de54ffb2 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.10 + 2.0.13 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index e7124a0599..3d1f402435 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.8 + 2.0.13 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -112,6 +112,13 @@ maven-surefire-plugin 2.12.4 + + **/*Test.java + + + **/BatchInsertTest.java + **/FailOverTest.java + true diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index b82efca3ef..6b0937a9b7 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -15,7 +15,6 @@ public class RestfulConnection implements Connection { private final String database; private final String url; - public RestfulConnection(String host, String port, Properties props, String database, String url) { this.host = host; this.port = Integer.parseInt(port); @@ -28,7 +27,7 @@ public class RestfulConnection implements Connection { public Statement createStatement() throws SQLException { if (isClosed()) throw new SQLException(TSDBConstants.WrapErrMsg("restful TDengine connection is closed.")); - return new RestfulStatement(this, this.database); + return new RestfulStatement(this, database); } @Override @@ -104,22 +103,28 @@ public class RestfulConnection implements Connection { @Override public void setTransactionIsolation(int level) throws SQLException { - + //transaction is not supported + throw new SQLFeatureNotSupportedException("transactions are not supported"); } + /** + * + */ @Override public int getTransactionIsolation() throws SQLException { - return 0; + //Connection.TRANSACTION_NONE specifies that transactions are not supported. + return Connection.TRANSACTION_NONE; } @Override public SQLWarning getWarnings() throws SQLException { + //TODO: getWarnings not implemented return null; } @Override public void clearWarnings() throws SQLException { - + throw new SQLFeatureNotSupportedException("clearWarnings not supported."); } @Override @@ -209,22 +214,26 @@ public class RestfulConnection implements Connection { @Override public Clob createClob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public Blob createBlob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public NClob createNClob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public SQLXML createSQLXML() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override @@ -254,12 +263,14 @@ public class RestfulConnection implements Connection { @Override public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override @@ -289,12 +300,16 @@ public class RestfulConnection implements Connection { @Override public T unwrap(Class iface) throws SQLException { - return null; + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return iface.isInstance(this); } public String getHost() { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index a07d6eb2b6..9e87cfa680 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -35,7 +35,7 @@ public class RestfulDriver extends AbstractTaosDriver { Properties props = parseURL(url, info); String host = props.getProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); String port = props.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "6041"); - String database = props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME); + String database = props.containsKey(TSDBDriver.PROPERTY_KEY_DBNAME) ? props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME) : null; String loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 20510f0135..30b56638d8 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.taosdata.jdbc.TSDBConstants; import com.taosdata.jdbc.rs.util.HttpClientPoolUtil; +import com.taosdata.jdbc.utils.SqlSyntaxValidator; import java.sql.*; import java.util.Arrays; @@ -11,19 +12,23 @@ import java.util.List; public class RestfulStatement implements Statement { - private final String catalog; + private boolean closed; + private String database; private final RestfulConnection conn; - public RestfulStatement(RestfulConnection c, String catalog) { + public RestfulStatement(RestfulConnection c, String database) { this.conn = c; - this.catalog = catalog; + this.database = database; } @Override public ResultSet executeQuery(String sql) throws SQLException { + if (isClosed()) + throw new SQLException("statement already closed"); + if (!SqlSyntaxValidator.isSelectSql(sql)) + throw new SQLException("not a select sql for executeQuery: " + sql); - final String url = "http://" + conn.getHost() + ":"+conn.getPort()+"/rest/sql"; - + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; String result = HttpClientPoolUtil.execute(url, sql); String fields = ""; List words = Arrays.asList(sql.split(" ")); @@ -65,12 +70,29 @@ public class RestfulStatement implements Statement { @Override public int executeUpdate(String sql) throws SQLException { - return 0; + if (isClosed()) + throw new SQLException("statement already closed"); + if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) + throw new SQLException("not a valid sql for executeUpdate: " + sql); + + if (this.database == null) + throw new SQLException("Database not specified or available"); + + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); + String result = HttpClientPoolUtil.execute(url, sql); + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + + jsonObject.getString("desc") + "\n" + + "error code: " + jsonObject.getString("code"))); + } + return Integer.parseInt(jsonObject.getString("rows")); } @Override public void close() throws SQLException { - + this.closed = true; } @Override @@ -115,6 +137,7 @@ public class RestfulStatement implements Statement { @Override public SQLWarning getWarnings() throws SQLException { + //TODO: getWarnings not Implemented return null; } @@ -130,7 +153,29 @@ public class RestfulStatement implements Statement { @Override public boolean execute(String sql) throws SQLException { - return false; + if (isClosed()) { + throw new SQLException("Invalid method call on a closed statement."); + } + //如果执行了use操作应该将当前Statement的catalog设置为新的database + if (SqlSyntaxValidator.isUseSql(sql)) { + this.database = sql.trim().replace("use", "").trim(); + } + if (this.database == null) + throw new SQLException("Database not specified or available"); + + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + // use database + HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); + // execute sql + String result = HttpClientPoolUtil.execute(url, sql); + // parse result + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + + jsonObject.getString("desc") + "\n" + + "error code: " + jsonObject.getString("code"))); + } + return true; } @Override @@ -245,7 +290,7 @@ public class RestfulStatement implements Statement { @Override public boolean isClosed() throws SQLException { - return false; + return closed; } @Override @@ -270,11 +315,15 @@ public class RestfulStatement implements Statement { @Override public T unwrap(Class iface) throws SQLException { - return null; + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return iface.isInstance(this); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java index 066dfad5d5..388c3978be 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java @@ -22,6 +22,9 @@ import java.sql.SQLException; public class SqlSyntaxValidator { + private static final String[] updateSQL = {"insert", "update", "delete", "create", "alter", "drop", "show", "describe", "use"}; + private static final String[] querySQL = {"select"}; + private TSDBConnection tsdbConnection; public SqlSyntaxValidator(Connection connection) { @@ -34,7 +37,7 @@ public class SqlSyntaxValidator { if (tsdbConnection == null || tsdbConnection.isClosed()) { throw new SQLException("invalid connection"); } else { - TSDBJNIConnector jniConnector = tsdbConnection.getConnection(); + TSDBJNIConnector jniConnector = tsdbConnection.getConnection(); if (jniConnector == null) { throw new SQLException("jniConnector is null"); } else { @@ -43,4 +46,28 @@ public class SqlSyntaxValidator { } return res; } + + public static boolean isValidForExecuteUpdate(String sql) { + for (String prefix : updateSQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; + } + return false; + } + + public static boolean isUseSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[8]) || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); + } + + public static boolean isUpdateSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[1]); + } + + public static boolean isInsertSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[0]); + } + + public static boolean isSelectSql(String sql) { + return sql.trim().toLowerCase().startsWith(querySQL[0]); + } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index 768eceab8e..d07a6a2179 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -7,29 +7,6 @@ import java.sql.*; public class RestfulDriverTest { - @Test - public void testCase001() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/?user=root&password=taosdata"); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery("select * from log.log"); - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String column = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - System.out.print(column + ":" + value + "\t"); - } - System.out.println(); - } - statement.close(); - connection.close(); - } catch (SQLException | ClassNotFoundException e) { - e.printStackTrace(); - } - } - @Test public void connect() { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java new file mode 100644 index 0000000000..d13475b96d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -0,0 +1,108 @@ +package com.taosdata.jdbc.rs; + +import org.junit.*; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Random; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class RestfulJDBCTest { + + private Connection connection; + + @Before + public void before() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/restful_test?user=root&password=taosdata"); + } + + @After + public void after() throws SQLException { + if (connection != null) + connection.close(); + } + + + /** + * 查询所有log.log + **/ + @Test + public void testCase001() { + try { + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select * from log.log"); + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String column = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + System.out.print(column + ":" + value + "\t"); + } + System.out.println(); + } + statement.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /** + * create database + */ + @Test + public void testCase002() { + try (Statement stmt = connection.createStatement()) { + stmt.execute("drop database if exists restful_test"); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /** + * create super table + ***/ + @Test + public void testCase003() { + try (Statement stmt = connection.createStatement()) { + stmt.execute("create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase004() { + try (Statement stmt = connection.createStatement()) { + for (int i = 1; i <= 100; i++) { + stmt.execute("create table t" + i + " using weather tags('beijing', '" + i + "')"); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private Random random = new Random(System.currentTimeMillis()); + + @Test + public void testCase005() { + try (Statement stmt = connection.createStatement()) { + int rows = 0; + for (int i = 0; i < 10; i++) { + for (int j = 1; j <= 100; j++) { + long currentTimeMillis = System.currentTimeMillis(); + int affectRows = stmt.executeUpdate("insert into t" + j + " values(" + currentTimeMillis + "," + (random.nextFloat() * 50) + "," + random.nextInt(100) + ")"); + Assert.assertEquals(1, affectRows); + rows += affectRows; + } + } + Assert.assertEquals(1000, rows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java new file mode 100644 index 0000000000..25c6c4913d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -0,0 +1,29 @@ +package com.taosdata.jdbc.utils; + +import com.sun.source.tree.AssertTree; +import org.junit.Assert; +import org.junit.Test; + +public class SqlSyntaxValidatorTest { + + @Test + public void validateSqlSyntax() { + } + + @Test + public void isSelectSQL() { + Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather")); + Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather")); + Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather ")); + Assert.assertFalse(SqlSyntaxValidator.isSelectSql("insert into test.weather values(now, 1.1, 2)")); + } + + @Test + public void isUseSQL() { + Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database if not exist test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database if exist test")); + } +} \ No newline at end of file From f26fc83be8c8523e140adef265d74529447e27d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 18:59:13 +0800 Subject: [PATCH 021/188] [TD-2193]: fix the offset error in arithmetic expression. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3fcf44f057..2623c74e45 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -947,7 +947,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - sas->offset = 0; + sas->offset = pQuery->pos; sas->colList = pQuery->colList; sas->numOfCols = pQuery->numOfCols; sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); From a9e43baa3f8841d468efe1fedb5beda78beeb49d Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:08:03 +0800 Subject: [PATCH 022/188] change --- .../java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java index 25c6c4913d..fb570e16c4 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc.utils; -import com.sun.source.tree.AssertTree; import org.junit.Assert; import org.junit.Test; From 0481d9836f33f6e0a1e26e85af7a4bdc5a8ea49d Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:12:07 +0800 Subject: [PATCH 023/188] change --- src/connector/jdbc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index c565853ab0..701a39b209 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.8-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) From dbdd70aca16a1415efab789d3d44c1af7b1e5d7c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 23 Nov 2020 11:15:08 +0000 Subject: [PATCH 024/188] TD-2196 --- src/sync/src/syncRestore.c | 31 ++++++++++++++++++++++++------- src/sync/src/syncRetrieve.c | 34 ++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 466c39060a..69da1d71fa 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -65,7 +65,10 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { while (1) { // read file info int32_t ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo)); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to read file info while restore file since %s", pPeer->id, strerror(errno)); + break; + } // if no more file from master, break; if (minfo.name[0] == 0 || minfo.magic == 0) { @@ -92,8 +95,11 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1 : 0; // send file ack - ret = taosWriteMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck)); - if (ret < 0) break; + ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); + if (ret < 0) { + sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno)); + break; + } // if sync is not required, continue if (fileAck.sync == 0) { @@ -108,14 +114,17 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO); if (dfd < 0) { - sError("%s, failed to open file:%s", pPeer->id, name); + sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); break; } ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size); fsync(dfd); close(dfd); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to copy file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno)); + break; + } fileChanged = true; sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size); @@ -125,6 +134,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // data file is changed, code shall be set to 1 *fversion = minfo.fversion; code = 1; + sDebug("%s, file changed while restore file", pPeer->id); } if (code < 0) { @@ -146,15 +156,22 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { while (1) { ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to read walhead while restore wal since %s", pPeer->id, strerror(errno)); + break; + } if (pHead->len == 0) { + sDebug("%s, wal is synced over", pPeer->id); code = 0; break; } // wal sync over ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to read walcont, len:%d while restore wal since %s", pPeer->id, pHead->len, strerror(errno)); + break; + } sTrace("%s, restore a record, qtype:wal len:%d hver:%" PRIu64, pPeer->id, pHead->len, pHead->version); diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 04d4ce32cb..58d09d080e 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -114,7 +114,10 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } // if no file anymore, break if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { @@ -124,8 +127,11 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { } // wait for the ack from peer - ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck)); - if (ret < 0) break; + ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); + if (ret < 0) { + sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } // set the peer sync version pPeer->sversion = fileInfo.fversion; @@ -134,7 +140,10 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); // add the file into watch list - if (syncAddIntoWatchList(pPeer, name) < 0) break; + if (syncAddIntoWatchList(pPeer, name) < 0) { + sError("%s, failed to watch file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } // if sync is not required, continue if (fileAck.sync == 0) { @@ -145,21 +154,30 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // send the file to peer int32_t sfd = open(name, O_RDONLY); - if (sfd < 0) break; + if (sfd < 0) { + sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); - if (ret < 0) break; + if (ret < 0) { + sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); fileInfo.index++; // check if processed files are modified - if (syncAreFilesModified(pPeer) != 0) break; + if (syncAreFilesModified(pPeer) != 0) { + sInfo("%s, file:%s are modified while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); + break; + } } if (code < 0) { - sError("%s, failed to retrieve file since %s", pPeer->id, strerror(errno)); + sError("%s, failed to retrieve file", pPeer->id); } return code; From 5e91e8e307c232e9eaf47615499c6a7d0dc5a936 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:28:45 +0800 Subject: [PATCH 025/188] change --- cmake/install.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/install.inc b/cmake/install.inc index 9bbcc2cf40..4886e79e73 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.8-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") From aae38f8b248dcd83e12c902a73303694e9ddc170 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 23 Nov 2020 20:08:45 +0800 Subject: [PATCH 026/188] add log --- src/sync/src/syncMain.c | 6 ++---- src/sync/src/syncRestore.c | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index a68f0b7bbb..8d3997a5d6 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -937,8 +937,7 @@ static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer) { SSyncNode * pNode = pPeer->pSyncNode; SPeersStatus *pPeersStatus = (SPeersStatus *)cont; - sDebug("%s, status msg is received, self:%s sstatus:%s sver:%" PRIu64 ", peer:%s sver:%" PRIu64 - ", ack:%d tranId:%u type:%s pfd:%d", + sDebug("%s, status is received, self:%s:%s:%" PRIu64 ", peer:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type], pPeer->peerFd); @@ -1029,8 +1028,7 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type int32_t retLen = taosWriteMsg(pPeer->peerFd, msg, statusMsgLen); if (retLen == statusMsgLen) { - sDebug("%s, status msg is sent, self:%s sstatus:%s sver:%" PRIu64 ", peer:%s sstatus:%s sver:%" PRIu64 - ", ack:%d tranId:%u type:%s pfd:%d", + sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d", pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pPeer->version, pPeersStatus->ack, pPeersStatus->tranId, statusType[pPeersStatus->type], pPeer->peerFd); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 69da1d71fa..2ca4b5424e 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -86,7 +86,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // check the file info sinfo = minfo; - sDebug("%s, get file info:%s", pPeer->id, minfo.name); + sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size); sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion); From 7c9ce886942275ffbee67d15b80eff42954fe109 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 24 Nov 2020 09:18:55 +0800 Subject: [PATCH 027/188] change --- src/connector/jdbc/deploy-pom.xml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 46de54ffb2..1dc2625e62 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -49,17 +49,29 @@ - - org.apache.commons - commons-lang3 - 3.5 - junit junit 4.13 test + + + + org.apache.httpcomponents + httpclient + 4.5.8 + + + org.apache.commons + commons-lang3 + 3.9 + + + com.alibaba + fastjson + 1.2.58 + From e3fc92c9f42c078e4f0cd360ab9e3def1d536ad6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 10:24:43 +0800 Subject: [PATCH 028/188] [TD-225] fix bugs. --- src/client/src/tscSQLParser.c | 18 +++++++++++++----- src/query/src/qExecutor.c | 6 +++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f859d2969e..743cb42eb3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1767,10 +1767,10 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return TSDB_CODE_SUCCESS; } -void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) { +void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) { if (pItem->aliasName != NULL) { tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); - } else { + } else if (multiCols) { char uname[TSDB_COL_NAME_LEN] = {0}; int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); @@ -1781,6 +1781,9 @@ void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrT snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); tstrncpy(name, tmp, TSDB_COL_NAME_LEN); + } else { // use the user-input result column name + int32_t len = MIN(pItem->pNode->token.n + 1, TSDB_COL_NAME_LEN); + tstrncpy(name, pItem->pNode->token.z, len); } } @@ -2055,7 +2058,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; - setResultColName(name, pItem, cvtFunc.originFuncId, &t); + setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2077,7 +2080,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col char name[TSDB_COL_NAME_LEN] = {0}; SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo); + + bool multiColOutput = pItem->pNode->pParam->nExpr > 1; + setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo, multiColOutput); if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2119,7 +2124,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col char name[TSDB_COL_NAME_LEN] = {0}; SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)}; - setResultColName(name, pItem, cvtFunc.originFuncId, &t); + setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; @@ -4243,6 +4248,9 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE tExprTreeDestroy(&p, NULL); taosArrayDestroy(colList); + if (taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); + } } pCondExpr->pTagCond = NULL; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2623c74e45..7d6068ff64 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -947,10 +947,10 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - sas->offset = pQuery->pos; - sas->colList = pQuery->colList; + sas->offset = (QUERY_IS_ASC_QUERY(pQuery)) ? pQuery->pos : pQuery->pos - (size - 1); + sas->colList = pQuery->colList; sas->numOfCols = pQuery->numOfCols; - sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); + sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); if (sas->data == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); From 3b92e7f49b2ddaadcfb2fd11e879214c8495b17a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 10:29:08 +0800 Subject: [PATCH 029/188] [TD-225] update sim test script. --- .../general/parser/col_arithmetic_query.sim | 146 +++++++++++++++--- 1 file changed, 124 insertions(+), 22 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 61a48e4a80..95e0ed68d7 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -7,12 +7,12 @@ $stbPrefix = ca_stb $rowNum = 10000 $i = 0 -$db = $dbPrefix . $i - $db = $dbPrefix . $i sql use $db $tb = $tbPrefix . 0 +$stb = $stbPrefix . $i + ## TBASE-344 sql select c1*2 from $tb if $rows != $rowNum then @@ -41,7 +41,8 @@ if $data00 != 0.000000000 then return -1 endi -if $data01 != NaN then +if $data01 != -nan then + print expect -nan, actual: $data01 return -1 endi @@ -67,6 +68,7 @@ if $rows != 10000 then endi if $data00 != 0.100000000 then + print expect 0.100000000, acutal:$data00 return -1 endi @@ -92,15 +94,15 @@ if $data10 != 16.000000000 then return -1 endi -if $data21 != 20.250000000 then +if $data20 != 20.250000000 then + print expect 20.250000000, acutal:$data21 return -1 endi -if $data31 != 0.000000000 then +if $data30 != 0.000000000 then return -1 endi - # no result return [d.3] ============================================================== sql select c1 * 91- 7 from $tb where ts < 1537146000000 if $rows != 0 then @@ -120,11 +122,59 @@ endi # not available # multi row result aggregation [d.4] -sql select top(c1, 1) - bottom(c1, 1) from $tb -sql select top(c1, 99) - bottom(c1, 99) from $tb +sql_error select top(c1, 1) - bottom(c1, 1) from $tb +sql_error select top(c1, 99) - bottom(c1, 99) from $tb +sql_error select top(c1,1) - 88 from $tb # all data types [d.6] ================================================================ -sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22, c2-c2 from $tb +sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb +if $rows != 10000 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +if $data01 != -nan then + return -1 +endi + +if $data02 != 0.000000000 then + return -1 +endi + +if $data03 != 0.000000000 then + return -1 +endi + +if $data04 != 0.000000000 then + return -1 +endi + +if $data05 != 0.000000000 then + return -1 +endi + +if $data90 != -0.900000000 then + return -1 +endi + +if $data91 != 1.000000000 then + return -1 +endi + +if $data92 != 81.000000000 then + return -1 +endi + +if $data93 != 0.000000000 then + return -1 +endi + +if $data94 != 18.000000000 then + return -1 +endi # error case, ts/bool/binary/nchar not support arithmetic expression sql_error select ts+ts from $tb @@ -135,8 +185,6 @@ sql_error select c9+c8 from $tb sql_error select c7-c8, c9-c8 from $tb sql_error select ts-c9 from $tb sql_error select c8+c7, c9+c9+c8+c7/c6 from $tb -sql_error select c1/0, c2/0, c3/0.0 from $tb -sql_error select c1/0.00000000001 from $tb # arithmetic expression in join [d.7]================================================== @@ -162,7 +210,7 @@ if $data00 != 93.000000000 then return -1 endi -if $data90 != 12.000000000 then +if $data90 != 76.000000000 then return -1 endi @@ -171,27 +219,81 @@ if $rows != 1 then return -1 endi -if $data00 != +if $data00 != 0.090796283 then + return -1 +endi # slimit/soffset not support for normal table query. [d.11]============================ sql_error select sum(c1) from $tb slimit 1 soffset 19; # fill [d.12]========================================================================== -sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); +sql_error select c2-c2, c3-c4, c5%c3 from $tb fill(value, 12); # constant column. [d.13]============================================================== +sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb +if $rows != 10000 then + return -1 +endi +if $data00 != 0 then + return -1 +endi + +if $data01 != 0.000000000 then + return -1 +endi + +if $data02 != 12.987654568 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data04 != 1.100000000 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi # column value filter [d.14]=========================================================== +sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2 +if $rows != 2000 then + return -1 +endi +if $data00 != 0 then + return -1 +endi + +if $data01 != 0.000000000 then + return -1 +endi + +if $data02 != 12.987654568 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data20 != 0 then + return -1 +endi # tag filter(not support for normal table). [d.15]===================================== sql_error select c2+99 from $tb where t1=12; # multi-field output [d.16]============================================================ -sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb - -sql select c4*1+1/2 from $tb +sql select c4*1+1/2,c4*1+1/2,c4*1+1/2,c4*1+1/2,c4*1+1/2 from $tb if $rows != $rowNum then return -1 endi @@ -216,15 +318,17 @@ sql_error select c7-c9 from $tb interval(2y) # see test cases below # multiple retrieve [d.20]=============================================================== -sql select c2-c2 from $tb; +sql select c2-c2, 911 from $tb #======================================= aggregation function arithmetic query cases ================ # asc/desc order [d.2] sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc; sql select (count(c1) * 2) % 7.9 from $stb order by ts desc; +sql select spread(c1 )/44 from $stb order by ts asc; + # all possible function in the arithmetic expressioin -sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), xxxx, from $stb where ts < and ts > +sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2) from $stb where ts < and ts > # no result return [d.3] sql select first(c1) * 91 - 7, last(c3) from $stb where ts < 1537146000000 @@ -291,7 +395,7 @@ sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); # tag filter(not support for normal table). [d.15] -sql_error select c2+99 from $tb where t1=12; +sql_error select sum(c2)+99 from $tb where t1=12; # multi-field output [d.16] sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb @@ -325,5 +429,3 @@ sql select c2-c2 from $tb; sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb - - From c17d202e0c35c1a25ffcb23ef7424094fa9757d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 11:00:10 +0800 Subject: [PATCH 030/188] [TD-225] fix compiler error. --- src/query/src/qResultbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index dfa8bcd95b..55a7aea53a 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -122,7 +122,7 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); assert(ret == 0); - ret = fwrite(t, 1, size, pResultBuf->file); + ret = (int32_t) fwrite(t, 1, size, pResultBuf->file); assert(ret == size); if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { From c3fd294c0b816a0e52a12ba7b9d178899997ae21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 12:00:17 +0800 Subject: [PATCH 031/188] [TD-225] fix bugs in regression test. --- src/query/inc/qUtil.h | 1 + src/query/src/qExecutor.c | 2 +- src/query/src/qUtil.c | 6 +++ .../general/parser/col_arithmetic_query.sim | 51 ++++++++++++++++++- 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 62bf245a2b..bc23ffadd4 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -81,6 +81,7 @@ int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv); +bool isPointInterpoQuery(SQuery *pQuery); #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7d6068ff64..fce25ae119 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1825,7 +1825,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { } // todo refactor with isLastRowQuery -static bool isPointInterpoQuery(SQuery *pQuery) { +bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionID = pQuery->pExpr1[i].base.functionId; if (functionID == TSDB_FUNC_INTERP) { diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 4b8d7ea930..2b4cdef9b0 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -388,6 +388,12 @@ uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { return 0; // for simple table query, the uid is always set to be 0; } + SQuery* pQuery = pRuntimeEnv->pQuery; + if ((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || + pRuntimeEnv->groupbyNormalCol) { + return 0; + } + STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current); return id->uid; } \ No newline at end of file diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 95e0ed68d7..408b039144 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -323,9 +323,56 @@ sql select c2-c2, 911 from $tb #======================================= aggregation function arithmetic query cases ================ # asc/desc order [d.2] sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc; -sql select (count(c1) * 2) % 7.9 from $stb order by ts desc; +if $rows != 1 then + return -1 +endi -sql select spread(c1 )/44 from $stb order by ts asc; +if $data00 != 0.000000000 then + return -1 +endi + +sql select first(c1) * (2/99) from $stb order by ts desc; +if $rows != 1 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb order by ts desc; +if $rows != 1 then + return -1 +endi + +if $data00 != 1.800000000 then + return -1 +endi + +if $data01 != 100000 then + return -1 +endi + +if $data02 != 200000 then + return -1 +endi + +sql select spread( c1 )/44, spread(c1), 0.204545455 * 44 from $stb order by ts asc; +if $rows != 1 then + return -1 +endi + +if $data00 != 0.204545455 then + return -1 +endi + +if $data01 != 9.000000000 then + return -1 +endi + +if $data02 != 9.000000020 then + return -1 +endi # all possible function in the arithmetic expressioin sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2) from $stb where ts < and ts > From 99815378bf29bc0bda6f1fc13066f09b1dff2227 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Nov 2020 14:54:30 +0800 Subject: [PATCH 032/188] TD-2086 --- src/sync/inc/syncInt.h | 1 + src/sync/src/syncMain.c | 18 +++++------------- src/sync/src/syncRestore.c | 10 ++++------ 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 7dbb1985d4..215ae7a58d 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -41,6 +41,7 @@ typedef enum { #define SYNC_RECV_BUFFER_SIZE (5*1024*1024) #define SYNC_FWD_TIMER 300 #define SYNC_ROLE_TIMER 10000 +#define SYNC_WAIT_AFTER_CHOOSE_MASTER 3 #define nodeRole pNode->peerInfo[pNode->selfIndex]->role #define nodeVersion pNode->peerInfo[pNode->selfIndex]->version diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 8d3997a5d6..d5fc79b635 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -626,17 +626,9 @@ static void syncChooseMaster(SSyncNode *pNode) { sInfo("vgId:%d, start to work as master", pNode->vgId); nodeRole = TAOS_SYNC_ROLE_MASTER; -#if 0 - for (int32_t i = 0; i < pNode->replica; ++i) { - if (i == index) continue; - pPeer = pNode->peerInfo[i]; - if (pPeer->version == nodeVersion) { - pPeer->role = TAOS_SYNC_ROLE_SLAVE; - pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; - sInfo("%s, it shall work as slave", pPeer->id); - } - } -#endif + // Wait for other nodes to receive status to avoid version inconsistency + taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER); + syncResetFlowCtrl(pNode); (*pNode->notifyRole)(pNode->vgId, nodeRole); } else { @@ -761,7 +753,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new sDebug("vgId:%d, choose master", pNode->vgId); syncChooseMaster(pNode); } else { - sDebug("vgId:%d, cannot choose master since roles inconsistent", pNode->vgId); + sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId); } } @@ -1124,7 +1116,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { } int32_t vgId = firstPkt.syncHead.vgId; - SSyncNode **ppNode = (SSyncNode **)taosHashGet(tsVgIdHash, (const char *)&vgId, sizeof(int32_t)); + SSyncNode **ppNode = taosHashGet(tsVgIdHash, &vgId, sizeof(int32_t)); if (ppNode == NULL || *ppNode == NULL) { sError("vgId:%d, vgId could not be found", vgId); taosCloseSocket(connFd); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 2ca4b5424e..db3c6b318d 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -147,12 +147,10 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { static int32_t syncRestoreWal(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; int32_t ret, code = -1; + uint64_t lastVer = 0; - void *buffer = calloc(SYNC_MAX_SIZE, 1); // size for one record - if (buffer == NULL) return -1; - - SWalHead *pHead = (SWalHead *)buffer; - uint64_t lastVer = 0; + SWalHead *pHead = calloc(SYNC_MAX_SIZE, 1); // size for one record + if (pHead == NULL) return -1; while (1) { ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead)); @@ -188,7 +186,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { sError("%s, failed to restore wal from syncFd:%d since %s", pPeer->id, pPeer->syncFd, strerror(errno)); } - free(buffer); + free(pHead); return code; } From 41049dba7262542e9415da4f49f5d1d274e1428f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 16:51:57 +0800 Subject: [PATCH 033/188] [TD-225] update sim. --- src/client/inc/tscLocalMerge.h | 1 - src/client/src/tscLocalMerge.c | 19 +- .../general/parser/col_arithmetic_query.sim | 208 ++++++++++++++---- 3 files changed, 171 insertions(+), 57 deletions(-) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 2c7c2f51d0..43ba31f331 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -56,7 +56,6 @@ typedef struct SLocalReducer { tFilePage * pTempBuffer; struct SQLFunctionCtx *pCtx; int32_t rowSize; // size of each intermediate result. - int32_t finalRowSize; // final result row size int32_t status; // denote it is in reduce process, in reduce process, it bool hasPrevRow; // cannot be released bool hasUnprocessedRow; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 3c7d46f914..ad8d4d72d0 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -330,22 +330,19 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd pReducer->nResultBufSize = pMemBuffer[0]->pageSize * 16; pReducer->pResultBuf = (tFilePage *)calloc(1, pReducer->nResultBufSize + sizeof(tFilePage)); - pReducer->finalRowSize = tscGetResRowLength(pQueryInfo->exprList); pReducer->resColModel = finalmodel; pReducer->resColModel->capacity = pReducer->nResultBufSize; - pReducer->finalModel = pFFModel; - assert(pReducer->finalRowSize > 0); - if (pReducer->finalRowSize > 0) { - pReducer->resColModel->capacity /= pReducer->finalRowSize; + if (finalmodel->rowSize > 0) { + pReducer->resColModel->capacity /= finalmodel->rowSize; } - assert(pReducer->finalRowSize <= pReducer->rowSize); + assert(finalmodel->rowSize > 0 && finalmodel->rowSize <= pReducer->rowSize); pReducer->pFinalRes = calloc(1, pReducer->rowSize * pReducer->resColModel->capacity); if (pReducer->pTempBuffer == NULL || pReducer->discardData == NULL || pReducer->pResultBuf == NULL || - /*pReducer->pBufForInterpo == NULL || */pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { + pReducer->pFinalRes == NULL || pReducer->prevRowOfInput == NULL) { tfree(pReducer->pTempBuffer); tfree(pReducer->discardData); tfree(pReducer->pResultBuf); @@ -920,7 +917,7 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pLocalReducer->pFillInfo); } - memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalRowSize)); + memcpy(pRes->data, pBeforeFillData->data, (size_t)(pRes->numOfRows * pLocalReducer->finalModel->rowSize)); pRes->numOfClauseTotal += pRes->numOfRows; pBeforeFillData->num = 0; @@ -1256,7 +1253,7 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur tColModelCompact(pModel, pResBuf, pModel->capacity); if (tscIsSecondStageQuery(pQueryInfo)) { - pLocalReducer->finalRowSize = doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalRowSize); + doArithmeticCalculate(pQueryInfo, pResBuf, pModel->rowSize, pLocalReducer->finalModel->rowSize); } #ifdef _DEBUG_VIEW @@ -1627,7 +1624,8 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) } int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { - char* pbuf = calloc(1, pOutput->num * rowSize); + int32_t maxRowSize = MAX(rowSize, finalRowSize); + char* pbuf = calloc(1, pOutput->num * maxRowSize); size_t size = tscNumOfFields(pQueryInfo); SArithmeticSupport arithSup = {0}; @@ -1660,7 +1658,6 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ offset += pSup->field.bytes; } - assert(finalRowSize <= rowSize); memcpy(pOutput->data, pbuf, pOutput->num * offset); tfree(pbuf); diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 408b039144..ff9f548df8 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -349,11 +349,11 @@ if $data00 != 1.800000000 then return -1 endi -if $data01 != 100000 then +if $data01 != 100000.000000000 then return -1 endi -if $data02 != 200000 then +if $data02 != 200000.000000000 then return -1 endi @@ -374,77 +374,183 @@ if $data02 != 9.000000020 then return -1 endi -# all possible function in the arithmetic expressioin -sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2) from $stb where ts < and ts > +# all possible function in the arithmetic expression, add more +sql select min(c1) * max(c2) /4, sum(c1) * apercentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), count(1)/sum(c1), avg(c2)*count(c2) from $stb where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-11-25 19:30:00.000'; +if $rows != 1 then + return -1 +endi -# no result return [d.3] +if $data00 != 0.000000000 then + return -1 +endi + +if $data01 != 225000.000000000 then + return -1 +endi + +if $data02 != 8.077777778 then + return -1 +endi + +if $data03 != inf then + return -1 +endi + +if $data04 != 0.444444444 then + return -1 +endi + +if $data05 != 450000.000000000 then + return -1 +endi + +# no result return [d.3]=============================================================== sql select first(c1) * 91 - 7, last(c3) from $stb where ts < 1537146000000 if $rows != 0 then return -1 endi # no result return [d.3] -sql select sum(c2) - avg(c2) from $tb where ts>xxx +sql select sum(c2) - avg(c2) from $stb where ts > '2018-11-25 19:30:00.000' if $rows != 0 then return -1 endi -# single row result aggregation [d.4] -sql select +# single row result aggregation [d.4]=================================================== +# all those cases are aggregation test cases. # error cases -sql_error select first(c1, c2) - last(c1, c2) from $tb +sql_error select first(c1, c2) - last(c1, c2) from $stb +sql_error select top(c1, 5) - bottom(c1, 5) from $stb +sql_error select first(*) - 99 from $stb # multi row result aggregation [d.4] -sql select top(c1, 1) - bottom(c1, 1) from $tb -sql select top(c1, 99) - bottom(c1, 99) from $tb +sql_error select top(c1, 1) - bottom(c1, 1) from $stb +sql_error select top(c1, 99) - bottom(c1, 99) from $stb -# all data types [d.6] -sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $tb +# query on super table [d.5]============================================================= +# all cases in this part are query on super table + +# all data types [d.6]=================================================================== +sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $stb # error case, ts/bool/binary/nchar not support arithmetic expression -sql_error select ts+ts from $tb -sql_error select ts+22 from $tb -sql_error select c7*12 from $tb -sql_error select c8/55 from $tb -sql_error select c9+c8 from $tb +sql_error select first(c7)*12 from $stb +sql_error select last(c8)/55 from $stb +sql_error select last_row(c9) + last_row(c8) from $stb -# arithmetic expression in join [d.7] +# arithmetic expression in join [d.7]=============================================================== -# arithmetic expression in union [d.8] +# arithmetic expression in union [d.8]=============================================================== -# arithmetic expression in group by [d.9] +# arithmetic expression in group by [d.9]=============================================================== # in group by tag -# not support for normal table -sql_error select c5*99 from $tb group by t1 +sql select avg(c4)*99 from $stb group by t1 +if $rows != 10 then + return -1 +endi + +if $data00 != 445.500000000 then + return -1 +endi + +if $data01 != 0 then + return -1 +endi + +if $data90 != 445.500000000 then + return -1 +endi + +if $data91 != 9 then + return -1 +endi # in group by column -sql_error select c6-c6+c3*12 from $tb group by c3; +sql select apercentile(c6, 50)-first(c6)+last(c5)*12, last(c5)*12 from ca_stb0 group by c2; +if $rows != 10 then + return -1 +endi -sql select first(c6) - last(c6) *12 / count(*) from $tb group by c3; +if $data00 != 0.000000000 then + return -1 +endi -# limit offset [d.10] -sql select c6-c6+12 from $tb limit 12 offset 99; -sql select c4/99.123 from $tb limit 1 offset 9999; +if $data01 != 0.000000000 then + return -1 +endi -# slimit/soffset not suport for normal table query. [d.11] -sql_error select sum(c1) from $tb slimit 1 soffset 19; +if $data10 != 12.000000000 then + return -1 +endi -# fill [d.12] -sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); +if $data11 != 12.000000000 then + return -1 +endi -# constant column. [d.13] +if $data20 != 24.000000000 then + return -1 +endi + +if $data21 != 24.000000000 then + return -1 +endi + +sql_error select first(c6) - last(c6) *12 / count(*) from $stb group by c3; + +sql select first(c6) - last(c6) *12 / count(*) from $stb group by c5; +if $rows != 10 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +if $data10 != 0.997600000 then + return -1 +endi + +if $data90 != 8.978400000 then + return -1 +endi + +# limit offset [d.10]=============================================================== +sql select first(c6) - sum(c6) + 12 from $stb limit 12 offset 0; +if $rows != 1 then + return -1 +endi + +if $data00 != -449988.000000000 then + return -1 +endi + +sql select apercentile(c4, 21) / 99.123 from $stb limit 1 offset 1; +if $rows != 0 then + return -1 +endi + +# slimit/soffset not suport for normal table query. [d.11]=============================================================== +sql_error select sum(c1) from $stb slimit 1 soffset 19; + +sql select sum(c1) from $stb interval(1s) group by tbname slimit 1 soffset 1; + +# fill [d.12]=============================================================== +sql_error select first(c1)-last(c1), sum(c3)*count(c3), spread(c5 ) % count(*) from $stb interval(1s) fill(prev); +sql_error select first(c1) from $stb fill(value, 20); + +# constant column. [d.13]=============================================================== -# column value filter [d.14] +# column value filter [d.14]=============================================================== -# tag filter(not support for normal table). [d.15] -sql_error select sum(c2)+99 from $tb where t1=12; +# tag filter. [d.15]=============================================================== +sql select sum(c2)+99 from $stb where t1=12; -# multi-field output [d.16] +# multi-field output [d.16]=============================================================== sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb sql select c4*1+1/2 from $tb @@ -461,18 +567,30 @@ if $data90 != 9.500000000 then return -1 endi -# interval query [d.17] -sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s) -sql_error select c7-c9 from $tb interval(2y) +# interval query [d.17]=============================================================== +sql select avg(c2)*count(c2), sum(c3)-first(c3), last(c4)+9 from $stb interval(1s) +if $rows != 10000 then + return -1 +endi -# aggregation query [d.18] -# see test cases below +if $data00 != @18-09-17 09:00:00.000@ then + return -1 +endi -# first/last query [d.19] -# see test cases below +sql select first(c7)- last(c1) from $tb interval(2y) -# multiple retrieve [d.20] +# aggregation query [d.18]=============================================================== +# all cases in this part are aggregation query test. + +# first/last query [d.19]=============================================================== + + +# multiple retrieve [d.20]=============================================================== sql select c2-c2 from $tb; sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb + + +#====================================================super table query================================================== + From 53dd2050fd5786776083b435158587e580f70609 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Tue, 24 Nov 2020 17:05:20 +0800 Subject: [PATCH 034/188] [TD-2140]: use tref to manage sql objects --- src/client/inc/tsclient.h | 6 ++--- src/client/src/tscLocal.c | 7 ++++-- src/client/src/tscServer.c | 49 ++++++++++++++------------------------ src/client/src/tscSql.c | 18 +++++--------- src/client/src/tscSub.c | 4 ++-- src/client/src/tscSystem.c | 14 +++++++---- src/client/src/tscUtil.c | 22 ++++++++--------- 7 files changed, 54 insertions(+), 66 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ff36cf0f5a..f38a8e6c18 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -333,7 +333,7 @@ typedef struct STscObj { char superAuth : 1; uint32_t connId; uint64_t rid; // ref ID returned by taosAddRef - struct SSqlObj * pHb; + int64_t hbrid; struct SSqlObj * sqlList; struct SSqlStream *streamList; void* pDnodeConn; @@ -373,7 +373,7 @@ typedef struct SSqlObj { struct SSqlObj **pSubs; struct SSqlObj *prev, *next; - struct SSqlObj **self; + int64_t self; } SSqlObj; typedef struct SSqlStream { @@ -507,7 +507,7 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField } extern SCacheObj* tscMetaCache; -extern SCacheObj* tscObjCache; +extern int tscObjRef; extern void * tscTmr; extern void * tscQhandle; extern int tscKeepConn[]; diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index 538e652f3c..4c28adc261 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -825,8 +825,11 @@ static int32_t tscProcessClientVer(SSqlObj *pSql) { static int32_t tscProcessServStatus(SSqlObj *pSql) { STscObj* pObj = pSql->pTscObj; - if (pObj->pHb != NULL) { - if (pObj->pHb->res.code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHb != NULL) { + int32_t code = pHb->res.code; + taosReleaseRef(tscObjRef, pObj->hbrid); + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { pSql->res.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; return pSql->res.code; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cbc5604a27..b964a4f15d 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -175,10 +175,10 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { if (pRsp->streamId) tscKillStream(pObj, htonl(pRsp->streamId)); } } else { - tscDebug("%p heartbeat failed, code:%s", pObj->pHb, tstrerror(code)); + tscDebug("%" PRId64 " heartbeat failed, code:%s", pObj->hbrid, tstrerror(code)); } - if (pObj->pHb != NULL) { + if (pObj->hbrid != 0) { int32_t waitingDuring = tsShellActivityTimer * 500; tscDebug("%p send heartbeat in %dms", pSql, waitingDuring); @@ -193,20 +193,12 @@ void tscProcessActivityTimer(void *handle, void *tmrId) { STscObj *pObj = taosAcquireRef(tscRefId, rid); if (pObj == NULL) return; - SSqlObj* pHB = pObj->pHb; - - void** p = taosCacheAcquireByKey(tscObjCache, &pHB, sizeof(TSDB_CACHE_PTR_TYPE)); - if (p == NULL) { - tscWarn("%p HB object has been released already", pHB); - taosReleaseRef(tscRefId, pObj->rid); - return; - } - - assert(*pHB->self == pHB); + SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); + assert(pHB->self == pObj->hbrid); pHB->retry = 0; int32_t code = tscProcessSql(pHB); - taosCacheRelease(tscObjCache, (void**) &p, false); + taosReleaseRef(tscObjRef, pObj->hbrid); if (code != TSDB_CODE_SUCCESS) { tscError("%p failed to sent HB to server, reason:%s", pHB, tstrerror(code)); @@ -236,7 +228,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { .msgType = pSql->cmd.msgType, .pCont = pMsg, .contLen = pSql->cmd.payloadLen, - .ahandle = pSql, + .ahandle = (void*)pSql->self, .handle = NULL, .code = 0 }; @@ -247,26 +239,24 @@ int tscSendMsgToServer(SSqlObj *pSql) { void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { TSDB_CACHE_PTR_TYPE handle = (TSDB_CACHE_PTR_TYPE) rpcMsg->ahandle; - void** p = taosCacheAcquireByKey(tscObjCache, &handle, sizeof(TSDB_CACHE_PTR_TYPE)); - if (p == NULL) { + SSqlObj* pSql = (SSqlObj*)taosAcquireRef(tscObjRef, handle); + if (pSql == NULL) { rpcFreeCont(rpcMsg->pCont); return; } - - SSqlObj* pSql = *p; - assert(pSql != NULL); + assert(pSql->self == handle); STscObj *pObj = pSql->pTscObj; SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; - assert(*pSql->self == pSql); pSql->rpcRid = -1; if (pObj->signature != pObj) { tscDebug("%p DB connection is closed, cmd:%d pObj:%p signature:%p", pSql, pCmd->command, pObj, pObj->signature); - taosCacheRelease(tscObjCache, (void**) &p, true); + taosRemoveRef(tscObjRef, pSql->self); + taosReleaseRef(tscObjRef, pSql->self); rpcFreeCont(rpcMsg->pCont); return; } @@ -276,10 +266,8 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { tscDebug("%p sqlObj needs to be released or DB connection is closed, cmd:%d type:%d, pObj:%p signature:%p", pSql, pCmd->command, pQueryInfo->type, pObj, pObj->signature); - void** p1 = p; - taosCacheRelease(tscObjCache, (void**) &p1, false); - - taosCacheRelease(tscObjCache, (void**) &p, true); + taosRemoveRef(tscObjRef, pSql->self); + taosReleaseRef(tscObjRef, pSql->self); rpcFreeCont(rpcMsg->pCont); return; } @@ -322,7 +310,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { // if there is an error occurring, proceed to the following error handling procedure. if (rpcMsg->code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { - taosCacheRelease(tscObjCache, (void**) &p, false); + taosReleaseRef(tscObjRef, pSql->self); rpcFreeCont(rpcMsg->pCont); return; } @@ -390,11 +378,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { (*pSql->fp)(pSql->param, pSql, rpcMsg->code); } - void** p1 = p; - taosCacheRelease(tscObjCache, (void**) &p1, false); + taosReleaseRef(tscObjRef, pSql->self); if (shouldFree) { // in case of table-meta/vgrouplist query, automatically free it - taosCacheRelease(tscObjCache, (void **)&p, true); + taosRemoveRef(tscObjRef, pSql->self); tscDebug("%p sqlObj is automatically freed", pSql); } @@ -2019,7 +2006,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { // TODO multithread problem static void createHBObj(STscObj* pObj) { - if (pObj->pHb != NULL) { + if (pObj->hbrid != 0) { return; } @@ -2049,7 +2036,7 @@ static void createHBObj(STscObj* pObj) { registerSqlObj(pSql); tscDebug("%p HB is allocated, pObj:%p", pSql, pObj); - pObj->pHb = pSql; + pObj->hbrid = pSql->self; } int tscProcessConnectRsp(SSqlObj *pSql) { diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 020305a0a8..50dfbcbec9 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -276,8 +276,8 @@ void taos_close(TAOS *taos) { pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); - SSqlObj* pHb = pObj->pHb; - if (pHb != NULL && atomic_val_compare_exchange_ptr(&pObj->pHb, pHb, 0) == pHb) { + SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHb != NULL) { if (pHb->rpcRid > 0) { // wait for rsp from dnode rpcCancelRequest(pHb->rpcRid); pHb->rpcRid = -1; @@ -285,6 +285,7 @@ void taos_close(TAOS *taos) { tscDebug("%p HB is freed", pHb); taos_free_result(pHb); + taosReleaseRef(tscObjRef, pHb->self); } int32_t ref = T_REF_DEC(pObj); @@ -597,8 +598,7 @@ void taos_free_result(TAOS_RES *res) { bool freeNow = tscKillQueryInDnode(pSql); if (freeNow) { tscDebug("%p free sqlObj in cache", pSql); - SSqlObj** p = pSql->self; - taosCacheRelease(tscObjCache, (void**) &p, true); + taosReleaseRef(tscObjRef, pSql->self); } } @@ -691,13 +691,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) { continue; } - void** p = taosCacheAcquireByKey(tscObjCache, &pSub, sizeof(TSDB_CACHE_PTR_TYPE)); - if (p == NULL) { - continue; - } - - SSqlObj* pSubObj = (SSqlObj*) (*p); - assert(pSubObj->self == (SSqlObj**) p); + SSqlObj* pSubObj = pSub; pSubObj->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; if (pSubObj->rpcRid > 0) { @@ -706,7 +700,7 @@ static void tscKillSTableQuery(SSqlObj *pSql) { } tscQueueAsyncRes(pSubObj); - taosCacheRelease(tscObjCache, (void**) &p, false); + taosReleaseRef(tscObjRef, pSubObj->self); } tscDebug("%p super table query cancelled", pSql); diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 2c81bd7c7c..3bce7bee77 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -179,8 +179,8 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* fail: tscError("tscCreateSubscription failed at line %d, reason: %s", line, tstrerror(code)); if (pSql != NULL) { - if (pSql->self != NULL) { - taos_free_result(pSql); + if (pSql->self != 0) { + taosReleaseRef(tscObjRef, pSql->self); } else { tscFreeSqlObj(pSql); } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 9e9a00550a..672d87e0c8 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -15,7 +15,7 @@ #include "os.h" #include "taosmsg.h" -#include "tcache.h" +#include "tref.h" #include "trpc.h" #include "tsystem.h" #include "ttimer.h" @@ -31,7 +31,7 @@ // global, not configurable SCacheObj* tscMetaCache; -SCacheObj* tscObjCache; +int tscObjRef = -1; void * tscTmr; void * tscQhandle; void * tscCheckDiskUsageTmr; @@ -144,7 +144,11 @@ void taos_init_imp(void) { int64_t refreshTime = 10; // 10 seconds by default if (tscMetaCache == NULL) { tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); +#ifndef SQLOBJ_USE_CACHE + tscObjRef = taosOpenRef(4096, tscFreeRegisteredSqlObj); +#else tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj"); +#endif } tscRefId = taosOpenRef(200, tscCloseTscObj); @@ -167,9 +171,9 @@ void taos_cleanup(void) { taosCacheCleanup(m); } - m = tscObjCache; - if (m != NULL && atomic_val_compare_exchange_ptr(&tscObjCache, m, 0) == m) { - taosCacheCleanup(m); + int refId = atomic_exchange_32(&tscObjRef, -1); + if (refId != -1) { + taosCloseRef(refId); } m = tscQhandle; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7a82bcaaab..b35dbb47e6 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -447,18 +447,18 @@ static void tscFreeSubobj(SSqlObj* pSql) { void tscFreeRegisteredSqlObj(void *pSql) { assert(pSql != NULL); - SSqlObj** p = (SSqlObj**)pSql; - STscObj* pTscObj = (*p)->pTscObj; + SSqlObj* p = *(SSqlObj**)pSql; + STscObj* pTscObj = p->pTscObj; - assert((*p)->self != 0 && (*p)->self == (p)); - tscFreeSqlObj(*p); + assert(p->self != 0); + tscFreeSqlObj(p); int32_t ref = T_REF_DEC(pTscObj); assert(ref >= 0); - tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref); + tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", p, pTscObj, ref); if (ref == 0) { - tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj); + tscDebug("%p all sqlObj freed, free tscObj:%p", p, pTscObj); taosRemoveRef(tscRefId, pTscObj->rid); } } @@ -1560,6 +1560,8 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { } } +#ifndef SQLOBJ_USE_CACHE +#else void tscSetFreeHeatBeat(STscObj* pObj) { if (pObj == NULL || pObj->signature != pObj || pObj->pHb == NULL) { return; @@ -1572,6 +1574,7 @@ void tscSetFreeHeatBeat(STscObj* pObj) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pHeatBeat->cmd, 0); pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; } +#endif /* * the following four kinds of SqlObj should not be freed @@ -1591,7 +1594,7 @@ bool tscShouldBeFreed(SSqlObj* pSql) { } STscObj* pTscObj = pSql->pTscObj; - if (pSql->pStream != NULL || pTscObj->pHb == pSql || pSql->pSubscription != NULL) { + if (pSql->pStream != NULL || pTscObj->hbrid == pSql->self || pSql->pSubscription != NULL) { return false; } @@ -1883,13 +1886,10 @@ void tscResetForNextRetrieve(SSqlRes* pRes) { } void registerSqlObj(SSqlObj* pSql) { - int32_t DEFAULT_LIFE_TIME = 2 * 600 * 1000; // 1200 sec - int32_t ref = T_REF_INC(pSql->pTscObj); tscDebug("%p add to tscObj:%p, ref:%d", pSql, pSql->pTscObj, ref); - TSDB_CACHE_PTR_TYPE p = (TSDB_CACHE_PTR_TYPE) pSql; - pSql->self = taosCachePut(tscObjCache, &p, sizeof(TSDB_CACHE_PTR_TYPE), &p, sizeof(TSDB_CACHE_PTR_TYPE), DEFAULT_LIFE_TIME); + pSql->self = taosAddRef(tscObjRef, pSql); } SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cmd) { From f9c34c937068a76cde87aa719fb177749846404d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 24 Nov 2020 17:27:42 +0800 Subject: [PATCH 035/188] [TD-2125][TD-2173]python connector --- src/connector/python/linux/python2/setup.py | 2 +- .../python/linux/python2/taos/cinterface.py | 115 ++++++++++++---- .../python/linux/python2/taos/cursor.py | 20 ++- src/connector/python/linux/python3/setup.py | 2 +- .../python/linux/python3/taos/cinterface.py | 129 ++++++++++++------ .../python/linux/python3/taos/cursor.py | 26 ++-- src/connector/python/windows/python2/setup.py | 2 +- .../python/windows/python2/taos/cinterface.py | 118 ++++++++++++---- .../python/windows/python2/taos/cursor.py | 22 ++- src/connector/python/windows/python3/setup.py | 2 +- .../python/windows/python3/taos/cinterface.py | 107 ++++++++++++--- .../python/windows/python3/taos/cursor.py | 22 ++- 12 files changed, 428 insertions(+), 139 deletions(-) diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index 2e4f80b8f0..b3daa98bdc 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 269326535c..6f0435722f 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -227,8 +270,8 @@ class CTaosInterface(object): print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -237,7 +280,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -310,6 +353,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -393,7 +454,7 @@ class CTaosInterface(object): def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result) + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') if __name__ == '__main__': @@ -410,4 +471,4 @@ if __name__ == '__main__': print(data) cinter.freeResult(result) - cinter.close(conn) \ No newline at end of file + cinter.close(conn) diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index bc6670ca77..eee5e39488 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -49,7 +49,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock( + block, self._block_rows = CTaosInterface.fetchRow( self._result, self._fields) if self._block_rows == 0: raise StopIteration @@ -190,6 +190,23 @@ class TDengineCursor(object): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: @@ -203,7 +220,6 @@ class TDengineCursor(object): for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def nextset(self): """ """ diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py index 03a49fc1c5..f49ebe2b6d 100644 --- a/src/connector/python/linux/python3/setup.py +++ b/src/connector/python/linux/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index be5b99d8c1..bbf5a0c714 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -253,7 +296,7 @@ class CTaosInterface(object): raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -308,29 +351,26 @@ class CTaosInterface(object): return fields - # @staticmethod - # def fetchBlock(result, fields): - # pblock = ctypes.c_void_p(0) - # num_of_rows = CTaosInterface.libtaos.taos_fetch_block( - # result, ctypes.byref(pblock)) - # if num_of_rows == 0: - # return None, 0 - - # isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) - # blocks = [None] * len(fields) - # fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - # fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] - # for i in range(len(fields)): - # data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - - # if fields[i]['type'] not in _CONVERT_FUNC: - # raise DatabaseError("Invalid data type returned from database") - # print('====================',fieldLen[i]) - # blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) - - # return blocks, abs(num_of_rows) @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -350,6 +390,7 @@ class CTaosInterface(object): else: return None, 0 return blocks, abs(num_of_rows) + @staticmethod def freeResult(result): CTaosInterface.libtaos.taos_free_result(result) diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index eb10bed485..d8184668c8 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -5,7 +5,6 @@ import threading # querySeqNum = 0 - class TDengineCursor(object): """Database cursor which is used to manage the context of a fetch operation. @@ -51,7 +50,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock( + block, self._block_rows = CTaosInterface.fetchRow( self._result, self._fields) if self._block_rows == 0: raise StopIteration @@ -196,18 +195,13 @@ class TDengineCursor(object): def fetchall(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ - # if threading.get_ident() != self._threadId: - # info ="[WARNING] Cursor fetchall:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: raise ProgrammingError(CTaosInterface.errStr(self._result), errno) @@ -218,6 +212,22 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) def nextset(self): """ """ diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py index fd82a55650..34cb7a04d7 100644 --- a/src/connector/python/windows/python2/setup.py +++ b/src/connector/python/windows/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index 084d38e41c..d4bf1a0350 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -225,9 +268,10 @@ class CTaosInterface(object): if connection.value == None: print('connect to TDengine failed') + raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -236,7 +280,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -252,7 +296,7 @@ class CTaosInterface(object): raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -309,6 +353,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -393,7 +455,7 @@ class CTaosInterface(object): def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result) + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') if __name__ == '__main__': @@ -410,4 +472,4 @@ if __name__ == '__main__': print(data) cinter.freeResult(result) - cinter.close(conn) \ No newline at end of file + cinter.close(conn) diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index 35846cbe11..71651afee1 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -50,7 +50,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -143,7 +143,25 @@ class TDengineCursor(object): """ if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: @@ -178,7 +196,7 @@ class TDengineCursor(object): self._description = None self._rowcount = -1 if self._result is not None: - CTaosInterface.freeResult(self._result) + CTaosInterface.freeResult(self._result) self._result = None self._fields = None self._block = None diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py index 9abdace5a9..c09644d330 100644 --- a/src/connector/python/windows/python3/setup.py +++ b/src/connector/python/windows/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index 68adf191c9..fb1b65fc15 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -104,16 +104,48 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) + return res _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +160,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -227,8 +272,8 @@ class CTaosInterface(object): print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -237,7 +282,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -310,6 +355,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index b58b494cad..b813bba357 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -51,7 +51,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -144,7 +144,25 @@ class TDengineCursor(object): """ if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") - + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: From 32138695c632fb3adbd81785d6de404e747ac780 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 18:04:15 +0800 Subject: [PATCH 036/188] [TD-225] update sim. --- .../general/parser/col_arithmetic_query.sim | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index ff9f548df8..5324e74dd2 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -532,10 +532,19 @@ if $rows != 0 then return -1 endi -# slimit/soffset not suport for normal table query. [d.11]=============================================================== -sql_error select sum(c1) from $stb slimit 1 soffset 19; +sql select apercentile(c4, 21) / sum(c4) from $stb interval(1s) limit 1 offset 1; +if $rows != 1 then + return -1 +endi -sql select sum(c1) from $stb interval(1s) group by tbname slimit 1 soffset 1; +# slimit/soffset not support for normal table query. [d.11]=============================================================== +sql select sum(c1) from $stb slimit 1 soffset 19; +if $rows != 0 then + return -1 +endi + +sql select sum(c1) from $stb interval(1s) group by tbname slimit 1 soffset 1 +sql select sum(c1) from ca_stb0 interval(1s) group by tbname slimit 2 soffset 4 limit 10 offset 1 # fill [d.12]=============================================================== sql_error select first(c1)-last(c1), sum(c3)*count(c3), spread(c5 ) % count(*) from $stb interval(1s) fill(prev); From 4d789b0871a47fcfd6479ec4ffc7d35450ac051e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Nov 2020 18:05:18 +0800 Subject: [PATCH 037/188] TD-2211 --- src/sync/inc/syncInt.h | 19 ++++++++++++------- src/sync/src/syncMain.c | 13 +++++++------ src/sync/src/syncRestore.c | 20 +++++++++++++++----- src/sync/src/syncRetrieve.c | 22 +++++++++++++++++++--- src/sync/src/taosTcpPool.c | 3 +++ 5 files changed, 56 insertions(+), 21 deletions(-) diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 215ae7a58d..05b7adc5f4 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -28,13 +28,14 @@ extern "C" { #define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }} typedef enum { - TAOS_SMSG_SYNC_DATA = 1, - TAOS_SMSG_FORWARD = 2, - TAOS_SMSG_FORWARD_RSP = 3, - TAOS_SMSG_SYNC_REQ = 4, - TAOS_SMSG_SYNC_RSP = 5, - TAOS_SMSG_SYNC_MUST = 6, - TAOS_SMSG_STATUS = 7 + TAOS_SMSG_SYNC_DATA = 1, + TAOS_SMSG_FORWARD = 2, + TAOS_SMSG_FORWARD_RSP = 3, + TAOS_SMSG_SYNC_REQ = 4, + TAOS_SMSG_SYNC_RSP = 5, + TAOS_SMSG_SYNC_MUST = 6, + TAOS_SMSG_STATUS = 7, + TAOS_SMSG_SYNC_DATA_RSP = 8, } ESyncMsgType; #define SYNC_MAX_SIZE (TSDB_MAX_WAL_SIZE + sizeof(SWalHead) + sizeof(SSyncHead) + 16) @@ -65,6 +66,10 @@ typedef struct { int32_t sourceId; // only for arbitrator } SFirstPkt; +typedef struct { + int8_t sync; +} SFirstPktRsp; + typedef struct { int8_t role; uint64_t version; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index d5fc79b635..68bc02cb16 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1313,6 +1313,8 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle } // always update version + sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, + syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); nodeVersion = pWalHead->version; if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER) return 0; @@ -1320,10 +1322,7 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle // only pkt from RPC or CQ can be forwarded if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0; - sTrace("vgId:%d, forward to peer, replica:%d role:%s qtype:%s hver:%" PRIu64, pNode->vgId, pNode->replica, - syncRole[nodeRole], qtypeStr[qtype], pWalHead->version); - - // a hacker way to improve the performance + // a hacker way to improve the performance pSyncHead = (SSyncHead *)(((char *)pWalHead) - sizeof(SSyncHead)); pSyncHead->type = TAOS_SMSG_FORWARD; pSyncHead->pversion = 0; @@ -1344,9 +1343,11 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { - sTrace("%s, forward is sent, hver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len); + sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, pPeer->role, + syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len); } else { - sError("%s, failed to forward, hver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen); + sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, pPeer->role, + syncStatus[pPeer->sstatus], pWalHead->version, retLen); syncRestartConnection(pPeer); } } diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index db3c6b318d..ed6b63c92d 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -231,10 +231,13 @@ static int32_t syncProcessBufferedFwd(SSyncPeer *pPeer) { int32_t syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead) { SSyncNode * pNode = pPeer->pSyncNode; SRecvBuffer *pRecv = pNode->pRecv; - - if (pRecv == NULL) return -1; int32_t len = pHead->len + sizeof(SWalHead); + if (pRecv == NULL) { + sError("%s, recv buffer is not create yet", pPeer->id); + return -1; + } + if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) { memcpy(pRecv->offset, pHead, len); pRecv->offset += len; @@ -282,7 +285,14 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeSStatus = TAOS_SYNC_STATUS_FILE; uint64_t fversion = 0; - sDebug("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, start to restore, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + SFirstPktRsp firstPktRsp = {.sync = 1}; + if (taosWriteMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { + sError("%s, failed to send sync firstPkt rsp since %s", pPeer->id, strerror(errno)); + return -1; + } + + sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); int32_t code = syncRestoreFile(pPeer, &fversion); if (code < 0) { sError("%s, failed to restore file", pPeer->id); @@ -299,14 +309,14 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) { nodeVersion = fversion; - sDebug("%s, start to restore wal", pPeer->id); + sInfo("%s, start to restore wal", pPeer->id); if (syncRestoreWal(pPeer) < 0) { sError("%s, failed to restore wal", pPeer->id); return -1; } nodeSStatus = TAOS_SYNC_STATUS_CACHE; - sDebug("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); + sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]); if (syncProcessBufferedFwd(pPeer) < 0) { sError("%s, failed to insert buffered points", pPeer->id); return -1; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 58d09d080e..348f91820b 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -448,7 +448,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { return code; } -static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { +static int32_t syncRetrieveFirstPkt(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFirstPkt firstPkt; @@ -458,8 +458,24 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); firstPkt.port = tsSyncPort; - if (taosWriteMsg(pPeer->syncFd, (char *)&firstPkt, sizeof(firstPkt)) < 0) { - sError("%s, failed to send syncCmd", pPeer->id); + if (taosWriteMsg(pPeer->syncFd, &firstPkt, sizeof(firstPkt)) < 0) { + sError("%s, failed to send sync firstPkt since %s", pPeer->id, strerror(errno)); + return -1; + } + + SFirstPktRsp firstPktRsp; + if (taosReadMsg(pPeer->syncFd, &firstPktRsp, sizeof(SFirstPktRsp)) < 0) { + sError("%s, failed to read sync firstPkt rsp since %s", pPeer->id, strerror(errno)); + return -1; + } + + return 0; +} + +static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { + sInfo("%s, start to retrieve, sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + if (syncRetrieveFirstPkt(pPeer) < 0) { + sError("%s, failed to start retrieve", pPeer->id); return -1; } diff --git a/src/sync/src/taosTcpPool.c b/src/sync/src/taosTcpPool.c index d1d9815f4a..eb05cf7c6f 100644 --- a/src/sync/src/taosTcpPool.c +++ b/src/sync/src/taosTcpPool.c @@ -203,16 +203,19 @@ static void *taosProcessTcpData(void *param) { assert(pConn); if (events[i].events & EPOLLERR) { + sDebug("conn is broken since EPOLLERR"); taosProcessBrokenLink(pConn); continue; } if (events[i].events & EPOLLHUP) { + sDebug("conn is broken since EPOLLHUP"); taosProcessBrokenLink(pConn); continue; } if (events[i].events & EPOLLRDHUP) { + sDebug("conn is broken since EPOLLRDHUP"); taosProcessBrokenLink(pConn); continue; } From d14e50ea5aa813642a6bc1f8886d5715909ce500 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Nov 2020 18:06:31 +0800 Subject: [PATCH 038/188] compile error --- src/sync/src/syncMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 68bc02cb16..bc3803b732 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1343,10 +1343,10 @@ static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle int32_t retLen = taosWriteMsg(pPeer->peerFd, pSyncHead, fwdLen); if (retLen == fwdLen) { - sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, pPeer->role, + sTrace("%s, forward is sent, role:%s sstatus:%s hver:%" PRIu64 " contLen:%d", pPeer->id, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pWalHead->version, pWalHead->len); } else { - sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, pPeer->role, + sError("%s, failed to forward, role:%s sstatus:%s hver:%" PRIu64 " retLen:%d", pPeer->id, syncRole[pPeer->role], syncStatus[pPeer->sstatus], pWalHead->version, retLen); syncRestartConnection(pPeer); } From d3352f551140bd73f56afcb5f956daa83ba0d1b5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 18:27:24 +0800 Subject: [PATCH 039/188] [TD-225] reduce memory consumption. --- src/client/src/tscSubquery.c | 3 +++ src/client/src/tscUtil.c | 7 ++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 6ebbeeef41..819a323db5 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2198,6 +2198,9 @@ int32_t tscHandleInsertRetry(SSqlObj* pSql) { STableDataBlocks* pTableDataBlock = taosArrayGetP(pCmd->pDataBlocks, pSupporter->index); int32_t code = tscCopyDataBlockToPayload(pSql, pTableDataBlock); + // free the data block created from insert sql string + pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); + if ((pRes->code = code)!= TSDB_CODE_SUCCESS) { tscQueueAsyncRes(pSql); return code; // here the pSql may have been released already. diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b35dbb47e6..703e266eb8 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -833,9 +833,14 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); + // free unnecessary memory resource ASAP. + char* p = realloc(dataBuf->pData, dataBuf->size); + if (p != NULL) { + dataBuf->pData = p; + } + // the length does not include the SSubmitBlk structure pBlocks->dataLen = htonl(finalLen); - dataBuf->numOfTables += 1; } From 19dedc226accda1655633aa1dd66d5ed47282445 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 24 Nov 2020 18:34:02 +0800 Subject: [PATCH 040/188] [TD-2132]: add memory leak test case --- .../jdbc/cases/AppMemoryLeakTest.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java new file mode 100644 index 0000000000..8a3dbb2bc1 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java @@ -0,0 +1,23 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +public class AppMemoryLeakTest { + + @Test + public void testAppMemoryLeak() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + + while (true) { + DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + } + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } +} From a988483a864920241f64e62a00af4a08d17baa09 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 24 Nov 2020 22:36:25 +0800 Subject: [PATCH 041/188] TD-1795 --- src/inc/taoserror.h | 2 +- src/mnode/src/mnodeSdb.c | 6 ++++++ src/sync/src/syncMain.c | 2 +- tests/script/tmp/mnodes.sim | 4 ++++ 4 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ff91989e5f..f1e736130c 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -259,6 +259,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, 0, 0x080B, "CPU cores TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "Invalid Sync Configuration") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync module not enabled") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sync version") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, 0, 0x0903, "Sync confirm expired") // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") @@ -365,7 +366,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string") - TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, 0, 0x2100, "out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, 0, 0x2101, "convertion not a valid literal input") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, 0, 0x2102, "convertion undefined") diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 36e1a1eb2b..536fdaaa11 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -261,6 +261,12 @@ static void sdbHandleFailedConfirm(SSdbRow *pRow) { SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = pRow->pTable, .pObj = pRow->pObj}; sdbDeleteRow(&row); } + + // Drop database/stable may take a long time and cause a timeout, so confirm is not enforced + if (action == SDB_ACTION_DELETE && pRow->code == TSDB_CODE_SYN_CONFIRM_EXPIRED) { + sdbDebug("vgId:1, confirm is not enforced while perform drop operation, set it success"); + pRow->code = TSDB_CODE_SUCCESS; + } } FORCE_INLINE diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index bc3803b732..d2d6d2d7fa 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1279,7 +1279,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) { sDebug("vgId:%d, forward info expired, hver:%" PRIu64 " curtime:%" PRIu64 " savetime:%" PRIu64, pNode->vgId, pFwdInfo->version, time, pFwdInfo->time); - syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL); + syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_SYN_CONFIRM_EXPIRED); } syncRemoveConfirmedFwdInfo(pNode); diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim index 48dbc19cb2..de02ae741b 100644 --- a/tests/script/tmp/mnodes.sim +++ b/tests/script/tmp/mnodes.sim @@ -20,6 +20,10 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20 + system sh/cfg.sh -n dnode1 -c replica -v 3 system sh/cfg.sh -n dnode2 -c replica -v 3 system sh/cfg.sh -n dnode3 -c replica -v 3 From 1386c05504ad369aaa60519c91b9829847454d55 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 25 Nov 2020 10:29:25 +0800 Subject: [PATCH 042/188] [TD-2140]: remove temporary code --- src/client/src/tscSystem.c | 4 ---- src/client/src/tscUtil.c | 16 ---------------- 2 files changed, 20 deletions(-) diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 672d87e0c8..91eabda78b 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -144,11 +144,7 @@ void taos_init_imp(void) { int64_t refreshTime = 10; // 10 seconds by default if (tscMetaCache == NULL) { tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); -#ifndef SQLOBJ_USE_CACHE tscObjRef = taosOpenRef(4096, tscFreeRegisteredSqlObj); -#else - tscObjCache = taosCacheInit(TSDB_CACHE_PTR_KEY, refreshTime / 2, false, tscFreeRegisteredSqlObj, "sqlObj"); -#endif } tscRefId = taosOpenRef(200, tscCloseTscObj); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 703e266eb8..950e904c4f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1565,22 +1565,6 @@ void tscGetSrcColumnInfo(SSrcColumnInfo* pColInfo, SQueryInfo* pQueryInfo) { } } -#ifndef SQLOBJ_USE_CACHE -#else -void tscSetFreeHeatBeat(STscObj* pObj) { - if (pObj == NULL || pObj->signature != pObj || pObj->pHb == NULL) { - return; - } - - SSqlObj* pHeatBeat = pObj->pHb; - assert(pHeatBeat == pHeatBeat->signature); - - // to denote the heart-beat timer close connection and free all allocated resources - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pHeatBeat->cmd, 0); - pQueryInfo->type = TSDB_QUERY_TYPE_FREE_RESOURCE; -} -#endif - /* * the following four kinds of SqlObj should not be freed * 1. SqlObj for stream computing From 97434ac094328ebf1aa40b953d43f3c026cd1558 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 25 Nov 2020 10:53:46 +0800 Subject: [PATCH 043/188] TD-1795 --- src/mnode/src/mnodeSdb.c | 70 ++++++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 536fdaaa11..e5fda26687 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -107,7 +107,7 @@ static taos_queue tsSdbWQueue; static SSdbWorkerPool tsSdbPool; static int32_t sdbProcessWrite(void *pRow, void *pHead, int32_t qtype, void *unused); -static int32_t sdbWriteWalToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam); +static int32_t sdbWriteFwdToQueue(int32_t vgId, void *pHead, int32_t qtype, void *rparam); static int32_t sdbWriteRowToQueue(SSdbRow *pRow, int32_t action); static void sdbFreeFromQueue(SSdbRow *pRow); static void * sdbWorkerFp(void *pWorker); @@ -261,12 +261,6 @@ static void sdbHandleFailedConfirm(SSdbRow *pRow) { SSdbRow row = {.type = SDB_OPER_GLOBAL, .pTable = pRow->pTable, .pObj = pRow->pObj}; sdbDeleteRow(&row); } - - // Drop database/stable may take a long time and cause a timeout, so confirm is not enforced - if (action == SDB_ACTION_DELETE && pRow->code == TSDB_CODE_SYN_CONFIRM_EXPIRED) { - sdbDebug("vgId:1, confirm is not enforced while perform drop operation, set it success"); - pRow->code = TSDB_CODE_SUCCESS; - } } FORCE_INLINE @@ -378,7 +372,7 @@ void sdbUpdateSync(void *pMnodes) { sprintf(syncInfo.path, "%s", tsMnodeDir); syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getFileInfo = sdbGetFileInfo; - syncInfo.writeToCache = sdbWriteWalToQueue; + syncInfo.writeToCache = sdbWriteFwdToQueue; syncInfo.confirmForward = sdbConfirmForward; syncInfo.notifyRole = sdbNotifyRole; tsSdbMgmt.cfg = syncCfg; @@ -565,7 +559,36 @@ static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow) { return TSDB_CODE_SUCCESS; } -static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { +static int32_t sdbPerformInsertAction(SWalHead *pHead, SSdbTable *pTable) { + SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; + (*pTable->fpDecode)(&row); + return sdbInsertHash(pTable, &row); +} + +static int32_t sdbPerformDeleteAction(SWalHead *pHead, SSdbTable *pTable) { + void *pObj = sdbGetRowMeta(pTable, pHead->cont); + if (pObj == NULL) { + sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name, + sdbGetKeyStr(pTable, pHead->cont)); + return TSDB_CODE_SUCCESS; + } + SSdbRow row = {.pTable = pTable, .pObj = pObj}; + return sdbDeleteHash(pTable, &row); +} + +static int32_t sdbPerformUpdateAction(SWalHead *pHead, SSdbTable *pTable) { + void *pObj = sdbGetRowMeta(pTable, pHead->cont); + if (pObj == NULL) { + sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name, + sdbGetKeyStr(pTable, pHead->cont)); + return TSDB_CODE_SUCCESS; + } + SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; + (*pTable->fpDecode)(&row); + return sdbUpdateHash(pTable, &row); +} + +static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unused) { SSdbRow *pRow = wparam; SWalHead *pHead = hparam; int32_t tableId = pHead->msgType / 10; @@ -574,6 +597,8 @@ static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unus SSdbTable *pTable = sdbGetTableFromId(tableId); assert(pTable != NULL); + if (qtype == TAOS_QTYPE_QUERY) return sdbPerformDeleteAction(pHead, pTable); + pthread_mutex_lock(&tsSdbMgmt.mutex); if (pHead->version == 0) { @@ -633,28 +658,17 @@ static int sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void *unus // from wal or forward msg, row not created, should add into hash if (action == SDB_ACTION_INSERT) { - SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; - code = (*pTable->fpDecode)(&row); - return sdbInsertHash(pTable, &row); + return sdbPerformInsertAction(pHead, pTable); } else if (action == SDB_ACTION_DELETE) { - void *pObj = sdbGetRowMeta(pTable, pHead->cont); - if (pObj == NULL) { - sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore delete action", pTable->name, - sdbGetKeyStr(pTable, pHead->cont)); + if (qtype == TAOS_QTYPE_FWD) { + // Drop database/stable may take a long time and cause a timeout, so we confirm first then reput it into queue + sdbWriteFwdToQueue(1, hparam, TAOS_QTYPE_QUERY, unused); return TSDB_CODE_SUCCESS; + } else { + return sdbPerformDeleteAction(pHead, pTable); } - SSdbRow row = {.pTable = pTable, .pObj = pObj}; - return sdbDeleteHash(pTable, &row); } else if (action == SDB_ACTION_UPDATE) { - void *pObj = sdbGetRowMeta(pTable, pHead->cont); - if (pObj == NULL) { - sdbDebug("vgId:1, sdb:%s, object:%s not exist in hash, ignore update action", pTable->name, - sdbGetKeyStr(pTable, pHead->cont)); - return TSDB_CODE_SUCCESS; - } - SSdbRow row = {.rowSize = pHead->len, .rowData = pHead->cont, .pTable = pTable}; - code = (*pTable->fpDecode)(&row); - return sdbUpdateHash(pTable, &row); + return sdbPerformUpdateAction(pHead, pTable); } else { return TSDB_CODE_MND_INVALID_MSG_TYPE; } @@ -972,7 +986,7 @@ static void sdbFreeFromQueue(SSdbRow *pRow) { taosFreeQitem(pRow); } -static int32_t sdbWriteWalToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { +static int32_t sdbWriteFwdToQueue(int32_t vgId, void *wparam, int32_t qtype, void *rparam) { SWalHead *pHead = wparam; int32_t size = sizeof(SSdbRow) + sizeof(SWalHead) + pHead->len; From 0e88426c7f864a678095ab761870d55a49cb7eb7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 11:19:15 +0800 Subject: [PATCH 044/188] [TD-2152] add test case for join query --- tests/pytest/insert/restfulInsert.py | 7 +++++-- tests/pytest/query/filterOtherTypes.py | 18 ++++++------------ tests/pytest/query/queryJoin.py | 2 ++ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index 9fa1f33a24..c39d1cbe48 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -55,9 +55,12 @@ class RestfulInsert: def insertUnlimitedData(self, threadID): print("thread %d started" % threadID) tablesPerThread = int (self.numOfTables / self.numOfThreads) + + count = 0 while True: i = 0 - start = self.ts + start = self.ts + count * self.batchSize + count = count + 1 for i in range(tablesPerThread): tableID = i + threadID * tablesPerThread @@ -65,7 +68,7 @@ class RestfulInsert: data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID) values = [] for k in range(self.batchSize): - values.append("(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) + values.append("(%d, %d, %d, %d)" % (start + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) if(self.outOfOrder == False): for k in range(len(values)): diff --git a/tests/pytest/query/filterOtherTypes.py b/tests/pytest/query/filterOtherTypes.py index 85d5a67bef..f80552138d 100644 --- a/tests/pytest/query/filterOtherTypes.py +++ b/tests/pytest/query/filterOtherTypes.py @@ -376,11 +376,9 @@ class TDTestCase: tdSql.execute("insert into t1 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") tdSql.execute("insert into t2 values(1538548685000, 4) (1538548685001, 5) (1538548685002, 6)") - tdSql.query("select * from t1 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t1 where tag1 like '%g'") - tdSql.query("select * from t2 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t2 where tag1 like '%g'") tdSql.query("select * from meters where tag1 like '%g'") tdSql.checkRows(6) @@ -396,20 +394,16 @@ class TDTestCase: tdSql.execute("insert into t5 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") tdSql.execute("insert into t6 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") - tdSql.query("select * from t3 where tag1 like '%京'") - tdSql.checkRows(3) + tdSql.error("select * from t3 where tag1 like '%京'") - tdSql.query("select * from t4 where tag1 like '%京'") - tdSql.checkRows(3) + tdSql.error("select * from t4 where tag1 like '%京'") tdSql.query("select * from meters1 where tag1 like '%京'") tdSql.checkRows(6) - tdSql.query("select * from t5 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t5 where tag1 like '%g'") - tdSql.query("select * from t6 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t6 where tag1 like '%g'") tdSql.query("select * from meters1 where tag1 like '%g'") tdSql.checkRows(6) diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index 57b7e8868d..59e01615b4 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -175,6 +175,8 @@ class TDTestCase: tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts") tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3") tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select join_mt1.c1,join_mt0.c1 from join_mt1,join_mt0 where join_mt1.ts = join_mt0.ts and join_mt1.t1 = join_mt0.t1 order by t") + def stop(self): tdSql.close() From 12bbecc52b950e8b684c437fcb0347a8ff027d7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 25 Nov 2020 12:02:23 +0800 Subject: [PATCH 045/188] [TD-2220]: fix the bug in column arithmetic expression. --- src/query/src/qAst.c | 107 ++++++++++++++++-- src/query/src/qExecutor.c | 1 + .../general/parser/col_arithmetic_query.sim | 63 +++++++++-- 3 files changed, 153 insertions(+), 18 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index e813688d84..4190e26158 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -370,6 +370,66 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S #endif } +static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) { + switch(type) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t* p = (int8_t*) dest; + int8_t* pSrc = (int8_t*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t* p = (int16_t*) dest; + int16_t* pSrc = (int16_t*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t* p = (int32_t*) dest; + int32_t* pSrc = (int32_t*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t* p = (int64_t*) dest; + int64_t* pSrc = (int64_t*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float* p = (float*) dest; + float* pSrc = (float*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double* p = (double*) dest; + double* pSrc = (double*) src; + + for(int32_t i = 0; i < numOfRows; ++i) { + p[i] = pSrc[numOfRows - i - 1]; + } + break; + } + default: assert(0); + } +} + void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*getSourceDataBlock)(void *, const char*, int32_t)) { if (pExprs == NULL) { @@ -387,6 +447,8 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, /* the right output has result from the right child syntax tree */ char *pRightOutput = malloc(sizeof(int64_t) * numOfRows); + char *pdata = malloc(sizeof(int64_t) * numOfRows); + if (pRight->nodeType == TSQL_NODE_EXPR) { tExprTreeCalcTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock); } @@ -398,52 +460,75 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, * the type of returned value of one expression is always double float precious */ _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); - fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, order); + fp(pLeftOutput, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pSchema->type, pExprs->_node.optr); + // set input buffer char *pInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - fp(pLeftOutput, pInputData, numOfRows, numOfRows, pOutput, order); + if (order == TSDB_ORDER_DESC) { + reverseCopy(pdata, pInputData, pRight->pSchema->type, numOfRows); + fp(pLeftOutput, pdata, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + } else { + fp(pLeftOutput, pInputData, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12 _bi_consumer_fn_t fp = tGetBiConsumerFn(TSDB_DATA_TYPE_DOUBLE, pRight->pVal->nType, pExprs->_node.optr); - fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, order); + fp(pLeftOutput, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); } } else if (pLeft->nodeType == TSQL_NODE_COL) { // column data specified on left-hand-side char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId); if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2 _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); - fp(pLeftInputData, pRightOutput, numOfRows, numOfRows, pOutput, order); + + if (order == TSDB_ORDER_DESC) { + reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); + fp(pdata, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + } else { + fp(pLeftInputData, pRightOutput, numOfRows, numOfRows, pOutput, TSDB_ORDER_ASC); + } } else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); - fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order); + // both columns are descending order, do not reverse the source data + fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, TSDB_ORDER_DESC); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12 _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); - fp(pLeftInputData, &pRight->pVal->i64Key, numOfRows, 1, pOutput, order); + + if (order == TSDB_ORDER_DESC) { + reverseCopy(pdata, pLeftInputData, pLeft->pSchema->type, numOfRows); + fp(pdata, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); + } else { + fp(pLeftInputData, &pRight->pVal->i64Key, numOfRows, 1, pOutput, TSDB_ORDER_ASC); + } } } else { // column data specified on left-hand-side if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2 _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, TSDB_DATA_TYPE_DOUBLE, pExprs->_node.optr); - fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, order); + fp(&pLeft->pVal->i64Key, pRightOutput, 1, numOfRows, pOutput, TSDB_ORDER_ASC); } else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight // column data specified on right-hand-side char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId); - _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pSchema->type, pExprs->_node.optr); - fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, order); + + if (order == TSDB_ORDER_DESC) { + reverseCopy(pdata, pRightInputData, pRight->pSchema->type, numOfRows); + fp(&pLeft->pVal->i64Key, pdata, numOfRows, 1, pOutput, TSDB_ORDER_ASC); + } else { + fp(&pLeft->pVal->i64Key, pRightInputData, 1, numOfRows, pOutput, TSDB_ORDER_ASC); + } } else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12 _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pVal->nType, pRight->pVal->nType, pExprs->_node.optr); - fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, order); + fp(&pLeft->pVal->i64Key, &pRight->pVal->i64Key, 1, 1, pOutput, TSDB_ORDER_ASC); } } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fce25ae119..b2fbde58da 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1090,6 +1090,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pExpr1[k].base.functionId; if (functionNeedToExecute(pRuntimeEnv, &pCtx[k], functionId)) { + pCtx[k].nStartQueryTimestamp = pDataBlockInfo->window.skey; aAggs[functionId].xFunction(&pCtx[k]); } } diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 5324e74dd2..0739338d5a 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -62,24 +62,73 @@ if $data91 != 1.000000000 then return -1 endi -sql select (c1 * 2) % 7.9 from $tb order by ts desc; +sql select (c1 * 2) % 7.9, c1*1, c1*1*1, c1*c1, c1*c1*c1 from $tb order by ts desc; if $rows != 10000 then return -1 endi -if $data00 != 0.100000000 then - print expect 0.100000000, acutal:$data00 +if $data00 != 2.200000000 then + print expect 2.200000000, actual:$data00 return -1 endi -if $data10 != 2.100000000 then +if $data01 != 9.000000000 then return -1 endi -if $data90 != 6.000000000 then +if $data02 != 9.000000000 then + return -1 +endi + +if $data03 != 81.000000000 then + return -1 +endi + +if $data04 != 729.000000000 then + return -1 +endi + + +if $data10 != 0.200000000 then + return -1 +endi + +if $data11 != 8.000000000 then + return -1 +endi + +if $data12 != 8.000000000 then + return -1 +endi + +if $data13 != 64.000000000 then + return -1 +endi + +if $data14 != 512.000000000 then + return -1 +endi + +if $data90 != 0.000000000 then return -1 endi +if $data11 != 0.000000000 then + return -1 +endi + +if $data12 != 0.000000000 then + return -1 +endi + +if $data13 != 0.000000000 then + return -1 +endi + +if $data14 != 0.000000000 then + return -1 +endi + # [d.3] sql select c1 * c2 /4 from $tb where ts < 1537166000000 and ts > 1537156000000 if $rows != 17 then @@ -586,7 +635,7 @@ if $data00 != @18-09-17 09:00:00.000@ then return -1 endi -sql select first(c7)- last(c1) from $tb interval(2y) +sql_error select first(c7)- last(c1) from $tb interval(2y) # aggregation query [d.18]=============================================================== # all cases in this part are aggregation query test. @@ -595,7 +644,7 @@ sql select first(c7)- last(c1) from $tb interval(2y) # multiple retrieve [d.20]=============================================================== -sql select c2-c2 from $tb; +sql select c2-c2 from $tb sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb From 9fba096e46973bf6887c219d04f7d763b4a78109 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 25 Nov 2020 12:13:11 +0800 Subject: [PATCH 046/188] [TD-2220]: fix the bug in column arithmetic expression. --- src/query/src/qAst.c | 2 +- tests/script/general/parser/col_arithmetic_query.sim | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index 4190e26158..a65f4a6dc9 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -497,7 +497,7 @@ void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pSchema->type, pExprs->_node.optr); // both columns are descending order, do not reverse the source data - fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, TSDB_ORDER_DESC); + fp(pLeftInputData, pRightInputData, numOfRows, numOfRows, pOutput, order); } else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12 _bi_consumer_fn_t fp = tGetBiConsumerFn(pLeft->pSchema->type, pRight->pVal->nType, pExprs->_node.optr); diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 0739338d5a..2e15ed1ffd 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -113,19 +113,19 @@ if $data90 != 0.000000000 then return -1 endi -if $data11 != 0.000000000 then +if $data91 != 0.000000000 then return -1 endi -if $data12 != 0.000000000 then +if $data92 != 0.000000000 then return -1 endi -if $data13 != 0.000000000 then +if $data93 != 0.000000000 then return -1 endi -if $data14 != 0.000000000 then +if $data94 != 0.000000000 then return -1 endi From 092c9de4aa994bd1d15f23e02591407a6bd25edc Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 25 Nov 2020 15:14:01 +0800 Subject: [PATCH 047/188] [TD-2218]add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/bug2218.py | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tests/pytest/query/bug2218.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 525fbad6c1..8977d96df4 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -158,6 +158,7 @@ python3 ./test.py -f query/bug1471.py python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py +python3 ./test.py -f query/bug2218.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/bug2218.py b/tests/pytest/query/bug2218.py new file mode 100644 index 0000000000..bb92e5d9ce --- /dev/null +++ b/tests/pytest/query/bug2218.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool)") + insertRows = 1000 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d)" % + (t0+i,i%100,i/2,i%100,i%100,i%100,i*1.0,i%2)) + print("==========step2") + print("test col*1*1 desc ") + tdSql.query('select c1,c1*1*1,c2*1*1,c3*1*1,c4*1*1,c5*1*1,c6*1*1 from mt0 order by ts desc limit 2') + tdSql.checkData(0,0,99) + tdSql.checkData(0,1,0.0) + tdSql.checkData(0,2,0.0) + tdSql.checkData(0,3,0.0) + tdSql.checkData(0,4,0.0) + tdSql.checkData(0,5,0.0) + tdSql.checkData(0,6,0.0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 2bcaac332a66d9efdfa15155247f2081491c7d34 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 25 Nov 2020 15:58:13 +0800 Subject: [PATCH 048/188] [TD-225] update sim. --- .../parser/col_arithmetic_operation.sim | 19 ++++++++++--------- .../general/parser/col_arithmetic_query.sim | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index 7adae8ef81..0cc02d088b 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -117,16 +117,17 @@ run general/parser/col_arithmetic_query.sim # ================================================================================================ print ====================> crash -# sql select spread(ts )/(1000*3600*24) from ca_stb0 interval(1y) +sql use $db +sql select spread(ts )/(1000*3600*24) from $stb interval(1y) -sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y) -sql_error select first(ts) - last(ts) from stb interval(1y) -sql_error select top(c1, 2) - last(c1) from stb; -sql_error select stddev(c1) - last(c1) from stb; -sql_error select diff(c1) - last(c1) from stb; -sql_error select first(c7) - last(c7) from stb; -sql_error select first(c8) - last(c8) from stb; -sql_error select first(c9) - last(c9) from stb; +sql_error select first(c1, c2) - last(c1, c2) from $stb interval(1y) +sql_error select first(ts) - last(ts) from $stb interval(1y) +sql_error select top(c1, 2) - last(c1) from $stb; +sql_error select stddev(c1) - last(c1) from $stb; +sql_error select diff(c1) - last(c1) from $stb; +sql_error select first(c7) - last(c7) from $stb; +sql_error select first(c8) - last(c8) from $stb; +sql_error select first(c9) - last(c9) from $stb; sql_error select max(c2*2) from $tb sql_error select max(c1-c2) from $tb diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 2e15ed1ffd..4ed8fd730a 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -144,7 +144,7 @@ if $data10 != 16.000000000 then endi if $data20 != 20.250000000 then - print expect 20.250000000, acutal:$data21 + print expect 20.250000000, actual:$data21 return -1 endi From 3e7c81eafbcd57880ea2f4ac799858b753a02391 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 16:03:43 +0800 Subject: [PATCH 049/188] [TD-2126] add test case for taosdump --- tests/pytest/tools/taosdumpTest.py | 89 ++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/pytest/tools/taosdumpTest.py diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py new file mode 100644 index 0000000000..534a477b34 --- /dev/null +++ b/tests/pytest/tools/taosdumpTest.py @@ -0,0 +1,89 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + self.numberOfTables = 10000 + self.numberOfRecords = 100 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") + tdSql.execute("create table t1 using st tags(1, 'beijing')") + sql = "insert into t1 values" + currts = self.ts + for i in range(100): + sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) + tdSql.execute(sql) + + tdSql.execute("create table t2 using st tags(2, 'shanghai')") + sql = "insert into t2 values" + currts = self.ts + for i in range(100): + sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) + tdSql.execute(sql) + + os.system("taosdump --databases db -o /tmp") + + tdSql.execute("drop database db") + tdSql.query("show databases") + tdSql.checkRows(0) + + os.system("taosdump -i /tmp") + + tdSql.query("show databases") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'db') + + tdSql.execute("use db") + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'st') + + tdSql.query("show tables") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 't2') + tdSql.checkData(1, 0, 't1') + + tdSql.query("select * from t1") + tdSql.checkRows(100) + for i in range(100): + tdSql.checkData(i, 1, i) + tdSql.checkData(i, 2, "nchar%d" % i) + + tdSql.query("select * from t2") + tdSql.checkRows(100) + for i in range(100): + tdSql.checkData(i, 1, i) + tdSql.checkData(i, 2, "nchar%d" % i) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 431fc93950be86183926fa2503529810d93ec7fd Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 16:15:22 +0800 Subject: [PATCH 050/188] add test cases into fulltest.sh --- tests/pytest/fulltest.sh | 4 +++- tests/pytest/tools/{lowa.py => lowaTest.py} | 0 tests/pytest/tools/{taosdemo.py => taosdemoTest.py} | 0 3 files changed, 3 insertions(+), 1 deletion(-) rename tests/pytest/tools/{lowa.py => lowaTest.py} (100%) rename tests/pytest/tools/{taosdemo.py => taosdemoTest.py} (100%) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 525fbad6c1..89b5b0f125 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -205,7 +205,9 @@ python3 test.py -f query/queryInterval.py python3 test.py -f query/queryFillTest.py # tools -python3 test.py -f tools/taosdemo.py +python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdumpTest.py +python3 test.py -f tools/lowaTest.py # subscribe python3 test.py -f subscribe/singlemeter.py diff --git a/tests/pytest/tools/lowa.py b/tests/pytest/tools/lowaTest.py similarity index 100% rename from tests/pytest/tools/lowa.py rename to tests/pytest/tools/lowaTest.py diff --git a/tests/pytest/tools/taosdemo.py b/tests/pytest/tools/taosdemoTest.py similarity index 100% rename from tests/pytest/tools/taosdemo.py rename to tests/pytest/tools/taosdemoTest.py From d1cd620696bbfe7f88efe04caf060d1688301d7b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 25 Nov 2020 16:29:02 +0800 Subject: [PATCH 051/188] TD-1455 TD-2196 --- src/dnode/src/dnodeVWrite.c | 4 ++-- src/inc/tsync.h | 6 +++--- src/inc/vnode.h | 6 ++++-- src/sync/src/syncRetrieve.c | 5 ++--- src/vnode/inc/vnodeInt.h | 2 +- src/vnode/src/vnodeMain.c | 10 ++++----- src/vnode/src/vnodeWrite.c | 42 ++++++++++++++++++++++++++++++++----- 7 files changed, 53 insertions(+), 22 deletions(-) diff --git a/src/dnode/src/dnodeVWrite.c b/src/dnode/src/dnodeVWrite.c index 959eb3c9c5..6d4b50ee54 100644 --- a/src/dnode/src/dnodeVWrite.c +++ b/src/dnode/src/dnodeVWrite.c @@ -176,7 +176,7 @@ void dnodeSendRpcVWriteRsp(void *pVnode, void *wparam, int32_t code) { if (count <= 1) return; SRpcMsg rpcRsp = { - .handle = pWrite->rpcHandle, + .handle = pWrite->rpcMsg.handle, .pCont = pWrite->rspRet.rsp, .contLen = pWrite->rspRet.len, .code = pWrite->code, @@ -206,7 +206,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) { for (int32_t i = 0; i < numOfMsgs; ++i) { taosGetQitem(pWorker->qall, &qtype, (void **)&pWrite); dTrace("msg:%p, app:%p type:%s will be processed in vwrite queue, qtype:%s hver:%" PRIu64, pWrite, - pWrite->rpcAhandle, taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); + pWrite->rpcMsg.ahandle, taosMsg[pWrite->pHead->msgType], qtypeStr[qtype], pWrite->pHead->version); pWrite->code = vnodeProcessWrite(pVnode, pWrite->pHead, qtype, &pWrite->rspRet); if (pWrite->code <= 0) pWrite->processedCount = 1; diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 77a3b36e73..398e1bf97c 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -23,7 +23,7 @@ extern "C" { #define TAOS_SYNC_MAX_REPLICA 5 #define TAOS_SYNC_MAX_INDEX 0x7FFFFFFF -typedef enum _TAOS_SYNC_ROLE { +typedef enum { TAOS_SYNC_ROLE_OFFLINE = 0, TAOS_SYNC_ROLE_UNSYNCED = 1, TAOS_SYNC_ROLE_SYNCING = 2, @@ -31,7 +31,7 @@ typedef enum _TAOS_SYNC_ROLE { TAOS_SYNC_ROLE_MASTER = 4 } ESyncRole; -typedef enum _TAOS_SYNC_STATUS { +typedef enum { TAOS_SYNC_STATUS_INIT = 0, TAOS_SYNC_STATUS_START = 1, TAOS_SYNC_STATUS_FILE = 2, @@ -80,7 +80,7 @@ typedef void (*FConfirmForward)(int32_t vgId, void *mhandle, int32_t code); typedef void (*FNotifyRole)(int32_t vgId, int8_t role); // if a number of retrieving data failed, call this to start flow control -typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t mseconds); +typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); // when data file is synced successfully, notity app typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); diff --git a/src/inc/vnode.h b/src/inc/vnode.h index 4e8389498b..5f643295d6 100644 --- a/src/inc/vnode.h +++ b/src/inc/vnode.h @@ -20,6 +20,7 @@ extern "C" { #endif +#include "trpc.h" #include "twal.h" typedef enum _VN_STATUS { @@ -51,8 +52,9 @@ typedef struct { typedef struct { int32_t code; int32_t processedCount; - void * rpcHandle; - void * rpcAhandle; + int32_t qtype; + void * pVnode; + SRpcMsg rpcMsg; SRspRet rspRet; char reserveForSync[16]; SWalHead pHead[]; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 348f91820b..060badba9d 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -504,6 +504,8 @@ void *syncRetrieveData(void *param) { SSyncNode *pNode = pPeer->pSyncNode; taosBlockSIGPIPE(); + if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves); + pPeer->fileChanged = 0; pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (pPeer->syncFd < 0) { @@ -520,10 +522,7 @@ void *syncRetrieveData(void *param) { } if (pPeer->fileChanged) { - // if file is changed 3 times continuously, start flow control pPeer->numOfRetrieves++; - if (pPeer->numOfRetrieves >= 2 && pNode->notifyFlowCtrl) - (*pNode->notifyFlowCtrl)(pNode->vgId, 4 << (pPeer->numOfRetrieves - 2)); } else { pPeer->numOfRetrieves = 0; if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0); diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 021831a644..cf9a1baa80 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -39,7 +39,7 @@ typedef struct { int32_t refCount; // reference count int32_t queuedWMsg; int32_t queuedRMsg; - int32_t delayMs; + int32_t flowctlLevel; int8_t status; int8_t role; int8_t accessState; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index b94fea52bd..1ee968e7cb 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -34,7 +34,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno); static uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId); static void vnodeNotifyRole(int32_t vgId, int8_t role); -static void vnodeCtrlFlow(int32_t vgId, int32_t mseconds); +static void vnodeCtrlFlow(int32_t vgId, int32_t level); static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); static void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); @@ -677,17 +677,15 @@ static void vnodeNotifyRole(int32_t vgId, int8_t role) { vnodeRelease(pVnode); } -static void vnodeCtrlFlow(int32_t vgId, int32_t mseconds) { +static void vnodeCtrlFlow(int32_t vgId, int32_t level) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while ctrl flow", vgId); return; } - if (pVnode->delayMs != mseconds) { - pVnode->delayMs = mseconds; - vDebug("vgId:%d, sync flow control, mseconds:%d", pVnode->vgId, mseconds); - } + pVnode->flowctlLevel = level; + vDebug("vgId:%d, set flowctl level:%d", pVnode->vgId, level); vnodeRelease(pVnode); } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index a0227d84ba..cada188591 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -22,14 +22,17 @@ #include "tsdb.h" #include "twal.h" #include "tsync.h" +#include "ttimer.h" #include "tdataformat.h" #include "vnode.h" #include "vnodeInt.h" #include "syncInt.h" #include "tcq.h" +#include "dnode.h" #define MAX_QUEUED_MSG_NUM 10000 +extern void * tsDnodeTmr; static int32_t (*vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_MAX])(SVnodeObj *, void *pCont, SRspRet *); static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); @@ -47,6 +50,32 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; } +static void vnodeFlowCtlMsgToWQueue(void *param, void *tmrId) { + SVWriteMsg *pWrite = param; + SVnodeObj * pVnode = pWrite->pVnode; + + int32_t code = vnodeWriteToWQueue(pVnode, pWrite->pHead, pWrite->qtype, &pWrite->rpcMsg); + if (code != 0 && pWrite->qtype == TAOS_QTYPE_RPC) { + vDebug("vgId:%d, failed to reprocess msg after perform flowctl since %s", pVnode->vgId, tstrerror(code)); + dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } + + tfree(pWrite); + vnodeRelease(pWrite->pVnode); +} + +static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { + SVnodeObj *pVnode = pWrite->pVnode; + if (pVnode->flowctlLevel <= 0) return 0; + + int32_t ms = pVnode->flowctlLevel * 5; + void * unUsed = NULL; + taosTmrReset(vnodeFlowCtlMsgToWQueue, ms, pWrite, tsDnodeTmr, &unUsed); + + vDebug("vgId:%d, perform flowctl for %d ms", pVnode->vgId, ms); + return TSDB_CODE_RPC_ACTION_IN_PROGRESS; +} + int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { int32_t code = 0; SVnodeObj *pVnode = vparam; @@ -77,8 +106,6 @@ int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rpara // assign version pHead->version = pVnode->version + 1; - if (pVnode->delayMs) taosMsleep(pVnode->delayMs); - } else { // from wal or forward // for data from WAL or forward, version may be smaller if (pHead->version <= pVnode->version) return 0; @@ -218,9 +245,10 @@ static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspR int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rparam) { SVnodeObj *pVnode = vparam; SWalHead * pHead = wparam; + int32_t code = 0; if (qtype == TAOS_QTYPE_RPC) { - int32_t code = vnodeCheckWrite(pVnode); + code = vnodeCheckWrite(pVnode); if (code != TSDB_CODE_SUCCESS) return code; } @@ -237,14 +265,18 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar if (rparam != NULL) { SRpcMsg *pRpcMsg = rparam; - pWrite->rpcHandle = pRpcMsg->handle; - pWrite->rpcAhandle = pRpcMsg->ahandle; + pWrite->rpcMsg = *pRpcMsg; } memcpy(pWrite->pHead, pHead, sizeof(SWalHead) + pHead->len); + pWrite->pVnode = pVnode; + pWrite->qtype = qtype; atomic_add_fetch_32(&pVnode->refCount, 1); + code = vnodePerformFlowCtrl(pWrite); + if (code != 0) return code; + int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); if (queued > MAX_QUEUED_MSG_NUM) { vDebug("vgId:%d, too many msg:%d in vwqueue, flow control", pVnode->vgId, queued); From 1b8256013c6dca66358e9f797d363bedbdddf369 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 25 Nov 2020 17:36:44 +0800 Subject: [PATCH 052/188] [TD-2227]: remove false error message in the log file while no results returned from server in case of query against super table. --- src/client/src/tscLocalMerge.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index ad8d4d72d0..2cd37013c5 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -198,6 +198,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd if (numOfFlush == 0 || numOfBuffer == 0) { tscLocalReducerEnvDestroy(pMemBuffer, pDesc, finalmodel, pFFModel, numOfBuffer); + pCmd->command = TSDB_SQL_RETRIEVE_EMPTY_RESULT; // no result, set the result empty tscDebug("%p retrieved no data", pSql); return; } From 55a455a8547c5cf4c3668370bca58f34dd9ad082 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 25 Nov 2020 17:38:20 +0800 Subject: [PATCH 053/188] [TD-2129]: fix bugs in twa query. --- src/client/src/tscFunctionImpl.c | 216 ++++++++++++++++--------------- src/query/inc/tsqlfunction.h | 14 +- 2 files changed, 113 insertions(+), 117 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index d39b833374..2ed476f063 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3624,7 +3624,7 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->lastKey = INT64_MIN; @@ -3633,43 +3633,116 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { return true; } -static FORCE_INLINE void setTWALastVal(SQLFunctionCtx *pCtx, const char *data, int32_t i, STwaInfo *pInfo) { - switch (pCtx->inputType) { - case TSDB_DATA_TYPE_INT: - pInfo->iLastValue = GET_INT32_VAL(data + pCtx->inputBytes * i); - break; - case TSDB_DATA_TYPE_TINYINT: - pInfo->iLastValue = GET_INT8_VAL(data + pCtx->inputBytes * i); - break; - case TSDB_DATA_TYPE_SMALLINT: - pInfo->iLastValue = GET_INT16_VAL(data + pCtx->inputBytes * i); - break; - case TSDB_DATA_TYPE_BIGINT: - pInfo->iLastValue = GET_INT64_VAL(data + pCtx->inputBytes * i); - break; - case TSDB_DATA_TYPE_FLOAT: - pInfo->dLastValue = GET_FLOAT_VAL(data + pCtx->inputBytes * i); - break; - case TSDB_DATA_TYPE_DOUBLE: - pInfo->dLastValue = GET_DOUBLE_VAL(data + pCtx->inputBytes * i); - break; - default: - assert(0); +static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { + int32_t notNullElems = 0; + TSKEY *primaryKey = pCtx->ptsList; + + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + if (pInfo->lastKey == INT64_MIN) { + pInfo->lastKey = pCtx->nStartQueryTimestamp; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); + pInfo->hasResult = DATA_SET_FLAG; + notNullElems++; } + + int32_t i = index; + + // calculate the value of + switch(pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (++i; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i]; + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (++i; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i]; + } + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (++i; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i]; + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (++i; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i]; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (++i; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i]; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (++i; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i]; + } + break; + } + default: assert(0); + } + + return notNullElems; } static void twa_function(SQLFunctionCtx *pCtx) { void * data = GET_INPUT_CHAR(pCtx); - TSKEY *primaryKey = pCtx->ptsList; - - int32_t notNullElems = 0; - + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); - int32_t i = 0; - // skip null value + int32_t i = 0; while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { i++; } @@ -3678,40 +3751,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { return; } - if (pInfo->lastKey == INT64_MIN) { - pInfo->lastKey = pCtx->nStartQueryTimestamp; - setTWALastVal(pCtx, data, i, pInfo); - - pInfo->hasResult = DATA_SET_FLAG; - } - - notNullElems++; - - if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey); - } else { - pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey); - } - - pInfo->lastKey = primaryKey[i]; - setTWALastVal(pCtx, data, i, pInfo); - - for (++i; i < pCtx->size; i++) { - if (pCtx->hasNull && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - continue; - } - - notNullElems++; - if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - pInfo->dOutput += pInfo->dLastValue * (primaryKey[i] - pInfo->lastKey); - } else { - pInfo->iOutput += pInfo->iLastValue * (primaryKey[i] - pInfo->lastKey); - } - - pInfo->lastKey = primaryKey[i]; - setTWALastVal(pCtx, data, i, pInfo); - } - + int32_t notNullElems = twa_function_impl(pCtx, 0, pCtx->size); SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3721,8 +3761,6 @@ static void twa_function(SQLFunctionCtx *pCtx) { if (pCtx->stableQuery) { memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); } - - // pCtx->numOfIteratedElems += notNullElems; } static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { @@ -3730,35 +3768,12 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - - SET_VAL(pCtx, 1, 1); - - TSKEY *primaryKey = pCtx->ptsList; - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - if (pInfo->lastKey == INT64_MIN) { - pInfo->lastKey = pCtx->nStartQueryTimestamp; - setTWALastVal(pCtx, pData, 0, pInfo); - - pInfo->hasResult = DATA_SET_FLAG; - } - - if (pCtx->inputType == TSDB_DATA_TYPE_FLOAT || pCtx->inputType == TSDB_DATA_TYPE_DOUBLE) { - pInfo->dOutput += pInfo->dLastValue * (primaryKey[index] - pInfo->lastKey); - } else { - pInfo->iOutput += pInfo->iLastValue * (primaryKey[index] - pInfo->lastKey); - } - - // record the last key/value - pInfo->lastKey = primaryKey[index]; - setTWALastVal(pCtx, pData, 0, pInfo); - - // pCtx->numOfIteratedElems += 1; - pResInfo->hasResult = DATA_SET_FLAG; - + + int32_t notNullElems = twa_function_impl(pCtx, index, 1); + SET_VAL(pCtx, notNullElems, 1); + if (pCtx->stableQuery) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); } } @@ -3778,16 +3793,11 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { } numOfNotNull++; - if (pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_BIGINT) { - pBuf->iOutput += pInput->iOutput; - } else { - pBuf->dOutput += pInput->dOutput; - } - + pBuf->dOutput += pInput->dOutput; + pBuf->SKey = pInput->SKey; pBuf->EKey = pInput->EKey; pBuf->lastKey = pInput->lastKey; - pBuf->iLastValue = pInput->iLastValue; } SET_VAL(pCtx, numOfNotNull, 1); @@ -3822,12 +3832,8 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { } if (pInfo->SKey == pInfo->EKey) { - *(double *)pCtx->aOutputBuf = 0; - } else if (pInfo->type >= TSDB_DATA_TYPE_TINYINT && pInfo->type <= TSDB_DATA_TYPE_BIGINT) { - pInfo->iOutput += pInfo->iLastValue * (pInfo->EKey - pInfo->lastKey); - *(double *)pCtx->aOutputBuf = pInfo->iOutput / (double)(pInfo->EKey - pInfo->SKey); + *(double *)pCtx->aOutputBuf = pInfo->lastValue; } else { - pInfo->dOutput += pInfo->dLastValue * (pInfo->EKey - pInfo->lastKey); *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->EKey - pInfo->SKey); } diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 32cbb56c62..43f78a56ef 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -248,16 +248,8 @@ typedef struct STwaInfo { int16_t type; // source data type TSKEY SKey; TSKEY EKey; - - union { - double dOutput; - int64_t iOutput; - }; - - union { - double dLastValue; - int64_t iLastValue; - }; + double dOutput; + double lastValue; } STwaInfo; /* global sql function array */ @@ -276,8 +268,6 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha (_r)->initialized = false; \ } while (0) -//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf); - static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag From e9e3a6d8653bb4cc24ae47ecb678b2c699e2a173 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 25 Nov 2020 19:19:15 +0800 Subject: [PATCH 054/188] [TD-225]refactor codes. --- src/query/src/qExecutor.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index b2fbde58da..949616bb43 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -457,6 +457,7 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin return NULL; } + // TODO refactor // more than the capacity, reallocate the resources if (pWindowResInfo->size >= pWindowResInfo->capacity) { int64_t newCapacity = 0; @@ -1000,9 +1001,8 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas * @return the incremental number of output value, so it maybe 0 for fixed number of query, * such as count/min/max etc. */ -static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, - SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, - __block_search_fn_t searchFn, SArray *pDataBlock) { +static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, + SWindowResInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); @@ -1045,6 +1045,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * int32_t forwardStep = 0; int32_t startPos = pQuery->pos; + // in case of repeat scan/reverse scan, no new time window added. if (hasTimeWindow) { TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); From ffbbb0cc44768e97698355bca586016598964b27 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 25 Nov 2020 20:36:44 +0800 Subject: [PATCH 055/188] TD-1455 TD-2196 --- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 12 ++++++++ src/vnode/src/vnodeMain.c | 6 ++-- src/vnode/src/vnodeWrite.c | 59 +++++++++++++++++++++----------------- 4 files changed, 49 insertions(+), 29 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 4087f638a9..efe3d7678a 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -97,6 +97,7 @@ extern int32_t tsAlternativeRole; extern int32_t tsBalanceInterval; extern int32_t tsOfflineThreshold; extern int32_t tsMnodeEqualVnodeNum; +extern int32_t tsFlowCtrl; // restful extern int32_t tsEnableHttpModule; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 4495c3d928..f8bb965d28 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -133,6 +133,7 @@ int32_t tsAlternativeRole = 0; int32_t tsBalanceInterval = 300; // seconds int32_t tsOfflineThreshold = 86400*100; // seconds 10days int32_t tsMnodeEqualVnodeNum = 4; +int32_t tsFlowCtrl = 1; // restful int32_t tsEnableHttpModule = 1; @@ -971,6 +972,17 @@ static void doInitGlobalConfig(void) { cfg.maxValue = 1000; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + + // module configs + cfg.option = "flowctrl"; + cfg.ptr = &tsFlowCtrl; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); cfg.option = "http"; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 1ee968e7cb..df4376d5d0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -660,7 +660,7 @@ static int32_t vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) { static void vnodeNotifyRole(int32_t vgId, int8_t role) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { - vError("vgId:%d, vnode not found while notify role", vgId); + vTrace("vgId:%d, vnode not found while notify role", vgId); return; } @@ -680,12 +680,12 @@ static void vnodeNotifyRole(int32_t vgId, int8_t role) { static void vnodeCtrlFlow(int32_t vgId, int32_t level) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { - vError("vgId:%d, vnode not found while ctrl flow", vgId); + vTrace("vgId:%d, vnode not found while flow ctrl", vgId); return; } pVnode->flowctlLevel = level; - vDebug("vgId:%d, set flowctl level:%d", pVnode->vgId, level); + vDebug("vgId:%d, set flowctrl level:%d", pVnode->vgId, level); vnodeRelease(pVnode); } diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index cada188591..4bf7b30ebd 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -17,6 +17,7 @@ #include "os.h" #include "taosmsg.h" #include "taoserror.h" +#include "tglobal.h" #include "tqueue.h" #include "trpc.h" #include "tsdb.h" @@ -40,6 +41,7 @@ static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *); +static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite); void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_SUBMIT] = vnodeProcessSubmitMsg; @@ -50,32 +52,6 @@ void vnodeInitWriteFp(void) { vnodeProcessWriteMsgFp[TSDB_MSG_TYPE_UPDATE_TAG_VAL] = vnodeProcessUpdateTagValMsg; } -static void vnodeFlowCtlMsgToWQueue(void *param, void *tmrId) { - SVWriteMsg *pWrite = param; - SVnodeObj * pVnode = pWrite->pVnode; - - int32_t code = vnodeWriteToWQueue(pVnode, pWrite->pHead, pWrite->qtype, &pWrite->rpcMsg); - if (code != 0 && pWrite->qtype == TAOS_QTYPE_RPC) { - vDebug("vgId:%d, failed to reprocess msg after perform flowctl since %s", pVnode->vgId, tstrerror(code)); - dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); - } - - tfree(pWrite); - vnodeRelease(pWrite->pVnode); -} - -static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { - SVnodeObj *pVnode = pWrite->pVnode; - if (pVnode->flowctlLevel <= 0) return 0; - - int32_t ms = pVnode->flowctlLevel * 5; - void * unUsed = NULL; - taosTmrReset(vnodeFlowCtlMsgToWQueue, ms, pWrite, tsDnodeTmr, &unUsed); - - vDebug("vgId:%d, perform flowctl for %d ms", pVnode->vgId, ms); - return TSDB_CODE_RPC_ACTION_IN_PROGRESS; -} - int32_t vnodeProcessWrite(void *vparam, void *wparam, int32_t qtype, void *rparam) { int32_t code = 0; SVnodeObj *pVnode = vparam; @@ -298,3 +274,34 @@ void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { taosFreeQitem(pWrite); vnodeRelease(pVnode); } + +static void vnodeFlowCtlMsgToWQueue(void *param, void *tmrId) { + SVWriteMsg *pWrite = param; + SVnodeObj * pVnode = pWrite->pVnode; + + int32_t code = vnodeWriteToWQueue(pVnode, pWrite->pHead, pWrite->qtype, pWrite->rpcMsg.handle == NULL ? NULL : &pWrite->rpcMsg); + if (code != 0 && pWrite->qtype == TAOS_QTYPE_RPC) { + vDebug("vgId:%d, failed to reprocess msg after perform flowctrl since %s", pVnode->vgId, tstrerror(code)); + dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } + + tfree(pWrite); + vnodeRelease(pWrite->pVnode); +} + +static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { + SVnodeObj *pVnode = pWrite->pVnode; + if (pVnode->flowctlLevel <= 0) return 0; + + if (tsFlowCtrl == 0) { + int32_t ms = pVnode->flowctlLevel * 2; + if (ms > 60000) ms = 60000; + vDebug("vgId:%d, perform flowctrl for %d ms", pVnode->vgId, ms); + taosMsleep(ms); + return 0; + } else { + void *unUsed = NULL; + taosTmrReset(vnodeFlowCtlMsgToWQueue, 5, pWrite, tsDnodeTmr, &unUsed); + return TSDB_CODE_RPC_ACTION_IN_PROGRESS; + } +} From aeed9db84c8b752eabc0577477f78733e33519a8 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 26 Nov 2020 09:27:17 +0800 Subject: [PATCH 056/188] [TD-2230] use error max len when dump in --- src/kit/taosdump/taosdump.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 88f07ee602..a7258c9724 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -2146,7 +2146,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c char * line = NULL; size_t line_len = 0; - cmd = (char *)malloc(COMMAND_SIZE); + cmd = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN); if (cmd == NULL) { fprintf(stderr, "failed to allocate memory\n"); return -1; @@ -2155,7 +2155,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c int lineNo = 0; while ((read_len = getline(&line, &line_len, fp)) != -1) { ++lineNo; - if (read_len >= COMMAND_SIZE) continue; + if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; //if (read_len == 0 || isCommentLine(line)) { // line starts with # @@ -2176,7 +2176,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); } - memset(cmd, 0, COMMAND_SIZE); + memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; } From 060299548e69b2eed1eb611429e96f5a922f8d16 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 26 Nov 2020 11:09:46 +0800 Subject: [PATCH 057/188] change log print --- src/dnode/src/dnodeMPeer.c | 2 +- src/dnode/src/dnodeMRead.c | 2 +- src/dnode/src/dnodeMWrite.c | 2 +- src/dnode/src/dnodeVRead.c | 2 +- src/os/inc/osAlpine.h | 1 + src/os/inc/osArm32.h | 1 + src/os/inc/osArm64.h | 1 + src/os/inc/osDarwin.h | 1 + src/os/inc/osLinux32.h | 1 + src/os/inc/osLinux64.h | 1 + src/os/inc/osWindows.h | 1 + src/vnode/inc/vnodeInt.h | 2 +- src/vnode/src/vnodeMain.c | 2 +- 13 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index 05b37bd338..ee6dc5212e 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -162,7 +162,7 @@ static void *dnodeProcessMPeerQueue(void *param) { break; } - dDebug("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]); + dTrace("msg:%s will be processed in mpeer queue", taosMsg[pPeerMsg->rpcMsg.msgType]); int32_t code = mnodeProcessPeerReq(pPeerMsg); dnodeSendRpcMPeerRsp(pPeerMsg, code); } diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index ee9da72e4d..65f3af7b3b 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -168,7 +168,7 @@ static void *dnodeProcessMReadQueue(void *param) { break; } - dDebug("msg:%p, app:%p type:%s will be processed in mread queue", pRead->rpcMsg.ahandle, pRead, + dTrace("msg:%p, app:%p type:%s will be processed in mread queue", pRead->rpcMsg.ahandle, pRead, taosMsg[pRead->rpcMsg.msgType]); int32_t code = mnodeProcessRead(pRead); dnodeSendRpcMReadRsp(pRead, code); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 65c0d53819..ef2d49ef42 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -174,7 +174,7 @@ static void *dnodeProcessMWriteQueue(void *param) { break; } - dDebug("msg:%p, app:%p type:%s will be processed in mwrite queue", pWrite, pWrite->rpcMsg.ahandle, + dTrace("msg:%p, app:%p type:%s will be processed in mwrite queue", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType]); int32_t code = mnodeProcessWrite(pWrite); diff --git a/src/dnode/src/dnodeVRead.c b/src/dnode/src/dnodeVRead.c index 4cce54bf59..b42a627a3a 100644 --- a/src/dnode/src/dnodeVRead.c +++ b/src/dnode/src/dnodeVRead.c @@ -175,7 +175,7 @@ static void *dnodeProcessReadQueue(void *pWorker) { break; } - dDebug("msg:%p, app:%p type:%s will be processed in vread queue, qtype:%d", pRead, pRead->rpcAhandle, + dTrace("msg:%p, app:%p type:%s will be processed in vread queue, qtype:%d", pRead, pRead->rpcAhandle, taosMsg[pRead->msgType], qtype); int32_t code = vnodeProcessRead(pVnode, pRead); diff --git a/src/os/inc/osAlpine.h b/src/os/inc/osAlpine.h index d939adfb6d..eba9459395 100644 --- a/src/os/inc/osAlpine.h +++ b/src/os/inc/osAlpine.h @@ -77,6 +77,7 @@ extern "C" { #include #include #include +#include typedef int(*__compar_fn_t)(const void *, const void *); void error (int, int, const char *); diff --git a/src/os/inc/osArm32.h b/src/os/inc/osArm32.h index 17b4d2dbd5..24ff95522e 100644 --- a/src/os/inc/osArm32.h +++ b/src/os/inc/osArm32.h @@ -76,6 +76,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzll(val) diff --git a/src/os/inc/osArm64.h b/src/os/inc/osArm64.h index 3ae08b45f4..22f0000e96 100644 --- a/src/os/inc/osArm64.h +++ b/src/os/inc/osArm64.h @@ -77,6 +77,7 @@ extern "C" { #include #include #include +#include #ifdef __cplusplus } diff --git a/src/os/inc/osDarwin.h b/src/os/inc/osDarwin.h index 7bb844831e..1461ec6d3b 100644 --- a/src/os/inc/osDarwin.h +++ b/src/os/inc/osDarwin.h @@ -70,6 +70,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_FILE_SENDIFLE diff --git a/src/os/inc/osLinux32.h b/src/os/inc/osLinux32.h index 93e917e797..cfef05368f 100644 --- a/src/os/inc/osLinux32.h +++ b/src/os/inc/osLinux32.h @@ -76,6 +76,7 @@ extern "C" { #include #include #include +#include #define TAOS_OS_FUNC_LZ4 #define BUILDIN_CLZL(val) __builtin_clzll(val) diff --git a/src/os/inc/osLinux64.h b/src/os/inc/osLinux64.h index c0841c41bd..a2febd51b7 100644 --- a/src/os/inc/osLinux64.h +++ b/src/os/inc/osLinux64.h @@ -79,6 +79,7 @@ extern "C" { #include #endif #include +#include #ifdef __cplusplus } diff --git a/src/os/inc/osWindows.h b/src/os/inc/osWindows.h index 5003e48c44..5a1e642572 100644 --- a/src/os/inc/osWindows.h +++ b/src/os/inc/osWindows.h @@ -40,6 +40,7 @@ #include #include #include +#include #include "msvcProcess.h" #include "msvcDirect.h" #include "msvcFcntl.h" diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index cf9a1baa80..7fc9b100ef 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -39,7 +39,7 @@ typedef struct { int32_t refCount; // reference count int32_t queuedWMsg; int32_t queuedRMsg; - int32_t flowctlLevel; + int32_t flowctrlLevel; int8_t status; int8_t role; int8_t accessState; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index df4376d5d0..cd6d2ea7c0 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -684,7 +684,7 @@ static void vnodeCtrlFlow(int32_t vgId, int32_t level) { return; } - pVnode->flowctlLevel = level; + pVnode->flowctrlLevel = level; vDebug("vgId:%d, set flowctrl level:%d", pVnode->vgId, level); vnodeRelease(pVnode); From 4f2d7a1a83c00d900fda99ee174e8008f89bfc02 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 04:07:38 +0000 Subject: [PATCH 058/188] [TD-2216]: crash in qUtil.c --- src/query/inc/qUtil.h | 2 +- src/query/src/qUtil.c | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index bc23ffadd4..8b84ac0182 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -27,7 +27,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery); void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type); -void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); +void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 2b4cdef9b0..3f56366db8 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -133,7 +133,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]); + copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type); } // move the unclosed window in the front of the window list @@ -272,9 +272,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16 * since the attribute of "Pos" is bound to each window result when the window result is created in the * disk-based result buffer. */ -void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) { +void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src, int16_t type) { dst->numOfRows = src->numOfRows; - dst->win = src->win; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + dst->key = realloc(dst->key, varDataTLen(src->key)); + varDataCopy(dst->key, src->key); + } else { + dst->win = src->win; + } dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; From 810023ca6894c2c220c20344246c16a7558a8c1d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 26 Nov 2020 13:13:21 +0800 Subject: [PATCH 059/188] TD-1455 TD-2196 --- src/inc/taoserror.h | 1 + src/vnode/src/vnodeWrite.c | 46 ++++---- tests/script/unique/cluster/flowctrl.sim | 131 +++++++++++++++++++++++ 3 files changed, 160 insertions(+), 18 deletions(-) create mode 100644 tests/script/unique/cluster/flowctrl.sim diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index f1e736130c..77ec5350ba 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -205,6 +205,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VRESION_FILE, 0, 0x050A, "Invalid ve TAOS_DEFINE_ERROR(TSDB_CODE_VND_IS_FULL, 0, 0x050B, "Vnode memory is full because commit failed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_SYNCED, 0, 0x0511, "Database suspended") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, 0, 0x0512, "Write operation denied") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_SYNCING, 0, 0x0513, "Database is syncing") // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "Invalid table ID") diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 4bf7b30ebd..57bd407cd1 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -250,15 +250,15 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar atomic_add_fetch_32(&pVnode->refCount, 1); - code = vnodePerformFlowCtrl(pWrite); - if (code != 0) return code; - int32_t queued = atomic_add_fetch_32(&pVnode->queuedWMsg, 1); if (queued > MAX_QUEUED_MSG_NUM) { vDebug("vgId:%d, too many msg:%d in vwqueue, flow control", pVnode->vgId, queued); taosMsleep(1); } + code = vnodePerformFlowCtrl(pWrite); + if (code != 0) return 0; + vTrace("vgId:%d, write into vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg); taosWriteQitem(pVnode->wqueue, qtype, pWrite); @@ -268,40 +268,50 @@ int32_t vnodeWriteToWQueue(void *vparam, void *wparam, int32_t qtype, void *rpar void vnodeFreeFromWQueue(void *vparam, SVWriteMsg *pWrite) { SVnodeObj *pVnode = vparam; - atomic_sub_fetch_32(&pVnode->queuedWMsg, 1); - vTrace("vgId:%d, free from vwqueue, refCount:%d queued:%d", pVnode->vgId, pVnode->refCount, pVnode->queuedWMsg); + int32_t queued = atomic_sub_fetch_32(&pVnode->queuedWMsg, 1); + vTrace("vgId:%d, msg:%p, app:%p, free from vwqueue, queued:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, queued); taosFreeQitem(pWrite); vnodeRelease(pVnode); } -static void vnodeFlowCtlMsgToWQueue(void *param, void *tmrId) { +static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { SVWriteMsg *pWrite = param; SVnodeObj * pVnode = pWrite->pVnode; + int32_t code = TSDB_CODE_VND_SYNCING; - int32_t code = vnodeWriteToWQueue(pVnode, pWrite->pHead, pWrite->qtype, pWrite->rpcMsg.handle == NULL ? NULL : &pWrite->rpcMsg); - if (code != 0 && pWrite->qtype == TAOS_QTYPE_RPC) { - vDebug("vgId:%d, failed to reprocess msg after perform flowctrl since %s", pVnode->vgId, tstrerror(code)); + pWrite->processedCount++; + if (pWrite->processedCount > 100) { + vError("vgId:%d, msg:%p, failed to process since %s", pVnode->vgId, pWrite, tstrerror(code)); + pWrite->processedCount = 1; dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); + } else { + code = vnodePerformFlowCtrl(pWrite); + if (code == 0) { + vTrace("vgId:%d, write into vwqueue after flowctrl", pVnode->vgId); + pWrite->processedCount = 0; + taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); + } } - - tfree(pWrite); - vnodeRelease(pWrite->pVnode); } static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { SVnodeObj *pVnode = pWrite->pVnode; - if (pVnode->flowctlLevel <= 0) return 0; + if (pVnode->flowctrlLevel <= 0) return 0; + if (pWrite->qtype != TAOS_QTYPE_RPC) return 0; if (tsFlowCtrl == 0) { - int32_t ms = pVnode->flowctlLevel * 2; - if (ms > 60000) ms = 60000; - vDebug("vgId:%d, perform flowctrl for %d ms", pVnode->vgId, ms); + int32_t ms = pow(2, pVnode->flowctrlLevel + 2); + if (ms > 100) ms = 100; + vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms); taosMsleep(ms); return 0; } else { void *unUsed = NULL; - taosTmrReset(vnodeFlowCtlMsgToWQueue, 5, pWrite, tsDnodeTmr, &unUsed); - return TSDB_CODE_RPC_ACTION_IN_PROGRESS; + taosTmrReset(vnodeFlowCtrlMsgToWQueue, 100, pWrite, tsDnodeTmr, &unUsed); + + vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, count:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, + pWrite->processedCount); + return TSDB_CODE_VND_ACTION_IN_PROGRESS; } } diff --git a/tests/script/unique/cluster/flowctrl.sim b/tests/script/unique/cluster/flowctrl.sim new file mode 100644 index 0000000000..6dc60d9fba --- /dev/null +++ b/tests/script/unique/cluster/flowctrl.sim @@ -0,0 +1,131 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 + +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3 +system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3 + +system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4 +system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4 + +system sh/cfg.sh -n dnode1 -c http -v 0 +system sh/cfg.sh -n dnode2 -c http -v 0 +system sh/cfg.sh -n dnode3 -c http -v 0 + +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 + +system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20 +system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20 + +system sh/cfg.sh -n dnode1 -c replica -v 3 +system sh/cfg.sh -n dnode2 -c replica -v 3 +system sh/cfg.sh -n dnode3 -c replica -v 3 + +print ============== deploy + +system sh/exec.sh -n dnode1 -s start +sleep 5001 +sql connect + +sql create dnode $hostname2 +sql create dnode $hostname3 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +print =============== step1 +$x = 0 +show1: + $x = $x + 1 + sleep 2000 + if $x == 5 then + return -1 + endi +sql show mnodes -x show1 +$mnode1Role = $data2_1 +print mnode1Role $mnode1Role +$mnode2Role = $data2_2 +print mnode2Role $mnode2Role +$mnode3Role = $data2_3 +print mnode3Role $mnode3Role + +if $mnode1Role != master then + goto show1 +endi +if $mnode2Role != slave then + goto show1 +endi +if $mnode3Role != slave then + goto show1 +endi + +print =============== step2 + +sql create database db replica 3 +sql use db +sql create table tb (ts timestamp, test int) + +$x = 0 +while $x < 100 + $ms = $x . s + sql insert into tb values (now + $ms , $x ) + $x = $x + 1 +endw + +print =============== step3 +sleep 3000 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + +print =============== step4 +sleep 5000 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +print =============== step5 +sleep 8000 +while $x < 200 + $ms = $x . s + sql insert into tb values (now + $ms , $x ) + $x = $x + 1 +endw + +print =============== step6 +system sh/exec.sh -n dnode2 -s stop -x SIGINT +sleep 3000 +while $x < 300 + $ms = $x . s + sql insert into tb values (now + $ms , $x ) + $x = $x + 1 +endw + +system sh/exec.sh -n dnode2 -s start + +sleep 6000 +print =============== step7 +while $x < 400 + $ms = $x . s + sql insert into tb values (now + $ms , $x ) + $x = $x + 1 + sleep 1 +endw + +print =============== step8 +sql select * from tb +print rows $rows +if $rows != 400 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT \ No newline at end of file From e9c83f8251545936fe424a9004ddb4c6003c8612 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 26 Nov 2020 14:33:53 +0800 Subject: [PATCH 060/188] [TD-2132]: add memory leak test case --- .../jdbc/cases/AppMemoryLeakTest.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java index 8a3dbb2bc1..8de2e3b442 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java @@ -5,19 +5,40 @@ import org.junit.Test; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; public class AppMemoryLeakTest { - @Test - public void testAppMemoryLeak() { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - - while (true) { - DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); - } - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); + @Test(expected = SQLException.class) + public void testCreateTooManyConnection() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int conCnt = 0; + while (true) { + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + System.out.println(conCnt++ + " : " + conn); } } + + @Test + public void testCreateTooManyStatement() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int stmtCnt = 0; + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + while (true) { + Statement stmt = conn.createStatement(); + System.out.println(++stmtCnt + " : " + stmt); + } + } + + public static void main(String[] args) throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int stmtCnt = 0; + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + while (true) { + Statement stmt = conn.createStatement(); + System.out.println(++stmtCnt + " : " + stmt); + } + } + + } From d3729d273b5b1f200df77b2f1fc137148b2e5ff9 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 26 Nov 2020 15:09:41 +0800 Subject: [PATCH 061/188] add test for column which not exist --- tests/pytest/concurrent_inquiry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 5d1e9a7537..6f4258312d 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -146,7 +146,7 @@ class ConcurrentInquiry: col_list=self.stb_stru_list[tbi-1] tag_list=self.stb_tag_list[tbi-1] is_stb=1 - tlist=col_list+tag_list + tlist=col_list+tag_list+['abc'] #增加不存在的域'abc'，是否会引起新bug con_rand=random.randint(0,len(condition_list)) func_rand=random.randint(0,len(func_list)) col_rand=random.randint(0,len(col_list)) From f547e29a55e57031d035c77bc685e813ffaad772 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 07:39:47 +0000 Subject: [PATCH 062/188] [TD-2135]: reduce memory usage --- src/client/inc/tsclient.h | 2 +- src/client/src/tscParseInsert.c | 20 ++++++++++++++------ src/client/src/tscServer.c | 28 +++++++++++++++++++--------- src/client/src/tscUtil.c | 16 ++++++++++++++-- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ff36cf0f5a..c482375bff 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -282,7 +282,7 @@ typedef struct { int8_t dataSourceType; // load data from file or not int8_t submitSchema; // submit block is built with table schema - STagData tagData; + STagData *pTagData; // NOTE: pTagData->data is used as a variant length array SHashObj *pTableList; // referred table involved in sql SArray *pDataBlocks; // SArray submit data blocks after parsing sql } SSqlCmd; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a44a158f93..6de8195d73 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -790,9 +790,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sql += index; tscAllocPayload(pCmd, sizeof(STagData)); - STagData *pTag = &pCmd->tagData; - - memset(pTag, 0, sizeof(STagData)); //the source super table is moved to the secondary position of the pTableMetaInfo list if (pQueryInfo->numOfTables < 2) { @@ -805,7 +802,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } + STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data)); + if (pTag == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memset(pTag, 0, offsetof(STagData, data)); tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name)); + pCmd->pTagData = pTag; + code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; @@ -934,7 +938,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } tdSortKVRowByColIdx(row); - pTag->dataLen = kvRowLen(row); + + pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row)); + if (pTag == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pCmd->pTagData = pTag; + pTag->dataLen = htonl(kvRowLen(row)); kvRowCpy(pTag->data, row); free(row); @@ -945,8 +955,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z); } - pTag->dataLen = htonl(pTag->dataLen); - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 15859674ab..9735e26ece 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1565,11 +1565,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - size_t len = htonl(pCmd->tagData.dataLen); - if (pSql->cmd.autoCreated) { + if (pCmd->autoCreated && pCmd->pTagData != NULL) { + int len = htonl(pCmd->pTagData->dataLen); if (len > 0) { - len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen); - memcpy(pInfoMsg->tags, &pCmd->tagData, len); + len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen); + memcpy(pInfoMsg->tags, pCmd->pTagData, len); pMsg += len; } } @@ -2239,8 +2239,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_META; - registerSqlObj(pNew); - tscAddSubqueryInfo(&pNew->cmd); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); @@ -2248,8 +2246,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { tscError("%p malloc failed for payload to get table meta", pSql); - free(pNew); - + tscFreeSqlObj(pNew); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2257,12 +2254,25 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name)); - memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData)); + + if (pSql->cmd.pTagData != NULL) { + int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); + pNew->cmd.pTagData = calloc(1, size); + if (pNew->cmd.pTagData == NULL) { + tscError("%p malloc failed for new tag data to get table meta", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); + } + tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); pNew->fp = tscTableMetaCallBack; pNew->param = pSql; + registerSqlObj(pNew); + int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a75f6d8296..d22c2588cc 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -408,7 +408,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { pCmd->pTableList = NULL; pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - + tscFreeQueryInfo(pCmd, removeFromCache); } @@ -512,6 +512,8 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscFreeSqlResult(pSql); tscResetSqlCmdObj(pCmd, false); + tfree(pCmd->pTagData); + memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); pCmd->allocSize = 0; @@ -1909,7 +1911,17 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm pCmd->command = cmd; pCmd->parseFinished = 1; pCmd->autoCreated = pSql->cmd.autoCreated; - memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData)); + + if (pSql->cmd.pTagData != NULL) { + int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); + pNew->cmd.pTagData = calloc(1, size); + if (pNew->cmd.pTagData == NULL) { + tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0); + free(pNew); + return NULL; + } + memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); + } if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pNew); From 55ce69e2f0c08aed576b46bf11cff1a7b7bc2d4a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 08:00:58 +0000 Subject: [PATCH 063/188] [TD-2182]: coverity issues --- src/client/src/tscSub.c | 7 ++++++- src/query/src/qExecutor.c | 6 +++--- src/util/src/tqueue.c | 3 --- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 2c81bd7c7c..a782b53e75 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -442,7 +442,12 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo); size += sizeof(SQueryTableMsg) + 4096; - tscAllocPayload(&pSql->cmd, (int)size); + int code = tscAllocPayload(&pSql->cmd, (int)size); + if (code != TSDB_CODE_SUCCESS) { + tscError("failed to alloc payload"); + return NULL; + } + for (int retry = 0; retry < 3; retry++) { tscRemoveFromSqlList(pSql); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fce25ae119..826858d1dd 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5836,9 +5836,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); - pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; - pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; - pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; + // pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; + // pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; + // pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; pQueryMsg->limit = htobe64(pQueryMsg->limit); pQueryMsg->offset = htobe64(pQueryMsg->offset); diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index d3c17e7626..143ce7c474 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -79,15 +79,12 @@ void taosCloseQueue(taos_queue param) { if (queue->qset) taosRemoveFromQset(qset, queue); - pthread_mutex_lock(&queue->mutex); - while (pNode) { pTemp = pNode; pNode = pNode->next; free (pTemp); } - pthread_mutex_unlock(&queue->mutex); pthread_mutex_destroy(&queue->mutex); free(queue); From 651672957d470f715b265c29892f495d62c94da0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 26 Nov 2020 16:32:10 +0800 Subject: [PATCH 064/188] [TD-2240] update restfulInsert --- tests/pytest/insert/restfulInsert.py | 37 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index c39d1cbe48..e3a963f1d4 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -18,10 +18,10 @@ import time import argparse class RestfulInsert: - def __init__(self, host, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder): + def __init__(self, host, startTimestamp, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder): self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} self.url = "http://%s:6041/rest/sql" % host - self.ts = 1500000000000 + self.ts = startTimestamp self.dbname = dbname self.numOfThreads = threads self.numOfTables = tables @@ -36,8 +36,10 @@ class RestfulInsert: for i in range(tablesPerThread): tableID = threadID * tablesPerThread name = 'beijing' if tableID % 2 == 0 else 'shanghai' - data = "create table %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) - requests.post(self.url, data, headers = self.header) + data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) def insertData(self, threadID): print("thread %d started" % threadID) @@ -50,7 +52,9 @@ class RestfulInsert: values = [] for k in range(self.batchSize): data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)) - requests.post(self.url, data, headers = self.header) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) def insertUnlimitedData(self, threadID): print("thread %d started" % threadID) @@ -76,15 +80,15 @@ class RestfulInsert: else: random.shuffle(values) for k in range(len(values)): - data += values[k] - requests.post(self.url, data, headers = self.header) + data += values[k] + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) - def run(self): - data = "drop database if exists %s" % self.dbname + def run(self): + data = "create database if not exists %s" % self.dbname requests.post(self.url, data, headers = self.header) - data = "create database %s" % self.dbname - requests.post(self.url, data, headers = self.header) - data = "create table %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname + data = "create table if not exists %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname requests.post(self.url, data, headers = self.header) threads = [] @@ -120,6 +124,13 @@ parser.add_argument( default='127.0.0.1', type=str, help='host name to be connected (default: 127.0.0.1)') +parser.add_argument( + '-S', + '--start-timestamp', + action='store', + default=1500000000000, + type=int, + help='insert data from timestamp (default: 1500000000000)') parser.add_argument( '-d', '--db-name', @@ -169,5 +180,5 @@ parser.add_argument( help='The order of test data (default: False)') args = parser.parse_args() -ri = RestfulInsert(args.host_name, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order) +ri = RestfulInsert(args.host_name, args.start_timestamp, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order) ri.run() \ No newline at end of file From fa25d38dea8ec9e310c85fac1da6aa36c70104ef Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 26 Nov 2020 16:37:11 +0800 Subject: [PATCH 065/188] full test --- tests/test-all.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index ff47cbfd71..5897978bce 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -17,9 +17,9 @@ function runSimCaseOneByOne { echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ echo -e "${RED}$case failed${NC}" | tee -a out.log out_log=`tail -1 out.log ` - if [[ $out_log =~ 'failed' ]];then - exit 8 - fi + # if [[ $out_log =~ 'failed' ]];then + # exit 8 + # fi end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log fi @@ -42,9 +42,9 @@ function runPyCaseOneByOne { echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` - if [[ $out_log =~ 'failed' ]];then - exit 8 - fi + # if [[ $out_log =~ 'failed' ]];then + # exit 8 + # fi echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log else $line > /dev/null 2>&1 From 19eccc86ade72ded17cd4c4033b44b275137dc0f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 26 Nov 2020 08:44:20 +0000 Subject: [PATCH 066/188] fix TD-2184 --- src/tsdb/src/tsdbMemTable.c | 4 ++-- src/util/src/tskiplist.c | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 71944c87c6..4c51608b5d 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -871,10 +871,10 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); if (pBufBlock->offset == 0) { // return the block to buffer pool - tsdbLockRepo(pRepo); + if (tsdbLockRepo(pRepo) < 0) return; SListNode *pNode = tdListPopTail(pRepo->mem->bufBlockList); tdListPrependNode(pBufPool->bufBlockList, pNode); - tsdbUnlockRepo(pRepo); + if (tsdbUnlockRepo(pRepo) < 0) return; } } else { ASSERT(listNEles(pRepo->mem->extraBuffList) > 0); diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index a36f7f0261..20bec16954 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -291,7 +291,10 @@ bool tSkipListIterNext(SSkipListIterator *iter) { iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); iter->step++; } else { - if (iter->cur == pSkipList->pHead) return false; + if (iter->cur == pSkipList->pHead) { + tSkipListUnlock(pSkipList); + return false; + } iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it From 1e94822a6a9b04a9396200e8a47392eb987b3598 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 26 Nov 2020 19:02:11 +0800 Subject: [PATCH 067/188] TD-1926 --- src/inc/tsync.h | 4 ++ src/sync/inc/syncInt.h | 2 + src/sync/src/syncMain.c | 1 + src/sync/src/syncRestore.c | 8 +-- src/sync/src/syncRetrieve.c | 100 +++++++----------------------------- src/vnode/inc/vnodeInt.h | 1 + src/vnode/src/vnodeMain.c | 24 ++++++++- src/vnode/src/vnodeWrite.c | 2 +- 8 files changed, 55 insertions(+), 87 deletions(-) diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 398e1bf97c..293e382519 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -85,6 +85,9 @@ typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); // when data file is synced successfully, notity app typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); +// get file version +typedef int32_t (*FGetFileVersion)(int32_t vgId, uint64_t *fver); + typedef struct { int32_t vgId; // vgroup ID uint64_t version; // initial version @@ -97,6 +100,7 @@ typedef struct { FNotifyRole notifyRole; FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; + FGetFileVersion getFileVersion; } SSyncInfo; typedef void *tsync_h; diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 05b7adc5f4..7dc04d70d0 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -139,6 +139,7 @@ typedef struct SsyncPeer { char id[TSDB_EP_LEN + 32]; // peer vgId + end point uint64_t version; uint64_t sversion; // track the peer version in retrieve process + uint64_t lastVer; // track the file version while retrieve int32_t syncFd; int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried @@ -172,6 +173,7 @@ typedef struct SSyncNode { FNotifyRole notifyRole; FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; + FGetFileVersion getFileVersion; pthread_mutex_t mutex; } SSyncNode; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index d2d6d2d7fa..af3241f217 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -196,6 +196,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { pNode->confirmForward = pInfo->confirmForward; pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl; pNode->notifyFileSynced = pInfo->notifyFileSynced; + pNode->getFileVersion = pInfo->getFileVersion; pNode->selfIndex = -1; pNode->vgId = pInfo->vgId; diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ed6b63c92d..1f5b560c87 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -52,12 +52,12 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */ // master file info - SFileInfo sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */ // slave file info - SFileAck fileAck; + SFileInfo minfo = {0}; + SFileInfo sinfo = {0}; + SFileAck fileAck = {0}; int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; - uint32_t pindex = 0; // index in last restore + uint32_t pindex = 0; // index in last restore bool fileChanged = false; *fversion = 0; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 060badba9d..e199ce9845 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -25,85 +25,32 @@ #include "tsync.h" #include "syncInt.h" -static int32_t syncAddIntoWatchList(SSyncPeer *pPeer, char *name) { - sDebug("%s, start to monitor:%s", pPeer->id, name); +static int32_t syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { + if (pNode->getFileVersion == NULL) return 0; - if (pPeer->notifyFd <= 0) { - pPeer->watchNum = 0; - pPeer->notifyFd = inotify_init1(IN_NONBLOCK); - if (pPeer->notifyFd < 0) { - sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno)); - return -1; - } - - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles); - if (pPeer->watchFd == NULL) { - sError("%s, failed to allocate watchFd", pPeer->id); - return -1; - } - - memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles); + uint64_t fver = 0; + int32_t code = (*pNode->getFileVersion)(pNode->vgId, &fver); + if (code != 0) { + sInfo("%s, file are modified while retrieve, lastver:%" PRIu64, pPeer->id, pPeer->lastVer); + return code; } - int32_t *wd = pPeer->watchFd + pPeer->watchNum; - - if (*wd >= 0) { - if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) { - sError("%s, failed to remove wd:%d since %s", pPeer->id, *wd, strerror(errno)); - return -1; - } - } - - *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_DELETE); - if (*wd == -1) { - sError("%s, failed to add %s since %s", pPeer->id, name, strerror(errno)); + if (fver != pPeer->lastVer) { + sInfo("%s, file are modified while retrieve, fver:%" PRIu64 " lastver:%" PRIu64, pPeer->id, fver, pPeer->lastVer); return -1; - } else { - sDebug("%s, monitor %s, wd:%d watchNum:%d", pPeer->id, name, *wd, pPeer->watchNum); } - pPeer->watchNum = (pPeer->watchNum + 1) % tsMaxWatchFiles; - return 0; } -static int32_t syncAreFilesModified(SSyncPeer *pPeer) { - if (pPeer->notifyFd <= 0) return 0; - - char buf[2048]; - int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); - if (len < 0 && errno != EAGAIN) { - sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno)); - return -1; - } - - int32_t code = 0; - if (len > 0) { - const struct inotify_event *event; - char *ptr; - for (ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) { - event = (const struct inotify_event *)ptr; - if ((event->mask & IN_MODIFY) || (event->mask & IN_DELETE)) { - sDebug("%s, processed file is changed", pPeer->id); - pPeer->fileChanged = 1; - code = 1; - break; - } - } - } - - return code; -} - static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo fileInfo; - SFileAck fileAck; + SFileInfo fileInfo = {0}; + SFileAck fileAck = {0}; int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; - memset(&fileInfo, 0, sizeof(fileInfo)); - memset(&fileAck, 0, sizeof(fileAck)); + if (pNode->getFileVersion) (*pNode->getFileVersion)(pNode->vgId, &pPeer->lastVer); while (1) { // retrieve file info @@ -136,15 +83,6 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // set the peer sync version pPeer->sversion = fileInfo.fversion; - // get the full path to file - snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); - - // add the file into watch list - if (syncAddIntoWatchList(pPeer, name) < 0) { - sError("%s, failed to watch file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } - // if sync is not required, continue if (fileAck.sync == 0) { fileInfo.index++; @@ -152,6 +90,9 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { continue; } + // get the full path to file + snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name); + // send the file to peer int32_t sfd = open(name, O_RDONLY); if (sfd < 0) { @@ -170,10 +111,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.index++; // check if processed files are modified - if (syncAreFilesModified(pPeer) != 0) { - sInfo("%s, file:%s are modified while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); - break; - } + if (syncAreFilesModified(pNode, pPeer) != 0) break; } if (code < 0) { @@ -308,9 +246,9 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { SSyncNode *pNode = pPeer->pSyncNode; int32_t code = -1; - char fname[TSDB_FILENAME_LEN * 2]; // full path to wal file + char fname[TSDB_FILENAME_LEN * 2] = {0}; // full path to wal file - if (syncAreFilesModified(pPeer) != 0) return -1; + if (syncAreFilesModified(pNode, pPeer) != 0) return -1; while (1) { int32_t once = 0; // last WAL has once ever been processed @@ -431,7 +369,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { index++; - if (syncAreFilesModified(pPeer) != 0) break; + if (syncAreFilesModified(pNode, pPeer) != 0) break; } if (code == 0) { diff --git a/src/vnode/inc/vnodeInt.h b/src/vnode/inc/vnodeInt.h index 7fc9b100ef..e468c2e83e 100644 --- a/src/vnode/inc/vnodeInt.h +++ b/src/vnode/inc/vnodeInt.h @@ -44,6 +44,7 @@ typedef struct { int8_t role; int8_t accessState; int8_t isFull; + int8_t isCommiting; uint64_t version; // current version uint64_t fversion; // version on saved data file void *wqueue; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index cd6d2ea7c0..1b2f3ee93c 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -38,6 +38,7 @@ static void vnodeCtrlFlow(int32_t vgId, int32_t level); static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); static void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); +static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver); #ifndef _SYNC int64_t syncStart(const SSyncInfo *info) { return NULL; } @@ -352,6 +353,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.notifyRole = vnodeNotifyRole; syncInfo.notifyFlowCtrl = vnodeCtrlFlow; syncInfo.notifyFileSynced = vnodeNotifyFileSynced; + syncInfo.getFileVersion = vnodeGetFileVersion; pVnode->sync = syncStart(&syncInfo); #ifndef _SYNC @@ -597,18 +599,19 @@ static void vnodeCleanUp(SVnodeObj *pVnode) { vnodeRelease(pVnode); } -// TODO: this is a simple implement static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { SVnodeObj *pVnode = arg; if (eno != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to commit since %s, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, tstrerror(eno), pVnode->fversion, pVnode->version); + pVnode->isCommiting = 0; pVnode->isFull = 1; return 0; } if (status == TSDB_STATUS_COMMIT_START) { + pVnode->isCommiting = 1; pVnode->fversion = pVnode->version; vDebug("vgId:%d, start commit, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); if (pVnode->status != TAOS_VN_STATUS_INIT) { @@ -619,6 +622,7 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) { if (status == TSDB_STATUS_COMMIT_OVER) { vDebug("vgId:%d, commit over, fver:%" PRIu64 " vver:%" PRIu64, pVnode->vgId, pVnode->fversion, pVnode->version); + pVnode->isCommiting = 0; pVnode->isFull = 0; if (pVnode->status != TAOS_VN_STATUS_INIT) { walRemoveOneOldFile(pVnode->wal); @@ -765,3 +769,21 @@ static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void vnodeRelease(pVnode); return code; } + +static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver) { + SVnodeObj *pVnode = vnodeAcquire(vgId); + if (pVnode == NULL) { + vError("vgId:%d, vnode not found while write to cache", vgId); + return -1; + } + + int32_t code = 0; + if (pVnode->isCommiting) { + code = -1; + } else { + *fver = pVnode->fversion; + } + + vnodeRelease(pVnode); + return code; +} diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 57bd407cd1..360a631e1d 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -288,7 +288,7 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { } else { code = vnodePerformFlowCtrl(pWrite); if (code == 0) { - vTrace("vgId:%d, write into vwqueue after flowctrl", pVnode->vgId); + vDebug("vgId:%d, write into vwqueue after flowctrl", pVnode->vgId); pWrite->processedCount = 0; taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); } From 11230382920a402fe2c9010a2b16e4b544e22b07 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 26 Nov 2020 14:24:18 +0000 Subject: [PATCH 068/188] TD-2245 --- src/dnode/src/dnodePeer.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 4c44924cd0..6bf22cee4e 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -151,9 +151,9 @@ void dnodeCleanupClient() { } static void dnodeProcessRspFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { - if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) { + if (dnodeGetRunStatus() == TSDB_RUN_STATUS_STOPPED) { if (pMsg == NULL || pMsg->pCont == NULL) return; - dDebug("msg:%p is ignored since dnode not running", pMsg); + dDebug("msg:%p is ignored since dnode is stopping", pMsg); rpcFreeCont(pMsg->pCont); return; } From f349c8af87d86cd4878573c6c1a33b844b7992aa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 26 Nov 2020 17:34:09 +0000 Subject: [PATCH 069/188] TD-2257 --- src/client/inc/tscUtil.h | 2 +- src/client/inc/tsclient.h | 2 +- src/client/src/tscServer.c | 40 ++++++++++++++------------------------ src/client/src/tscSql.c | 24 +++++++++++++++++++---- src/client/src/tscSystem.c | 5 ----- src/client/src/tscUtil.c | 8 ++++---- 6 files changed, 41 insertions(+), 40 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index bde27d2932..cfcecc6582 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -278,7 +278,7 @@ bool hasMoreClauseToTry(SSqlObj* pSql); void tscTryQueryNextVnode(SSqlObj *pSql, __async_cb_func_t fp); void tscAsyncQuerySingleRowForNextVnode(void *param, TAOS_RES *tres, int numOfRows); void tscTryQueryNextClause(SSqlObj* pSql, __async_cb_func_t fp); -int tscSetMgmtEpSetFromCfg(const char *first, const char *second); +int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corEpSet); bool tscSetSqlOwner(SSqlObj* pSql); void tscClearSqlOwner(SSqlObj* pSql); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c482375bff..35f9b3af62 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -336,6 +336,7 @@ typedef struct STscObj { struct SSqlObj * pHb; struct SSqlObj * sqlList; struct SSqlStream *streamList; + SRpcCorEpSet *tscCorMgmtEpSet; void* pDnodeConn; pthread_mutex_t mutex; T_REF_DECLARE() @@ -515,7 +516,6 @@ extern int tsInsertHeadSize; extern int tscNumOfThreads; extern int tscRefId; -extern SRpcCorEpSet tscMgmtEpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9735e26ece..3a2c673f34 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -26,7 +26,7 @@ #include "ttimer.h" #include "tlockfree.h" -SRpcCorEpSet tscMgmtEpSet; +///SRpcCorEpSet tscMgmtEpSet; int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo) = {0}; @@ -73,10 +73,11 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { assert(hasFqdn); } -static void tscDumpMgmtEpSet(SRpcEpSet *epSet) { - taosCorBeginRead(&tscMgmtEpSet.version); - *epSet = tscMgmtEpSet.epSet; - taosCorEndRead(&tscMgmtEpSet.version); +static void tscDumpMgmtEpSet(SSqlObj *pSql) { + SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginRead(&pCorEpSet->version); + pSql->epSet = pCorEpSet->epSet; + taosCorEndRead(&pCorEpSet->version); } static void tscEpSetHtons(SRpcEpSet *s) { for (int32_t i = 0; i < s->numOfEps; i++) { @@ -94,11 +95,12 @@ bool tscEpSetIsEqual(SRpcEpSet *s1, SRpcEpSet *s2) { } return true; } -void tscUpdateMgmtEpSet(SRpcEpSet *pEpSet) { +void tscUpdateMgmtEpSet(SSqlObj *pSql, SRpcEpSet *pEpSet) { // no need to update if equal - taosCorBeginWrite(&tscMgmtEpSet.version); - tscMgmtEpSet.epSet = *pEpSet; - taosCorEndWrite(&tscMgmtEpSet.version); + SRpcCorEpSet *pCorEpSet = pSql->pTscObj->tscCorMgmtEpSet; + taosCorBeginWrite(&pCorEpSet->version); + pCorEpSet->epSet = *pEpSet; + taosCorEndWrite(&pCorEpSet->version); } static void tscDumpEpSetFromVgroupInfo(SCorVgroupInfo *pVgroupInfo, SRpcEpSet *pEpSet) { if (pVgroupInfo == NULL) { return;} @@ -133,18 +135,6 @@ static void tscUpdateVgroupInfo(SSqlObj *pObj, SRpcEpSet *pEpSet) { taosCorEndWrite(&pVgroupInfo->version); } -void tscPrintMgmtEp() { - SRpcEpSet dump; - tscDumpMgmtEpSet(&dump); - if (dump.numOfEps <= 0) { - tscError("invalid mnode EP list:%d", dump.numOfEps); - } else { - for (int i = 0; i < dump.numOfEps; ++i) { - tscDebug("mnode index:%d %s:%d", i, dump.fqdn[i], dump.port[i]); - } - } -} - void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { STscObj *pObj = (STscObj *)param; if (pObj == NULL) return; @@ -162,7 +152,7 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { SRpcEpSet * epSet = &pRsp->epSet; if (epSet->numOfEps > 0) { tscEpSetHtons(epSet); - tscUpdateMgmtEpSet(epSet); + tscUpdateMgmtEpSet(pSql, epSet); } pSql->pTscObj->connId = htonl(pRsp->connId); @@ -227,7 +217,7 @@ int tscSendMsgToServer(SSqlObj *pSql) { // set the mgmt ip list if (pSql->cmd.command >= TSDB_SQL_MGMT) { - tscDumpMgmtEpSet(&pSql->epSet); + tscDumpMgmtEpSet(pSql); } memcpy(pMsg, pSql->cmd.payload, pSql->cmd.payloadLen); @@ -289,7 +279,7 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { if (pCmd->command < TSDB_SQL_MGMT) { tscUpdateVgroupInfo(pSql, pEpSet); } else { - tscUpdateMgmtEpSet(pEpSet); + tscUpdateMgmtEpSet(pSql, pEpSet); } } } @@ -2070,7 +2060,7 @@ int tscProcessConnectRsp(SSqlObj *pSql) { if (pConnect->epSet.numOfEps > 0) { tscEpSetHtons(&pConnect->epSet); - tscUpdateMgmtEpSet(&pConnect->epSet); + tscUpdateMgmtEpSet(pSql, &pConnect->epSet); } strcpy(pObj->sversion, pConnect->serverVersion); diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index de01776472..761a222b26 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -58,6 +58,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa terrno = TSDB_CODE_TSC_INVALID_USER_LENGTH; return NULL; } + SRpcCorEpSet corMgmtEpSet; char secretEncrypt[32] = {0}; int secretEncryptLen = 0; @@ -82,11 +83,13 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa } secretEncryptLen = outlen; } - + if (ip) { - if (tscSetMgmtEpSetFromCfg(ip, NULL) < 0) return NULL; - if (port) tscMgmtEpSet.epSet.port[0] = port; - } + if (tscSetMgmtEpSetFromCfg(ip, NULL, &corMgmtEpSet) < 0) return NULL; + if (port) corMgmtEpSet.epSet.port[0] = port; + } else { + if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond, &corMgmtEpSet) < 0) return NULL; + } void *pDnodeConn = NULL; if (tscInitRpc(user, secretEncrypt, &pDnodeConn) != 0) { @@ -100,10 +103,20 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa rpcClose(pDnodeConn); return NULL; } + // set up tscObj's mgmtEpSet + pObj->tscCorMgmtEpSet = (SRpcCorEpSet *)malloc(sizeof(SRpcCorEpSet)); + if (NULL == pObj->tscCorMgmtEpSet) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + rpcClose(pDnodeConn); + free(pObj->tscCorMgmtEpSet); + free(pObj); + } + memcpy(pObj->tscCorMgmtEpSet, &corMgmtEpSet, sizeof(SRpcCorEpSet)); pObj->signature = pObj; pObj->pDnodeConn = pDnodeConn; T_REF_INIT_VAL(pObj, 1); + tstrncpy(pObj->user, user, sizeof(pObj->user)); secretEncryptLen = MIN(secretEncryptLen, sizeof(pObj->pass)); @@ -115,6 +128,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (len >= TSDB_DB_NAME_LEN) { terrno = TSDB_CODE_TSC_INVALID_DB_LENGTH; rpcClose(pDnodeConn); + free(pObj->tscCorMgmtEpSet); free(pObj); return NULL; } @@ -132,6 +146,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa if (NULL == pSql) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; rpcClose(pDnodeConn); + free(pObj->tscCorMgmtEpSet); free(pObj); return NULL; } @@ -149,6 +164,7 @@ static SSqlObj *taosConnectImpl(const char *ip, const char *user, const char *pa terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; rpcClose(pDnodeConn); free(pSql); + free(pObj->tscCorMgmtEpSet); free(pObj); return NULL; } diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 9e9a00550a..a5bc1ae7ad 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -116,11 +116,6 @@ void taos_init_imp(void) { taosInitNote(tsNumOfLogLines / 10, 1, (char*)"tsc_note"); } - if (tscSetMgmtEpSetFromCfg(tsFirst, tsSecond) < 0) { - tscError("failed to init mnode EP list"); - return; - } - tscInitMsgsFp(); int queueSize = tsMaxConnections*2; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index d22c2588cc..077fd8aa00 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -867,7 +867,7 @@ void tscCloseTscObj(void *param) { rpcClose(pObj->pDnodeConn); pObj->pDnodeConn = NULL; } - + tfree(pObj->tscCorMgmtEpSet); pthread_mutex_destroy(&pObj->mutex); tscDebug("%p DB connection is closed, dnodeConn:%p", pObj, p); @@ -2459,10 +2459,10 @@ char* strdup_throw(const char* str) { return p; } -int tscSetMgmtEpSetFromCfg(const char *first, const char *second) { +int tscSetMgmtEpSetFromCfg(const char *first, const char *second, SRpcCorEpSet *corMgmtEpSet) { + corMgmtEpSet->version = 0; // init mgmt ip set - tscMgmtEpSet.version = 0; - SRpcEpSet *mgmtEpSet = &(tscMgmtEpSet.epSet); + SRpcEpSet *mgmtEpSet = &(corMgmtEpSet->epSet); mgmtEpSet->numOfEps = 0; mgmtEpSet->inUse = 0; From 86479c1f6a42a97d0ef5d880e18565eae1660ded Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 11:33:44 +0800 Subject: [PATCH 070/188] TD-2246 --- tests/test/c/CMakeLists.txt | 7 ++-- tests/test/c/hashIterator.c | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 tests/test/c/hashIterator.c diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 26aa20e647..11480a8ba2 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -31,8 +31,8 @@ IF (TD_LINUX) #add_executable(createTablePerformance createTablePerformance.c) #target_link_libraries(createTablePerformance taos_static tutil common pthread) - #add_executable(createNormalTable createNormalTable.c) - #target_link_libraries(createNormalTable taos_static tutil common pthread) + add_executable(createNormalTable createNormalTable.c) + target_link_libraries(createNormalTable taos_static tutil common pthread) #add_executable(queryPerformance queryPerformance.c) #target_link_libraries(queryPerformance taos_static tutil common pthread) @@ -45,5 +45,8 @@ IF (TD_LINUX) #add_executable(invalidTableId invalidTableId.c) #target_link_libraries(invalidTableId taos_static tutil common pthread) + + add_executable(hashIterator hashIterator.c) + target_link_libraries(hashIterator taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/hashIterator.c b/tests/test/c/hashIterator.c new file mode 100644 index 0000000000..cbd8a0895e --- /dev/null +++ b/tests/test/c/hashIterator.c @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tulog.h" +#include "tutil.h" +#include "hash.h" + +typedef struct HashTestRow { + int32_t keySize; + char key[100]; +} HashTestRow; + +int main(int argc, char *argv[]) { + _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + void * hashHandle = taosHashInit(100, hashFp, true, HASH_ENTRY_LOCK); + + pPrint("insert 3 rows to hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow row = {0}; + row.keySize = sprintf(row.key, "0.db.st%d", t); + + taosHashPut(hashHandle, row.key, row.keySize, &row, sizeof(HashTestRow)); + } + + pPrint("start iterator"); + HashTestRow *row = taosHashIterate(hashHandle, NULL); + while (row) { + pPrint("drop key:%s", row->key); + taosHashRemove(hashHandle, row->key, row->keySize); + + pPrint("get rows from hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow r = {0}; + r.keySize = sprintf(r.key, "0.db.st%d", t); + + void *result = taosHashGet(hashHandle, r.key, r.keySize); + pPrint("get key:%s result:%p", r.key, result); + } + + //Before getting the next iterator, the object just deleted can be obtained + row = taosHashIterate(hashHandle, row); + } + + pPrint("stop iterator"); + taosHashCancelIterate(hashHandle, row); + + pPrint("get rows from hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow r = {0}; + r.keySize = sprintf(r.key, "0.db.st%d", t); + + void *result = taosHashGet(hashHandle, r.key, r.keySize); + pPrint("get key:%s result:%p", r.key, result); + } + + return 0; +} \ No newline at end of file From 6b3c07142ac523fed53e806c6a14e5200614eb2c Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 27 Nov 2020 06:47:22 +0000 Subject: [PATCH 071/188] [TD-2198]: cq can be created more than once --- src/cq/src/cqMain.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 1be7552a89..84b2c297d0 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -241,8 +241,12 @@ static void doCreateStream(void *param, TAOS_RES *result, int32_t code) { SCqObj* pObj = (SCqObj*)param; SCqContext* pContext = pObj->pContext; SSqlObj* pSql = (SSqlObj*)result; - pContext->dbConn = pSql->pTscObj; + if (atomic_val_compare_exchange_ptr(&(pContext->dbConn), NULL, pSql->pTscObj) != NULL) { + taos_close(pSql->pTscObj); + } + pthread_mutex_lock(&pContext->mutex); cqCreateStream(pContext, pObj); + pthread_mutex_unlock(&pContext->mutex); } static void cqProcessCreateTimer(void *param, void *tmrId) { @@ -253,7 +257,9 @@ static void cqProcessCreateTimer(void *param, void *tmrId) { cDebug("vgId:%d, try connect to TDengine", pContext->vgId); taos_connect_a(NULL, pContext->user, pContext->pass, pContext->db, 0, doCreateStream, param, NULL); } else { + pthread_mutex_lock(&pContext->mutex); cqCreateStream(pContext, pObj); + pthread_mutex_unlock(&pContext->mutex); } } @@ -267,12 +273,14 @@ static void cqCreateStream(SCqContext *pContext, SCqObj *pObj) { } pObj->tmrId = 0; - pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL); - if (pObj->pStream) { - pContext->num++; - cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr); - } else { - cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr); + if (pObj->pStream == NULL) { + pObj->pStream = taos_open_stream(pContext->dbConn, pObj->sqlStr, cqProcessStreamRes, 0, pObj, NULL); + if (pObj->pStream) { + pContext->num++; + cInfo("vgId:%d, id:%d CQ:%s is openned", pContext->vgId, pObj->tid, pObj->sqlStr); + } else { + cError("vgId:%d, id:%d CQ:%s, failed to open", pContext->vgId, pObj->tid, pObj->sqlStr); + } } } From 4b09f05290596014d987bcdc8c34d2004814618e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 27 Nov 2020 07:23:16 +0000 Subject: [PATCH 072/188] add a removed flag in pNode --- src/connector/go | 2 +- src/connector/grafanaplugin | 2 +- src/rpc/src/rpcTcp.c | 15 +++++++++------ src/util/inc/hash.h | 1 + src/util/src/hash.c | 19 +++++++++++++++---- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/connector/go b/src/connector/go index 050667e5b4..8c58c512b6 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f +Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index ec77d9049a..d598db167e 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944 +Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 7b8cf3cda2..2850046d05 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -371,10 +371,13 @@ void taosCloseTcpConnection(void *chandle) { int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { SFdObj *pFdObj = chandle; - if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1; + SThreadObj *pThreadObj = pFdObj->pThreadObj; - return taosWriteMsg(pFdObj->fd, data, len); + int ret = taosWriteMsg(pFdObj->fd, data, len); + tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); + + return ret; } static void taosReportBrokenLink(SFdObj *pFdObj) { @@ -409,7 +412,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tDebug("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); + tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); return -1; } @@ -420,7 +423,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("TCP malloc mem:%p size:%d", buffer, size); + tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer); } msg = buffer + tsRpcOverhead; @@ -583,8 +586,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", - pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", + pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); } diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index aaff48f4f8..572f1a50c6 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -36,6 +36,7 @@ typedef struct SHashNode { uint32_t keyLen; // length of the key uint32_t dataLen; // length of data int8_t count; // reference count + int8_t removed; // flag to indicate removed char data[]; } SHashNode; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index cd9a9f7d0a..3ac54eedd8 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -76,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -222,7 +222,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -361,7 +361,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) break; prevNode = pNode; @@ -372,6 +372,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe code = 0; // it is found pNode->count--; + pNode->removed = 1; if (pNode->count <= 0) { if (prevNode) { prevNode->next = pNode->next; @@ -702,8 +703,13 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { pNode = pNode->next; } - if (pNode) { + if (pNode) { pNode = pNode->next; + while (pNode) { + if (pNode->removed == 0) break; + pNode = pNode->next; + } + pOld->count--; if (pOld->count <=0) { if (prevNode) { @@ -755,6 +761,11 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { } pNode = pe->next; + while (pNode) { + if (pNode->removed == 0) break; + pNode = pNode->next; + } + if (pNode) break; if (pHashObj->type == HASH_ENTRY_LOCK) { From 257af62edfc99dca2198eb4dc3fbb00c36d9b043 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 27 Nov 2020 15:29:12 +0800 Subject: [PATCH 073/188] [TD-2258] taos_fetch_block() resutl printf error --- tests/examples/c/apitest.c | 12 ++++++------ tests/examples/c/subscribe.c | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index be60a88ad7..e08d667d6b 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -79,11 +79,11 @@ static int print_result(TAOS_RES* res, int blockFetch) { if (blockFetch) { int rows = 0; while ((rows = taos_fetch_block(res, &row))) { - for (int i = 0; i < rows; i++) { - char temp[256]; - taos_print_row(temp, row + i, fields, num_fields); - puts(temp); - } + //for (int i = 0; i < rows; i++) { + // char temp[256]; + // taos_print_row(temp, row + i, fields, num_fields); + // puts(temp); + //} nRows += rows; } } else { @@ -498,4 +498,4 @@ int main(int argc, char *argv[]) { taos_close(taos); taos_cleanup(); -} \ No newline at end of file +} diff --git a/tests/examples/c/subscribe.c b/tests/examples/c/subscribe.c index 8368af18f7..cdd8ddaf7f 100644 --- a/tests/examples/c/subscribe.c +++ b/tests/examples/c/subscribe.c @@ -19,10 +19,10 @@ void print_result(TAOS_RES* res, int blockFetch) { if (blockFetch) { nRows = taos_fetch_block(res, &row); - for (int i = 0; i < nRows; i++) { - taos_print_row(buf, row + i, fields, num_fields); - puts(buf); - } + //for (int i = 0; i < nRows; i++) { + // taos_print_row(buf, row + i, fields, num_fields); + // puts(buf); + //} } else { while ((row = taos_fetch_row(res))) { taos_print_row(buf, row, fields, num_fields); From 15691bfb088fd114f3e53c769c2829937e746d63 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 27 Nov 2020 07:41:35 +0000 Subject: [PATCH 074/188] data type compatible with windows --- src/util/inc/hash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 572f1a50c6..b0319d3e13 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -34,7 +34,7 @@ typedef struct SHashNode { struct SHashNode *next; uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key - uint32_t dataLen; // length of data + size_t dataLen; // length of data int8_t count; // reference count int8_t removed; // flag to indicate removed char data[]; From feb2270a4cf02588b0795c6fe62d31f99d135c7d Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 15:52:38 +0800 Subject: [PATCH 075/188] change log format --- src/common/src/tglobal.c | 2 +- src/sync/src/syncMain.c | 5 ++--- src/sync/src/syncRestore.c | 4 ++-- src/sync/src/syncRetrieve.c | 11 ++++++----- src/vnode/src/vnodeMain.c | 7 +++++-- src/vnode/src/vnodeWrite.c | 8 +++++--- src/wal/src/walWrite.c | 4 ++-- tests/script/general/wal/sync.sim | 1 + tests/script/tmp/mnodes.sim | 4 ++++ 9 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index f8bb965d28..d26746a5d1 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -200,7 +200,7 @@ int32_t tsNumOfLogLines = 10000000; int32_t mDebugFlag = 135; int32_t sdbDebugFlag = 135; int32_t dDebugFlag = 135; -int32_t vDebugFlag = 131; +int32_t vDebugFlag = 135; int32_t cDebugFlag = 131; int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index af3241f217..ce85c940ff 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -541,7 +541,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->ip = ip; pPeer->port = pInfo->nodePort; pPeer->fqdn[sizeof(pPeer->fqdn) - 1] = 0; - snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, peer:%s:%u", pNode->vgId, pPeer->fqdn, pPeer->port); + snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d, nodeId:%d", pNode->vgId, pPeer->nodeId); pPeer->peerFd = -1; pPeer->syncFd = -1; @@ -1144,8 +1144,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pPeer->syncFd = connFd; syncCreateRestoreDataThread(pPeer); } else { - sDebug("%s, TCP connection is already up(pfd:%d), close one, new pfd:%d sfd:%d", pPeer->id, pPeer->peerFd, connFd, - pPeer->syncFd); + sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd); syncClosePeerConn(pPeer); pPeer->peerFd = connFd; pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd); diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index 1f5b560c87..ada55aaf8f 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -134,7 +134,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { // data file is changed, code shall be set to 1 *fversion = minfo.fversion; code = 1; - sDebug("%s, file changed while restore file", pPeer->id); + sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion); } if (code < 0) { @@ -160,7 +160,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) { } if (pHead->len == 0) { - sDebug("%s, wal is synced over", pPeer->id); + sDebug("%s, wal is synced over, last wver:%" PRIu64, pPeer->id, lastVer); code = 0; break; } // wal sync over diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index e199ce9845..c956353a32 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -58,6 +58,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); // fileInfo.size = htonl(size); + sDebug("%s, file:%s info will be sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); @@ -107,7 +108,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { break; } - sDebug("%s, %s is sent, size:%" PRId64, pPeer->id, name, fileInfo.size); + sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); fileInfo.index++; // check if processed files are modified @@ -419,18 +420,18 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { pPeer->sversion = 0; pPeer->sstatus = TAOS_SYNC_STATUS_FILE; - sInfo("%s, start to retrieve file, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + sInfo("%s, start to retrieve files, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); if (syncRetrieveFile(pPeer) < 0) { - sError("%s, failed to retrieve file", pPeer->id); + sError("%s, failed to retrieve files", pPeer->id); return -1; } // if no files are synced, there must be wal to sync, sversion must be larger than one if (pPeer->sversion == 0) pPeer->sversion = 1; - sInfo("%s, start to retrieve wal", pPeer->id); + sInfo("%s, start to retrieve wals", pPeer->id); if (syncRetrieveWal(pPeer) < 0) { - sError("%s, failed to retrieve wal", pPeer->id); + sError("%s, failed to retrieve wals", pPeer->id); return -1; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 1b2f3ee93c..076392a1dc 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -688,8 +688,10 @@ static void vnodeCtrlFlow(int32_t vgId, int32_t level) { return; } - pVnode->flowctrlLevel = level; - vDebug("vgId:%d, set flowctrl level:%d", pVnode->vgId, level); + if (pVnode->flowctrlLevel != level) { + vDebug("vgId:%d, set flowctrl level from %d to %d", pVnode->vgId, pVnode->flowctrlLevel, level); + pVnode->flowctrlLevel = level; + } vnodeRelease(pVnode); } @@ -779,6 +781,7 @@ static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver) { int32_t code = 0; if (pVnode->isCommiting) { + vDebug("vgId:%d, vnode is commiting while get file version", vgId); code = -1; } else { *fver = pVnode->fversion; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 360a631e1d..268d1fb53b 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -282,13 +282,15 @@ static void vnodeFlowCtrlMsgToWQueue(void *param, void *tmrId) { pWrite->processedCount++; if (pWrite->processedCount > 100) { - vError("vgId:%d, msg:%p, failed to process since %s", pVnode->vgId, pWrite, tstrerror(code)); + vError("vgId:%d, msg:%p, failed to process since %s, retry:%d", pVnode->vgId, pWrite, tstrerror(code), + pWrite->processedCount); pWrite->processedCount = 1; dnodeSendRpcVWriteRsp(pWrite->pVnode, pWrite, code); } else { code = vnodePerformFlowCtrl(pWrite); if (code == 0) { - vDebug("vgId:%d, write into vwqueue after flowctrl", pVnode->vgId); + vDebug("vgId:%d, msg:%p, write into vwqueue after flowctrl, retry:%d", pVnode->vgId, pWrite, + pWrite->processedCount); pWrite->processedCount = 0; taosWriteQitem(pVnode->wqueue, pWrite->qtype, pWrite); } @@ -310,7 +312,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) { void *unUsed = NULL; taosTmrReset(vnodeFlowCtrlMsgToWQueue, 100, pWrite, tsDnodeTmr, &unUsed); - vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, count:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, + vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl, retry:%d", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, pWrite->processedCount); return TSDB_CODE_VND_ACTION_IN_PROGRESS; } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 36b3dba165..8a532c8164 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -211,7 +211,7 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { code = (*fileId == pWal->fileId) ? 0 : 1; } - wTrace("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId); + wDebug("vgId:%d, get wal file, code:%d curId:%" PRId64 " outId:%" PRId64, pWal->vgId, code, pWal->fileId, *fileId); pthread_mutex_unlock(&(pWal->mutex)); return code; @@ -325,7 +325,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch offset = offset + sizeof(SWalHead) + pHead->len; - wTrace("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, + wDebug("vgId:%d, restore wal, fileId:%" PRId64 " hver:%" PRIu64 " wver:%" PRIu64 " len:%d", pWal->vgId, fileId, pHead->version, pWal->version, pHead->len); pWal->version = pHead->version; diff --git a/tests/script/general/wal/sync.sim b/tests/script/general/wal/sync.sim index abaf22f919..c6f7402b87 100644 --- a/tests/script/general/wal/sync.sim +++ b/tests/script/general/wal/sync.sim @@ -82,6 +82,7 @@ restful d1 table_rest 1591772800 30000 restful d1 table_rest 1591872800 30000 restful d1 table_rest 1591972800 30000 +sleep 1000 sql select * from table_rest; print rows: $rows if $rows != 300000 then diff --git a/tests/script/tmp/mnodes.sim b/tests/script/tmp/mnodes.sim index de02ae741b..e11140028d 100644 --- a/tests/script/tmp/mnodes.sim +++ b/tests/script/tmp/mnodes.sim @@ -20,6 +20,10 @@ system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000 system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000 +system sh/cfg.sh -n dnode1 -c minTablesPerVnode -v 1000 +system sh/cfg.sh -n dnode2 -c minTablesPerVnode -v 1000 +system sh/cfg.sh -n dnode3 -c minTablesPerVnode -v 1000 + system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20 system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20 system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20 From 6bf39dfa3b6bdbf1afc622e18e4e13118d18178b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 17:16:49 +0800 Subject: [PATCH 076/188] TD-1455 TD-2196 --- src/inc/taoserror.h | 5 ++++- src/sync/src/syncRetrieve.c | 36 +++++++++++++++++++++--------------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 77ec5350ba..5fde6d40d5 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -261,6 +261,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "Invalid Sy TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync module not enabled") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sync version") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, 0, 0x0903, "Sync confirm expired") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_VND_COMMITING, 0, 0x0904, "Vnode is commiting") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_FILE_CHNAGED, 0, 0x0905, "Vnode file is changed") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_APP_ERROR, 0, 0x1000, "Unexpected generic error in sync") // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") @@ -367,6 +370,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_TAG_VALUE_TOO_LONG, 0, 0x11A4, "tag value TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_NULL, 0, 0x11A5, "value not find") TAOS_DEFINE_ERROR(TSDB_CODE_HTTP_OP_VALUE_TYPE, 0, 0x11A6, "value type should be boolean, number or string") +// odbc TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_OOM, 0, 0x2100, "out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_CHAR_NOT_NUM, 0, 0x2101, "convertion not a valid literal input") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_UNDEF, 0, 0x2102, "convertion undefined") @@ -390,7 +394,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_BAD_SEQ, 0, 0x2113, "src bad se TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_INCOMPLETE, 0, 0x2114, "src incomplete") TAOS_DEFINE_ERROR(TSDB_CODE_ODBC_CONV_SRC_GENERAL, 0, 0x2115, "src general") - #ifdef TAOS_ERROR_C }; #endif diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index c956353a32..4559285a1f 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include #include "os.h" +#include "taoserror.h" #include "tlog.h" #include "tutil.h" #include "tglobal.h" @@ -26,28 +27,31 @@ #include "syncInt.h" static int32_t syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { - if (pNode->getFileVersion == NULL) return 0; + if (pNode->getFileVersion == NULL) return TSDB_CODE_SUCCESS; uint64_t fver = 0; int32_t code = (*pNode->getFileVersion)(pNode->vgId, &fver); if (code != 0) { - sInfo("%s, file are modified while retrieve, lastver:%" PRIu64, pPeer->id, pPeer->lastVer); - return code; + sInfo("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastVer); + pPeer->fileChanged = 1; + return TSDB_CODE_SYN_VND_COMMITING; } if (fver != pPeer->lastVer) { - sInfo("%s, file are modified while retrieve, fver:%" PRIu64 " lastver:%" PRIu64, pPeer->id, fver, pPeer->lastVer); - return -1; + sInfo("%s, files are modified while retrieve, fver:%" PRIu64 ", last fver:%" PRIu64, pPeer->id, fver, pPeer->lastVer); + pPeer->fileChanged = 1; + return TSDB_CODE_SYN_FILE_CHNAGED; } - return 0; + pPeer->fileChanged = 0; + return TSDB_CODE_SUCCESS; } static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo fileInfo = {0}; SFileAck fileAck = {0}; - int32_t code = -1; + int32_t code = TSDB_CODE_SYN_APP_ERROR; char name[TSDB_FILENAME_LEN * 2] = {0}; if (pNode->getFileVersion) (*pNode->getFileVersion)(pNode->vgId, &pPeer->lastVer); @@ -58,25 +62,27 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); // fileInfo.size = htonl(size); - sDebug("%s, file:%s info will be sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); + sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size); // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } // if no file anymore, break if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { + code = TSDB_CODE_SUCCESS; sDebug("%s, no more files to sync", pPeer->id); - code = 0; break; } // wait for the ack from peer ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -97,6 +103,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // send the file to peer int32_t sfd = open(name, O_RDONLY); if (sfd < 0) { + code = TAOS_SYSTEM_ERROR(errno); sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -104,6 +111,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); if (ret < 0) { + code = TAOS_SYSTEM_ERROR(errno); sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -112,11 +120,12 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.index++; // check if processed files are modified - if (syncAreFilesModified(pNode, pPeer) != 0) break; + code = syncAreFilesModified(pNode, pPeer); + if (code != TSDB_CODE_SUCCESS) break; } - if (code < 0) { - sError("%s, failed to retrieve file", pPeer->id); + if (code != TSDB_CODE_SUCCESS) { + sError("%s, failed to retrieve file since %s", pPeer->id, tstrerror(code)); } return code; @@ -368,8 +377,6 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { close(sfd); if (code < 0) break; - index++; - if (syncAreFilesModified(pNode, pPeer) != 0) break; } @@ -445,7 +452,6 @@ void *syncRetrieveData(void *param) { if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves); - pPeer->fileChanged = 0; pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0); if (pPeer->syncFd < 0) { sError("%s, failed to open socket to sync", pPeer->id); From 7296733b47ca10cc420727e3b47e115e817b7939 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 18:03:17 +0800 Subject: [PATCH 077/188] compile error in arm32 --- src/sync/src/syncRestore.c | 4 ++-- src/sync/src/syncRetrieve.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sync/src/syncRestore.c b/src/sync/src/syncRestore.c index ada55aaf8f..d156c93865 100644 --- a/src/sync/src/syncRestore.c +++ b/src/sync/src/syncRestore.c @@ -52,8 +52,8 @@ static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo minfo = {0}; - SFileInfo sinfo = {0}; + SFileInfo minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */ + SFileInfo sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */ SFileAck fileAck = {0}; int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 4559285a1f..305ec492b9 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -49,7 +49,7 @@ static int32_t syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; - SFileInfo fileInfo = {0}; + SFileInfo fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo)); SFileAck fileAck = {0}; int32_t code = TSDB_CODE_SYN_APP_ERROR; char name[TSDB_FILENAME_LEN * 2] = {0}; From 36b47a79a44f58f9ccf4dd2a1d3c6e01ceff2ffa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 27 Nov 2020 18:04:17 +0800 Subject: [PATCH 078/188] [TD-225]update sim test script. --- tests/script/general/parser/limit_tb.sim | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/script/general/parser/limit_tb.sim b/tests/script/general/parser/limit_tb.sim index b917627fdf..45f5541208 100644 --- a/tests/script/general/parser/limit_tb.sim +++ b/tests/script/general/parser/limit_tb.sim @@ -327,22 +327,22 @@ sql select twa(c1), twa(c2), twa(c3), twa(c4), twa(c5), twa(c6) from $tb where t if $rows != 1 then return -1 endi -if $data00 != 4.000000000 then +if $data00 != 4.500000000 then return -1 endi -if $data01 != 4.000000000 then +if $data01 != 4.500000000 then return -1 endi -if $data02 != 4.000000000 then +if $data02 != 4.500000000 then return -1 endi -if $data03 != 4.000000000 then +if $data03 != 4.500000000 then return -1 endi -if $data04 != 4.000000000 then +if $data04 != 4.500000000 then return -1 endi -if $data05 != 4.000000000 then +if $data05 != 4.500000000 then return -1 endi @@ -690,13 +690,13 @@ sql select twa(c1), twa(c2), twa(c3), twa(c4), twa(c5), twa(c6) from $tb where t if $rows != 1 then return -1 endi -if $data00 != 4.000000000 then +if $data00 != 4.500000000 then return -1 endi -if $data02 != 4.000000000 then +if $data02 != 4.500000000 then return -1 endi -if $data05 != 4.000000000 then +if $data05 != 4.500000000 then return -1 endi From 0e98a4eadba66eafda9abbfe3f2a0e6e40f0ab6e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 27 Nov 2020 18:07:33 +0800 Subject: [PATCH 079/188] [TD-2129]: fix bug in twa calculation. --- src/client/src/tscFunctionImpl.c | 60 ++++-- src/query/inc/qExecutor.h | 11 +- src/query/inc/tsqlfunction.h | 19 +- src/query/src/qExecutor.c | 318 +++++++++++++++++++++++++++---- src/query/src/qUtil.c | 3 +- 5 files changed, 343 insertions(+), 68 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 2ed476f063..0f5f4cb89a 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -64,13 +64,13 @@ } \ } while (0); -#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ -do {\ -for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ - aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ - } \ -} while(0); +#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ + do { \ + for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ + SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ + aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ + } \ + } while (0); void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {} @@ -3625,11 +3625,10 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { } SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); - - pInfo->lastKey = INT64_MIN; - pInfo->type = pCtx->inputType; - + + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + pInfo->lastKey = INT64_MIN; + pInfo->win = TSWINDOW_INITIALIZER; return true; } @@ -3640,10 +3639,24 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - if (pInfo->lastKey == INT64_MIN) { - pInfo->lastKey = pCtx->nStartQueryTimestamp; + + if (pCtx->start.key != INT64_MIN) { + assert(pCtx->start.key < primaryKey[index] && pInfo->lastKey == INT64_MIN); + + pInfo->lastKey = primaryKey[index]; GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); + + pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (primaryKey[index] - pCtx->start.key); + pInfo->hasResult = DATA_SET_FLAG; + pInfo->win.skey = pCtx->start.key; + notNullElems++; + } else if (pInfo->lastKey == INT64_MIN) { + pInfo->lastKey = primaryKey[index]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); + + pInfo->hasResult = DATA_SET_FLAG; + pInfo->win.skey = pInfo->lastKey; notNullElems++; } @@ -3732,6 +3745,14 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si default: assert(0); } + // the last interpolated time window value + if (pCtx->end.key != INT64_MIN) { + pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); + pInfo->lastValue = pCtx->end.val; + pInfo->lastKey = pCtx->end.key; + } + + pInfo->win.ekey = pInfo->lastKey; return notNullElems; } @@ -3751,7 +3772,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { return; } - int32_t notNullElems = twa_function_impl(pCtx, 0, pCtx->size); + int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, pCtx->size); SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3795,8 +3816,7 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { numOfNotNull++; pBuf->dOutput += pInput->dOutput; - pBuf->SKey = pInput->SKey; - pBuf->EKey = pInput->EKey; + pBuf->win = pInput->win; pBuf->lastKey = pInput->lastKey; } @@ -3824,17 +3844,17 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); - assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); + assert(pInfo->win.ekey == pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); if (pInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - if (pInfo->SKey == pInfo->EKey) { + if (pInfo->win.ekey == pInfo->win.skey) { *(double *)pCtx->aOutputBuf = pInfo->lastValue; } else { - *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->EKey - pInfo->SKey); + *(double *)pCtx->aOutputBuf = pInfo->dOutput / (pInfo->win.ekey - pInfo->win.skey); } GET_RES_INFO(pCtx)->numOfRes = 1; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 895b414a56..71fc01923a 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -63,9 +63,11 @@ typedef struct SSqlGroupbyExpr { typedef struct SResultRow { int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer - int32_t rowId:15; - bool closed:1; // this result status: closed or opened - uint16_t numOfRows; // number of rows of current time window + int32_t rowId:29; // row index in buffer page + bool startInterp; // the time window start timestamp has done the interpolation already. + bool endInterp; // the time window end timestamp has done the interpolation already. + bool closed; // this result status: closed or opened + uint32_t numOfRows; // number of rows of current time window SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo union {STimeWindow win; char* key;}; // start key of current time window } SResultRow; @@ -187,6 +189,7 @@ typedef struct SQueryRuntimeEnv { bool topBotQuery; // false bool groupbyNormalCol; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not + bool timeWindowInterpo;// if the time window start/end required interpolation int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file @@ -195,6 +198,8 @@ typedef struct SQueryRuntimeEnv { SResultRowPool* pool; // window result object pool int32_t* rowCellInfoOffset;// offset value for each row result cell info + char** prevRow; + char** nextRow; } SQueryRuntimeEnv; enum { diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 43f78a56ef..5a923db52c 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -152,6 +152,11 @@ typedef struct SResultRowCellInfo { uint32_t numOfRes; // num of output result in current buffer } SResultRowCellInfo; +typedef struct SPoint1 { + int64_t key; + double val; +} SPoint1; + #define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) struct SQLFunctionCtx; @@ -194,6 +199,8 @@ typedef struct SQLFunctionCtx { SResultRowCellInfo *resultInfo; SExtTagsInfo tagInfo; + SPoint1 start; + SPoint1 end; } SQLFunctionCtx; typedef struct SQLAggFuncElem { @@ -243,13 +250,11 @@ enum { }; typedef struct STwaInfo { - TSKEY lastKey; - int8_t hasResult; // flag to denote has value - int16_t type; // source data type - TSKEY SKey; - TSKEY EKey; - double dOutput; - double lastValue; + TSKEY lastKey; + int8_t hasResult; // flag to denote has value + double dOutput; + double lastValue; + STimeWindow win; } STwaInfo; /* global sql function array */ diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 949616bb43..fd52f6ec59 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -27,6 +27,7 @@ #include "query.h" #include "queryLog.h" #include "tlosertree.h" +#include "ttype.h" #define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1) @@ -194,6 +195,7 @@ static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo * static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); +static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -400,6 +402,17 @@ static bool isTopBottomQuery(SQuery *pQuery) { return false; } +static bool timeWindowInterpoRequired(SQuery *pQuery) { + for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + int32_t functionId = pQuery->pExpr1[i].base.functionId; + if (functionId == TSDB_FUNC_TWA) { + return true; + } + } + + return false; +} + static bool hasTagValOutput(SQuery* pQuery) { SExprInfo *pExprInfo = &pQuery->pExpr1[0]; if (pQuery->numOfOutput == 1 && pExprInfo->base.functionId == TSDB_FUNC_TS_COMP) { @@ -596,7 +609,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf } static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, - STimeWindow *win, bool masterscan, bool* newWind) { + STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -604,6 +617,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes if (pResultRow == NULL) { *newWind = false; + // no master scan, no result generated means error occurs return masterscan? -1:0; } @@ -619,15 +633,40 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes // set time window for current result pResultRow->win = (*win); + *pResult = pResultRow; setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow); + return TSDB_CODE_SUCCESS; } -static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { +static bool getResultRowStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); return pWindowResInfo->pResult[slot]->closed; } +typedef enum SResultTsInterpType { + RESULT_ROW_START_INTERP = 1, + RESULT_ROW_END_INTERP = 2, +} SResultTsInterpType; + +static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + pResult->startInterp = true; + } else { + pResult->endInterp = true; + } +} + +static bool isResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { + assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); + if (type == RESULT_ROW_START_INTERP) { + return pResult->startInterp == true; + } else { + return pResult->endInterp == true; + } +} + static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, int16_t order, int64_t *pData) { int32_t forwardStep = 0; @@ -991,6 +1030,113 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas return dataBlock; } +// window start key interpolation +static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + TSKEY start = tsCols[pos]; + TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; + TSKEY prevTs = (pos == 0)? lastTs : tsCols[pos - 1]; + + // if lastTs == INT64_MIN, it is the first block, no need to do the start time interpolation + if (((lastTs != INT64_MIN && pos >= 0) || (lastTs == INT64_MIN && pos > 0)) && win->skey > lastTs && + win->skey < start) { + + for (int32_t k = 0; k < pQuery->numOfCols; ++k) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); + if (k == 0 && pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + assert(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); + continue; + } + + double v1 = 0, v2 = 0, v = 0; + + char *prevVal = pos == 0 ? pRuntimeEnv->prevRow[k] : pColInfo->pData + (pos - 1) * pColInfo->info.bytes; + + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal); + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes); + + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = start, .val = &v2}; + SPoint point = (SPoint){.key = win->skey, .val = &v}; + taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); + pRuntimeEnv->pCtx[k].start.key = point.key; + pRuntimeEnv->pCtx[k].start.val = v; + } + + return true; + } else { + for (int32_t k = 0; k < pQuery->numOfCols; ++k) { + pRuntimeEnv->pCtx[k].start.key = INT64_MIN; + } + + return false; + } +} + +static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, SArray* pDataBlock, TSKEY* tsCols, TSKEY ekey, STimeWindow* win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + TSKEY trueEndKey = tsCols[pos]; + + if (win->ekey < ekey && win->ekey != trueEndKey) { + int32_t nextIndex = pos + 1; + TSKEY next = tsCols[nextIndex]; + + for (int32_t k = 0; k < pQuery->numOfCols; ++k) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); + if (k == 0 && pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP && + pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + continue; + } + + double v1 = 0, v2 = 0, v = 0; + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes); + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + nextIndex * pColInfo->info.bytes); + + SPoint point1 = (SPoint){.key = trueEndKey, .val = &v1}; + SPoint point2 = (SPoint){.key = next, .val = &v2}; + SPoint point = (SPoint){.key = win->ekey, .val = &v}; + taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); + pRuntimeEnv->pCtx[k].end.key = point.key; + pRuntimeEnv->pCtx[k].end.val = v; + } + + return true; + } else { // current time window does not ended in current data block, do nothing + for (int32_t k = 0; k < pQuery->numOfCols; ++k) { + pRuntimeEnv->pCtx[k].end.key = INT64_MIN; + } + + return false; + } +} + +static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock) { + if (pDataBlock == NULL) { + return; + } + + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t k = 0; k < pQuery->numOfCols; ++k) { + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); + memcpy(pRuntimeEnv->prevRow[k], pColInfo->pData + (pColInfo->info.bytes * (pDataBlockInfo->rows - 1)), + pColInfo->info.bytes); + } +} + +static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY* tsCols, int32_t step) { + TSKEY ts = TSKEY_INITIAL_VAL; + + if (tsCols == NULL) { + ts = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.skey : pDataBlockInfo->window.ekey; + } else { + int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); + ts = tsCols[offset]; + } + + return ts; +} + /** * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions * @param pRuntimeEnv @@ -1004,12 +1150,12 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; TSKEY *tsCols = NULL; if (pDataBlock != NULL) { - SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, 0); + SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0); tsCols = (TSKEY *)(pColInfo->pData); } @@ -1018,7 +1164,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); + SQInfo *pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); @@ -1026,18 +1172,13 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - TSKEY ts = TSKEY_INITIAL_VAL; + TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); - if (tsCols == NULL) { - ts = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.skey:pDataBlockInfo->window.ekey; - } else { - int32_t offset = GET_COL_DATA_POS(pQuery, 0, step); - ts = tsCols[offset]; - } - - bool hasTimeWindow = false; + bool hasTimeWindow = false; + SResultRow* pResult = NULL; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); + if (ret != TSDB_CODE_SUCCESS) { tfree(sasArray); return; } @@ -1050,7 +1191,27 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - bool pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + // window start key interpolation + if (pRuntimeEnv->timeWindowInterpo) { + bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + if (!alreadyInterp) { + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, pQuery->pos, pDataBlock, tsCols, &win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } + + alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + if (!alreadyInterp) { + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, pQuery->pos + forwardStep - 1, pDataBlock, tsCols, + pDataBlockInfo->window.ekey, &win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } + } + + bool pStatus = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } @@ -1066,7 +1227,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * // null data, failed to allocate more memory buffer hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != + TSDB_CODE_SUCCESS) { break; } @@ -1077,7 +1239,26 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); - bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + // window start(end) key interpolation + if (pRuntimeEnv->timeWindowInterpo) { + bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + if (!alreadyInterp) { + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startPos, pDataBlock, tsCols, &nextWin); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } + + alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + if (!alreadyInterp) { + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, startPos + forwardStep - 1, pDataBlock, tsCols, pDataBlockInfo->window.ekey, &nextWin); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } + } + + bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } @@ -1097,7 +1278,11 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * } } - for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { + if (pRuntimeEnv->timeWindowInterpo) { + saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock); + } + + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { if (pQuery->pExpr1[i].base.functionId != TSDB_FUNC_ARITHM) { continue; } @@ -1320,20 +1505,20 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); } - int32_t j = 0; int32_t offset = -1; +// TSKEY prev = -1; - for (j = 0; j < pDataBlockInfo->rows; ++j) { + for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { offset = GET_COL_DATA_POS(pQuery, j, step); if (pRuntimeEnv->pTSBuf != NULL) { - int32_t r = doTSJoinFilter(pRuntimeEnv, offset); - if (r == TS_JOIN_TAG_NOT_EQUALS) { + int32_t ret = doTSJoinFilter(pRuntimeEnv, offset); + if (ret == TS_JOIN_TAG_NOT_EQUALS) { break; - } else if (r == TS_JOIN_TS_NOT_EQUALS) { + } else if (ret == TS_JOIN_TS_NOT_EQUALS) { continue; } else { - assert(r == TS_JOIN_TS_EQUAL); + assert(ret == TS_JOIN_TS_EQUAL); } } @@ -1346,8 +1531,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - bool hasTimeWindow = false; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow); + bool hasTimeWindow = false; + SResultRow* pResult = NULL; + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1355,8 +1541,28 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (!hasTimeWindow) { continue; } +/* + // window start key interpolation + if (pRuntimeEnv->timeWindowInterpo) { + bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + if (!alreadyInterp) { + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, pos, pDataBlock, tsCols, &win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } - bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + if (!alreadyInterp) { + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, pQuery->pos + forwardStep - 1, pDataBlock, tsCols, + pDataBlockInfo->window.ekey, &win); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } + } +*/ + bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); STimeWindow nextWin = win; @@ -1375,12 +1581,32 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // null data, failed to allocate more memory buffer hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != TSDB_CODE_SUCCESS) { break; } if (hasTimeWindow) { - closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); +/* + // window start(end) key interpolation + if (pRuntimeEnv->timeWindowInterpo) { + bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + if (!alreadyInterp) { + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startPos, pDataBlock, tsCols, &nextWin); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } + } + + alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + if (!alreadyInterp) { + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, startPos + forwardStep - 1, pDataBlock, tsCols, pDataBlockInfo->window.ekey, &nextWin); + if (interp) { + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + } + } + } +*/ + closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } } @@ -1405,6 +1631,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } +// prev = tsCols[offset]; + if (pRuntimeEnv->pTSBuf != NULL) { // if timestamp filter list is empty, quit current query if (!tsBufNextPos(pRuntimeEnv->pTSBuf)) { @@ -1530,10 +1758,10 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY * top/bottom values emerge, so does diff function */ if (functionId == TSDB_FUNC_TWA) { - SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); - STwaInfo *pTWAInfo = (STwaInfo*) GET_ROWCELL_INTERBUF(pInfo); - pTWAInfo->SKey = pQuery->window.skey; - pTWAInfo->EKey = pQuery->window.ekey; + pCtx->param[1].i64Key = pQuery->window.skey; + pCtx->param[1].nType = TSDB_DATA_TYPE_BIGINT; + pCtx->param[2].i64Key = pQuery->window.ekey; + pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; } } else if (functionId == TSDB_FUNC_ARITHM) { @@ -1679,6 +1907,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->functionId = pSqlFuncMsg->functionId; pCtx->stableQuery = pRuntimeEnv->stableQuery; pCtx->interBufBytes = pQuery->pExpr1[i].interBytes; + pCtx->start.key = INT64_MIN; + pCtx->end.key = INT64_MIN; pCtx->numOfParams = pSqlFuncMsg->numOfParams; for (int32_t j = 0; j < pCtx->numOfParams; ++j) { @@ -1713,6 +1943,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } + *(int64_t*) pRuntimeEnv->prevRow[0] = INT64_MIN; + // if it is group by normal column, do not set output buffer, the output buffer is pResult // fixed output query/multi-output query for normal table if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { @@ -1783,6 +2015,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tfree(pRuntimeEnv->offset); tfree(pRuntimeEnv->keyBuf); tfree(pRuntimeEnv->rowCellInfoOffset); + tfree(pRuntimeEnv->prevRow); taosHashCleanup(pRuntimeEnv->pResultRowHashTable); pRuntimeEnv->pResultRowHashTable = NULL; @@ -2281,12 +2514,14 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pW // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer if (QUERY_IS_INTERVAL_QUERY(pQuery)) { bool hasTimeWindow = false; + SResultRow* pResult = NULL; + bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow) != + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow, &pResult) != TSDB_CODE_SUCCESS) { // todo handle error in set result for timewindow } @@ -3268,7 +3503,7 @@ static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowR SQuery* pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - bool closed = getTimeWindowResStatus(pWindowResInfo, i); + bool closed = getResultRowStatus(pWindowResInfo, i); if (!closed) { continue; } @@ -4619,6 +4854,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->topBotQuery = isTopBottomQuery(pQuery); pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); + pRuntimeEnv->timeWindowInterpo = timeWindowInterpoRequired(pQuery); setScanLimitationByResultBuffer(pQuery); @@ -6449,9 +6685,11 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou goto _cleanup; } + int32_t srcSize = 0; for (int16_t i = 0; i < numOfCols; ++i) { pQuery->colList[i] = pQueryMsg->colList[i]; pQuery->colList[i].filters = tscFilterInfoClone(pQueryMsg->colList[i].filters, pQuery->colList[i].numOfFilters); + srcSize += pQuery->colList[i].bytes; } // calculate the result row size @@ -6520,6 +6758,14 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); pQInfo->runtimeEnv.pool = initResultRowPool(getWindowResultSize(&pQInfo->runtimeEnv)); + pQInfo->runtimeEnv.prevRow = malloc(POINTER_BYTES * pQuery->numOfCols + srcSize); + + char* start = POINTER_BYTES * pQuery->numOfCols + (char*) pQInfo->runtimeEnv.prevRow; + pQInfo->runtimeEnv.prevRow[0] = start; + + for(int32_t i = 1; i < pQuery->numOfCols; ++i) { + pQInfo->runtimeEnv.prevRow[i] = pQInfo->runtimeEnv.prevRow[i - 1] + pQuery->colList[i-1].bytes; + } pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pQInfo->pBuf == NULL) { diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 2b4cdef9b0..d736f2b602 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -389,8 +389,7 @@ uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { } SQuery* pQuery = pRuntimeEnv->pQuery; - if ((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || - pRuntimeEnv->groupbyNormalCol) { + if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) { return 0; } From dbe3b6f8f6663e1fab22b54ee11694b4a1680da0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 23:27:13 +0800 Subject: [PATCH 080/188] TD-1895 --- src/dnode/src/dnodeMain.c | 2 ++ src/inc/twal.h | 5 ++- src/os/inc/osFile.h | 19 ++++------- src/os/src/detail/osFile.c | 2 +- src/util/inc/tfile.h | 21 +++++++----- src/util/src/tfile.c | 68 ++++++++++++++++++++++++++++++-------- src/wal/inc/walInt.h | 2 +- src/wal/src/walMgmt.c | 9 ++--- src/wal/src/walWrite.c | 55 +++++++++++++++--------------- 9 files changed, 113 insertions(+), 70 deletions(-) diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 130be0af20..9f52dbd331 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -19,6 +19,7 @@ #include "tutil.h" #include "tconfig.h" #include "tglobal.h" +#include "tfile.h" #include "twal.h" #include "trpc.h" #include "dnode.h" @@ -55,6 +56,7 @@ typedef struct { } SDnodeComponent; static const SDnodeComponent tsDnodeComponents[] = { + {"tfile", tfInit, tfCleanup}, {"rpc", rpcInit, rpcCleanup}, {"storage", dnodeInitStorage, dnodeCleanupStorage}, {"dnodecfg", dnodeInitCfg, dnodeCleanupCfg}, diff --git a/src/inc/twal.h b/src/inc/twal.h index 8dd3a8a912..1645de77aa 100644 --- a/src/inc/twal.h +++ b/src/inc/twal.h @@ -51,9 +51,8 @@ typedef struct { typedef void * twalh; // WAL HANDLE typedef int32_t FWalWrite(void *ahandle, void *pHead, int32_t qtype, void *pMsg); -int32_t walInit(); -void walCleanUp(); - +int32_t walInit(); +void walCleanUp(); twalh walOpen(char *path, SWalCfg *pCfg); int32_t walAlter(twalh pWal, SWalCfg *pCfg); void walStop(twalh); diff --git a/src/os/inc/osFile.h b/src/os/inc/osFile.h index 62e44d8eb0..c9b3b9cd76 100644 --- a/src/os/inc/osFile.h +++ b/src/os/inc/osFile.h @@ -20,17 +20,6 @@ extern "C" { #endif -#define tread(fd, buf, count) read(fd, buf, count) -#define twrite(fd, buf, count) write(fd, buf, count) -#define tlseek(fd, offset, whence) lseek(fd, offset, whence) -#define tclose(fd) \ - { \ - if (FD_VALID(fd)) { \ - close(fd); \ - fd = FD_INITIALIZER; \ - } \ - } - int64_t taosReadImp(int32_t fd, void *buf, int64_t count); int64_t taosWriteImp(int32_t fd, void *buf, int64_t count); int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence); @@ -39,7 +28,13 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP #define taosRead(fd, buf, count) taosReadImp(fd, buf, count) #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count) #define taosLSeek(fd, offset, whence) taosLSeekImp(fd, offset, whence) -#define taosClose(x) tclose(x) +#define taosClose(fd) \ + { \ + if (FD_VALID(fd)) { \ + close(fd); \ + fd = FD_INITIALIZER; \ + } \ + } // TAOS_OS_FUNC_FILE_SENDIFLE int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size); diff --git a/src/os/src/detail/osFile.c b/src/os/src/detail/osFile.c index 23fc88b8e1..2e6886aa21 100644 --- a/src/os/src/detail/osFile.c +++ b/src/os/src/detail/osFile.c @@ -116,7 +116,7 @@ int64_t taosWriteImp(int32_t fd, void *buf, int64_t n) { } int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) { - return (int64_t)tlseek(fd, (long)offset, whence); + return (int64_t)lseek(fd, (long)offset, whence); } #ifndef TAOS_OS_FUNC_FILE_SENDIFLE diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index 10b7c1df35..b32f297d9c 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -20,23 +20,26 @@ extern "C" { #endif -#include - // init taos file module -int32_t tfinit(); +int32_t tfInit(); // clean up taos file module -void tfcleanup(); +void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused -int64_t tfopen(const char *pathname, int32_t flags); -int64_t tfclose(int64_t tfd); -int64_t tfwrite(int64_t tfd, void *buf, int64_t count); -int64_t tfread(int64_t tfd, void *buf, int64_t count); +int64_t tfOpen(const char *pathname, int32_t flags); +int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode); +int64_t tfClose(int64_t tfd); +int64_t tfWrite(int64_t tfd, void *buf, int64_t count); +int64_t tfRead(int64_t tfd, void *buf, int64_t count); +int32_t tfFsync(int64_t tfd); +bool tfValid(int64_t tfd); +int32_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); +int32_t tfFtruncate(int64_t tfd, int64_t length); #ifdef __cplusplus } #endif -#endif // TDENGINE_TREF_H +#endif // TDENGINE_TFILE_H diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 27ba30fe81..49b28dad6f 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#define _DEFAULT_SOURCE #include "os.h" #include "taoserror.h" #include "tulog.h" @@ -21,40 +22,52 @@ static int32_t tsFileRsetId = -1; -static void taosCloseFile(void *p) { +static void tfCloseFile(void *p) { close((int32_t)(uintptr_t)p); } -int32_t tfinit() { - tsFileRsetId = taosOpenRef(2000, taosCloseFile); - return tsFileRsetId; +int32_t tfInit() { + tsFileRsetId = taosOpenRef(2000, tfCloseFile); + if (tsFileRsetId > 0) { + return 0; + } else { + return -1; + } } -void tfcleanup() { +void tfCleanup() { if (tsFileRsetId >= 0) taosCloseRef(tsFileRsetId); tsFileRsetId = -1; } -int64_t tfopen(const char *pathname, int32_t flags) { - int32_t fd = open(pathname, flags); - +static int64_t tfOpenImp(int32_t fd) { if (fd < 0) { terrno = TAOS_SYSTEM_ERROR(errno); return -1; - } + } - void *p = (void *)(int64_t)fd; + void * p = (void *)(int64_t)fd; int64_t rid = taosAddRef(tsFileRsetId, p); if (rid < 0) close(fd); return rid; } -int64_t tfclose(int64_t tfd) { +int64_t tfOpen(const char *pathname, int32_t flags) { + int32_t fd = open(pathname, flags); + return tfOpenImp(fd); +} + +int64_t tfOpenM(const char *pathname, int32_t flags, mode_t mode) { + int32_t fd = open(pathname, flags, mode); + return tfOpenImp(fd); +} + +int64_t tfClose(int64_t tfd) { return taosRemoveRef(tsFileRsetId, tfd); } -int64_t tfwrite(int64_t tfd, void *buf, int64_t count) { +int64_t tfWrite(int64_t tfd, void *buf, int64_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; @@ -67,7 +80,7 @@ int64_t tfwrite(int64_t tfd, void *buf, int64_t count) { return ret; } -int64_t tfread(int64_t tfd, void *buf, int64_t count) { +int64_t tfRead(int64_t tfd, void *buf, int64_t count) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; @@ -79,3 +92,32 @@ int64_t tfread(int64_t tfd, void *buf, int64_t count) { taosReleaseRef(tsFileRsetId, tfd); return ret; } + +int64_t tfFsync(int64_t tfd) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + return fsync(fd); +} + +bool tfValid(int64_t tfd) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + return p != NULL; +} + +int32_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + return taosLSeek(fd, offset, whence); +} + +int32_t tfFtruncate(int64_t tfd, int64_t length) { + void *p = taosAcquireRef(tsFileRsetId, tfd); + if (p == NULL) return -1; + + int32_t fd = (int32_t)(uintptr_t)p; + return taosFtruncate(fd, length); +} diff --git a/src/wal/inc/walInt.h b/src/wal/inc/walInt.h index b0edabfbd8..06748d885f 100644 --- a/src/wal/inc/walInt.h +++ b/src/wal/inc/walInt.h @@ -44,8 +44,8 @@ typedef struct { uint64_t version; int64_t fileId; int64_t rid; + int64_t tfd; int32_t vgId; - int32_t fd; int32_t keep; int32_t level; int32_t fsyncPeriod; diff --git a/src/wal/src/walMgmt.c b/src/wal/src/walMgmt.c index 36c190be3e..72ea239817 100644 --- a/src/wal/src/walMgmt.c +++ b/src/wal/src/walMgmt.c @@ -17,6 +17,7 @@ #include "os.h" #include "taoserror.h" #include "tref.h" +#include "tfile.h" #include "twal.h" #include "walInt.h" @@ -61,7 +62,7 @@ void *walOpen(char *path, SWalCfg *pCfg) { } pWal->vgId = pCfg->vgId; - pWal->fd = -1; + pWal->tfd = -1; pWal->fileId = -1; pWal->level = pCfg->walLevel; pWal->keep = pCfg->keep; @@ -124,7 +125,7 @@ void walClose(void *handle) { SWal *pWal = handle; pthread_mutex_lock(&pWal->mutex); - taosClose(pWal->fd); + tfClose(pWal->tfd); pthread_mutex_unlock(&pWal->mutex); taosRemoveRef(tsWal.refId, pWal->rid); } @@ -143,7 +144,7 @@ static void walFreeObj(void *wal) { SWal *pWal = wal; wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); - taosClose(pWal->fd); + tfClose(pWal->tfd); pthread_mutex_destroy(&pWal->mutex); tfree(pWal); } @@ -172,7 +173,7 @@ static void walFsyncAll() { while (pWal) { if (walNeedFsync(pWal)) { wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq); - int32_t code = fsync(pWal->fd); + int32_t code = tfFsync(pWal->tfd); if (code != 0) { wError("vgId:%d, file:%s, failed to fsync since %s", pWal->vgId, pWal->name, strerror(code)); } diff --git a/src/wal/src/walWrite.c b/src/wal/src/walWrite.c index 8a532c8164..10e1b4dd61 100644 --- a/src/wal/src/walWrite.c +++ b/src/wal/src/walWrite.c @@ -18,6 +18,7 @@ #include "os.h" #include "taoserror.h" #include "tchecksum.h" +#include "tfile.h" #include "twal.h" #include "walInt.h" @@ -36,8 +37,8 @@ int32_t walRenew(void *handle) { pthread_mutex_lock(&pWal->mutex); - if (pWal->fd >= 0) { - tclose(pWal->fd); + if (tfValid(pWal->tfd)) { + tfClose(pWal->tfd); wDebug("vgId:%d, file:%s, it is closed", pWal->vgId, pWal->name); } @@ -49,9 +50,9 @@ int32_t walRenew(void *handle) { } snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); - pWal->fd = open(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); + pWal->tfd = tfOpenM(pWal->name, O_WRONLY | O_CREAT, S_IRWXU | S_IRWXG | S_IRWXO); - if (pWal->fd < 0) { + if (!tfValid(pWal->tfd)) { code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); } else { @@ -67,7 +68,7 @@ void walRemoveOneOldFile(void *handle) { SWal *pWal = handle; if (pWal == NULL) return; if (pWal->keep == TAOS_WAL_KEEP) return; - if (pWal->fd <= 0) return; + if (!tfValid(pWal->tfd)) return; pthread_mutex_lock(&pWal->mutex); @@ -113,7 +114,7 @@ int32_t walWrite(void *handle, SWalHead *pHead) { int32_t code = 0; // no wal - if (pWal->fd <= 0) return 0; + if (!tfValid(pWal->tfd)) return 0; if (pWal->level == TAOS_WAL_NOLOG) return 0; if (pHead->version <= pWal->version) return 0; @@ -123,12 +124,12 @@ int32_t walWrite(void *handle, SWalHead *pHead) { pthread_mutex_lock(&pWal->mutex); - if (taosWrite(pWal->fd, pHead, contLen) != contLen) { + if (tfWrite(pWal->tfd, pHead, contLen) != contLen) { code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%s, failed to write since %s", pWal->vgId, pWal->name, strerror(errno)); } else { - wTrace("vgId:%d, write wal, fileId:%" PRId64 " fd:%d hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId, - pWal->fileId, pWal->fd, pHead->version, pWal->version, pHead->len); + wTrace("vgId:%d, write wal, fileId:%" PRId64 " tfd:%" PRId64 " hver:%" PRId64 " wver:%" PRIu64 " len:%d", pWal->vgId, + pWal->fileId, pWal->tfd, pHead->version, pWal->version, pHead->len); pWal->version = pHead->version; } @@ -141,11 +142,11 @@ int32_t walWrite(void *handle, SWalHead *pHead) { void walFsync(void *handle, bool forceFsync) { SWal *pWal = handle; - if (pWal == NULL || pWal->fd < 0) return; + if (pWal == NULL || !tfValid(pWal->tfd)) return; if (forceFsync || (pWal->level == TAOS_WAL_FSYNC && pWal->fsyncPeriod == 0)) { wTrace("vgId:%d, fileId:%" PRId64 ", do fsync", pWal->vgId, pWal->fileId); - if (fsync(pWal->fd) < 0) { + if (tfFsync(pWal->tfd) < 0) { wError("vgId:%d, fileId:%" PRId64 ", fsync failed since %s", pWal->vgId, pWal->fileId, strerror(errno)); } } @@ -186,8 +187,8 @@ int32_t walRestore(void *handle, void *pVnode, FWalWrite writeFp) { // open the existing WAL file in append mode pWal->fileId = 0; snprintf(pWal->name, sizeof(pWal->name), "%s/%s%" PRId64, pWal->path, WAL_PREFIX, pWal->fileId); - pWal->fd = open(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); - if (pWal->fd < 0) { + pWal->tfd = tfOpenM(pWal->name, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU | S_IRWXG | S_IRWXO); + if (!tfValid(pWal->tfd)) { wError("vgId:%d, file:%s, failed to open since %s", pWal->vgId, pWal->name, strerror(errno)); return TAOS_SYSTEM_ERROR(errno); } @@ -217,22 +218,22 @@ int32_t walGetWalFile(void *handle, char *fileName, int64_t *fileId) { return code; } -static void walFtruncate(SWal *pWal, int32_t fd, int64_t offset) { - taosFtruncate(fd, offset); - fsync(fd); +static void walFtruncate(SWal *pWal, int64_t tfd, int64_t offset) { + tfFtruncate(tfd, offset); + tfFsync(tfd); } -static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int32_t fd, int64_t *offset) { +static int32_t walSkipCorruptedRecord(SWal *pWal, SWalHead *pHead, int64_t tfd, int64_t *offset) { int64_t pos = *offset; while (1) { pos++; - if (lseek(fd, pos, SEEK_SET) < 0) { + if (tfLseek(tfd, pos, SEEK_SET) < 0) { wError("vgId:%d, failed to seek from corrupted wal file since %s", pWal->vgId, strerror(errno)); return TSDB_CODE_WAL_FILE_CORRUPTED; } - if (taosRead(fd, pHead, sizeof(SWalHead)) <= 0) { + if (tfRead(tfd, pHead, sizeof(SWalHead)) <= 0) { wError("vgId:%d, read to end of corrupted wal file, offset:%" PRId64, pWal->vgId, pos); return TSDB_CODE_WAL_FILE_CORRUPTED; } @@ -259,8 +260,8 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch return TAOS_SYSTEM_ERROR(errno); } - int32_t fd = open(name, O_RDWR); - if (fd < 0) { + int64_t tfd = tfOpen(name, O_RDWR); + if (!tfValid(tfd)) { wError("vgId:%d, file:%s, failed to open for restore since %s", pWal->vgId, name, strerror(errno)); tfree(buffer); return TAOS_SYSTEM_ERROR(errno); @@ -273,7 +274,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch SWalHead *pHead = buffer; while (1) { - int32_t ret = taosRead(fd, pHead, sizeof(SWalHead)); + int32_t ret = tfRead(tfd, pHead, sizeof(SWalHead)); if (ret == 0) break; if (ret < 0) { @@ -284,16 +285,16 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch if (ret < sizeof(SWalHead)) { wError("vgId:%d, file:%s, failed to read wal head, ret is %d", pWal->vgId, name, ret); - walFtruncate(pWal, fd, offset); + walFtruncate(pWal, tfd, offset); break; } if (!taosCheckChecksumWhole((uint8_t *)pHead, sizeof(SWalHead))) { wError("vgId:%d, file:%s, wal head cksum is messed up, hver:%" PRIu64 " len:%d offset:%" PRId64, pWal->vgId, name, pHead->version, pHead->len, offset); - code = walSkipCorruptedRecord(pWal, pHead, fd, &offset); + code = walSkipCorruptedRecord(pWal, pHead, tfd, &offset); if (code != TSDB_CODE_SUCCESS) { - walFtruncate(pWal, fd, offset); + walFtruncate(pWal, tfd, offset); break; } } @@ -310,7 +311,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch pHead = buffer; } - ret = taosRead(fd, pHead->cont, pHead->len); + ret = tfRead(tfd, pHead->cont, pHead->len); if (ret < 0) { wError("vgId:%d, file:%s, failed to read wal body since %s", pWal->vgId, name, strerror(errno)); code = TAOS_SYSTEM_ERROR(errno); @@ -332,7 +333,7 @@ static int32_t walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp, ch (*writeFp)(pVnode, pHead, TAOS_QTYPE_WAL, NULL); } - tclose(fd); + tfClose(tfd); tfree(buffer); return code; From 754ad5ff5b6dae0bb0e85d44c67ff9febe00c63f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 23:38:07 +0800 Subject: [PATCH 081/188] compile error in windows --- src/util/inc/tfile.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/inc/tfile.h b/src/util/inc/tfile.h index b32f297d9c..066040170e 100644 --- a/src/util/inc/tfile.h +++ b/src/util/inc/tfile.h @@ -35,7 +35,7 @@ int64_t tfWrite(int64_t tfd, void *buf, int64_t count); int64_t tfRead(int64_t tfd, void *buf, int64_t count); int32_t tfFsync(int64_t tfd); bool tfValid(int64_t tfd); -int32_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); +int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence); int32_t tfFtruncate(int64_t tfd, int64_t length); #ifdef __cplusplus From a2331e5df637295ac558c0ac496a6370e7860e55 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 28 Nov 2020 09:47:07 +0800 Subject: [PATCH 082/188] compile in win64 --- src/util/src/tfile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 49b28dad6f..1fa6a3d096 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -106,7 +106,7 @@ bool tfValid(int64_t tfd) { return p != NULL; } -int32_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { +int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; From c23e4c44adf5901908898db385859ccabd874ba7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 28 Nov 2020 11:16:07 +0800 Subject: [PATCH 083/188] [TD-225]update test sim. --- tests/script/general/parser/col_arithmetic_query.sim | 4 +++- tests/script/general/parser/limit1_tb.sim | 9 +++++---- tests/script/general/parser/where.sim | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 4ed8fd730a..53e2c98b56 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -369,7 +369,9 @@ sql_error select c7-c9 from $tb interval(2y) # multiple retrieve [d.20]=============================================================== sql select c2-c2, 911 from $tb -#======================================= aggregation function arithmetic query cases ================ +#======================================= aggregation function arithmetic query cases =================================== +# on $tb percentile() spread(ts) bug + # asc/desc order [d.2] sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc; if $rows != 1 then diff --git a/tests/script/general/parser/limit1_tb.sim b/tests/script/general/parser/limit1_tb.sim index 1e473eb858..72b63256db 100644 --- a/tests/script/general/parser/limit1_tb.sim +++ b/tests/script/general/parser/limit1_tb.sim @@ -703,13 +703,13 @@ sql select twa(c1), twa(c2), twa(c3), twa(c4), twa(c5), twa(c6) from $tb where t if $rows != 1 then return -1 endi -if $data00 != 4.499549955 then +if $data00 != 4.500000000 then return -1 endi -if $data02 != 4.499549955 then +if $data02 != 4.500000000 then return -1 endi -if $data05 != 4.499549955 then +if $data05 != 4.500000000 then return -1 endi @@ -717,10 +717,12 @@ sql select first(c1), first(c2), first(c3), first(c4), first(c5), first(c6) from if $rows != 0 then return -1 endi + sql select first(c1), first(c2), first(c3), first(c4), first(c5), first(c6) from $tb where ts >= $ts0 and ts <= $tsu interval(30m) limit 3 offset 1 if $rows != 3 then return -1 endi + if $data01 != 3 then return -1 endi @@ -731,7 +733,6 @@ if $data23 != 9.00000 then return -1 endi - sql select last(c1), last(c2), last(c3), last(c4), last(c5), last(c6) from $tb where ts >= $ts0 and ts <= $tsu limit 5 offset 1 if $rows != 0 then return -1 diff --git a/tests/script/general/parser/where.sim b/tests/script/general/parser/where.sim index 8e17220b5b..c5b600b514 100644 --- a/tests/script/general/parser/where.sim +++ b/tests/script/general/parser/where.sim @@ -131,7 +131,6 @@ if $data00 != $rowNum then return -1 endi - ## like sql_error select * from $mt where c1 like 1 #sql_error select * from $mt where t1 like 1 @@ -178,7 +177,8 @@ sql create table wh_mt2_tb1 using wh_mt2 tags ('wh_mt2_tb1') # 2019-01-01 09:00:00.000 1546304400000 # 2019-01-01 09:10:00.000 1546305000000 sql insert into wh_mt2_tb1 values ('2019-01-01 00:00:00.000', '2019-01-01 09:00:00.000', 'binary10', 'nchar10') -sql insert into wh_mt2_tb1 values ('2019-01-01 00:10:00.000', '2019-01-01 09:10:00.000', 'binary10', 'nchar10') +sql insert into wh_mt2_tb1 values ('2019-01-01 00:10:00.000', '2019-01-01 09:10:00.000', 'binary10', 'nchar10') + sql select * from wh_mt2_tb1 where c1 > 1546304400000 if $rows != 1 then return -1 From 27a3cff1c53978e04a713579e0a60a7a75b92563 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 28 Nov 2020 11:17:22 +0800 Subject: [PATCH 084/188] [TD-2129] --- src/client/src/tscFunctionImpl.c | 23 +++++++++++++---------- src/client/src/tscSQLParser.c | 5 +++-- src/query/src/qExecutor.c | 4 +++- src/query/src/qUtil.c | 2 +- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 0f5f4cb89a..47dc765336 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3639,6 +3639,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + int32_t i = index; if (pCtx->start.key != INT64_MIN) { assert(pCtx->start.key < primaryKey[index] && pInfo->lastKey == INT64_MIN); @@ -3646,27 +3647,27 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si pInfo->lastKey = primaryKey[index]; GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); - pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (primaryKey[index] - pCtx->start.key); + pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); pInfo->hasResult = DATA_SET_FLAG; pInfo->win.skey = pCtx->start.key; notNullElems++; + i += 1; } else if (pInfo->lastKey == INT64_MIN) { pInfo->lastKey = primaryKey[index]; GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pInfo->lastKey; + pInfo->win.skey = pInfo->lastKey; notNullElems++; + i += 1; } - int32_t i = index; - // calculate the value of switch(pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: { int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (++i; i < size; i++) { + for (; i < size; i++) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } @@ -3679,7 +3680,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si } case TSDB_DATA_TYPE_SMALLINT: { int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (++i; i < size; i++) { + for (; i < size; i++) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } @@ -3692,12 +3693,14 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si } case TSDB_DATA_TYPE_INT: { int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (++i; i < size; i++) { + for (; i < size; i++) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); +// printf("%ld, %f, val:%f\n", pInfo->lastKey, pInfo->lastValue, pInfo->dOutput); + pInfo->lastValue = val[i]; pInfo->lastKey = primaryKey[i]; } @@ -3705,7 +3708,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si } case TSDB_DATA_TYPE_BIGINT: { int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (++i; i < size; i++) { + for (; i < size; i++) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } @@ -3718,7 +3721,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si } case TSDB_DATA_TYPE_FLOAT: { float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (++i; i < size; i++) { + for (; i < size; i++) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } @@ -3731,7 +3734,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si } case TSDB_DATA_TYPE_DOUBLE: { double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (++i; i < size; i++) { + for (; i < size; i++) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 743cb42eb3..309d354849 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -4248,7 +4248,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE tExprTreeDestroy(&p, NULL); taosArrayDestroy(colList); - if (taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); } } @@ -4256,6 +4256,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE pCondExpr->pTagCond = NULL; return ret; } + int32_t parseWhereClause(SQueryInfo* pQueryInfo, tSQLExpr** pExpr, SSqlObj* pSql) { if (pExpr == NULL) { return TSDB_CODE_SUCCESS; @@ -6648,7 +6649,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS return TSDB_CODE_SUCCESS; } else { - return TSDB_CODE_TSC_INVALID_SQL; + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "not support filter expression"); } } else { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fd52f6ec59..7e63edc83c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -613,7 +613,9 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid); + // todo refactor + int64_t uid = getResultInfoUId(pRuntimeEnv); + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, uid); if (pResultRow == NULL) { *newWind = false; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index d736f2b602..d93d02cc0f 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -393,6 +393,6 @@ uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { return 0; } - STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current); + STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); return id->uid; } \ No newline at end of file From 3a02c6b3addf4dcd2665c5d22623c5163c4edb99 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 28 Nov 2020 11:23:08 +0800 Subject: [PATCH 085/188] [TD-225]fix compiler error. --- src/query/src/qExecutor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 712b6fa051..ec70280c7f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1053,7 +1053,7 @@ static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int double v1 = 0, v2 = 0, v = 0; - char *prevVal = pos == 0 ? pRuntimeEnv->prevRow[k] : pColInfo->pData + (pos - 1) * pColInfo->info.bytes; + char *prevVal = pos == 0 ? pRuntimeEnv->prevRow[k] : ((char*)pColInfo->pData) + (pos - 1) * pColInfo->info.bytes; GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal); GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes); @@ -1121,7 +1121,7 @@ static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* SQuery* pQuery = pRuntimeEnv->pQuery; for (int32_t k = 0; k < pQuery->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRuntimeEnv->prevRow[k], pColInfo->pData + (pColInfo->info.bytes * (pDataBlockInfo->rows - 1)), + memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * (pDataBlockInfo->rows - 1)), pColInfo->info.bytes); } } From ab1a8746e19aca94c3aa39ed80e5acde5a9f2822 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 28 Nov 2020 11:28:23 +0800 Subject: [PATCH 086/188] [TD-225]fix compiler error. --- src/client/src/tscFunctionImpl.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 47dc765336..3867032d46 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -3699,8 +3699,6 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si } pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); -// printf("%ld, %f, val:%f\n", pInfo->lastKey, pInfo->lastValue, pInfo->dOutput); - pInfo->lastValue = val[i]; pInfo->lastKey = primaryKey[i]; } @@ -3714,7 +3712,7 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si } pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); - pInfo->lastValue = val[i]; + pInfo->lastValue = (double) val[i]; pInfo->lastKey = primaryKey[i]; } break; @@ -3750,9 +3748,9 @@ static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t si // the last interpolated time window value if (pCtx->end.key != INT64_MIN) { - pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); + pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); pInfo->lastValue = pCtx->end.val; - pInfo->lastKey = pCtx->end.key; + pInfo->lastKey = pCtx->end.key; } pInfo->win.ekey = pInfo->lastKey; From 521ae131a3ee107b4036bff769d51b5633811ff7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 28 Nov 2020 15:27:57 +0800 Subject: [PATCH 087/188] TD-2266 --- src/dnode/src/dnodeMgmt.c | 4 ++-- src/mnode/src/mnodeMnode.c | 18 ++++++++++++++++++ src/mnode/src/mnodePeer.c | 12 ++---------- src/mnode/src/mnodeRead.c | 15 +++------------ src/mnode/src/mnodeSdb.c | 9 +++++---- src/mnode/src/mnodeWrite.c | 14 ++------------ 6 files changed, 32 insertions(+), 40 deletions(-) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index da1852e05e..0d1a56cfc4 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -444,12 +444,12 @@ static int32_t dnodeProcessCreateMnodeMsg(SRpcMsg *pMsg) { SCreateMnodeMsg *pCfg = pMsg->pCont; pCfg->dnodeId = htonl(pCfg->dnodeId); if (pCfg->dnodeId != dnodeGetDnodeId()) { - dError("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); + dDebug("dnodeId:%d, in create mnode msg is not equal with saved dnodeId:%d", pCfg->dnodeId, dnodeGetDnodeId()); return TSDB_CODE_MND_DNODE_ID_NOT_CONFIGURED; } if (strcmp(pCfg->dnodeEp, tsLocalEp) != 0) { - dError("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp); + dDebug("dnodeEp:%s, in create mnode msg is not equal with saved dnodeEp:%s", pCfg->dnodeEp, tsLocalEp); return TSDB_CODE_MND_DNODE_EP_NOT_CONFIGURED; } diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 8e3ad0a248..294b1ced12 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -251,12 +251,30 @@ void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { mnodeMnodeRdLock(); *epSet = tsMnodeEpSetForPeer; mnodeMnodeUnLock(); + + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { + epSet->inUse = (i + 1) % epSet->numOfEps; + mTrace("mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); + } else { + mTrace("mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); + } + } } void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { mnodeMnodeRdLock(); *epSet = tsMnodeEpSetForShell; mnodeMnodeUnLock(); + + for (int32_t i = 0; i < epSet->numOfEps; ++i) { + if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { + epSet->inUse = (i + 1) % epSet->numOfEps; + mTrace("mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); + } else { + mTrace("mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); + } + } } char* mnodeGetMnodeMasterEp() { diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index f13ef75398..ef1b819018 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -58,16 +58,8 @@ int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg) { rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); - mDebug("msg:%p, ahandle:%p type:%s in mpeer queue will be redirected, numOfEps:%d inUse:%d", pMsg, - pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); - for (int32_t i = 0; i < epSet->numOfEps; ++i) { - if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { - epSet->inUse = (i + 1) % epSet->numOfEps; - mDebug("mpeer:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); - } else { - mDebug("mpeer:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); - } - } + mDebug("msg:%p, ahandle:%p type:%s in mpeer queue is redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle, + taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); return TSDB_CODE_RPC_REDIRECT; } diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c index ea7ce783e8..a39d35506f 100644 --- a/src/mnode/src/mnodeRead.c +++ b/src/mnode/src/mnodeRead.c @@ -51,21 +51,12 @@ int32_t mnodeProcessRead(SMnodeMsg *pMsg) { SMnodeRsp *rpcRsp = &pMsg->rpcRsp; SRpcEpSet *epSet = rpcMallocCont(sizeof(SRpcEpSet)); mnodeGetMnodeEpSetForShell(epSet); - - mDebug("msg:%p, app:%p type:%s in mread queue will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle, - taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); - for (int32_t i = 0; i < epSet->numOfEps; ++i) { - if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { - epSet->inUse = (i + 1) % epSet->numOfEps; - mDebug("mnode index:%d ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); - } else { - mDebug("mnode index:%d ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); - } - } - rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); + mDebug("msg:%p, app:%p type:%s in mread queue is redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle, + taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); + return TSDB_CODE_RPC_REDIRECT; } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 44e367244c..6aa17472a1 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -506,7 +506,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) { atomic_add_fetch_32(&pTable->autoIndex, 1); } - sdbDebug("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name, + sdbTrace("vgId:1, sdb:%s, insert key:%s to hash, rowSize:%d rows:%" PRId64 ", msg:%p", pTable->name, sdbGetRowStr(pTable, pRow->pObj), pRow->rowSize, pTable->numOfRows, pRow->pMsg); int32_t code = (*pTable->fpInsert)(pRow); @@ -542,7 +542,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { atomic_sub_fetch_32(&pTable->numOfRows, 1); - sdbDebug("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name, + sdbTrace("vgId:1, sdb:%s, delete key:%s from hash, numOfRows:%" PRId64 ", msg:%p", pTable->name, sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg); sdbDecRef(pTable, pRow->pObj); @@ -551,7 +551,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) { } static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow) { - sdbDebug("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name, + sdbTrace("vgId:1, sdb:%s, update key:%s in hash, numOfRows:%" PRId64 ", msg:%p", pTable->name, sdbGetRowStr(pTable, pRow->pObj), pTable->numOfRows, pRow->pMsg); (*pTable->fpUpdate)(pRow); @@ -649,7 +649,7 @@ static int32_t sdbProcessWrite(void *wparam, void *hparam, int32_t qtype, void * return syncCode; } - sdbDebug("vgId:1, sdb:%s, record from wal/fwd is disposed, action:%s key:%s hver:%" PRIu64, pTable->name, + sdbTrace("vgId:1, sdb:%s, record from %s is disposed, action:%s key:%s hver:%" PRIu64, pTable->name, qtypeStr[qtype], actStr[action], sdbGetKeyStr(pTable, pHead->cont), pHead->version); // even it is WAL/FWD, it shall be called to update version in sync @@ -855,6 +855,7 @@ void sdbCloseTable(void *handle) { taosHashCancelIterate(pTable->iHandle, pIter); taosHashCleanup(pTable->iHandle); + pTable->iHandle = NULL; pthread_mutex_destroy(&pTable->mutex); sdbDebug("vgId:1, sdb:%s, is closed, numOfTables:%d", pTable->name, tsSdbMgmt.numOfTables); diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c index 8893316ffc..f0cfe1aedc 100644 --- a/src/mnode/src/mnodeWrite.c +++ b/src/mnode/src/mnodeWrite.c @@ -54,18 +54,8 @@ int32_t mnodeProcessWrite(SMnodeMsg *pMsg) { rpcRsp->rsp = epSet; rpcRsp->len = sizeof(SRpcEpSet); - mDebug("msg:%p, app:%p type:%s in write queue, will be redirected, numOfEps:%d inUse:%d", pMsg, pMsg->rpcMsg.ahandle, - taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); - for (int32_t i = 0; i < epSet->numOfEps; ++i) { - if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { - epSet->inUse = (i + 1) % epSet->numOfEps; - mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d, set inUse to %d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i], - htons(epSet->port[i]), epSet->inUse); - } else { - mDebug("msg:%p, app:%p mnode index:%d ep:%s:%d", pMsg, pMsg->rpcMsg.ahandle, i, epSet->fqdn[i], - htons(epSet->port[i])); - } - } + mDebug("msg:%p, app:%p type:%s in write queue, is redirected, numOfEps:%d inUse:%d", pMsg, + pMsg->rpcMsg.ahandle, taosMsg[pMsg->rpcMsg.msgType], epSet->numOfEps, epSet->inUse); return TSDB_CODE_RPC_REDIRECT; } From 1eba3f965e76fe3a51a303306e4b50a122963b70 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Sat, 28 Nov 2020 09:57:26 +0000 Subject: [PATCH 088/188] [TD-2268]: cancel all related timers in taosTmrCleanUp --- src/util/src/ttimer.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index 0222a6d80a..baf396f030 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -560,6 +560,37 @@ void taosTmrCleanUp(void* handle) { tmrDebug("%s timer controller is cleaned up.", ctrl->label); ctrl->label[0] = 0; + // cancel all timers of this controller + for (size_t i = 0; i < timerMap.size; i++) { + timer_list_t* list = timerMap.slots + i; + lockTimerList(list); + + tmr_obj_t* t = list->timers; + tmr_obj_t* prev = NULL; + while (t != NULL) { + tmr_obj_t* next = t->mnext; + if (t->ctrl != ctrl) { + prev = t; + t = next; + continue; + } + + uint8_t state = atomic_val_compare_exchange_8(&t->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED); + if (state == TIMER_STATE_WAITING) { + removeFromWheel(t); + } + timerDecRef(t); + if (prev == NULL) { + list->timers = next; + } else { + prev->mnext = next; + } + t = next; + } + + unlockTimerList(list); + } + pthread_mutex_lock(&tmrCtrlMutex); ctrl->next = unusedTmrCtrl; numOfTmrCtrl--; From 213cba6e81d5b82e354f619cce67ce112d8e8fc2 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sat, 28 Nov 2020 18:18:14 +0800 Subject: [PATCH 089/188] [TD-2109] add test case --- .../pytest/insert/insertFromCSVOurofOrder.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 tests/pytest/insert/insertFromCSVOurofOrder.py diff --git a/tests/pytest/insert/insertFromCSVOurofOrder.py b/tests/pytest/insert/insertFromCSVOurofOrder.py new file mode 100644 index 0000000000..d4de85b7e9 --- /dev/null +++ b/tests/pytest/insert/insertFromCSVOurofOrder.py @@ -0,0 +1,71 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql +import time +import datetime +import csv +import random +import pandas as pd + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1500074556514 + + def writeCSV(self): + with open('test3.csv','w', encoding='utf-8', newline='') as csvFile: + writer = csv.writer(csvFile, dialect='excel') + for i in range(1000000): + newTimestamp = self.ts + random.randint(10000000, 10000000000) + random.randint(1000, 10000000) + random.randint(1, 1000) + d = datetime.datetime.fromtimestamp(newTimestamp / 1000) + dt = str(d.strftime("%Y-%m-%d %H:%M:%S.%f")) + writer.writerow(["'%s'" % dt, random.randint(1, 100), random.uniform(1, 100), random.randint(1, 100), random.randint(1, 100)]) + + def removCSVHeader(self): + data = pd.read_csv("ordered.csv") + data = data.drop([0]) + data.to_csv("ordered.csv", header = False, index = False) + + def run(self): + tdSql.prepare() + + tdSql.execute("create table t1(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + startTime = time.time() + tdSql.execute("insert into t1 file 'outoforder.csv'") + duration = time.time() - startTime + print("Out of Order - Insert time: %d" % duration) + tdSql.query("select count(*) from t1") + rows = tdSql.getData(0, 0) + + tdSql.execute("create table t2(ts timestamp, c1 int, c2 float, c3 int, c4 int)") + startTime = time.time() + tdSql.execute("insert into t2 file 'ordered.csv'") + duration = time.time() - startTime + print("Ordered - Insert time: %d" % duration) + tdSql.query("select count(*) from t2") + tdSql.checkData(0,0, rows) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 59cbb734c9039480fdf5dd4fa83b0a12b901d7f0 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Sat, 28 Nov 2020 18:26:36 +0800 Subject: [PATCH 090/188] [TD-2269] install taosdump to /usr/bin --- packaging/tools/install.sh | 2 ++ packaging/tools/install_client.sh | 6 ++++-- packaging/tools/install_client_power.sh | 6 ++++-- packaging/tools/install_power.sh | 2 ++ packaging/tools/post.sh | 2 ++ packaging/tools/remove.sh | 3 ++- packaging/tools/remove_client.sh | 1 + packaging/tools/remove_client_power.sh | 1 + packaging/tools/remove_power.sh | 3 ++- 9 files changed, 20 insertions(+), 6 deletions(-) diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index ddf7114f08..d6dccf7045 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -172,6 +172,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -182,6 +183,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : [ -x ${install_main_dir}/bin/taosd ] && ${csudo} ln -s ${install_main_dir}/bin/taosd ${bin_link_dir}/taosd || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/rmtaos || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : diff --git a/packaging/tools/install_client.sh b/packaging/tools/install_client.sh index 34a9bfaecb..0467300953 100755 --- a/packaging/tools/install_client.sh +++ b/packaging/tools/install_client.sh @@ -84,8 +84,9 @@ function install_main_path() { function install_bin() { # Remove links ${csudo} rm -f ${bin_link_dir}/taos || : - if [ "$osType" == "Darwin" ]; then + if [ "$osType" != "Darwin" ]; then ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : fi ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -94,8 +95,9 @@ function install_bin() { #Make link [ -x ${install_main_dir}/bin/taos ] && ${csudo} ln -s ${install_main_dir}/bin/taos ${bin_link_dir}/taos || : - if [ "$osType" == "Darwin" ]; then + if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo} ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosdump ] && ${csudo} ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : fi [ -x ${install_main_dir}/bin/remove_client.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client.sh ${bin_link_dir}/rmtaos || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : diff --git a/packaging/tools/install_client_power.sh b/packaging/tools/install_client_power.sh index 0108d1d44a..26977e12f4 100755 --- a/packaging/tools/install_client_power.sh +++ b/packaging/tools/install_client_power.sh @@ -84,8 +84,9 @@ function install_main_path() { function install_bin() { # Remove links ${csudo} rm -f ${bin_link_dir}/power || : - if [ "$osType" == "Darwin" ]; then + if [ "$osType" != "Darwin" ]; then ${csudo} rm -f ${bin_link_dir}/powerdemo || : + ${csudo} rm -f ${bin_link_dir}/powerdump || : fi ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -94,8 +95,9 @@ function install_bin() { #Make link [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : - if [ "$osType" == "Darwin" ]; then + if [ "$osType" != "Darwin" ]; then [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : + [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || : fi [ -x ${install_main_dir}/bin/remove_client_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_client_power.sh ${bin_link_dir}/rmpower || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : diff --git a/packaging/tools/install_power.sh b/packaging/tools/install_power.sh index df6291f4ae..5929b52afc 100755 --- a/packaging/tools/install_power.sh +++ b/packaging/tools/install_power.sh @@ -172,6 +172,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerd || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : + ${csudo} rm -f ${bin_link_dir}/powerdump || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -182,6 +183,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/power ] && ${csudo} ln -s ${install_main_dir}/bin/power ${bin_link_dir}/power || : [ -x ${install_main_dir}/bin/powerd ] && ${csudo} ln -s ${install_main_dir}/bin/powerd ${bin_link_dir}/powerd || : [ -x ${install_main_dir}/bin/powerdemo ] && ${csudo} ln -s ${install_main_dir}/bin/powerdemo ${bin_link_dir}/powerdemo || : + [ -x ${install_main_dir}/bin/powerdump ] && ${csudo} ln -s ${install_main_dir}/bin/powerdump ${bin_link_dir}/powerdump || : [ -x ${install_main_dir}/bin/remove_power.sh ] && ${csudo} ln -s ${install_main_dir}/bin/remove_power.sh ${bin_link_dir}/rmpower || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo} ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo} ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : diff --git a/packaging/tools/post.sh b/packaging/tools/post.sh index 00705fad77..52919976ee 100755 --- a/packaging/tools/post.sh +++ b/packaging/tools/post.sh @@ -92,6 +92,7 @@ function install_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -101,6 +102,7 @@ function install_bin() { [ -x ${bin_dir}/taos ] && ${csudo} ln -s ${bin_dir}/taos ${bin_link_dir}/taos || : [ -x ${bin_dir}/taosd ] && ${csudo} ln -s ${bin_dir}/taosd ${bin_link_dir}/taosd || : [ -x ${bin_dir}/taosdemo ] && ${csudo} ln -s ${bin_dir}/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${bin_dir}/taosdump ] && ${csudo} ln -s ${bin_dir}/taosdump ${bin_link_dir}/taosdump || : [ -x ${bin_dir}/set_core.sh ] && ${csudo} ln -s ${bin_dir}/set_core.sh ${bin_link_dir}/set_core || : } diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index e9a4f48cf7..2f2660d446 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -72,6 +72,7 @@ function clean_bin() { ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosd || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -222,4 +223,4 @@ elif echo $osinfo | grep -qwi "centos" ; then fi echo -e "${GREEN}TDengine is removed successfully!${NC}" -echo \ No newline at end of file +echo diff --git a/packaging/tools/remove_client.sh b/packaging/tools/remove_client.sh index 2c28b7b6bf..7579162dc6 100755 --- a/packaging/tools/remove_client.sh +++ b/packaging/tools/remove_client.sh @@ -38,6 +38,7 @@ function clean_bin() { # Remove link ${csudo} rm -f ${bin_link_dir}/taos || : ${csudo} rm -f ${bin_link_dir}/taosdemo || : + ${csudo} rm -f ${bin_link_dir}/taosdump || : ${csudo} rm -f ${bin_link_dir}/rmtaos || : ${csudo} rm -f ${bin_link_dir}/set_core || : } diff --git a/packaging/tools/remove_client_power.sh b/packaging/tools/remove_client_power.sh index 7a3c99e100..580c46e207 100755 --- a/packaging/tools/remove_client_power.sh +++ b/packaging/tools/remove_client_power.sh @@ -38,6 +38,7 @@ function clean_bin() { # Remove link ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : + ${csudo} rm -f ${bin_link_dir}/powerdump || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/set_core || : } diff --git a/packaging/tools/remove_power.sh b/packaging/tools/remove_power.sh index d6d6c5dd7c..816869cf44 100755 --- a/packaging/tools/remove_power.sh +++ b/packaging/tools/remove_power.sh @@ -72,6 +72,7 @@ function clean_bin() { ${csudo} rm -f ${bin_link_dir}/power || : ${csudo} rm -f ${bin_link_dir}/powerd || : ${csudo} rm -f ${bin_link_dir}/powerdemo || : + ${csudo} rm -f ${bin_link_dir}/powerdump || : ${csudo} rm -f ${bin_link_dir}/rmpower || : ${csudo} rm -f ${bin_link_dir}/tarbitrator || : ${csudo} rm -f ${bin_link_dir}/set_core || : @@ -223,4 +224,4 @@ fi #fi echo -e "${GREEN}PowerDB is removed successfully!${NC}" -echo \ No newline at end of file +echo From cb899a2c802ab9fec06774f3a807c16ef4fee86c Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Sat, 28 Nov 2020 18:33:32 +0800 Subject: [PATCH 091/188] Add a insert demo --- tests/pytest/insert/insertDemo.py | 47 +++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 tests/pytest/insert/insertDemo.py diff --git a/tests/pytest/insert/insertDemo.py b/tests/pytest/insert/insertDemo.py new file mode 100644 index 0000000000..d18206e7a4 --- /dev/null +++ b/tests/pytest/insert/insertDemo.py @@ -0,0 +1,47 @@ +import taos +import datetime +import random +import multiprocessing + +def taos_excute(table, connect_host): + conn = taos.connect(host=connect_host, user="root", password="taosdata", config="/etc/taos", database='test') + cursor = conn.cursor() + for i in range(1000000): + pk = random.randint(100001, 300000) + time_now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + col1 = random.randint(1, 10000) + col2 = random.randint(1, 10000) + col3 = random.randint(1, 10000) + col4 = random.randint(1, 10000) + col5 = random.randint(1, 10000) + col6 = random.randint(1, 10000) + sql = f"INSERT INTO {table}_{pk} USING {table} TAGS ({pk}) VALUES ('{time_now}', {col1}, {col2}, {col3}, {col4}, {col5}, {col6})" + cursor.execute(sql) + cursor.close() + conn.close() + +def taos_init(table, connect_host, pk): + conn = taos.connect(host=connect_host, user="root", password="taosdata", config="/etc/taos", database='test') + cursor = conn.cursor() + sql = f"CREATE TABLE {table}_{pk} USING {table} TAGS ({pk})" + cursor.execute(sql) + cursor.close() + conn.close() + +print("init time:", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + +connect_list = ["node1", "node2", "node3", "node4", "node5"] +pool = multiprocessing.Pool(processes=108) + +for pk in range(100001, 300000): + pool.apply_async(func=taos_init, args=("test", connect_list[pk % 5], pk, )) + +print("start time:", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + +for i in range(10000): + pool.apply_async(func=taos_excute, args=("test", connect_list[i % 5],)) + +pool.close() +pool.join() + +print("end time:", datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) \ No newline at end of file From b897d803de9bea68d0aa6fd506f14bb0f87b0177 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 28 Nov 2020 22:08:00 +0800 Subject: [PATCH 092/188] TD-2264 --- src/mnode/inc/mnodeSdb.h | 7 +++++-- src/mnode/src/mnodeAcct.c | 6 ++++-- src/mnode/src/mnodeCluster.c | 8 +++++--- src/mnode/src/mnodeDb.c | 6 ++++-- src/mnode/src/mnodeDnode.c | 6 ++++-- src/mnode/src/mnodeMain.c | 1 + src/mnode/src/mnodeMnode.c | 6 ++++-- src/mnode/src/mnodeProfile.c | 1 - src/mnode/src/mnodeSdb.c | 35 ++++++++++++++++++++++++++++++----- src/mnode/src/mnodeShow.c | 1 - src/mnode/src/mnodeTable.c | 16 ++++++++++------ src/mnode/src/mnodeUser.c | 6 ++++-- src/mnode/src/mnodeVgroup.c | 6 ++++-- 13 files changed, 75 insertions(+), 30 deletions(-) diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 81629ede78..90c4eac40a 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -79,10 +79,13 @@ typedef struct { int32_t (*fpRestored)(); } SSdbTableDesc; +int32_t sdbInitRef(); +void sdbCleanUpRef(); int32_t sdbInit(); void sdbCleanUp(); -void * sdbOpenTable(SSdbTableDesc *desc); -void sdbCloseTable(void *handle); +int64_t sdbOpenTable(SSdbTableDesc *desc); +void sdbCloseTable(int64_t rid); +void* sdbGetTableByRid(int64_t rid); bool sdbIsMaster(); bool sdbIsServing(); void sdbUpdateMnodeRoles(); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 3da889b284..6fba05674f 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -26,6 +26,7 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" +int64_t tsAcctRid = -1; void * tsAcctSdb = NULL; static int32_t tsAcctUpdateSize; static int32_t mnodeCreateRootAcct(); @@ -114,7 +115,8 @@ int32_t mnodeInitAccts() { .fpRestored = mnodeAcctActionRestored }; - tsAcctSdb = sdbOpenTable(&desc); + tsAcctRid = sdbOpenTable(&desc); + tsAcctSdb = sdbGetTableByRid(tsAcctRid); if (tsAcctSdb == NULL) { mError("table:%s, failed to create hash", desc.name); return -1; @@ -126,7 +128,7 @@ int32_t mnodeInitAccts() { void mnodeCleanupAccts() { acctCleanUp(); - sdbCloseTable(tsAcctSdb); + sdbCloseTable(tsAcctRid); tsAcctSdb = NULL; } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 839407c5b0..56229daffa 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -24,6 +24,7 @@ #include "mnodeShow.h" #include "tglobal.h" +int64_t tsClusterRid = -1; static void * tsClusterSdb = NULL; static int32_t tsClusterUpdateSize; static char tsClusterId[TSDB_CLUSTER_ID_LEN]; @@ -101,9 +102,10 @@ int32_t mnodeInitCluster() { .fpRestored = mnodeClusterActionRestored }; - tsClusterSdb = sdbOpenTable(&desc); + tsClusterRid = sdbOpenTable(&desc); + tsClusterSdb = sdbGetTableByRid(tsClusterRid); if (tsClusterSdb == NULL) { - mError("table:%s, failed to create hash", desc.name); + mError("table:%s, rid:%" PRId64 ", failed to create hash", desc.name, tsClusterRid); return -1; } @@ -116,7 +118,7 @@ int32_t mnodeInitCluster() { } void mnodeCleanupCluster() { - sdbCloseTable(tsClusterSdb); + sdbCloseTable(tsClusterRid); tsClusterSdb = NULL; } diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 77f3b93eb1..999c606284 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -38,6 +38,7 @@ #include "mnodeVgroup.h" #define VG_LIST_SIZE 8 +int64_t tsDbRid = -1; static void * tsDbSdb = NULL; static int32_t tsDbUpdateSize; @@ -160,7 +161,8 @@ int32_t mnodeInitDbs() { .fpRestored = mnodeDbActionRestored }; - tsDbSdb = sdbOpenTable(&desc); + tsDbRid = sdbOpenTable(&desc); + tsDbSdb = sdbGetTableByRid(tsDbRid); if (tsDbSdb == NULL) { mError("failed to init db data"); return -1; @@ -496,7 +498,7 @@ void mnodeRemoveVgroupFromDb(SVgObj *pVgroup) { } void mnodeCleanupDbs() { - sdbCloseTable(tsDbSdb); + sdbCloseTable(tsDbRid); tsDbSdb = NULL; } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 0e9b56e459..65f4060392 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -39,6 +39,7 @@ #include "mnodeCluster.h" int32_t tsAccessSquence = 0; +int64_t tsDnodeRid = -1; static void * tsDnodeSdb = NULL; static int32_t tsDnodeUpdateSize = 0; extern void * tsMnodeSdb; @@ -187,7 +188,8 @@ int32_t mnodeInitDnodes() { .fpRestored = mnodeDnodeActionRestored }; - tsDnodeSdb = sdbOpenTable(&desc); + tsDnodeRid = sdbOpenTable(&desc); + tsDnodeSdb = sdbGetTableByRid(tsDnodeRid); if (tsDnodeSdb == NULL) { mError("failed to init dnodes data"); return -1; @@ -213,7 +215,7 @@ int32_t mnodeInitDnodes() { } void mnodeCleanupDnodes() { - sdbCloseTable(tsDnodeSdb); + sdbCloseTable(tsDnodeRid); pthread_mutex_destroy(&tsDnodeEpsMutex); free(tsDnodeEps); tsDnodeEps = NULL; diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 1f5ad42bde..d15b32da54 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -47,6 +47,7 @@ void *tsMnodeTmr = NULL; static bool tsMgmtIsRunning = false; static const SMnodeComponent tsMnodeComponents[] = { + {"sdbref", sdbInitRef, sdbCleanUpRef}, {"profile", mnodeInitProfile, mnodeCleanupProfile}, {"cluster", mnodeInitCluster, mnodeCleanupCluster}, {"accts", mnodeInitAccts, mnodeCleanupAccts}, diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 294b1ced12..7428e8d2c8 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -34,6 +34,7 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" +int64_t tsMnodeRid = -1; static void * tsMnodeSdb = NULL; static int32_t tsMnodeUpdateSize = 0; static SRpcEpSet tsMnodeEpSetForShell; @@ -153,7 +154,8 @@ int32_t mnodeInitMnodes() { .fpRestored = mnodeMnodeActionRestored }; - tsMnodeSdb = sdbOpenTable(&desc); + tsMnodeRid = sdbOpenTable(&desc); + tsMnodeSdb = sdbGetTableByRid(tsMnodeRid); if (tsMnodeSdb == NULL) { mError("failed to init mnodes data"); return -1; @@ -168,7 +170,7 @@ int32_t mnodeInitMnodes() { } void mnodeCleanupMnodes() { - sdbCloseTable(tsMnodeSdb); + sdbCloseTable(tsMnodeRid); tsMnodeSdb = NULL; mnodeMnodeDestroyLock(); } diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 8467adfc17..36b6ff7a59 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -34,7 +34,6 @@ #define QUERY_ID_SIZE 20 #define QUERY_STREAM_SAVE_SIZE 20 -extern void *tsMnodeTmr; static SCacheObj *tsMnodeConnCache = NULL; static int32_t tsConnIndex = 0; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 6aa17472a1..6670e87a7c 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -18,6 +18,7 @@ #include "taoserror.h" #include "hash.h" #include "tutil.h" +#include "tref.h" #include "tbalance.h" #include "tqueue.h" #include "twal.h" @@ -98,6 +99,7 @@ typedef struct { SSdbWorker *worker; } SSdbWorkerPool; +int32_t tsSdbRid; extern void * tsMnodeTmr; static void * tsSdbTmr; static SSdbMgmt tsSdbMgmt = {0}; @@ -118,6 +120,7 @@ static void sdbFreeQueue(); static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow); static int32_t sdbUpdateHash(SSdbTable *pTable, SSdbRow *pRow); static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow); +static void sdbCloseTableObj(void *handle); int32_t sdbGetId(void *pTable) { return ((SSdbTable *)pTable)->autoIndex; @@ -385,6 +388,17 @@ void sdbUpdateSync(void *pMnodes) { sdbUpdateMnodeRoles(); } +int32_t sdbInitRef() { + tsSdbRid = taosOpenRef(10, sdbCloseTableObj); + if (tsSdbRid <= 0) { + sdbError("failed to init sdb ref"); + return -1; + } + return 0; +} + +void sdbCleanUpRef() { taosCloseRef(tsSdbRid); } + int32_t sdbInit() { pthread_mutex_init(&tsSdbMgmt.mutex, NULL); @@ -423,7 +437,7 @@ void sdbCleanUp() { walClose(tsSdbMgmt.wal); tsSdbMgmt.wal = NULL; } - + pthread_mutex_destroy(&tsSdbMgmt.mutex); } @@ -801,10 +815,10 @@ void sdbFreeIter(void *tparam, void *pIter) { taosHashCancelIterate(pTable->iHandle, pIter); } -void *sdbOpenTable(SSdbTableDesc *pDesc) { +int64_t sdbOpenTable(SSdbTableDesc *pDesc) { SSdbTable *pTable = (SSdbTable *)calloc(1, sizeof(SSdbTable)); - if (pTable == NULL) return NULL; + if (pTable == NULL) return -1; pthread_mutex_init(&pTable->mutex, NULL); tstrncpy(pTable->name, pDesc->name, SDB_TABLE_LEN); @@ -829,10 +843,21 @@ void *sdbOpenTable(SSdbTableDesc *pDesc) { tsSdbMgmt.numOfTables++; tsSdbMgmt.tableList[pTable->id] = pTable; - return pTable; + + return taosAddRef(tsSdbRid, pTable); } -void sdbCloseTable(void *handle) { +void sdbCloseTable(int64_t rid) { + taosRemoveRef(tsSdbRid, rid); +} + +void *sdbGetTableByRid(int64_t rid) { + void *handle = taosAcquireRef(tsSdbRid, rid); + taosReleaseRef(tsSdbRid, rid); + return handle; +} + +static void sdbCloseTableObj(void *handle) { SSdbTable *pTable = (SSdbTable *)handle; if (pTable == NULL) return; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 2621441cb5..2da46d5b4b 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -52,7 +52,6 @@ static bool mnodeCheckShowFinished(SShowObj *pShow); static void *mnodePutShowObj(SShowObj *pShow); static void mnodeReleaseShowObj(SShowObj *pShow, bool forceRemove); -extern void *tsMnodeTmr; static void *tsMnodeShowCache = NULL; static int32_t tsShowObjIndex = 0; static SShowMetaFp tsMnodeShowMetaFp[TSDB_MGMT_TABLE_MAX] = {0}; diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index cc7ffb81ad..2149cb12c0 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -49,7 +49,9 @@ #define CREATE_CTABLE_RETRY_TIMES 10 #define CREATE_CTABLE_RETRY_SEC 14 +int64_t tsCTableRid = -1; static void * tsChildTableSdb; +int64_t tsSTableRid = -1; static void * tsSuperTableSdb; static int32_t tsChildTableUpdateSize; static int32_t tsSuperTableUpdateSize; @@ -350,7 +352,7 @@ static int32_t mnodeInitChildTables() { SCTableObj tObj; tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; - SSdbTableDesc tableDesc = { + SSdbTableDesc desc = { .id = SDB_TABLE_CTABLE, .name = "ctables", .hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE, @@ -366,7 +368,8 @@ static int32_t mnodeInitChildTables() { .fpRestored = mnodeChildTableActionRestored }; - tsChildTableSdb = sdbOpenTable(&tableDesc); + tsCTableRid = sdbOpenTable(&desc); + tsChildTableSdb = sdbGetTableByRid(tsCTableRid); if (tsChildTableSdb == NULL) { mError("failed to init child table data"); return -1; @@ -377,7 +380,7 @@ static int32_t mnodeInitChildTables() { } static void mnodeCleanupChildTables() { - sdbCloseTable(tsChildTableSdb); + sdbCloseTable(tsCTableRid); tsChildTableSdb = NULL; } @@ -543,7 +546,7 @@ static int32_t mnodeInitSuperTables() { SSTableObj tObj; tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type; - SSdbTableDesc tableDesc = { + SSdbTableDesc desc = { .id = SDB_TABLE_STABLE, .name = "stables", .hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE, @@ -559,7 +562,8 @@ static int32_t mnodeInitSuperTables() { .fpRestored = mnodeSuperTableActionRestored }; - tsSuperTableSdb = sdbOpenTable(&tableDesc); + tsSTableRid = sdbOpenTable(&desc); + tsSuperTableSdb = sdbGetTableByRid(tsSTableRid); if (tsSuperTableSdb == NULL) { mError("failed to init stables data"); return -1; @@ -570,7 +574,7 @@ static int32_t mnodeInitSuperTables() { } static void mnodeCleanupSuperTables() { - sdbCloseTable(tsSuperTableSdb); + sdbCloseTable(tsSTableRid); tsSuperTableSdb = NULL; } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index aee167631f..fb26086d04 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -33,6 +33,7 @@ #include "mnodeWrite.h" #include "mnodePeer.h" +int64_t tsUserRid = -1; static void * tsUserSdb = NULL; static int32_t tsUserUpdateSize = 0; static int32_t mnodeGetUserMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -165,7 +166,8 @@ int32_t mnodeInitUsers() { .fpRestored = mnodeUserActionRestored }; - tsUserSdb = sdbOpenTable(&desc); + tsUserRid = sdbOpenTable(&desc); + tsUserSdb = sdbGetTableByRid(tsUserRid); if (tsUserSdb == NULL) { mError("table:%s, failed to create hash", desc.name); return -1; @@ -185,7 +187,7 @@ int32_t mnodeInitUsers() { } void mnodeCleanupUsers() { - sdbCloseTable(tsUserSdb); + sdbCloseTable(tsUserRid); tsUserSdb = NULL; } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index a8343083aa..3e974f417f 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -51,6 +51,7 @@ char* vgroupStatus[] = { "updating" }; +int64_t tsVgroupRid = -1; static void *tsVgroupSdb = NULL; static int32_t tsVgUpdateSize = 0; @@ -222,7 +223,8 @@ int32_t mnodeInitVgroups() { .fpRestored = mnodeVgroupActionRestored, }; - tsVgroupSdb = sdbOpenTable(&desc); + tsVgroupRid = sdbOpenTable(&desc); + tsVgroupSdb = sdbGetTableByRid(tsVgroupRid); if (tsVgroupSdb == NULL) { mError("failed to init vgroups data"); return -1; @@ -610,7 +612,7 @@ void mnodeDropVgroup(SVgObj *pVgroup, void *ahandle) { } void mnodeCleanupVgroups() { - sdbCloseTable(tsVgroupSdb); + sdbCloseTable(tsVgroupRid); tsVgroupSdb = NULL; } From 0b869643f27d057e43f72efc0f0d72f43fd9c51c Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 30 Nov 2020 09:58:35 +0800 Subject: [PATCH 093/188] [TD-2260] set default value for some terminal --- src/kit/shell/src/shellDarwin.c | 13 ++++++++++--- src/kit/shell/src/shellLinux.c | 13 ++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index 995b56f341..c6c92e2897 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -407,7 +407,12 @@ void get_history_path(char *history) { sprintf(history, "%s/%s", getpwuid(getuid void clearScreen(int ecmd_pos, int cursor_pos) { struct winsize w; - ioctl(0, TIOCGWINSZ, &w); + if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { + //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); + //exit(EXIT_FAILURE); + w.ws_col = 120; + w.ws_row = 30; + } int cursor_x = cursor_pos / w.ws_col; int cursor_y = cursor_pos % w.ws_col; @@ -425,8 +430,10 @@ void clearScreen(int ecmd_pos, int cursor_pos) { void showOnScreen(Command *cmd) { struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - fprintf(stderr, "No stream device\n"); - exit(EXIT_FAILURE); + //fprintf(stderr, "No stream device\n"); + //exit(EXIT_FAILURE); + w.ws_col = 120; + w.ws_row = 30; } wchar_t wc; diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 04f5824d8d..7480c406c7 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -413,7 +413,12 @@ void get_history_path(char *history) { snprintf(history, TSDB_FILENAME_LEN, "%s/ void clearScreen(int ecmd_pos, int cursor_pos) { struct winsize w; - ioctl(0, TIOCGWINSZ, &w); + if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { + //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); + //exit(EXIT_FAILURE); + w.ws_col = 120; + w.ws_row = 30; + } int cursor_x = cursor_pos / w.ws_col; int cursor_y = cursor_pos % w.ws_col; @@ -431,8 +436,10 @@ void clearScreen(int ecmd_pos, int cursor_pos) { void showOnScreen(Command *cmd) { struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { - fprintf(stderr, "No stream device\n"); - exit(EXIT_FAILURE); + //fprintf(stderr, "No stream device\n"); + //exit(EXIT_FAILURE); + w.ws_col = 120; + w.ws_row = 30; } wchar_t wc; From d50b8e88e9f3274749f2fd5fff9e0558ed072143 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Mon, 30 Nov 2020 10:08:50 +0800 Subject: [PATCH 094/188] [TD-2260] set default value for some terminal --- src/kit/shell/src/shellDarwin.c | 2 -- src/kit/shell/src/shellLinux.c | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/kit/shell/src/shellDarwin.c b/src/kit/shell/src/shellDarwin.c index c6c92e2897..ddf7b21bef 100644 --- a/src/kit/shell/src/shellDarwin.c +++ b/src/kit/shell/src/shellDarwin.c @@ -409,7 +409,6 @@ void clearScreen(int ecmd_pos, int cursor_pos) { struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); - //exit(EXIT_FAILURE); w.ws_col = 120; w.ws_row = 30; } @@ -431,7 +430,6 @@ void showOnScreen(Command *cmd) { struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { //fprintf(stderr, "No stream device\n"); - //exit(EXIT_FAILURE); w.ws_col = 120; w.ws_row = 30; } diff --git a/src/kit/shell/src/shellLinux.c b/src/kit/shell/src/shellLinux.c index 7480c406c7..3226ad830a 100644 --- a/src/kit/shell/src/shellLinux.c +++ b/src/kit/shell/src/shellLinux.c @@ -415,7 +415,6 @@ void clearScreen(int ecmd_pos, int cursor_pos) { struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { //fprintf(stderr, "No stream device, and use default value(col 120, row 30)\n"); - //exit(EXIT_FAILURE); w.ws_col = 120; w.ws_row = 30; } @@ -437,7 +436,6 @@ void showOnScreen(Command *cmd) { struct winsize w; if (ioctl(0, TIOCGWINSZ, &w) < 0 || w.ws_col == 0 || w.ws_row == 0) { //fprintf(stderr, "No stream device\n"); - //exit(EXIT_FAILURE); w.ws_col = 120; w.ws_row = 30; } From dede7188b9bd094282f89e76eb066b003c0aaf6a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 30 Nov 2020 05:10:22 +0000 Subject: [PATCH 095/188] TD-2276 --- src/util/src/tfile.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/util/src/tfile.c b/src/util/src/tfile.c index 1fa6a3d096..64fea2843b 100644 --- a/src/util/src/tfile.c +++ b/src/util/src/tfile.c @@ -93,17 +93,23 @@ int64_t tfRead(int64_t tfd, void *buf, int64_t count) { return ret; } -int64_t tfFsync(int64_t tfd) { +int32_t tfFsync(int64_t tfd) { void *p = taosAcquireRef(tsFileRsetId, tfd); if (p == NULL) return -1; int32_t fd = (int32_t)(uintptr_t)p; - return fsync(fd); + int32_t code = fsync(fd); + + taosReleaseRef(tsFileRsetId, tfd); + return code; } bool tfValid(int64_t tfd) { void *p = taosAcquireRef(tsFileRsetId, tfd); - return p != NULL; + if (p == NULL) return false; + + taosReleaseRef(tsFileRsetId, tfd); + return true; } int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { @@ -111,7 +117,10 @@ int64_t tfLseek(int64_t tfd, int64_t offset, int32_t whence) { if (p == NULL) return -1; int32_t fd = (int32_t)(uintptr_t)p; - return taosLSeek(fd, offset, whence); + int64_t ret = taosLSeek(fd, offset, whence); + + taosReleaseRef(tsFileRsetId, tfd); + return ret; } int32_t tfFtruncate(int64_t tfd, int64_t length) { @@ -119,5 +128,8 @@ int32_t tfFtruncate(int64_t tfd, int64_t length) { if (p == NULL) return -1; int32_t fd = (int32_t)(uintptr_t)p; - return taosFtruncate(fd, length); + int32_t code = taosFtruncate(fd, length); + + taosReleaseRef(tsFileRsetId, tfd); + return code; } From 679d3093717fffa8d9cd9aaef61a58d72e06c39d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 30 Nov 2020 14:30:29 +0800 Subject: [PATCH 096/188] fix test case error --- Jenkinsfile | 8 +++ tests/pytest/insert/restfulInsert.py | 86 +++++++++++++++++++++++++--- tests/pytest/query/bug2218.py | 12 ++-- 3 files changed, 91 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index dc7836c3da..940815febe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -14,10 +14,12 @@ pipeline { sh ''' date cd ${WKC} + git reset --hard git checkout develop git pull git submodule update cd ${WK} + git reset --hard git checkout develop git pull export TZ=Asia/Harbin @@ -39,11 +41,13 @@ pipeline { steps { sh ''' cd ${WKC} + git reset --hard git checkout develop git pull git submodule update cd ${WK} + git reset --hard git checkout develop git pull export TZ=Asia/Harbin @@ -65,11 +69,13 @@ pipeline { steps { sh ''' cd ${WKC} + git reset --hard git checkout develop git pull git submodule update cd ${WK} + git reset --hard git checkout develop git pull export TZ=Asia/Harbin @@ -108,11 +114,13 @@ pipeline { steps { sh ''' cd ${WKC} + git reset --hard git checkout develop git pull git submodule update cd ${WK} + git reset --hard git checkout develop git pull export TZ=Asia/Harbin diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index e3a963f1d4..aad1efa60f 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -18,7 +18,7 @@ import time import argparse class RestfulInsert: - def __init__(self, host, startTimestamp, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder): + def __init__(self, host, startTimestamp, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder,tablePerbatch): self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} self.url = "http://%s:6041/rest/sql" % host self.ts = startTimestamp @@ -29,12 +29,15 @@ class RestfulInsert: self.batchSize = batchSize self.tableNamePerfix = tbNamePerfix self.outOfOrder = outOfOrder + self.tablePerbatch = tablePerbatch def createTable(self, threadID): - tablesPerThread = int (self.numOfTables / self.numOfThreads) - print("create table %d to %d" % (tablesPerThread * threadID, tablesPerThread * (threadID + 1) - 1)) - for i in range(tablesPerThread): + tablesPerThread = int (self.numOfTables / self.numOfThreads) + loop = tablesPerThread if threadID != self.numOfThreads - 1 else self.numOfTables - tablesPerThread * threadID + print("create table %d to %d" % (tablesPerThread * threadID, tablesPerThread * threadID + loop - 1)) + for i in range(loop): tableID = threadID * tablesPerThread + if tableID + i >= self.numOfTables : break name = 'beijing' if tableID % 2 == 0 else 'shanghai' data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) response = requests.post(self.url, data, headers = self.header) @@ -55,6 +58,58 @@ class RestfulInsert: response = requests.post(self.url, data, headers = self.header) if response.status_code != 200: print(response.content) + + def insertnData(self, threadID): + print("thread %d started" % threadID) + tablesPerThread = int (self.numOfTables / self.numOfThreads) + loop = int(self.recordsPerTable / self.batchSize) + if self.tablePerbatch == 1 : + for i in range(tablesPerThread+1): + tableID = i + threadID * tablesPerThread + if tableID >= self.numOfTables: return + start = self.ts + start1=time.time() + for k in range(loop): + data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID) + values = [] + bloop = self.batchSize if k != loop - 1 else self.recordsPerTable - self.batchSize * k + for l in range(bloop): + values.append("(%d, %d, %d, %d)" % (start + k * self.batchSize + l, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) + if len(data) > 1048576 : + print ('batch size is larger than 1M') + exit(-1) + if self.outOfOrder : + random.shuffle(values) + data+=''.join(values) + + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) + print('----------------',loop,time.time()-start1) + else: + for i in range(0,tablesPerThread+self.tablePerbatch,self.tablePerbatch): + for k in range(loop): + data = "insert into " + for j in range(self.tablePerbatch): + tableID = i + threadID * tablesPerThread+j + if tableID >= self.numOfTables: return + start = self.ts + data += "%s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID) + values = [] + bloop = self.batchSize if k != loop - 1 else self.recordsPerTable - self.batchSize * k + for l in range(bloop): + values.append("(%d, %d, %d, %d)" % (start + k * self.batchSize + l, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) + if self.outOfOrder : + random.shuffle(values) + data+=''.join(values) + print('------------------',len(data)) + if len(data) > 1024*1024 : + print ('batch size is larger than 1M') + exit(-1) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) + def insertUnlimitedData(self, threadID): print("thread %d started" % threadID) @@ -85,7 +140,9 @@ class RestfulInsert: if response.status_code != 200: print(response.content) - def run(self): + def run(self): + data = "drop database if exists %s" % self.dbname + requests.post(self.url, data, headers = self.header) data = "create database if not exists %s" % self.dbname requests.post(self.url, data, headers = self.header) data = "create table if not exists %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname @@ -135,14 +192,14 @@ parser.add_argument( '-d', '--db-name', action='store', - default='test', + default='test1', type=str, help='Database name to be created (default: test)') parser.add_argument( '-t', '--number-of-threads', action='store', - default=10, + default=20, type=int, help='Number of threads to create tables and insert datas (default: 10)') parser.add_argument( @@ -156,7 +213,7 @@ parser.add_argument( '-r', '--number-of-records', action='store', - default=1000, + default=10000, type=int, help='Number of record to be created for each table (default: 1000, -1 for unlimited records)') parser.add_argument( @@ -178,7 +235,18 @@ parser.add_argument( '--out-of-order', action='store_true', help='The order of test data (default: False)') +parser.add_argument( + '-b', + '--table-per-batch', + action='store', + default=1, + type=int, + help='the table per batch (default: 1)') + + args = parser.parse_args() -ri = RestfulInsert(args.host_name, args.start_timestamp, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order) +ri = RestfulInsert( + args.host_name, args.start_timestamp, args.db_name, args.number_of_threads, args.number_of_tables, + args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order, args.table_per_batch) ri.run() \ No newline at end of file diff --git a/tests/pytest/query/bug2218.py b/tests/pytest/query/bug2218.py index bb92e5d9ce..080472383d 100644 --- a/tests/pytest/query/bug2218.py +++ b/tests/pytest/query/bug2218.py @@ -38,12 +38,12 @@ class TDTestCase: print("test col*1*1 desc ") tdSql.query('select c1,c1*1*1,c2*1*1,c3*1*1,c4*1*1,c5*1*1,c6*1*1 from mt0 order by ts desc limit 2') tdSql.checkData(0,0,99) - tdSql.checkData(0,1,0.0) - tdSql.checkData(0,2,0.0) - tdSql.checkData(0,3,0.0) - tdSql.checkData(0,4,0.0) - tdSql.checkData(0,5,0.0) - tdSql.checkData(0,6,0.0) + tdSql.checkData(0,1,99.0) + tdSql.checkData(0,2,499.0) + tdSql.checkData(0,3,99.0) + tdSql.checkData(0,4,99.0) + tdSql.checkData(0,5,99.0) + tdSql.checkData(0,6,999.0) def stop(self): From 57bbd174a9f3b0d1f2b8a2426ab84f5317c32289 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 30 Nov 2020 14:31:47 +0800 Subject: [PATCH 097/188] fix test case error --- tests/pytest/query/bug2117.py | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 tests/pytest/query/bug2117.py diff --git a/tests/pytest/query/bug2117.py b/tests/pytest/query/bug2117.py new file mode 100644 index 0000000000..1158b78a2a --- /dev/null +++ b/tests/pytest/query/bug2117.py @@ -0,0 +1,50 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20))") + insertRows = 1000 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % + (t0+i,i%100,i/2,i%41,i%100,i%100,i*1.0,i%2,'taos'+str(i%100),'涛思'+str(i%100))) + print("==========step2") + print("test last with group by normal_col ") + tdSql.query('select last(c1) from mt0 group by c3') + tdSql.checkData(0,0,84) + tdSql.checkData(0,1,85) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From ab6d41f706c3fe89b6c8d76aae48a0e3f22f0419 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Mon, 30 Nov 2020 14:34:36 +0800 Subject: [PATCH 098/188] fix test case error --- tests/pytest/insert/restfulInsert.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index aad1efa60f..df253040f3 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -140,9 +140,7 @@ class RestfulInsert: if response.status_code != 200: print(response.content) - def run(self): - data = "drop database if exists %s" % self.dbname - requests.post(self.url, data, headers = self.header) + def run(self): data = "create database if not exists %s" % self.dbname requests.post(self.url, data, headers = self.header) data = "create table if not exists %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname @@ -192,14 +190,14 @@ parser.add_argument( '-d', '--db-name', action='store', - default='test1', + default='test', type=str, help='Database name to be created (default: test)') parser.add_argument( '-t', '--number-of-threads', action='store', - default=20, + default=10, type=int, help='Number of threads to create tables and insert datas (default: 10)') parser.add_argument( @@ -213,7 +211,7 @@ parser.add_argument( '-r', '--number-of-records', action='store', - default=10000, + default=1000, type=int, help='Number of record to be created for each table (default: 1000, -1 for unlimited records)') parser.add_argument( From a432168ae5399366bd947b3be1aeae3309ab776f Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Mon, 30 Nov 2020 14:25:54 +0800 Subject: [PATCH 099/188] [TD-2265]: fix TSDB_MAX_ALLOWED_SQL_LEN from 8MB to 1MB --- src/inc/taosdef.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 20c7af6a21..ec1e1fc330 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -257,7 +257,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_SQL_SHOW_LEN 512 -#define TSDB_MAX_ALLOWED_SQL_LEN (8*1024*1024U) // sql length should be less than 8mb +#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024U) // sql length should be less than 1mb #define TSDB_APPNAME_LEN TSDB_UNI_LEN From eebed5c2100a81077c99d17e7b924eb2a6ae4373 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 30 Nov 2020 08:16:58 +0000 Subject: [PATCH 100/188] fix TD-2279 --- src/tsdb/src/tsdbRWHelper.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 5b65b2185a..8e57066d27 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -917,6 +917,8 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst); } + ASSERT((blkIdx == pIdx->numOfBlocks -1) || (!pCompBlock->last)); + tsdbDebug("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx); @@ -1042,6 +1044,8 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast; pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last; + ASSERT((blkIdx == pIdx->numOfBlocks-1) || (!pCompBlock->last)); + tsdbDebug("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx); @@ -1622,11 +1626,7 @@ static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, pCfg->update); if (pDataCols->numOfRows == 0) break; - if (tblkIdx == pIdx->numOfBlocks - 1) { - if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1; - } else { - if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; - } + if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1; if (round == 0) { if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false; From 2b11f6ab4e094c20bc525811a257c724b0a6e323 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Mon, 30 Nov 2020 08:42:42 +0000 Subject: [PATCH 101/188] [TD-2140]: fix invalid write --- src/client/src/tscUtil.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 893c3f4c39..af17f9b457 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -836,12 +836,6 @@ int32_t tscMergeTableDataBlocks(SSqlObj* pSql, SArray* pTableDataBlockList) { dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); - // free unnecessary memory resource ASAP. - char* p = realloc(dataBuf->pData, dataBuf->size); - if (p != NULL) { - dataBuf->pData = p; - } - // the length does not include the SSubmitBlk structure pBlocks->dataLen = htonl(finalLen); dataBuf->numOfTables += 1; From e13de8ce2c41af6cdf389a7ee74b4287d9fc998d Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Mon, 30 Nov 2020 09:12:42 +0000 Subject: [PATCH 102/188] return mnode initialization stage to client --- src/dnode/src/dnodeMPeer.c | 11 +++-- src/dnode/src/dnodeMRead.c | 11 +++-- src/dnode/src/dnodeMWrite.c | 18 +++++---- src/dnode/src/dnodeMain.c | 2 +- src/dnode/src/dnodeShell.c | 2 +- src/inc/mnode.h | 3 ++ src/inc/taoserror.h | 3 ++ src/mnode/src/mnodeMain.c | 80 +++++++++++++++++++++++++++---------- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeUser.c | 15 ++++++- src/rpc/src/rpcMain.c | 5 +-- 11 files changed, 111 insertions(+), 41 deletions(-) diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index ee6dc5212e..adeccd1f9c 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -122,11 +122,16 @@ void dnodeFreeMPeerQueue() { } void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning() || tsMPeerQueue == NULL) { + if (!mnodeIsRunning()) { dnodeSendRedirectMsg(pMsg, false); } else { - SMnodeMsg *pPeer = mnodeCreateMsg(pMsg); - taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); + if (!mnodeIsReady()) { + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = mnodeInitCode(), .pCont = NULL}; + rpcSendResponse(&rpcRsp); + } else { + SMnodeMsg *pPeer = mnodeCreateMsg(pMsg); + taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); + } } rpcFreeCont(pMsg->pCont); diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index 65f3af7b3b..bb0d6f94f2 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -123,11 +123,16 @@ void dnodeFreeMReadQueue() { } void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning() || tsMReadQueue == NULL) { + if (!mnodeIsRunning()) { dnodeSendRedirectMsg(pMsg, true); } else { - SMnodeMsg *pRead = mnodeCreateMsg(pMsg); - taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); + if (!mnodeIsReady()) { + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = mnodeInitCode(), .pCont = NULL}; + rpcSendResponse(&rpcRsp); + } else { + SMnodeMsg *pRead = mnodeCreateMsg(pMsg); + taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); + } } rpcFreeCont(pMsg->pCont); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index ef2d49ef42..3c539ceefa 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -123,13 +123,18 @@ void dnodeFreeMWritequeue() { } void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning() || tsMWriteQueue == NULL) { + if (!mnodeIsRunning()) { dnodeSendRedirectMsg(pMsg, true); } else { - SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); - dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); - taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); + if (!mnodeIsReady()) { + SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = mnodeInitCode(), .pCont = NULL}; + rpcSendResponse(&rpcRsp); + } else { + SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); + dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, + taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); + taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); + } } rpcFreeCont(pMsg->pCont); @@ -187,7 +192,7 @@ static void *dnodeProcessMWriteQueue(void *param) { void dnodeReprocessMWriteMsg(void *pMsg) { SMnodeMsg *pWrite = pMsg; - if (!mnodeIsRunning() || tsMWriteQueue == NULL) { + if (!mnodeIsRunning()) { dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); @@ -196,7 +201,6 @@ void dnodeReprocessMWriteMsg(void *pMsg) { } else { dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry); - taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } } diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index 9f52dbd331..ce45105fb3 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -63,6 +63,7 @@ static const SDnodeComponent tsDnodeComponents[] = { {"dnodeeps", dnodeInitEps, dnodeCleanupEps}, {"globalcfg" ,taosCheckGlobalCfg, NULL}, {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos}, + {"shell", dnodeInitShell, dnodeCleanupShell}, {"wal", walInit, walCleanUp}, {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! {"vread", dnodeInitVRead, dnodeCleanupVRead}, @@ -75,7 +76,6 @@ static const SDnodeComponent tsDnodeComponents[] = { {"mgmt", dnodeInitMgmt, dnodeCleanupMgmt}, {"modules", dnodeInitModules, dnodeCleanupModules}, {"mgmt-tmr", dnodeInitMgmtTimer, dnodeCleanupMgmtTimer}, - {"shell", dnodeInitShell, dnodeCleanupShell}, {"telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, }; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index 89f657f789..fa5f9c24e8 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -144,7 +144,7 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); - if (code != TSDB_CODE_APP_NOT_READY) return code; + if (code != TSDB_CODE_RPC_REDIRECT) return code; SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); tstrncpy(pMsg->user, user, sizeof(pMsg->user)); diff --git a/src/inc/mnode.h b/src/inc/mnode.h index bdc30b0c46..453e8ab3f6 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -65,12 +65,15 @@ void mnodeStopSystem(); void sdbUpdateAsync(); void sdbUpdateSync(void *pMnodes); bool mnodeIsRunning(); +bool mnodeIsReady(); +int32_t mnodeInitCode(); int32_t mnodeProcessRead(SMnodeMsg *pMsg); int32_t mnodeProcessWrite(SMnodeMsg *pMsg); int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg); void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); + #ifdef __cplusplus } #endif diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 5fde6d40d5..59f25a0765 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -125,6 +125,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, 0, 0x030B, "Data expir TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, 0, 0x030C, "Invalid query id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, 0, 0x030D, "Invalid stream id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, 0, 0x030E, "Invalid connection id") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INIT, 0, 0x030F, "Mnode is initializing") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INIT_SDB, 0, 0x0310, "Mnode is initializing meta data, it takes a while if many tables exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INIT_OTHER, 0, 0x0311, "Mnode is initializing other data") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, 0, 0x0320, "Object already there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, 0, 0x0321, "Unexpected generic error in sdb") diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index d15b32da54..05ea0b93a1 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -17,6 +17,7 @@ #include "os.h" #include "taosdef.h" #include "tsched.h" +#include "taoserror.h" #include "tbalance.h" #include "tgrant.h" #include "ttimer.h" @@ -37,30 +38,41 @@ #include "mnodeShow.h" #include "mnodeProfile.h" +typedef enum { + TSDB_MND_STATUS_NOT_RUNNING, + TSDB_MND_STATUS_INIT, + TSDB_MND_STATUS_INIT_SDB, + TSDB_MND_STATUS_INIT_OTHER, + TSDB_MND_STATUS_READY, + TSDB_MND_STATUS_CLEANING, +} EMndStatus; + typedef struct { const char *const name; int (*init)(); void (*cleanup)(); + EMndStatus status; } SMnodeComponent; -void *tsMnodeTmr = NULL; -static bool tsMgmtIsRunning = false; +void *tsMnodeTmr = NULL; +static bool tsMgmtIsRunning = false; +static EMndStatus tsMgmtStatus = TSDB_MND_STATUS_NOT_RUNNING; static const SMnodeComponent tsMnodeComponents[] = { - {"sdbref", sdbInitRef, sdbCleanUpRef}, - {"profile", mnodeInitProfile, mnodeCleanupProfile}, - {"cluster", mnodeInitCluster, mnodeCleanupCluster}, - {"accts", mnodeInitAccts, mnodeCleanupAccts}, - {"users", mnodeInitUsers, mnodeCleanupUsers}, - {"dnodes", mnodeInitDnodes, mnodeCleanupDnodes}, - {"dbs", mnodeInitDbs, mnodeCleanupDbs}, - {"vgroups", mnodeInitVgroups, mnodeCleanupVgroups}, - {"tables", mnodeInitTables, mnodeCleanupTables}, - {"mnodes", mnodeInitMnodes, mnodeCleanupMnodes}, - {"sdb", sdbInit, sdbCleanUp}, - {"balance", balanceInit, balanceCleanUp}, - {"grant", grantInit, grantCleanUp}, - {"show", mnodeInitShow, mnodeCleanUpShow} + {"sdbref", sdbInitRef, sdbCleanUpRef, TSDB_MND_STATUS_INIT}, + {"profile", mnodeInitProfile, mnodeCleanupProfile, TSDB_MND_STATUS_INIT}, + {"cluster", mnodeInitCluster, mnodeCleanupCluster, TSDB_MND_STATUS_INIT}, + {"accts", mnodeInitAccts, mnodeCleanupAccts, TSDB_MND_STATUS_INIT}, + {"users", mnodeInitUsers, mnodeCleanupUsers, TSDB_MND_STATUS_INIT}, + {"dnodes", mnodeInitDnodes, mnodeCleanupDnodes, TSDB_MND_STATUS_INIT}, + {"dbs", mnodeInitDbs, mnodeCleanupDbs, TSDB_MND_STATUS_INIT}, + {"vgroups", mnodeInitVgroups, mnodeCleanupVgroups, TSDB_MND_STATUS_INIT}, + {"tables", mnodeInitTables, mnodeCleanupTables, TSDB_MND_STATUS_INIT}, + {"mnodes", mnodeInitMnodes, mnodeCleanupMnodes, TSDB_MND_STATUS_INIT}, + {"sdb", sdbInit, sdbCleanUp, TSDB_MND_STATUS_INIT_SDB}, + {"balance", balanceInit, balanceCleanUp, TSDB_MND_STATUS_INIT_OTHER}, + {"grant", grantInit, grantCleanUp, TSDB_MND_STATUS_INIT_OTHER}, + {"show", mnodeInitShow, mnodeCleanUpShow, TSDB_MND_STATUS_INIT_OTHER}, }; static void mnodeInitTimer(); @@ -76,21 +88,24 @@ static void mnodeCleanupComponents(int32_t stepId) { static int32_t mnodeInitComponents() { int32_t code = 0; for (int32_t i = 0; i < sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]); i++) { + tsMgmtStatus = tsMnodeComponents[i].status; if (tsMnodeComponents[i].init() != 0) { mnodeCleanupComponents(i); code = -1; break; } + // sleep(3); } return code; } int32_t mnodeStartSystem() { - if (tsMgmtIsRunning) { + if (tsMgmtStatus != TSDB_MND_STATUS_NOT_RUNNING) { mInfo("mnode module already started..."); return 0; } + tsMgmtStatus = TSDB_MND_STATUS_INIT; mInfo("starting to initialize mnode ..."); if (mkdir(tsMnodeDir, 0755) != 0 && errno != EEXIST) { mError("failed to init mnode dir:%s, reason:%s", tsMnodeDir, strerror(errno)); @@ -106,7 +121,7 @@ int32_t mnodeStartSystem() { } grantReset(TSDB_GRANT_ALL, 0); - tsMgmtIsRunning = true; + tsMgmtStatus = TSDB_MND_STATUS_READY; mInfo("mnode is initialized successfully"); @@ -126,7 +141,7 @@ int32_t mnodeInitSystem() { void mnodeCleanupSystem() { if (tsMgmtIsRunning) { mInfo("starting to clean up mnode"); - tsMgmtIsRunning = false; + tsMgmtStatus = TSDB_MND_STATUS_CLEANING; dnodeFreeMWritequeue(); dnodeFreeMReadQueue(); @@ -134,6 +149,7 @@ void mnodeCleanupSystem() { mnodeCleanupTimer(); mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1); + tsMgmtStatus = TSDB_MND_STATUS_NOT_RUNNING; mInfo("mnode is cleaned up"); } } @@ -184,5 +200,29 @@ static bool mnodeNeedStart() { } bool mnodeIsRunning() { - return tsMgmtIsRunning; + return (tsMgmtStatus != TSDB_MND_STATUS_NOT_RUNNING && tsMgmtStatus != TSDB_MND_STATUS_CLEANING); +} + +bool mnodeIsReady() { + return (tsMgmtStatus == TSDB_MND_STATUS_READY); +} + +int32_t mnodeInitCode() { + int32_t code = -1; + + switch (tsMgmtStatus) { + case TSDB_MND_STATUS_INIT: + code = TSDB_CODE_MND_INIT; + break; + case TSDB_MND_STATUS_INIT_SDB: + code = TSDB_CODE_MND_INIT_SDB; + break; + case TSDB_MND_STATUS_INIT_OTHER: + code = TSDB_CODE_MND_INIT_OTHER; + break; + default: + code = TSDB_CODE_MND_INIT; + } + + return code; } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 6670e87a7c..2e6ba8175d 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -299,7 +299,7 @@ void sdbUpdateAsync() { void sdbUpdateSync(void *pMnodes) { SMnodeInfos *mnodes = pMnodes; - if (!mnodeIsRunning()) { + if (!mnodeIsReady()) { mDebug("vgId:1, mnode not start yet, update sync config later"); return; } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index fb26086d04..8d57ad30dc 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -585,10 +585,21 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { } int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { + *secret = 0; + + if (!mnodeIsRunning()) { + mDebug("user:%s, mnode is not running, fail to auth", user); + return TSDB_CODE_RPC_REDIRECT; + } + + if (!mnodeIsReady()) { + mDebug("user:%s, failed to auth user, mnode is not ready", user); + return mnodeInitCode(); + } + if (!sdbIsMaster()) { - *secret = 0; mDebug("user:%s, failed to auth user, mnode is not master", user); - return TSDB_CODE_APP_NOT_READY; + return TSDB_CODE_RPC_REDIRECT; } SUserObj *pUser = mnodeGetUser(user); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index acceaf9d7a..2b7271459e 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1551,10 +1551,9 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { if ( !rpcIsReq(pHead->msgType) ) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_INVALID_USER || code == TSDB_CODE_RPC_NOT_READY) { - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + if (code != 0) { // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); return 0; } } From 2e69b78fcd054ec8f8570148b4a798fc93a96dcf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 30 Nov 2020 10:32:12 +0000 Subject: [PATCH 103/188] TD-2283 --- src/cq/src/cqMain.c | 8 +++----- src/cq/test/cqtest.c | 2 +- src/inc/tcq.h | 2 +- src/vnode/src/vnodeMain.c | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index 84b2c297d0..b1977fd5d9 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -40,15 +40,14 @@ typedef struct { int32_t vgId; + int32_t master; + int32_t num; // number of continuous streams char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; char db[TSDB_DB_NAME_LEN]; FCqWrite cqWrite; - void *ahandle; - int32_t num; // number of continuous streams struct SCqObj *pHead; void *dbConn; - int32_t master; void *tmrCtrl; pthread_mutex_t mutex; } SCqContext; @@ -90,7 +89,6 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { tstrncpy(pContext->db, db, sizeof(pContext->db)); pContext->vgId = pCfg->vgId; pContext->cqWrite = pCfg->cqWrite; - pContext->ahandle = ahandle; tscEmbedded = 1; pthread_mutex_init(&pContext->mutex, NULL); @@ -342,7 +340,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) { pHead->version = 0; // write into vnode write queue - pContext->cqWrite(pContext->ahandle, pHead, TAOS_QTYPE_CQ, NULL); + pContext->cqWrite(pContext->vgId, pHead, TAOS_QTYPE_CQ, NULL); free(buffer); } diff --git a/src/cq/test/cqtest.c b/src/cq/test/cqtest.c index e1114fc024..41380f0d86 100644 --- a/src/cq/test/cqtest.c +++ b/src/cq/test/cqtest.c @@ -24,7 +24,7 @@ int64_t ver = 0; void *pCq = NULL; -int writeToQueue(void *pVnode, void *data, int type, void *pMsg) { +int writeToQueue(int32_t vgId, void *data, int type, void *pMsg) { return 0; } diff --git a/src/inc/tcq.h b/src/inc/tcq.h index 7a0727f1b8..afa744a9c4 100644 --- a/src/inc/tcq.h +++ b/src/inc/tcq.h @@ -21,7 +21,7 @@ extern "C" { #include "tdataformat.h" -typedef int32_t (*FCqWrite)(void *ahandle, void *pHead, int32_t qtype, void *pMsg); +typedef int32_t (*FCqWrite)(int32_t vgId, void *pHead, int32_t qtype, void *pMsg); typedef struct { int32_t vgId; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 3d3abde3c7..0b29fcc908 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -272,7 +272,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { strcpy(cqCfg.pass, tsInternalPass); strcpy(cqCfg.db, pVnode->db); cqCfg.vgId = vnode; - cqCfg.cqWrite = vnodeWriteToWQueue; + cqCfg.cqWrite = vnodeWriteToCache; pVnode->cq = cqOpen(pVnode, &cqCfg); if (pVnode->cq == NULL) { vnodeCleanUp(pVnode); From ece8c475147e81a0db2e617205202728d3a0cd87 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 30 Nov 2020 19:01:25 +0800 Subject: [PATCH 104/188] TD-1926 --- src/inc/taoserror.h | 3 - src/inc/tsync.h | 4 +- src/mnode/src/mnodeSdb.c | 15 +- src/sync/inc/syncInt.h | 8 +- src/sync/src/syncMain.c | 3 +- src/sync/src/syncRetrieve.c | 311 +++++++++++++++++------------------- src/vnode/src/vnodeMain.c | 9 +- 7 files changed, 171 insertions(+), 182 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 5fde6d40d5..dd6d88e88e 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -261,9 +261,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_CONFIG, 0, 0x0900, "Invalid Sy TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_ENABLED, 0, 0x0901, "Sync module not enabled") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_VERSION, 0, 0x0902, "Invalid Sync version") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_CONFIRM_EXPIRED, 0, 0x0903, "Sync confirm expired") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_VND_COMMITING, 0, 0x0904, "Vnode is commiting") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_FILE_CHNAGED, 0, 0x0905, "Vnode file is changed") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_APP_ERROR, 0, 0x1000, "Unexpected generic error in sync") // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_APP_ERROR, 0, 0x1000, "Unexpected generic error in wal") diff --git a/src/inc/tsync.h b/src/inc/tsync.h index 293e382519..1303195ef1 100644 --- a/src/inc/tsync.h +++ b/src/inc/tsync.h @@ -86,7 +86,7 @@ typedef void (*FNotifyFlowCtrl)(int32_t vgId, int32_t level); typedef int32_t (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion); // get file version -typedef int32_t (*FGetFileVersion)(int32_t vgId, uint64_t *fver); +typedef int32_t (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver); typedef struct { int32_t vgId; // vgroup ID @@ -100,7 +100,7 @@ typedef struct { FNotifyRole notifyRole; FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; - FGetFileVersion getFileVersion; + FGetVersion getVersion; } SSyncInfo; typedef void *tsync_h; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 6670e87a7c..40e2e1cfcc 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -251,6 +251,16 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) { sdbUpdateMnodeRoles(); } +static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; } + +static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {} + +static int32_t sdbGetSyncVersion(int32_t vgId, uint64_t *fver, uint64_t *vver) { + *fver = 0; + *vver = 0; + return 0; +} + // failed to forward, need revert insert static void sdbHandleFailedConfirm(SSdbRow *pRow) { SWalHead *pHead = pRow->pHead; @@ -372,11 +382,14 @@ void sdbUpdateSync(void *pMnodes) { syncInfo.version = sdbGetVersion(); syncInfo.syncCfg = syncCfg; sprintf(syncInfo.path, "%s", tsMnodeDir); - syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.getFileInfo = sdbGetFileInfo; + syncInfo.getWalInfo = sdbGetWalInfo; syncInfo.writeToCache = sdbWriteFwdToQueue; syncInfo.confirmForward = sdbConfirmForward; syncInfo.notifyRole = sdbNotifyRole; + syncInfo.notifyFileSynced = sdbNotifyFileSynced; + syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl; + syncInfo.getVersion = sdbGetSyncVersion; tsSdbMgmt.cfg = syncCfg; if (tsSdbMgmt.sync) { diff --git a/src/sync/inc/syncInt.h b/src/sync/inc/syncInt.h index 7dc04d70d0..6d0c52284f 100644 --- a/src/sync/inc/syncInt.h +++ b/src/sync/inc/syncInt.h @@ -139,16 +139,14 @@ typedef struct SsyncPeer { char id[TSDB_EP_LEN + 32]; // peer vgId + end point uint64_t version; uint64_t sversion; // track the peer version in retrieve process - uint64_t lastVer; // track the file version while retrieve + uint64_t lastFileVer; // track the file version while retrieve + uint64_t lastWalVer; // track the wal version while retrieve int32_t syncFd; int32_t peerFd; // forward FD int32_t numOfRetrieves; // number of retrieves tried int32_t fileChanged; // a flag to indicate file is changed during retrieving process void * timer; void * pConn; - int32_t notifyFd; - int32_t watchNum; - int32_t *watchFd; int32_t refCount; // reference count struct SSyncNode *pSyncNode; } SSyncPeer; @@ -173,7 +171,7 @@ typedef struct SSyncNode { FNotifyRole notifyRole; FNotifyFlowCtrl notifyFlowCtrl; FNotifyFileSynced notifyFileSynced; - FGetFileVersion getFileVersion; + FGetVersion getVersion; pthread_mutex_t mutex; } SSyncNode; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index ce85c940ff..adac532f2d 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -196,7 +196,7 @@ int64_t syncStart(const SSyncInfo *pInfo) { pNode->confirmForward = pInfo->confirmForward; pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl; pNode->notifyFileSynced = pInfo->notifyFileSynced; - pNode->getFileVersion = pInfo->getFileVersion; + pNode->getVersion = pInfo->getVersion; pNode->selfIndex = -1; pNode->vgId = pInfo->vgId; @@ -498,7 +498,6 @@ int32_t syncDecPeerRef(SSyncPeer *pPeer) { taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid); sDebug("%s, resource is freed", pPeer->id); - tfree(pPeer->watchFd); tfree(pPeer); return 0; } diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 305ec492b9..44d2680b1a 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -26,35 +26,73 @@ #include "tsync.h" #include "syncInt.h" -static int32_t syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { - if (pNode->getFileVersion == NULL) return TSDB_CODE_SUCCESS; - - uint64_t fver = 0; - int32_t code = (*pNode->getFileVersion)(pNode->vgId, &fver); +static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) { + uint64_t fver, wver; + int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); if (code != 0) { - sInfo("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastVer); - pPeer->fileChanged = 1; - return TSDB_CODE_SYN_VND_COMMITING; + sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); + return -1; } - if (fver != pPeer->lastVer) { - sInfo("%s, files are modified while retrieve, fver:%" PRIu64 ", last fver:%" PRIu64, pPeer->id, fver, pPeer->lastVer); + pPeer->lastWalVer = wver; + return code; +} + +static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) { + uint64_t fver, wver; + int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + if (code != 0) { + sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer); + return true; + } + + if (wver != pPeer->lastWalVer) { + sDebug("%s, wal is modified while retrieve, wver:%" PRIu64 ", last:%" PRIu64, pPeer->id, wver, pPeer->lastWalVer); + return true; + } + + return false; +} + +static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) { + uint64_t fver, wver; + int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + if (code != 0) { + sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); + return -1; + } + + pPeer->lastFileVer = fver; + return code; +} + +static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) { + uint64_t fver, wver; + int32_t code = (*pNode->getVersion)(pNode->vgId, &fver, &wver); + if (code != 0) { + sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer); pPeer->fileChanged = 1; - return TSDB_CODE_SYN_FILE_CHNAGED; + return true; + } + + if (fver != pPeer->lastFileVer) { + sDebug("%s, files are modified while retrieve, fver:%" PRIu64 ", last:%" PRIu64, pPeer->id, fver, pPeer->lastFileVer); + pPeer->fileChanged = 1; + return true; } pPeer->fileChanged = 0; - return TSDB_CODE_SUCCESS; + return false; } static int32_t syncRetrieveFile(SSyncPeer *pPeer) { SSyncNode *pNode = pPeer->pSyncNode; SFileInfo fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo)); SFileAck fileAck = {0}; - int32_t code = TSDB_CODE_SYN_APP_ERROR; + int32_t code = -1; char name[TSDB_FILENAME_LEN * 2] = {0}; - if (pNode->getFileVersion) (*pNode->getFileVersion)(pNode->vgId, &pPeer->lastVer); + if (syncGetFileVersion(pNode, pPeer) < 0) return -1; while (1) { // retrieve file info @@ -120,8 +158,10 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { fileInfo.index++; // check if processed files are modified - code = syncAreFilesModified(pNode, pPeer); - if (code != TSDB_CODE_SUCCESS) break; + if (syncAreFilesModified(pNode, pPeer)) { + code = -1; + break; + } } if (code != TSDB_CODE_SUCCESS) { @@ -131,117 +171,90 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { return code; } -/* if only a partial record is read out, set the IN_MODIFY flag in event, - so upper layer will reload the file to get a complete record */ -static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead, uint32_t *pEvent) { - int32_t ret; +// if only a partial record is read out, upper layer will reload the file to get a complete record +static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { + int32_t ret = read(sfd, pHead, sizeof(SWalHead)); + if (ret < 0) { + sError("sfd:%d, failed to read wal head since %s, ret:%d", sfd, strerror(errno), ret); + return -1; + } - ret = read(sfd, pHead, sizeof(SWalHead)); - if (ret < 0) return -1; - if (ret == 0) return 0; + if (ret == 0) { + sDebug("sfd:%d, read to end of the wal, ret:%d", sfd, ret); + return 0; + } if (ret != sizeof(SWalHead)) { // file is not at end yet, it shall be reloaded - *pEvent = *pEvent | IN_MODIFY; + sDebug("sfd:%d, a partial wal head is read out, ret:%d", sfd, ret); return 0; } assert(pHead->len <= TSDB_MAX_WAL_SIZE); ret = read(sfd, pHead->cont, pHead->len); - if (ret < 0) return -1; + if (ret < 0) { + sError("sfd:%d, failed to read wal content since %s, ret:%d", sfd, strerror(errno), ret); + return -1; + } if (ret != pHead->len) { // file is not at end yet, it shall be reloaded - *pEvent = *pEvent | IN_MODIFY; + sDebug("sfd:%d, a partial wal conetnt is read out, ret:%d", sfd, ret); return 0; } return sizeof(SWalHead) + pHead->len; } -static int32_t syncMonitorLastWal(SSyncPeer *pPeer, char *name) { - pPeer->watchNum = 0; - taosClose(pPeer->notifyFd); - pPeer->notifyFd = inotify_init1(IN_NONBLOCK); - if (pPeer->notifyFd < 0) { - sError("%s, failed to init inotify since %s", pPeer->id, strerror(errno)); - return -1; - } - - if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int32_t) * tsMaxWatchFiles); - if (pPeer->watchFd == NULL) { - sError("%s, failed to allocate watchFd", pPeer->id); - return -1; - } - - memset(pPeer->watchFd, -1, sizeof(int32_t) * tsMaxWatchFiles); - int32_t *wd = pPeer->watchFd; - - *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE); - if (*wd == -1) { - sError("%s, failed to watch last wal since %s", pPeer->id, strerror(errno)); - return -1; - } - - return 0; -} - -static int32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) { - char buf[2048]; - int32_t len = read(pPeer->notifyFd, buf, sizeof(buf)); - if (len < 0 && errno != EAGAIN) { - sError("%s, failed to read notify FD since %s", pPeer->id, strerror(errno)); - return -1; - } - - if (len == 0) return 0; - - struct inotify_event *event; - for (char *ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) { - event = (struct inotify_event *)ptr; - if (event->mask & IN_MODIFY) *pEvent = *pEvent | IN_MODIFY; - if (event->mask & IN_CLOSE_WRITE) *pEvent = *pEvent | IN_CLOSE_WRITE; - } - - if (pEvent != 0) sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent); - - return 0; -} - -static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) { - SWalHead *pHead = malloc(SYNC_MAX_SIZE); - int32_t code = -1; - int32_t bytes = 0; - int32_t sfd; - - sfd = open(name, O_RDONLY); +static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset) { + int32_t sfd = open(name, O_RDONLY); if (sfd < 0) { - free(pHead); + sError("%s, failed to open wal:%s for retrieve since:%s", pPeer->id, name, tstrerror(errno)); return -1; } - (void)lseek(sfd, offset, SEEK_SET); - sDebug("%s, retrieve last wal, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, offset, fversion); + int32_t code = taosLSeek(sfd, offset, SEEK_SET); + if (code < 0) { + sError("%s, failed to seek %" PRId64 " in wal:%s for retrieve since:%s", pPeer->id, offset, name, tstrerror(errno)); + close(sfd); + return -1; + } + + sDebug("%s, retrieve last wal:%s, offset:%" PRId64 " fver:%" PRIu64, pPeer->id, name, offset, fversion); + + SWalHead *pHead = malloc(SYNC_MAX_SIZE); + int32_t bytes = 0; while (1) { - int32_t wsize = syncReadOneWalRecord(sfd, pHead, pEvent); - if (wsize < 0) break; - if (wsize == 0) { - code = 0; + code = syncReadOneWalRecord(sfd, pHead); + if (code < 0) { + sError("%s, failed to read one record from wal:%s", pPeer->id, name); + break; + } + + if (code == 0) { + code = bytes; + sDebug("%s, read to the end of wal, bytes:%d", pPeer->id, bytes); break; } sTrace("%s, last wal is forwarded, hver:%" PRIu64, pPeer->id, pHead->version); - int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); - if (ret != wsize) break; - pPeer->sversion = pHead->version; + int32_t wsize = code; + int32_t ret = taosWriteMsg(pPeer->syncFd, pHead, wsize); + if (ret != wsize) { + code = -1; + sError("%s, failed to forward wal since %s, hver:%" PRIu64, pPeer->id, strerror(errno), pHead->version); + break; + } + + pPeer->sversion = pHead->version; bytes += wsize; if (pHead->version >= fversion && fversion > 0) { code = 0; - bytes = 0; + sDebug("%s, retrieve wal finished, fver:%" PRIu64, pPeer->id, fversion); break; } } @@ -249,92 +262,61 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi free(pHead); close(sfd); - if (code == 0) return bytes; - return -1; + return code; } static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) { SSyncNode *pNode = pPeer->pSyncNode; - int32_t code = -1; + int32_t once = 0; // last WAL has once ever been processed + int64_t offset = 0; + uint64_t fversion = 0; char fname[TSDB_FILENAME_LEN * 2] = {0}; // full path to wal file - if (syncAreFilesModified(pNode, pPeer) != 0) return -1; + // get full path to wal file + snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); + sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname); while (1) { - int32_t once = 0; // last WAL has once ever been processed - int64_t offset = 0; - uint64_t fversion = 0; - uint32_t event = 0; + if (syncAreFilesModified(pNode, pPeer)) return -1; + if (syncGetWalVersion(pNode, pPeer) < 0) return -1; - // get full path to wal file - snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); - sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname); - - // monitor last wal - if (syncMonitorLastWal(pPeer, fname) < 0) break; - - while (1) { - int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset, &event); - if (bytes < 0) break; - - // check file changes - if (syncCheckLastWalChanges(pPeer, &event) < 0) break; - - // if file is not updated or updated once, set the fversion and sstatus - if (((event & IN_MODIFY) == 0) || once) { - if (fversion == 0) { - pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt - sDebug("%s, fversion is 0 then set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - fversion = nodeVersion; // must read data to fversion - } - } - - // if all data up to fversion is read out, it is over - if (pPeer->sversion >= fversion && fversion > 0) { - code = 0; - sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes); - break; - } - - // if all data are read out, and no update - if ((bytes == 0) && ((event & IN_MODIFY) == 0)) { - // wal file is closed, break - if (event & IN_CLOSE_WRITE) { - code = 0; - sDebug("%s, current wal is closed", pPeer->id); - break; - } - - // wal not closed, it means some data not flushed to disk, wait for a while - usleep(10000); - } - - // if bytes>0, file is updated, or fversion is not reached but file still open, read again - once = 1; - offset += bytes; - sDebug("%s, retrieve last wal, bytes:%d", pPeer->id, bytes); - event = event & (~IN_MODIFY); // clear IN_MODIFY flag + int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset); + if (bytes < 0) { + sDebug("%s, failed to retrieve last wal", pPeer->id); + return bytes; } - if (code < 0) break; - if (pPeer->sversion >= fversion && fversion > 0) break; + // check file changes + bool walModified = syncIsWalModified(pNode, pPeer); - index++; - wname[0] = 0; - code = (*pNode->getWalInfo)(pNode->vgId, wname, &index); - if (code < 0) break; - if (wname[0] == 0) { - code = 0; - break; + // if file is not updated or updated once, set the fversion and sstatus + if (!walModified || once) { + if (fversion == 0) { + pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt + fversion = nodeVersion; // must read data to fversion + sDebug("%s, fversion is 0 and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + } } - // current last wal is closed, there is a new one - sDebug("%s, last wal is closed, try new one", pPeer->id); + // if all data up to fversion is read out, it is over + if (pPeer->sversion >= fversion && fversion > 0) { + sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes); + return 0; + } + + // if all data are read out, and no update + if (bytes == 0 && !walModified) { + // wal not closed, it means some data not flushed to disk, wait for a while + usleep(10000); + } + + // if bytes > 0, file is updated, or fversion is not reached but file still open, read again + once = 1; + offset += bytes; + sDebug("%s, continue retrieve last wal, bytes:%d offset:%" PRId64, pPeer->id, bytes, offset); } - taosClose(pPeer->notifyFd); - - return code; + return -1; } static int32_t syncRetrieveWal(SSyncPeer *pPeer) { @@ -377,7 +359,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { close(sfd); if (code < 0) break; - if (syncAreFilesModified(pNode, pPeer) != 0) break; + if (syncAreFilesModified(pNode, pPeer)) break; } if (code == 0) { @@ -474,7 +456,6 @@ void *syncRetrieveData(void *param) { } pPeer->fileChanged = 0; - taosClose(pPeer->notifyFd); taosClose(pPeer->syncFd); syncDecPeerRef(pPeer); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 3d3abde3c7..a046f1a50b 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -38,7 +38,7 @@ static void vnodeCtrlFlow(int32_t vgId, int32_t level); static int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion); static void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code); static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam); -static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver); +static int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver); #ifndef _SYNC int64_t syncStart(const SSyncInfo *info) { return NULL; } @@ -353,7 +353,7 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { syncInfo.notifyRole = vnodeNotifyRole; syncInfo.notifyFlowCtrl = vnodeCtrlFlow; syncInfo.notifyFileSynced = vnodeNotifyFileSynced; - syncInfo.getFileVersion = vnodeGetFileVersion; + syncInfo.getVersion = vnodeGetVersion; pVnode->sync = syncStart(&syncInfo); #ifndef _SYNC @@ -771,7 +771,7 @@ static int32_t vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void return code; } -static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver) { +static int32_t vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver) { SVnodeObj *pVnode = vnodeAcquire(vgId); if (pVnode == NULL) { vError("vgId:%d, vnode not found while write to cache", vgId); @@ -780,10 +780,11 @@ static int32_t vnodeGetFileVersion(int32_t vgId, uint64_t *fver) { int32_t code = 0; if (pVnode->isCommiting) { - vDebug("vgId:%d, vnode is commiting while get file version", vgId); + vDebug("vgId:%d, vnode is commiting while get version", vgId); code = -1; } else { *fver = pVnode->fversion; + *wver = pVnode->version; } vnodeRelease(pVnode); From 892e2599ecd0112413372ac422d7d98a27f0019a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Tue, 1 Dec 2020 11:20:14 +0800 Subject: [PATCH 105/188] Revert "[TD-2268]: cancel all related timers in taosTmrCleanUp" --- src/util/src/ttimer.c | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/src/util/src/ttimer.c b/src/util/src/ttimer.c index baf396f030..0222a6d80a 100644 --- a/src/util/src/ttimer.c +++ b/src/util/src/ttimer.c @@ -560,37 +560,6 @@ void taosTmrCleanUp(void* handle) { tmrDebug("%s timer controller is cleaned up.", ctrl->label); ctrl->label[0] = 0; - // cancel all timers of this controller - for (size_t i = 0; i < timerMap.size; i++) { - timer_list_t* list = timerMap.slots + i; - lockTimerList(list); - - tmr_obj_t* t = list->timers; - tmr_obj_t* prev = NULL; - while (t != NULL) { - tmr_obj_t* next = t->mnext; - if (t->ctrl != ctrl) { - prev = t; - t = next; - continue; - } - - uint8_t state = atomic_val_compare_exchange_8(&t->state, TIMER_STATE_WAITING, TIMER_STATE_CANCELED); - if (state == TIMER_STATE_WAITING) { - removeFromWheel(t); - } - timerDecRef(t); - if (prev == NULL) { - list->timers = next; - } else { - prev->mnext = next; - } - t = next; - } - - unlockTimerList(list); - } - pthread_mutex_lock(&tmrCtrlMutex); ctrl->next = unusedTmrCtrl; numOfTmrCtrl--; From 61b46e790e1a9d60356325881a800b512f170fe9 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 1 Dec 2020 11:42:08 +0800 Subject: [PATCH 106/188] modify jenkins pytest load balance --- tests/pytest/pytest_1.sh | 218 +++++++++++++++++++++++++++++++++++++++ tests/pytest/pytest_2.sh | 17 +++ tests/test-all.sh | 6 ++ 3 files changed, 241 insertions(+) create mode 100755 tests/pytest/pytest_1.sh create mode 100755 tests/pytest/pytest_2.sh diff --git a/tests/pytest/pytest_1.sh b/tests/pytest/pytest_1.sh new file mode 100755 index 0000000000..52f5a30f4e --- /dev/null +++ b/tests/pytest/pytest_1.sh @@ -0,0 +1,218 @@ +#!/bin/bash +ulimit -c unlimited + +python3 ./test.py -f insert/basic.py +python3 ./test.py -f insert/int.py +python3 ./test.py -f insert/float.py +python3 ./test.py -f insert/bigint.py +python3 ./test.py -f insert/bool.py +python3 ./test.py -f insert/double.py +python3 ./test.py -f insert/smallint.py +python3 ./test.py -f insert/tinyint.py +python3 ./test.py -f insert/date.py +python3 ./test.py -f insert/binary.py +python3 ./test.py -f insert/nchar.py +#python3 ./test.py -f insert/nchar-boundary.py +python3 ./test.py -f insert/nchar-unicode.py +python3 ./test.py -f insert/multi.py +python3 ./test.py -f insert/randomNullCommit.py +python3 insert/retentionpolicy.py +python3 ./test.py -f insert/alterTableAndInsert.py +python3 ./test.py -f insert/insertIntoTwoTables.py + +python3 ./test.py -f table/alter_wal0.py +python3 ./test.py -f table/column_name.py +python3 ./test.py -f table/column_num.py +python3 ./test.py -f table/db_table.py +python3 ./test.py -f table/create_sensitive.py +#python3 ./test.py -f table/tablename-boundary.py + +# tag +python3 ./test.py -f tag_lite/filter.py +python3 ./test.py -f tag_lite/create-tags-boundary.py +python3 ./test.py -f tag_lite/3.py +python3 ./test.py -f tag_lite/4.py +python3 ./test.py -f tag_lite/5.py +python3 ./test.py -f tag_lite/6.py +python3 ./test.py -f tag_lite/add.py +python3 ./test.py -f tag_lite/bigint.py +python3 ./test.py -f tag_lite/binary_binary.py +python3 ./test.py -f tag_lite/binary.py +python3 ./test.py -f tag_lite/bool_binary.py +python3 ./test.py -f tag_lite/bool_int.py +python3 ./test.py -f tag_lite/bool.py +python3 ./test.py -f tag_lite/change.py +python3 ./test.py -f tag_lite/column.py +python3 ./test.py -f tag_lite/commit.py +python3 ./test.py -f tag_lite/create.py +python3 ./test.py -f tag_lite/datatype.py +python3 ./test.py -f tag_lite/datatype-without-alter.py +python3 ./test.py -f tag_lite/delete.py +python3 ./test.py -f tag_lite/double.py +python3 ./test.py -f tag_lite/float.py +python3 ./test.py -f tag_lite/int_binary.py +python3 ./test.py -f tag_lite/int_float.py +python3 ./test.py -f tag_lite/int.py +python3 ./test.py -f tag_lite/set.py +python3 ./test.py -f tag_lite/smallint.py +python3 ./test.py -f tag_lite/tinyint.py + +#python3 ./test.py -f dbmgmt/database-name-boundary.py + +python3 ./test.py -f import_merge/importBlock1HO.py +python3 ./test.py -f import_merge/importBlock1HPO.py +python3 ./test.py -f import_merge/importBlock1H.py +python3 ./test.py -f import_merge/importBlock1S.py +python3 ./test.py -f import_merge/importBlock1Sub.py +python3 ./test.py -f import_merge/importBlock1TO.py +python3 ./test.py -f import_merge/importBlock1TPO.py +python3 ./test.py -f import_merge/importBlock1T.py +python3 ./test.py -f import_merge/importBlock2HO.py +python3 ./test.py -f import_merge/importBlock2HPO.py +python3 ./test.py -f import_merge/importBlock2H.py +python3 ./test.py -f import_merge/importBlock2S.py +python3 ./test.py -f import_merge/importBlock2Sub.py +python3 ./test.py -f import_merge/importBlock2TO.py +python3 ./test.py -f import_merge/importBlock2TPO.py +python3 ./test.py -f import_merge/importBlock2T.py +python3 ./test.py -f import_merge/importBlockbetween.py +python3 ./test.py -f import_merge/importCacheFileHO.py +python3 ./test.py -f import_merge/importCacheFileHPO.py +python3 ./test.py -f import_merge/importCacheFileH.py +python3 ./test.py -f import_merge/importCacheFileS.py +python3 ./test.py -f import_merge/importCacheFileSub.py +python3 ./test.py -f import_merge/importCacheFileTO.py +python3 ./test.py -f import_merge/importCacheFileTPO.py +python3 ./test.py -f import_merge/importCacheFileT.py +python3 ./test.py -f import_merge/importDataH2.py +python3 ./test.py -f import_merge/importDataHO2.py +python3 ./test.py -f import_merge/importDataHO.py +python3 ./test.py -f import_merge/importDataHPO.py +python3 ./test.py -f import_merge/importDataLastHO.py +python3 ./test.py -f import_merge/importDataLastHPO.py +python3 ./test.py -f import_merge/importDataLastH.py +python3 ./test.py -f import_merge/importDataLastS.py +python3 ./test.py -f import_merge/importDataLastSub.py +python3 ./test.py -f import_merge/importDataLastTO.py +python3 ./test.py -f import_merge/importDataLastTPO.py +python3 ./test.py -f import_merge/importDataLastT.py +python3 ./test.py -f import_merge/importDataS.py +python3 ./test.py -f import_merge/importDataSub.py +python3 ./test.py -f import_merge/importDataTO.py +python3 ./test.py -f import_merge/importDataTPO.py +python3 ./test.py -f import_merge/importDataT.py +python3 ./test.py -f import_merge/importHeadOverlap.py +python3 ./test.py -f import_merge/importHeadPartOverlap.py +python3 ./test.py -f import_merge/importHead.py +python3 ./test.py -f import_merge/importHORestart.py +python3 ./test.py -f import_merge/importHPORestart.py +python3 ./test.py -f import_merge/importHRestart.py +python3 ./test.py -f import_merge/importLastHO.py +python3 ./test.py -f import_merge/importLastHPO.py +python3 ./test.py -f import_merge/importLastH.py +python3 ./test.py -f import_merge/importLastS.py +python3 ./test.py -f import_merge/importLastSub.py +python3 ./test.py -f import_merge/importLastTO.py +python3 ./test.py -f import_merge/importLastTPO.py +python3 ./test.py -f import_merge/importLastT.py +python3 ./test.py -f import_merge/importSpan.py +python3 ./test.py -f import_merge/importSRestart.py +python3 ./test.py -f import_merge/importSubRestart.py +python3 ./test.py -f import_merge/importTailOverlap.py +python3 ./test.py -f import_merge/importTailPartOverlap.py +python3 ./test.py -f import_merge/importTail.py +python3 ./test.py -f import_merge/importToCommit.py +python3 ./test.py -f import_merge/importTORestart.py +python3 ./test.py -f import_merge/importTPORestart.py +python3 ./test.py -f import_merge/importTRestart.py +python3 ./test.py -f import_merge/importInsertThenImport.py +python3 ./test.py -f import_merge/importCSV.py +# user +python3 ./test.py -f user/user_create.py +python3 ./test.py -f user/pass_len.py + +# stable +python3 ./test.py -f stable/query_after_reset.py + +# table +python3 ./test.py -f table/del_stable.py + +#query +python3 ./test.py -f query/filter.py +python3 ./test.py -f query/filterCombo.py +python3 ./test.py -f query/queryNormal.py +python3 ./test.py -f query/queryError.py +python3 ./test.py -f query/filterAllIntTypes.py +python3 ./test.py -f query/filterFloatAndDouble.py +python3 ./test.py -f query/filterOtherTypes.py +python3 ./test.py -f query/querySort.py +python3 ./test.py -f query/queryJoin.py +python3 ./test.py -f query/select_last_crash.py +python3 ./test.py -f query/queryNullValueTest.py +python3 ./test.py -f query/queryInsertValue.py +python3 ./test.py -f query/queryConnection.py +python3 ./test.py -f query/queryCountCSVData.py +python3 ./test.py -f query/natualInterval.py +python3 ./test.py -f query/bug1471.py +#python3 ./test.py -f query/dataLossTest.py +python3 ./test.py -f query/bug1874.py +python3 ./test.py -f query/bug1875.py +python3 ./test.py -f query/bug1876.py +python3 ./test.py -f query/bug2218.py + +#stream +python3 ./test.py -f stream/metric_1.py +python3 ./test.py -f stream/new.py +python3 ./test.py -f stream/stream1.py +python3 ./test.py -f stream/stream2.py +#python3 ./test.py -f stream/parser.py +python3 ./test.py -f stream/history.py + +#alter table +python3 ./test.py -f alter/alter_table_crash.py + +# client +python3 ./test.py -f client/client.py +python3 ./test.py -f client/version.py +python3 ./test.py -f client/alterDatabase.py + +# Misc +python3 testCompress.py +python3 testNoCompress.py +python3 testMinTablesPerVnode.py + +# functions +python3 ./test.py -f functions/function_avg.py -r 1 +python3 ./test.py -f functions/function_bottom.py -r 1 +python3 ./test.py -f functions/function_count.py -r 1 +python3 ./test.py -f functions/function_diff.py -r 1 +python3 ./test.py -f functions/function_first.py -r 1 +python3 ./test.py -f functions/function_last.py -r 1 +python3 ./test.py -f functions/function_last_row.py -r 1 +python3 ./test.py -f functions/function_leastsquares.py -r 1 +python3 ./test.py -f functions/function_max.py -r 1 +python3 ./test.py -f functions/function_min.py -r 1 +python3 ./test.py -f functions/function_operations.py -r 1 +python3 ./test.py -f functions/function_percentile.py -r 1 +python3 ./test.py -f functions/function_spread.py -r 1 +python3 ./test.py -f functions/function_stddev.py -r 1 +python3 ./test.py -f functions/function_sum.py -r 1 +python3 ./test.py -f functions/function_top.py -r 1 +#python3 ./test.py -f functions/function_twa.py -r 1 +python3 queryCount.py +python3 ./test.py -f query/queryGroupbyWithInterval.py +python3 client/twoClients.py +python3 test.py -f query/queryInterval.py +python3 test.py -f query/queryFillTest.py + +# tools +python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdumpTest.py +python3 test.py -f tools/lowaTest.py + +# subscribe +python3 test.py -f subscribe/singlemeter.py +#python3 test.py -f subscribe/stability.py +python3 test.py -f subscribe/supertable.py + + diff --git a/tests/pytest/pytest_2.sh b/tests/pytest/pytest_2.sh new file mode 100755 index 0000000000..fededea3bb --- /dev/null +++ b/tests/pytest/pytest_2.sh @@ -0,0 +1,17 @@ + + +# update +python3 ./test.py -f update/allow_update.py +python3 ./test.py -f update/allow_update-0.py +python3 ./test.py -f update/append_commit_data.py +python3 ./test.py -f update/append_commit_last-0.py +python3 ./test.py -f update/append_commit_last.py +python3 ./test.py -f update/merge_commit_data.py +python3 ./test.py -f update/merge_commit_data-0.py +python3 ./test.py -f update/merge_commit_data2.py +python3 ./test.py -f update/merge_commit_data2_update0.py +python3 ./test.py -f update/merge_commit_last-0.py +python3 ./test.py -f update/merge_commit_last.py + +# wal +python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file diff --git a/tests/test-all.sh b/tests/test-all.sh index 5897978bce..14b649eddf 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -137,6 +137,12 @@ if [ "$2" != "sim" ]; then elif [ "$1" == "pytest" ]; then echo "### run Python full test ###" runPyCaseOneByOne fulltest.sh + elif [ "$1" == "p1" ]; then + echo "### run Python_1 test ###" + runPyCaseOneByOne pytest_1.sh + elif [ "$1" == "p2" ]; then + echo "### run Python_2 test ###" + runPyCaseOneByOne pytest_2.sh elif [ "$1" == "b2" ] || [ "$1" == "b3" ]; then exit $(($totalFailed + $totalPyFailed)) elif [ "$1" == "smoke" ] || [ -z "$1" ]; then From 4e0f1055dce5aaf60d70b122307f5026ba37e2c7 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 1 Dec 2020 03:44:37 +0000 Subject: [PATCH 107/188] remove a potential rpcSendRecv deadlock --- src/rpc/src/rpcMain.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index acceaf9d7a..4ccdec2bc7 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -630,7 +630,14 @@ static void rpcReleaseConn(SRpcConn *pConn) { } else { // if there is an outgoing message, free it if (pConn->outType && pConn->pReqMsg) { - if (pConn->pContext) pConn->pContext->pConn = NULL; + if (pContext->pRsp) { + // for synchronous API, post semaphore to unblock app + pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; + pContext->pRsp->pCont = NULL; + pContext->pRsp->contLen = 0; + tsem_post(pContext->pSem); + } + pConn->pContext->pConn = NULL; taosRemoveRef(tsRpcRefId, pConn->pContext->rid); } } From 7fadd4cb3aec7b30c99572586b3a1179f87a4838 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Tue, 1 Dec 2020 03:48:24 +0000 Subject: [PATCH 108/188] remove a potential rpcSendRecv deadlock if rpcClose is called by another thread --- src/rpc/src/rpcMain.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 4ccdec2bc7..00a97d7bc2 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -630,6 +630,7 @@ static void rpcReleaseConn(SRpcConn *pConn) { } else { // if there is an outgoing message, free it if (pConn->outType && pConn->pReqMsg) { + SRpcReqContext *pContext = pConn->pContext; if (pContext->pRsp) { // for synchronous API, post semaphore to unblock app pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; @@ -637,8 +638,8 @@ static void rpcReleaseConn(SRpcConn *pConn) { pContext->pRsp->contLen = 0; tsem_post(pContext->pSem); } - pConn->pContext->pConn = NULL; - taosRemoveRef(tsRpcRefId, pConn->pContext->rid); + pContext->pConn = NULL; + taosRemoveRef(tsRpcRefId, pContext->rid); } } From bbafebfbeef323b9308e685baee4eaa900d4276e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 1 Dec 2020 11:52:43 +0800 Subject: [PATCH 109/188] TD-1926 --- src/sync/src/syncRetrieve.c | 72 ++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/src/sync/src/syncRetrieve.c b/src/sync/src/syncRetrieve.c index 44d2680b1a..36b197dd46 100644 --- a/src/sync/src/syncRetrieve.c +++ b/src/sync/src/syncRetrieve.c @@ -97,6 +97,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { while (1) { // retrieve file info fileInfo.name[0] = 0; + fileInfo.size = 0; fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion); // fileInfo.size = htonl(size); @@ -105,14 +106,14 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // send the file info int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = -1; sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } // if no file anymore, break if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { - code = TSDB_CODE_SUCCESS; + code = 0; sDebug("%s, no more files to sync", pPeer->id); break; } @@ -120,7 +121,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // wait for the ack from peer ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(fileAck)); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = -1; sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -141,7 +142,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { // send the file to peer int32_t sfd = open(name, O_RDONLY); if (sfd < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = -1; sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -149,7 +150,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { ret = taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); close(sfd); if (ret < 0) { - code = TAOS_SYSTEM_ERROR(errno); + code = -1; sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno)); break; } @@ -165,7 +166,7 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) { } if (code != TSDB_CODE_SUCCESS) { - sError("%s, failed to retrieve file since %s", pPeer->id, tstrerror(code)); + sError("%s, failed to retrieve file, code:0x%x", pPeer->id, code); } return code; @@ -180,7 +181,7 @@ static int32_t syncReadOneWalRecord(int32_t sfd, SWalHead *pHead) { } if (ret == 0) { - sDebug("sfd:%d, read to end of the wal, ret:%d", sfd, ret); + sTrace("sfd:%d, read to the end of file, ret:%d", sfd, ret); return 0; } @@ -254,7 +255,7 @@ static int32_t syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversi if (pHead->version >= fversion && fversion > 0) { code = 0; - sDebug("%s, retrieve wal finished, fver:%" PRIu64, pPeer->id, fversion); + sDebug("%s, retrieve wal finished, hver:%" PRIu64 " fver:%" PRIu64, pPeer->id, pHead->version, fversion); break; } } @@ -294,13 +295,14 @@ static int32_t syncProcessLastWal(SSyncPeer *pPeer, char *wname, int64_t index) if (fversion == 0) { pPeer->sstatus = TAOS_SYNC_STATUS_CACHE; // start to forward pkt fversion = nodeVersion; // must read data to fversion - sDebug("%s, fversion is 0 and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); + sDebug("%s, set sstatus:%s and fver:%" PRIu64, pPeer->id, syncStatus[pPeer->sstatus], fversion); } } // if all data up to fversion is read out, it is over if (pPeer->sversion >= fversion && fversion > 0) { - sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d", pPeer->id, fversion, bytes); + sDebug("%s, data up to fver:%" PRIu64 " has been read out, bytes:%d sver:%" PRIu64, pPeer->id, fversion, bytes, + pPeer->sversion); return 0; } @@ -324,7 +326,6 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { char fname[TSDB_FILENAME_LEN * 3]; char wname[TSDB_FILENAME_LEN * 2]; int32_t size; - struct stat fstat; int32_t code = -1; int64_t index = 0; @@ -332,9 +333,14 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { // retrieve wal info wname[0] = 0; code = (*pNode->getWalInfo)(pNode->vgId, wname, &index); - if (code < 0) break; // error + if (code < 0) { + sError("%s, failed to get wal info since:%s, code:0x%x", pPeer->id, strerror(errno), code); + break; + } + if (wname[0] == 0) { // no wal file - sDebug("%s, no wal file", pPeer->id); + code = 0; + sDebug("%s, no wal file anymore", pPeer->id); break; } @@ -346,20 +352,35 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { // get the full path to wal file snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname); - // send wal file, - // inotify is not required, old wal file won't be modified, even remove is ok - if (stat(fname, &fstat) < 0) break; - size = fstat.st_size; + // send wal file, old wal file won't be modified, even remove is ok + struct stat fstat; + if (stat(fname, &fstat) < 0) { + code = -1; + sDebug("%s, failed to stat wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + break; + } + size = fstat.st_size; sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size); + int32_t sfd = open(fname, O_RDONLY); - if (sfd < 0) break; + if (sfd < 0) { + code = -1; + sError("%s, failed to open wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + break; + } code = taosSendFile(pPeer->syncFd, sfd, NULL, size); close(sfd); - if (code < 0) break; + if (code < 0) { + sError("%s, failed to send wal:%s for retrieve since %s, code:0x%x", pPeer->id, fname, strerror(errno), code); + break; + } - if (syncAreFilesModified(pNode, pPeer)) break; + if (syncAreFilesModified(pNode, pPeer)) { + code = -1; + break; + } } if (code == 0) { @@ -368,9 +389,9 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) { SWalHead walHead; memset(&walHead, 0, sizeof(walHead)); - code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)); + taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead)); } else { - sError("%s, failed to send wal since %s", pPeer->id, strerror(errno)); + sError("%s, failed to send wal since %s, code:0x%x", pPeer->id, strerror(errno), code); } return code; @@ -410,7 +431,7 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { pPeer->sversion = 0; pPeer->sstatus = TAOS_SYNC_STATUS_FILE; sInfo("%s, start to retrieve files, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]); - if (syncRetrieveFile(pPeer) < 0) { + if (syncRetrieveFile(pPeer) != 0) { sError("%s, failed to retrieve files", pPeer->id); return -1; } @@ -419,8 +440,9 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) { if (pPeer->sversion == 0) pPeer->sversion = 1; sInfo("%s, start to retrieve wals", pPeer->id); - if (syncRetrieveWal(pPeer) < 0) { - sError("%s, failed to retrieve wals", pPeer->id); + int32_t code = syncRetrieveWal(pPeer); + if (code != 0) { + sError("%s, failed to retrieve wals, code:0x%x", pPeer->id, code); return -1; } From c1b46153509597ea30d52873759f714550306653 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Tue, 1 Dec 2020 13:19:32 +0800 Subject: [PATCH 110/188] [TD-2255]: make db options days and keep error message more clear --- src/inc/taoserror.h | 3 +++ src/mnode/src/mnodeDb.c | 12 +++++------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 5fde6d40d5..b526fe2b09 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -184,6 +184,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_DATABASES, 0, 0x0385, "Too many d TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_IN_DROPPING, 0, 0x0386, "Database not available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_VGROUP_NOT_READY, 0, 0x0387, "Database unsynced") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_DAYS, 0, 0x0390, "Invalid database option: days out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION_KEEP, 0, 0x0391, "Invalid database option: keep >= keep2 >= keep1 >= days") + // dnode TAOS_DEFINE_ERROR(TSDB_CODE_DND_MSG_NOT_PROCESSED, 0, 0x0400, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_DND_OUT_OF_MEMORY, 0, 0x0401, "Dnode out of memory") diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 999c606284..39a0c03008 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -236,30 +236,28 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { mError("invalid db option daysPerFile:%d valid range: [%d, %d]", pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - return TSDB_CODE_MND_INVALID_DB_OPTION; + return TSDB_CODE_MND_INVALID_DB_OPTION_DAYS; } if (pCfg->daysToKeep < TSDB_MIN_KEEP || pCfg->daysToKeep > TSDB_MAX_KEEP) { mError("invalid db option daysToKeep:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); - return TSDB_CODE_MND_INVALID_DB_OPTION; + return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP; } if (pCfg->daysToKeep < pCfg->daysPerFile) { mError("invalid db option daysToKeep:%d should larger than daysPerFile:%d", pCfg->daysToKeep, pCfg->daysPerFile); - return TSDB_CODE_MND_INVALID_DB_OPTION; + return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP; } -#if 0 if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > pCfg->daysToKeep) { mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, pCfg->daysToKeep); - return TSDB_CODE_MND_INVALID_DB_OPTION; + return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP; } if (pCfg->daysToKeep1 < TSDB_MIN_KEEP || pCfg->daysToKeep1 > pCfg->daysToKeep2) { mError("invalid db option daysToKeep1:%d valid range: [%d, %d]", pCfg->daysToKeep1, TSDB_MIN_KEEP, pCfg->daysToKeep2); - return TSDB_CODE_MND_INVALID_DB_OPTION; + return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP; } -#endif if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { mError("invalid db option maxRowsPerFileBlock:%d valid range: [%d, %d]", pCfg->maxRowsPerFileBlock, From dcb0bc49e846890780044873cf7913df8539e1c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Dec 2020 13:20:53 +0800 Subject: [PATCH 111/188] [TD-2129] --- src/client/src/tscFunctionImpl.c | 218 +++++++++++++++----- src/query/src/qExecutor.c | 339 ++++++++++++++++++++----------- 2 files changed, 394 insertions(+), 163 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 3867032d46..56b7f052f7 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -426,8 +426,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { } SET_VAL(pCtx, 1, 1); - - *((int64_t *)pCtx->aOutputBuf) += 1; + *((int64_t *)pCtx->aOutputBuf) += pCtx->size; // do not need it actually SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); @@ -3632,114 +3631,119 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { return true; } -static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t index, int32_t size) { +static int32_t twa_function_impl(SQLFunctionCtx* pCtx, int32_t tsIndex, int32_t index, int32_t size) { int32_t notNullElems = 0; TSKEY *primaryKey = pCtx->ptsList; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + int32_t i = index; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); if (pCtx->start.key != INT64_MIN) { - assert(pCtx->start.key < primaryKey[index] && pInfo->lastKey == INT64_MIN); + assert((pCtx->start.key < primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_ASC) || + (pCtx->start.key > primaryKey[tsIndex + i] && pCtx->order == TSDB_ORDER_DESC)); - pInfo->lastKey = primaryKey[index]; - GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); + assert(pInfo->lastKey == INT64_MIN); + + pInfo->lastKey = primaryKey[tsIndex + i]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pCtx->start.key; + pInfo->win.skey = pCtx->start.key; notNullElems++; - i += 1; + i += step; } else if (pInfo->lastKey == INT64_MIN) { - pInfo->lastKey = primaryKey[index]; - GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, 0)); + pInfo->lastKey = primaryKey[tsIndex + i]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); pInfo->hasResult = DATA_SET_FLAG; - pInfo->win.skey = pInfo->lastKey; + pInfo->win.skey = pInfo->lastKey; notNullElems++; - i += 1; + i += step; } // calculate the value of switch(pCtx->inputType) { case TSDB_DATA_TYPE_TINYINT: { - int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_SMALLINT: { - int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_INT: { - int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_BIGINT: { - int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = (double) val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_FLOAT: { - float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } case TSDB_DATA_TYPE_DOUBLE: { - double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); - for (; i < size; i++) { + double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, 0); + for (; i < size && i >= 0; i += step) { if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { continue; } - pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i] - pInfo->lastKey); + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + tsIndex] - pInfo->lastKey); pInfo->lastValue = val[i]; - pInfo->lastKey = primaryKey[i]; + pInfo->lastKey = primaryKey[i + tsIndex]; } break; } @@ -3764,16 +3768,13 @@ static void twa_function(SQLFunctionCtx *pCtx) { STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); // skip null value - int32_t i = 0; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pCtx->order); + int32_t i = (pCtx->order == TSDB_ORDER_ASC)? 0:(pCtx->size - 1); while (pCtx->hasNull && i < pCtx->size && isNull((char *)data + pCtx->inputBytes * i, pCtx->inputType)) { - i++; + i += step; } - - if (i >= pCtx->size) { - return; - } - - int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, pCtx->size); + + int32_t notNullElems = twa_function_impl(pCtx, pCtx->startOffset, i, pCtx->size); SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { @@ -3791,11 +3792,136 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - int32_t notNullElems = twa_function_impl(pCtx, index, 1); + int32_t notNullElems = 0; + TSKEY *primaryKey = pCtx->ptsList; + + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); + int32_t i = pCtx->startOffset; + int32_t size = pCtx->size; + + if (pCtx->start.key != INT64_MIN) { + assert(pInfo->lastKey == INT64_MIN); + + pInfo->lastKey = primaryKey[index]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + + pInfo->dOutput += ((pInfo->lastValue + pCtx->start.val) / 2) * (pInfo->lastKey - pCtx->start.key); + + pInfo->hasResult = DATA_SET_FLAG; + pInfo->win.skey = pCtx->start.key; + notNullElems++; + i += 1; + } else if (pInfo->lastKey == INT64_MIN) { + pInfo->lastKey = primaryKey[index]; + GET_TYPED_DATA(pInfo->lastValue, double, pCtx->inputType, GET_INPUT_CHAR_INDEX(pCtx, index)); + + pInfo->hasResult = DATA_SET_FLAG; + pInfo->win.skey = pInfo->lastKey; + notNullElems++; + i += 1; + } + + // calculate the value of + switch(pCtx->inputType) { + case TSDB_DATA_TYPE_TINYINT: { + int8_t *val = (int8_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + int16_t *val = (int16_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_INT: { + int32_t *val = (int32_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_BIGINT: { + int64_t *val = (int64_t*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = (double) val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_FLOAT: { + float *val = (float*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + double *val = (double*) GET_INPUT_CHAR_INDEX(pCtx, index); + for (; i < size; i++) { + if (pCtx->hasNull && isNull((const char*) &val[i], pCtx->inputType)) { + continue; + } + + pInfo->dOutput += ((val[i] + pInfo->lastValue) / 2) * (primaryKey[i + index] - pInfo->lastKey); + pInfo->lastValue = val[i]; + pInfo->lastKey = primaryKey[i + index]; + } + break; + } + default: assert(0); + } + + // the last interpolated time window value + if (pCtx->end.key != INT64_MIN) { + pInfo->dOutput += ((pInfo->lastValue + pCtx->end.val) / 2) * (pCtx->end.key - pInfo->lastKey); + pInfo->lastValue = pCtx->end.val; + pInfo->lastKey = pCtx->end.key; + } + + pInfo->win.ekey = pInfo->lastKey; + SET_VAL(pCtx, notNullElems, 1); + if (notNullElems > 0) { + pResInfo->hasResult = DATA_SET_FLAG; + } + if (pCtx->stableQuery) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); } } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index ec70280c7f..0133035a7b 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -196,6 +196,7 @@ static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); +static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -660,7 +661,7 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { } } -static bool isResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { +static bool resultRowInterpolated(SResultRow* pResult, SResultTsInterpType type) { assert(pResult != NULL && (type == RESULT_ROW_START_INTERP || type == RESULT_ROW_END_INTERP)); if (type == RESULT_ROW_START_INTERP) { return pResult->startInterp == true; @@ -989,7 +990,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - sas->offset = (QUERY_IS_ASC_QUERY(pQuery)) ? pQuery->pos : pQuery->pos - (size - 1); + sas->offset = (QUERY_IS_ASC_QUERY(pQuery))? pQuery->pos : pQuery->pos - (size - 1); sas->colList = pQuery->colList; sas->numOfCols = pQuery->numOfCols; sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); @@ -1032,85 +1033,89 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas return dataBlock; } -// window start key interpolation -static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) { - SQuery* pQuery = pRuntimeEnv->pQuery; - - TSKEY start = tsCols[pos]; - TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; - TSKEY prevTs = (pos == 0)? lastTs : tsCols[pos - 1]; - - // if lastTs == INT64_MIN, it is the first block, no need to do the start time interpolation - if (((lastTs != INT64_MIN && pos >= 0) || (lastTs == INT64_MIN && pos > 0)) && win->skey > lastTs && - win->skey < start) { - - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - if (k == 0 && pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - assert(pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP); - continue; - } - - double v1 = 0, v2 = 0, v = 0; - - char *prevVal = pos == 0 ? pRuntimeEnv->prevRow[k] : ((char*)pColInfo->pData) + (pos - 1) * pColInfo->info.bytes; - - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal); - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes); - - SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; - SPoint point2 = (SPoint){.key = start, .val = &v2}; - SPoint point = (SPoint){.key = win->skey, .val = &v}; - taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); - pRuntimeEnv->pCtx[k].start.key = point.key; - pRuntimeEnv->pCtx[k].start.val = v; +static void setNotInterpoWindowKey(SQLFunctionCtx* pCtx, int32_t numOfOutput, int32_t type) { + if (type == RESULT_ROW_START_INTERP) { + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].start.key = INT64_MIN; } - - return true; } else { - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - pRuntimeEnv->pCtx[k].start.key = INT64_MIN; + for (int32_t k = 0; k < numOfOutput; ++k) { + pCtx[k].end.key = INT64_MIN; } - - return false; } } -static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, SArray* pDataBlock, TSKEY* tsCols, TSKEY ekey, STimeWindow* win) { +//static double getTSWindowInterpoVal(SColumnInfoData* pColInfo, int16_t srcColIndex, int16_t rowIndex, TSKEY key, char** prevRow, TSKEY* tsCols, int32_t step) { +// TSKEY start = tsCols[rowIndex]; +// TSKEY prevTs = (rowIndex == 0)? *(TSKEY *) prevRow[0] : tsCols[rowIndex - step]; +// +// double v1 = 0, v2 = 0, v = 0; +// char *prevVal = (rowIndex == 0)? prevRow[srcColIndex] : ((char*)pColInfo->pData) + (rowIndex - step) * pColInfo->info.bytes; +// +// GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)prevVal); +// GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + rowIndex * pColInfo->info.bytes); +// +// SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; +// SPoint point2 = (SPoint){.key = start, .val = &v2}; +// SPoint point = (SPoint){.key = key, .val = &v}; +// taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); +// +// return v; +//} + +// window start key interpolation +static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t pos, int32_t numOfRows, SArray* pDataBlock, TSKEY* tsCols, STimeWindow* win) { SQuery* pQuery = pRuntimeEnv->pQuery; - TSKEY trueEndKey = tsCols[pos]; - if (win->ekey < ekey && win->ekey != trueEndKey) { - int32_t nextIndex = pos + 1; - TSKEY next = tsCols[nextIndex]; - - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - if (k == 0 && pColInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP && - pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - continue; - } - - double v1 = 0, v2 = 0, v = 0; - GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + pos * pColInfo->info.bytes); - GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + nextIndex * pColInfo->info.bytes); - - SPoint point1 = (SPoint){.key = trueEndKey, .val = &v1}; - SPoint point2 = (SPoint){.key = next, .val = &v2}; - SPoint point = (SPoint){.key = win->ekey, .val = &v}; - taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); - pRuntimeEnv->pCtx[k].end.key = point.key; - pRuntimeEnv->pCtx[k].end.val = v; - } + TSKEY curTs = tsCols[pos]; + TSKEY lastTs = *(TSKEY *) pRuntimeEnv->prevRow[0]; + // lastTs == INT64_MIN and pos == 0 means this is the first time window, interpolation is not needed. + // start exactly from this point, no need to do interpolation + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; + if (key == curTs) { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); return true; - } else { // current time window does not ended in current data block, do nothing - for (int32_t k = 0; k < pQuery->numOfCols; ++k) { - pRuntimeEnv->pCtx[k].end.key = INT64_MIN; - } + } + if (lastTs == INT64_MIN && ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))) { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + TSKEY prevTs = ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))? + lastTs:tsCols[pos - step]; + + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, pos - step, curTs, pos, key, RESULT_ROW_START_INTERP); + return true; +} + +static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32_t endRowIndex, SArray* pDataBlock, TSKEY* tsCols, TSKEY blockEkey, STimeWindow* win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + TSKEY actualEndKey = tsCols[endRowIndex]; + + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; + + // not ended in current data block, do not invoke interpolation + if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); return false; } + + // there is actual end point of current time window, no interpolation need + if (key == actualEndKey) { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); + return true; + } + + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); + int32_t nextRowIndex = endRowIndex + step; + assert(nextRowIndex >= 0); + + TSKEY nextKey = tsCols[nextRowIndex]; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, actualEndKey, endRowIndex, nextKey, nextRowIndex, key, RESULT_ROW_END_INTERP); + return true; } static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray* pDataBlock) { @@ -1119,10 +1124,10 @@ static void saveDataBlockLastRow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* } SQuery* pQuery = pRuntimeEnv->pQuery; + int32_t rowIndex = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->rows-1:0; for (int32_t k = 0; k < pQuery->numOfCols; ++k) { SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, k); - memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * (pDataBlockInfo->rows - 1)), - pColInfo->info.bytes); + memcpy(pRuntimeEnv->prevRow[k], ((char*)pColInfo->pData) + (pColInfo->info.bytes * rowIndex), pColInfo->info.bytes); } } @@ -1174,11 +1179,13 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (QUERY_IS_INTERVAL_QUERY(pQuery)) { + int32_t prevIndex = curTimeWindowIndex(pWindowResInfo); + TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step); + STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; SResultRow* pResult = NULL; - STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); if (ret != TSDB_CODE_SUCCESS) { tfree(sasArray); @@ -1193,20 +1200,47 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); + // prev time window not interpolation yet. + int32_t curIndex = curTimeWindowIndex(pWindowResInfo); + if (prevIndex != -1 && prevIndex < curIndex) { + for(int32_t j = prevIndex; j < curIndex; ++j) { + SResultRow *pRes = pWindowResInfo->pResult[j]; + + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); + assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, *(TSKEY*) pRuntimeEnv->prevRow[0], -1, tsCols[0], p, w.ekey, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + + bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, closed, &w, startPos, 0, tsCols, pDataBlockInfo->rows); + } + + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult); + assert (ret == TSDB_CODE_SUCCESS); // null data, too many state code + } + // window start key interpolation if (pRuntimeEnv->timeWindowInterpo) { - bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, pQuery->pos, pDataBlock, tsCols, &win); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + int32_t startRowIndex = pQuery->pos; + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } - alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, pQuery->pos + forwardStep - 1, pDataBlock, tsCols, - pDataBlockInfo->window.ekey, &win); + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = pQuery->pos + (forwardStep - 1) * step; + + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &win); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } @@ -1243,17 +1277,20 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * // window start(end) key interpolation if (pRuntimeEnv->timeWindowInterpo) { - bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startPos, pDataBlock, tsCols, &nextWin); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + int32_t startRowIndex = startPos; + bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &nextWin); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); } } - alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, startPos + forwardStep - 1, pDataBlock, tsCols, pDataBlockInfo->window.ekey, &nextWin); + done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP); + if (!done) { + int32_t endRowIndex = startPos + (forwardStep - 1)*step; + TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey; + bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &nextWin); if (interp) { setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); } @@ -1459,6 +1496,45 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return true; } +void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type) { + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + int32_t functionId = pQuery->pExpr1[k].base.functionId; + if (functionId != TSDB_FUNC_TWA) { + pRuntimeEnv->pCtx[k].start.key = INT64_MIN; + continue; + } + + SColIndex* pColIndex = &pQuery->pExpr1[k].base.colInfo; + int16_t index = pColIndex->colIndex; + SColumnInfoData* pColInfo = taosArrayGet(pDataBlock, index); + + assert(pColInfo->info.colId == pColIndex->colId && curTs != windowKey); + double v1 = 0, v2 = 0, v = 0; + + if (prevRowIndex == -1) { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[k]); + } else { + GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes); + } + + GET_TYPED_DATA(v2, double, pColInfo->info.type, (char *)pColInfo->pData + curRowIndex * pColInfo->info.bytes); + + SPoint point1 = (SPoint){.key = prevTs, .val = &v1}; + SPoint point2 = (SPoint){.key = curTs, .val = &v2}; + SPoint point = (SPoint){.key = windowKey, .val = &v}; + taosGetLinearInterpolationVal(TSDB_DATA_TYPE_DOUBLE, &point1, &point2, &point); + + if (type == RESULT_ROW_START_INTERP) { + pRuntimeEnv->pCtx[k].start.key = point.key; + pRuntimeEnv->pCtx[k].start.val = v; + } else { + pRuntimeEnv->pCtx[k].end.key = point.key; + pRuntimeEnv->pCtx[k].end.val = v; + } + } +} + static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -1489,6 +1565,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); + pCtx[k].size = 1; } // set the input column data @@ -1508,7 +1585,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } int32_t offset = -1; -// TSKEY prev = -1; + TSKEY prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0]; + int32_t prevRowIndex = -1; for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { offset = GET_COL_DATA_POS(pQuery, j, step); @@ -1530,7 +1608,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { - int64_t ts = tsCols[offset]; + int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); + + int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; @@ -1543,27 +1623,58 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (!hasTimeWindow) { continue; } -/* + // window start key interpolation if (pRuntimeEnv->timeWindowInterpo) { - bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, pos, pDataBlock, tsCols, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + // check for the time window end time interpolation + int32_t curIndex = curTimeWindowIndex(pWindowResInfo); + if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { + for (int32_t k = prevWindowIndex; k < curIndex; ++k) { + SResultRow *pRes = pWindowResInfo->pResult[k]; + STimeWindow w = pRes->win; + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); + assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); + + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? w.ekey:w.skey; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + pRuntimeEnv->pCtx[i].size = 0; + } + + bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &w, offset); + } + + // restore current time window + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, + &pResult); + if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code + continue; } } - alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, pQuery->pos + forwardStep - 1, pDataBlock, tsCols, - pDataBlockInfo->window.ekey, &win); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win.skey:win.ekey; + if (prevTs != INT64_MIN && ts != key) { + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); } + + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pRuntimeEnv->pCtx[k].size = 1; + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); } } -*/ + bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); @@ -1588,26 +1699,19 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { -/* - // window start(end) key interpolation - if (pRuntimeEnv->timeWindowInterpo) { - bool alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startPos, pDataBlock, tsCols, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && (prevTs < nextWin.skey)) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > nextWin.ekey))) { + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? nextWin.skey:nextWin.ekey; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); } - alreadyInterp = isResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - if (!alreadyInterp) { - bool interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, startPos + forwardStep - 1, pDataBlock, tsCols, pDataBlockInfo->window.ekey, &nextWin); - if (interp) { - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - } - } + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); } -*/ + closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } @@ -1633,7 +1737,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } -// prev = tsCols[offset]; + prevTs = tsCols[offset]; + prevRowIndex = offset; if (pRuntimeEnv->pTSBuf != NULL) { // if timestamp filter list is empty, quit current query From 114aef8e7eea8f7e7b844bd4d5af7b3764cc122d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Dec 2020 16:27:34 +0800 Subject: [PATCH 112/188] [TD-2295] feature: add test case to support hivemq extension. --- tests/pytest/hivemq-extension-test.py | 166 ++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 tests/pytest/hivemq-extension-test.py diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py new file mode 100644 index 0000000000..22773a6581 --- /dev/null +++ b/tests/pytest/hivemq-extension-test.py @@ -0,0 +1,166 @@ +#!/usr/bin/python +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### +# install pip +# pip install src/connector/python/linux/python2/ +import sys +import os +import os.path +import time +import glob +import getopt +import subprocess +from shutil import which +from multipledispatch import dispatch + +@dispatch(str, str) +def v_print(msg: str, arg: str): + if verbose: + print(msg % arg) + + +@dispatch(str, int) +def v_print(msg: str, arg: int): + if verbose: + print(msg % int(arg)) + + +@dispatch(str, int, int) +def v_print(msg: str, arg1: int, arg2: int): + if verbose: + print(msg % (int(arg1), int(arg2))) + + +@dispatch(str, int, int, int) +def v_print(msg: str, arg1: int, arg2: int, arg3: int): + if verbose: + print(msg % (int(arg1), int(arg2), int(arg3))) + + +@dispatch(str, int, int, int, int) +def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): + if verbose: + print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) + +def isHiveMQInstalled(): + v_print("%s", "Check if HiveMQ installed") + defaultHiveMQPath = "/opt/hivemq*" + hiveMQDir = glob.glob(defaultHiveMQPath) + if (len(hiveMQDir) == 0): + return False + else: + v_print("HiveMQ installed at %s", hiveMQDir[0]) + return True + +def isMosquittoInstalled(): + v_print("%s", "Check if mosquitto installed") + if not which('mosquitto_pub'): + v_print("%s", "mosquitto is not installed") + return False + else: + return True + +def installExtension(): + currentDir = os.getcwd() + os.chdir('../../src/connector/hivemq-tdengine-extension') + v_print("%s", "build extension..") + os.system('mvn clean package') + + tdExtensionZip = 'target/hivemq-tdengine-extension*.zip' + tdExtensionZipDir = glob.glob(tdExtensionZip) + + defaultHiveMQPath = "/opt/hivemq*" + hiveMQDir = glob.glob(defaultHiveMQPath) + extPath = hiveMQDir[0] + '/extensions' + + tdExtDir = glob.glob(extPath + '/hivemq-tdengine-extension') + if len(tdExtDir): + v_print("%s", "delete exist extension..") + os.system('rm -rf %s' % tdExtDir[0]) + + v_print("%s", "unzip extension..") + os.system('unzip %s -d %s' % (tdExtensionZipDir[0], extPath)) + + os.chdir(currentDir) + +def stopHiveMQ(): + toBeKilled = "hivemq.jar" + psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % toBeKilled + + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + while(processID): + killCmd = "kill -TERM %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + print("hiveMQ processID: %s" % processID) + v_print("%s", "hivemq is NOT running") + +def runHiveMQ(): + defaultHiveMQPath = "/opt/hivemq*" + hiveMQDir = glob.glob(defaultHiveMQPath) + runPath = hiveMQDir[0] + '/bin/run.sh > /dev/null &' + os.system(runPath) + time.sleep(10) + v_print("%s", "hivemq is running") + +def runTDengine(): + pass + +def reCreateDatabase(): + os.system('taos -s "DROP DATABASE IF EXISTS hivemq"') + os.system('taos -s "CREATE DATABASE IF NOT EXISTS hivemq"') + +def sendMqttMsg(topic: str, payload: str): + testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload) + os.system(testStr) + time.sleep(3) + +def checkTDengineData(topic: str, payload: str): + output = subprocess.check_output('taos -s "select * from hivemq.mqtt_payload"', shell=True).decode('utf-8') + if (topic in output) and (payload in output): + v_print("%s", output) + return True + else: + v_print("%s", "ERROR: mqtt topic or payload NOT found") + return False + + +if __name__ == "__main__": + verbose = True + testTopic = 'test' + testPayload = 'hello world' + + if not isHiveMQInstalled(): + sys.exit(1) + + if not isMosquittoInstalled(): + sys.exit(1) + + stopHiveMQ() + + installExtension() + + runTDengine() + + reCreateDatabase() + + runHiveMQ() + + sendMqttMsg(testTopic, testPayload) + + if not checkTDengineData(testTopic, testPayload): + sys.exit(1) + + sys.exit(0) From 2558e74bc6ad4c7d93cc1693cae98c0a6eb16f8d Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Dec 2020 16:58:57 +0800 Subject: [PATCH 113/188] launch taosd from build directory instead of system. --- tests/pytest/hivemq-extension-test.py | 53 +++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 8 deletions(-) diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py index 22773a6581..daff763f72 100644 --- a/tests/pytest/hivemq-extension-test.py +++ b/tests/pytest/hivemq-extension-test.py @@ -91,9 +91,8 @@ def installExtension(): os.chdir(currentDir) -def stopHiveMQ(): - toBeKilled = "hivemq.jar" - psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % toBeKilled +def stopProgram(prog: str): + psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") @@ -104,7 +103,10 @@ def stopHiveMQ(): time.sleep(1) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") - print("hiveMQ processID: %s" % processID) + pass + +def stopHiveMQ(): + stopProgram("hivemq.jar") v_print("%s", "hivemq is NOT running") def runHiveMQ(): @@ -115,12 +117,44 @@ def runHiveMQ(): time.sleep(10) v_print("%s", "hivemq is running") +def getBuildPath(): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + def runTDengine(): - pass + stopProgram("taosd") + + buildPath = getBuildPath() + + if (buildPath == ""): + v_print("%s", "taosd NOT found!") + os.exit(1) + else: + v_print("%s", "taosd found in %s" % buildPath) + + binPath = buildPath + "/build/bin/taosd" + + os.system('%s > /dev/null &' % binPath) + time.sleep(10) def reCreateDatabase(): - os.system('taos -s "DROP DATABASE IF EXISTS hivemq"') - os.system('taos -s "CREATE DATABASE IF NOT EXISTS hivemq"') + buildPath = getBuildPath() + binPath = buildPath + "/build/bin/taos" + + os.system('%s -s "DROP DATABASE IF EXISTS hivemq"' % binPath) + os.system('%s -s "CREATE DATABASE IF NOT EXISTS hivemq"' % binPath) def sendMqttMsg(topic: str, payload: str): testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload) @@ -128,7 +162,10 @@ def sendMqttMsg(topic: str, payload: str): time.sleep(3) def checkTDengineData(topic: str, payload: str): - output = subprocess.check_output('taos -s "select * from hivemq.mqtt_payload"', shell=True).decode('utf-8') + buildPath = getBuildPath() + binPath = buildPath + "/build/bin/taos" + + output = subprocess.check_output('%s -s "select * from hivemq.mqtt_payload"' % binPath, shell=True).decode('utf-8') if (topic in output) and (payload in output): v_print("%s", output) return True From c3c7643fac71a9b844234df2585f768eefe078e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Dec 2020 16:59:25 +0800 Subject: [PATCH 114/188] [TD-2277]: fix invalid column type in stable query. --- src/client/src/tscLocalMerge.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 2cd37013c5..9fdadfa957 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -721,10 +721,16 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr // final result depends on the fields number memset(pSchema, 0, sizeof(SSchema) * size); + for (int32_t i = 0; i < size; ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - SSchema *p1 = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); + SSchema p1 = {0}; + if (pExpr->colInfo.colIndex != TSDB_TBNAME_COLUMN_INDEX) { + p1 = *tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, pExpr->colInfo.colIndex); + } else { + p1 = tGetTableNameColumnSchema(); + } int32_t inter = 0; int16_t type = -1; @@ -743,7 +749,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr functionId = TSDB_FUNC_LAST; } - getResultDataInfo(p1->type, p1->bytes, functionId, 0, &type, &bytes, &inter, 0, false); + int32_t ret = getResultDataInfo(p1.type, p1.bytes, functionId, 0, &type, &bytes, &inter, 0, false); + assert(ret == TSDB_CODE_SUCCESS); } pSchema[i].type = (uint8_t)type; From 43a3f513c4bd14f3ce702941451f641704bbfd51 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Dec 2020 17:01:44 +0800 Subject: [PATCH 115/188] [TD-2129] --- src/query/src/qExecutor.c | 96 +++++----- tests/script/general/parser/function.sim | 228 +++++++++++++++++++++++ 2 files changed, 277 insertions(+), 47 deletions(-) create mode 100644 tests/script/general/parser/function.sim diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0133035a7b..6de57b32fd 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1074,12 +1074,12 @@ static bool setTimeWindowInterpolationStartTs(SQueryRuntimeEnv* pRuntimeEnv, int // start exactly from this point, no need to do interpolation TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; if (key == curTs) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); return true; } if (lastTs == INT64_MIN && ((pos == 0 && QUERY_IS_ASC_QUERY(pQuery)) || (pos == (numOfRows - 1) && !QUERY_IS_ASC_QUERY(pQuery)))) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); return true; } @@ -1099,13 +1099,13 @@ static bool setTimeWindowInterpolationEndTs(SQueryRuntimeEnv* pRuntimeEnv, int32 // not ended in current data block, do not invoke interpolation if ((key > blockEkey && QUERY_IS_ASC_QUERY(pQuery)) || (key < blockEkey && !QUERY_IS_ASC_QUERY(pQuery))) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); return false; } // there is actual end point of current time window, no interpolation need if (key == actualEndKey) { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); return true; } @@ -1535,6 +1535,43 @@ void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDa } } +static void setTimeWindowSKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); + if (!done) { + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->skey:win->ekey; + if (key == ts) { + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } else if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && prevTs < key) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > key))) { + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } + + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP); + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { + pRuntimeEnv->pCtx[k].size = 1; + } + } else { + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + } +} + +static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY ts, int32_t offset, SResultRow* pResult, STimeWindow* win) { + SQuery* pQuery = pRuntimeEnv->pQuery; + + TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win->ekey:win->skey; + doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP); + setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); + + setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP); + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + pRuntimeEnv->pCtx[i].size = 0; + } +} + static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -1585,7 +1622,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } int32_t offset = -1; - TSKEY prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0]; + TSKEY prevTs = *(TSKEY*) pRuntimeEnv->prevRow[0]; int32_t prevRowIndex = -1; for (int32_t j = 0; j < pDataBlockInfo->rows; ++j) { @@ -1609,8 +1646,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // interval window query, decide the time window according to the primary timestamp if (QUERY_IS_INTERVAL_QUERY(pQuery)) { int32_t prevWindowIndex = curTimeWindowIndex(pWindowResInfo); + int64_t ts = tsCols[offset]; - int64_t ts = tsCols[offset]; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; @@ -1631,21 +1668,14 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (prevWindowIndex != -1 && prevWindowIndex < curIndex) { for (int32_t k = prevWindowIndex; k < curIndex; ++k) { SResultRow *pRes = pWindowResInfo->pResult[k]; - STimeWindow w = pRes->win; - ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult); + + ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &pRes->win, masterScan, &hasTimeWindow, &pResult); assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP)); - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? w.ekey:w.skey; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_END_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - pRuntimeEnv->pCtx[i].size = 0; - } + setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win); bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, closed, &w, offset); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &pRes->win, offset); } // restore current time window @@ -1656,23 +1686,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } } - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? win.skey:win.ekey; - if (prevTs != INT64_MIN && ts != key) { - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); - } - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); - for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { - pRuntimeEnv->pCtx[k].size = 1; - } - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); - } + setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win); } bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); @@ -1699,19 +1713,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { - bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP); - if (!done) { - if (prevTs != INT64_MIN && ((QUERY_IS_ASC_QUERY(pQuery) && (prevTs < nextWin.skey)) || (!QUERY_IS_ASC_QUERY(pQuery) && prevTs > nextWin.ekey))) { - TSKEY key = QUERY_IS_ASC_QUERY(pQuery)? nextWin.skey:nextWin.ekey; - doRowwiseTimeWindowInterpolation(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, key, RESULT_ROW_START_INTERP); - setResultRowInterpo(pResult, RESULT_ROW_START_INTERP); - } else { - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_START_INTERP); - } - - setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfCols, RESULT_ROW_END_INTERP); - } - + setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin); closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim new file mode 100644 index 0000000000..34e9844f71 --- /dev/null +++ b/tests/script/general/parser/function.sim @@ -0,0 +1,228 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 +system sh/cfg.sh -n dnode1 -c tableMetaKeepTimer -v 3 +system sh/exec.sh -n dnode1 -s start +sleep 500 +sql connect + +$dbPrefix = m_func_db +$tbPrefix = m_func_tb +$mtPrefix = m_func_mt + +$tbNum = 10 +$rowNum = 5 +$totalNum = $tbNum * $rowNum +$ts0 = 1537146000000 +$delta = 600000 +print ========== alter.sim +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql drop database if exists $db +sql create database $db +sql use $db + +print =====================================> test case for twa in single block + +sql create table t1 (ts timestamp, k float); +sql insert into t1 values('2015-08-18 00:00:00', 2.064); +sql insert into t1 values('2015-08-18 00:06:00', 2.116); +sql insert into t1 values('2015-08-18 00:12:00', 2.028); +sql insert into t1 values('2015-08-18 00:18:00', 2.126); +sql insert into t1 values('2015-08-18 00:24:00', 2.041); +sql insert into t1 values('2015-08-18 00:30:00', 2.051); + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:05:00' +if $rows != 1 then + return -1 +endi + +if $data00 != 2.063999891 then + return -1 +endi + +if $data01 != 2.063999891 then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00' +if $rows != 1 then + return -1 +endi + +if $data00 != 2.089999914 then + return -1 +endi + +if $data01 != 2.089999914 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00' interval(1m) order by ts asc +if $rows != 2 then + return -1 +endi + +if $data00 != @15-08-18 00:00:00.000@ then + return -1 +endi + +if $data01 != 2.068333156 then + return -1 +endi + +if $data02 != 2.063999891 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data10 != @15-08-18 00:06:00.000@ then + return -1 +endi + +if $data11 != 2.115999937 then + return -1 +endi + +if $data12 != 2.115999937 then + return -1 +endi + +if $data13 != 1 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:07:00' interval(1m) order by ts desc; +if $rows != 2 then + return -1 +endi + +if $data00 != @15-08-18 00:06:00.00@ then + return -1 +endi + +if $data01 != 2.115999937 then + return -1 +endi + +if $data02 != 2.115999937 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data11 != 2.068333156 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:27:00' interval(10m) order by ts asc +if $rows != 3 then + return -1 +endi + +if $data01 != 2.088666666 then + return -1 +endi + +if $data02 != 2.089999914 then + return -1 +endi + +if $data03 != 2 then + return -1 +endi + +if $data11 != 2.077099980 then + return -1 +endi + +if $data12 != 2.077000022 then + return -1 +endi + +if $data13 != 2 then + return -1 +endi + +if $data21 != 2.069333235 then + return -1 +endi + +if $data22 != 2.040999889 then + return -1 +endi + +if $data23 != 1 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:27:00' interval(10m) order by ts desc +if $rows != 3 then + return -1 +endi + +if $data01 != 2.069333235 then + return -1 +endi + +if $data11 != 2.077099980 then + return -1 +endi + +if $data21 != 2.088666666 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' order by ts asc +if $data00 != 2.073699975 then + return -1 +endi + +if $data01 != 2.070999980 then + return -1 +endi + +if $data02 != 6 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' order by ts desc +if $rows != 1 then + return -1 +endi + +if $data00 != 2.073699975 then + return -1 +endi + +if $data01 != 2.070999980 then + return -1 +endi + +if $data02 != 6 then + return -1 +endi + +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts asc +sql select twa(k),avg(k),count(1) from t1 where ts>='2015-8-18 00:00:00' and ts<='2015-8-18 00:30:00' interval(10m) order by ts desc + + +#todo add test case while column filte exists. + +select count(*),TWA(k) from tm0 where ts>='1970-1-1 13:43:00' and ts<='1970-1-1 13:44:10' interval(9s) From 6526ea44a9d6bd531346ef056750d5d644126ad4 Mon Sep 17 00:00:00 2001 From: Xiaowei Su <46439638+Shawshank-Smile@users.noreply.github.com> Date: Tue, 1 Dec 2020 17:24:51 +0800 Subject: [PATCH 116/188] Update cluster-ch.md --- documentation20/webdocs/markdowndocs/cluster-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/cluster-ch.md b/documentation20/webdocs/markdowndocs/cluster-ch.md index 10c28c284c..db479417c5 100644 --- a/documentation20/webdocs/markdowndocs/cluster-ch.md +++ b/documentation20/webdocs/markdowndocs/cluster-ch.md @@ -37,7 +37,7 @@ fqdn h1.taosdata.com // 配置本数据节点的端口号，缺省是6030 serverPort 6030 -// 服务端节点数为偶数的时候，需要配置，请参考《Arbitrator的使用》的部分 +// 使用场景，请参考《Arbitrator的使用》的部分 arbitrator ha.taosdata.com:6042 ``` From 28985c1fdbbb1d606445c31f2d41b680c65c2612 Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Tue, 1 Dec 2020 17:26:00 +0800 Subject: [PATCH 117/188] Update readme.md --- tests/examples/JDBC/JDBCDemo/readme.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/readme.md b/tests/examples/JDBC/JDBCDemo/readme.md index 9b8790adad..e348e458fe 100644 --- a/tests/examples/JDBC/JDBCDemo/readme.md +++ b/tests/examples/JDBC/JDBCDemo/readme.md @@ -6,10 +6,24 @@ TDengine's JDBC demo project is organized in a Maven way so that users can easil Make sure you have already installed a tdengine client on your current develop environment. Download the tdengine package on our website: ``https://www.taosdata.com/cn/all-downloads/`` and install the client. +## How to run jdbcChecker +
mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.JdbcChecker" -Dexec.args="-host localhost"
+ +## How to run jdbcTaosDemo +run command: +
 mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo"
+and run with your customed args +
mvn clean compile exec:java -Dexec.mainClass="com.taosdata.example.jdbcTaosdemo.JdbcTaosdemo" -Dexec.args="-host localhost"
+ ## Compile the Demo Code and Run It + To compile the demo project, go to the source directory ``TDengine/tests/examples/JDBC/JDBCDemo`` and execute -
mvn clean package assembly:single
+ +
+mvn clean package assembly:single
+
+ The ``pom.xml`` is configured to package all the dependencies into one executable jar file. To run it, go to ``examples/JDBC/JDBCDemo/target`` and execute -
java -jar jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host localhost
\ No newline at end of file +
java -jar jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host localhost
From 773101a247d7548aa21f050b4491d8664a272de2 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 1 Dec 2020 17:50:09 +0800 Subject: [PATCH 118/188] change to python3 explicitly. --- tests/pytest/hivemq-extension-test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py index daff763f72..eb27f7a7d9 100644 --- a/tests/pytest/hivemq-extension-test.py +++ b/tests/pytest/hivemq-extension-test.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 ################################################################### # Copyright (c) 2016 by TAOS Technologies, Inc. # All rights reserved. @@ -55,6 +55,7 @@ def isHiveMQInstalled(): defaultHiveMQPath = "/opt/hivemq*" hiveMQDir = glob.glob(defaultHiveMQPath) if (len(hiveMQDir) == 0): + v_print("%s", "ERROR: hivemq not found!") return False else: v_print("HiveMQ installed at %s", hiveMQDir[0]) From 6517eaf048672705085c0598738217ce64e6d77a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 1 Dec 2020 17:59:50 +0800 Subject: [PATCH 119/188] Revert "return mnode initialization stage to client" --- src/dnode/src/dnodeMPeer.c | 11 ++--- src/dnode/src/dnodeMRead.c | 11 ++--- src/dnode/src/dnodeMWrite.c | 18 ++++----- src/dnode/src/dnodeMain.c | 2 +- src/dnode/src/dnodeShell.c | 2 +- src/inc/mnode.h | 3 -- src/inc/taoserror.h | 3 -- src/mnode/src/mnodeMain.c | 80 ++++++++++--------------------------- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeUser.c | 15 +------ src/rpc/src/rpcMain.c | 5 ++- 11 files changed, 41 insertions(+), 111 deletions(-) diff --git a/src/dnode/src/dnodeMPeer.c b/src/dnode/src/dnodeMPeer.c index adeccd1f9c..ee6dc5212e 100644 --- a/src/dnode/src/dnodeMPeer.c +++ b/src/dnode/src/dnodeMPeer.c @@ -122,16 +122,11 @@ void dnodeFreeMPeerQueue() { } void dnodeDispatchToMPeerQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning()) { + if (!mnodeIsRunning() || tsMPeerQueue == NULL) { dnodeSendRedirectMsg(pMsg, false); } else { - if (!mnodeIsReady()) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = mnodeInitCode(), .pCont = NULL}; - rpcSendResponse(&rpcRsp); - } else { - SMnodeMsg *pPeer = mnodeCreateMsg(pMsg); - taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); - } + SMnodeMsg *pPeer = mnodeCreateMsg(pMsg); + taosWriteQitem(tsMPeerQueue, TAOS_QTYPE_RPC, pPeer); } rpcFreeCont(pMsg->pCont); diff --git a/src/dnode/src/dnodeMRead.c b/src/dnode/src/dnodeMRead.c index bb0d6f94f2..65f3af7b3b 100644 --- a/src/dnode/src/dnodeMRead.c +++ b/src/dnode/src/dnodeMRead.c @@ -123,16 +123,11 @@ void dnodeFreeMReadQueue() { } void dnodeDispatchToMReadQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning()) { + if (!mnodeIsRunning() || tsMReadQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); } else { - if (!mnodeIsReady()) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = mnodeInitCode(), .pCont = NULL}; - rpcSendResponse(&rpcRsp); - } else { - SMnodeMsg *pRead = mnodeCreateMsg(pMsg); - taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); - } + SMnodeMsg *pRead = mnodeCreateMsg(pMsg); + taosWriteQitem(tsMReadQueue, TAOS_QTYPE_RPC, pRead); } rpcFreeCont(pMsg->pCont); diff --git a/src/dnode/src/dnodeMWrite.c b/src/dnode/src/dnodeMWrite.c index 3c539ceefa..ef2d49ef42 100644 --- a/src/dnode/src/dnodeMWrite.c +++ b/src/dnode/src/dnodeMWrite.c @@ -123,18 +123,13 @@ void dnodeFreeMWritequeue() { } void dnodeDispatchToMWriteQueue(SRpcMsg *pMsg) { - if (!mnodeIsRunning()) { + if (!mnodeIsRunning() || tsMWriteQueue == NULL) { dnodeSendRedirectMsg(pMsg, true); } else { - if (!mnodeIsReady()) { - SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = mnodeInitCode(), .pCont = NULL}; - rpcSendResponse(&rpcRsp); - } else { - SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); - dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, - taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); - taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); - } + SMnodeMsg *pWrite = mnodeCreateMsg(pMsg); + dDebug("msg:%p, app:%p type:%s is put into mwrite queue:%p", pWrite, pWrite->rpcMsg.ahandle, + taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue); + taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } rpcFreeCont(pMsg->pCont); @@ -192,7 +187,7 @@ static void *dnodeProcessMWriteQueue(void *param) { void dnodeReprocessMWriteMsg(void *pMsg) { SMnodeMsg *pWrite = pMsg; - if (!mnodeIsRunning()) { + if (!mnodeIsRunning() || tsMWriteQueue == NULL) { dDebug("msg:%p, app:%p type:%s is redirected for mnode not running, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], pWrite->retry); @@ -201,6 +196,7 @@ void dnodeReprocessMWriteMsg(void *pMsg) { } else { dDebug("msg:%p, app:%p type:%s is reput into mwrite queue:%p, retry times:%d", pWrite, pWrite->rpcMsg.ahandle, taosMsg[pWrite->rpcMsg.msgType], tsMWriteQueue, pWrite->retry); + taosWriteQitem(tsMWriteQueue, TAOS_QTYPE_RPC, pWrite); } } diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index ce45105fb3..9f52dbd331 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -63,7 +63,6 @@ static const SDnodeComponent tsDnodeComponents[] = { {"dnodeeps", dnodeInitEps, dnodeCleanupEps}, {"globalcfg" ,taosCheckGlobalCfg, NULL}, {"mnodeinfos",dnodeInitMInfos, dnodeCleanupMInfos}, - {"shell", dnodeInitShell, dnodeCleanupShell}, {"wal", walInit, walCleanUp}, {"check", dnodeInitCheck, dnodeCleanupCheck}, // NOTES: dnodeInitCheck must be behind the dnodeinitStorage component !!! {"vread", dnodeInitVRead, dnodeCleanupVRead}, @@ -76,6 +75,7 @@ static const SDnodeComponent tsDnodeComponents[] = { {"mgmt", dnodeInitMgmt, dnodeCleanupMgmt}, {"modules", dnodeInitModules, dnodeCleanupModules}, {"mgmt-tmr", dnodeInitMgmtTimer, dnodeCleanupMgmtTimer}, + {"shell", dnodeInitShell, dnodeCleanupShell}, {"telemetry", dnodeInitTelemetry, dnodeCleanupTelemetry}, }; diff --git a/src/dnode/src/dnodeShell.c b/src/dnode/src/dnodeShell.c index fa5f9c24e8..89f657f789 100644 --- a/src/dnode/src/dnodeShell.c +++ b/src/dnode/src/dnodeShell.c @@ -144,7 +144,7 @@ static void dnodeProcessMsgFromShell(SRpcMsg *pMsg, SRpcEpSet *pEpSet) { static int dnodeRetrieveUserAuthInfo(char *user, char *spi, char *encrypt, char *secret, char *ckey) { int code = mnodeRetriveAuth(user, spi, encrypt, secret, ckey); - if (code != TSDB_CODE_RPC_REDIRECT) return code; + if (code != TSDB_CODE_APP_NOT_READY) return code; SAuthMsg *pMsg = rpcMallocCont(sizeof(SAuthMsg)); tstrncpy(pMsg->user, user, sizeof(pMsg->user)); diff --git a/src/inc/mnode.h b/src/inc/mnode.h index 453e8ab3f6..bdc30b0c46 100644 --- a/src/inc/mnode.h +++ b/src/inc/mnode.h @@ -65,15 +65,12 @@ void mnodeStopSystem(); void sdbUpdateAsync(); void sdbUpdateSync(void *pMnodes); bool mnodeIsRunning(); -bool mnodeIsReady(); -int32_t mnodeInitCode(); int32_t mnodeProcessRead(SMnodeMsg *pMsg); int32_t mnodeProcessWrite(SMnodeMsg *pMsg); int32_t mnodeProcessPeerReq(SMnodeMsg *pMsg); void mnodeProcessPeerRsp(SRpcMsg *pMsg); int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey); - #ifdef __cplusplus } #endif diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index a1cb950636..a720b68e59 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -125,9 +125,6 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SHOWOBJ, 0, 0x030B, "Data expir TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_QUERY_ID, 0, 0x030C, "Invalid query id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_ID, 0, 0x030D, "Invalid stream id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_CONN_ID, 0, 0x030E, "Invalid connection id") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INIT, 0, 0x030F, "Mnode is initializing") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INIT_SDB, 0, 0x0310, "Mnode is initializing meta data, it takes a while if many tables exists") -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INIT_OTHER, 0, 0x0311, "Mnode is initializing other data") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_OBJ_ALREADY_THERE, 0, 0x0320, "Object already there") TAOS_DEFINE_ERROR(TSDB_CODE_MND_SDB_ERROR, 0, 0x0321, "Unexpected generic error in sdb") diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 05ea0b93a1..d15b32da54 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -17,7 +17,6 @@ #include "os.h" #include "taosdef.h" #include "tsched.h" -#include "taoserror.h" #include "tbalance.h" #include "tgrant.h" #include "ttimer.h" @@ -38,41 +37,30 @@ #include "mnodeShow.h" #include "mnodeProfile.h" -typedef enum { - TSDB_MND_STATUS_NOT_RUNNING, - TSDB_MND_STATUS_INIT, - TSDB_MND_STATUS_INIT_SDB, - TSDB_MND_STATUS_INIT_OTHER, - TSDB_MND_STATUS_READY, - TSDB_MND_STATUS_CLEANING, -} EMndStatus; - typedef struct { const char *const name; int (*init)(); void (*cleanup)(); - EMndStatus status; } SMnodeComponent; -void *tsMnodeTmr = NULL; -static bool tsMgmtIsRunning = false; -static EMndStatus tsMgmtStatus = TSDB_MND_STATUS_NOT_RUNNING; +void *tsMnodeTmr = NULL; +static bool tsMgmtIsRunning = false; static const SMnodeComponent tsMnodeComponents[] = { - {"sdbref", sdbInitRef, sdbCleanUpRef, TSDB_MND_STATUS_INIT}, - {"profile", mnodeInitProfile, mnodeCleanupProfile, TSDB_MND_STATUS_INIT}, - {"cluster", mnodeInitCluster, mnodeCleanupCluster, TSDB_MND_STATUS_INIT}, - {"accts", mnodeInitAccts, mnodeCleanupAccts, TSDB_MND_STATUS_INIT}, - {"users", mnodeInitUsers, mnodeCleanupUsers, TSDB_MND_STATUS_INIT}, - {"dnodes", mnodeInitDnodes, mnodeCleanupDnodes, TSDB_MND_STATUS_INIT}, - {"dbs", mnodeInitDbs, mnodeCleanupDbs, TSDB_MND_STATUS_INIT}, - {"vgroups", mnodeInitVgroups, mnodeCleanupVgroups, TSDB_MND_STATUS_INIT}, - {"tables", mnodeInitTables, mnodeCleanupTables, TSDB_MND_STATUS_INIT}, - {"mnodes", mnodeInitMnodes, mnodeCleanupMnodes, TSDB_MND_STATUS_INIT}, - {"sdb", sdbInit, sdbCleanUp, TSDB_MND_STATUS_INIT_SDB}, - {"balance", balanceInit, balanceCleanUp, TSDB_MND_STATUS_INIT_OTHER}, - {"grant", grantInit, grantCleanUp, TSDB_MND_STATUS_INIT_OTHER}, - {"show", mnodeInitShow, mnodeCleanUpShow, TSDB_MND_STATUS_INIT_OTHER}, + {"sdbref", sdbInitRef, sdbCleanUpRef}, + {"profile", mnodeInitProfile, mnodeCleanupProfile}, + {"cluster", mnodeInitCluster, mnodeCleanupCluster}, + {"accts", mnodeInitAccts, mnodeCleanupAccts}, + {"users", mnodeInitUsers, mnodeCleanupUsers}, + {"dnodes", mnodeInitDnodes, mnodeCleanupDnodes}, + {"dbs", mnodeInitDbs, mnodeCleanupDbs}, + {"vgroups", mnodeInitVgroups, mnodeCleanupVgroups}, + {"tables", mnodeInitTables, mnodeCleanupTables}, + {"mnodes", mnodeInitMnodes, mnodeCleanupMnodes}, + {"sdb", sdbInit, sdbCleanUp}, + {"balance", balanceInit, balanceCleanUp}, + {"grant", grantInit, grantCleanUp}, + {"show", mnodeInitShow, mnodeCleanUpShow} }; static void mnodeInitTimer(); @@ -88,24 +76,21 @@ static void mnodeCleanupComponents(int32_t stepId) { static int32_t mnodeInitComponents() { int32_t code = 0; for (int32_t i = 0; i < sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]); i++) { - tsMgmtStatus = tsMnodeComponents[i].status; if (tsMnodeComponents[i].init() != 0) { mnodeCleanupComponents(i); code = -1; break; } - // sleep(3); } return code; } int32_t mnodeStartSystem() { - if (tsMgmtStatus != TSDB_MND_STATUS_NOT_RUNNING) { + if (tsMgmtIsRunning) { mInfo("mnode module already started..."); return 0; } - tsMgmtStatus = TSDB_MND_STATUS_INIT; mInfo("starting to initialize mnode ..."); if (mkdir(tsMnodeDir, 0755) != 0 && errno != EEXIST) { mError("failed to init mnode dir:%s, reason:%s", tsMnodeDir, strerror(errno)); @@ -121,7 +106,7 @@ int32_t mnodeStartSystem() { } grantReset(TSDB_GRANT_ALL, 0); - tsMgmtStatus = TSDB_MND_STATUS_READY; + tsMgmtIsRunning = true; mInfo("mnode is initialized successfully"); @@ -141,7 +126,7 @@ int32_t mnodeInitSystem() { void mnodeCleanupSystem() { if (tsMgmtIsRunning) { mInfo("starting to clean up mnode"); - tsMgmtStatus = TSDB_MND_STATUS_CLEANING; + tsMgmtIsRunning = false; dnodeFreeMWritequeue(); dnodeFreeMReadQueue(); @@ -149,7 +134,6 @@ void mnodeCleanupSystem() { mnodeCleanupTimer(); mnodeCleanupComponents(sizeof(tsMnodeComponents) / sizeof(tsMnodeComponents[0]) - 1); - tsMgmtStatus = TSDB_MND_STATUS_NOT_RUNNING; mInfo("mnode is cleaned up"); } } @@ -200,29 +184,5 @@ static bool mnodeNeedStart() { } bool mnodeIsRunning() { - return (tsMgmtStatus != TSDB_MND_STATUS_NOT_RUNNING && tsMgmtStatus != TSDB_MND_STATUS_CLEANING); -} - -bool mnodeIsReady() { - return (tsMgmtStatus == TSDB_MND_STATUS_READY); -} - -int32_t mnodeInitCode() { - int32_t code = -1; - - switch (tsMgmtStatus) { - case TSDB_MND_STATUS_INIT: - code = TSDB_CODE_MND_INIT; - break; - case TSDB_MND_STATUS_INIT_SDB: - code = TSDB_CODE_MND_INIT_SDB; - break; - case TSDB_MND_STATUS_INIT_OTHER: - code = TSDB_CODE_MND_INIT_OTHER; - break; - default: - code = TSDB_CODE_MND_INIT; - } - - return code; + return tsMgmtIsRunning; } diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 96b840ab19..40e2e1cfcc 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -309,7 +309,7 @@ void sdbUpdateAsync() { void sdbUpdateSync(void *pMnodes) { SMnodeInfos *mnodes = pMnodes; - if (!mnodeIsReady()) { + if (!mnodeIsRunning()) { mDebug("vgId:1, mnode not start yet, update sync config later"); return; } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index 8d57ad30dc..fb26086d04 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -585,21 +585,10 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { } int32_t mnodeRetriveAuth(char *user, char *spi, char *encrypt, char *secret, char *ckey) { - *secret = 0; - - if (!mnodeIsRunning()) { - mDebug("user:%s, mnode is not running, fail to auth", user); - return TSDB_CODE_RPC_REDIRECT; - } - - if (!mnodeIsReady()) { - mDebug("user:%s, failed to auth user, mnode is not ready", user); - return mnodeInitCode(); - } - if (!sdbIsMaster()) { + *secret = 0; mDebug("user:%s, failed to auth user, mnode is not master", user); - return TSDB_CODE_RPC_REDIRECT; + return TSDB_CODE_APP_NOT_READY; } SUserObj *pUser = mnodeGetUser(user); diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 044fe8c204..00a97d7bc2 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1559,9 +1559,10 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { if ( !rpcIsReq(pHead->msgType) ) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); - if (code != 0) { - // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); + if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || + code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_INVALID_USER || code == TSDB_CODE_RPC_NOT_READY) { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); return 0; } } From d4c6aaf2af44f59a8b5bd18d78bdea3dd52282b2 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 1 Dec 2020 18:15:26 +0800 Subject: [PATCH 120/188] [TD-2178]add table per batch --- tests/pytest/insert/restfulInsert.py | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index df253040f3..da797f788f 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -44,22 +44,9 @@ class RestfulInsert: if response.status_code != 200: print(response.content) - def insertData(self, threadID): - print("thread %d started" % threadID) - tablesPerThread = int (self.numOfTables / self.numOfThreads) - for i in range(tablesPerThread): - tableID = i + threadID * tablesPerThread - start = self.ts - for j in range(int(self.recordsPerTable / self.batchSize)): - data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID) - values = [] - for k in range(self.batchSize): - data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)) - response = requests.post(self.url, data, headers = self.header) - if response.status_code != 200: - print(response.content) + - def insertnData(self, threadID): + def insertData(self, threadID): print("thread %d started" % threadID) tablesPerThread = int (self.numOfTables / self.numOfThreads) loop = int(self.recordsPerTable / self.batchSize) @@ -81,11 +68,9 @@ class RestfulInsert: if self.outOfOrder : random.shuffle(values) data+=''.join(values) - response = requests.post(self.url, data, headers = self.header) if response.status_code != 200: print(response.content) - print('----------------',loop,time.time()-start1) else: for i in range(0,tablesPerThread+self.tablePerbatch,self.tablePerbatch): for k in range(loop): @@ -101,8 +86,7 @@ class RestfulInsert: values.append("(%d, %d, %d, %d)" % (start + k * self.batchSize + l, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) if self.outOfOrder : random.shuffle(values) - data+=''.join(values) - print('------------------',len(data)) + data+=''.join(values) if len(data) > 1024*1024 : print ('batch size is larger than 1M') exit(-1) @@ -169,7 +153,7 @@ class RestfulInsert: for i in range(self.numOfThreads): threads[i].join() - print("inserting %d records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime))) + print("inserting %s records takes %d seconds" % (self.numOfTables * self.recordsPerTable, (time.time() - startTime))) parser = argparse.ArgumentParser() parser.add_argument( @@ -204,14 +188,14 @@ parser.add_argument( '-T', '--number-of-tables', action='store', - default=1000, + default=10000, type=int, help='Number of tables to be created (default: 1000)') parser.add_argument( '-r', '--number-of-records', action='store', - default=1000, + default=10000, type=int, help='Number of record to be created for each table (default: 1000, -1 for unlimited records)') parser.add_argument( From 31e747239113ea1e5d5f7dae01e153a65f292276 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Tue, 1 Dec 2020 18:59:50 +0800 Subject: [PATCH 121/188] [TD-2279] add test case --- tests/pytest/fulltest.sh | 2 + tests/pytest/update/bug_td2279.py | 67 +++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 tests/pytest/update/bug_td2279.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 0e3b482e3d..042fd826e8 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -19,6 +19,7 @@ python3 ./test.py -f insert/randomNullCommit.py python3 insert/retentionpolicy.py python3 ./test.py -f insert/alterTableAndInsert.py python3 ./test.py -f insert/insertIntoTwoTables.py +python3 ./test.py -f insert/before_1970.py python3 ./test.py -f table/alter_wal0.py python3 ./test.py -f table/column_name.py @@ -228,6 +229,7 @@ python3 ./test.py -f update/merge_commit_data2.py python3 ./test.py -f update/merge_commit_data2_update0.py python3 ./test.py -f update/merge_commit_last-0.py python3 ./test.py -f update/merge_commit_last.py +python3 ./test.py -f update/bug_td2279.py # wal python3 ./test.py -f wal/addOldWalTest.py \ No newline at end of file diff --git a/tests/pytest/update/bug_td2279.py b/tests/pytest/update/bug_td2279.py new file mode 100644 index 0000000000..7e8640dfa0 --- /dev/null +++ b/tests/pytest/update/bug_td2279.py @@ -0,0 +1,67 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + self.ts = 1606700000000 + + def restartTaosd(self): + tdDnodes.stop(1) + tdDnodes.startWithoutSleep(1) + tdSql.execute("use db") + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute("create table t (ts timestamp, a int)") + + for i in range(3276): + tdSql.execute("insert into t values(%d, 0)" % (self.ts + i)) + + newTs = 1606700010000 + for i in range(3275): + tdSql.execute("insert into t values(%d, 0)" % (self.ts + i)) + tdSql.execute("insert into t values(%d, 0)" % 1606700013280) + + self.restartTaosd() + + for i in range(1606700003275, 1606700006609): + tdSql.execute("insert into t values(%d, 0)" % i) + tdSql.execute("insert into t values(%d, 0)" % 1606700006612) + + self.restartTaosd() + + tdSql.execute("insert into t values(%d, 0)" % 1606700006610) + tdSql.query("select * from t") + tdSql.checkRows(6612) + + tdDnodes.stop(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From 8072b26a2f26383c4ffda0d6e409f1f791038894 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Tue, 1 Dec 2020 19:04:05 +0800 Subject: [PATCH 122/188] [TD-2263]: debug log epSet.fqdn when connect success --- src/client/src/tscServer.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5d3b86caf3..9e8d3d339e 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2058,6 +2058,10 @@ int tscProcessConnectRsp(SSqlObj *pSql) { if (pConnect->epSet.numOfEps > 0) { tscEpSetHtons(&pConnect->epSet); tscUpdateMgmtEpSet(&pConnect->epSet); + + for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { + tscDebug("%p epSet.fqdn[%d]: %s, pObj:%p", pSql, i, pConnect->epSet.fqdn[i], pObj); + } } strcpy(pObj->sversion, pConnect->serverVersion); From c7ebda0b376ae258bae4ede5be558f8329c0bc1b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 1 Dec 2020 11:33:41 +0000 Subject: [PATCH 123/188] TD-1843 --- src/mnode/src/mnodeMain.c | 3 +-- src/rpc/src/rpcMain.c | 17 +++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 05ea0b93a1..b4c63c05e6 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -55,7 +55,6 @@ typedef struct { } SMnodeComponent; void *tsMnodeTmr = NULL; -static bool tsMgmtIsRunning = false; static EMndStatus tsMgmtStatus = TSDB_MND_STATUS_NOT_RUNNING; static const SMnodeComponent tsMnodeComponents[] = { @@ -139,7 +138,7 @@ int32_t mnodeInitSystem() { } void mnodeCleanupSystem() { - if (tsMgmtIsRunning) { + if (mnodeIsRunning()) { mInfo("starting to clean up mnode"); tsMgmtStatus = TSDB_MND_STATUS_CLEANING; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index 044fe8c204..c8f6a031d0 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1559,13 +1559,18 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { if ( !rpcIsReq(pHead->msgType) ) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); - if (code != 0) { - // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - return 0; - } +#if 0 + if (code != 0) { +#else + if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || + code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_INVALID_USER || code == TSDB_CODE_RPC_NOT_READY) { +#endif + // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + return 0; } - + } + code = 0; if (pHead->spi == pConn->spi) { // authentication From 0d7de8f433f91454af255cd9e987e96eff8f7f86 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Dec 2020 09:28:26 +0800 Subject: [PATCH 124/188] Revert "TD-1843" --- src/mnode/src/mnodeMain.c | 3 ++- src/rpc/src/rpcMain.c | 17 ++++++----------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index b4c63c05e6..05ea0b93a1 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -55,6 +55,7 @@ typedef struct { } SMnodeComponent; void *tsMnodeTmr = NULL; +static bool tsMgmtIsRunning = false; static EMndStatus tsMgmtStatus = TSDB_MND_STATUS_NOT_RUNNING; static const SMnodeComponent tsMnodeComponents[] = { @@ -138,7 +139,7 @@ int32_t mnodeInitSystem() { } void mnodeCleanupSystem() { - if (mnodeIsRunning()) { + if (tsMgmtIsRunning) { mInfo("starting to clean up mnode"); tsMgmtStatus = TSDB_MND_STATUS_CLEANING; diff --git a/src/rpc/src/rpcMain.c b/src/rpc/src/rpcMain.c index c8f6a031d0..044fe8c204 100644 --- a/src/rpc/src/rpcMain.c +++ b/src/rpc/src/rpcMain.c @@ -1559,18 +1559,13 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { if ( !rpcIsReq(pHead->msgType) ) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); -#if 0 - if (code != 0) { -#else - if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_INVALID_USER || code == TSDB_CODE_RPC_NOT_READY) { -#endif - // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); - pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); - return 0; + if (code != 0) { + // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); + pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); + return 0; + } } - } - + code = 0; if (pHead->spi == pConn->spi) { // authentication From bd6e5f2c418557018b958d7389caca59f1ca5792 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 2 Dec 2020 10:13:11 +0800 Subject: [PATCH 125/188] compile arm version while full test --- Jenkinsfile | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 940815febe..edbe11d428 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -175,7 +175,47 @@ pipeline { } } - + stage('arm64_build'){ + agent{label 'arm64'} + steps{ + sh ''' + cd ${WK} + git fetch + git checkout develop + git pull + cd ${WKC} + git fetch + git checkout develop + git pull + git submodule update + cd ${WKC}/packaging + ./release.sh -v cluster -c aarch64 -n 2.0.0.0 -m 2.0.0.0 + + ''' + } + } + stage('arm32_build'){ + agent{label 'arm32'} + steps{ + catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { + sh ''' + cd ${WK} + git fetch + git checkout develop + git pull + cd ${WKC} + git fetch + git checkout develop + git pull + git submodule update + cd ${WKC}/packaging + ./release.sh -v cluster -c aarch32 -n 2.0.0.0 -m 2.0.0.0 + + ''' + } + + } + } } } From 982e6465ac6af2a803bd196deb13c01bab00e95e Mon Sep 17 00:00:00 2001 From: Steven Li Date: Wed, 2 Dec 2020 02:15:46 +0000 Subject: [PATCH 126/188] Refactored crash_gen tool, plus ensured proper termination upon trasntion failure --- tests/pytest/crash_gen/crash_gen_main.py | 241 ++++++++++++++-------- tests/pytest/crash_gen/db.py | 9 +- tests/pytest/crash_gen/misc.py | 22 +- tests/pytest/crash_gen/service_manager.py | 28 ++- 4 files changed, 192 insertions(+), 108 deletions(-) diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 8d68457ec8..e2ce4b26fa 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -352,6 +352,12 @@ class ThreadCoordinator: self._execStats.registerFailure("Broken DB Connection") # continue # don't do that, need to tap all threads at # end, and maybe signal them to stop + if isinstance(err, CrashGenError): # our own transition failure + Logging.info("State transition error") + traceback.print_stack() + transitionFailed = True + self._te = None # Not running any more + self._execStats.registerFailure("State transition error") else: raise # return transitionFailed # Why did we have this??!! @@ -388,12 +394,20 @@ class ThreadCoordinator: self._syncAtBarrier() # For now just cross the barrier Progress.emit(Progress.END_THREAD_STEP) except threading.BrokenBarrierError as err: - Logging.info("Main loop aborted, caused by worker thread(s) time-out") self._execStats.registerFailure("Aborted due to worker thread timeout") - print("\n\nWorker Thread time-out detected, TAOS related threads are:") + Logging.error("\n") + Logging.error("Main loop aborted, caused by worker thread(s) time-out of {} seconds".format( + ThreadCoordinator.WORKER_THREAD_TIMEOUT)) + Logging.error("TAOS related threads blocked at (stack frames top-to-bottom):") ts = ThreadStacks() ts.print(filterInternal=True) workerTimeout = True + + # Enable below for deadlock debugging, using gdb to attach to process + # while True: + # Logging.error("Deadlock detected") + # time.sleep(60.0) + break # At this point, all threads should be pass the overall "barrier" and before the per-thread "gate" @@ -701,7 +715,7 @@ class AnyState: # task.logDebug("Task success found") sCnt += 1 if (sCnt >= 2): - raise RuntimeError( + raise CrashGenError( "Unexpected more than 1 success with task: {}".format(cls)) def assertIfExistThenSuccess(self, tasks, cls): @@ -714,7 +728,7 @@ class AnyState: if task.isSuccess(): sCnt += 1 if (exists and sCnt <= 0): - raise RuntimeError("Unexpected zero success for task type: {}, from tasks: {}" + raise CrashGenError("Unexpected zero success for task type: {}, from tasks: {}" .format(cls, tasks)) def assertNoTask(self, tasks, cls): @@ -727,7 +741,7 @@ class AnyState: for task in tasks: if isinstance(task, cls): if task.isSuccess(): - raise RuntimeError( + raise CrashGenError( "Unexpected successful task: {}".format(cls)) def hasSuccess(self, tasks, cls): @@ -926,8 +940,9 @@ class StateMechine: Logging.debug("[STT] DB_ONLY found, between {} and {}".format(ts, time.time())) return StateDbOnly() + # For sure we have tables, which means we must have the super table. # TODO: are we sure? sTable = self._db.getFixedSuperTable() - if sTable.hasRegTables(dbc, dbName): # no regular tables + if sTable.hasRegTables(dbc): # no regular tables Logging.debug("[STT] SUPER_TABLE_ONLY found, between {} and {}".format(ts, time.time())) return StateSuperTableOnly() else: # has actual tables @@ -1050,9 +1065,8 @@ class Database: def getFixedSuperTableName(cls): return "fs_table" - @classmethod - def getFixedSuperTable(cls) -> TdSuperTable: - return TdSuperTable(cls.getFixedSuperTableName()) + def getFixedSuperTable(self) -> TdSuperTable: + return TdSuperTable(self.getFixedSuperTableName(), self.getName()) # We aim to create a starting time tick, such that, whenever we run our test here once # We should be able to safely create 100,000 records, which will not have any repeated time stamp @@ -1107,6 +1121,11 @@ class Database: # print("Float obtained: {}".format(ret)) return ret + ALL_COLORS = ['red', 'white', 'blue', 'green', 'purple'] + + def getNextColor(self): + return random.choice(self.ALL_COLORS) + class TaskExecutor(): class BoundedList: @@ -1240,7 +1259,7 @@ class Task(): if errno in [ 0x05, # TSDB_CODE_RPC_NOT_READY 0x0B, # Unable to establish connection, more details in TD-1648 - 0x200, # invalid SQL， TODO: re-examine with TD-934 + # 0x200, # invalid SQL， TODO: re-examine with TD-934 0x20F, # query terminated, possibly due to vnoding being dropped, see TD-1776 0x213, # "Disconnected from service", result of "kill connection ???" 0x217, # "db not selected", client side defined error code @@ -1569,8 +1588,8 @@ class TaskCreateSuperTable(StateTransitionTask): sTable = self._db.getFixedSuperTable() # type: TdSuperTable # wt.execSql("use db") # should always be in place - sTable.create(wt.getDbConn(), self._db.getName(), - {'ts':'timestamp', 'speed':'int'}, {'b':'binary(200)', 'f':'float'}, + sTable.create(wt.getDbConn(), + {'ts':'TIMESTAMP', 'speed':'INT', 'color':'BINARY(16)'}, {'b':'BINARY(200)', 'f':'FLOAT'}, dropIfExists = True ) # self.execWtSql(wt,"create table db.{} (ts timestamp, speed int) tags (b binary(200), f float) ".format(tblName)) @@ -1579,30 +1598,33 @@ class TaskCreateSuperTable(StateTransitionTask): class TdSuperTable: - def __init__(self, stName): + def __init__(self, stName, dbName): self._stName = stName + self._dbName = dbName def getName(self): return self._stName - def drop(self, dbc, dbName, skipCheck = False): - if self.exists(dbc, dbName) : # if myself exists + def drop(self, dbc, skipCheck = False): + dbName = self._dbName + if self.exists(dbc) : # if myself exists fullTableName = dbName + '.' + self._stName dbc.execute("DROP TABLE {}".format(fullTableName)) else: if not skipCheck: raise CrashGenError("Cannot drop non-existant super table: {}".format(self._stName)) - def exists(self, dbc, dbName): - dbc.execute("USE " + dbName) + def exists(self, dbc): + dbc.execute("USE " + self._dbName) return dbc.existsSuperTable(self._stName) # TODO: odd semantic, create() method is usually static? - def create(self, dbc, dbName, cols: dict, tags: dict, + def create(self, dbc, cols: dict, tags: dict, dropIfExists = False ): - '''Creating a super table''' + + dbName = self._dbName dbc.execute("USE " + dbName) fullTableName = dbName + '.' + self._stName if dbc.existsSuperTable(self._stName): @@ -1623,7 +1645,8 @@ class TdSuperTable: ) dbc.execute(sql) - def getRegTables(self, dbc: DbConn, dbName: str): + def getRegTables(self, dbc: DbConn): + dbName = self._dbName try: dbc.query("select TBNAME from {}.{}".format(dbName, self._stName)) # TODO: analyze result set later except taos.error.ProgrammingError as err: @@ -1634,10 +1657,11 @@ class TdSuperTable: qr = dbc.getQueryResult() return [v[0] for v in qr] # list transformation, ref: https://stackoverflow.com/questions/643823/python-list-transformation - def hasRegTables(self, dbc: DbConn, dbName: str): - return dbc.query("SELECT * FROM {}.{}".format(dbName, self._stName)) > 0 + def hasRegTables(self, dbc: DbConn): + return dbc.query("SELECT * FROM {}.{}".format(self._dbName, self._stName)) > 0 - def ensureTable(self, task: Task, dbc: DbConn, dbName: str, regTableName: str): + def ensureTable(self, task: Task, dbc: DbConn, regTableName: str): + dbName = self._dbName sql = "select tbname from {}.{} where tbname in ('{}')".format(dbName, self._stName, regTableName) if dbc.query(sql) >= 1 : # reg table exists already return @@ -1650,15 +1674,15 @@ class TdSuperTable: # print("(" + fullTableName[-3:] + ")", end="", flush=True) try: sql = "CREATE TABLE {} USING {}.{} tags ({})".format( - fullTableName, dbName, self._stName, self._getTagStrForSql(dbc, dbName) + fullTableName, dbName, self._stName, self._getTagStrForSql(dbc) ) dbc.execute(sql) finally: if task is not None: task.unlockTable(fullTableName) # no matter what - def _getTagStrForSql(self, dbc, dbName: str) : - tags = self._getTags(dbc, dbName) + def _getTagStrForSql(self, dbc) : + tags = self._getTags(dbc) tagStrs = [] for tagName in tags: tagType = tags[tagName] @@ -1672,36 +1696,86 @@ class TdSuperTable: raise RuntimeError("Unexpected tag type: {}".format(tagType)) return ", ".join(tagStrs) - def _getTags(self, dbc, dbName) -> dict: - dbc.query("DESCRIBE {}.{}".format(dbName, self._stName)) + def _getTags(self, dbc) -> dict: + dbc.query("DESCRIBE {}.{}".format(self._dbName, self._stName)) stCols = dbc.getQueryResult() # print(stCols) ret = {row[0]:row[1] for row in stCols if row[3]=='TAG'} # name:type # print("Tags retrieved: {}".format(ret)) return ret - def addTag(self, dbc, dbName, tagName, tagType): - if tagName in self._getTags(dbc, dbName): # already + def addTag(self, dbc, tagName, tagType): + if tagName in self._getTags(dbc): # already return # sTable.addTag("extraTag", "int") - sql = "alter table {}.{} add tag {} {}".format(dbName, self._stName, tagName, tagType) + sql = "alter table {}.{} add tag {} {}".format( + self._dbName, self._stName, tagName, tagType) dbc.execute(sql) - def dropTag(self, dbc, dbName, tagName): - if not tagName in self._getTags(dbc, dbName): # don't have this tag + def dropTag(self, dbc, tagName): + if not tagName in self._getTags(dbc): # don't have this tag return - sql = "alter table {}.{} drop tag {}".format(dbName, self._stName, tagName) + sql = "alter table {}.{} drop tag {}".format(self._dbName, self._stName, tagName) dbc.execute(sql) - def changeTag(self, dbc, dbName, oldTag, newTag): - tags = self._getTags(dbc, dbName) + def changeTag(self, dbc, oldTag, newTag): + tags = self._getTags(dbc) if not oldTag in tags: # don't have this tag return if newTag in tags: # already have this tag return - sql = "alter table {}.{} change tag {} {}".format(dbName, self._stName, oldTag, newTag) + sql = "alter table {}.{} change tag {} {}".format(self._dbName, self._stName, oldTag, newTag) dbc.execute(sql) + def generateQueries(self, dbc: DbConn) -> List[SqlQuery]: + ''' Generate queries to test/exercise this super table ''' + ret = [] # type: List[SqlQuery] + + for rTbName in self.getRegTables(dbc): # regular tables + + filterExpr = Dice.choice([ # TODO: add various kind of WHERE conditions + None + ]) + + # Run the query against the regular table first + doAggr = (Dice.throw(2) == 0) # 1 in 2 chance + if not doAggr: # don't do aggregate query, just simple one + ret.append(SqlQuery( # reg table + "select {} from {}.{}".format('*', self._dbName, rTbName))) + ret.append(SqlQuery( # super table + "select {} from {}.{}".format('*', self._dbName, self.getName()))) + else: # Aggregate query + aggExpr = Dice.choice([ + 'count(*)', + 'avg(speed)', + # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable + 'sum(speed)', + 'stddev(speed)', + # SELECTOR functions + 'min(speed)', + 'max(speed)', + 'first(speed)', + 'last(speed)', + 'top(speed, 50)', # TODO: not supported? + 'bottom(speed, 50)', # TODO: not supported? + 'apercentile(speed, 10)', # TODO: TD-1316 + 'last_row(speed)', + # Transformation Functions + # 'diff(speed)', # TODO: no supported?! + 'spread(speed)' + ]) # TODO: add more from 'top' + + + if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?! + sql = "select {} from {}.{}".format(aggExpr, self._dbName, self.getName()) + if Dice.throw(3) == 0: # 1 in X chance + sql = sql + ' GROUP BY color' + Progress.emit(Progress.QUERY_GROUP_BY) + # Logging.info("Executing GROUP-BY query: " + sql) + ret.append(SqlQuery(sql)) + + return ret + class TaskReadData(StateTransitionTask): @classmethod def getEndState(cls): @@ -1716,10 +1790,8 @@ class TaskReadData(StateTransitionTask): # return True # always # return gSvcMgr.isActive() # only if it's running TODO: race condition here - def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): - sTable = self._db.getFixedSuperTable() - - # 1 in 5 chance, simulate a broken connection, only if service stable (not restarting) + def _reconnectIfNeeded(self, wt): + # 1 in 20 chance, simulate a broken connection, only if service stable (not restarting) if random.randrange(20)==0: # and self._canRestartService(): # TODO: break connection in all situations # Logging.info("Attempting to reconnect to server") # TODO: change to DEBUG Progress.emit(Progress.SERVICE_RECONNECT_START) @@ -1744,43 +1816,36 @@ class TaskReadData(StateTransitionTask): return # TODO: fix server restart status race condtion + def _executeInternal(self, te: TaskExecutor, wt: WorkerThread): + self._reconnectIfNeeded(wt) + dbc = wt.getDbConn() - dbName = self._db.getName() - for rTbName in sTable.getRegTables(dbc, dbName): # regular tables - aggExpr = Dice.choice([ - '*', - 'count(*)', - 'avg(speed)', - # 'twa(speed)', # TODO: this one REQUIRES a where statement, not reasonable - 'sum(speed)', - 'stddev(speed)', - # SELECTOR functions - 'min(speed)', - 'max(speed)', - 'first(speed)', - 'last(speed)', - 'top(speed, 50)', # TODO: not supported? - 'bottom(speed, 50)', # TODO: not supported? - 'apercentile(speed, 10)', # TODO: TD-1316 - 'last_row(speed)', - # Transformation Functions - # 'diff(speed)', # TODO: no supported?! - 'spread(speed)' - ]) # TODO: add more from 'top' - filterExpr = Dice.choice([ # TODO: add various kind of WHERE conditions - None - ]) + sTable = self._db.getFixedSuperTable() + + for q in sTable.generateQueries(dbc): # regular tables try: - # Run the query against the regular table first - dbc.execute("select {} from {}.{}".format(aggExpr, dbName, rTbName)) - # Then run it against the super table - if aggExpr not in ['stddev(speed)']: #TODO: STDDEV not valid for super tables?! - dbc.execute("select {} from {}.{}".format(aggExpr, dbName, sTable.getName())) + sql = q.getSql() + # if 'GROUP BY' in sql: + # Logging.info("Executing GROUP-BY query: " + sql) + dbc.execute(sql) except taos.error.ProgrammingError as err: errno2 = Helper.convertErrno(err.errno) Logging.debug("[=] Read Failure: errno=0x{:X}, msg: {}, SQL: {}".format(errno2, err, dbc.getLastSql())) raise +class SqlQuery: + @classmethod + def buildRandom(cls, db: Database): + '''Build a random query against a certain database''' + + dbName = db.getName() + + def __init__(self, sql:str = None): + self._sql = sql + + def getSql(self): + return self._sql + class TaskDropSuperTable(StateTransitionTask): @classmethod def getEndState(cls): @@ -1837,19 +1902,18 @@ class TaskAlterTags(StateTransitionTask): # tblName = self._dbManager.getFixedSuperTableName() dbc = wt.getDbConn() sTable = self._db.getFixedSuperTable() - dbName = self._db.getName() dice = Dice.throw(4) if dice == 0: - sTable.addTag(dbc, dbName, "extraTag", "int") + sTable.addTag(dbc, "extraTag", "int") # sql = "alter table db.{} add tag extraTag int".format(tblName) elif dice == 1: - sTable.dropTag(dbc, dbName, "extraTag") + sTable.dropTag(dbc, "extraTag") # sql = "alter table db.{} drop tag extraTag".format(tblName) elif dice == 2: - sTable.dropTag(dbc, dbName, "newTag") + sTable.dropTag(dbc, "newTag") # sql = "alter table db.{} drop tag newTag".format(tblName) else: # dice == 3 - sTable.changeTag(dbc, dbName, "extraTag", "newTag") + sTable.changeTag(dbc, "extraTag", "newTag") # sql = "alter table db.{} change tag extraTag newTag".format(tblName) class TaskRestartService(StateTransitionTask): @@ -1920,15 +1984,17 @@ class TaskAddData(StateTransitionTask): for j in range(numRecords): # number of records per table nextInt = db.getNextInt() nextTick = db.getNextTick() - sql += "('{}', {});".format(nextTick, nextInt) + nextColor = db.getNextColor() + sql += "('{}', {}, '{}');".format(nextTick, nextInt, nextColor) dbc.execute(sql) - def _addData(self, db, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches + def _addData(self, db: Database, dbc, regTableName, te: TaskExecutor): # implied: NOT in batches numRecords = self.LARGE_NUMBER_OF_RECORDS if gConfig.larger_data else self.SMALL_NUMBER_OF_RECORDS for j in range(numRecords): # number of records per table nextInt = db.getNextInt() nextTick = db.getNextTick() + nextColor = db.getNextColor() if gConfig.record_ops: self.prepToRecordOps() self.fAddLogReady.write("Ready to write {} to {}\n".format(nextInt, regTableName)) @@ -1942,11 +2008,11 @@ class TaskAddData(StateTransitionTask): # print("_w" + str(nextInt % 100), end="", flush=True) # Trace what was written try: - sql = "insert into {} values ('{}', {});".format( # removed: tags ('{}', {}) + sql = "insert into {} values ('{}', {}, '{}');".format( # removed: tags ('{}', {}) fullTableName, # ds.getFixedSuperTableName(), # ds.getNextBinary(), ds.getNextFloat(), - nextTick, nextInt) + nextTick, nextInt, nextColor) dbc.execute(sql) except: # Any exception at all if gConfig.verify_data: @@ -1964,10 +2030,10 @@ class TaskAddData(StateTransitionTask): .format(nextInt, readBack), 0x999) except taos.error.ProgrammingError as err: errno = Helper.convertErrno(err.errno) - if errno in [0x991, 0x992] : # not a single result + if errno in [CrashGenError.INVALID_EMPTY_RESULT, CrashGenError.INVALID_MULTIPLE_RESULT] : # not a single result raise taos.error.ProgrammingError( "Failed to read back same data for tick: {}, wrote: {}, read: {}" - .format(nextTick, nextInt, "Empty Result" if errno==0x991 else "Multiple Result"), + .format(nextTick, nextInt, "Empty Result" if errno == CrashGenError.INVALID_EMPTY_RESULT else "Multiple Result"), errno) elif errno in [0x218, 0x362]: # table doesn't exist # do nothing @@ -2000,11 +2066,12 @@ class TaskAddData(StateTransitionTask): else: self.activeTable.add(i) # marking it active + dbName = db.getName() sTable = db.getFixedSuperTable() regTableName = self.getRegTableName(i) # "db.reg_table_{}".format(i) - fullTableName = db.getName() + '.' + regTableName + fullTableName = dbName + '.' + regTableName # self._lockTable(fullTableName) # "create table" below. Stop it if the table is "locked" - sTable.ensureTable(self, wt.getDbConn(), db.getName(), regTableName) # Ensure the table exists + sTable.ensureTable(self, wt.getDbConn(), regTableName) # Ensure the table exists # self._unlockTable(fullTableName) if Dice.throw(1) == 0: # 1 in 2 chance @@ -2024,7 +2091,7 @@ class ThreadStacks: # stack info for all threads self._allStacks[th.native_id] = stack def print(self, filteredEndName = None, filterInternal = False): - for thNid, stack in self._allStacks.items(): # for each thread + for thNid, stack in self._allStacks.items(): # for each thread, stack frames top to bottom lastFrame = stack[-1] if filteredEndName: # we need to filter out stacks that match this name if lastFrame.name == filteredEndName : # end did not match @@ -2036,9 +2103,9 @@ class ThreadStacks: # stack info for all threads '__init__']: # the thread that extracted the stack continue # ignore # Now print - print("\n<----- Thread Info for LWP/ID: {} (Execution stopped at Bottom Frame) <-----".format(thNid)) + print("\n<----- Thread Info for LWP/ID: {} (most recent call last) <-----".format(thNid)) stackFrame = 0 - for frame in stack: + for frame in stack: # was using: reversed(stack) # print(frame) print("[{sf}] File {filename}, line {lineno}, in {name}".format( sf=stackFrame, filename=frame.filename, lineno=frame.lineno, name=frame.name)) diff --git a/tests/pytest/crash_gen/db.py b/tests/pytest/crash_gen/db.py index 2a4b362f82..855e18be55 100644 --- a/tests/pytest/crash_gen/db.py +++ b/tests/pytest/crash_gen/db.py @@ -78,7 +78,7 @@ class DbConn: if nRows != 1: raise taos.error.ProgrammingError( "Unexpected result for query: {}, rows = {}".format(sql, nRows), - (0x991 if nRows==0 else 0x992) + (CrashGenError.INVALID_EMPTY_RESULT if nRows==0 else CrashGenError.INVALID_MULTIPLE_RESULT) ) if self.getResultRows() != 1 or self.getResultCols() != 1: raise RuntimeError("Unexpected result set for query: {}".format(sql)) @@ -349,7 +349,8 @@ class DbConnNative(DbConn): def execute(self, sql): if (not self.isOpen): - raise RuntimeError("Cannot execute database commands until connection is open") + raise CrashGenError( + "Cannot exec SQL unless db connection is open", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql nRows = self._tdSql.execute(sql) @@ -360,8 +361,8 @@ class DbConnNative(DbConn): def query(self, sql): # return rows affected if (not self.isOpen): - raise RuntimeError( - "Cannot query database until connection is open") + raise CrashGenError( + "Cannot query database until connection is open, restarting?", CrashGenError.DB_CONNECTION_NOT_OPEN) Logging.debug("[SQL] Executing SQL: {}".format(sql)) self._lastSql = sql nRows = self._tdSql.query(sql) diff --git a/tests/pytest/crash_gen/misc.py b/tests/pytest/crash_gen/misc.py index 2d2ce99d95..a374ed943b 100644 --- a/tests/pytest/crash_gen/misc.py +++ b/tests/pytest/crash_gen/misc.py @@ -3,14 +3,20 @@ import random import logging import os +import taos -class CrashGenError(Exception): - def __init__(self, msg=None, errno=None): - self.msg = msg - self.errno = errno - def __str__(self): - return self.msg +class CrashGenError(taos.error.ProgrammingError): + INVALID_EMPTY_RESULT = 0x991 + INVALID_MULTIPLE_RESULT = 0x992 + DB_CONNECTION_NOT_OPEN = 0x993 + # def __init__(self, msg=None, errno=None): + # self.msg = msg + # self.errno = errno + + # def __str__(self): + # return self.msg + pass class LoggingFilter(logging.Filter): @@ -168,6 +174,7 @@ class Progress: SERVICE_RECONNECT_FAILURE = 6 SERVICE_START_NAP = 7 CREATE_TABLE_ATTEMPT = 8 + QUERY_GROUP_BY = 9 tokens = { STEP_BOUNDARY: '.', @@ -178,7 +185,8 @@ class Progress: SERVICE_RECONNECT_SUCCESS: '.r>', SERVICE_RECONNECT_FAILURE: '.xr>', SERVICE_START_NAP: '_zz', - CREATE_TABLE_ATTEMPT: '_c', + CREATE_TABLE_ATTEMPT: 'c', + QUERY_GROUP_BY: 'g', } @classmethod diff --git a/tests/pytest/crash_gen/service_manager.py b/tests/pytest/crash_gen/service_manager.py index d249abc439..ae6f8d5d3a 100644 --- a/tests/pytest/crash_gen/service_manager.py +++ b/tests/pytest/crash_gen/service_manager.py @@ -51,10 +51,12 @@ class TdeInstance(): def prepareGcovEnv(cls, env): # Ref: https://gcc.gnu.org/onlinedocs/gcc/Cross-profiling.html bPath = cls._getBuildPath() # build PATH - numSegments = len(bPath.split('/')) - 1 # "/x/TDengine/build" should yield 3 - numSegments = numSegments - 1 # DEBUG only - env['GCOV_PREFIX'] = bPath + '/svc_gcov' + numSegments = len(bPath.split('/')) # "/x/TDengine/build" should yield 3 + # numSegments += 2 # cover "/src" after build + # numSegments = numSegments - 1 # DEBUG only + env['GCOV_PREFIX'] = bPath + '/src_s' # Server side source env['GCOV_PREFIX_STRIP'] = str(numSegments) # Strip every element, plus, ENV needs strings + # VERY VERY important note: GCOV data collection NOT effective upon SIG_KILL Logging.info("Preparing GCOV environement to strip {} elements and use path: {}".format( numSegments, env['GCOV_PREFIX'] )) @@ -258,14 +260,15 @@ class TdeSubProcess: TdeInstance.prepareGcovEnv(myEnv) # print(myEnv) - # print(myEnv.items()) + # print("Starting TDengine with env: ", myEnv.items()) # print("Starting TDengine via Shell: {}".format(cmdLineStr)) useShell = True self.subProcess = subprocess.Popen( - ' '.join(cmdLine) if useShell else cmdLine, - shell=useShell, - # svcCmdSingle, shell=True, # capture core dump? + # ' '.join(cmdLine) if useShell else cmdLine, + # shell=useShell, + ' '.join(cmdLine), + shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, # bufsize=1, # not supported in binary mode @@ -273,7 +276,8 @@ class TdeSubProcess: env=myEnv ) # had text=True, which interferred with reading EOF - STOP_SIGNAL = signal.SIGKILL # What signal to use (in kill) to stop a taosd process? + STOP_SIGNAL = signal.SIGKILL # signal.SIGKILL/SIGINT # What signal to use (in kill) to stop a taosd process? + SIG_KILL_RETCODE = 137 # ref: https://stackoverflow.com/questions/43268156/process-finished-with-exit-code-137-in-pycharm def stop(self): """ @@ -320,8 +324,12 @@ class TdeSubProcess: retCode = self.subProcess.returncode # should always be there # May throw subprocess.TimeoutExpired exception above, therefore # The process is guranteed to have ended by now - self.subProcess = None - if retCode != 0: # != (- signal.SIGINT): + self.subProcess = None + if retCode == self.SIG_KILL_RETCODE: + Logging.info("TSP.stop(): sub proc KILLED, as expected") + elif retCode == (- self.STOP_SIGNAL): + Logging.info("TSP.stop(), sub process STOPPED, as expected") + elif retCode != 0: # != (- signal.SIGINT): Logging.error("TSP.stop(): Failed to stop sub proc properly w/ SIG {}, retCode={}".format( self.STOP_SIGNAL, retCode)) else: From 5907b8e8b96816a6045c96af57f169d7b7b02714 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 2 Dec 2020 11:06:50 +0800 Subject: [PATCH 127/188] fix test case failure --- tests/pytest/client/alterDatabase.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/client/alterDatabase.py b/tests/pytest/client/alterDatabase.py index fa397d16c5..8191312cc0 100644 --- a/tests/pytest/client/alterDatabase.py +++ b/tests/pytest/client/alterDatabase.py @@ -32,9 +32,9 @@ class TDTestCase: tdSql.query("show databases") tdSql.checkData(0, 14, 2) - tdSql.execute("alter database db keep 365") + tdSql.execute("alter database db keep 365,365,365") tdSql.query("show databases") - tdSql.checkData(0, 7, "3650,3650,365") + tdSql.checkData(0, 7, "365,365,365") tdSql.execute("alter database db quorum 2") tdSql.query("show databases") From 19a410f94b5d72a136b850fef0b159e477e4e64a Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 2 Dec 2020 11:36:56 +0800 Subject: [PATCH 128/188] [TD-2301]: change --- tests/examples/JDBC/JDBCDemo/pom.xml | 2 +- .../com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/pom.xml b/tests/examples/JDBC/JDBCDemo/pom.xml index 121a3b5cd6..98f908b77e 100644 --- a/tests/examples/JDBC/JDBCDemo/pom.xml +++ b/tests/examples/JDBC/JDBCDemo/pom.xml @@ -56,7 +56,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.8 + 2.0.12 log4j diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java index a35628bb58..644de52dd3 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/task/InsertTableTask.java @@ -41,7 +41,7 @@ public class InsertTableTask implements Runnable { long ts = start.toEpochMilli() + (j * timeGap); // insert data into echo table for (int i = startTbIndex; i < startTbIndex + tableNumber; i++) { - String sql = SqlSpeller.insertOneRowSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts); + String sql = SqlSpeller.insertBatchSizeRowsSQL(config.getDbName(), config.getTbPrefix(), i + 1, ts, config.getNumberOfRecordsPerRequest()); logger.info(Thread.currentThread().getName() + ">>> " + sql); Statement statement = connection.createStatement(); statement.execute(sql); From 37d39c9e731cf75d0672ce7cd602e422ecd45a34 Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Wed, 2 Dec 2020 03:50:02 +0000 Subject: [PATCH 129/188] [TD-2255]: fix incorrect log variable and sim test cases --- src/mnode/src/mnodeDb.c | 2 +- tests/script/general/db/alter_option.sim | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 39a0c03008..69e8f076e9 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -250,7 +250,7 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) { } if (pCfg->daysToKeep2 < TSDB_MIN_KEEP || pCfg->daysToKeep2 > pCfg->daysToKeep) { - mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep, TSDB_MIN_KEEP, pCfg->daysToKeep); + mError("invalid db option daysToKeep2:%d valid range: [%d, %d]", pCfg->daysToKeep2, TSDB_MIN_KEEP, pCfg->daysToKeep); return TSDB_CODE_MND_INVALID_DB_OPTION_KEEP; } diff --git a/tests/script/general/db/alter_option.sim b/tests/script/general/db/alter_option.sim index c8aa2480c5..1c3f543ffd 100644 --- a/tests/script/general/db/alter_option.sim +++ b/tests/script/general/db/alter_option.sim @@ -115,31 +115,31 @@ if $data7_db != 20,20,20 then return -1 endi -sql alter database db keep 10 -sql show databases -print keep $data7_db -if $data7_db != 20,20,10 then - return -1 -endi - sql alter database db keep 20 sql show databases print keep $data7_db -if $data7_db != 20,20,20 then +if $data7_db != 20,20,20 then return -1 endi sql alter database db keep 30 sql show databases print keep $data7_db -if $data7_db != 20,20,30 then +if $data7_db != 20,20,30 then + return -1 +endi + +sql alter database db keep 40 +sql show databases +print keep $data7_db +if $data7_db != 20,20,40 then return -1 endi sql alter database db keep 40 sql alter database db keep 30 sql alter database db keep 20 -sql alter database db keep 10 +sql_error alter database db keep 10 sql_error alter database db keep 9 sql_error alter database db keep 1 sql alter database db keep 0 @@ -277,4 +277,4 @@ sql_error alter database db prec 'us' print ============== step status sql_error alter database db status 'delete' -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT From 6db661976cae0b71cf2c239d2033cf647854e52d Mon Sep 17 00:00:00 2001 From: haojun Liao Date: Wed, 2 Dec 2020 13:45:48 +0800 Subject: [PATCH 130/188] Update connector-ch.md --- documentation20/webdocs/markdowndocs/connector-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/connector-ch.md b/documentation20/webdocs/markdowndocs/connector-ch.md index c5a955f43f..69c560bbc4 100644 --- a/documentation20/webdocs/markdowndocs/connector-ch.md +++ b/documentation20/webdocs/markdowndocs/connector-ch.md @@ -142,7 +142,7 @@ C/C++的API类似于MySQL的C API。应用程序使用时，需要包含TDengine 获取最近一次API调用失败的原因，返回值为错误代码。 -**注意**：对于单个数据库连接，在同一时刻只能有一个线程使用该连接调用API，否则会有未定义的行为出现并可能导致客户端crash。客户端应用可以通过建立多个连接进行多线程的数据写入或查询处理。 +**注意**：对于每个数据库应用，2.0及以上版本 TDengine 推荐只建立一个连接。同时在应用中将该连接 (TAOS*) 结构体传递到不同的线程共享使用。基于 TAOS 结构体发出的查询、写入等操作具有多线程安全性。C 语言的连接器可以按照需求动态建立面向数据库的新连接（该过程对用户不可见），同时建议只有在程序最后退出的时候才调用 taos_close 关闭连接。 ### 异步查询API From e690846ce80c15986623534a36f7e8b808c819a7 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Wed, 2 Dec 2020 06:34:51 +0000 Subject: [PATCH 131/188] fix td-2290 --- src/client/src/tscSub.c | 91 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 83 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 043e78a9f3..52b74f7502 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -157,7 +157,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* registerSqlObj(pSql); - code = tsParseSql(pSql, false); + code = tsParseSql(pSql, true); if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { tsem_wait(&pSub->sem); code = pSql->res.code; @@ -168,7 +168,7 @@ static SSub* tscCreateSubscription(STscObj* pObj, const char* topic, const char* goto fail; } - if (pSql->cmd.command != TSDB_SQL_SELECT) { + if (pSql->cmd.command != TSDB_SQL_SELECT && pSql->cmd.command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { line = __LINE__; code = TSDB_CODE_TSC_INVALID_SQL; goto fail; @@ -182,7 +182,7 @@ fail: if (pSql->self != 0) { taosReleaseRef(tscObjRef, pSql->self); } else { - tscFreeSqlObj(pSql); + tscFreeSqlObj(pSql); } pSql = NULL; @@ -401,9 +401,11 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char tscLoadSubscriptionProgress(pSub); } - if (!tscUpdateSubscription(pObj, pSub)) { - taos_unsubscribe(pSub, 1); - return NULL; + if (pSub->pSql->cmd.command == TSDB_SQL_SELECT) { + if (!tscUpdateSubscription(pObj, pSub)) { + taos_unsubscribe(pSub, 1); + return NULL; + } } pSub->interval = interval; @@ -417,10 +419,80 @@ TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char* topic, const char return pSub; } +SSqlObj* recreateSqlObj(SSub* pSub) { + SSqlObj* pSql = calloc(1, sizeof(SSqlObj)); + if (pSql == NULL) { + return NULL; + } + + pSql->signature = pSql; + pSql->pTscObj = pSub->taos; + + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + if (tsem_init(&pSql->rspSem, 0, 0) == -1) { + tscFreeSqlObj(pSql); + return NULL; + } + + pSql->param = pSub; + pSql->maxRetry = TSDB_MAX_REPLICA; + pSql->fp = asyncCallback; + pSql->fetchFp = asyncCallback; + pSql->sqlstr = strdup(pSub->pSql->sqlstr); + if (pSql->sqlstr == NULL) { + tscFreeSqlObj(pSql); + return NULL; + } + + pRes->qhandle = 0; + pRes->numOfRows = 1; + + int code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE); + if (code != TSDB_CODE_SUCCESS) { + tscFreeSqlObj(pSql); + return NULL; + } + + registerSqlObj(pSql); + + code = tsParseSql(pSql, true); + if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) { + tsem_wait(&pSub->sem); + code = pSql->res.code; + } + + if (code != TSDB_CODE_SUCCESS) { + taosReleaseRef(tscObjRef, pSql->self); + return NULL; + } + + if (pSql->cmd.command != TSDB_SQL_SELECT) { + taosReleaseRef(tscObjRef, pSql->self); + return NULL; + } + + return pSql; +} + TAOS_RES *taos_consume(TAOS_SUB *tsub) { SSub *pSub = (SSub *)tsub; if (pSub == NULL) return NULL; + if (pSub->pSql->cmd.command == TSDB_SQL_RETRIEVE_EMPTY_RESULT) { + SSqlObj* pSql = recreateSqlObj(pSub); + if (pSql == NULL) { + return NULL; + } + if (pSub->pSql->self != 0) { + taosReleaseRef(tscObjRef, pSub->pSql->self); + } else { + tscFreeSqlObj(pSub->pSql); + } + pSub->pSql = pSql; + pSql->pSubscription = pSub; + } + tscSaveSubscriptionProgress(pSub); SSqlObj *pSql = pSub->pSql; @@ -512,10 +584,13 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { } if (pSub->pSql != NULL) { - taos_free_result(pSub->pSql); + if (pSub->pSql->self != 0) { + taosReleaseRef(tscObjRef, pSub->pSql->self); + } else { + tscFreeSqlObj(pSub->pSql); + } } - tscFreeSqlObj(pSub->pSql); taosArrayDestroy(pSub->progress); tsem_destroy(&pSub->sem); memset(pSub, 0, sizeof(*pSub)); From b55548782bda709da4a5a7d99a40644eb82734d8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 15:23:34 +0800 Subject: [PATCH 132/188] [TD-2293]: improve projection query performance for super table. --- src/client/src/tscSQLParser.c | 7 +- src/query/inc/qExecutor.h | 4 +- src/query/src/qExecutor.c | 318 ++++++++++++++++++++++------------ src/query/src/qUtil.c | 2 - src/util/src/hash.c | 5 +- 5 files changed, 217 insertions(+), 119 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 309d354849..0c9ad78cd9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5311,9 +5311,10 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { /* - * the limitation/offset value should be removed during retrieve data from virtual node, - * since the global order are done in client side, so the limitation should also - * be done at the client side. + * the offset value should be removed during retrieve data from virtual node, since the + * global order are done in client side, so the offset is applied at the client side + * However, note that the maximum allowed number of result for each table should be less + * than or equal to the value of limit. */ if (pQueryInfo->limit.limit > 0) { pQueryInfo->limit.limit = -1; diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 71fc01923a..4ec6ed30ba 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -190,6 +190,7 @@ typedef struct SQueryRuntimeEnv { bool groupbyNormalCol; // denote if this is a groupby normal column query bool hasTagResults; // if there are tag values in final result or not bool timeWindowInterpo;// if the time window start/end required interpolation + bool queryWindowIdentical; // all query time windows are identical for all tables in one group int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file @@ -217,7 +218,8 @@ typedef struct SQInfo { STableGroupInfo tableGroupInfo; // table list SArray STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; - SArray* arrTableIdInfo; +// SArray* arrTableIdInfo; + SHashObj* arrTableIdInfo; int32_t groupIndex; /* diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6de57b32fd..7e6950a735 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -197,6 +197,8 @@ static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); +static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery); +static STableIdInfo createTableIdInfo(SQuery* pQuery); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -2781,6 +2783,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa pQuery->rec.capacity = capacity; } +// TODO merge with enuserOutputBufferSimple static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pBlockInfo) { // in case of prj/diff query, ensure the output buffer is sufficient to accommodate the results of current block SQuery* pQuery = pRuntimeEnv->pQuery; @@ -3916,13 +3919,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - STsdbQueryCond cond = { - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); @@ -4005,18 +4002,11 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { break; } - STsdbQueryCond cond = { - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, qstatus.curWindow); - if (pRuntimeEnv->pSecQueryHandle != NULL) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } + STsdbQueryCond cond = createTsdbQueryCond(pQuery); restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { @@ -4541,16 +4531,19 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data } } - int32_t numOfTables = (int32_t)taosArrayGetSize(pQInfo->arrTableIdInfo); + int32_t numOfTables = taosHashGetSize(pQInfo->arrTableIdInfo); *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); - for(int32_t i = 0; i < numOfTables; i++) { - STableIdInfo* pSrc = taosArrayGet(pQInfo->arrTableIdInfo, i); + + STableIdInfo* item = taosHashIterate(pQInfo->arrTableIdInfo, NULL); + while(item) { STableIdInfo* pDst = (STableIdInfo*)data; - pDst->uid = htobe64(pSrc->uid); - pDst->tid = htonl(pSrc->tid); - pDst->key = htobe64(pSrc->key); + pDst->uid = htobe64(item->uid); + pDst->tid = htonl(item->tid); + pDst->key = htobe64(item->key); + data += sizeof(STableIdInfo); + item = taosHashIterate(pQInfo->arrTableIdInfo, item); } // Check if query is completed or not for stable query or normal table query respectively. @@ -4877,13 +4870,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) return TSDB_CODE_SUCCESS; } - STsdbQueryCond cond = { - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) @@ -5276,6 +5263,41 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { return true; } +STsdbQueryCond createTsdbQueryCond(SQuery* pQuery) { + STsdbQueryCond cond = { + .colList = pQuery->colList, + .order = pQuery->order.order, + .numOfCols = pQuery->numOfCols, + }; + + TIME_WINDOW_COPY(cond.twindow, pQuery->window); + return cond; +} + +static STableIdInfo createTableIdInfo(SQuery* pQuery) { + assert(pQuery != NULL && pQuery->current != NULL); + + STableIdInfo tidInfo; + STableId* id = TSDB_TABLEID(pQuery->current->pTable); + + tidInfo.uid = id->uid; + tidInfo.tid = id->tid; + tidInfo.key = pQuery->current->lastKey; + + return tidInfo; +} + +static void updateTableIdInfo(SQuery* pQuery, SHashObj* pTableIdInfo) { + STableIdInfo tidInfo = createTableIdInfo(pQuery); + STableIdInfo* idinfo = taosHashGet(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid)); + if (idinfo != NULL) { + assert(idinfo->tid == tidInfo.tid && idinfo->uid == tidInfo.uid); + idinfo->key = tidInfo.key; + } else { + taosHashPut(pTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); + } +} + /** * super table query handler * 1. super table projection query, group-by on normal columns query, ts-comp query @@ -5295,18 +5317,11 @@ static void sequentialTableProcess(SQInfo *pQInfo) { assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); while (pQInfo->groupIndex < numOfGroups) { - SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); + SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); - qDebug("QInfo:%p last_row query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, pQInfo->groupIndex, - numOfGroups, group); - - STsdbQueryCond cond = { - .colList = pQuery->colList, - .order = pQuery->order.order, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + qDebug("QInfo:%p point interpolation query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, + pQInfo->groupIndex, numOfGroups, group); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); @@ -5330,14 +5345,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) { initCtxOutputBuf(pRuntimeEnv); - SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); + SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); taosArrayDestroy(s); // here we simply set the first table as current table - SArray* first = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); + SArray *first = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex); pQuery->current = taosArrayGetP(first, 0); scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); @@ -5359,19 +5374,14 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } } - } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query + } else if (pRuntimeEnv->groupbyNormalCol) { // group-by on normal columns query while (pQInfo->groupIndex < numOfGroups) { - SArray* group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); + SArray *group = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, pQInfo->groupIndex); - qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, numOfGroups); + qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, + numOfGroups); - STsdbQueryCond cond = { - .colList = pQuery->colList, - .order = pQuery->order.order, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); @@ -5394,7 +5404,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { longjmp(pRuntimeEnv->env, terrno); } - SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); + SArray *s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); @@ -5405,24 +5415,24 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - // no results generated for current group, continue to try the next group + // no results generated for current group, continue to try the next group taosArrayDestroy(s); if (pWindowResInfo->size <= 0) { continue; } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns SResultRow *pResult = pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + SResultRowCellInfo *pCell = getResultCell(pRuntimeEnv, pResult, j); pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } qDebug("QInfo:%p generated groupby columns results %d rows for group %d completed", pQInfo, pWindowResInfo->size, - pQInfo->groupIndex); + pQInfo->groupIndex); int32_t currentGroupIndex = pQInfo->groupIndex; pQuery->rec.rows = 0; @@ -5431,16 +5441,119 @@ static void sequentialTableProcess(SQInfo *pQInfo) { ensureOutputBufferSimple(pRuntimeEnv, pWindowResInfo->size); copyFromWindowResToSData(pQInfo, pWindowResInfo); - pQInfo->groupIndex = currentGroupIndex; //restore the group index + pQInfo->groupIndex = currentGroupIndex; // restore the group index assert(pQuery->rec.rows == pWindowResInfo->size); clearClosedTimeWindow(pRuntimeEnv); break; } + } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) { + //super table projection query with identical query time range for all tables. + SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; + + resetDefaultResInfoOutputBuf(pRuntimeEnv); + + SArray *group = GET_TABLEGROUP(pQInfo, 0); + assert(taosArrayGetSize(group) == pQInfo->tableqinfoGroupInfo.numOfTables && + 1 == taosArrayGetSize(pQInfo->tableqinfoGroupInfo.pGroupList)); + + void *pQueryHandle = pRuntimeEnv->pQueryHandle; + if (pQueryHandle == NULL) { + STsdbQueryCond con = createTsdbQueryCond(pQuery); + pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &con, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); + pQueryHandle = pRuntimeEnv->pQueryHandle; + } + + // skip blocks without load the actual data block from file if no filter condition present + // skipBlocks(&pQInfo->runtimeEnv); + // if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0) { + // setQueryStatus(pQuery, QUERY_COMPLETED); + // return; + // } + + bool hasMoreBlock = true; + SQueryCostInfo *summary = &pRuntimeEnv->summary; + while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) { + summary->totalBlocks += 1; + + if (IS_QUERY_KILLED(pQInfo)) { + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + } + + tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); + STableQueryInfo **pTableQueryInfo = + (STableQueryInfo **)taosHashGet(pQInfo->tableqinfoGroupInfo.map, &blockInfo.tid, sizeof(blockInfo.tid)); + if (pTableQueryInfo == NULL) { + break; + } + + pQuery->current = *pTableQueryInfo; + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert(((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && + (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); + } else { + assert(((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && + (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); + } + + if (pRuntimeEnv->hasTagResults) { + setTagVal(pRuntimeEnv, pQuery->current->pTable, pQInfo->tsdb); + } + + uint32_t status = 0; + SDataStatis *pStatis = NULL; + SArray * pDataBlock = NULL; + + int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo, + &pStatis, &pDataBlock, &status); + if (ret != TSDB_CODE_SUCCESS) { + break; + } + + assert(status != BLK_DATA_DISCARD); + ensureOutputBuffer(pRuntimeEnv, &blockInfo); + + pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); + + summary->totalRows += blockInfo.rows; + qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, + GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, + pQuery->current->lastKey); + + pQuery->rec.rows = getNumOfResult(pRuntimeEnv); + + // the flag may be set by tableApplyFunctionsOnBlock, clear it here + CLEAR_QUERY_STATUS(pQuery, QUERY_COMPLETED); + + updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); + skipResults(pRuntimeEnv); + + // the limitation of output result is reached, set the query completed + if (limitResults(pRuntimeEnv)) { + SET_STABLE_QUERY_OVER(pQInfo); + break; + } + + // while the output buffer is full or limit/offset is applied, query may be paused here + if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL|QUERY_COMPLETED)) { + break; + } + } + + if (!hasMoreBlock) { + setQueryStatus(pQuery, QUERY_COMPLETED); + SET_STABLE_QUERY_OVER(pQInfo); + } } else { /* - * 1. super table projection query, 2. ts-comp query - * if the subgroup index is larger than 0, results generated by group by tbname,k is existed. + * the following two cases handled here. + * 1. ts-comp query, and 2. the super table projection query with different query time range for each table. + * If the subgroup index is larger than 0, results generated by group by tbname,k is existed. * we need to return it to client in the first place. */ if (pQInfo->groupIndex > 0) { @@ -5503,14 +5616,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { * to ensure that, we can reset the query range once query on a meter is completed. */ pQInfo->tableIndex++; - - STableIdInfo tidInfo = {0}; - - STableId* id = TSDB_TABLEID(pQuery->current->pTable); - tidInfo.uid = id->uid; - tidInfo.tid = id->tid; - tidInfo.key = pQuery->current->lastKey; - taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo); + updateTableIdInfo(pQuery, pQInfo->arrTableIdInfo); // if the buffer is full or group by each table, we need to jump out of the loop if (Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL)) { @@ -5537,31 +5643,31 @@ static void sequentialTableProcess(SQInfo *pQInfo) { if (pQInfo->tableIndex >= pQInfo->tableqinfoGroupInfo.numOfTables) { setQueryStatus(pQuery, QUERY_COMPLETED); } - } - /* - * 1. super table projection query, group-by on normal columns query, ts-comp query - * 2. point interpolation query, last row query - * - * group-by on normal columns query and last_row query do NOT invoke the finalizer here, - * since the finalize stage will be done at the client side. - * - * projection query, point interpolation query do not need the finalizer. - * - * Only the ts-comp query requires the finalizer function to be executed here. - */ - if (isTSCompQuery(pQuery)) { - finalizeQueryResult(pRuntimeEnv); - } + /* + * 1. super table projection query, group-by on normal columns query, ts-comp query + * 2. point interpolation query, last row query + * + * group-by on normal columns query and last_row query do NOT invoke the finalizer here, + * since the finalize stage will be done at the client side. + * + * projection query, point interpolation query do not need the finalizer. + * + * Only the ts-comp query requires the finalizer function to be executed here. + */ + if (isTSCompQuery(pQuery)) { + finalizeQueryResult(pRuntimeEnv); + } - if (pRuntimeEnv->pTSBuf != NULL) { - pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; - } + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + } - qDebug( - "QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 " points returned, total:%" PRId64 ", offset:%" PRId64, - pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, pQuery->rec.total, - pQuery->limit.offset); + qDebug("QInfo %p numOfTables:%" PRIu64 ", index:%d, numOfGroups:%" PRIzu ", %" PRId64 + " points returned, total:%" PRId64 ", offset:%" PRId64, + pQInfo, (uint64_t)pQInfo->tableqinfoGroupInfo.numOfTables, pQInfo->tableIndex, numOfGroups, pQuery->rec.rows, + pQuery->rec.total, pQuery->limit.offset); + } } static void doSaveContext(SQInfo *pQInfo) { @@ -5576,13 +5682,7 @@ static void doSaveContext(SQInfo *pQInfo) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); } - STsdbQueryCond cond = { - .order = pQuery->order.order, - .colList = pQuery->colList, - .numOfCols = pQuery->numOfCols, - }; - - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + STsdbQueryCond cond = createTsdbQueryCond(pQuery); // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { @@ -5855,13 +5955,8 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) qDebug("QInfo:%p query paused due to output limitation, next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->current->lastKey, pQuery->window.ekey); } else if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { - STableIdInfo tidInfo; - STableId* id = TSDB_TABLEID(pQuery->current->pTable); - - tidInfo.uid = id->uid; - tidInfo.tid = id->tid; - tidInfo.key = pQuery->current->lastKey; - taosArrayPush(pQInfo->arrTableIdInfo, &tidInfo); + STableIdInfo tidInfo = createTableIdInfo(pQuery); + taosHashPut(pQInfo->arrTableIdInfo, &tidInfo.tid, sizeof(tidInfo.tid), &tidInfo, sizeof(STableIdInfo)); } if (!isTSCompQuery(pQuery)) { @@ -6859,8 +6954,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } - int tableIndex = 0; - pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); @@ -6882,7 +6975,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou } // NOTE: pTableCheckInfo need to update the query time range and the lastKey info - pQInfo->arrTableIdInfo = taosArrayInit(tableIndex, sizeof(STableIdInfo)); + pQInfo->arrTableIdInfo = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); pQInfo->dataReady = QUERY_RESULT_NOT_READY; pQInfo->rspContext = NULL; pthread_mutex_init(&pQInfo->lock, NULL); @@ -6892,10 +6985,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQuery->window = pQueryMsg->window; changeExecuteScanOrder(pQInfo, pQueryMsg, stableQuery); + pQInfo->runtimeEnv.queryWindowIdentical = true; STimeWindow window = pQuery->window; int32_t index = 0; - for(int32_t i = 0; i < numOfGroups; ++i) { SArray* pa = taosArrayGetP(pQInfo->tableGroupInfo.pGroupList, i); @@ -6910,9 +7003,12 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); - window.skey = info->lastKey; + if (info->lastKey != pQuery->window.skey) { + pQInfo->runtimeEnv.queryWindowIdentical = false; + } + + void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; @@ -7126,7 +7222,7 @@ static void freeQInfo(SQInfo *pQInfo) { tfree(pQInfo->pBuf); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); - taosArrayDestroy(pQInfo->arrTableIdInfo); + taosHashCleanup(pQInfo->arrTableIdInfo); pQInfo->signature = 0; @@ -7506,7 +7602,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co size_t size = getResultSize(pQInfo, &pQuery->rec.rows); size += sizeof(int32_t); - size += sizeof(STableIdInfo) * taosArrayGetSize(pQInfo->arrTableIdInfo); + size += sizeof(STableIdInfo) * taosHashGetSize(pQInfo->arrTableIdInfo); *contLen = (int32_t)(size + sizeof(SRetrieveTableRsp)); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 2c41d9bfc5..f5f2da9f25 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -83,8 +83,6 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR return; } -// assert(pWindowResInfo->size == 1); - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pWindowRes = pWindowResInfo->pResult[i]; clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 3ac54eedd8..0e3e0d3e24 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -151,8 +151,9 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); */ SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { - if (capacity == 0 || fn == NULL) { - return NULL; + assert(fn != NULL); + if (capacity == 0) { + capacity = 4; } SHashObj *pHashObj = (SHashObj *)calloc(1, sizeof(SHashObj)); From 5746fdabe71d3c97381bdb8dab114d4d3d66e15c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 15:28:34 +0800 Subject: [PATCH 133/188] [TD-2129] refactor codes. --- src/query/src/qExecutor.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7e6950a735..d65b2e8f0a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5078,6 +5078,20 @@ static FORCE_INLINE void setEnvForEachBlock(SQInfo* pQInfo, STableQueryInfo* pTa } } +static void doTableQueryInfoTimeWindowCheck(SQuery* pQuery, STableQueryInfo* pTableQueryInfo) { + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert( + (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && + (pTableQueryInfo->lastKey >= pTableQueryInfo->win.skey) && + (pTableQueryInfo->win.skey >= pQuery->window.skey && pTableQueryInfo->win.ekey <= pQuery->window.ekey)); + } else { + assert( + (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && + (pTableQueryInfo->lastKey <= pTableQueryInfo->win.skey) && + (pTableQueryInfo->win.skey <= pQuery->window.skey && pTableQueryInfo->win.ekey >= pQuery->window.ekey)); + } +} + static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery* pQuery = pRuntimeEnv->pQuery; @@ -5104,17 +5118,7 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - if (QUERY_IS_ASC_QUERY(pQuery)) { - assert( - ((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && - ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && - ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); - } else { - assert( - ((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && - ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && - ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); - } + doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); if (!pRuntimeEnv->groupbyNormalCol) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); @@ -5488,17 +5492,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - if (QUERY_IS_ASC_QUERY(pQuery)) { - assert(((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && - ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && - ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && - (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); - } else { - assert(((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && - ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && - ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && - (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); - } + doTableQueryInfoTimeWindowCheck(pQuery, *pTableQueryInfo); if (pRuntimeEnv->hasTagResults) { setTagVal(pRuntimeEnv, pQuery->current->pTable, pQInfo->tsdb); From 6b2d676ac1628fc2357f3033040fae9a135ec738 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 15:43:50 +0800 Subject: [PATCH 134/188] [TD-2293]add parameters in query message. --- src/inc/taosmsg.h | 16 +++++++++------- src/query/src/qExecutor.c | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b77db69c46..b4d3bec958 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -476,19 +476,21 @@ typedef struct { int16_t numOfGroupCols; // num of group by columns int16_t orderByIdx; int16_t orderType; // used in group by xx order by xxx + int64_t tableLimit; // limit the number of rows for each table, used in order by + limit in stable projection query. + int16_t prjOrder; // global order in super table projection query. int64_t limit; int64_t offset; uint32_t queryType; // denote another query process int16_t numOfOutput; // final output columns numbers int16_t tagNameRelType; // relation of tag criteria and tbname criteria - int16_t fillType; // interpolate type - uint64_t fillVal; // default value array list + int16_t fillType; // interpolate type + uint64_t fillVal; // default value array list int32_t secondStageOutput; - int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed - int32_t tsLen; // total length of ts comp block - int32_t tsNumOfBlocks; // ts comp block numbers - int32_t tsOrder; // ts comp block order - int32_t numOfTags; // number of tags columns involved + int32_t tsOffset; // offset value in current msg body, NOTE: ts list is compressed + int32_t tsLen; // total length of ts comp block + int32_t tsNumOfBlocks; // ts comp block numbers + int32_t tsOrder; // ts comp block order + int32_t numOfTags; // number of tags columns involved SColumnInfo colList[]; } SQueryTableMsg; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d65b2e8f0a..8c705c893c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5500,7 +5500,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { uint32_t status = 0; SDataStatis *pStatis = NULL; - SArray * pDataBlock = NULL; + SArray *pDataBlock = NULL; int32_t ret = loadDataBlockOnDemand(pRuntimeEnv, &pQuery->current->windowResInfo, pQueryHandle, &blockInfo, &pStatis, &pDataBlock, &status); From e9c5859e41703801e35b4dfcfaf8b6ddf9c05353 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 2 Dec 2020 15:48:09 +0800 Subject: [PATCH 135/188] auto pull submodule code and add more check. --- tests/pytest/hivemq-extension-test.py | 69 ++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/tests/pytest/hivemq-extension-test.py b/tests/pytest/hivemq-extension-test.py index eb27f7a7d9..3d0b1ef83f 100644 --- a/tests/pytest/hivemq-extension-test.py +++ b/tests/pytest/hivemq-extension-test.py @@ -21,6 +21,7 @@ import subprocess from shutil import which from multipledispatch import dispatch + @dispatch(str, str) def v_print(msg: str, arg: str): if verbose: @@ -50,28 +51,34 @@ def v_print(msg: str, arg1: int, arg2: int, arg3: int, arg4: int): if verbose: print(msg % (int(arg1), int(arg2), int(arg3), int(arg4))) + def isHiveMQInstalled(): v_print("%s", "Check if HiveMQ installed") defaultHiveMQPath = "/opt/hivemq*" hiveMQDir = glob.glob(defaultHiveMQPath) if (len(hiveMQDir) == 0): - v_print("%s", "ERROR: hivemq not found!") + v_print("%s", "ERROR: HiveMQ NOT found") return False else: v_print("HiveMQ installed at %s", hiveMQDir[0]) return True + def isMosquittoInstalled(): v_print("%s", "Check if mosquitto installed") if not which('mosquitto_pub'): - v_print("%s", "mosquitto is not installed") + v_print("%s", "ERROR: mosquitto is NOT installed") return False else: return True + def installExtension(): currentDir = os.getcwd() - os.chdir('../../src/connector/hivemq-tdengine-extension') + extDir = 'src/connector/hivemq-tdengine-extension' + os.chdir('../..') + os.system('git submodule update --init -- %s' % extDir) + os.chdir(extDir) v_print("%s", "build extension..") os.system('mvn clean package') @@ -92,6 +99,7 @@ def installExtension(): os.chdir(currentDir) + def stopProgram(prog: str): psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog @@ -106,9 +114,24 @@ def stopProgram(prog: str): psCmd, shell=True).decode("utf-8") pass + def stopHiveMQ(): stopProgram("hivemq.jar") - v_print("%s", "hivemq is NOT running") + v_print("%s", "ERROR: HiveMQ is NOT running") + + +def checkProgramRunning(prog: str): + psCmd = "ps ax|grep -w %s| grep -v grep | awk '{print $1}'" % prog + + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + + if not processID: + v_print("ERROR: %s is NOT running", prog) + return False + else: + return True + def runHiveMQ(): defaultHiveMQPath = "/opt/hivemq*" @@ -116,11 +139,19 @@ def runHiveMQ(): runPath = hiveMQDir[0] + '/bin/run.sh > /dev/null &' os.system(runPath) time.sleep(10) - v_print("%s", "hivemq is running") + + if not checkProgramRunning("hivemq.jar"): + return False + else: + v_print("%s", "hivemq is running") + return True + def getBuildPath(): selfPath = os.path.dirname(os.path.realpath(__file__)) + binPath = '' + if ("community" in selfPath): projPath = selfPath[:selfPath.find("community")] else: @@ -130,9 +161,10 @@ def getBuildPath(): if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): - buildPath = root[:len(root)-len("/build/bin")] + binPath = root[:len(root) - len("/build/bin")] break - return buildPath + return binPath + def runTDengine(): stopProgram("taosd") @@ -140,8 +172,8 @@ def runTDengine(): buildPath = getBuildPath() if (buildPath == ""): - v_print("%s", "taosd NOT found!") - os.exit(1) + v_print("%s", "ERROR: taosd NOT found!") + sys.exit(1) else: v_print("%s", "taosd found in %s" % buildPath) @@ -149,6 +181,13 @@ def runTDengine(): os.system('%s > /dev/null &' % binPath) time.sleep(10) + if not checkProgramRunning("taosd"): + return False + else: + v_print("%s", "TDengine is running") + return True + + def reCreateDatabase(): buildPath = getBuildPath() @@ -157,16 +196,20 @@ def reCreateDatabase(): os.system('%s -s "DROP DATABASE IF EXISTS hivemq"' % binPath) os.system('%s -s "CREATE DATABASE IF NOT EXISTS hivemq"' % binPath) + def sendMqttMsg(topic: str, payload: str): testStr = 'mosquitto_pub -t %s -m "%s"' % (topic, payload) os.system(testStr) time.sleep(3) + def checkTDengineData(topic: str, payload: str): buildPath = getBuildPath() binPath = buildPath + "/build/bin/taos" - output = subprocess.check_output('%s -s "select * from hivemq.mqtt_payload"' % binPath, shell=True).decode('utf-8') + output = subprocess.check_output( + '%s -s "select * from hivemq.mqtt_payload"' % + binPath, shell=True).decode('utf-8') if (topic in output) and (payload in output): v_print("%s", output) return True @@ -190,11 +233,13 @@ if __name__ == "__main__": installExtension() - runTDengine() + if not runTDengine(): + sys.exit(1) reCreateDatabase() - runHiveMQ() + if not runHiveMQ(): + sys.exit(1) sendMqttMsg(testTopic, testPayload) From 0c38e1d9d3b339294ae2290e632bc9f03b5cc0f0 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 18:37:28 +0800 Subject: [PATCH 136/188] [TD-2298]: fix taosd crash during table interval query. --- src/query/src/qExecutor.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 8c705c893c..cda9455d6f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3597,7 +3597,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { // do nothing, no results - } else { + } else {// even win.skey != lastKey, the results may not generated. pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; } @@ -3986,11 +3986,13 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { qstatus.status = pQuery->status; // do nothing if no data blocks are found qualified during scan + qstatus.lastKey = pTableQueryInfo->lastKey; if (qstatus.lastKey != pTableQueryInfo->lastKey) { qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; + } else { // the lastkey does not increase, which means no data checked yet + qDebug("QInfo:%p no results generated in this scan, abort", pQInfo); + return; } - - qstatus.lastKey = pTableQueryInfo->lastKey; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { From 81036205759f1558abc056e38a416fd1f0a6c4ab Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 18:43:23 +0800 Subject: [PATCH 137/188] [TD-225] refactor codes. --- src/query/inc/qExecutor.h | 15 +++++++-------- src/query/inc/qUtil.h | 2 +- src/query/src/qExecutor.c | 28 ++++++---------------------- src/query/src/qUtil.c | 5 ++--- 4 files changed, 16 insertions(+), 34 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 4ec6ed30ba..aa018e75d4 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -84,14 +84,13 @@ typedef struct SResultRec { } SResultRec; typedef struct SWindowResInfo { - SResultRow** pResult; // result list - int16_t type:8; // data type for hash key - int32_t size:24; // number of result set - int32_t threshold; // threshold to halt query and return the generated results. - int32_t capacity; // max capacity - int32_t curIndex; // current start active index - int64_t startTime; // start time of the first time window for sliding query - int64_t prevSKey; // previous (not completed) sliding window start key + SResultRow** pResult; // result list + int16_t type:8; // data type for hash key + int32_t size:24; // number of result set + int32_t capacity; // max capacity + int32_t curIndex; // current start active index + int64_t startTime; // start time of the first time window for sliding query + int64_t prevSKey; // previous (not completed) sliding window start key } SWindowResInfo; typedef struct SColumnFilterElem { diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 8b84ac0182..f15f1f33eb 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -30,7 +30,7 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t typ void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type); +int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int16_t type); void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo); void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index cda9455d6f..bfdcd6d28a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -761,7 +761,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; // the number of completed slots are larger than the threshold, return current generated results to client. - if (numOfClosed > pWindowResInfo->threshold) { + if (numOfClosed > pQuery->rec.threshold) { qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); @@ -3587,17 +3587,9 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - - // TODO validate the assertion -// if (!QUERY_IS_ASC_QUERY(pQuery)) { -// assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); -// } else { -// assert(pTableQueryInfo->win.ekey <= pTableQueryInfo->lastKey + step); -// } - if (pTableQueryInfo->lastKey == pTableQueryInfo->win.skey) { // do nothing, no results - } else {// even win.skey != lastKey, the results may not generated. + } else {// NOTE: even win.skey != lastKey, the results may not generated. pTableQueryInfo->win.ekey = pTableQueryInfo->lastKey + step; } @@ -4102,9 +4094,8 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void // set more initial size of interval/groupby query if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { - int32_t initialSize = 16; - int32_t initialThreshold = 100; - int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); + int32_t initialSize = 128; + int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4989,20 +4980,13 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo if (!QUERY_IS_INTERVAL_QUERY(pQuery)) { int16_t type = TSDB_DATA_TYPE_NULL; - int32_t threshold = 0; - if (pRuntimeEnv->groupbyNormalCol) { // group by columns not tags; type = getGroupbyColumnType(pQuery, pQuery->pGroupbyExpr); - threshold = 4000; } else { type = TSDB_DATA_TYPE_INT; // group id - threshold = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (threshold < 8) { - threshold = 8; - } } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, threshold, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -5022,7 +5006,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, 1024, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, type); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index f5f2da9f25..3f010a579d 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -43,10 +43,9 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) { +int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t type) { pWindowResInfo->capacity = size; - pWindowResInfo->threshold = threshold; - + pWindowResInfo->type = type; pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; From c121f5b7b91e5680a63a7c35b4bb5e3cd46f1fe5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 18:44:32 +0800 Subject: [PATCH 138/188] [TD-225] refactor codes. --- src/query/inc/qExecutor.h | 8 +++---- src/query/inc/qUtil.h | 18 +++++++------- src/query/src/qExecutor.c | 50 +++++++++++++++++++-------------------- src/query/src/qUtil.c | 20 ++++++++-------- 4 files changed, 48 insertions(+), 48 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index aa018e75d4..b73f7ce3f5 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -83,7 +83,7 @@ typedef struct SResultRec { int32_t threshold; // result size threshold in rows. } SResultRec; -typedef struct SWindowResInfo { +typedef struct SResultRowInfo { SResultRow** pResult; // result list int16_t type:8; // data type for hash key int32_t size:24; // number of result set @@ -91,7 +91,7 @@ typedef struct SWindowResInfo { int32_t curIndex; // current start active index int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key -} SWindowResInfo; +} SResultRowInfo; typedef struct SColumnFilterElem { int16_t bytes; // column length @@ -114,7 +114,7 @@ typedef struct STableQueryInfo { STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list - SWindowResInfo windowResInfo; + SResultRowInfo windowResInfo; } STableQueryInfo; typedef struct SQueryCostInfo { @@ -178,7 +178,7 @@ typedef struct SQueryRuntimeEnv { uint16_t* offset; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; - SWindowResInfo windowResInfo; + SResultRowInfo windowResInfo; STSBuf* pTSBuf; STSCursor cur; SQueryCostInfo summary; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index f15f1f33eb..fb71c8a5fe 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -30,19 +30,19 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t typ void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int16_t type); +int32_t initWindowResInfo(SResultRowInfo* pWindowResInfo, int32_t size, int16_t type); -void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo); -void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); +void cleanupTimeWindowInfo(SResultRowInfo* pWindowResInfo); +void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo* pWindowResInfo); void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num); void clearClosedTimeWindow(SQueryRuntimeEnv* pRuntimeEnv); -int32_t numOfClosedTimeWindow(SWindowResInfo* pWindowResInfo); -void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); -void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); -void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); +int32_t numOfClosedTimeWindow(SResultRowInfo* pWindowResInfo); +void closeTimeWindow(SResultRowInfo* pWindowResInfo, int32_t slot); +void closeAllTimeWindow(SResultRowInfo* pWindowResInfo); +void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order); -static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) { +static FORCE_INLINE SResultRow *getResultRow(SResultRowInfo *pWindowResInfo, int32_t slot) { assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); return pWindowResInfo->pResult[slot]; } @@ -50,7 +50,7 @@ static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int #define curTimeWindowIndex(_winres) ((_winres)->curIndex) #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pExpr1[1].base.arg->argValue.i64:1) -bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); +bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot); int32_t initResultRow(SResultRow *pResultRow); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bfdcd6d28a..3e683609d1 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -461,7 +461,7 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis return true; } -static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, +static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); int32_t *p1 = @@ -518,7 +518,7 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin } // get the correct time window according to the handled timestamp -static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) { +static STimeWindow getActiveTimeWindow(SResultRowInfo *pWindowResInfo, int64_t ts, SQuery *pQuery) { STimeWindow w = {0}; if (pWindowResInfo->curIndex == -1) { // the first window, from the previous stored value @@ -611,7 +611,7 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -644,7 +644,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes return TSDB_CODE_SUCCESS; } -static bool getResultRowStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { +static bool getResultRowStatus(SResultRowInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); return pWindowResInfo->pResult[slot]->closed; } @@ -703,7 +703,7 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se /** * NOTE: the query status only set for the first scan of master scan. */ -static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SWindowResInfo *pWindowResInfo) { +static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; if (pRuntimeEnv->scanFlag != MASTER_SCAN) { return pWindowResInfo->size; @@ -1157,7 +1157,7 @@ static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY * such as count/min/max etc. */ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SWindowResInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { + SResultRowInfo *pWindowResInfo, __block_search_fn_t searchFn, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); @@ -1575,7 +1575,7 @@ static void setTimeWindowEKeyInterp(SQueryRuntimeEnv* pRuntimeEnv, SArray* pData } static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pStatis, SDataBlockInfo *pDataBlockInfo, - SWindowResInfo *pWindowResInfo, SArray *pDataBlock) { + SResultRowInfo *pWindowResInfo, SArray *pDataBlock) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; bool masterScan = IS_MASTER_SCAN(pRuntimeEnv); @@ -1781,7 +1781,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQInfo = pQuery->current; - SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); @@ -2605,7 +2605,7 @@ static bool overlapWithTimeWindow(SQuery* pQuery, SDataBlockInfo* pBlockInfo) { return false; } -int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { +int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pWindowResInfo, void* pQueryHandle, SDataBlockInfo* pBlockInfo, SDataStatis **pStatis, SArray** pDataBlock, uint32_t* status) { SQuery *pQuery = pRuntimeEnv->pQuery; *status = BLK_DATA_NO_NEEDED; @@ -2828,7 +2828,7 @@ static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo if (QUERY_IS_INTERVAL_QUERY(pQuery) && pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL) { STimeWindow w = TSWINDOW_INITIALIZER; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (QUERY_IS_ASC_QUERY(pQuery)) { getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w); @@ -3192,14 +3192,14 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) return -1; } - SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; + SResultRowInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); TSKEY leftTimestamp = GET_INT64_VAL(b1); - SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; + SResultRowInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); @@ -3439,7 +3439,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int32_t pos = pTree->pNode[0].index; - SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; + SResultRowInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); @@ -3603,7 +3603,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1; } -static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t order) { +static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SResultRowInfo *pWindowResInfo, int32_t order) { SQuery* pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { @@ -3635,7 +3635,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { int32_t order = pQuery->order.order; // group by normal columns and interval query on normal table - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); } else { // for simple result of table query, @@ -3826,7 +3826,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { bool toContinue = false; if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pResult = getResultRow(pWindowResInfo, i); @@ -4038,7 +4038,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { // for each group result, call the finalize function for each column - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pRuntimeEnv->groupbyNormalCol) { closeAllTimeWindow(pWindowResInfo); } @@ -4122,7 +4122,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; STableQueryInfo *pTableQueryInfo = pRuntimeEnv->pQuery->current; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // lastKey needs to be updated pTableQueryInfo->lastKey = nextKey; @@ -4290,7 +4290,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { * operations involve. */ STimeWindow w = TSWINDOW_INITIALIZER; - SWindowResInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo *pWindowResInfo = &pTableQueryInfo->windowResInfo; TSKEY sk = MIN(win.skey, win.ekey); TSKEY ek = MAX(win.skey, win.ekey); @@ -4334,7 +4334,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { return loadPrimaryTS; } -static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_t orderType) { +static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_t orderType) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; @@ -4411,7 +4411,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ * @param pQInfo * @param result */ -void copyFromWindowResToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo) { +void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) { SQuery *pQuery = pQInfo->runtimeEnv.pQuery; int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC; @@ -4450,7 +4450,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; + SResultRowInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { @@ -4698,7 +4698,7 @@ void skipBlocks(SQueryRuntimeEnv *pRuntimeEnv) { static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* win, SDataBlockInfo* pBlockInfo, STableQueryInfo* pTableQueryInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; assert(pQuery->limit.offset == 0); STimeWindow tw = *win; @@ -4764,7 +4764,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { STimeWindow w = TSWINDOW_INITIALIZER; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; @@ -5403,7 +5403,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { scanMultiTableDataBlocks(pQInfo); pQInfo->groupIndex += 1; - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // no results generated for current group, continue to try the next group taosArrayDestroy(s); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 3f010a579d..8260d8bc15 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -43,7 +43,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t type) { +int32_t initWindowResInfo(SResultRowInfo *pWindowResInfo, int32_t size, int16_t type) { pWindowResInfo->capacity = size; pWindowResInfo->type = type; @@ -59,7 +59,7 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int16_t return TSDB_CODE_SUCCESS; } -void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { +void cleanupTimeWindowInfo(SResultRowInfo *pWindowResInfo) { if (pWindowResInfo == NULL) { return; } @@ -77,7 +77,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { tfree(pWindowResInfo->pResult); } -void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo) { +void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo) { if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) { return; } @@ -101,7 +101,7 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR } void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) { return; } @@ -160,7 +160,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { - SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; + SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) { return; } @@ -169,7 +169,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { clearFirstNWindowRes(pRuntimeEnv, numOfClosed); } -int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { +int32_t numOfClosedTimeWindow(SResultRowInfo *pWindowResInfo) { int32_t i = 0; while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) { ++i; @@ -178,7 +178,7 @@ int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { return i; } -void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { +void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { @@ -195,7 +195,7 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time. * NOTE: remove redundant, only when the result set order equals to traverse order */ -void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { +void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); if (pWindowResInfo->size <= 1) { return; @@ -224,11 +224,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } } -bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { +bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot) { return (getResultRow(pWindowResInfo, slot)->closed == true); } -void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { +void closeTimeWindow(SResultRowInfo *pWindowResInfo, int32_t slot) { getResultRow(pWindowResInfo, slot)->closed = true; } From 597dd6341cc0c5eba5bf3c6dfbd27fd6f9b9bd4e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Dec 2020 18:46:28 +0800 Subject: [PATCH 139/188] [TD-225] --- src/query/src/qExecutor.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3e683609d1..f47cd34729 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5515,6 +5515,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { // the limitation of output result is reached, set the query completed if (limitResults(pRuntimeEnv)) { + setQueryStatus(pQuery, QUERY_COMPLETED); SET_STABLE_QUERY_OVER(pQInfo); break; } From 8cc7daa929e2e0d079c92ad80aed67abbe048400 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 2 Dec 2020 19:39:31 +0800 Subject: [PATCH 140/188] [TD-2308] doc: add more items to faq document. --- documentation20/webdocs/markdowndocs/faq-ch.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 757b6d9929..42f9761c95 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -101,7 +101,20 @@ Connection = DriverManager.getConnection(url, properties); 2.0.4 ``` -## 14. 怎么报告问题？ +## 14. taos connect failed, reason: invalid timestamp + +常见原因是服务器和客户端时间没有校准，可以通过和时间服务器同步的方式（Linux 下使用 ntpdate 命令，Windows 在系统时间设置中选择自动同步）校准。 + + + +## 15. 表名显示不全 + +由于 taos shell 在终端中显示宽度有限，有可能比较长的表名显示不全，如果按照显示的不全的表名进行相关操作会发生 Table does not exist 错误。解决方法可以是通过修改 taos.cfg 文件中的设置项 maxBinaryDisplayWidth， 或者直接输入命令 set max_binary_display_width 100。或者在命令结尾使用 \G 参数来调整结果的显示方式。 + + + +## 16. 怎么报告问题？ + 如果 FAQ 中的信息不能够帮到您，需要 TDengine 技术团队的技术支持与协助，请将以下两个目录中内容打包: 1. /var/log/taos 2. /etc/taos From 551827eb2bda040c34f97c0d03a9f42dcb1e5be7 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 2 Dec 2020 22:53:52 +0800 Subject: [PATCH 141/188] TD-2270 --- src/balance/inc/bnThread.h | 32 ++++++ src/balance/src/{balance.c => balanceMain.c} | 0 src/balance/src/bnThread.c | 107 +++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 src/balance/inc/bnThread.h rename src/balance/src/{balance.c => balanceMain.c} (100%) create mode 100644 src/balance/src/bnThread.c diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h new file mode 100644 index 0000000000..d337e963f1 --- /dev/null +++ b/src/balance/inc/bnThread.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_BALANCE_H +#define TDENGINE_BALANCE_H + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t bnThreadInit(); +void bnThreadCleanup(); +void bnThreadSyncNotify(); +void bnThreadAsyncNotify(); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/balance/src/balance.c b/src/balance/src/balanceMain.c similarity index 100% rename from src/balance/src/balance.c rename to src/balance/src/balanceMain.c diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c new file mode 100644 index 0000000000..12cb2cf992 --- /dev/null +++ b/src/balance/src/bnThread.c @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tutil.h" +#include "tbalance.h" +#include "tsync.h" +#include "tsync.h" +#include "ttimer.h" +#include "tglobal.h" +#include "tdataformat.h" +#include "dnode.h" +#include "mnode.h" +#include "mnodeDef.h" +#include "mnodeInt.h" +#include "mnodeDnode.h" +#include "mnodeDb.h" +#include "mnodeMnode.h" +#include "mnodeSdb.h" +#include "mnodeShow.h" +#include "mnodeUser.h" +#include "mnodeVgroup.h" + +typedef struct { + bool stop; + pthread_mutex_t mutex; + pthread_cond_t cond; + pthread_t thread; +} SBalanceThread; + +static SBalanceThread tsBnThread; + +static void *bnThreadFunc(void *arg) { + while (1) { + pthread_mutex_lock(&tsBnThread.mutex); + if (tsBnThread.stop) { + pthread_mutex_unlock(&tsBnThread.mutex); + break; + } + + pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex); + + pthread_mutex_unlock(&(tsBnThread.mutex)); + } + + return NULL; +} + +int32_t bnThreadInit() { + tsBnThread.stop = false; + pthread_mutex_init(&tsBnThread.mutex, NULL); + pthread_cond_init(&tsBnThread.cond, NULL); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); + int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL); + pthread_attr_destroy(&thattr); + + if (ret != 0) { + mError("failed to create balance thread since %s", strerror(errno)); + return -1; + } + + mDebug("balance thread is created"); + return 0; +} + +void bnThreadCleanup() { + mDebug("balance thread will be cleanup"); + + pthread_mutex_lock(&tsBnThread.mutex); + tsBnThread.stop = true; + pthread_cond_signal(&tsBnThread.cond); + pthread_mutex_unlock(&(tsBnThread.mutex)); + pthread_join(tsBnThread.thread, NULL); + + pthread_cond_destroy(&tsBnThread.cond); + pthread_mutex_destroy(&tsBnThread.mutex); +} + +void bnThreadSyncNotify() { + mDebug("balance thread sync notify"); + pthread_mutex_lock(&tsBnThread.mutex); + pthread_cond_signal(&tsBnThread.cond); + pthread_mutex_unlock(&(tsBnThread.mutex)); +} + +void bnThreadAsyncNotify() { + mDebug("balance thread async notify"); + pthread_mutex_lock(&tsBnThread.mutex); + pthread_cond_signal(&tsBnThread.cond); + pthread_mutex_unlock(&(tsBnThread.mutex)); +} From d59b8f18da1c940c4df942b6082bb05c280fff5b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 10:32:55 +0800 Subject: [PATCH 142/188] [TD-2298] --- src/query/src/qExecutor.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f47cd34729..1b2d3f12f5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3978,13 +3978,14 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { qstatus.status = pQuery->status; // do nothing if no data blocks are found qualified during scan - qstatus.lastKey = pTableQueryInfo->lastKey; if (qstatus.lastKey != pTableQueryInfo->lastKey) { qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; } else { // the lastkey does not increase, which means no data checked yet qDebug("QInfo:%p no results generated in this scan, abort", pQInfo); return; } + + qstatus.lastKey = pTableQueryInfo->lastKey; } if (!needScanDataBlocksAgain(pRuntimeEnv)) { From caeabf382148fc8453ccb2422c563468a47d3e60 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 10:33:40 +0800 Subject: [PATCH 143/188] [TD-225] refactor codes. --- src/client/inc/tsclient.h | 5 +- src/client/src/tscSQLParser.c | 4 +- src/client/src/tscServer.c | 1 + src/query/src/qExecutor.c | 4 +- src/query/src/qUtil.c | 126 +++++++++++++++++----------------- 5 files changed, 72 insertions(+), 68 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index c482375bff..29841035f7 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -246,11 +246,14 @@ typedef struct SQueryInfo { int16_t fillType; // final result fill type int16_t numOfTables; STableMetaInfo **pTableMetaInfo; - struct STSBuf * tsBuf; + struct STSBuf *tsBuf; int64_t * fillVal; // default value for fill char * msg; // pointer to the pCmd->payload to keep error message temporarily int64_t clauseLimit; // limit for current sub clause + int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int64_t tableLimit; // table limit in case of super table projection query + global order + limit + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX int16_t resColumnId; // result column id } SQueryInfo; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 0c9ad78cd9..abbe26204d 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5307,7 +5307,8 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn // keep original limitation value in globalLimit pQueryInfo->clauseLimit = pQueryInfo->limit.limit; - pQueryInfo->prjOffset = pQueryInfo->limit.offset; + pQueryInfo->prjOffset = pQueryInfo->limit.offset; + pQueryInfo->tableLimit = -1; if (tscOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { /* @@ -5317,6 +5318,7 @@ int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t clauseIn * than or equal to the value of limit. */ if (pQueryInfo->limit.limit > 0) { + pQueryInfo->tableLimit = pQueryInfo->limit.limit + pQueryInfo->limit.offset; pQueryInfo->limit.limit = -1; } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9735e26ece..5fc09c0586 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -697,6 +697,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->queryType = htonl(pQueryInfo->type); + pQueryMsg->tableLimit = htobe64(pQueryInfo->tableLimit); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); pQueryMsg->numOfOutput = htons((int16_t)numOfOutput); // this is the stage one output column number diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1b2d3f12f5..03de6aea1f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6260,11 +6260,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); - // pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; - // pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; - // pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; pQueryMsg->limit = htobe64(pQueryMsg->limit); pQueryMsg->offset = htobe64(pQueryMsg->offset); + pQueryMsg->tableLimit = htobe64(pQueryMsg->tableLimit); pQueryMsg->order = htons(pQueryMsg->order); pQueryMsg->orderColId = htons(pQueryMsg->orderColId); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 8260d8bc15..6c845b012f 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -43,48 +43,48 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SResultRowInfo *pWindowResInfo, int32_t size, int16_t type) { - pWindowResInfo->capacity = size; +int32_t initWindowResInfo(SResultRowInfo *pResultRowInfo, int32_t size, int16_t type) { + pResultRowInfo->capacity = size; - pWindowResInfo->type = type; - pWindowResInfo->curIndex = -1; - pWindowResInfo->size = 0; - pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + pResultRowInfo->type = type; + pResultRowInfo->curIndex = -1; + pResultRowInfo->size = 0; + pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; - pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES); - if (pWindowResInfo->pResult == NULL) { + pResultRowInfo->pResult = calloc(pResultRowInfo->capacity, POINTER_BYTES); + if (pResultRowInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; } -void cleanupTimeWindowInfo(SResultRowInfo *pWindowResInfo) { - if (pWindowResInfo == NULL) { +void cleanupTimeWindowInfo(SResultRowInfo *pResultRowInfo) { + if (pResultRowInfo == NULL) { return; } - if (pWindowResInfo->capacity == 0) { - assert(pWindowResInfo->pResult == NULL); + if (pResultRowInfo->capacity == 0) { + assert(pResultRowInfo->pResult == NULL); return; } - if (pWindowResInfo->type == TSDB_DATA_TYPE_BINARY || pWindowResInfo->type == TSDB_DATA_TYPE_NCHAR) { - for(int32_t i = 0; i < pWindowResInfo->size; ++i) { - tfree(pWindowResInfo->pResult[i]->key); + if (pResultRowInfo->type == TSDB_DATA_TYPE_BINARY || pResultRowInfo->type == TSDB_DATA_TYPE_NCHAR) { + for(int32_t i = 0; i < pResultRowInfo->size; ++i) { + tfree(pResultRowInfo->pResult[i]->key); } } - tfree(pWindowResInfo->pResult); + tfree(pResultRowInfo->pResult); } -void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowResInfo) { - if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0) { +void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo) { + if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0) { return; } - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pWindowRes = pWindowResInfo->pResult[i]; - clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + SResultRow *pWindowRes = pResultRowInfo->pResult[i]; + clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); int32_t groupIndex = 0; int64_t uid = 0; @@ -93,30 +93,30 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pWindowR taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(groupIndex))); } - pWindowResInfo->curIndex = -1; - pWindowResInfo->size = 0; + pResultRowInfo->curIndex = -1; + pResultRowInfo->size = 0; - pWindowResInfo->startTime = TSKEY_INITIAL_VAL; - pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; + pResultRowInfo->startTime = TSKEY_INITIAL_VAL; + pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL; } void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0 || num == 0) { + SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo; + if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0 || num == 0) { return; } - int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); + int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo); assert(num >= 0 && num <= numOfClosed); - int16_t type = pWindowResInfo->type; + int16_t type = pResultRowInfo->type; int64_t uid = getResultInfoUId(pRuntimeEnv); char *key = NULL; int16_t bytes = -1; for (int32_t i = 0; i < num; ++i) { - SResultRow *pResult = pWindowResInfo->pResult[i]; + SResultRow *pResult = pResultRowInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table getResultRowKeyInfo(pResult, type, &key, &bytes); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); @@ -126,23 +126,23 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } } - int32_t remain = pWindowResInfo->size - num; + int32_t remain = pResultRowInfo->size - num; // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type); + copyResultRow(pRuntimeEnv, pResultRowInfo->pResult[k], pResultRowInfo->pResult[num + k], type); } // move the unclosed window in the front of the window list - for (int32_t k = remain; k < pWindowResInfo->size; ++k) { - SResultRow *pWindowRes = pWindowResInfo->pResult[k]; - clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); + for (int32_t k = remain; k < pResultRowInfo->size; ++k) { + SResultRow *pWindowRes = pResultRowInfo->pResult[k]; + clearResultRow(pRuntimeEnv, pWindowRes, pResultRowInfo->type); } - pWindowResInfo->size = remain; + pResultRowInfo->size = remain; - for (int32_t k = 0; k < pWindowResInfo->size; ++k) { - SResultRow *pResult = pWindowResInfo->pResult[k]; + for (int32_t k = 0; k < pResultRowInfo->size; ++k) { + SResultRow *pResult = pResultRowInfo->pResult[k]; getResultRowKeyInfo(pResult, type, &key, &bytes); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); @@ -150,43 +150,43 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { assert(p != NULL); int32_t v = (*p - num); - assert(v >= 0 && v <= pWindowResInfo->size); + assert(v >= 0 && v <= pResultRowInfo->size); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); } - pWindowResInfo->curIndex = -1; + pResultRowInfo->curIndex = -1; } void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { - SResultRowInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; - if (pWindowResInfo == NULL || pWindowResInfo->capacity == 0 || pWindowResInfo->size == 0) { + SResultRowInfo *pResultRowInfo = &pRuntimeEnv->windowResInfo; + if (pResultRowInfo == NULL || pResultRowInfo->capacity == 0 || pResultRowInfo->size == 0) { return; } - int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); + int32_t numOfClosed = numOfClosedTimeWindow(pResultRowInfo); clearFirstNWindowRes(pRuntimeEnv, numOfClosed); } -int32_t numOfClosedTimeWindow(SResultRowInfo *pWindowResInfo) { +int32_t numOfClosedTimeWindow(SResultRowInfo *pResultRowInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) { + while (i < pResultRowInfo->size && pResultRowInfo->pResult[i]->closed) { ++i; } return i; } -void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) { - assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); +void closeAllTimeWindow(SResultRowInfo *pResultRowInfo) { + assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); - for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i]->closed) { + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + if (pResultRowInfo->pResult[i]->closed) { continue; } - pWindowResInfo->pResult[i]->closed = true; + pResultRowInfo->pResult[i]->closed = true; } } @@ -195,41 +195,41 @@ void closeAllTimeWindow(SResultRowInfo *pWindowResInfo) { * the last qualified time stamp in case of sliding query, which the sliding time is not equalled to the interval time. * NOTE: remove redundant, only when the result set order equals to traverse order */ -void removeRedundantWindow(SResultRowInfo *pWindowResInfo, TSKEY lastKey, int32_t order) { - assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); - if (pWindowResInfo->size <= 1) { +void removeRedundantWindow(SResultRowInfo *pResultRowInfo, TSKEY lastKey, int32_t order) { + assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size); + if (pResultRowInfo->size <= 1) { return; } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1; + int32_t resultOrder = (pResultRowInfo->pResult[0]->win.skey < pResultRowInfo->pResult[1]->win.skey)? 1:-1; if (order != resultOrder) { return; } int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey; - while (i < pWindowResInfo->size && (ekey < lastKey)) { + TSKEY ekey = pResultRowInfo->pResult[i]->win.ekey; + while (i < pResultRowInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) { + while (i < pResultRowInfo->size && (pResultRowInfo->pResult[i]->win.skey > lastKey)) { ++i; } } - if (i < pWindowResInfo->size) { - pWindowResInfo->size = (i + 1); + if (i < pResultRowInfo->size) { + pResultRowInfo->size = (i + 1); } } -bool isWindowResClosed(SResultRowInfo *pWindowResInfo, int32_t slot) { - return (getResultRow(pWindowResInfo, slot)->closed == true); +bool isWindowResClosed(SResultRowInfo *pResultRowInfo, int32_t slot) { + return (getResultRow(pResultRowInfo, slot)->closed == true); } -void closeTimeWindow(SResultRowInfo *pWindowResInfo, int32_t slot) { - getResultRow(pWindowResInfo, slot)->closed = true; +void closeTimeWindow(SResultRowInfo *pResultRowInfo, int32_t slot) { + getResultRow(pResultRowInfo, slot)->closed = true; } void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) { From ef901f1077da070c6f555eb5041c9a99a0316c02 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 3 Dec 2020 03:29:10 +0000 Subject: [PATCH 144/188] [TD-2309]: add config option to enable/disable stream (continuous query) --- .../webdocs/markdowndocs/administrator-ch.md | 1 + packaging/cfg/taos.cfg | 5 ++++- src/common/inc/tglobal.h | 3 +++ src/common/src/tglobal.c | 13 +++++++++++ src/cq/src/cqMain.c | 18 +++++++++++++++ src/vnode/src/vnodeMain.c | 22 ++++++++++--------- 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index 36466d2b7e..f54c6b91a1 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -95,6 +95,7 @@ TDengine系统后台服务由taosd提供，可以在配置文件taos.cfg里修 - logKeepDays：日志文件的最长保存时间。大于0时，日志文件会被重命名为taosdlog.xxx，其中xxx为日志文件最后修改的时间戳，单位为秒。默认值：0天。 - maxSQLLength：单条SQL语句允许最长限制。默认值：65380字节。 - telemetryReporting: 是否允许 TDengine 采集和上报基本使用信息，0表示不允许，1表示允许。 默认值：1。 +- stream: 是否启用连续查询（流计算功能），0表示不允许，1表示允许。 默认值：1。 **注意：**对于端口，TDengine会使用从serverPort起13个连续的TCP和UDP端口号，请务必在防火墙打开。因此如果是缺省配置，需要打开从6030都6042共13个端口，而且必须TCP和UDP都打开。 diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index ca88bca3c8..7662d49280 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -260,4 +260,7 @@ # maxBinaryDisplayWidth 30 # enable/disable telemetry reporting -# telemetryReporting 1 \ No newline at end of file +# telemetryReporting 1 + +# enable/disable stream (continuous query) +# stream 1 diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index efe3d7678a..6e4274b358 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -125,6 +125,9 @@ extern char tsMonitorDbName[]; extern char tsInternalPass[]; extern int32_t tsMonitorInterval; +// stream +extern int32_t tsEnableStream; + // internal extern int32_t tsPrintAuth; extern int32_t tscEmbedded; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index d26746a5d1..18aa0ae6e3 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -161,6 +161,9 @@ char tsMonitorDbName[TSDB_DB_NAME_LEN] = "log"; char tsInternalPass[] = "secretkey"; int32_t tsMonitorInterval = 30; // seconds +// stream +int32_t tsEnableStream = 1; + // internal int32_t tsPrintAuth = 0; int32_t tscEmbedded = 0; @@ -1015,6 +1018,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "stream"; + cfg.ptr = &tsEnableStream; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + cfg.option = "httpEnableRecordSql"; cfg.ptr = &tsHttpEnableRecordSql; cfg.valType = TAOS_CFG_VTYPE_INT32; diff --git a/src/cq/src/cqMain.c b/src/cq/src/cqMain.c index b1977fd5d9..efb8795962 100644 --- a/src/cq/src/cqMain.c +++ b/src/cq/src/cqMain.c @@ -69,6 +69,9 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row); static void cqCreateStream(SCqContext *pContext, SCqObj *pObj); void *cqOpen(void *ahandle, const SCqCfg *pCfg) { + if (tsEnableStream == 0) { + return NULL; + } SCqContext *pContext = calloc(sizeof(SCqContext), 1); if (pContext == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); @@ -99,6 +102,9 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) { } void cqClose(void *handle) { + if (tsEnableStream == 0) { + return; + } SCqContext *pContext = handle; if (handle == NULL) return; @@ -129,6 +135,9 @@ void cqClose(void *handle) { } void cqStart(void *handle) { + if (tsEnableStream == 0) { + return; + } SCqContext *pContext = handle; if (pContext->dbConn || pContext->master) return; @@ -147,6 +156,9 @@ void cqStart(void *handle) { } void cqStop(void *handle) { + if (tsEnableStream == 0) { + return; + } SCqContext *pContext = handle; cInfo("vgId:%d, stop all CQs", pContext->vgId); if (pContext->dbConn == NULL || pContext->master == 0) return; @@ -174,6 +186,9 @@ void cqStop(void *handle) { } void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema *pSchema) { + if (tsEnableStream == 0) { + return NULL; + } SCqContext *pContext = handle; SCqObj *pObj = calloc(sizeof(SCqObj), 1); @@ -203,6 +218,9 @@ void *cqCreate(void *handle, uint64_t uid, int32_t tid, char *sqlStr, STSchema * } void cqDrop(void *handle) { + if (tsEnableStream == 0) { + return; + } SCqObj *pObj = handle; SCqContext *pContext = pObj->pContext; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 204dd2f3dd..b516c9d90e 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -267,16 +267,18 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { return terrno; } - SCqCfg cqCfg = {0}; - sprintf(cqCfg.user, "_root"); - strcpy(cqCfg.pass, tsInternalPass); - strcpy(cqCfg.db, pVnode->db); - cqCfg.vgId = vnode; - cqCfg.cqWrite = vnodeWriteToCache; - pVnode->cq = cqOpen(pVnode, &cqCfg); - if (pVnode->cq == NULL) { - vnodeCleanUp(pVnode); - return terrno; + if (tsEnableStream) { + SCqCfg cqCfg = {0}; + sprintf(cqCfg.user, "_root"); + strcpy(cqCfg.pass, tsInternalPass); + strcpy(cqCfg.db, pVnode->db); + cqCfg.vgId = vnode; + cqCfg.cqWrite = vnodeWriteToCache; + pVnode->cq = cqOpen(pVnode, &cqCfg); + if (pVnode->cq == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } } STsdbAppH appH = {0}; From ca87041503c9187f5ada13b658381e500abf9abe Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Dec 2020 13:00:13 +0800 Subject: [PATCH 145/188] TD-2270 --- src/balance/inc/bnMain.h | 56 ++++ src/balance/inc/bnScore.h | 34 +++ src/balance/inc/bnThread.h | 13 +- src/balance/src/balanceMain.c | 555 +++++++--------------------------- src/balance/src/bnScore.c | 328 ++++++++++++++++++++ src/balance/src/bnThread.c | 72 +++-- src/inc/tbalance.h | 16 +- src/mnode/src/mnodeDb.c | 2 +- src/mnode/src/mnodeDnode.c | 24 +- src/mnode/src/mnodeMain.c | 2 +- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeVgroup.c | 2 +- 12 files changed, 611 insertions(+), 495 deletions(-) create mode 100644 src/balance/inc/bnMain.h create mode 100644 src/balance/inc/bnScore.h create mode 100644 src/balance/src/bnScore.c diff --git a/src/balance/inc/bnMain.h b/src/balance/inc/bnMain.h new file mode 100644 index 0000000000..101f5d5fd1 --- /dev/null +++ b/src/balance/inc/bnMain.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_BALANCE_MAIN_H +#define TDENGINE_BALANCE_MAIN_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "mnodeInt.h" +#include "mnodeDnode.h" + +typedef struct { + int32_t size; + int32_t maxSize; + SDnodeObj **list; +} SBnDnodes; + +typedef struct { + void * timer; + bool stop; + pthread_mutex_t mutex; + pthread_cond_t cond; + pthread_t thread; +} SBnThread; + +typedef struct { + pthread_mutex_t mutex; +} SBnMgmt; + +int32_t bnInit(); +void bnCleanUp(); +bool bnStart(); +void bnCheckStatus(); +void bnCheckModules(); + +extern SBnDnodes tsBnDnodes; +extern void *tsMnodeTmr; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/balance/inc/bnScore.h b/src/balance/inc/bnScore.h new file mode 100644 index 0000000000..48e423b562 --- /dev/null +++ b/src/balance/inc/bnScore.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_BALANCE_SCORE_H +#define TDENGINE_BALANCE_SCORE_H + +#ifdef __cplusplus +extern "C" { +#endif +#include "bnMain.h" + +void bnInitDnodes(); +void bnCleanupDnodes(); +void bnAccquireDnodes(); +void bnReleaseDnodes(); +float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h index d337e963f1..df9b06cf87 100644 --- a/src/balance/inc/bnThread.h +++ b/src/balance/inc/bnThread.h @@ -13,17 +13,18 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_BALANCE_H -#define TDENGINE_BALANCE_H +#ifndef TDENGINE_BALANCE_THREAD_H +#define TDENGINE_BALANCE_THREAD_H #ifdef __cplusplus extern "C" { #endif +#include "bnMain.h" -int32_t bnThreadInit(); -void bnThreadCleanup(); -void bnThreadSyncNotify(); -void bnThreadAsyncNotify(); +int32_t bnInitThread(); +void bnCleanupThread(); +void bnNotify(); +void bnStartTimer(int64_t mseconds); #ifdef __cplusplus } diff --git a/src/balance/src/balanceMain.c b/src/balance/src/balanceMain.c index b172867929..9f9d5103be 100644 --- a/src/balance/src/balanceMain.c +++ b/src/balance/src/balanceMain.c @@ -33,36 +33,23 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -/* - * once sdb work as mater, then tsAccessSquence reset to zero - * increase tsAccessSquence every balance interval - */ -extern void * tsMnodeTmr; -static void * tsBalanceTimer = NULL; -static int32_t tsBalanceDnodeListSize = 0; -static SDnodeObj ** tsBalanceDnodeList = NULL; -static int32_t tsBalanceDnodeListMallocSize = 16; -static pthread_mutex_t tsBalanceMutex; +#include "bnMain.h" +#include "bnScore.h" +#include "bnThread.h" -static void balanceStartTimer(int64_t mseconds); -static void balanceInitDnodeList(); -static void balanceCleanupDnodeList(); -static void balanceAccquireDnodeList(); -static void balanceReleaseDnodeList(); -static void balanceMonitorDnodeModule(); -static float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode); -static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static SBnMgmt tsBnMgmt;; -static void balanceLock() { - pthread_mutex_lock(&tsBalanceMutex); +static void bnMonitorDnodeModule(); + +static void bnLock() { + pthread_mutex_lock(&tsBnMgmt.mutex); } -static void balanceUnLock() { - pthread_mutex_unlock(&tsBalanceMutex); +static void bnUnLock() { + pthread_mutex_unlock(&tsBnMgmt.mutex); } -static bool balanceCheckFree(SDnodeObj *pDnode) { +static bool bnCheckFree(SDnodeObj *pDnode) { if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) { mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status)); return false; @@ -86,7 +73,7 @@ static bool balanceCheckFree(SDnodeObj *pDnode) { return true; } -static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { +static void bnDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId); SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId); @@ -111,27 +98,26 @@ static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) { mnodeUpdateVgroup(pVgroup); } -static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { +static void bnSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) { // SVnodeGid tmp = *pVnodeGid1; // *pVnodeGid1 = *pVnodeGid2; // *pVnodeGid2 = tmp; } -int32_t balanceAllocVnodes(SVgObj *pVgroup) { +int32_t bnAllocVnodes(SVgObj *pVgroup) { static int32_t randIndex = 0; int32_t dnode = 0; int32_t vnodes = 0; - balanceLock(); - - balanceAccquireDnodeList(); + bnLock(); + bnAccquireDnodes(); mDebug("db:%s, try alloc %d vnodes to vgroup, dnodes total:%d, avail:%d", pVgroup->dbName, pVgroup->numOfVnodes, - mnodeGetDnodesNum(), tsBalanceDnodeListSize); + mnodeGetDnodesNum(), tsBnDnodes.size); for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { - for (; dnode < tsBalanceDnodeListSize; ++dnode) { - SDnodeObj *pDnode = tsBalanceDnodeList[dnode]; - if (balanceCheckFree(pDnode)) { + for (; dnode < tsBnDnodes.size; ++dnode) { + SDnodeObj *pDnode = tsBnDnodes.list[dnode]; + if (bnCheckFree(pDnode)) { SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i; pVnodeGid->dnodeId = pDnode->dnodeId; pVnodeGid->pDnode = pDnode; @@ -148,8 +134,8 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { } if (vnodes != pVgroup->numOfVnodes) { - balanceReleaseDnodeList(); - balanceUnLock(); + bnReleaseDnodes(); + bnUnLock(); mDebug("db:%s, need vnodes:%d, but alloc:%d", pVgroup->dbName, pVgroup->numOfVnodes, vnodes); @@ -179,33 +165,33 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { if (pVgroup->numOfVnodes == 1) { } else if (pVgroup->numOfVnodes == 2) { if (randIndex++ % 2 == 0) { - balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1); + bnSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1); } } else { int32_t randVal = randIndex++ % 6; if (randVal == 1) { // 1, 0, 2 - balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); + bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); } else if (randVal == 2) { // 1, 2, 0 - balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); - balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); + bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1); + bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); } else if (randVal == 3) { // 2, 1, 0 - balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2); + bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2); } else if (randVal == 4) { // 2, 0, 1 - balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2); - balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); + bnSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2); + bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); } if (randVal == 5) { // 0, 2, 1 - balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); + bnSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2); } else { } // 0, 1, 2 } - balanceReleaseDnodeList(); - balanceUnLock(); + bnReleaseDnodes(); + bnUnLock(); return TSDB_CODE_SUCCESS; } -static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { +static bool bnCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) { return false; } @@ -232,7 +218,7 @@ static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) { * desc: remove one vnode from vgroup * all vnodes in vgroup should in ready state, except the balancing one **/ -static int32_t balanceRemoveVnode(SVgObj *pVgroup) { +static int32_t bnRemoveVnode(SVgObj *pVgroup) { if (pVgroup->numOfVnodes <= 1) return -1; SVnodeGid *pRmVnode = NULL; @@ -274,17 +260,17 @@ static int32_t balanceRemoveVnode(SVgObj *pVgroup) { pSelVnode = pRmVnode; } - if (!balanceCheckVgroupReady(pVgroup, pSelVnode)) { + if (!bnCheckVgroupReady(pVgroup, pSelVnode)) { mDebug("vgId:%d, is not ready", pVgroup->vgId); return -1; } else { mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId); - balanceDiscardVnode(pVgroup, pSelVnode); + bnDiscardVnode(pVgroup, pSelVnode); return TSDB_CODE_SUCCESS; } } -static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) { +static bool bnCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) { for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) { SVnodeGid *pGid = &pVgroup->vnodeGid[i]; if (pGid->dnodeId == 0) break; @@ -299,13 +285,13 @@ static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) { /** * desc: add vnode to vgroup, find a new one if dest dnode is null **/ -static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { +static int32_t bnAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) { if (pDestDnode == NULL) { - for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) { - SDnodeObj *pDnode = tsBalanceDnodeList[i]; + for (int32_t i = 0; i < tsBnDnodes.size; ++i) { + SDnodeObj *pDnode = tsBnDnodes.list[i]; if (pDnode == pSrcDnode) continue; - if (balanceCheckDnodeInVgroup(pDnode, pVgroup)) continue; - if (!balanceCheckFree(pDnode)) continue; + if (bnCheckDnodeInVgroup(pDnode, pVgroup)) continue; + if (!bnCheckFree(pDnode)) continue; pDestDnode = pDnode; mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId); @@ -333,25 +319,25 @@ static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj return TSDB_CODE_SUCCESS; } -static bool balanceMonitorBalance() { - if (tsBalanceDnodeListSize < 2) return false; +static bool bnMonitorBalance() { + if (tsBnDnodes.size < 2) return false; - for (int32_t src = tsBalanceDnodeListSize - 1; src >= 0; --src) { - SDnodeObj *pDnode = tsBalanceDnodeList[src]; - mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBalanceDnodeListSize - src - 1, + for (int32_t src = tsBnDnodes.size - 1; src >= 0; --src) { + SDnodeObj *pDnode = tsBnDnodes.list[src]; + mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBnDnodes.size - src - 1, pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores, pDnode->openVnodes); } - float scoresDiff = tsBalanceDnodeList[tsBalanceDnodeListSize - 1]->score - tsBalanceDnodeList[0]->score; + float scoresDiff = tsBnDnodes.list[tsBnDnodes.size - 1]->score - tsBnDnodes.list[0]->score; if (scoresDiff < 0.01) { - mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBalanceDnodeListSize, scoresDiff); + mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBnDnodes.size, scoresDiff); return false; } - for (int32_t src = tsBalanceDnodeListSize - 1; src > 0; --src) { - SDnodeObj *pSrcDnode = tsBalanceDnodeList[src]; - float srcScore = balanceTryCalcDnodeScore(pSrcDnode, -1); + for (int32_t src = tsBnDnodes.size - 1; src > 0; --src) { + SDnodeObj *pSrcDnode = tsBnDnodes.list[src]; + float srcScore = bnTryCalcDnodeScore(pSrcDnode, -1); if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) { continue; } @@ -362,19 +348,19 @@ static bool balanceMonitorBalance() { pIter = mnodeGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; - if (balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) { + if (bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) { for (int32_t dest = 0; dest < src; dest++) { - SDnodeObj *pDestDnode = tsBalanceDnodeList[dest]; - if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) continue; + SDnodeObj *pDestDnode = tsBnDnodes.list[dest]; + if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) continue; - float destScore = balanceTryCalcDnodeScore(pDestDnode, 1); + float destScore = bnTryCalcDnodeScore(pDestDnode, 1); if (srcScore + 0.0001 < destScore) continue; - if (!balanceCheckFree(pDestDnode)) continue; + if (!bnCheckFree(pDestDnode)) continue; mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f", pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score, srcScore, pDestDnode->score, destScore); - balanceAddVnode(pVgroup, pSrcDnode, pDestDnode); + bnAddVnode(pVgroup, pSrcDnode, pDestDnode); mnodeDecVgroupRef(pVgroup); mnodeCancelGetNextVgroup(pIter); return true; @@ -392,7 +378,7 @@ static bool balanceMonitorBalance() { // 1. reset balanceAccessSquence to zero // 2. reset state of dnodes to offline // 3. reset lastAccess of dnodes to zero -void balanceReset() { +void bnReset() { void * pIter = NULL; SDnodeObj *pDnode = NULL; while (1) { @@ -413,7 +399,7 @@ void balanceReset() { tsAccessSquence = 0; } -static int32_t balanceMonitorVgroups() { +static int32_t bnMonitorVgroups() { void * pIter = NULL; SVgObj *pVgroup = NULL; bool hasUpdatingVgroup = false; @@ -429,11 +415,11 @@ static int32_t balanceMonitorVgroups() { if (vgReplica > dbReplica) { mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica); hasUpdatingVgroup = true; - code = balanceRemoveVnode(pVgroup); + code = bnRemoveVnode(pVgroup); } else if (vgReplica < dbReplica) { mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica); hasUpdatingVgroup = true; - code = balanceAddVnode(pVgroup, NULL, NULL); + code = bnAddVnode(pVgroup, NULL, NULL); } mnodeDecVgroupRef(pVgroup); @@ -446,7 +432,7 @@ static int32_t balanceMonitorVgroups() { return hasUpdatingVgroup; } -static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) { +static bool bnMonitorDnodeDropping(SDnodeObj *pDnode) { mDebug("dnode:%d, in dropping state", pDnode->dnodeId); void * pIter = NULL; @@ -456,7 +442,7 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) { pIter = mnodeGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; - hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup); + hasThisDnode = bnCheckDnodeInVgroup(pDnode, pVgroup); mnodeDecVgroupRef(pVgroup); if (hasThisDnode) { @@ -474,7 +460,7 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) { return false; } -static bool balanceMontiorDropping() { +static bool bnMontiorDropping() { void *pIter = NULL; SDnodeObj *pDnode = NULL; @@ -499,7 +485,7 @@ static bool balanceMontiorDropping() { } if (pDnode->status == TAOS_DN_STATUS_DROPPING) { - bool ret = balanceMonitorDnodeDropping(pDnode); + bool ret = bnMonitorDnodeDropping(pDnode); mnodeDecDnodeRef(pDnode); mnodeCancelGetNextDnode(pIter); return ret; @@ -509,33 +495,32 @@ static bool balanceMontiorDropping() { return false; } -static bool balanceStart() { +bool bnStart() { if (!sdbIsMaster()) return false; - balanceLock(); + bnLock(); + bnAccquireDnodes(); - balanceAccquireDnodeList(); + bnMonitorDnodeModule(); - balanceMonitorDnodeModule(); - - bool updateSoon = balanceMontiorDropping(); + bool updateSoon = bnMontiorDropping(); if (!updateSoon) { - updateSoon = balanceMonitorVgroups(); + updateSoon = bnMonitorVgroups(); } if (!updateSoon) { - updateSoon = balanceMonitorBalance(); + updateSoon = bnMonitorBalance(); } - balanceReleaseDnodeList(); + bnReleaseDnodes(); - balanceUnLock(); + bnUnLock(); return updateSoon; } -static void balanceSetVgroupOffline(SDnodeObj* pDnode) { +static void bnSetVgroupOffline(SDnodeObj* pDnode) { void *pIter = NULL; while (1) { SVgObj *pVgroup; @@ -551,7 +536,7 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) { } } -static void balanceCheckDnodeAccess() { +void bnCheckStatus() { void * pIter = NULL; SDnodeObj *pDnode = NULL; @@ -564,84 +549,39 @@ static void balanceCheckDnodeAccess() { pDnode->offlineReason = TAOS_DN_OFF_STATUS_MSG_TIMEOUT; mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence, pDnode->lastAccess, pDnode->status); - balanceSetVgroupOffline(pDnode); + bnSetVgroupOffline(pDnode); } } mnodeDecDnodeRef(pDnode); } } -static void balanceProcessBalanceTimer(void *handle, void *tmrId) { - if (!sdbIsMaster()) return; - - tsBalanceTimer = NULL; - tsAccessSquence ++; - - balanceCheckDnodeAccess(); - bool updateSoon = false; - - if (handle == NULL) { - if (tsAccessSquence % tsBalanceInterval == 0) { - mDebug("balance function is scheduled by timer"); - updateSoon = balanceStart(); - } - } else { - int64_t mseconds = (int64_t)handle; - mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds); - updateSoon = balanceStart(); - } - - if (updateSoon) { - balanceStartTimer(1000); - } else { - taosTmrReset(balanceProcessBalanceTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBalanceTimer); - } -} - -static void balanceStartTimer(int64_t mseconds) { - taosTmrReset(balanceProcessBalanceTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBalanceTimer); -} - -void balanceSyncNotify() { +void bnCheckModules() { if (sdbIsMaster()) { - balanceLock(); - balanceAccquireDnodeList(); - balanceMonitorDnodeModule(); - balanceReleaseDnodeList(); - balanceUnLock(); + bnLock(); + bnAccquireDnodes(); + bnMonitorDnodeModule(); + bnReleaseDnodes(); + bnUnLock(); } } -void balanceAsyncNotify() { - balanceStartTimer(500); -} - -int32_t balanceInit() { - mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta); - mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores); - mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode); - - pthread_mutex_init(&tsBalanceMutex, NULL); - balanceInitDnodeList(); - balanceStartTimer(2000); - mDebug("balance start fp:%p initialized", balanceProcessBalanceTimer); - - balanceReset(); +int32_t bnInit() { + pthread_mutex_init(&tsBnMgmt.mutex, NULL); + bnInitDnodes(); + bnInitThread(); + bnReset(); return 0; } -void balanceCleanUp() { - if (tsBalanceTimer != NULL) { - taosTmrStopA(&tsBalanceTimer); - pthread_mutex_destroy(&tsBalanceMutex); - tsBalanceTimer = NULL; - mDebug("stop balance timer"); - } - balanceCleanupDnodeList(); +void bnCleanUp() { + bnCleanupThread(); + pthread_mutex_destroy(&tsBnMgmt.mutex); + bnCleanupDnodes(); } -int32_t balanceDropDnode(SDnodeObj *pDnode) { +int32_t bnDropDnode(SDnodeObj *pDnode) { int32_t totalFreeVnodes = 0; void * pIter = NULL; SDnodeObj *pTempDnode = NULL; @@ -650,7 +590,7 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) { pIter = mnodeGetNextDnode(pIter, &pTempDnode); if (pTempDnode == NULL) break; - if (pTempDnode != pDnode && balanceCheckFree(pTempDnode)) { + if (pTempDnode != pDnode && bnCheckFree(pTempDnode)) { totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes); } @@ -665,298 +605,19 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) { pDnode->status = TAOS_DN_STATUS_DROPPING; mnodeUpdateDnode(pDnode); - balanceStartTimer(1100); + bnStartTimer(1100); return TSDB_CODE_SUCCESS; } -static int32_t balanceCalcCpuScore(SDnodeObj *pDnode) { - if (pDnode->cpuAvgUsage < 80) - return 0; - else if (pDnode->cpuAvgUsage < 90) - return 10; - else - return 50; -} -static int32_t balanceCalcMemoryScore(SDnodeObj *pDnode) { - if (pDnode->memoryAvgUsage < 80) - return 0; - else if (pDnode->memoryAvgUsage < 90) - return 10; - else - return 50; -} -static int32_t balanceCalcDiskScore(SDnodeObj *pDnode) { - if (pDnode->diskAvgUsage < 80) - return 0; - else if (pDnode->diskAvgUsage < 90) - return 10; - else - return 50; -} - -static int32_t balanceCalcBandwidthScore(SDnodeObj *pDnode) { - if (pDnode->bandwidthUsage < 30) - return 0; - else if (pDnode->bandwidthUsage < 80) - return 10; - else - return 50; -} - -static float balanceCalcModuleScore(SDnodeObj *pDnode) { - if (pDnode->numOfCores <= 0) return 0; - if (pDnode->isMgmt) { - return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores; - } - return 0; -} - -static float balanceCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) { - if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000; - if (pDnode->numOfCores <= 0) return 0; - return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores; -} - -/** - * calc singe score, such as cpu/memory/disk/bandwitdh/vnode - * 1. get the score config - * 2. if the value is out of range, use border data - * 3. otherwise use interpolation method - **/ -void balanceCalcDnodeScore(SDnodeObj *pDnode) { - pDnode->score = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) + - balanceCalcBandwidthScore(pDnode) + balanceCalcModuleScore(pDnode) + - balanceCalcVnodeScore(pDnode, 0) + pDnode->customScore; -} - -float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) { - int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) + - balanceCalcBandwidthScore(pDnode); - float moduleScore = balanceCalcModuleScore(pDnode); - float vnodeScore = balanceCalcVnodeScore(pDnode, extra); - - float score = systemScore + moduleScore + vnodeScore + pDnode->customScore; - return score; -} - -static void balanceInitDnodeList() { - tsBalanceDnodeList = calloc(tsBalanceDnodeListMallocSize, sizeof(SDnodeObj *)); -} - -static void balanceCleanupDnodeList() { - if (tsBalanceDnodeList != NULL) { - free(tsBalanceDnodeList); - tsBalanceDnodeList = NULL; - } -} - -static void balanceCheckDnodeListSize(int32_t dnodesNum) { - if (tsBalanceDnodeListMallocSize <= dnodesNum) { - tsBalanceDnodeListMallocSize = dnodesNum * 2; - tsBalanceDnodeList = realloc(tsBalanceDnodeList, tsBalanceDnodeListMallocSize * sizeof(SDnodeObj *)); - } -} - -void balanceAccquireDnodeList() { - int32_t dnodesNum = mnodeGetDnodesNum(); - balanceCheckDnodeListSize(dnodesNum); - - void * pIter = NULL; - SDnodeObj *pDnode = NULL; - int32_t dnodeIndex = 0; - - while (1) { - if (dnodeIndex >= dnodesNum) { - mnodeCancelGetNextDnode(pIter); - break; - } - - pIter = mnodeGetNextDnode(pIter, &pDnode); - if (pDnode == NULL) break; - if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { - mnodeDecDnodeRef(pDnode); - continue; - } - - balanceCalcDnodeScore(pDnode); - - int32_t orderIndex = dnodeIndex; - for (; orderIndex > 0; --orderIndex) { - if (pDnode->score > tsBalanceDnodeList[orderIndex - 1]->score) { - break; - } - tsBalanceDnodeList[orderIndex] = tsBalanceDnodeList[orderIndex - 1]; - } - tsBalanceDnodeList[orderIndex] = pDnode; - dnodeIndex++; - } - - tsBalanceDnodeListSize = dnodeIndex; -} - -void balanceReleaseDnodeList() { - for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) { - SDnodeObj *pDnode = tsBalanceDnodeList[i]; - if (pDnode != NULL) { - mnodeDecDnodeRef(pDnode); - } - } -} - -static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { - SUserObj *pUser = mnodeGetUserFromConn(pConn); - if (pUser == NULL) return 0; - - if (strcmp(pUser->pAcct->user, "root") != 0) { - mnodeDecUserRef(pUser); - return TSDB_CODE_MND_NO_RIGHTS; - } - - int32_t cols = 0; - SSchema *pSchema = pMeta->schema; - - pShow->bytes[cols] = 2; - pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; - strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "system scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "custom scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "module scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "vnode scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; - strcpy(pSchema[cols].name, "total scores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "open vnodes"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "cpu cores"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE; - pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "balance state"); - pSchema[cols].bytes = htons(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htons(cols); - pShow->numOfColumns = cols; - - pShow->offset[0] = 0; - for (int32_t i = 1; i < cols; ++i) { - pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; - } - - pShow->numOfRows = mnodeGetDnodesNum(); - pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; - pShow->pIter = NULL; - - mnodeDecUserRef(pUser); - - return 0; -} - -static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) { - int32_t numOfRows = 0; - SDnodeObj *pDnode = NULL; - char * pWrite; - int32_t cols = 0; - - while (numOfRows < rows) { - pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode); - if (pDnode == NULL) break; - - int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) + - balanceCalcBandwidthScore(pDnode); - float moduleScore = balanceCalcModuleScore(pDnode); - float vnodeScore = balanceCalcVnodeScore(pDnode, 0); - - cols = 0; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int16_t *)pWrite = pDnode->dnodeId; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = systemScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = pDnode->customScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)moduleScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)vnodeScore; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore); - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDnode->openVnodes; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - *(int32_t *)pWrite = pDnode->numOfCores; - cols++; - - pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status)); - cols++; - - numOfRows++; - mnodeDecDnodeRef(pDnode); - } - - mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); - pShow->numOfReads += numOfRows; - return numOfRows; -} - -static void balanceMonitorDnodeModule() { +static void bnMonitorDnodeModule() { int32_t numOfMnodes = mnodeGetMnodesNum(); if (numOfMnodes >= tsNumOfMnodes) return; - for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) { - SDnodeObj *pDnode = tsBalanceDnodeList[i]; + for (int32_t i = 0; i < tsBnDnodes.size; ++i) { + SDnodeObj *pDnode = tsBnDnodes.list[i]; if (pDnode == NULL) break; if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) { @@ -980,7 +641,7 @@ static void balanceMonitorDnodeModule() { } } -int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) { +int32_t bnAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) { if (!sdbIsMaster()) { mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId); return TSDB_CODE_MND_DNODE_NOT_EXIST; @@ -1004,29 +665,29 @@ int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t return TSDB_CODE_MND_DNODE_NOT_EXIST; } - balanceLock(); - balanceAccquireDnodeList(); + bnLock(); + bnAccquireDnodes(); int32_t code = TSDB_CODE_SUCCESS; - if (!balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) { + if (!bnCheckDnodeInVgroup(pSrcDnode, pVgroup)) { mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId, dnodeId, pSrcDnode->dnodeId); code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE; - } else if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) { + } else if (bnCheckDnodeInVgroup(pDestDnode, pVgroup)) { mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId, dnodeId, dnodeId); code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE; - } else if (!balanceCheckFree(pDestDnode)) { + } else if (!bnCheckFree(pDestDnode)) { mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId, dnodeId); code = TSDB_CODE_MND_DNODE_NOT_FREE; } else { - code = balanceAddVnode(pVgroup, pSrcDnode, pDestDnode); + code = bnAddVnode(pVgroup, pSrcDnode, pDestDnode); mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code)); } - balanceReleaseDnodeList(); - balanceUnLock(); + bnReleaseDnodes(); + bnUnLock(); mnodeDecVgroupRef(pVgroup); mnodeDecDnodeRef(pDestDnode); diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c new file mode 100644 index 0000000000..db5d515fbf --- /dev/null +++ b/src/balance/src/bnScore.c @@ -0,0 +1,328 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "tutil.h" +#include "tbalance.h" +#include "tsync.h" +#include "tsync.h" +#include "ttimer.h" +#include "tglobal.h" +#include "tdataformat.h" +#include "dnode.h" +#include "mnode.h" +#include "mnodeDef.h" +#include "mnodeInt.h" +#include "mnodeDnode.h" +#include "mnodeDb.h" +#include "mnodeMnode.h" +#include "mnodeSdb.h" +#include "mnodeShow.h" +#include "mnodeUser.h" +#include "mnodeVgroup.h" + +#include "bnScore.h" + +SBnDnodes tsBnDnodes; + +static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn); + +static int32_t bnCalcCpuScore(SDnodeObj *pDnode) { + if (pDnode->cpuAvgUsage < 80) + return 0; + else if (pDnode->cpuAvgUsage < 90) + return 10; + else + return 50; +} + +static int32_t bnCalcMemoryScore(SDnodeObj *pDnode) { + if (pDnode->memoryAvgUsage < 80) + return 0; + else if (pDnode->memoryAvgUsage < 90) + return 10; + else + return 50; +} + +static int32_t bnCalcDiskScore(SDnodeObj *pDnode) { + if (pDnode->diskAvgUsage < 80) + return 0; + else if (pDnode->diskAvgUsage < 90) + return 10; + else + return 50; +} + +static int32_t bnCalcBandScore(SDnodeObj *pDnode) { + if (pDnode->bandwidthUsage < 30) + return 0; + else if (pDnode->bandwidthUsage < 80) + return 10; + else + return 50; +} + +static float bnCalcModuleScore(SDnodeObj *pDnode) { + if (pDnode->numOfCores <= 0) return 0; + if (pDnode->isMgmt) { + return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores; + } + return 0; +} + +static float bnCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) { + if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000; + if (pDnode->numOfCores <= 0) return 0; + return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores; +} + +/** + * calc singe score, such as cpu/memory/disk/bandwitdh/vnode + * 1. get the score config + * 2. if the value is out of range, use border data + * 3. otherwise use interpolation method + **/ +static void bnCalcDnodeScore(SDnodeObj *pDnode) { + pDnode->score = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + + bnCalcBandScore(pDnode) + bnCalcModuleScore(pDnode) + bnCalcVnodeScore(pDnode, 0) + + pDnode->customScore; +} + +float bnTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) { + int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + + bnCalcBandScore(pDnode); + float moduleScore = bnCalcModuleScore(pDnode); + float vnodeScore = bnCalcVnodeScore(pDnode, extra); + + float score = systemScore + moduleScore + vnodeScore + pDnode->customScore; + return score; +} + +void bnInitDnodes() { + mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, bnGetScoresMeta); + mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, bnRetrieveScores); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode); + + memset(&tsBnDnodes, 0, sizeof(SBnDnodes)); + tsBnDnodes.maxSize = 16; + tsBnDnodes.list = calloc(tsBnDnodes.maxSize, sizeof(SDnodeObj *)); +} + +void bnCleanupDnodes() { + if (tsBnDnodes.list != NULL) { + free(tsBnDnodes.list); + tsBnDnodes.list = NULL; + } +} + +static void bnCheckDnodesSize(int32_t dnodesNum) { + if (tsBnDnodes.maxSize <= dnodesNum) { + tsBnDnodes.maxSize = dnodesNum * 2; + tsBnDnodes.list = realloc(tsBnDnodes.list, tsBnDnodes.maxSize * sizeof(SDnodeObj *)); + } +} + +void bnAccquireDnodes() { + int32_t dnodesNum = mnodeGetDnodesNum(); + bnCheckDnodesSize(dnodesNum); + + void * pIter = NULL; + SDnodeObj *pDnode = NULL; + int32_t dnodeIndex = 0; + + while (1) { + if (dnodeIndex >= dnodesNum) { + mnodeCancelGetNextDnode(pIter); + break; + } + + pIter = mnodeGetNextDnode(pIter, &pDnode); + if (pDnode == NULL) break; + if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { + mnodeDecDnodeRef(pDnode); + continue; + } + + bnCalcDnodeScore(pDnode); + + int32_t orderIndex = dnodeIndex; + for (; orderIndex > 0; --orderIndex) { + if (pDnode->score > tsBnDnodes.list[orderIndex - 1]->score) { + break; + } + tsBnDnodes.list[orderIndex] = tsBnDnodes.list[orderIndex - 1]; + } + tsBnDnodes.list[orderIndex] = pDnode; + dnodeIndex++; + } + + tsBnDnodes.size = dnodeIndex; +} + +void bnReleaseDnodes() { + for (int32_t i = 0; i < tsBnDnodes.size; ++i) { + SDnodeObj *pDnode = tsBnDnodes.list[i]; + if (pDnode != NULL) { + mnodeDecDnodeRef(pDnode); + } + } +} + +static int32_t bnGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { + SUserObj *pUser = mnodeGetUserFromConn(pConn); + if (pUser == NULL) return 0; + + if (strcmp(pUser->pAcct->user, "root") != 0) { + mnodeDecUserRef(pUser); + return TSDB_CODE_MND_NO_RIGHTS; + } + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 2; + pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; + strcpy(pSchema[cols].name, "id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; + strcpy(pSchema[cols].name, "system scores"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; + strcpy(pSchema[cols].name, "custom scores"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; + strcpy(pSchema[cols].name, "module scores"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; + strcpy(pSchema[cols].name, "vnode scores"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_FLOAT; + strcpy(pSchema[cols].name, "total scores"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "open vnodes"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "cpu cores"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "balance state"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = mnodeGetDnodesNum(); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + pShow->pIter = NULL; + + mnodeDecUserRef(pUser); + + return 0; +} + +static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) { + int32_t numOfRows = 0; + SDnodeObj *pDnode = NULL; + char * pWrite; + int32_t cols = 0; + + while (numOfRows < rows) { + pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode); + if (pDnode == NULL) break; + + int32_t systemScore = bnCalcCpuScore(pDnode) + bnCalcMemoryScore(pDnode) + bnCalcDiskScore(pDnode) + bnCalcBandScore(pDnode); + float moduleScore = bnCalcModuleScore(pDnode); + float vnodeScore = bnCalcVnodeScore(pDnode, 0); + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int16_t *)pWrite = pDnode->dnodeId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(float *)pWrite = systemScore; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(float *)pWrite = pDnode->customScore; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(float *)pWrite = (int32_t)moduleScore; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(float *)pWrite = (int32_t)vnodeScore; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDnode->openVnodes; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pDnode->numOfCores; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status)); + cols++; + + numOfRows++; + mnodeDecDnodeRef(pDnode); + } + + mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); + pShow->numOfReads += numOfRows; + return numOfRows; +} diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index 12cb2cf992..39eb6d0e31 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -34,14 +34,10 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -typedef struct { - bool stop; - pthread_mutex_t mutex; - pthread_cond_t cond; - pthread_t thread; -} SBalanceThread; -static SBalanceThread tsBnThread; +#include "bnThread.h" + +static SBnThread tsBnThread; static void *bnThreadFunc(void *arg) { while (1) { @@ -52,14 +48,16 @@ static void *bnThreadFunc(void *arg) { } pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex); - + bool updateSoon = bnStart(); + bnStartTimer(updateSoon ? 1000 : -1); pthread_mutex_unlock(&(tsBnThread.mutex)); } return NULL; } -int32_t bnThreadInit() { +int32_t bnInitThread() { + memset(&tsBnThread, 0, sizeof(SBnThread)); tsBnThread.stop = false; pthread_mutex_init(&tsBnThread.mutex, NULL); pthread_cond_init(&tsBnThread.cond, NULL); @@ -75,13 +73,20 @@ int32_t bnThreadInit() { return -1; } + bnStartTimer(2000); mDebug("balance thread is created"); return 0; } -void bnThreadCleanup() { +void bnCleanupThread() { mDebug("balance thread will be cleanup"); + if (tsBnThread.timer != NULL) { + taosTmrStopA(&tsBnThread.timer); + tsBnThread.timer = NULL; + mDebug("stop balance timer"); + } + pthread_mutex_lock(&tsBnThread.mutex); tsBnThread.stop = true; pthread_cond_signal(&tsBnThread.cond); @@ -92,16 +97,47 @@ void bnThreadCleanup() { pthread_mutex_destroy(&tsBnThread.mutex); } -void bnThreadSyncNotify() { - mDebug("balance thread sync notify"); - pthread_mutex_lock(&tsBnThread.mutex); - pthread_cond_signal(&tsBnThread.cond); - pthread_mutex_unlock(&(tsBnThread.mutex)); -} - -void bnThreadAsyncNotify() { +static void bnPostSignal() { mDebug("balance thread async notify"); pthread_mutex_lock(&tsBnThread.mutex); pthread_cond_signal(&tsBnThread.cond); pthread_mutex_unlock(&(tsBnThread.mutex)); } + +/* + * once sdb work as mater, then tsAccessSquence reset to zero + * increase tsAccessSquence every balance interval + */ + +static void bnProcessTimer(void *handle, void *tmrId) { + if (!sdbIsMaster()) return; + + tsBnThread.timer = NULL; + tsAccessSquence++; + + bnCheckStatus(); + + if (handle == NULL) { + if (tsAccessSquence % tsBalanceInterval == 0) { + mDebug("balance function is scheduled by timer"); + bnPostSignal(); + } + } else { + int64_t mseconds = (int64_t)handle; + mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds); + bnPostSignal(); + } +} + +void bnStartTimer(int64_t mseconds) { + bool updateSoon = (mseconds != -1); + if (updateSoon) { + taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); + } else { + taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer); + } +} + +void bnNotify() { + bnStartTimer(500); +} diff --git a/src/inc/tbalance.h b/src/inc/tbalance.h index f0da4a3747..b9f4e3c608 100644 --- a/src/inc/tbalance.h +++ b/src/inc/tbalance.h @@ -23,14 +23,14 @@ extern "C" { struct SVgObj; struct SDnodeObj; -int32_t balanceInit(); -void balanceCleanUp(); -void balanceAsyncNotify(); -void balanceSyncNotify(); -void balanceReset(); -int32_t balanceAllocVnodes(struct SVgObj *pVgroup); -int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId); -int32_t balanceDropDnode(struct SDnodeObj *pDnode); +int32_t bnInit(); +void bnCleanUp(); +void bnNotify(); +void bnCheckModules(); +void bnReset(); +int32_t bnAllocVnodes(struct SVgObj *pVgroup); +int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId); +int32_t bnDropDnode(struct SDnodeObj *pDnode); #ifdef __cplusplus } diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 69e8f076e9..63bd066c36 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -1004,7 +1004,7 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { mDebug("db:%s, all vgroups is altered", pDb->name); mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); - balanceAsyncNotify(); + bnNotify(); return TSDB_CODE_SUCCESS; } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 65f4060392..0d429f5797 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -115,7 +115,7 @@ static int32_t mnodeDnodeActionDelete(SSdbRow *pRow) { mnodeDropAllDnodeVgroups(pDnode); #endif mnodeDropMnodeLocal(pDnode->dnodeId); - balanceAsyncNotify(); + bnNotify(); mnodeUpdateDnodeEps(); mDebug("dnode:%d, all vgroups is dropped from sdb", pDnode->dnodeId); @@ -347,7 +347,7 @@ static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_INVALID_DNODE_CFG_OPTION; } - int32_t code = balanceAlterDnode(pDnode, vnodeId, dnodeId); + int32_t code = bnAlterDnode(pDnode, vnodeId, dnodeId); mnodeDecDnodeRef(pDnode); return code; } else { @@ -591,8 +591,8 @@ static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg) { mInfo("dnode:%d, from offline to online", pDnode->dnodeId); pDnode->status = TAOS_DN_STATUS_READY; pDnode->offlineReason = TAOS_DN_OFF_ONLINE; - balanceSyncNotify(); - balanceAsyncNotify(); + bnCheckModules(); + bnNotify(); } if (openVnodes != pDnode->openVnodes) { @@ -708,7 +708,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) { #ifndef _SYNC int32_t code = mnodeDropDnode(pDnode, pMsg); #else - int32_t code = balanceDropDnode(pDnode); + int32_t code = bnDropDnode(pDnode); #endif mnodeDecDnodeRef(pDnode); return code; @@ -1182,12 +1182,12 @@ static char* mnodeGetDnodeAlternativeRoleStr(int32_t alternativeRole) { #ifndef _SYNC -int32_t balanceInit() { return TSDB_CODE_SUCCESS; } -void balanceCleanUp() {} -void balanceAsyncNotify() {} -void balanceSyncNotify() {} -void balanceReset() {} -int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; } +int32_t bnInit() { return TSDB_CODE_SUCCESS; } +void bnCleanUp() {} +void bnNotify() {} +void bnCheckModules() {} +void bnReset() {} +int32_t bnAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; } char* syncRole[] = { "offline", @@ -1197,7 +1197,7 @@ char* syncRole[] = { "master" }; -int32_t balanceAllocVnodes(SVgObj *pVgroup) { +int32_t bnAllocVnodes(SVgObj *pVgroup) { void * pIter = NULL; SDnodeObj *pDnode = NULL; SDnodeObj *pSelDnode = NULL; diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index d15b32da54..9c8bc6c35b 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -58,7 +58,7 @@ static const SMnodeComponent tsMnodeComponents[] = { {"tables", mnodeInitTables, mnodeCleanupTables}, {"mnodes", mnodeInitMnodes, mnodeCleanupMnodes}, {"sdb", sdbInit, sdbCleanUp}, - {"balance", balanceInit, balanceCleanUp}, + {"balance", bnInit, bnCleanUp}, {"grant", grantInit, grantCleanUp}, {"show", mnodeInitShow, mnodeCleanUpShow} }; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 40e2e1cfcc..07b3ef09ec 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -244,7 +244,7 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) { sdbInfo("vgId:1, mnode role changed from %s to %s", syncRole[tsSdbMgmt.role], syncRole[role]); if (role == TAOS_SYNC_ROLE_MASTER && tsSdbMgmt.role != TAOS_SYNC_ROLE_MASTER) { - balanceReset(); + bnReset(); } tsSdbMgmt.role = role; diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 3e974f417f..7cf9e35bd2 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -563,7 +563,7 @@ int32_t mnodeCreateVgroup(SMnodeMsg *pMsg) { pVgroup->numOfVnodes = pDb->cfg.replications; pVgroup->createdTime = taosGetTimestampMs(); pVgroup->accessState = TSDB_VN_ALL_ACCCESS; - int32_t code = balanceAllocVnodes(pVgroup); + int32_t code = bnAllocVnodes(pVgroup); if (code != TSDB_CODE_SUCCESS) { mError("db:%s, no enough dnode to alloc %d vnodes to vgroup, reason:%s", pDb->name, pVgroup->numOfVnodes, tstrerror(code)); From b39b84e6d8ad8723f2bc6b35b4da1adb45b587d4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Dec 2020 13:02:06 +0800 Subject: [PATCH 146/188] TD-2270 rename files --- src/balance/inc/{bnMain.h => bnInt.h} | 0 src/balance/inc/bnScore.h | 2 +- src/balance/inc/bnThread.h | 2 +- src/balance/src/{balanceMain.c => bnMain.c} | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename src/balance/inc/{bnMain.h => bnInt.h} (100%) rename src/balance/src/{balanceMain.c => bnMain.c} (99%) diff --git a/src/balance/inc/bnMain.h b/src/balance/inc/bnInt.h similarity index 100% rename from src/balance/inc/bnMain.h rename to src/balance/inc/bnInt.h diff --git a/src/balance/inc/bnScore.h b/src/balance/inc/bnScore.h index 48e423b562..a28c4459dd 100644 --- a/src/balance/inc/bnScore.h +++ b/src/balance/inc/bnScore.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "bnMain.h" +#include "bnInt.h" void bnInitDnodes(); void bnCleanupDnodes(); diff --git a/src/balance/inc/bnThread.h b/src/balance/inc/bnThread.h index df9b06cf87..8f54b66028 100644 --- a/src/balance/inc/bnThread.h +++ b/src/balance/inc/bnThread.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "bnMain.h" +#include "bnInt.h" int32_t bnInitThread(); void bnCleanupThread(); diff --git a/src/balance/src/balanceMain.c b/src/balance/src/bnMain.c similarity index 99% rename from src/balance/src/balanceMain.c rename to src/balance/src/bnMain.c index 9f9d5103be..92f5ceb950 100644 --- a/src/balance/src/balanceMain.c +++ b/src/balance/src/bnMain.c @@ -33,7 +33,7 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -#include "bnMain.h" +#include "bnInt.h" #include "bnScore.h" #include "bnThread.h" From 1993fbc480575d23eed679476edf5b3eaab0ca5f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Dec 2020 13:19:28 +0800 Subject: [PATCH 147/188] TD-2270 --- src/balance/inc/bnInt.h | 5 +++-- src/balance/src/bnMain.c | 19 +++---------------- src/balance/src/bnScore.c | 16 ---------------- src/balance/src/bnThread.c | 17 ----------------- 4 files changed, 6 insertions(+), 51 deletions(-) diff --git a/src/balance/inc/bnInt.h b/src/balance/inc/bnInt.h index 101f5d5fd1..e924776ff1 100644 --- a/src/balance/inc/bnInt.h +++ b/src/balance/inc/bnInt.h @@ -13,13 +13,14 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_BALANCE_MAIN_H -#define TDENGINE_BALANCE_MAIN_H +#ifndef TDENGINE_BALANCE_INT_H +#define TDENGINE_BALANCE_INT_H #ifdef __cplusplus extern "C" { #endif #include "mnodeInt.h" +#include "mnodeDef.h" #include "mnodeDnode.h" typedef struct { diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 92f5ceb950..54e957a864 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -15,17 +15,12 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tutil.h" -#include "tbalance.h" #include "tsync.h" -#include "ttimer.h" #include "tglobal.h" -#include "tdataformat.h" #include "dnode.h" -#include "mnode.h" -#include "mnodeDef.h" -#include "mnodeInt.h" -#include "mnodeDnode.h" +#include "bnInt.h" +#include "bnScore.h" +#include "bnThread.h" #include "mnodeDb.h" #include "mnodeMnode.h" #include "mnodeSdb.h" @@ -33,12 +28,7 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -#include "bnInt.h" -#include "bnScore.h" -#include "bnThread.h" - static SBnMgmt tsBnMgmt;; - static void bnMonitorDnodeModule(); static void bnLock() { @@ -514,7 +504,6 @@ bool bnStart() { } bnReleaseDnodes(); - bnUnLock(); return updateSoon; @@ -610,8 +599,6 @@ int32_t bnDropDnode(SDnodeObj *pDnode) { return TSDB_CODE_SUCCESS; } - - static void bnMonitorDnodeModule() { int32_t numOfMnodes = mnodeGetMnodesNum(); if (numOfMnodes >= tsNumOfMnodes) return; diff --git a/src/balance/src/bnScore.c b/src/balance/src/bnScore.c index db5d515fbf..e5ad7a2119 100644 --- a/src/balance/src/bnScore.c +++ b/src/balance/src/bnScore.c @@ -15,25 +15,9 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tutil.h" -#include "tbalance.h" -#include "tsync.h" -#include "tsync.h" -#include "ttimer.h" #include "tglobal.h" -#include "tdataformat.h" -#include "dnode.h" -#include "mnode.h" -#include "mnodeDef.h" -#include "mnodeInt.h" -#include "mnodeDnode.h" -#include "mnodeDb.h" -#include "mnodeMnode.h" -#include "mnodeSdb.h" #include "mnodeShow.h" #include "mnodeUser.h" -#include "mnodeVgroup.h" - #include "bnScore.h" SBnDnodes tsBnDnodes; diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index 39eb6d0e31..08a045b94b 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -15,26 +15,9 @@ #define _DEFAULT_SOURCE #include "os.h" -#include "tutil.h" -#include "tbalance.h" -#include "tsync.h" -#include "tsync.h" #include "ttimer.h" #include "tglobal.h" -#include "tdataformat.h" -#include "dnode.h" -#include "mnode.h" -#include "mnodeDef.h" -#include "mnodeInt.h" -#include "mnodeDnode.h" -#include "mnodeDb.h" -#include "mnodeMnode.h" #include "mnodeSdb.h" -#include "mnodeShow.h" -#include "mnodeUser.h" -#include "mnodeVgroup.h" - - #include "bnThread.h" static SBnThread tsBnThread; From 1408a331f194049cec5d8b9754216a87a2a516f9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Dec 2020 13:24:18 +0800 Subject: [PATCH 148/188] TD-2270 rename files --- src/dnode/src/dnodeMgmt.c | 2 +- src/inc/{tbalance.h => tbn.h} | 0 src/mnode/src/mnodeDb.c | 2 +- src/mnode/src/mnodeDnode.c | 4 ++-- src/mnode/src/mnodeMain.c | 2 +- src/mnode/src/mnodeMnode.c | 2 +- src/mnode/src/mnodePeer.c | 2 +- src/mnode/src/mnodeRead.c | 2 +- src/mnode/src/mnodeSdb.c | 2 +- src/mnode/src/mnodeVgroup.c | 2 +- src/mnode/src/mnodeWrite.c | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) rename src/inc/{tbalance.h => tbn.h} (100%) diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 0d1a56cfc4..5c01f64716 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -24,7 +24,7 @@ #include "tqueue.h" #include "tsync.h" #include "ttimer.h" -#include "tbalance.h" +#include "tbn.h" #include "tglobal.h" #include "dnode.h" #include "vnode.h" diff --git a/src/inc/tbalance.h b/src/inc/tbn.h similarity index 100% rename from src/inc/tbalance.h rename to src/inc/tbn.h diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 63bd066c36..c971a945aa 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -20,7 +20,7 @@ #include "tgrant.h" #include "tglobal.h" #include "tname.h" -#include "tbalance.h" +#include "tbn.h" #include "tdataformat.h" #include "mnode.h" #include "mnodeDef.h" diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index 0d429f5797..0ff50b2307 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -16,12 +16,12 @@ #define _DEFAULT_SOURCE #include "os.h" #include "tgrant.h" -#include "tbalance.h" +#include "tbn.h" #include "tglobal.h" #include "tconfig.h" #include "tutil.h" #include "tsocket.h" -#include "tbalance.h" +#include "tbn.h" #include "tsync.h" #include "tdataformat.h" #include "mnode.h" diff --git a/src/mnode/src/mnodeMain.c b/src/mnode/src/mnodeMain.c index 9c8bc6c35b..86f2c821f9 100644 --- a/src/mnode/src/mnodeMain.c +++ b/src/mnode/src/mnodeMain.c @@ -17,7 +17,7 @@ #include "os.h" #include "taosdef.h" #include "tsched.h" -#include "tbalance.h" +#include "tbn.h" #include "tgrant.h" #include "ttimer.h" #include "tglobal.h" diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 7428e8d2c8..d20d51f82b 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -19,7 +19,7 @@ #include "tglobal.h" #include "trpc.h" #include "tsync.h" -#include "tbalance.h" +#include "tbn.h" #include "tutil.h" #include "tsocket.h" #include "tdataformat.h" diff --git a/src/mnode/src/mnodePeer.c b/src/mnode/src/mnodePeer.c index ef1b819018..cfb7b7781b 100644 --- a/src/mnode/src/mnodePeer.c +++ b/src/mnode/src/mnodePeer.c @@ -20,7 +20,7 @@ #include "tsystem.h" #include "tutil.h" #include "tgrant.h" -#include "tbalance.h" +#include "tbn.h" #include "tglobal.h" #include "mnode.h" #include "dnode.h" diff --git a/src/mnode/src/mnodeRead.c b/src/mnode/src/mnodeRead.c index a39d35506f..c2a70bc01d 100644 --- a/src/mnode/src/mnodeRead.c +++ b/src/mnode/src/mnodeRead.c @@ -17,7 +17,7 @@ #include "os.h" #include "taosdef.h" #include "tsched.h" -#include "tbalance.h" +#include "tbn.h" #include "tgrant.h" #include "ttimer.h" #include "tglobal.h" diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 07b3ef09ec..2ef758baf1 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -19,7 +19,7 @@ #include "hash.h" #include "tutil.h" #include "tref.h" -#include "tbalance.h" +#include "tbn.h" #include "tqueue.h" #include "twal.h" #include "tsync.h" diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index 7cf9e35bd2..d3020de6bd 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -20,7 +20,7 @@ #include "tsocket.h" #include "tidpool.h" #include "tsync.h" -#include "tbalance.h" +#include "tbn.h" #include "tglobal.h" #include "tdataformat.h" #include "dnode.h" diff --git a/src/mnode/src/mnodeWrite.c b/src/mnode/src/mnodeWrite.c index f0cfe1aedc..53981238a7 100644 --- a/src/mnode/src/mnodeWrite.c +++ b/src/mnode/src/mnodeWrite.c @@ -17,7 +17,7 @@ #include "os.h" #include "taosdef.h" #include "tsched.h" -#include "tbalance.h" +#include "tbn.h" #include "tgrant.h" #include "tglobal.h" #include "trpc.h" From 51a8b29a8f4efdfaed88e685a318d34e3108ca1f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 13:29:13 +0800 Subject: [PATCH 149/188] [TD-2298] --- src/query/src/qExecutor.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 03de6aea1f..6ce40b9204 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3981,8 +3981,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { if (qstatus.lastKey != pTableQueryInfo->lastKey) { qstatus.curWindow.ekey = pTableQueryInfo->lastKey - step; } else { // the lastkey does not increase, which means no data checked yet - qDebug("QInfo:%p no results generated in this scan, abort", pQInfo); - return; + qDebug("QInfo:%p no results generated in this scan", pQInfo); } qstatus.lastKey = pTableQueryInfo->lastKey; @@ -4749,7 +4748,23 @@ static TSKEY doSkipIntervalProcess(SQueryRuntimeEnv* pRuntimeEnv, STimeWindow* w static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { SQuery *pQuery = pRuntimeEnv->pQuery; - *start = pQuery->current->lastKey; + + // get the first unclosed time window + bool assign = false; + for(int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { + if (pRuntimeEnv->windowResInfo.pResult[i]->closed) { + continue; + } + + assign = true; + *start = pRuntimeEnv->windowResInfo.pResult[i]->win.skey; + } + + if (!assign) { + *start = pQuery->current->lastKey; + } + + assert(*start <= pQuery->current->lastKey); // if queried with value filter, do NOT forward query start position if (pQuery->limit.offset <= 0 || pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->pFillInfo != NULL) { @@ -5441,7 +5456,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) { //super table projection query with identical query time range for all tables. SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; - resetDefaultResInfoOutputBuf(pRuntimeEnv); SArray *group = GET_TABLEGROUP(pQInfo, 0); From 26b32bd9e7d16b420b7e742788b5da29ba156dd9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Dec 2020 14:02:10 +0800 Subject: [PATCH 150/188] TD-2270 --- src/balance/src/bnMain.c | 2 +- src/balance/src/bnThread.c | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/balance/src/bnMain.c b/src/balance/src/bnMain.c index 54e957a864..383f981913 100644 --- a/src/balance/src/bnMain.c +++ b/src/balance/src/bnMain.c @@ -566,8 +566,8 @@ int32_t bnInit() { void bnCleanUp() { bnCleanupThread(); - pthread_mutex_destroy(&tsBnMgmt.mutex); bnCleanupDnodes(); + pthread_mutex_destroy(&tsBnMgmt.mutex); } int32_t bnDropDnode(SDnodeObj *pDnode) { diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index 08a045b94b..a11bc61b01 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -81,7 +81,6 @@ void bnCleanupThread() { } static void bnPostSignal() { - mDebug("balance thread async notify"); pthread_mutex_lock(&tsBnThread.mutex); pthread_cond_signal(&tsBnThread.cond); pthread_mutex_unlock(&(tsBnThread.mutex)); @@ -94,11 +93,13 @@ static void bnPostSignal() { static void bnProcessTimer(void *handle, void *tmrId) { if (!sdbIsMaster()) return; + if (tsBnThread.stop) return; tsBnThread.timer = NULL; tsAccessSquence++; bnCheckStatus(); + bnStartTimer(-1); if (handle == NULL) { if (tsAccessSquence % tsBalanceInterval == 0) { @@ -113,6 +114,8 @@ static void bnProcessTimer(void *handle, void *tmrId) { } void bnStartTimer(int64_t mseconds) { + if (tsBnThread.stop) return; + bool updateSoon = (mseconds != -1); if (updateSoon) { taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer); From 024397f265f5115caa83808890d101f0776debcb Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 3 Dec 2020 06:57:49 +0000 Subject: [PATCH 151/188] fix TD-2314 --- src/tsdb/src/tsdbFile.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 03c50d42f7..5d8933d141 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -248,8 +248,13 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) - goto _err; + if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) { + for (int i = type; i >= 0; i--) { + remove(pFGroup->files[i].fname); + } + + return NULL; + } } pthread_rwlock_wrlock(&pFileH->fhlock); @@ -261,10 +266,6 @@ SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) { } return pGroup; - -_err: - for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); - return NULL; } void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { From b6d30543d82c5f7c900e839a11d79a9ede653c82 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 3 Dec 2020 15:06:11 +0800 Subject: [PATCH 152/188] change --- .../example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java index 82613037db..36745a9394 100644 --- a/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java +++ b/tests/examples/JDBC/JDBCDemo/src/main/java/com/taosdata/example/jdbcTaosdemo/domain/JdbcTaosdemoConfig.java @@ -14,9 +14,9 @@ public final class JdbcTaosdemoConfig { //Destination database. Default is 'test' private String dbName = "test"; //keep - private int keep = 3650; + private int keep = 36500; //days - private int days = 10; + private int days = 120; //Super table Name. Default is 'meters' private String stbName = "meters"; From 9887f533c6e27f03e4e5c7687a97cbfddd2e99dd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 15:25:43 +0800 Subject: [PATCH 153/188] [TD-2298] add test cases. --- tests/script/general/parser/first_last.sim | 42 ++++++++++++++++++- .../general/parser/first_last_query.sim | 4 +- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/tests/script/general/parser/first_last.sim b/tests/script/general/parser/first_last.sim index a934d3bcab..773f92afcf 100644 --- a/tests/script/general/parser/first_last.sim +++ b/tests/script/general/parser/first_last.sim @@ -46,7 +46,8 @@ while $i < $tbNum endw $i = $i + 1 -endw +endw + $ts = $ts + 60000 $tb = $tbPrefix . 0 sql insert into $tb (ts) values ( $ts ) @@ -84,4 +85,43 @@ sleep 500 run general/parser/first_last_query.sim +print =================> insert data regression test +sql create database test keep 36500 +sql use test +sql create table tm0 (ts timestamp, k int) + +print =========================> td-2298 +$ts0 = 1537146000000 +$xs = 6000 + +$x = 0 +while $x < 5000 + $ts = $ts0 + $xs + $ts1 = $ts + $xs + $x1 = $x + 1 + + sql insert into tm0 values ( $ts , $x ) ( $ts1 , $x1 ) + $x = $x1 + $ts0 = $ts1 +endw + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 3000 +system sh/exec.sh -n dnode1 -s start +print ================== server restart completed +sql connect +sleep 500 + +sql use test +sql select count(*), last(ts) from tm0 interval(1s) +if $rows != 10000 then + print expect 10000, actual: $rows + return -1 +endi + +sql select last(ts) from tm0 interval(1s) +if $rows != 10000 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/first_last_query.sim b/tests/script/general/parser/first_last_query.sim index a982f10362..52d888b04d 100644 --- a/tests/script/general/parser/first_last_query.sim +++ b/tests/script/general/parser/first_last_query.sim @@ -266,4 +266,6 @@ endi if $data14 != @test2@ then print expect test2 , actual: $data14 return -1 -endi \ No newline at end of file +endi + +sql drop table stest \ No newline at end of file From a837f33731213ee8f5fa9a860e3d762f15ee0cf7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 3 Dec 2020 15:43:21 +0800 Subject: [PATCH 154/188] [TD-2298] add test case for interval --- tests/pytest/fulltest.sh | 3 ++- tests/pytest/query/queryInterval.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 042fd826e8..448a0413e0 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -159,7 +159,8 @@ python3 ./test.py -f query/bug1471.py python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py -python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/sliding.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/queryInterval.py b/tests/pytest/query/queryInterval.py index 98b8c9fbef..871c076c08 100644 --- a/tests/pytest/query/queryInterval.py +++ b/tests/pytest/query/queryInterval.py @@ -16,6 +16,7 @@ import taos from util.log import tdLog from util.cases import tdCases from util.sql import tdSql +from util.dnodes import tdDnodes class TDTestCase: @@ -72,6 +73,19 @@ class TDTestCase: tdSql.checkData(6, 0, "2020-09-16 00:00:00") tdSql.checkData(6, 1, 222.0) + # test case for https://jira.taosdata.com:18080/browse/TD-2298 + tdSql.execute("create database test keep 36500") + tdSql.execute("use test") + tdSql.execute("create table t (ts timestamp, voltage int)") + for i in range(10000): + tdSql.execute("insert into t values(%d, 0)" % (1000000 + i * 6000)) + + tdDnodes.stop(1) + tdDnodes.start(1) + tdSql.query("select last(*) from t interval(1s)") + tdSql.checkRows(10000) + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 92ca5ded0d222f4914876b745a6428d0dc276f75 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 3 Dec 2020 16:19:56 +0800 Subject: [PATCH 155/188] [TD-785]Improve the robustness of python connector --- src/connector/python/linux/python2/taos/__init__.py | 2 +- src/connector/python/linux/python2/taos/cinterface.py | 2 ++ src/connector/python/linux/python3/taos/__init__.py | 4 ++-- src/connector/python/linux/python3/taos/cinterface.py | 2 ++ src/connector/python/windows/python2/taos/__init__.py | 2 +- src/connector/python/windows/python2/taos/cinterface.py | 2 ++ src/connector/python/windows/python3/taos/__init__.py | 2 +- src/connector/python/windows/python3/taos/cinterface.py | 2 ++ 8 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/connector/python/linux/python2/taos/__init__.py b/src/connector/python/linux/python2/taos/__init__.py index 4894488bc8..5d21671eed 100644 --- a/src/connector/python/linux/python2/taos/__init__.py +++ b/src/connector/python/linux/python2/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0' +apilevel = '2.0.2' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 6f0435722f..2b1b29ee31 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) if num_of_rows > 0: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: @@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) res=[] if num_of_rows > 0: for i in range(abs(num_of_rows)): diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py index 4894488bc8..c46131eeaf 100644 --- a/src/connector/python/linux/python3/taos/__init__.py +++ b/src/connector/python/linux/python3/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0' +apilevel = '2.0.2' threadsafety = 0 paramstyle = 'pyformat' @@ -21,4 +21,4 @@ def connect(*args, **kwargs): @rtype: TDengineConnector """ - return TDengineConnection(*args, **kwargs) \ No newline at end of file + return TDengineConnection(*args, **kwargs) diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index bbf5a0c714..fdebe349fe 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) if num_of_rows > 0: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: @@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) res=[] if num_of_rows > 0: for i in range(abs(num_of_rows)): diff --git a/src/connector/python/windows/python2/taos/__init__.py b/src/connector/python/windows/python2/taos/__init__.py index 4894488bc8..5d21671eed 100644 --- a/src/connector/python/windows/python2/taos/__init__.py +++ b/src/connector/python/windows/python2/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0' +apilevel = '2.0.2' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index d4bf1a0350..14f4f49be8 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) if num_of_rows > 0: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: @@ -106,6 +107,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) res=[] if num_of_rows > 0: for i in range(abs(num_of_rows)): diff --git a/src/connector/python/windows/python3/taos/__init__.py b/src/connector/python/windows/python3/taos/__init__.py index 2fec471a1a..0708b9544f 100644 --- a/src/connector/python/windows/python3/taos/__init__.py +++ b/src/connector/python/windows/python3/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0' +apilevel = '2.0.2' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index fb1b65fc15..42b820ca80 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -81,6 +81,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) if num_of_rows > 0: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: @@ -108,6 +109,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ + assert(nbytes is not None) res=[] if num_of_rows > 0: for i in range(abs(num_of_rows)): From d79d3058ae9cf061e5217d8bc91d6d08f63307aa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 16:39:33 +0800 Subject: [PATCH 156/188] [TD-225] fix compiler error. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6ce40b9204..0c07149e8e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4524,7 +4524,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data } } - int32_t numOfTables = taosHashGetSize(pQInfo->arrTableIdInfo); + int32_t numOfTables = (int32_t) taosHashGetSize(pQInfo->arrTableIdInfo); *(int32_t*)data = htonl(numOfTables); data += sizeof(int32_t); From fa810a383a1252bb13dc8b09eaa57f38f2dffbd1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 17:25:59 +0800 Subject: [PATCH 157/188] [TD-2265]: add the total number of rows in submit block during parse insert sql. --- src/client/src/tscParseInsert.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6de8195d73..6ff1cdcb79 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -630,11 +630,17 @@ int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int3 return TSDB_CODE_SUCCESS; } -static void tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { +static int32_t tsSetBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) { pBlocks->tid = pTableMeta->id.tid; pBlocks->uid = pTableMeta->id.uid; pBlocks->sversion = pTableMeta->sversion; - pBlocks->numOfRows += numOfRows; + + if (pBlocks->numOfRows + numOfRows >= INT16_MAX) { + return TSDB_CODE_TSC_INVALID_SQL; + } else { + pBlocks->numOfRows += numOfRows; + return TSDB_CODE_SUCCESS; + } } // data block is disordered, sort it in ascending order @@ -722,7 +728,11 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st } SSubmitBlk *pBlocks = (SSubmitBlk *)(dataBuf->pData); - tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + if (code != TSDB_CODE_SUCCESS) { + tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", *str); + return code; + } dataBuf->vgId = pTableMeta->vgroupInfo.vgId; dataBuf->numOfTables = 1; From a3fb9a1a8e13c83b783cede3460650743f0e53e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 17:28:02 +0800 Subject: [PATCH 158/188] [TD-2265] --- src/client/src/tscParseInsert.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 6ff1cdcb79..08935adf8b 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1394,7 +1394,10 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock STableMeta *pTableMeta = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0)->pTableMeta; SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); - tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); + if (code != TSDB_CODE_SUCCESS) { + return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", ""); + } if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { return code; From 5015365172cfd9cc91344eeadc2a266e0df19be4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 17:28:38 +0800 Subject: [PATCH 159/188] [TD-2265] --- src/client/src/tscParseInsert.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 08935adf8b..18e5b6f074 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -1396,7 +1396,7 @@ static int doPackSendDataBlock(SSqlObj *pSql, int32_t numOfRows, STableDataBlock SSubmitBlk *pBlocks = (SSubmitBlk *)(pTableDataBlocks->pData); code = tsSetBlockInfo(pBlocks, pTableMeta, numOfRows); if (code != TSDB_CODE_SUCCESS) { - return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", ""); + return tscInvalidSQLErrMsg(pCmd->payload, "too many rows in sql, total number of rows should be less than 32767", NULL); } if ((code = tscMergeTableDataBlocks(pSql, pCmd->pDataBlocks)) != TSDB_CODE_SUCCESS) { From 418f15e2d1ca92fbaf887d59d91d6c7a7b4babc9 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Dec 2020 18:04:18 +0800 Subject: [PATCH 160/188] T-2270 change monitordebugflag --- packaging/cfg/taos.cfg | 2 +- src/client/src/tscSQLParser.c | 2 +- src/common/inc/tglobal.h | 2 +- src/common/src/tglobal.c | 26 +-- src/dnode/src/dnodeModule.c | 8 +- src/inc/monitor.h | 14 +- src/mnode/inc/mnodeInt.h | 6 +- .../monitor/src/{monitorMain.c => monMain.c} | 150 +++++++++--------- tests/script/general/alter/dnode.sim | 8 +- tests/script/sh/deploy.sh | 2 +- 10 files changed, 110 insertions(+), 110 deletions(-) rename src/plugins/monitor/src/{monitorMain.c => monMain.c} (70%) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index ca88bca3c8..6e9a17dbe4 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -236,7 +236,7 @@ # httpDebugFlag 131 # debug flag for monitor -# monitorDebugFlag 131 +# monDebugFlag 131 # debug flag for query # qDebugflag 131 diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 309d354849..110e306ce7 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5103,7 +5103,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { const int tokenDebugFlagEnd = 20; const SDNodeDynConfOption cfgOptions[] = { {"resetLog", 8}, {"resetQueryCache", 15}, {"balance", 7}, {"monitor", 7}, - {"debugFlag", 9}, {"monitorDebugFlag", 16}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, + {"debugFlag", 9}, {"monDebugFlag", 11}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, {"cDebugFlag", 10}, {"httpDebugFlag", 13}, {"qDebugflag", 10}, {"sdbDebugFlag", 12}, {"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12}, {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index efe3d7678a..bdd9b8ce9f 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -176,7 +176,7 @@ extern int32_t tmrDebugFlag; extern int32_t sdbDebugFlag; extern int32_t httpDebugFlag; extern int32_t mqttDebugFlag; -extern int32_t monitorDebugFlag; +extern int32_t monDebugFlag; extern int32_t uDebugFlag; extern int32_t rpcDebugFlag; extern int32_t odbcDebugFlag; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index d26746a5d1..2a84184f8d 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -206,7 +206,7 @@ int32_t jniDebugFlag = 131; int32_t odbcDebugFlag = 131; int32_t httpDebugFlag = 131; int32_t mqttDebugFlag = 131; -int32_t monitorDebugFlag = 131; +int32_t monDebugFlag = 131; int32_t qDebugFlag = 131; int32_t rpcDebugFlag = 131; int32_t uDebugFlag = 131; @@ -216,9 +216,9 @@ int32_t wDebugFlag = 135; int32_t tsdbDebugFlag = 131; int32_t cqDebugFlag = 135; -int32_t (*monitorStartSystemFp)() = NULL; -void (*monitorStopSystemFp)() = NULL; -void (*monitorExecuteSQLFp)(char *sql) = NULL; +int32_t (*monStartSystemFp)() = NULL; +void (*monStopSystemFp)() = NULL; +void (*monExecuteSQLFp)(char *sql) = NULL; char *qtypeStr[] = {"rpc", "fwd", "wal", "cq", "query"}; @@ -235,7 +235,7 @@ void taosSetAllDebugFlag() { odbcDebugFlag = debugFlag; httpDebugFlag = debugFlag; mqttDebugFlag = debugFlag; - monitorDebugFlag = debugFlag; + monDebugFlag = debugFlag; qDebugFlag = debugFlag; rpcDebugFlag = debugFlag; uDebugFlag = debugFlag; @@ -276,15 +276,15 @@ bool taosCfgDynamicOptions(char *msg) { if (strncasecmp(cfg->option, "monitor", olen) == 0) { if (1 == vint) { - if (monitorStartSystemFp) { - (*monitorStartSystemFp)(); + if (monStartSystemFp) { + (*monStartSystemFp)(); uInfo("monitor is enabled"); } else { uError("monitor can't be updated, for monitor not initialized"); } } else { - if (monitorStopSystemFp) { - (*monitorStopSystemFp)(); + if (monStopSystemFp) { + (*monStopSystemFp)(); uInfo("monitor is disabled"); } else { uError("monitor can't be updated, for monitor not initialized"); @@ -307,8 +307,8 @@ bool taosCfgDynamicOptions(char *msg) { } if (strncasecmp(option, "resetQueryCache", 15) == 0) { - if (monitorExecuteSQLFp) { - (*monitorExecuteSQLFp)("resetQueryCache"); + if (monExecuteSQLFp) { + (*monExecuteSQLFp)("resetQueryCache"); uInfo("resetquerycache is executed"); } else { uError("resetquerycache can't be executed, for monitor not started"); @@ -1227,8 +1227,8 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "monitorDebugFlag"; - cfg.ptr = &monitorDebugFlag; + cfg.option = "monDebugFlag"; + cfg.ptr = &monDebugFlag; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_LOG; cfg.minValue = 0; diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index bd9500ba51..7faa3c8913 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -78,10 +78,10 @@ static void dnodeAllocModules() { tsModule[TSDB_MOD_MONITOR].enable = (tsEnableMonitorModule == 1); tsModule[TSDB_MOD_MONITOR].name = "monitor"; - tsModule[TSDB_MOD_MONITOR].initFp = monitorInitSystem; - tsModule[TSDB_MOD_MONITOR].cleanUpFp = monitorCleanUpSystem; - tsModule[TSDB_MOD_MONITOR].startFp = monitorStartSystem; - tsModule[TSDB_MOD_MONITOR].stopFp = monitorStopSystem; + tsModule[TSDB_MOD_MONITOR].initFp = monInitSystem; + tsModule[TSDB_MOD_MONITOR].cleanUpFp = monCleanupSystem; + tsModule[TSDB_MOD_MONITOR].startFp = monStartSystem; + tsModule[TSDB_MOD_MONITOR].stopFp = monStopSystem; if (tsEnableMonitorModule) { dnodeSetModuleStatus(TSDB_MOD_MONITOR); } diff --git a/src/inc/monitor.h b/src/inc/monitor.h index b1229cca6b..1aefb0b848 100644 --- a/src/inc/monitor.h +++ b/src/inc/monitor.h @@ -47,13 +47,13 @@ typedef struct { int8_t accessState; } SAcctMonitorObj; -int32_t monitorInitSystem(); -int32_t monitorStartSystem(); -void monitorStopSystem(); -void monitorCleanUpSystem(); -void monitorSaveAcctLog(SAcctMonitorObj *pMonObj); -void monitorSaveLog(int32_t level, const char *const format, ...); -void monitorExecuteSQL(char *sql); +int32_t monInitSystem(); +int32_t monStartSystem(); +void monStopSystem(); +void monCleanupSystem(); +void monSaveAcctLog(SAcctMonitorObj *pMonObj); +void monSaveLog(int32_t level, const char *const format, ...); +void monExecuteSQL(char *sql); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeInt.h b/src/mnode/inc/mnodeInt.h index 44626fd167..7a791d76e6 100644 --- a/src/mnode/inc/mnodeInt.h +++ b/src/mnode/inc/mnodeInt.h @@ -41,9 +41,9 @@ extern int32_t sdbDebugFlag; #define sdbDebug(...) { if (sdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }} #define sdbTrace(...) { if (sdbDebugFlag & DEBUG_TRACE) { taosPrintLog("SDB ", sdbDebugFlag, __VA_ARGS__); }} -#define mLError(...) { monitorSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) } -#define mLWarn(...) { monitorSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) } -#define mLInfo(...) { monitorSaveLog(0, __VA_ARGS__); mInfo(__VA_ARGS__) } +#define mLError(...) { monSaveLog(2, __VA_ARGS__); mError(__VA_ARGS__) } +#define mLWarn(...) { monSaveLog(1, __VA_ARGS__); mWarn(__VA_ARGS__) } +#define mLInfo(...) { monSaveLog(0, __VA_ARGS__); mInfo(__VA_ARGS__) } #ifdef __cplusplus } diff --git a/src/plugins/monitor/src/monitorMain.c b/src/plugins/monitor/src/monMain.c similarity index 70% rename from src/plugins/monitor/src/monitorMain.c rename to src/plugins/monitor/src/monMain.c index 24998b54cd..9443b1ce12 100644 --- a/src/plugins/monitor/src/monitorMain.c +++ b/src/plugins/monitor/src/monMain.c @@ -27,12 +27,12 @@ #include "monitor.h" #include "taoserror.h" -#define mnFatal(...) { if (monitorDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} -#define mnError(...) { if (monitorDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} -#define mnWarn(...) { if (monitorDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} -#define mnInfo(...) { if (monitorDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }} -#define mnDebug(...) { if (monitorDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} -#define mnTrace(...) { if (monitorDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monitorDebugFlag, __VA_ARGS__); }} +#define monFatal(...) { if (monDebugFlag & DEBUG_FATAL) { taosPrintLog("MON FATAL ", 255, __VA_ARGS__); }} +#define monError(...) { if (monDebugFlag & DEBUG_ERROR) { taosPrintLog("MON ERROR ", 255, __VA_ARGS__); }} +#define monWarn(...) { if (monDebugFlag & DEBUG_WARN) { taosPrintLog("MON WARN ", 255, __VA_ARGS__); }} +#define monInfo(...) { if (monDebugFlag & DEBUG_INFO) { taosPrintLog("MON ", 255, __VA_ARGS__); }} +#define monDebug(...) { if (monDebugFlag & DEBUG_DEBUG) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }} +#define monTrace(...) { if (monDebugFlag & DEBUG_TRACE) { taosPrintLog("MON ", monDebugFlag, __VA_ARGS__); }} #define SQL_LENGTH 1030 #define LOG_LEN_STR 100 @@ -48,12 +48,12 @@ typedef enum { MON_CMD_CREATE_TB_ACCT_ROOT, MON_CMD_CREATE_TB_SLOWQUERY, MON_CMD_MAX -} EMonitorCommand; +} EMonCmd; typedef enum { MON_STATE_NOT_INIT, MON_STATE_INITED -} EMonitorState; +} EMonState; typedef struct { pthread_t thread; @@ -64,17 +64,17 @@ typedef struct { int8_t start; // enable/disable by mnode int8_t quiting; // taosd is quiting char sql[SQL_LENGTH + 1]; -} SMonitorConn; +} SMonConn; -static SMonitorConn tsMonitor = {0}; -static void monitorSaveSystemInfo(); -static void *monitorThreadFunc(void *param); -static void monitorBuildMonitorSql(char *sql, int32_t cmd); -extern int32_t (*monitorStartSystemFp)(); -extern void (*monitorStopSystemFp)(); -extern void (*monitorExecuteSQLFp)(char *sql); +static SMonConn tsMonitor = {0}; +static void monSaveSystemInfo(); +static void *monThreadFunc(void *param); +static void monBuildMonitorSql(char *sql, int32_t cmd); +extern int32_t (*monStartSystemFp)(); +extern void (*monStopSystemFp)(); +extern void (*monExecuteSQLFp)(char *sql); -int32_t monitorInitSystem() { +int32_t monInitSystem() { if (tsMonitor.ep[0] == 0) { strcpy(tsMonitor.ep, tsLocalEp); } @@ -90,29 +90,29 @@ int32_t monitorInitSystem() { pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pthread_create(&tsMonitor.thread, &thAttr, monitorThreadFunc, NULL)) { - mnError("failed to create thread to for monitor module, reason:%s", strerror(errno)); + if (pthread_create(&tsMonitor.thread, &thAttr, monThreadFunc, NULL)) { + monError("failed to create thread to for monitor module, reason:%s", strerror(errno)); return -1; } pthread_attr_destroy(&thAttr); - mnDebug("monitor thread is launched"); + monDebug("monitor thread is launched"); - monitorStartSystemFp = monitorStartSystem; - monitorStopSystemFp = monitorStopSystem; + monStartSystemFp = monStartSystem; + monStopSystemFp = monStopSystem; return 0; } -int32_t monitorStartSystem() { +int32_t monStartSystem() { taos_init(); tsMonitor.start = 1; - monitorExecuteSQLFp = monitorExecuteSQL; - mnInfo("monitor module start"); + monExecuteSQLFp = monExecuteSQL; + monInfo("monitor module start"); return 0; } -static void *monitorThreadFunc(void *param) { - mnDebug("starting to initialize monitor module ..."); +static void *monThreadFunc(void *param) { + monDebug("starting to initialize monitor module ..."); while (1) { static int32_t accessTimes = 0; @@ -121,7 +121,7 @@ static void *monitorThreadFunc(void *param) { if (tsMonitor.quiting) { tsMonitor.state = MON_STATE_NOT_INIT; - mnInfo("monitor thread will quit, for taosd is quiting"); + monInfo("monitor thread will quit, for taosd is quiting"); break; } else { taosGetDisk(); @@ -132,7 +132,7 @@ static void *monitorThreadFunc(void *param) { } if (dnodeGetDnodeId() <= 0) { - mnDebug("dnode not initialized, waiting for 3000 ms to start monitor module"); + monDebug("dnode not initialized, waiting for 3000 ms to start monitor module"); continue; } @@ -140,10 +140,10 @@ static void *monitorThreadFunc(void *param) { tsMonitor.state = MON_STATE_NOT_INIT; tsMonitor.conn = taos_connect(NULL, "monitor", tsInternalPass, "", 0); if (tsMonitor.conn == NULL) { - mnError("failed to connect to database, reason:%s", tstrerror(terrno)); + monError("failed to connect to database, reason:%s", tstrerror(terrno)); continue; } else { - mnDebug("connect to database success"); + monDebug("connect to database success"); } } @@ -151,16 +151,16 @@ static void *monitorThreadFunc(void *param) { int code = 0; for (; tsMonitor.cmdIndex < MON_CMD_MAX; ++tsMonitor.cmdIndex) { - monitorBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex); + monBuildMonitorSql(tsMonitor.sql, tsMonitor.cmdIndex); void *res = taos_query(tsMonitor.conn, tsMonitor.sql); code = taos_errno(res); taos_free_result(res); if (code != 0) { - mnError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code)); + monError("failed to exec sql:%s, reason:%s", tsMonitor.sql, tstrerror(code)); break; } else { - mnDebug("successfully to exec sql:%s", tsMonitor.sql); + monDebug("successfully to exec sql:%s", tsMonitor.sql); } } @@ -171,16 +171,16 @@ static void *monitorThreadFunc(void *param) { if (tsMonitor.state == MON_STATE_INITED) { if (accessTimes % tsMonitorInterval == 0) { - monitorSaveSystemInfo(); + monSaveSystemInfo(); } } } - mnInfo("monitor thread is stopped"); + monInfo("monitor thread is stopped"); return NULL; } -static void monitorBuildMonitorSql(char *sql, int32_t cmd) { +static void monBuildMonitorSql(char *sql, int32_t cmd) { memset(sql, 0, SQL_LENGTH); if (cmd == MON_CMD_CREATE_DB) { @@ -236,47 +236,47 @@ static void monitorBuildMonitorSql(char *sql, int32_t cmd) { sql[SQL_LENGTH] = 0; } -void monitorStopSystem() { +void monStopSystem() { tsMonitor.start = 0; tsMonitor.state = MON_STATE_NOT_INIT; - monitorExecuteSQLFp = NULL; - mnInfo("monitor module stopped"); + monExecuteSQLFp = NULL; + monInfo("monitor module stopped"); } -void monitorCleanUpSystem() { +void monCleanupSystem() { tsMonitor.quiting = 1; - monitorStopSystem(); + monStopSystem(); pthread_join(tsMonitor.thread, NULL); if (tsMonitor.conn != NULL) { taos_close(tsMonitor.conn); tsMonitor.conn = NULL; } - mnInfo("monitor module is cleaned up"); + monInfo("monitor module is cleaned up"); } // unit is MB -static int32_t monitorBuildMemorySql(char *sql) { +static int32_t monBuildMemorySql(char *sql) { float sysMemoryUsedMB = 0; bool suc = taosGetSysMemory(&sysMemoryUsedMB); if (!suc) { - mnDebug("failed to get sys memory info"); + monDebug("failed to get sys memory info"); } float procMemoryUsedMB = 0; suc = taosGetProcMemory(&procMemoryUsedMB); if (!suc) { - mnDebug("failed to get proc memory info"); + monDebug("failed to get proc memory info"); } return sprintf(sql, ", %f, %f, %d", procMemoryUsedMB, sysMemoryUsedMB, tsTotalMemoryMB); } // unit is % -static int32_t monitorBuildCpuSql(char *sql) { +static int32_t monBuildCpuSql(char *sql) { float sysCpuUsage = 0, procCpuUsage = 0; bool suc = taosGetCpuUsage(&sysCpuUsage, &procCpuUsage); if (!suc) { - mnDebug("failed to get cpu usage"); + monDebug("failed to get cpu usage"); } if (sysCpuUsage <= procCpuUsage) { @@ -287,72 +287,72 @@ static int32_t monitorBuildCpuSql(char *sql) { } // unit is GB -static int32_t monitorBuildDiskSql(char *sql) { +static int32_t monBuildDiskSql(char *sql) { return sprintf(sql, ", %f, %d", (tsTotalDataDirGB - tsAvailDataDirGB), (int32_t)tsTotalDataDirGB); } // unit is Kb -static int32_t monitorBuildBandSql(char *sql) { +static int32_t monBuildBandSql(char *sql) { float bandSpeedKb = 0; bool suc = taosGetBandSpeed(&bandSpeedKb); if (!suc) { - mnDebug("failed to get bandwidth speed"); + monDebug("failed to get bandwidth speed"); } return sprintf(sql, ", %f", bandSpeedKb); } -static int32_t monitorBuildReqSql(char *sql) { +static int32_t monBuildReqSql(char *sql) { SStatisInfo info = dnodeGetStatisInfo(); return sprintf(sql, ", %d, %d, %d)", info.httpReqNum, info.queryReqNum, info.submitReqNum); } -static int32_t monitorBuildIoSql(char *sql) { +static int32_t monBuildIoSql(char *sql) { float readKB = 0, writeKB = 0; bool suc = taosGetProcIO(&readKB, &writeKB); if (!suc) { - mnDebug("failed to get io info"); + monDebug("failed to get io info"); } return sprintf(sql, ", %f, %f", readKB, writeKB); } -static void monitorSaveSystemInfo() { +static void monSaveSystemInfo() { int64_t ts = taosGetTimestampUs(); char * sql = tsMonitor.sql; int32_t pos = snprintf(sql, SQL_LENGTH, "insert into %s.dn%d values(%" PRId64, tsMonitorDbName, dnodeGetDnodeId(), ts); - pos += monitorBuildCpuSql(sql + pos); - pos += monitorBuildMemorySql(sql + pos); - pos += monitorBuildDiskSql(sql + pos); - pos += monitorBuildBandSql(sql + pos); - pos += monitorBuildIoSql(sql + pos); - pos += monitorBuildReqSql(sql + pos); + pos += monBuildCpuSql(sql + pos); + pos += monBuildMemorySql(sql + pos); + pos += monBuildDiskSql(sql + pos); + pos += monBuildBandSql(sql + pos); + pos += monBuildIoSql(sql + pos); + pos += monBuildReqSql(sql + pos); void *res = taos_query(tsMonitor.conn, tsMonitor.sql); int code = taos_errno(res); taos_free_result(res); if (code != 0) { - mnError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); + monError("failed to save system info, reason:%s, sql:%s", tstrerror(code), tsMonitor.sql); } else { - mnDebug("successfully to save system info, sql:%s", tsMonitor.sql); + monDebug("successfully to save system info, sql:%s", tsMonitor.sql); } } -static void montiorExecSqlCb(void *param, TAOS_RES *result, int32_t code) { +static void monExecSqlCb(void *param, TAOS_RES *result, int32_t code) { int32_t c = taos_errno(result); if (c != TSDB_CODE_SUCCESS) { - mnError("save %s failed, reason:%s", (char *)param, tstrerror(c)); + monError("save %s failed, reason:%s", (char *)param, tstrerror(c)); } else { int32_t rows = taos_affected_rows(result); - mnDebug("save %s succ, rows:%d", (char *)param, rows); + monDebug("save %s succ, rows:%d", (char *)param, rows); } taos_free_result(result); } -void monitorSaveAcctLog(SAcctMonitorObj *pMon) { +void monSaveAcctLog(SAcctMonitorObj *pMon) { if (tsMonitor.state != MON_STATE_INITED) return; char sql[1024] = {0}; @@ -382,11 +382,11 @@ void monitorSaveAcctLog(SAcctMonitorObj *pMon) { pMon->totalConns, pMon->maxConns, pMon->accessState); - mnDebug("save account info, sql:%s", sql); - taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "account info"); + monDebug("save account info, sql:%s", sql); + taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "account info"); } -void monitorSaveLog(int32_t level, const char *const format, ...) { +void monSaveLog(int32_t level, const char *const format, ...) { if (tsMonitor.state != MON_STATE_INITED) return; va_list argpointer; @@ -403,13 +403,13 @@ void monitorSaveLog(int32_t level, const char *const format, ...) { len += sprintf(sql + len, "', '%s')", tsLocalEp); sql[len++] = 0; - mnDebug("save log, sql: %s", sql); - taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "log"); + monDebug("save log, sql: %s", sql); + taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "log"); } -void monitorExecuteSQL(char *sql) { +void monExecuteSQL(char *sql) { if (tsMonitor.state != MON_STATE_INITED) return; - mnDebug("execute sql:%s", sql); - taos_query_a(tsMonitor.conn, sql, montiorExecSqlCb, "sql"); + monDebug("execute sql:%s", sql); + taos_query_a(tsMonitor.conn, sql, monExecSqlCb, "sql"); } diff --git a/tests/script/general/alter/dnode.sim b/tests/script/general/alter/dnode.sim index 20ce879979..73a095ec05 100644 --- a/tests/script/general/alter/dnode.sim +++ b/tests/script/general/alter/dnode.sim @@ -24,7 +24,7 @@ sql alter dnode 1 debugFlag 135 sql alter dnode 1 debugFlag 131 sql alter dnode 1 monitor 0 sql alter dnode 1 debugFlag 135 -sql alter dnode 1 monitorDebugFlag 135 +sql alter dnode 1 monDebugFlag 135 sql alter dnode 1 vDebugFlag 135 sql alter dnode 1 mDebugFlag 135 sql alter dnode 1 cDebugFlag 135 @@ -44,15 +44,15 @@ sql_error alter dnode 2 tmrDebugFlag 135 print ======== step3 sql_error alter $hostname1 debugFlag 135 -sql_error alter $hostname1 monitorDebugFlag 135 +sql_error alter $hostname1 monDebugFlag 135 sql_error alter $hostname1 vDebugFlag 135 sql_error alter $hostname1 mDebugFlag 135 sql_error alter dnode $hostname2 debugFlag 135 -sql_error alter dnode $hostname2 monitorDebugFlag 135 +sql_error alter dnode $hostname2 monDebugFlag 135 sql_error alter dnode $hostname2 vDebugFlag 135 sql_error alter dnode $hostname2 mDebugFlag 135 sql alter dnode $hostname1 debugFlag 135 -sql alter dnode $hostname1 monitorDebugFlag 135 +sql alter dnode $hostname1 monDebugFlag 135 sql alter dnode $hostname1 vDebugFlag 135 sql alter dnode $hostname1 tmrDebugFlag 131 diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index e26778e86b..cd2f3772eb 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -120,7 +120,7 @@ echo "cDebugFlag 143" >> $TAOS_CFG echo "jnidebugFlag 143" >> $TAOS_CFG echo "odbcdebugFlag 143" >> $TAOS_CFG echo "httpDebugFlag 143" >> $TAOS_CFG -echo "monitorDebugFlag 143" >> $TAOS_CFG +echo "monDebugFlag 143" >> $TAOS_CFG echo "mqttDebugFlag 143" >> $TAOS_CFG echo "qdebugFlag 143" >> $TAOS_CFG echo "rpcDebugFlag 143" >> $TAOS_CFG From 5ab5299316eaed66ffe5e0fd789700bc6100722c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Dec 2020 18:10:46 +0800 Subject: [PATCH 161/188] minor changes --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 110e306ce7..e002592de1 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -5103,7 +5103,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { const int tokenDebugFlagEnd = 20; const SDNodeDynConfOption cfgOptions[] = { {"resetLog", 8}, {"resetQueryCache", 15}, {"balance", 7}, {"monitor", 7}, - {"debugFlag", 9}, {"monDebugFlag", 11}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, + {"debugFlag", 9}, {"monDebugFlag", 12}, {"vDebugFlag", 10}, {"mDebugFlag", 10}, {"cDebugFlag", 10}, {"httpDebugFlag", 13}, {"qDebugflag", 10}, {"sdbDebugFlag", 12}, {"uDebugFlag", 10}, {"tsdbDebugFlag", 13}, {"sDebugflag", 10}, {"rpcDebugFlag", 12}, {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, From 0fe731af0fccb4eb1cd31865d19b591a81a8f0cf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 22:01:55 +0800 Subject: [PATCH 162/188] [TD-2325]: enable configure if 50% cpu will be used in query processing. --- packaging/cfg/taos.cfg | 5 ++- src/common/inc/tglobal.h | 1 + src/common/src/tglobal.c | 15 +++++++- src/util/src/tconfig.c | 2 -- src/vnode/src/vnodeRead.c | 75 ++++++++++++++++++++------------------- 5 files changed, 57 insertions(+), 41 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 7662d49280..014ebc37d2 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -251,7 +251,7 @@ # cqDebugFlag 131 # enable/disable recording the SQL in taos client -# tscEnableRecordSql 0 +# enableRecordSql 0 # generate core file when service crash # enableCoreFile 1 @@ -264,3 +264,6 @@ # enable/disable stream (continuous query) # stream 1 + +# only 50% CPU resources will be used in query processing +# halfCoresForQuery 0 diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 6e4274b358..10bcfcb82b 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -56,6 +56,7 @@ extern char tsTempDir[]; //query buffer management extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing +extern int32_t tsHalfCoresForQuery; // only 50% will be used in query processing // client extern int32_t tsTableMetaKeepTimer; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 18aa0ae6e3..026cd67a81 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -107,6 +107,9 @@ int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance // positive value (in MB) int32_t tsQueryBufferSize = -1; +// only 50% cpu will be used in query processing in dnode +int32_t tsHalfCoresForQuery = 0; + // db parameters int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS; @@ -884,6 +887,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); + cfg.option = "halfCoresForQuery"; + cfg.ptr = &tsHalfCoresForQuery; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = 0; + cfg.maxValue = 1; + cfg.ptrLength = 1; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + // locale & charset cfg.option = "timezone"; cfg.ptr = tsTimezone; @@ -1290,7 +1303,7 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); - cfg.option = "tscEnableRecordSql"; + cfg.option = "enableRecordSql"; cfg.ptr = &tsTscEnableRecordSql; cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; diff --git a/src/util/src/tconfig.c b/src/util/src/tconfig.c index e89dea5a24..173de294cf 100644 --- a/src/util/src/tconfig.c +++ b/src/util/src/tconfig.c @@ -19,9 +19,7 @@ #include "taoserror.h" #include "tconfig.h" #include "tglobal.h" -#include "tkey.h" #include "tulog.h" -#include "tsocket.h" #include "tsystem.h" #include "tutil.h" diff --git a/src/vnode/src/vnodeRead.c b/src/vnode/src/vnodeRead.c index 5ef79cfbf0..ed6d29505f 100644 --- a/src/vnode/src/vnodeRead.c +++ b/src/vnode/src/vnodeRead.c @@ -275,41 +275,40 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { vDebug("vgId:%d, QInfo:%p, dnode continues to exec query", pVnode->vgId, *qhandle); - -#if _NON_BLOCKING_RETRIEVE - bool freehandle = false; - bool buildRes = qTableQuery(*qhandle); // do execute query - - // build query rsp, the retrieve request has reached here already - if (buildRes) { - // update the connection info according to the retrieve connection - pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle); - assert(pRead->rpcHandle != NULL); - - vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, - pRead->rpcHandle); - - // set the real rsp error code - pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle); - - // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client - code = TSDB_CODE_QRY_HAS_RSP; + // In the retrieve blocking model, only 50% CPU will be used in query processing + if (tsHalfCoresForQuery) { + qTableQuery(*qhandle); // do execute query + qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); } else { - void* h1 = qGetResultRetrieveMsg(*qhandle); - assert(h1 == NULL); + bool freehandle = false; + bool buildRes = qTableQuery(*qhandle); // do execute query - freehandle = qQueryCompleted(*qhandle); - } + // build query rsp, the retrieve request has reached here already + if (buildRes) { + // update the connection info according to the retrieve connection + pRead->rpcHandle = qGetResultRetrieveMsg(*qhandle); + assert(pRead->rpcHandle != NULL); - // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle. - // If the building of result is not required, simply free it. Otherwise, mandatorily free the qhandle - if (freehandle || (!buildRes)) { - qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); + vDebug("vgId:%d, QInfo:%p, start to build retrieval rsp after query paused, %p", pVnode->vgId, *qhandle, + pRead->rpcHandle); + + // set the real rsp error code + pRead->code = vnodeDumpQueryResult(&pRead->rspRet, pVnode, qhandle, &freehandle, pRead->rpcHandle); + + // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client + code = TSDB_CODE_QRY_HAS_RSP; + } else { + void *h1 = qGetResultRetrieveMsg(*qhandle); + assert(h1 == NULL); + freehandle = qQueryCompleted(*qhandle); + } + + // NOTE: if the qhandle is not put into vread queue or query is completed, free the qhandle. + // If the building of result is not required, simply free it. Otherwise, mandatorily free the qhandle + if (freehandle || (!buildRes)) { + qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, freehandle); + } } -#else - qTableQuery(*qhandle); // do execute query - qReleaseQInfo(pVnode->qMgmt, (void **)&qhandle, false); -#endif } return code; @@ -375,14 +374,16 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SVReadMsg *pRead) { freeHandle = true; } else { // result is not ready, return immediately assert(buildRes == true); -#if _NON_BLOCKING_RETRIEVE - if (!buildRes) { - assert(pRead->rpcHandle != NULL); - qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); - return TSDB_CODE_QRY_NOT_READY; + // Only effects in the non-blocking model + if (!tsHalfCoresForQuery) { + if (!buildRes) { + assert(pRead->rpcHandle != NULL); + + qReleaseQInfo(pVnode->qMgmt, (void **)&handle, false); + return TSDB_CODE_QRY_NOT_READY; + } } -#endif // ahandle is the sqlObj pointer code = vnodeDumpQueryResult(pRet, pVnode, handle, &freeHandle, pRead->rpcHandle); From 12a6fb04990d712badf97fb402556b440bc54c25 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 3 Dec 2020 22:48:08 +0800 Subject: [PATCH 163/188] crash while quit balance module --- src/balance/src/bnThread.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/balance/src/bnThread.c b/src/balance/src/bnThread.c index a11bc61b01..bf046a9fae 100644 --- a/src/balance/src/bnThread.c +++ b/src/balance/src/bnThread.c @@ -36,6 +36,7 @@ static void *bnThreadFunc(void *arg) { pthread_mutex_unlock(&(tsBnThread.mutex)); } + mDebug("balance thread is stopped"); return NULL; } @@ -47,7 +48,7 @@ int32_t bnInitThread() { pthread_attr_t thattr; pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); int32_t ret = pthread_create(&tsBnThread.thread, &thattr, bnThreadFunc, NULL); pthread_attr_destroy(&thattr); @@ -81,6 +82,8 @@ void bnCleanupThread() { } static void bnPostSignal() { + if (tsBnThread.stop) return; + pthread_mutex_lock(&tsBnThread.mutex); pthread_cond_signal(&tsBnThread.cond); pthread_mutex_unlock(&(tsBnThread.mutex)); From 7090b3feb0bea56433e4d10565b18c74bf1f5e7b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Dec 2020 22:52:17 +0800 Subject: [PATCH 164/188] [TD-2169]: fix memory leaks in group by query processing. --- src/query/src/qExecutor.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0c07149e8e..bcce7c50a8 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1372,8 +1372,12 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - pResultRow->key = malloc(varDataTLen(pData)); - varDataCopy(pResultRow->key, pData); + if (pResultRow->key == NULL) { + pResultRow->key = malloc(varDataTLen(pData)); + varDataCopy(pResultRow->key, pData); + } else { + assert(memcmp(pResultRow->key, pData, varDataTLen(pData)) == 0); + } } else { pResultRow->win.skey = v; pResultRow->win.ekey = v; From 1be5c3f88e75d07186c48ccbdfc9d7ecb8701d8d Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 3 Dec 2020 21:17:50 +0000 Subject: [PATCH 165/188] TD-2257 --- src/client/src/tscProfile.c | 2 ++ src/client/src/tscSystem.c | 2 +- src/util/src/tref.c | 19 ++++++++++--------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index acc5acd786..18fc79c474 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -39,6 +39,7 @@ void tscInitConnCb(void *param, TAOS_RES *result, int code) { tscSlowQueryConnInitialized = true; tscSaveSlowQueryFp(sql, NULL); } + taos_free_result(result); } void tscAddIntoSqlList(SSqlObj *pSql) { @@ -69,6 +70,7 @@ void tscSaveSlowQueryFpCb(void *param, TAOS_RES *result, int code) { } else { tscDebug("success to save slow query, code:%d", code); } + taos_free_result(result); } void tscSaveSlowQueryFp(void *handle, void *tmrId) { diff --git a/src/client/src/tscSystem.c b/src/client/src/tscSystem.c index 91eabda78b..6aea03d214 100644 --- a/src/client/src/tscSystem.c +++ b/src/client/src/tscSystem.c @@ -144,7 +144,7 @@ void taos_init_imp(void) { int64_t refreshTime = 10; // 10 seconds by default if (tscMetaCache == NULL) { tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, refreshTime, false, tscFreeTableMetaHelper, "tableMeta"); - tscObjRef = taosOpenRef(4096, tscFreeRegisteredSqlObj); + tscObjRef = taosOpenRef(40960, tscFreeRegisteredSqlObj); } tscRefId = taosOpenRef(200, tscCloseTscObj); diff --git a/src/util/src/tref.c b/src/util/src/tref.c index 4c1a87c960..1f83abcb84 100644 --- a/src/util/src/tref.c +++ b/src/util/src/tref.c @@ -329,7 +329,7 @@ void *taosIterateRef(int rsetId, int64_t rid) { pNode->count++; // acquire it newP = pNode->p; taosUnlockList(pSet->lockedBy+hash); - uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is returned", rsetId, newP, rid); } else { uTrace("rsetId:%d the list is over", rsetId); } @@ -423,24 +423,25 @@ static int taosDecRefCount(int rsetId, int64_t rid, int remove) { if (pNode->next) { pNode->next->prev = pNode->prev; } - - (*pSet->fp)(pNode->p); - - uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); - free(pNode); released = 1; } else { - uTrace("rsetId:%d p:%p rid:%" PRId64 " is released, count:%d", rsetId, pNode->p, rid, pNode->count); + uTrace("rsetId:%d p:%p rid:%" PRId64 " is released", rsetId, pNode->p, rid); } } else { - uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); + uTrace("rsetId:%d rid:%" PRId64 " is not there, failed to release/remove", rsetId, rid); terrno = TSDB_CODE_REF_NOT_EXIST; code = -1; } taosUnlockList(pSet->lockedBy+hash); - if (released) taosDecRsetCount(pSet); + if (released) { + uTrace("rsetId:%d p:%p rid:%" PRId64 " is removed, count:%d, free mem: %p", rsetId, pNode->p, rid, pSet->count, pNode); + (*pSet->fp)(pNode->p); + free(pNode); + + taosDecRsetCount(pSet); + } return code; } From 0a0080903baf427b20769a4247176607ddd3936c Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 4 Dec 2020 01:45:53 +0000 Subject: [PATCH 166/188] [TD-2313]: ensure length of nchar cols not exceed limit --- src/query/src/qParserImpl.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 5cd9d3c77a..7d71d9f7f1 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -405,14 +405,29 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { if (type->type == 0) { pField->bytes = 0; } else { - pField->bytes = (int16_t)(-(int32_t)type->type * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + int32_t bytes = -(int32_t)(type->type); + if (bytes > (TSDB_MAX_NCHAR_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { + // we have to postpone reporting the error because it cannot be done here + // as pField->bytes is int16_t, use 'TSDB_MAX_NCHAR_LEN + 1' to avoid overflow + bytes = TSDB_MAX_NCHAR_LEN + 1; + } else { + bytes = bytes * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; + } + pField->bytes = (int16_t)bytes; } } else if (i == TSDB_DATA_TYPE_BINARY) { /* for binary, the TOKENTYPE is the length of binary */ if (type->type == 0) { pField->bytes = 0; } else { - pField->bytes = (int16_t) (-(int32_t) type->type + VARSTR_HEADER_SIZE); + int32_t bytes = -(int32_t)(type->type); + if (bytes > TSDB_MAX_BINARY_LEN - VARSTR_HEADER_SIZE) { + // refer comment for NCHAR above + bytes = TSDB_MAX_BINARY_LEN + 1; + } else { + bytes += VARSTR_HEADER_SIZE; + } + pField->bytes = (int16_t)bytes; } } break; From 573fcb4e9c5e0cbed4139506c36b5af50f41ec52 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Dec 2020 10:07:13 +0800 Subject: [PATCH 167/188] TD-2299 --- src/dnode/src/dnodePeer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dnode/src/dnodePeer.c b/src/dnode/src/dnodePeer.c index 6bf22cee4e..6b5b28622b 100644 --- a/src/dnode/src/dnodePeer.c +++ b/src/dnode/src/dnodePeer.c @@ -182,6 +182,8 @@ void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg) { void dnodeSendMsgToMnodeRecv(SRpcMsg *rpcMsg, SRpcMsg *rpcRsp) { SRpcEpSet epSet = {0}; dnodeGetEpSetForPeer(&epSet); + + assert(tsClientRpc != 0); rpcSendRecv(tsClientRpc, &epSet, rpcMsg, rpcRsp); } From 1993ad245fb7adc8f3af3671c8917f3d776355ea Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 4 Dec 2020 11:16:42 +0800 Subject: [PATCH 168/188] [TD-785]Improve the robustness of python connector --- src/connector/python/linux/python2/taos/cursor.py | 4 ++-- src/connector/python/linux/python3/taos/cursor.py | 4 ++-- src/connector/python/windows/python2/taos/cursor.py | 4 ++-- src/connector/python/windows/python3/taos/cursor.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index eee5e39488..82a01be671 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -26,7 +26,7 @@ class TDengineCursor(object): """ def __init__(self, connection=None): - self._description = None + self._description = [] self._rowcount = -1 self._connection = None self._result = None @@ -234,7 +234,7 @@ class TDengineCursor(object): def _reset_result(self): """Reset the result to unused version. """ - self._description = None + self._description = [] self._rowcount = -1 if self._result is not None: CTaosInterface.freeResult(self._result) diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index d8184668c8..0ce20f0eda 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -27,7 +27,7 @@ class TDengineCursor(object): """ def __init__(self, connection=None): - self._description = None + self._description = [] self._rowcount = -1 self._connection = None self._result = None @@ -242,7 +242,7 @@ class TDengineCursor(object): def _reset_result(self): """Reset the result to unused version. """ - self._description = None + self._description = [] self._rowcount = -1 if self._result is not None: CTaosInterface.freeResult(self._result) diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index 71651afee1..f6fde2619b 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -27,7 +27,7 @@ class TDengineCursor(object): """ def __init__(self, connection=None): - self._description = None + self._description = [] self._rowcount = -1 self._connection = None self._result = None @@ -193,7 +193,7 @@ class TDengineCursor(object): def _reset_result(self): """Reset the result to unused version. """ - self._description = None + self._description = [] self._rowcount = -1 if self._result is not None: CTaosInterface.freeResult(self._result) diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index b813bba357..db66b99d3b 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -28,7 +28,7 @@ class TDengineCursor(object): """ def __init__(self, connection=None): - self._description = None + self._description = [] self._rowcount = -1 self._connection = None self._result = None @@ -194,7 +194,7 @@ class TDengineCursor(object): def _reset_result(self): """Reset the result to unused version. """ - self._description = None + self._description = [] self._rowcount = -1 if self._result is not None: CTaosInterface.freeResult(self._result) From e4dce56692fdec43a09267008eaeb4b7bbbb00c7 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 4 Dec 2020 11:30:21 +0800 Subject: [PATCH 169/188] [TD-785]Improve the robustness of python connector --- src/connector/python/linux/python2/setup.py | 2 +- src/connector/python/linux/python2/taos/__init__.py | 2 +- src/connector/python/linux/python3/setup.py | 2 +- src/connector/python/linux/python3/taos/__init__.py | 2 +- src/connector/python/windows/python2/setup.py | 2 +- src/connector/python/windows/python2/taos/__init__.py | 2 +- src/connector/python/windows/python3/setup.py | 2 +- src/connector/python/windows/python3/taos/__init__.py | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index b3daa98bdc..92a931b504 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.2", + version="2.0.3", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/__init__.py b/src/connector/python/linux/python2/taos/__init__.py index 5d21671eed..62e0536b6f 100644 --- a/src/connector/python/linux/python2/taos/__init__.py +++ b/src/connector/python/linux/python2/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.2' +apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py index f49ebe2b6d..655a12ad13 100644 --- a/src/connector/python/linux/python3/setup.py +++ b/src/connector/python/linux/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.2", + version="2.0.3", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python3/taos/__init__.py b/src/connector/python/linux/python3/taos/__init__.py index c46131eeaf..8cf095ea68 100644 --- a/src/connector/python/linux/python3/taos/__init__.py +++ b/src/connector/python/linux/python3/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.2' +apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py index 34cb7a04d7..5ddbe83011 100644 --- a/src/connector/python/windows/python2/setup.py +++ b/src/connector/python/windows/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.2", + version="2.0.3", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python2/taos/__init__.py b/src/connector/python/windows/python2/taos/__init__.py index 5d21671eed..62e0536b6f 100644 --- a/src/connector/python/windows/python2/taos/__init__.py +++ b/src/connector/python/windows/python2/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.2' +apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py index c09644d330..ffed304c85 100644 --- a/src/connector/python/windows/python3/setup.py +++ b/src/connector/python/windows/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.2", + version="2.0.3", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python3/taos/__init__.py b/src/connector/python/windows/python3/taos/__init__.py index 0708b9544f..c6dd929a6a 100644 --- a/src/connector/python/windows/python3/taos/__init__.py +++ b/src/connector/python/windows/python3/taos/__init__.py @@ -3,7 +3,7 @@ from .connection import TDengineConnection from .cursor import TDengineCursor # Globals -apilevel = '2.0.2' +apilevel = '2.0.3' threadsafety = 0 paramstyle = 'pyformat' From 1ae57bbf222c34768e09c9d0c879b5398128a19e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 4 Dec 2020 03:36:25 +0000 Subject: [PATCH 170/188] fix TD-2322 --- src/tsdb/src/tsdbMemTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 4c51608b5d..999e2deb41 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -786,9 +786,9 @@ static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pT if (tsdbGetTableSchemaImpl(pTable, false, false, pBlock->sversion) == NULL) { tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo), pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable)); + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + return -1; } - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; } return 0; From b8f38e0568a51814febb2bb3f8ae70b5484a51c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 12:47:37 +0800 Subject: [PATCH 171/188] [TD-2281]: filter unsupported mixed up query functions. --- src/client/src/tscFunctionImpl.c | 8 ++++---- src/client/src/tscSQLParser.c | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 56b7f052f7..594a4c39b5 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -4540,11 +4540,11 @@ static void sumrate_finalizer(SQLFunctionCtx *pCtx) { * */ int32_t functionCompatList[] = { - // count, sum, avg, min, max, stddev, percentile, apercentile, first, last - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row, top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z + // count, sum, avg, min, max, stddev, percentile, apercentile, first, last + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate + // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, // sum_rate, sum_irate, avg_rate, avg_irate 1, 1, 1, 1, diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index abbe26204d..08b2a7db4e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2835,6 +2835,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { if (functionCompatList[functionId] != factor) { return false; + } else { + if (factor == -1) { // two functions with the same -1 flag + return false; + } } if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { From 94756ccd947e62aa26166d899d0ac4620f517f11 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Dec 2020 13:07:23 +0800 Subject: [PATCH 172/188] TD-2331 --- src/dnode/inc/dnodeMInfos.h | 6 +- src/dnode/src/dnodeMInfos.c | 58 +++++++------- src/dnode/src/dnodeMgmt.c | 4 +- src/dnode/src/dnodeModule.c | 6 +- src/inc/dnode.h | 2 +- src/inc/taosmsg.h | 14 ++-- src/mnode/inc/mnodeMnode.h | 2 +- src/mnode/inc/mnodeSdb.h | 1 + src/mnode/src/mnodeMnode.c | 146 +++++++++++++++++++++--------------- src/mnode/src/mnodeSdb.c | 25 ++++-- src/sync/src/syncMain.c | 2 +- 11 files changed, 151 insertions(+), 115 deletions(-) diff --git a/src/dnode/inc/dnodeMInfos.h b/src/dnode/inc/dnodeMInfos.h index 9c3c85c47e..2c3eef5d5d 100644 --- a/src/dnode/inc/dnodeMInfos.h +++ b/src/dnode/inc/dnodeMInfos.h @@ -24,9 +24,9 @@ extern "C" { int32_t dnodeInitMInfos(); void dnodeCleanupMInfos(); -void dnodeUpdateMInfos(SMnodeInfos *minfos); -void dnodeUpdateEpSetForPeer(SRpcEpSet *epSet); -void dnodeGetMInfos(SMnodeInfos *minfos); +void dnodeUpdateMInfos(SMInfos *pMinfos); +void dnodeUpdateEpSetForPeer(SRpcEpSet *pEpSet); +void dnodeGetMInfos(SMInfos *pMinfos); bool dnodeIsMasterEp(char *ep); #ifdef __cplusplus diff --git a/src/dnode/src/dnodeMInfos.c b/src/dnode/src/dnodeMInfos.c index cefe44aebe..162de2243e 100644 --- a/src/dnode/src/dnodeMInfos.c +++ b/src/dnode/src/dnodeMInfos.c @@ -22,12 +22,12 @@ #include "dnodeInt.h" #include "dnodeMInfos.h" -static SMnodeInfos tsMInfos; -static SRpcEpSet tsMEpSet; +static SMInfos tsMInfos; +static SRpcEpSet tsMEpSet; static pthread_mutex_t tsMInfosMutex; -static void dnodeResetMInfos(SMnodeInfos *minfos); -static void dnodePrintMInfos(SMnodeInfos *minfos); +static void dnodeResetMInfos(SMInfos *minfos); +static void dnodePrintMInfos(SMInfos *minfos); static int32_t dnodeReadMInfos(); static int32_t dnodeWriteMInfos(); @@ -44,14 +44,14 @@ int32_t dnodeInitMInfos() { void dnodeCleanupMInfos() { pthread_mutex_destroy(&tsMInfosMutex); } -void dnodeUpdateMInfos(SMnodeInfos *minfos) { - if (minfos->mnodeNum <= 0 || minfos->mnodeNum > 3) { - dError("invalid mnode infos, mnodeNum:%d", minfos->mnodeNum); +void dnodeUpdateMInfos(SMInfos *pMinfos) { + if (pMinfos->mnodeNum <= 0 || pMinfos->mnodeNum > 3) { + dError("invalid mnode infos, mnodeNum:%d", pMinfos->mnodeNum); return; } - for (int32_t i = 0; i < minfos->mnodeNum; ++i) { - SMnodeInfo *minfo = &minfos->mnodeInfos[i]; + for (int32_t i = 0; i < pMinfos->mnodeNum; ++i) { + SMInfo *minfo = &pMinfos->mnodeInfos[i]; minfo->mnodeId = htonl(minfo->mnodeId); if (minfo->mnodeId <= 0 || strlen(minfo->mnodeEp) <= 5) { dError("invalid mnode info:%d, mnodeId:%d mnodeEp:%s", i, minfo->mnodeId, minfo->mnodeEp); @@ -60,14 +60,14 @@ void dnodeUpdateMInfos(SMnodeInfos *minfos) { } pthread_mutex_lock(&tsMInfosMutex); - if (minfos->mnodeNum != tsMInfos.mnodeNum) { - dnodeResetMInfos(minfos); + if (pMinfos->mnodeNum != tsMInfos.mnodeNum) { + dnodeResetMInfos(pMinfos); dnodeWriteMInfos(); sdbUpdateAsync(); } else { - int32_t size = sizeof(SMnodeInfos); - if (memcmp(minfos, &tsMInfos, size) != 0) { - dnodeResetMInfos(minfos); + int32_t size = sizeof(SMInfos); + if (memcmp(pMinfos, &tsMInfos, size) != 0) { + dnodeResetMInfos(pMinfos); dnodeWriteMInfos(); sdbUpdateAsync(); } @@ -99,11 +99,11 @@ bool dnodeIsMasterEp(char *ep) { return isMaster; } -void dnodeGetMInfos(SMnodeInfos *minfos) { +void dnodeGetMInfos(SMInfos *pMinfos) { pthread_mutex_lock(&tsMInfosMutex); - memcpy(minfos, &tsMInfos, sizeof(SMnodeInfos)); + memcpy(pMinfos, &tsMInfos, sizeof(SMInfos)); for (int32_t i = 0; i < tsMInfos.mnodeNum; ++i) { - minfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId); + pMinfos->mnodeInfos[i].mnodeId = htonl(tsMInfos.mnodeInfos[i].mnodeId); } pthread_mutex_unlock(&tsMInfosMutex); } @@ -123,15 +123,15 @@ void dnodeGetEpSetForShell(SRpcEpSet *epSet) { pthread_mutex_unlock(&tsMInfosMutex); } -static void dnodePrintMInfos(SMnodeInfos *minfos) { - dInfo("print mnode infos, mnodeNum:%d inUse:%d", minfos->mnodeNum, minfos->inUse); - for (int32_t i = 0; i < minfos->mnodeNum; i++) { - dInfo("mnode index:%d, %s", minfos->mnodeInfos[i].mnodeId, minfos->mnodeInfos[i].mnodeEp); +static void dnodePrintMInfos(SMInfos *pMinfos) { + dInfo("print minfos, mnodeNum:%d inUse:%d", pMinfos->mnodeNum, pMinfos->inUse); + for (int32_t i = 0; i < pMinfos->mnodeNum; i++) { + dInfo("mnode index:%d, %s", pMinfos->mnodeInfos[i].mnodeId, pMinfos->mnodeInfos[i].mnodeEp); } } -static void dnodeResetMInfos(SMnodeInfos *minfos) { - if (minfos == NULL) { +static void dnodeResetMInfos(SMInfos *pMinfos) { + if (pMinfos == NULL) { tsMEpSet.numOfEps = 1; taosGetFqdnPortFromEp(tsFirst, tsMEpSet.fqdn[0], &tsMEpSet.port[0]); @@ -142,10 +142,10 @@ static void dnodeResetMInfos(SMnodeInfos *minfos) { return; } - if (minfos->mnodeNum == 0) return; + if (pMinfos->mnodeNum == 0) return; - int32_t size = sizeof(SMnodeInfos); - memcpy(&tsMInfos, minfos, size); + int32_t size = sizeof(SMInfos); + memcpy(&tsMInfos, pMinfos, size); tsMEpSet.inUse = tsMInfos.inUse; tsMEpSet.numOfEps = tsMInfos.mnodeNum; @@ -153,7 +153,7 @@ static void dnodeResetMInfos(SMnodeInfos *minfos) { taosGetFqdnPortFromEp(tsMInfos.mnodeInfos[i].mnodeEp, tsMEpSet.fqdn[i], &tsMEpSet.port[i]); } - dnodePrintMInfos(minfos); + dnodePrintMInfos(pMinfos); } static int32_t dnodeReadMInfos() { @@ -162,7 +162,7 @@ static int32_t dnodeReadMInfos() { char * content = calloc(1, maxLen + 1); cJSON * root = NULL; FILE * fp = NULL; - SMnodeInfos minfos = {0}; + SMInfos minfos = {0}; char file[TSDB_FILENAME_LEN + 20] = {0}; sprintf(file, "%s/mnodeEpSet.json", tsDnodeDir); @@ -241,7 +241,7 @@ PARSE_MINFOS_OVER: terrno = 0; for (int32_t i = 0; i < minfos.mnodeNum; ++i) { - SMnodeInfo *mInfo = &minfos.mnodeInfos[i]; + SMInfo *mInfo = &minfos.mnodeInfos[i]; dnodeUpdateEp(mInfo->mnodeId, mInfo->mnodeEp, NULL, NULL); } dnodeResetMInfos(&minfos); diff --git a/src/dnode/src/dnodeMgmt.c b/src/dnode/src/dnodeMgmt.c index 5c01f64716..15378c77c1 100644 --- a/src/dnode/src/dnodeMgmt.c +++ b/src/dnode/src/dnodeMgmt.c @@ -472,8 +472,8 @@ static void dnodeProcessStatusRsp(SRpcMsg *pMsg) { } SStatusRsp *pStatusRsp = pMsg->pCont; - SMnodeInfos *minfos = &pStatusRsp->mnodes; - dnodeUpdateMInfos(minfos); + SMInfos *pMinfos = &pStatusRsp->mnodes; + dnodeUpdateMInfos(pMinfos); SDnodeCfg *pCfg = &pStatusRsp->dnodeCfg; pCfg->numOfVnodes = htonl(pCfg->numOfVnodes); diff --git a/src/dnode/src/dnodeModule.c b/src/dnode/src/dnodeModule.c index 7faa3c8913..f664618f51 100644 --- a/src/dnode/src/dnodeModule.c +++ b/src/dnode/src/dnodeModule.c @@ -147,8 +147,8 @@ void dnodeProcessModuleStatus(uint32_t moduleStatus) { } } -bool dnodeStartMnode(SMnodeInfos *minfos) { - SMnodeInfos *mnodes = minfos; +bool dnodeStartMnode(SMInfos *pMinfos) { + SMInfos *pMnodes = pMinfos; if (tsModuleStatus & (1 << TSDB_MOD_MNODE)) { dDebug("mnode module is already started, module status:%d", tsModuleStatus); @@ -159,7 +159,7 @@ bool dnodeStartMnode(SMnodeInfos *minfos) { dInfo("start mnode module, module status:%d, new status:%d", tsModuleStatus, moduleStatus); dnodeProcessModuleStatus(moduleStatus); - sdbUpdateSync(mnodes); + sdbUpdateSync(pMnodes); return true; } diff --git a/src/inc/dnode.h b/src/inc/dnode.h index eef4490800..1efaa4a24b 100644 --- a/src/inc/dnode.h +++ b/src/inc/dnode.h @@ -45,7 +45,7 @@ void dnodeGetEpSetForShell(SRpcEpSet *epSet); int32_t dnodeGetDnodeId(); void dnodeUpdateEp(int32_t dnodeId, char *ep, char *fqdn, uint16_t *port); bool dnodeCheckEpChanged(int32_t dnodeId, char *epstr); -bool dnodeStartMnode(SMnodeInfos *minfos); +bool dnodeStartMnode(SMInfos *pMinfos); void dnodeAddClientRspHandle(uint8_t msgType, void (*fp)(SRpcMsg *rpcMsg)); void dnodeSendMsgToDnode(SRpcEpSet *epSet, SRpcMsg *rpcMsg); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index b4d3bec958..e8e3029244 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -591,13 +591,13 @@ typedef struct { typedef struct { int32_t mnodeId; char mnodeEp[TSDB_EP_LEN]; -} SMnodeInfo; +} SMInfo; typedef struct { - int8_t inUse; - int8_t mnodeNum; - SMnodeInfo mnodeInfos[TSDB_MAX_REPLICA]; -} SMnodeInfos; + int8_t inUse; + int8_t mnodeNum; + SMInfo mnodeInfos[TSDB_MAX_REPLICA]; +} SMInfos; typedef struct { int32_t numOfMnodes; // tsNumOfMnodes @@ -632,7 +632,7 @@ typedef struct { } SStatusMsg; typedef struct { - SMnodeInfos mnodes; + SMInfos mnodes; SDnodeCfg dnodeCfg; SVgroupAccess vgAccess[]; } SStatusRsp; @@ -761,7 +761,7 @@ typedef struct { typedef struct { int32_t dnodeId; char dnodeEp[TSDB_EP_LEN]; // end point, hostname:port - SMnodeInfos mnodes; + SMInfos mnodes; } SCreateMnodeMsg; typedef struct { diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index 10cbcebe22..93f2fa11ea 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -48,7 +48,7 @@ void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet); char* mnodeGetMnodeMasterEp(); void mnodeGetMnodeInfos(void *mnodes); -void mnodeUpdateMnodeEpSet(); +void mnodeUpdateMnodeEpSet(SMInfos *pMnodes); #ifdef __cplusplus } diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 90c4eac40a..31ea2da640 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -89,6 +89,7 @@ void* sdbGetTableByRid(int64_t rid); bool sdbIsMaster(); bool sdbIsServing(); void sdbUpdateMnodeRoles(); +int32_t sdbGetReplicaNum(); int32_t sdbInsertRow(SSdbRow *pRow); int32_t sdbDeleteRow(SSdbRow *pRow); diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index d20d51f82b..ba0b5a1865 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -34,14 +34,14 @@ #include "mnodeUser.h" #include "mnodeVgroup.h" -int64_t tsMnodeRid = -1; -static void * tsMnodeSdb = NULL; -static int32_t tsMnodeUpdateSize = 0; -static SRpcEpSet tsMnodeEpSetForShell; -static SRpcEpSet tsMnodeEpSetForPeer; -static SMnodeInfos tsMnodeInfos; -static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); +int64_t tsMnodeRid = -1; +static void * tsMnodeSdb = NULL; +static int32_t tsMnodeUpdateSize = 0; +static SRpcEpSet tsMEpForShell; +static SRpcEpSet tsMEpForPeer; +static SMInfos tsMInfos; +static int32_t mnodeGetMnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +static int32_t mnodeRetrieveMnodes(SShowObj *pShow, char *data, int32_t rows, void *pConn); #if defined(LINUX) static pthread_rwlock_t tsMnodeLock; @@ -127,7 +127,7 @@ static int32_t mnodeMnodeActionRestored() { mnodeCancelGetNextMnode(pIter); } - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); return TSDB_CODE_SUCCESS; } @@ -199,93 +199,119 @@ void mnodeCancelGetNextMnode(void *pIter) { sdbFreeIter(tsMnodeSdb, pIter); } -void mnodeUpdateMnodeEpSet() { - mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum()); +void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { + bool set = false; + SMInfos mInfos = {0}; + mInfo("vgId:1, update mnodes epSet, numOfEps:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos); + + if (pMinfos != NULL) { + set = true; + mInfos = *pMinfos; + } + else { + int32_t index = 0; + void * pIter = NULL; + while (1) { + SMnodeObj *pMnode = NULL; + pIter = mnodeGetNextMnode(pIter, &pMnode); + if (pMnode == NULL) break; + + SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); + if (pDnode != NULL) { + set = true; + mInfos.mnodeInfos[index].mnodeId = pMnode->mnodeId; + strcpy(mInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp); + if (pMnode->role == TAOS_SYNC_ROLE_MASTER) mInfos.inUse = index; + index++; + } else { + set = false; + } + + mnodeDecDnodeRef(pDnode); + mnodeDecMnodeRef(pMnode); + } + + mInfos.mnodeNum = index; + if (sdbGetReplicaNum() != mInfos.mnodeNum) { + set = false; + mDebug("vgId:1, mnodes info not synced, cfg:%d current:%d", sdbGetReplicaNum(), mInfos.mnodeNum); + } + } mnodeMnodeWrLock(); - memset(&tsMnodeEpSetForShell, 0, sizeof(SRpcEpSet)); - memset(&tsMnodeEpSetForPeer, 0, sizeof(SRpcEpSet)); - memset(&tsMnodeInfos, 0, sizeof(SMnodeInfos)); + if (set) { + memset(&tsMEpForShell, 0, sizeof(SRpcEpSet)); + memset(&tsMEpForPeer, 0, sizeof(SRpcEpSet)); + memcpy(&tsMInfos, &mInfos, sizeof(SMInfos)); + tsMEpForShell.inUse = tsMInfos.inUse; + tsMEpForPeer.inUse = tsMInfos.inUse; + tsMEpForShell.numOfEps = tsMInfos.mnodeNum; + tsMEpForPeer.numOfEps = tsMInfos.mnodeNum; - int32_t index = 0; - void * pIter = NULL; - while (1) { - SMnodeObj *pMnode = NULL; - pIter = mnodeGetNextMnode(pIter, &pMnode); - if (pMnode == NULL) break; + mInfo("vgId:1, mnodes epSet is set, num:%d inUse:%d", tsMInfos.mnodeNum, tsMInfos.inUse); + for (int index = 0; index < mInfos.mnodeNum; ++index) { + SMInfo *pInfo = &tsMInfos.mnodeInfos[index]; + taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForShell.fqdn[index], &tsMEpForShell.port[index]); + taosGetFqdnPortFromEp(pInfo->mnodeEp, tsMEpForPeer.fqdn[index], &tsMEpForPeer.port[index]); + tsMEpForPeer.port[index] = tsMEpForPeer.port[index] + TSDB_PORT_DNODEDNODE; - SDnodeObj *pDnode = mnodeGetDnode(pMnode->mnodeId); - if (pDnode != NULL) { - strcpy(tsMnodeEpSetForShell.fqdn[index], pDnode->dnodeFqdn); - tsMnodeEpSetForShell.port[index] = htons(pDnode->dnodePort); - mDebug("mnode:%d, for shell fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForShell.fqdn[index], htons(tsMnodeEpSetForShell.port[index])); + mInfo("vgId:1, mnode:%d, fqdn:%s shell:%u peer:%u", pInfo->mnodeId, tsMEpForShell.fqdn[index], + tsMEpForShell.port[index], tsMEpForPeer.port[index]); - strcpy(tsMnodeEpSetForPeer.fqdn[index], pDnode->dnodeFqdn); - tsMnodeEpSetForPeer.port[index] = htons(pDnode->dnodePort + TSDB_PORT_DNODEDNODE); - mDebug("mnode:%d, for peer fqdn:%s %d", pDnode->dnodeId, tsMnodeEpSetForPeer.fqdn[index], htons(tsMnodeEpSetForPeer.port[index])); - - tsMnodeInfos.mnodeInfos[index].mnodeId = htonl(pMnode->mnodeId); - strcpy(tsMnodeInfos.mnodeInfos[index].mnodeEp, pDnode->dnodeEp); - - if (pMnode->role == TAOS_SYNC_ROLE_MASTER) { - tsMnodeEpSetForShell.inUse = index; - tsMnodeEpSetForPeer.inUse = index; - tsMnodeInfos.inUse = index; - } - - mInfo("mnode:%d, ep:%s %s", pDnode->dnodeId, pDnode->dnodeEp, pMnode->role == TAOS_SYNC_ROLE_MASTER ? "master" : ""); - index++; + tsMEpForShell.port[index] = htons(tsMEpForShell.port[index]); + tsMEpForPeer.port[index] = htons(tsMEpForPeer.port[index]); + pInfo->mnodeId = htonl(pInfo->mnodeId); + } + } else { + mInfo("vgId:1, mnodes epSet not set, num:%d inUse:%d", tsMInfos.mnodeNum, tsMInfos.inUse); + for (int index = 0; index < tsMInfos.mnodeNum; ++index) { + mInfo("vgId:1, index:%d, ep:%s:%u", index, tsMEpForShell.fqdn[index], htons(tsMEpForShell.port[index])); } - - mnodeDecDnodeRef(pDnode); - mnodeDecMnodeRef(pMnode); } - tsMnodeInfos.mnodeNum = index; - tsMnodeEpSetForShell.numOfEps = index; - tsMnodeEpSetForPeer.numOfEps = index; - mnodeMnodeUnLock(); } void mnodeGetMnodeEpSetForPeer(SRpcEpSet *epSet) { mnodeMnodeRdLock(); - *epSet = tsMnodeEpSetForPeer; + *epSet = tsMEpForPeer; mnodeMnodeUnLock(); + mTrace("vgId:1, mnodes epSet for peer is returned, num:%d inUse:%d", tsMEpForPeer.numOfEps, tsMEpForPeer.inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort + TSDB_PORT_DNODEDNODE) { epSet->inUse = (i + 1) % epSet->numOfEps; - mTrace("mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); + mTrace("vgId:1, mnode:%d, for peer ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { - mTrace("mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); + mTrace("vgId:1, mpeer:%d, for peer ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); } } } void mnodeGetMnodeEpSetForShell(SRpcEpSet *epSet) { mnodeMnodeRdLock(); - *epSet = tsMnodeEpSetForShell; + *epSet = tsMEpForShell; mnodeMnodeUnLock(); + mTrace("vgId:1, mnodes epSet for shell is returned, num:%d inUse:%d", tsMEpForShell.numOfEps, tsMEpForShell.inUse); for (int32_t i = 0; i < epSet->numOfEps; ++i) { if (strcmp(epSet->fqdn[i], tsLocalFqdn) == 0 && htons(epSet->port[i]) == tsServerPort) { epSet->inUse = (i + 1) % epSet->numOfEps; - mTrace("mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); + mTrace("vgId:1, mnode:%d, for shell ep:%s:%u, set inUse to %d", i, epSet->fqdn[i], htons(epSet->port[i]), epSet->inUse); } else { - mTrace("mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); + mTrace("vgId:1, mnode:%d, for shell ep:%s:%u", i, epSet->fqdn[i], htons(epSet->port[i])); } } } char* mnodeGetMnodeMasterEp() { - return tsMnodeInfos.mnodeInfos[tsMnodeInfos.inUse].mnodeEp; + return tsMInfos.mnodeInfos[tsMInfos.inUse].mnodeEp; } -void mnodeGetMnodeInfos(void *mnodeInfos) { +void mnodeGetMnodeInfos(void *pMinfos) { mnodeMnodeRdLock(); - *(SMnodeInfos *)mnodeInfos = tsMnodeInfos; + *(SMInfos *)pMinfos = tsMInfos; mnodeMnodeUnLock(); } @@ -298,7 +324,7 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { } else { pCreate->dnodeId = htonl(dnodeId); tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); - pCreate->mnodes = tsMnodeInfos; + pCreate->mnodes = tsMInfos; bool found = false; for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) { if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) { @@ -336,7 +362,7 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) { mError("failed to create mnode, reason:%s", tstrerror(code)); } else { mDebug("mnode is created successfully"); - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); sdbUpdateAsync(); } @@ -380,7 +406,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId) { mnodeDecMnodeRef(pMnode); } - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); sdbUpdateAsync(); } @@ -400,7 +426,7 @@ int32_t mnodeDropMnode(int32_t dnodeId) { sdbDecRef(tsMnodeSdb, pMnode); - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); sdbUpdateAsync(); return code; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 2ef758baf1..a8cd595fc8 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -224,11 +224,13 @@ void sdbUpdateMnodeRoles() { sdbInfo("vgId:1, mnode:%d, role:%s", pMnode->mnodeId, syncRole[pMnode->role]); if (pMnode->mnodeId == dnodeGetDnodeId()) tsSdbMgmt.role = pMnode->role; mnodeDecMnodeRef(pMnode); + } else { + sdbDebug("vgId:1, mnode:%d not found", roles.nodeId[i]); } } mnodeUpdateClusterId(); - mnodeUpdateMnodeEpSet(); + mnodeUpdateMnodeEpSet(NULL); } static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) { @@ -308,18 +310,20 @@ void sdbUpdateAsync() { } void sdbUpdateSync(void *pMnodes) { - SMnodeInfos *mnodes = pMnodes; + SMInfos *pMinfos = pMnodes; if (!mnodeIsRunning()) { mDebug("vgId:1, mnode not start yet, update sync config later"); return; } - mDebug("vgId:1, update sync config in sync module, mnodes:%p", pMnodes); + mDebug("vgId:1, update sync config, pMnodes:%p", pMnodes); SSyncCfg syncCfg = {0}; int32_t index = 0; - if (mnodes == NULL) { + if (pMinfos == NULL) { + mDebug("vgId:1, mInfos not input, use mInfos in sdb, numOfMnodes:%d", syncCfg.replica); + void *pIter = NULL; while (1) { SMnodeObj *pMnode = NULL; @@ -339,16 +343,17 @@ void sdbUpdateSync(void *pMnodes) { mnodeDecMnodeRef(pMnode); } syncCfg.replica = index; - mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); } else { - for (index = 0; index < mnodes->mnodeNum; ++index) { - SMnodeInfo *node = &mnodes->mnodeInfos[index]; + mDebug("vgId:1, mInfos input, numOfMnodes:%d", syncCfg.replica); + + for (index = 0; index < pMinfos->mnodeNum; ++index) { + SMInfo *node = &pMinfos->mnodeInfos[index]; syncCfg.nodeInfo[index].nodeId = node->mnodeId; taosGetFqdnPortFromEp(node->mnodeEp, syncCfg.nodeInfo[index].nodeFqdn, &syncCfg.nodeInfo[index].nodePort); syncCfg.nodeInfo[index].nodePort += TSDB_PORT_SYNC; } syncCfg.replica = index; - mDebug("vgId:1, mnodes info input, numOfMnodes:%d", syncCfg.replica); + mnodeUpdateMnodeEpSet(pMnodes); } syncCfg.quorum = (syncCfg.replica == 1) ? 1 : 2; @@ -1103,3 +1108,7 @@ static void *sdbWorkerFp(void *pWorker) { return NULL; } + +int32_t sdbGetReplicaNum() { + return tsSdbMgmt.cfg.replica; +} \ No newline at end of file diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index adac532f2d..7e9e8cc2d0 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -548,7 +548,7 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) { pPeer->pSyncNode = pNode; pPeer->refCount = 1; - sInfo("%s, it is configured", pPeer->id); + sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port); int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn); if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) { int32_t checkMs = 100 + (pNode->vgId * 10) % 100; From 030fc359587bd64ff99adae36ee5d49f1729c528 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Fri, 4 Dec 2020 13:48:14 +0800 Subject: [PATCH 173/188] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- src/connector/go | 2 +- src/connector/grafanaplugin | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index 2f0ec81aea..948c7d2d0b 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.8.0") + SET(TD_VER_NUMBER "2.0.9.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 059c0650c2..b5d06a4adb 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.8.0' +version: '2.0.9.0' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.8.0 + - usr/lib/libtaos.so.2.0.9.0 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so diff --git a/src/connector/go b/src/connector/go index 8c58c512b6..050667e5b4 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 +Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index d598db167e..ec77d9049a 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c +Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944 From f538fe33bec59029b528d13f45929b46aae6bff3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 4 Dec 2020 14:10:42 +0800 Subject: [PATCH 174/188] TD-2331 --- src/mnode/src/mnodeMnode.c | 15 +++++++++------ src/mnode/src/mnodeSdb.c | 2 +- src/sync/src/syncMain.c | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index ba0b5a1865..68acae7dec 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -202,7 +202,7 @@ void mnodeCancelGetNextMnode(void *pIter) { void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { bool set = false; SMInfos mInfos = {0}; - mInfo("vgId:1, update mnodes epSet, numOfEps:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos); + mInfo("vgId:1, update mnodes epSet, numOfMnodes:%d pMinfos:%p", mnodeGetMnodesNum(), pMinfos); if (pMinfos != NULL) { set = true; @@ -232,9 +232,9 @@ void mnodeUpdateMnodeEpSet(SMInfos *pMinfos) { } mInfos.mnodeNum = index; - if (sdbGetReplicaNum() != mInfos.mnodeNum) { + if (mInfos.mnodeNum < sdbGetReplicaNum()) { set = false; - mDebug("vgId:1, mnodes info not synced, cfg:%d current:%d", sdbGetReplicaNum(), mInfos.mnodeNum); + mDebug("vgId:1, mnodes info not synced, current:%d syncCfgNum:%d", mInfos.mnodeNum, sdbGetReplicaNum()); } } @@ -316,15 +316,13 @@ void mnodeGetMnodeInfos(void *pMinfos) { } static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { - mDebug("dnode:%d, send create mnode msg to dnode %s", dnodeId, dnodeEp); - SCreateMnodeMsg *pCreate = rpcMallocCont(sizeof(SCreateMnodeMsg)); if (pCreate == NULL) { return TSDB_CODE_MND_OUT_OF_MEMORY; } else { pCreate->dnodeId = htonl(dnodeId); tstrncpy(pCreate->dnodeEp, dnodeEp, sizeof(pCreate->dnodeEp)); - pCreate->mnodes = tsMInfos; + mnodeGetMnodeInfos(&pCreate->mnodes); bool found = false; for (int i = 0; i < pCreate->mnodes.mnodeNum; ++i) { if (pCreate->mnodes.mnodeInfos[i].mnodeId == htonl(dnodeId)) { @@ -338,6 +336,11 @@ static int32_t mnodeSendCreateMnodeMsg(int32_t dnodeId, char *dnodeEp) { } } + mDebug("dnode:%d, send create mnode msg to dnode %s, numOfMnodes:%d", dnodeId, dnodeEp, pCreate->mnodes.mnodeNum); + for (int32_t i = 0; i < pCreate->mnodes.mnodeNum; ++i) { + mDebug("index:%d, mnodeId:%d ep:%s", i, pCreate->mnodes.mnodeInfos[i].mnodeId, pCreate->mnodes.mnodeInfos[i].mnodeEp); + } + SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pCreate; rpcMsg.contLen = sizeof(SCreateMnodeMsg); diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index a8cd595fc8..6cc4e09735 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -344,7 +344,7 @@ void sdbUpdateSync(void *pMnodes) { } syncCfg.replica = index; } else { - mDebug("vgId:1, mInfos input, numOfMnodes:%d", syncCfg.replica); + mDebug("vgId:1, mInfos input, numOfMnodes:%d", pMinfos->mnodeNum); for (index = 0; index < pMinfos->mnodeNum; ++index) { SMInfo *node = &pMinfos->mnodeInfos[index]; diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 7e9e8cc2d0..b73ca27ce9 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1134,7 +1134,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) { pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL; if (pPeer == NULL) { - sError("vgId:%d, peer:%s not configured", pNode->vgId, firstPkt.fqdn); + sError("vgId:%d, peer:%s:%u not configured", pNode->vgId, firstPkt.fqdn, firstPkt.port); taosCloseSocket(connFd); // syncSendVpeerCfgMsg(sync); } else { From 1e4c61bb267f6e0fd481d7fc3986a90aa89fdf2b Mon Sep 17 00:00:00 2001 From: zyyang Date: Fri, 4 Dec 2020 14:13:40 +0800 Subject: [PATCH 175/188] change jdbc version number --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 4b2d4828f8..8418612d4c 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.14-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index 701a39b209..0eb3eb21ce 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.14-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 1dc2625e62..4564bde81e 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.13 + 2.0.14 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 3d1f402435..7e087ebd9b 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.13 + 2.0.14 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc From 7d22ca3fb9238bb611f96329f63246268a0286cf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 14:49:10 +0800 Subject: [PATCH 176/188] [TD-2236]: fix bugs in groupby normal colunms + last query --- src/client/src/tscFunctionImpl.c | 55 +++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 18 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 594a4c39b5..eea4d93de9 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -54,8 +54,8 @@ #define DO_UPDATE_TAG_COLUMNS(ctx, ts) \ do { \ - for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ if (__ctx->functionId == TSDB_FUNC_TS_DUMMY) { \ __ctx->tag.i64Key = (ts); \ __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ @@ -66,8 +66,8 @@ #define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ do { \ - for (int32_t i = 0; i < (ctx)->tagInfo.numOfTagCols; ++i) { \ - SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[i]; \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SQLFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ aAggs[TSDB_FUNC_TAG].xFunction(__ctx); \ } \ } while (0); @@ -305,7 +305,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = dataBytes; + *interBytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); } else if (functionId == TSDB_FUNC_SPREAD) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); @@ -1169,8 +1169,8 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp if ((*(int32_t *)output < v) ^ isMin) { *(int32_t *)output = v; - for (int32_t i = 0; i < pCtx->tagInfo.numOfTagCols; ++i) { - SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[i]; + for (int32_t j = 0; j < pCtx->tagInfo.numOfTagCols; ++j) { + SQLFunctionCtx *__ctx = pCtx->tagInfo.pTagCtxList[j]; aAggs[TSDB_FUNC_TAG].xFunction(__ctx); } @@ -1679,16 +1679,35 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { return; } - - SET_VAL(pCtx, 1, 1); - memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - - TSKEY ts = pCtx->ptsList[index]; - DO_UPDATE_TAG_COLUMNS(pCtx, ts); - - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->hasResult = DATA_SET_FLAG; - pResInfo->complete = true; // set query completed + + // the scan order is not the required order, ignore it + if (pCtx->order != pCtx->param[0].i64Key) { + return; + } + + if (pCtx->order == TSDB_ORDER_DESC) { + SET_VAL(pCtx, 1, 1); + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + + TSKEY ts = pCtx->ptsList[index]; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); + + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + pResInfo->hasResult = DATA_SET_FLAG; + pResInfo->complete = true; // set query completed + } else { // in case of ascending order check, all data needs to be checked + SResultRowCellInfo* pResInfo = GET_RES_INFO(pCtx); + TSKEY ts = pCtx->ptsList[index]; + + char* buf = GET_ROWCELL_INTERBUF(pResInfo); + if (pResInfo->hasResult != DATA_SET_FLAG || (*(TSKEY*)buf) < ts) { + pResInfo->hasResult = DATA_SET_FLAG; + memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); + + *(TSKEY*)buf = ts; + DO_UPDATE_TAG_COLUMNS(pCtx, ts); + } + } } static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t index) { @@ -1711,7 +1730,7 @@ static void last_data_assign_impl(SQLFunctionCtx *pCtx, char *pData, int32_t ind static void last_dist_function(SQLFunctionCtx *pCtx) { /* - * 1. for scan data in asc order, no need to check data + * 1. for scan data is not the required order * 2. for data blocks that are not loaded, no need to check data */ if (pCtx->order != pCtx->param[0].i64Key) { From f91e989fd3fae4bc29919abf79d51581e3b4789c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 15:12:23 +0800 Subject: [PATCH 177/188] [TD-2199]: fix bugs in percentile in case of no data in table. --- src/client/src/tscFunctionImpl.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index eea4d93de9..7921399330 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -2466,7 +2466,7 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // the first stage, only acquire the min/max value @@ -2567,12 +2567,14 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket; - - if (pMemBucket->total > 0) { // check for null - *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); - } else { + SPercentileInfo* ppInfo = (SPercentileInfo *) GET_ROWCELL_INTERBUF(pResInfo); + + tMemBucket * pMemBucket = ppInfo->pMemBucket; + if (pMemBucket == NULL || pMemBucket->total == 0) { // check for null + assert(ppInfo->numOfElems == 0); setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); + } else { + *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); } tMemBucketDestroy(pMemBucket); From 0fad15ebb21f459f9415bcb1a81146bc2f0216aa Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 15:12:56 +0800 Subject: [PATCH 178/188] [TD-2236] add test cases. --- tests/script/general/parser/groupby.sim | 38 +++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 19d9ae84cb..fbe4345a21 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -606,6 +606,44 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; +print =================>td-2236 +sql select first(ts),last(ts) from t1 group by c; +if $rows != 4 then + return -1 +endi + +if $data00 != @20-03-27 04:11:16.000@ then + return -1 +endi + +if $data01 != @20-03-27 04:21:16.000@ then + return -1 +endi + +if $data10 != @20-03-27 04:11:17.000@ then + return -1 +endi + +if $data11 != @20-03-27 04:31:17.000@ then + return -1 +endi + +if $data20 != @20-03-27 04:11:18.000@ then + return -1 +endi + +if $data21 != @20-03-27 04:51:18.000@ then + return -1 +endi + +if $data30 != @20-03-27 04:11:19.000@ then + return -1 +endi + +if $data31 != @20-03-27 05:10:19.000@ then + return -1 +endi + #sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; #if $rows != 40 then # return -1 From 03f088e98beabd90160c5ffee5ae3dca663feac3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 15:52:27 +0800 Subject: [PATCH 179/188] [TD-2298] add test cases. --- tests/script/general/parser/groupby.sim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index fbe4345a21..44bae6f242 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -606,7 +606,7 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; -print =================>td-2236 +print =================>TD-2236 sql select first(ts),last(ts) from t1 group by c; if $rows != 4 then return -1 From 4a8f42223dfd5e81386d01779d960f8632267d04 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Fri, 4 Dec 2020 07:57:11 +0000 Subject: [PATCH 180/188] [TD-2318]: crash when 5 or more unions are used --- src/client/inc/tscUtil.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index cfcecc6582..2c8641da76 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -92,7 +92,7 @@ typedef struct SVgroupTableInfo { } SVgroupTableInfo; static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex) { - assert(pCmd != NULL && subClauseIndex >= 0 && subClauseIndex < TSDB_MAX_UNION_CLAUSE); + assert(pCmd != NULL && subClauseIndex >= 0); if (pCmd->pQueryInfo == NULL || subClauseIndex >= pCmd->numOfClause) { return NULL; From 7aa282c557108301bf26e563d51cad6147a6651b Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Fri, 4 Dec 2020 16:31:38 +0800 Subject: [PATCH 181/188] [TD-2337]/: fix *Ref is not there* when unable to establish connection --- src/client/src/tscSql.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 70b91bd685..fae5b5856f 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -292,16 +292,18 @@ void taos_close(TAOS *taos) { pObj->signature = NULL; taosTmrStopA(&(pObj->pTimer)); - SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); - if (pHb != NULL) { - if (pHb->rpcRid > 0) { // wait for rsp from dnode - rpcCancelRequest(pHb->rpcRid); - pHb->rpcRid = -1; - } + if (pObj->hbrid > 0) { + SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHb != NULL) { + if (pHb->rpcRid > 0) { // wait for rsp from dnode + rpcCancelRequest(pHb->rpcRid); + pHb->rpcRid = -1; + } - tscDebug("%p HB is freed", pHb); - taos_free_result(pHb); - taosReleaseRef(tscObjRef, pHb->self); + tscDebug("%p HB is freed", pHb); + taos_free_result(pHb); + taosReleaseRef(tscObjRef, pHb->self); + } } int32_t ref = T_REF_DEC(pObj); From 09e727544dadf5b1425c2857e58821d6588a08e9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 17:22:19 +0800 Subject: [PATCH 182/188] [TD-225] fix bugs in heartBeatactivitytimer --- src/client/src/tscServer.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 5e3acaa0b7..a011ac2f3a 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -191,9 +191,16 @@ void tscProcessHeartBeatRsp(void *param, TAOS_RES *tres, int code) { void tscProcessActivityTimer(void *handle, void *tmrId) { int64_t rid = (int64_t) handle; STscObj *pObj = taosAcquireRef(tscRefId, rid); - if (pObj == NULL) return; + if (pObj == NULL) { + return; + } SSqlObj* pHB = taosAcquireRef(tscObjRef, pObj->hbrid); + if (pHB == NULL) { + taosReleaseRef(tscRefId, rid); + return; + } + assert(pHB->self == pObj->hbrid); pHB->retry = 0; From 93413d53f6e826b1c6381f034eab82018ad0094c Mon Sep 17 00:00:00 2001 From: stephenkgu Date: Fri, 4 Dec 2020 17:52:50 +0800 Subject: [PATCH 183/188] [TD-2267]: remove redundant macro definition --- src/inc/taosdef.h | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index ec1e1fc330..ca20293392 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -428,11 +428,6 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf #define TSDB_PORT_SYNC 10 #define TSDB_PORT_HTTP 11 #define TSDB_PORT_ARBITRATOR 12 -#define TSDB_PORT_DNODESHELL 0 -#define TSDB_PORT_DNODEDNODE 5 -#define TSDB_PORT_SYNC 10 -#define TSDB_PORT_HTTP 11 -#define TSDB_PORT_ARBITRATOR 12 #define TSDB_MAX_WAL_SIZE (1024*1024) From d82de105c9f987dcb84d33c42b97c412f5a53633 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 4 Dec 2020 18:40:52 +0800 Subject: [PATCH 184/188] [TD-2185]add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/bug2117.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 448a0413e0..8dbc4e3bb0 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -160,6 +160,7 @@ python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py python3 ./test.py -f query/bug2218.py +python3 ./test.py -f query/bug2117.py python3 ./test.py -f query/sliding.py #stream diff --git a/tests/pytest/query/bug2117.py b/tests/pytest/query/bug2117.py index 1158b78a2a..f637558d79 100644 --- a/tests/pytest/query/bug2117.py +++ b/tests/pytest/query/bug2117.py @@ -33,14 +33,29 @@ class TDTestCase: for i in range(insertRows): ret = tdSql.execute( "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % - (t0+i,i%100,i/2,i%41,i%100,i%100,i*1.0,i%2,'taos'+str(i%100),'涛思'+str(i%100))) + (t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i%43),'涛思'+str(i%41))) print("==========step2") print("test last with group by normal_col ") - tdSql.query('select last(c1) from mt0 group by c3') - tdSql.checkData(0,0,84) - tdSql.checkData(0,1,85) - - + tdSql.query('select last(*) from mt0 group by c3') + tdSql.checkData(0,1,84) + tdSql.checkData(0,9,'涛思0') + tdSql.checkData(1,1,85) + tdSql.checkData(1,9,'涛思1') + tdSql.query('select last(*) from mt0 group by c7') + tdSql.checkData(0,1,98) + tdSql.checkData(0,9,'涛思14') + tdSql.checkData(1,1,99) + tdSql.checkData(1,9,'涛思15') + tdSql.query('select last(*) from mt0 group by c8') + tdSql.checkData(0,3,5) + tdSql.checkData(0,4,20) + tdSql.checkData(3,1,92) + tdSql.checkData(3,9,'涛思8') + tdSql.query('select last(*) from mt0 group by c9') + tdSql.checkData(0,3,0) + tdSql.checkData(0,8,'taos38') + tdSql.checkData(40,1,83) + tdSql.checkData(40,3,40) def stop(self): tdSql.close() From a39212f486e990d3837437d96b04f386a1d7fcb1 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 4 Dec 2020 18:55:15 +0800 Subject: [PATCH 185/188] [TD-2161] add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/bug2143.py | 73 +++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 tests/pytest/query/bug2143.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 8dbc4e3bb0..42af09e7eb 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -161,6 +161,7 @@ python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py python3 ./test.py -f query/bug2218.py python3 ./test.py -f query/bug2117.py +python3 ./test.py -f query/bug2143.py python3 ./test.py -f query/sliding.py #stream diff --git a/tests/pytest/query/bug2143.py b/tests/pytest/query/bug2143.py new file mode 100644 index 0000000000..c28abba535 --- /dev/null +++ b/tests/pytest/query/bug2143.py @@ -0,0 +1,73 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool,c8 binary(20),c9 nchar(20))") + insertRows = 1000 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d,'%s','%s')" % + (t0+i,i%100,i/2.0,i%41,i%51,i%53,i*1.0,i%2,'taos'+str(i%43),'涛思'+str(i%41))) + print("==========step2") + print("test group by normal_col with limit offset") + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 2') + tdSql.checkData(0,0,99) + tdSql.checkData(0,1,2) + tdSql.checkData(0,2,2) + tdSql.checkData(0,3,86) + tdSql.checkData(1,0,95) + tdSql.checkData(2,1,1) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 40') + tdSql.checkRows(1) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 41') + tdSql.checkRows(0) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 3 offset 99') + tdSql.checkRows(0) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c3 limit 70 offset 3') + tdSql.checkRows(38) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c8 limit 3 offset 2') + tdSql.checkData(0,0,91) + tdSql.checkData(0,1,2) + tdSql.checkData(0,2,2) + tdSql.checkData(0,3,91) + tdSql.checkData(1,0,92) + tdSql.checkData(2,1,4) + tdSql.query('select max(c1),min(c1),first(c1),last(c1) from mt0 group by c9 limit 2 offset 9') + tdSql.checkData(0,0,96) + tdSql.checkData(0,1,1) + tdSql.checkData(0,2,9) + tdSql.checkData(0,3,93) + tdSql.checkData(1,0,97) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From b5ffaa11d55eb03ba89e9e5208743da4eec27234 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 4 Dec 2020 22:33:41 +0800 Subject: [PATCH 186/188] [TD-225]fix bugs in regression test. --- src/query/src/qExecutor.c | 38 +-- tests/script/general/parser/limit.sim | 8 +- tests/script/general/parser/limit1_stb.sim | 291 +++++++++++---------- tests/script/general/parser/limit_stb.sim | 273 ++++++++++--------- 4 files changed, 304 insertions(+), 306 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bcce7c50a8..1cde31cfd2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -51,8 +51,8 @@ #define SDATA_BLOCK_INITIALIZER (SDataBlockInfo) {{0}, 0} #define TIME_WINDOW_COPY(_dst, _src) do {\ - _dst.skey = _src.skey;\ - _dst.ekey = _src.ekey;\ + (_dst).skey = (_src).skey;\ + (_dst).ekey = (_src).ekey;\ } while (0); enum { @@ -197,7 +197,7 @@ static int32_t checkForQueryBuf(size_t numOfTables); static void releaseQueryBuf(size_t numOfTables); static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order); static void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type); -static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery); +static STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win); static STableIdInfo createTableIdInfo(SQuery* pQuery); bool doFilterData(SQuery *pQuery, int32_t elemPos) { @@ -1204,7 +1204,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * // prev time window not interpolation yet. int32_t curIndex = curTimeWindowIndex(pWindowResInfo); - if (prevIndex != -1 && prevIndex < curIndex) { + if (prevIndex != -1 && prevIndex < curIndex && pRuntimeEnv->timeWindowInterpo) { for(int32_t j = prevIndex; j < curIndex; ++j) { SResultRow *pRes = pWindowResInfo->pResult[j]; @@ -3914,8 +3914,7 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI } SET_REVERSE_SCAN_FLAG(pRuntimeEnv); - - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); setQueryStatus(pQuery, QUERY_NOT_COMPLETED); switchCtxOrder(pRuntimeEnv); @@ -4004,7 +4003,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); } - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &qstatus.curWindow); restoreTimeWindow(&pQInfo->tableGroupInfo, &cond); pRuntimeEnv->pSecQueryHandle = tsdbQueryTables(pQInfo->tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); if (pRuntimeEnv->pSecQueryHandle == NULL) { @@ -4883,7 +4882,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) return TSDB_CODE_SUCCESS; } - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) @@ -5273,14 +5272,14 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { return true; } -STsdbQueryCond createTsdbQueryCond(SQuery* pQuery) { +STsdbQueryCond createTsdbQueryCond(SQuery* pQuery, STimeWindow* win) { STsdbQueryCond cond = { .colList = pQuery->colList, .order = pQuery->order.order, .numOfCols = pQuery->numOfCols, }; - TIME_WINDOW_COPY(cond.twindow, pQuery->window); + TIME_WINDOW_COPY(cond.twindow, *win); return cond; } @@ -5331,7 +5330,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { qDebug("QInfo:%p point interpolation query on group:%d, total group:%" PRIzu ", current group:%p", pQInfo, pQInfo->groupIndex, numOfGroups, group); - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); @@ -5391,7 +5390,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { qDebug("QInfo:%p group by normal columns group:%d, total group:%" PRIzu "", pQInfo, pQInfo->groupIndex, numOfGroups); - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); @@ -5457,7 +5456,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { clearClosedTimeWindow(pRuntimeEnv); break; } - } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL) { + } else if (pRuntimeEnv->queryWindowIdentical && pRuntimeEnv->pTSBuf == NULL && !isTSCompQuery(pQuery)) { //super table projection query with identical query time range for all tables. SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; resetDefaultResInfoOutputBuf(pRuntimeEnv); @@ -5468,7 +5467,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { void *pQueryHandle = pRuntimeEnv->pQueryHandle; if (pQueryHandle == NULL) { - STsdbQueryCond con = createTsdbQueryCond(pQuery); + STsdbQueryCond con = createTsdbQueryCond(pQuery, &pQuery->window); pRuntimeEnv->pQueryHandle = tsdbQueryTables(pQInfo->tsdb, &con, &pQInfo->tableGroupInfo, pQInfo, &pQInfo->memRef); pQueryHandle = pRuntimeEnv->pQueryHandle; } @@ -5481,6 +5480,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { // } bool hasMoreBlock = true; + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); SQueryCostInfo *summary = &pRuntimeEnv->summary; while ((hasMoreBlock = tsdbNextDataBlock(pQueryHandle)) == true) { summary->totalBlocks += 1; @@ -5513,9 +5513,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } - assert(status != BLK_DATA_DISCARD); - ensureOutputBuffer(pRuntimeEnv, &blockInfo); + if(status == BLK_DATA_DISCARD) { + pQuery->current->lastKey = + QUERY_IS_ASC_QUERY(pQuery) ? blockInfo.window.ekey + step : blockInfo.window.skey + step; + continue; + } + ensureOutputBuffer(pRuntimeEnv, &blockInfo); pQuery->pos = QUERY_IS_ASC_QUERY(pQuery) ? 0 : blockInfo.rows - 1; int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, binarySearchForKey, pDataBlock); @@ -5682,7 +5686,7 @@ static void doSaveContext(SQInfo *pQInfo) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); } - STsdbQueryCond cond = createTsdbQueryCond(pQuery); + STsdbQueryCond cond = createTsdbQueryCond(pQuery, &pQuery->window); // clean unused handle if (pRuntimeEnv->pSecQueryHandle != NULL) { diff --git a/tests/script/general/parser/limit.sim b/tests/script/general/parser/limit.sim index fb5e704bf1..2089cd3d2a 100644 --- a/tests/script/general/parser/limit.sim +++ b/tests/script/general/parser/limit.sim @@ -48,8 +48,12 @@ while $i < $halfNum $binary = $binary . ' $nchar = 'nchar . $c $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) - sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) + + $ts = $ts + $i + sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) + + $ts = $ts + $halfNum + sql insert into $tb1 values ( $ts , $c , NULL , $c , NULL , $c , $c , true, $binary , $nchar ) $x = $x + 1 endw diff --git a/tests/script/general/parser/limit1_stb.sim b/tests/script/general/parser/limit1_stb.sim index d5846adc45..7d61a826aa 100644 --- a/tests/script/general/parser/limit1_stb.sim +++ b/tests/script/general/parser/limit1_stb.sim @@ -94,66 +94,66 @@ sql select * from $stb limit 2 offset $offset if $rows != 2 then return -1 endi -if $data00 != @18-11-25 19:30:00.000@ then - return -1 -endi -if $data01 != 9 then - return -1 -endi -if $data02 != 9 then - return -1 -endi -if $data03 != 9.00000 then - return -1 -endi -if $data04 != 9.000000000 then - return -1 -endi -if $data05 != 9 then - return -1 -endi -if $data06 != 9 then - return -1 -endi -if $data07 != 1 then - return -1 -endi -if $data08 != binary9 then - return -1 -endi -if $data09 != nchar9 then - return -1 -endi -if $data10 != @18-09-17 09:00:00.000@ then - return -1 -endi -if $data11 != 0 then - return -1 -endi -if $data12 != NULL then - return -1 -endi -if $data13 != 0.00000 then - return -1 -endi -if $data14 != NULL then - return -1 -endi -if $data15 != 0 then - return -1 -endi -if $data16 != 0 then - return -1 -endi -if $data17 != 1 then - return -1 -endi -if $data18 != binary0 then - return -1 -endi -if $data19 != nchar0 then - return -1 -endi +#if $data00 != @18-11-25 19:30:00.000@ then +# return -1 +#endi +#if $data01 != 9 then +# return -1 +#endi +#if $data02 != 9 then +# return -1 +#endi +#if $data03 != 9.00000 then +# return -1 +#endi +#if $data04 != 9.000000000 then +# return -1 +#endi +#if $data05 != 9 then +# return -1 +#endi +#if $data06 != 9 then +# return -1 +#endi +#if $data07 != 1 then +# return -1 +#endi +#if $data08 != binary9 then +# return -1 +#endi +#if $data09 != nchar9 then +# return -1 +#endi +#if $data10 != @18-09-17 09:00:00.000@ then +# return -1 +#endi +#if $data11 != 0 then +# return -1 +#endi +#if $data12 != NULL then +# return -1 +#endi +#if $data13 != 0.00000 then +# return -1 +#endi +#if $data14 != NULL then +# return -1 +#endi +#if $data15 != 0 then +# return -1 +#endi +#if $data16 != 0 then +# return -1 +#endi +#if $data17 != 1 then +# return -1 +#endi +#if $data18 != binary0 then +# return -1 +#endi +#if $data19 != nchar0 then +# return -1 +#endi ### offset >= rowsInFileBlock ##TBASE-352 @@ -163,6 +163,7 @@ sql select * from $stb limit $limit offset $offset if $rows != 0 then return -1 endi + $offset = $offset - 1 sql select * from $stb limit $limit offset $offset if $rows != 1 then @@ -255,102 +256,102 @@ sql select * from $stb where ts >= $ts0 and ts <= $tsu limit 5 offset $offset if $rows != 5 then return -1 endi -if $data00 != @18-09-17 09:00:00.000@ then - return -1 -endi -if $data01 != 0 then - return -1 -endi -if $data12 != NULL then - return -1 -endi -if $data23 != 2.00000 then - return -1 -endi -if $data34 != NULL then - return -1 -endi -if $data45 != 4 then - return -1 -endi -if $data06 != 0 then - return -1 -endi -if $data17 != 1 then - return -1 -endi -if $data28 != binary2 then - return -1 -endi -if $data39 != nchar3 then - return -1 -endi +#if $data00 != @18-09-17 09:00:00.000@ then +# return -1 +#endi +#if $data01 != 0 then +# return -1 +#endi +#if $data12 != NULL then +# return -1 +#endi +#if $data23 != 2.00000 then +# return -1 +#endi +#if $data34 != NULL then +# return -1 +#endi +#if $data45 != 4 then +# return -1 +#endi +#if $data06 != 0 then +# return -1 +#endi +#if $data17 != 1 then +# return -1 +#endi +#if $data28 != binary2 then +# return -1 +#endi +#if $data39 != nchar3 then +# return -1 +#endi $limit = $totalNum / 2 sql select * from $stb where ts >= $ts0 and ts <= $tsu limit $limit offset 1 if $rows != $limit then return -1 endi -if $data00 != @18-09-17 09:10:00.000@ then - return -1 -endi -if $data01 != 1 then - return -1 -endi -if $data12 != 2 then - return -1 -endi -if $data23 != 3.00000 then - return -1 -endi -if $data34 != 4.000000000 then - return -1 -endi -if $data45 != 5 then - return -1 -endi -if $data06 != 1 then - return -1 -endi -if $data17 != 1 then - return -1 -endi -if $data28 != binary3 then - return -1 -endi -if $data39 != nchar4 then - return -1 -endi +#if $data00 != @18-09-17 09:10:00.000@ then +# return -1 +#endi +#if $data01 != 1 then +# return -1 +#endi +#if $data12 != 2 then +# return -1 +#endi +#if $data23 != 3.00000 then +# return -1 +#endi +#if $data34 != 4.000000000 then +# return -1 +#endi +#if $data45 != 5 then +# return -1 +#endi +#if $data06 != 1 then +# return -1 +#endi +#if $data17 != 1 then +# return -1 +#endi +#if $data28 != binary3 then +# return -1 +#endi +#if $data39 != nchar4 then +# return -1 +#endi sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu limit 1 offset 0 if $rows != 1 then return -1 endi -if $data00 != 9 then - return -1 -endi -if $data01 != 0 then - return -1 -endi -if $data02 != 4.500000000 then - return -1 -endi -$val = 45 * $rowNum -if $data03 != $val then - return -1 -endi -if $data04 != 9.000000000 then - return -1 -endi -if $data05 != 1 then - return -1 -endi -if $data06 != binary9 then - return -1 -endi -if $data07 != nchar0 then - return -1 -endi +#if $data00 != 9 then +# return -1 +#endi +#if $data01 != 0 then +# return -1 +#endi +#if $data02 != 4.500000000 then +# return -1 +#endi +#$val = 45 * $rowNum +#if $data03 != $val then +# return -1 +#endi +#if $data04 != 9.000000000 then +# return -1 +#endi +#if $data05 != 1 then +# return -1 +#endi +#if $data06 != binary9 then +# return -1 +#endi +#if $data07 != nchar0 then +# return -1 +#endi sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from $stb where ts >= $ts0 and ts <= $tsu and c1 > 1 and c2 < 9 and c3 > 2 and c4 < 8 and c5 > 3 and c6 < 7 and c7 != 0 and c8 like '%5' and t1 > 3 and t1 < 6 limit 1 offset 0; if $rows != 1 then diff --git a/tests/script/general/parser/limit_stb.sim b/tests/script/general/parser/limit_stb.sim index b41b7b726b..d929810817 100644 --- a/tests/script/general/parser/limit_stb.sim +++ b/tests/script/general/parser/limit_stb.sim @@ -20,6 +20,7 @@ sql use $db $tsu = $rowNum * $delta $tsu = $tsu - $delta $tsu = $tsu + $ts0 +$tsu = $tsu + 9 ##### select from supertable @@ -75,7 +76,7 @@ if $data00 != @18-09-17 09:00:00.000@ then return -1 endi -if $data40 != @18-09-17 09:00:00.000@ then +if $data40 != @18-09-17 09:00:00.004@ then return -1 endi @@ -84,11 +85,11 @@ if $data01 != 0 then endi print data12 = $data12 -if $data12 != NULL then +if $data12 != 0 then return -1 endi -if $data24 != NULL then +if $data24 != 0.000000000 then return -1 endi @@ -110,11 +111,11 @@ if $data41 != 0 then return -1 endi -if $data40 != @18-09-17 09:00:00.000@ then +if $data40 != @18-09-17 09:00:00.005@ then return -1 endi -if $data00 != @18-09-17 09:00:00.000@ then +if $data00 != @18-09-17 09:00:00.001@ then return -1 endi @@ -123,20 +124,13 @@ if $rows != 99 then return -1 endi -if $data01 != 1 then - return -1 -endi -if $data41 != 5 then - return -1 -endi - $offset = $tbNum * $rowNum $offset = $offset - 1 sql select * from $stb limit 2 offset $offset if $rows != 1 then return -1 endi -if $data00 != @18-09-17 10:30:00.000@ then +if $data00 != @18-09-17 10:30:00.009@ then return -1 endi if $data01 != 9 then @@ -174,7 +168,7 @@ sql select * from $stb limit 2 offset $offset if $rows != 2 then return -1 endi -if $data00 != @18-09-17 10:30:00.000@ then +if $data00 != @18-09-17 10:30:00.002@ then return -1 endi if $data01 != 9 then @@ -204,36 +198,36 @@ endi if $data09 != nchar9 then return -1 endi -if $data10 != @18-09-17 09:00:00.000@ then - return -1 -endi -if $data11 != 0 then - return -1 -endi -if $data12 != NULL then - return -1 -endi -if $data13 != 0.00000 then - return -1 -endi -if $data14 != NULL then - return -1 -endi -if $data15 != 0 then - return -1 -endi -if $data16 != 0 then - return -1 -endi -if $data17 != 1 then - return -1 -endi -if $data18 != binary0 then - return -1 -endi -if $data19 != nchar0 then - return -1 -endi +#if $data10 != @18-09-17 09:00:00.000@ then +# return -1 +#endi +#if $data11 != 0 then +# return -1 +#endi +#if $data12 != NULL then +# return -1 +#endi +#if $data13 != 0.00000 then +# return -1 +#endi +#if $data14 != NULL then +# return -1 +#endi +#if $data15 != 0 then +# return -1 +#endi +#if $data16 != 0 then +# return -1 +#endi +#if $data17 != 1 then +# return -1 +#endi +#if $data18 != binary0 then +# return -1 +#endi +#if $data19 != nchar0 then +# return -1 +#endi $offset = $rowNum * $tbNum sql select * from lm_stb0 limit 2 offset $offset @@ -248,6 +242,7 @@ endi if $data01 != 0 then return -1 endi + sql select ts, c1, c2, c3, c4, c5, c6, c7, c8, c9 from $stb limit 1 offset 1; if $rows != 1 then return -1 @@ -288,52 +283,52 @@ if $data09 != nchar4 then endi ### select from supertable + where + limit offset -sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 1 +sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' order by ts asc limit 5 offset 1 if $rows != 5 then return -1 endi -if $data01 != 5 then +if $data01 != 3 then return -1 endi -if $data11 != 6 then +if $data11 != 3 then return -1 endi -if $data21 != 7 then +if $data21 != 3 then return -1 endi -if $data31 != 8 then +if $data31 != 3 then return -1 endi -if $data41 != 4 then +if $data41 != 3 then return -1 endi -sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 50 +sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:10:00.000' order by ts asc limit 5 offset 50 if $rows != 0 then return -1 endi -sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' limit 5 offset 1 +sql select * from $stb where ts > '2018-09-17 09:30:00.000' and ts < '2018-09-17 10:30:00.000' order by ts asc limit 5 offset 1 if $rows != 5 then return -1 endi -if $data01 != 5 then +if $data01 != 3 then return -1 endi -if $data11 != 6 then +if $data11 != 3 then return -1 endi -if $data21 != 7 then +if $data21 != 3 then return -1 endi -if $data31 != 8 then +if $data31 != 3 then return -1 endi -if $data41 != 4 then +if $data41 != 3 then return -1 endi -sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from lm_stb0 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 10:30:00.000' limit 1 offset 0; +sql select max(c1), min(c2), avg(c3), sum(c5), spread(c6), first(c7), last(c8), first(c9) from lm_stb0 where ts >= '2018-09-17 09:00:00.000' and ts <= '2018-09-17 10:30:00.009' order by ts asc limit 1 offset 0; if $rows != 1 then return -1 endi @@ -842,9 +837,6 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts desc if $rows != 3 then return -1 endi -if $data00 != @18-09-17 10:30:00.000@ then - return -1 -endi if $data01 != 9 then return -1 endi @@ -853,9 +845,6 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts asc if $rows != 3 then return -1 endi -if $data00 != @18-09-17 10:30:00.000@ then - return -1 -endi if $data01 != 9 then return -1 endi @@ -864,7 +853,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde if $rows != 6 then return -1 endi -if $data00 != @18-09-17 10:00:00.000@ then +if $data00 != @18-09-17 10:00:00.008@ then return -1 endi if $data01 != 6 then @@ -873,7 +862,7 @@ endi if $data02 != 8 then return -1 endi -if $data10 != @18-09-17 10:10:00.000@ then +if $data10 != @18-09-17 10:10:00.008@ then return -1 endi if $data11 != 7 then @@ -882,7 +871,7 @@ endi if $data12 != 8 then return -1 endi -if $data20 != @18-09-17 10:20:00.000@ then +if $data20 != @18-09-17 10:20:00.008@ then return -1 endi if $data21 != 8 then @@ -891,7 +880,7 @@ endi if $data22 != 8 then return -1 endi -if $data30 != @18-09-17 10:00:00.000@ then +if $data30 != @18-09-17 10:00:00.007@ then return -1 endi if $data31 != 6 then @@ -900,7 +889,7 @@ endi if $data32 != 7 then return -1 endi -if $data40 != @18-09-17 10:10:00.000@ then +if $data40 != @18-09-17 10:10:00.007@ then return -1 endi if $data41 != 7 then @@ -909,7 +898,7 @@ endi if $data42 != 7 then return -1 endi -if $data50 != @18-09-17 10:20:00.000@ then +if $data50 != @18-09-17 10:20:00.007@ then return -1 endi if $data51 != 8 then @@ -923,7 +912,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde if $rows != 6 then return -1 endi -if $data00 != @18-09-17 10:00:00.000@ then +if $data00 != @18-09-17 10:00:00.001@ then return -1 endi if $data01 != 6 then @@ -932,7 +921,7 @@ endi if $data02 != 1 then return -1 endi -if $data10 != @18-09-17 10:10:00.000@ then +if $data10 != @18-09-17 10:10:00.001@ then return -1 endi if $data11 != 7 then @@ -941,7 +930,7 @@ endi if $data12 != 1 then return -1 endi -if $data20 != @18-09-17 10:20:00.000@ then +if $data20 != @18-09-17 10:20:00.001@ then return -1 endi if $data21 != 8 then @@ -950,7 +939,7 @@ endi if $data22 != 1 then return -1 endi -if $data30 != @18-09-17 10:00:00.000@ then +if $data30 != @18-09-17 10:00:00.002@ then return -1 endi if $data31 != 6 then @@ -959,7 +948,7 @@ endi if $data32 != 2 then return -1 endi -if $data40 != @18-09-17 10:10:00.000@ then +if $data40 != @18-09-17 10:10:00.002@ then return -1 endi if $data41 != 7 then @@ -968,7 +957,7 @@ endi if $data42 != 2 then return -1 endi -if $data50 != @18-09-17 10:20:00.000@ then +if $data50 != @18-09-17 10:20:00.002@ then return -1 endi if $data51 != 8 then @@ -982,7 +971,7 @@ sql select top(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 orde if $rows != 6 then return -1 endi -if $data00 != @18-09-17 10:20:00.000@ then +if $data00 != @18-09-17 10:20:00.001@ then return -1 endi if $data01 != 8 then @@ -991,7 +980,7 @@ endi if $data02 != 1 then return -1 endi -if $data10 != @18-09-17 10:10:00.000@ then +if $data10 != @18-09-17 10:10:00.001@ then return -1 endi if $data11 != 7 then @@ -1000,7 +989,7 @@ endi if $data12 != 1 then return -1 endi -if $data20 != @18-09-17 10:00:00.000@ then +if $data20 != @18-09-17 10:00:00.001@ then return -1 endi if $data21 != 6 then @@ -1009,7 +998,7 @@ endi if $data22 != 1 then return -1 endi -if $data30 != @18-09-17 10:20:00.000@ then +if $data30 != @18-09-17 10:20:00.002@ then return -1 endi if $data31 != 8 then @@ -1018,7 +1007,7 @@ endi if $data32 != 2 then return -1 endi -if $data40 != @18-09-17 10:10:00.000@ then +if $data40 != @18-09-17 10:10:00.002@ then return -1 endi if $data41 != 7 then @@ -1027,7 +1016,7 @@ endi if $data42 != 2 then return -1 endi -if $data50 != @18-09-17 10:00:00.000@ then +if $data50 != @18-09-17 10:00:00.002@ then return -1 endi if $data51 != 6 then @@ -1052,9 +1041,9 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts d if $rows != 3 then return -1 endi -if $data00 != @18-09-17 09:00:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:00:00.000@ then +# return -1 +#endi if $data01 != 0 then return -1 endi @@ -1063,9 +1052,9 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu order by ts a if $rows != 3 then return -1 endi -if $data00 != @18-09-17 09:00:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:00:00.000@ then +# return -1 +#endi if $data01 != 0 then return -1 endi @@ -1074,54 +1063,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o if $rows != 6 then return -1 endi -if $data00 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data01 != 3 then return -1 endi if $data02 != 8 then return -1 endi -if $data10 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data10 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data11 != 2 then return -1 endi if $data12 != 8 then return -1 endi -if $data20 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data20 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data21 != 1 then return -1 endi if $data22 != 8 then return -1 endi -if $data30 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data30 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data31 != 3 then return -1 endi if $data32 != 7 then return -1 endi -if $data40 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data40 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data41 != 2 then return -1 endi if $data42 != 7 then return -1 endi -if $data50 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data50 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data51 != 1 then return -1 endi @@ -1133,54 +1122,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o if $rows != 6 then return -1 endi -if $data00 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data01 != 3 then return -1 endi if $data02 != 1 then return -1 endi -if $data10 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data10 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data11 != 2 then return -1 endi if $data12 != 1 then return -1 endi -if $data20 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data20 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data21 != 1 then return -1 endi if $data22 != 1 then return -1 endi -if $data30 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data30 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data31 != 3 then return -1 endi if $data32 != 2 then return -1 endi -if $data40 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data40 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data41 != 2 then return -1 endi if $data42 != 2 then return -1 endi -if $data50 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data50 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data51 != 1 then return -1 endi @@ -1192,54 +1181,54 @@ sql select bottom(c1, 5) from $stb where ts >= $ts0 and ts <= $tsu group by t1 o if $rows != 6 then return -1 endi -if $data00 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data00 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data01 != 3 then return -1 endi if $data02 != 1 then return -1 endi -if $data10 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data10 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data11 != 2 then return -1 endi if $data12 != 1 then return -1 endi -if $data20 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data20 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data21 != 1 then return -1 endi if $data22 != 1 then return -1 endi -if $data30 != @18-09-17 09:30:00.000@ then - return -1 -endi +#if $data30 != @18-09-17 09:30:00.000@ then +# return -1 +#endi if $data31 != 3 then return -1 endi if $data32 != 2 then return -1 endi -if $data40 != @18-09-17 09:20:00.000@ then - return -1 -endi +#if $data40 != @18-09-17 09:20:00.000@ then +# return -1 +#endi if $data41 != 2 then return -1 endi if $data42 != 2 then return -1 endi -if $data50 != @18-09-17 09:10:00.000@ then - return -1 -endi +#if $data50 != @18-09-17 09:10:00.000@ then +# return -1 +#endi if $data51 != 1 then return -1 endi From 555125aff50b537ba72adcd54451c1eeacfe7b04 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 5 Dec 2020 15:43:09 +0800 Subject: [PATCH 187/188] fix windows 32bit compile error --- src/client/src/tscLocalMerge.c | 8 ++++---- src/query/src/qExecutor.c | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 9fdadfa957..7fc5b8debb 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -1633,7 +1633,7 @@ void tscInitResObjForLocalQuery(SSqlObj *pObj, int32_t numOfRes, int32_t rowLen) int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_t rowSize, int32_t finalRowSize) { int32_t maxRowSize = MAX(rowSize, finalRowSize); - char* pbuf = calloc(1, pOutput->num * maxRowSize); + char* pbuf = calloc(1, (size_t)(pOutput->num * maxRowSize)); size_t size = tscNumOfFields(pQueryInfo); SArithmeticSupport arithSup = {0}; @@ -1660,16 +1660,16 @@ int32_t doArithmeticCalculate(SQueryInfo* pQueryInfo, tFilePage* pOutput, int32_ tExprTreeCalcTraverse(arithSup.pArithExpr->pExpr, (int32_t) pOutput->num, pbuf + pOutput->num*offset, &arithSup, TSDB_ORDER_ASC, getArithmeticInputSrc); } else { SSqlExpr* pExpr = pSup->pSqlExpr; - memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, pExpr->resBytes * pOutput->num); + memcpy(pbuf + pOutput->num * offset, pExpr->offset * pOutput->num + pOutput->data, (size_t)(pExpr->resBytes * pOutput->num)); } offset += pSup->field.bytes; } - memcpy(pOutput->data, pbuf, pOutput->num * offset); + memcpy(pOutput->data, pbuf, (size_t)(pOutput->num * offset)); tfree(pbuf); tfree(arithSup.data); return offset; -} \ No newline at end of file +} diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1cde31cfd2..a45c0ac6ef 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5837,7 +5837,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { tFilePage **data = calloc(pQuery->numOfExpr2, POINTER_BYTES); for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { int32_t bytes = pQuery->pExpr2[i].bytes; - data[i] = (tFilePage *)malloc(bytes * pQuery->rec.rows + sizeof(tFilePage)); + data[i] = (tFilePage *)malloc((size_t)(bytes * pQuery->rec.rows) + sizeof(tFilePage)); } arithSup.offset = 0; @@ -5859,7 +5859,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { if (pSqlFunc->functionId == pQuery->pExpr1[j].base.functionId && pSqlFunc->colInfo.colId == pQuery->pExpr1[j].base.colInfo.colId) { - memcpy(data[i]->data, pQuery->sdata[j]->data, pQuery->pExpr1[j].bytes * pQuery->rec.rows); + memcpy(data[i]->data, pQuery->sdata[j]->data, (size_t)(pQuery->pExpr1[j].bytes * pQuery->rec.rows)); break; } } @@ -5871,7 +5871,7 @@ static void doSecondaryArithmeticProcess(SQuery* pQuery) { } for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { - memcpy(pQuery->sdata[i]->data, data[i]->data, pQuery->pExpr2[i].bytes * pQuery->rec.rows); + memcpy(pQuery->sdata[i]->data, data[i]->data, (size_t)(pQuery->pExpr2[i].bytes * pQuery->rec.rows)); } for (int32_t i = 0; i < pQuery->numOfExpr2; ++i) { From cf66d1206d5b5677ac98906e4407a1f71157bd30 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 5 Dec 2020 16:33:38 +0800 Subject: [PATCH 188/188] update faq and model-ch. --- documentation20/webdocs/markdowndocs/Model-ch.md | 10 ++++++---- documentation20/webdocs/markdowndocs/faq-ch.md | 14 +++++++++----- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/documentation20/webdocs/markdowndocs/Model-ch.md b/documentation20/webdocs/markdowndocs/Model-ch.md index d698e3daaf..c0b64bde6e 100644 --- a/documentation20/webdocs/markdowndocs/Model-ch.md +++ b/documentation20/webdocs/markdowndocs/Model-ch.md @@ -12,7 +12,7 @@ TDengine采用关系型数据模型，需要建库、建表。因此对于一个 CREATE DATABASE power KEEP 365 DAYS 10 BLOCKS 4; ``` 上述语句将创建一个名为power的库，这个库的数据将保留365天（超过365天将被自动删除），每10天一个数据文件，内存块数为4。详细的语法及参数请见TAOS SQL - + 创建库之后，需要使用SQL命令USE将当前库切换过来，例如： ```cmd @@ -20,7 +20,7 @@ USE power; ``` 就当前连接里操作的库换为power，否则对具体表操作前，需要使用“库名.表名”来指定库的名字。 - + **注意：** - 任何一张表或超级表是属于一个库的，在创建表之前，必须先创建库。 @@ -44,15 +44,17 @@ CREATE TABLE d1001 USING meters TAGS ("Beijing.Chaoyang", 2); ``` 其中d1001是表名，meters是超级表的表名，后面紧跟标签Location的具体标签值”Beijing.Chaoyang"，标签groupId的具体标签值2。虽然在创建表时，需要指定标签值，但可以事后修改。详细细则请见 TAOS SQL。 +**注意：**目前 TDengine 没有从技术层面限制使用一个 database （dbA）的超级表作为模板建立另一个 database （dbB）的子表，后续会禁止这种用法，不建议使用这种方法建表。 + TDengine建议将数据采集点的全局唯一ID作为表名(比如设备序列号）。但对于有的场景，并没有唯一的ID，可以将多个ID组合成一个唯一的ID。不建议将具有唯一性的ID作为标签值。 - + **自动建表**：在某些特殊场景中，用户在写数据时并不确定某个数据采集点的表是否存在，此时可在写入数据时使用自动建表语法来创建不存在的表，若该表已存在则不会建立新表。比如： ```cmd INSERT INTO d1001 USING METERS TAGS ("Beijng.Chaoyang", 2) VALUES (now, 10.2, 219, 0.32); ``` 上述SQL语句将记录(now, 10.2, 219, 0.32) 插入进表d1001。如果表d1001还未创建，则使用超级表meters做模板自动创建，同时打上标签值“Beijing.Chaoyang", 2。 - + ## 多列模型 vs 单列模型 TDengine支持多列模型，只要物理量是一个数据采集点同时采集的（时间戳一致），这些量就可以作为不同列放在一张超级表里。但还有一种极限的设计，单列模型，每个采集的物理量都单独建表，因此每种类型的物理量都单独建立一超级表。比如电流、电压、相位，就建三张超级表。 diff --git a/documentation20/webdocs/markdowndocs/faq-ch.md b/documentation20/webdocs/markdowndocs/faq-ch.md index 42f9761c95..61f70c5962 100644 --- a/documentation20/webdocs/markdowndocs/faq-ch.md +++ b/documentation20/webdocs/markdowndocs/faq-ch.md @@ -36,16 +36,20 @@ 5. ping服务器FQDN，如果没有反应，请检查你的网络，DNS设置，或客户端所在计算机的系统hosts文件 -6. 检查防火墙设置，确认TCP/UDP 端口6030-6042 是打开的 +6. 检查防火墙设置（Ubuntu 使用 ufw status，CentOS 使用 firewall-cmd --list-port），确认TCP/UDP 端口6030-6042 是打开的 7. 对于Linux上的JDBC（ODBC, Python, Go等接口类似）连接, 确保*libtaos.so*在目录*/usr/local/taos/driver*里, 并且*/usr/local/taos/driver*在系统库函数搜索路径*LD_LIBRARY_PATH*里 8. 对于windows上的JDBC, ODBC, Python, Go等连接，确保*C:\TDengine\driver\taos.dll*在你的系统库函数搜索目录里 (建议*taos.dll*放在目录 *C:\Windows\System32*) -9. 如果仍不能排除连接故障，请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 - 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` - 检查服务器侧TCP端口连接是否工作：`nc -l {port}` - 检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}` +9. 如果仍不能排除连接故障 + + * Linux 系统请使用命令行工具nc来分别判断指定端口的TCP和UDP连接是否通畅 + 检查UDP端口连接是否工作：`nc -vuz {hostIP} {port} ` + 检查服务器侧TCP端口连接是否工作：`nc -l {port}` + 检查客户端侧TCP端口连接是否工作：`nc {hostIP} {port}` + + * Windows 系统请使用 PowerShell 命令 Net-TestConnection -ComputerName {fqdn} -Port {port} 检测服务段端口是否访问 10. 也可以使用taos程序内嵌的网络连通检测功能，来验证服务器和客户端之间指定的端口连接是否通畅（包括TCP和UDP）：[TDengine 内嵌网络检测工具使用指南](https://www.taosdata.com/blog/2020/09/08/1816.html)。