From ec8ac48a606e311c3c030a408510e74df30cd073 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 13 Nov 2020 12:44:53 +0000 Subject: [PATCH 01/41] return value is pointer --- src/sync/src/syncMain.c | 2 +- src/util/tests/trefTest.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sync/src/syncMain.c b/src/sync/src/syncMain.c index 8d90315c48..4b51c770e7 100644 --- a/src/sync/src/syncMain.c +++ b/src/sync/src/syncMain.c @@ -1097,7 +1097,7 @@ static void syncProcessBrokenLink(void *param) { SSyncPeer *pPeer = param; SSyncNode *pNode = pPeer->pSyncNode; - if (taosAcquireRef(tsSyncRefId, pNode->rid) < 0) return; + if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return; pthread_mutex_lock(&(pNode->mutex)); sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno)); diff --git a/src/util/tests/trefTest.c b/src/util/tests/trefTest.c index 6887b24abd..454860410b 100644 --- a/src/util/tests/trefTest.c +++ b/src/util/tests/trefTest.c @@ -77,8 +77,8 @@ void *acquireRelease(void *param) { printf("a"); id = random() % pSpace->refNum; - code = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); - if (code >= 0) { + void *p = taosAcquireRef(pSpace->rsetId, pSpace->p[id]); + if (p) { usleep(id % 5 + 1); taosReleaseRef(pSpace->rsetId, pSpace->p[id]); } From 4c11e46075d65d1bc29a116ed9359dd36c80d421 Mon Sep 17 00:00:00 2001 From: zyyang Date: Wed, 18 Nov 2020 17:40:00 +0800 Subject: [PATCH 02/41] change --- .../com/taosdata/jdbc/rs/RestfulDriver.java | 3 +- .../taosdata/jdbc/rs/RestfulDriverTest.java | 35 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index c267f660de..a07d6eb2b6 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -86,6 +86,7 @@ public class RestfulDriver extends AbstractTaosDriver { @Override public Logger getParentLogger() throws SQLFeatureNotSupportedException { - return null; + //TODO SQLFeatureNotSupportedException + throw new SQLFeatureNotSupportedException(); } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index a91d1c2d6b..768eceab8e 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -31,10 +31,43 @@ public class RestfulDriverTest { } @Test - public void testAcceptUrl() throws SQLException { + public void connect() { + + } + + @Test + public void acceptsURL() throws SQLException { Driver driver = new RestfulDriver(); boolean isAccept = driver.acceptsURL("jdbc:TAOS-RS://master:6041"); Assert.assertTrue(isAccept); + isAccept = driver.acceptsURL("jdbc:TAOS://master:6041"); + Assert.assertFalse(isAccept); } + @Test + public void getPropertyInfo() throws SQLException { + Driver driver = new RestfulDriver(); + final String url = ""; + DriverPropertyInfo[] propertyInfo = driver.getPropertyInfo(url, null); + } + + @Test + public void getMajorVersion() { + Assert.assertEquals(2, new RestfulDriver().getMajorVersion()); + } + + @Test + public void getMinorVersion() { + Assert.assertEquals(0, new RestfulDriver().getMinorVersion()); + } + + @Test + public void jdbcCompliant() { + Assert.assertFalse(new RestfulDriver().jdbcCompliant()); + } + + @Test(expected = SQLFeatureNotSupportedException.class) + public void getParentLogger() throws SQLFeatureNotSupportedException { + new RestfulDriver().getParentLogger(); + } } From a8b87ce0cd79c498fc370f7c4656f335115256fd Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 20 Nov 2020 01:07:33 +0000 Subject: [PATCH 03/41] change applications --- src/query/src/qResultbuf.c | 8 +- src/util/inc/hash.h | 53 ++---- src/util/src/hash.c | 332 ++++++++++++++++++------------------- src/util/src/tkvstore.c | 19 +-- src/vnode/src/vnodeMain.c | 28 ++-- 5 files changed, 191 insertions(+), 249 deletions(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 2645cff678..e9ed8dc1c3 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -420,9 +420,8 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { unlink(pResultBuf->path); tfree(pResultBuf->path); - SHashMutableIterator* iter = taosHashCreateIter(pResultBuf->groupSet); - while(taosHashIterNext(iter)) { - SArray** p = (SArray**) taosHashIterGet(iter); + SArray** p = taosHashIterate(pResultBuf->groupSet, NULL); + while(p) { size_t n = taosArrayGetSize(*p); for(int32_t i = 0; i < n; ++i) { SPageInfo* pi = taosArrayGetP(*p, i); @@ -431,10 +430,9 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } taosArrayDestroy(*p); + p = taosHashIterate(pResultBuf->groupSet, p); } - taosHashDestroyIter(iter); - tdListFree(pResultBuf->lruList); taosArrayDestroy(pResultBuf->emptyDummyIdList); taosHashCleanup(pResultBuf->groupSet); diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 42bc136584..aaff48f4f8 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -31,16 +31,17 @@ extern "C" { typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { -// char *key; struct SHashNode *next; uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key -// char *data; + uint32_t dataLen; // length of data + int8_t count; // reference count + char data[]; } SHashNode; -#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode)) -#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen) - +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((char*)(_n) - sizeof(SHashNode)); typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, @@ -65,15 +66,6 @@ typedef struct SHashObj { SArray *pMemBlock; // memory block allocated for SHashEntry } SHashObj; -typedef struct SHashMutableIterator { - SHashObj *pHashObj; - int32_t entryIndex; - SHashNode *pCur; - SHashNode *pNext; // current node can be deleted for mutable iterator, so keep the next one before return current - size_t numOfChecked; // already check number of elements in hash table - size_t numOfEntries; // number of entries while the iterator is created -} SHashMutableIterator; - /** * init the hash table * @@ -142,33 +134,9 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi */ void taosHashCleanup(SHashObj *pHashObj); -/** - * - * @param pHashObj - * @return - */ -SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj); - -/** - * - * @param iter - * @return - */ -bool taosHashIterNext(SHashMutableIterator *iter); - -/** - * - * @param iter - * @return - */ -void *taosHashIterGet(SHashMutableIterator *iter); - -/** - * - * @param iter - * @return - */ -void* taosHashDestroyIter(SHashMutableIterator* iter); +/* +void *SHashMutableIterator* taosHashCreateIter(SHashObj *pHashObj, void *); +*/ /** * @@ -179,6 +147,9 @@ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); size_t taosHashGetMemSize(const SHashObj *pHashObj); +void *taosHashIterate(SHashObj *pHashObj, void *p); +void taosHashCancelIterate(SHashObj *pHashObj, void *p); + #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 03a7342497..cd9a9f7d0a 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -114,15 +114,25 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashObj *pHashObj, SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); + + pNode->count--; if (prev != NULL) { prev->next = pNewNode; } else { pe->next = pNewNode; } - pNewNode->next = pNode->next; + if (pNode->count <= 0) { + pNewNode->next = pNode->next; + DO_FREE_HASH_NODE(pNode); + } else { + pNewNode->next = pNode; + pe->num++; + atomic_add_fetch_64(&pHashObj->size, 1); + } + return pNewNode; } @@ -139,7 +149,6 @@ static void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode); * @param pIter * @return */ -static SHashNode *getNextHashNode(SHashMutableIterator *pIter); SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type) { if (capacity == 0 || fn == NULL) { @@ -244,8 +253,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } else { // not support the update operation, return error if (pHashObj->enableUpdate) { - doUpdateHashNode(pe, prev, pNode, pNewNode); - DO_FREE_HASH_NODE(pNode); + doUpdateHashNode(pHashObj, pe, prev, pNode, pNewNode); } else { DO_FREE_HASH_NODE(pNewNode); } @@ -335,22 +343,10 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity); SHashEntry *pe = pHashObj->hashList[slot]; - // no data, return directly - if (pe->num == 0) { - __rd_unlock(&pHashObj->lock, pHashObj->type); - return -1; - } - if (pHashObj->type == HASH_ENTRY_LOCK) { taosWLockLatch(&pe->latch); } - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } - // double check after locked if (pe->num == 0) { assert(pe->next == NULL); @@ -360,37 +356,36 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe return -1; } + int code = -1; SHashNode *pNode = pe->next; - SHashNode *pRes = NULL; + SHashNode *prevNode = NULL; - // remove it - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { - pe->num -= 1; - pRes = pNode; - pe->next = pNode->next; - } else { - while (pNode->next != NULL) { - if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) { - assert((pNode->next)->hashVal == hashVal); - break; + while (pNode) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) + break; + + prevNode = pNode; + pNode = pNode->next; + } + + if (pNode) { + code = 0; // it is found + + pNode->count--; + if (pNode->count <= 0) { + if (prevNode) { + prevNode->next = pNode->next; + } else { + pe->next = pNode->next; } + + if (data) memcpy(data, GET_HASH_NODE_DATA(pNode), dsize); - pNode = pNode->next; + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pNode); } - - - if (pNode->next != NULL) { - pe->num -= 1; - pRes = pNode->next; - pNode->next = pNode->next->next; - } - } - - if (pe->num == 0) { - assert(pe->next == NULL); - } else { - assert(pe->next != NULL); - } + } if (pHashObj->type == HASH_ENTRY_LOCK) { taosWUnLockLatch(&pe->latch); @@ -398,17 +393,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe __rd_unlock(&pHashObj->lock, pHashObj->type); - if (data != NULL && pRes != NULL) { - memcpy(data, GET_HASH_NODE_DATA(pRes), dsize); - } - - if (pRes != NULL) { - atomic_sub_fetch_64(&pHashObj->size, 1); - FREE_HASH_NODE(pHashObj, pRes); - return 0; - } else { - return -1; - } + return code; } int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), void *param) { @@ -531,98 +516,6 @@ void taosHashCleanup(SHashObj *pHashObj) { free(pHashObj); } -SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { - SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator)); - if (pIter == NULL) { - return NULL; - } - - pIter->pHashObj = pHashObj; - - // keep it in local variable, in case the resize operation expand the size - pIter->numOfEntries = pHashObj->capacity; - return pIter; -} - -bool taosHashIterNext(SHashMutableIterator *pIter) { - if (pIter == NULL) { - return false; - } - - size_t size = taosHashGetSize(pIter->pHashObj); - if (size == 0) { - return false; - } - - // check the first one - if (pIter->numOfChecked == 0) { - assert(pIter->pCur == NULL && pIter->pNext == NULL); - - while (1) { - SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry->num == 0) { - assert(pEntry->next == NULL); - - pIter->entryIndex++; - continue; - } - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pEntry->latch); - } - - pIter->pCur = pEntry->next; - - if (pIter->pCur->next) { - pIter->pNext = pIter->pCur->next; - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - } else { - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - - pIter->pNext = getNextHashNode(pIter); - } - - break; - } - - pIter->numOfChecked++; - return true; - } else { - assert(pIter->pCur != NULL); - if (pIter->pNext) { - pIter->pCur = pIter->pNext; - } else { // no more data in the hash list - return false; - } - - pIter->numOfChecked++; - - if (pIter->pCur->next) { - pIter->pNext = pIter->pCur->next; - } else { - pIter->pNext = getNextHashNode(pIter); - } - - return true; - } -} - -void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); } - -void *taosHashDestroyIter(SHashMutableIterator *iter) { - if (iter == NULL) { - return NULL; - } - - free(iter); - return NULL; -} - // for profile only int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj) { if (pHashObj == NULL || pHashObj->size == 0) { @@ -759,6 +652,8 @@ SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, s pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; + pNewNode->dataLen = dsize; + pNewNode->count = 1; memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); @@ -775,35 +670,6 @@ void pushfrontNodeInEntryList(SHashEntry *pEntry, SHashNode *pNode) { pEntry->num += 1; } -SHashNode *getNextHashNode(SHashMutableIterator *pIter) { - assert(pIter != NULL); - - pIter->entryIndex++; - SHashNode *p = NULL; - - while (pIter->entryIndex < pIter->numOfEntries) { - SHashEntry *pEntry = pIter->pHashObj->hashList[pIter->entryIndex]; - if (pEntry->num == 0) { - pIter->entryIndex++; - continue; - } - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRLockLatch(&pEntry->latch); - } - - p = pEntry->next; - - if (pIter->pHashObj->type == HASH_ENTRY_LOCK) { - taosRUnLockLatch(&pEntry->latch); - } - - return p; - } - - return NULL; -} - size_t taosHashGetMemSize(const SHashObj *pHashObj) { if (pHashObj == NULL) { return 0; @@ -811,3 +677,119 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } + +// release the pNode, return next pNode, and lock the current entry +static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { + + SHashNode *pOld = (SHashNode *)GET_HASH_PNODE(p); + SHashNode *prevNode = NULL; + + *slot = HASH_INDEX(pOld->hashVal, pHashObj->capacity); + SHashEntry *pe = pHashObj->hashList[*slot]; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + SHashNode *pNode = pe->next; + + while (pNode) { + if (pNode == pOld) + break; + + prevNode = pNode; + pNode = pNode->next; + } + + if (pNode) { + pNode = pNode->next; + pOld->count--; + if (pOld->count <=0) { + if (prevNode) { + prevNode->next = pOld->next; + } else { + pe->next = pOld->next; + } + + pe->num--; + atomic_sub_fetch_64(&pHashObj->size, 1); + FREE_HASH_NODE(pHashObj, pOld); + } + } else { + uError("pNode:%p data:%p is not there!!!", pNode, p); + } + + return pNode; +} + +void *taosHashIterate(SHashObj *pHashObj, void *p) { + if (pHashObj == NULL) return NULL; + + int slot = 0; + char *data = NULL; + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + SHashNode *pNode = NULL; + if (p) { + pNode = taosHashReleaseNode(pHashObj, p, &slot); + if (pNode == NULL) { + SHashEntry *pe = pHashObj->hashList[slot]; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + slot = slot + 1; + } + } + + if (pNode == NULL) { + for (; slot < pHashObj->capacity; ++slot) { + SHashEntry *pe = pHashObj->hashList[slot]; + + // lock entry + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWLockLatch(&pe->latch); + } + + pNode = pe->next; + if (pNode) break; + + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + } + } + + if (pNode) { + SHashEntry *pe = pHashObj->hashList[slot]; + pNode->count++; + data = GET_HASH_NODE_DATA(pNode); + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); + return data; + +} + +void taosHashCancelIterate(SHashObj *pHashObj, void *p) { + if (pHashObj == NULL || p == NULL) return; + + // only add the read lock to disable the resize process + __rd_lock(&pHashObj->lock, pHashObj->type); + + int slot; + taosHashReleaseNode(pHashObj, p, &slot); + + SHashEntry *pe = pHashObj->hashList[slot]; + if (pHashObj->type == HASH_ENTRY_LOCK) { + taosWUnLockLatch(&pe->latch); + } + + __rd_unlock(&pHashObj->lock, pHashObj->type); +} diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 31641ac9a7..2b1d13c78b 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -529,7 +529,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { void * buf = NULL; int64_t maxBufSize = 0; SKVRecord rInfo = {0}; - SHashMutableIterator *pIter = NULL; + SKVRecord *pRecord = NULL; ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR)); ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE); @@ -582,16 +582,8 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } - pIter = taosHashCreateIter(pStore->map); - if (pIter == NULL) { - uError("failed to create hash iter while opening KV store %s", pStore->fname); - terrno = TSDB_CODE_COM_OUT_OF_MEMORY; - goto _err; - } - - while (taosHashIterNext(pIter)) { - SKVRecord *pRecord = taosHashIterGet(pIter); - + pRecord = taosHashIterate(pStore->map, NULL); + while (pRecord) { if (lseek(pStore->fd, (off_t)(pRecord->offset + sizeof(SKVRecord)), SEEK_SET) < 0) { uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); @@ -613,16 +605,17 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } } + + pRecord = taosHashIterate(pStore->map, pRecord); } if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); - taosHashDestroyIter(pIter); tfree(buf); return 0; _err: - taosHashDestroyIter(pIter); + taosHashCancelIterate(pStore->map, pRecord); tfree(buf); return -1; } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 7447acc488..e3eb35ea32 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -517,11 +517,10 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { } int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { - SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); - while (taosHashIterNext(pIter)) { - SVnodeObj **pVnode = taosHashIterGet(pIter); - if (pVnode == NULL) continue; - if (*pVnode == NULL) continue; + void *pIter = taosHashIterate(tsVnodesHash, NULL); + while (pIter) { + SVnodeObj **pVnode = pIter; + if (*pVnode) { (*numOfVnodes)++; if (*numOfVnodes >= TSDB_MAX_VNODES) { @@ -530,25 +529,24 @@ int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { } else { vnodeList[*numOfVnodes - 1] = (*pVnode)->vgId; } - } - taosHashDestroyIter(pIter); + } + + pIter = taosHashIterate(tsVnodesHash, pIter); + } return TSDB_CODE_SUCCESS; } void vnodeBuildStatusMsg(void *param) { SStatusMsg *pStatus = param; - SHashMutableIterator *pIter = taosHashCreateIter(tsVnodesHash); - - while (taosHashIterNext(pIter)) { - SVnodeObj **pVnode = taosHashIterGet(pIter); - if (pVnode == NULL) continue; - if (*pVnode == NULL) continue; + void *pIter = taosHashIterate(tsVnodesHash, NULL); + while (pIter) { + SVnodeObj **pVnode = pIter; + if (*pVnode) { vnodeBuildVloadMsg(*pVnode, pStatus); + } } - - taosHashDestroyIter(pIter); } void vnodeSetAccess(SVgroupAccess *pAccess, int32_t numOfVnodes) { From 3e43a6533189e060a5e7f848959fd0bda55685da Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 21 Nov 2020 20:45:06 +0800 Subject: [PATCH 04/41] TD-1983 change hash in mnode --- src/balance/src/balance.c | 40 ++++++++++++------------------ src/mnode/inc/mnodeAcct.h | 1 + src/mnode/inc/mnodeDb.h | 1 + src/mnode/inc/mnodeDnode.h | 1 + src/mnode/inc/mnodeMnode.h | 1 + src/mnode/inc/mnodeSdb.h | 6 ++--- src/mnode/inc/mnodeShow.h | 2 ++ src/mnode/inc/mnodeTable.h | 2 ++ src/mnode/inc/mnodeUser.h | 1 + src/mnode/inc/mnodeVgroup.h | 1 + src/mnode/src/mnodeAcct.c | 6 +++-- src/mnode/src/mnodeCluster.c | 8 +++++- src/mnode/src/mnodeDb.c | 12 +++++---- src/mnode/src/mnodeDnode.c | 23 +++++++++--------- src/mnode/src/mnodeMnode.c | 9 ++++--- src/mnode/src/mnodeProfile.c | 24 +++++++++--------- src/mnode/src/mnodeSdb.c | 36 ++++++++++++--------------- src/mnode/src/mnodeShow.c | 9 ++++++- src/mnode/src/mnodeTable.c | 47 ++++++++++++++++++------------------ src/mnode/src/mnodeUser.c | 9 ++++--- src/mnode/src/mnodeVgroup.c | 15 +++++------- src/vnode/src/vnodeMain.c | 7 +++--- tests/script/basicSuite.sim | 1 - 23 files changed, 139 insertions(+), 123 deletions(-) diff --git a/src/balance/src/balance.c b/src/balance/src/balance.c index df78f4fe27..b172867929 100644 --- a/src/balance/src/balance.c +++ b/src/balance/src/balance.c @@ -162,7 +162,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { pDnode->openVnodes, pDnode->diskAvailable, pDnode->alternativeRole); mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); if (mnodeGetOnlineDnodesNum() == 0) { return TSDB_CODE_MND_NOT_READY; @@ -377,15 +376,13 @@ static bool balanceMonitorBalance() { srcScore, pDestDnode->score, destScore); balanceAddVnode(pVgroup, pSrcDnode, pDestDnode); mnodeDecVgroupRef(pVgroup); - sdbFreeIter(pIter); + mnodeCancelGetNextVgroup(pIter); return true; } } mnodeDecVgroupRef(pVgroup); } - - sdbFreeIter(pIter); } return false; @@ -413,8 +410,6 @@ void balanceReset() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - tsAccessSquence = 0; } @@ -443,12 +438,11 @@ static int32_t balanceMonitorVgroups() { mnodeDecVgroupRef(pVgroup); if (code == TSDB_CODE_SUCCESS) { + mnodeCancelGetNextVgroup(pIter); break; } } - sdbFreeIter(pIter); - return hasUpdatingVgroup; } @@ -465,11 +459,12 @@ static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) { hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup); mnodeDecVgroupRef(pVgroup); - if (hasThisDnode) break; + if (hasThisDnode) { + mnodeCancelGetNextVgroup(pIter); + break; + } } - sdbFreeIter(pIter); - if (!hasThisDnode) { mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId); mnodeDropDnode(pDnode, NULL); @@ -499,20 +494,18 @@ static bool balanceMontiorDropping() { pDnode->status = TAOS_DN_STATUS_DROPPING; mnodeUpdateDnode(pDnode); mnodeDecDnodeRef(pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return true; } if (pDnode->status == TAOS_DN_STATUS_DROPPING) { bool ret = balanceMonitorDnodeDropping(pDnode); mnodeDecDnodeRef(pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return ret; } } - sdbFreeIter(pIter); - return false; } @@ -556,8 +549,6 @@ static void balanceSetVgroupOffline(SDnodeObj* pDnode) { } mnodeDecVgroupRef(pVgroup); } - - sdbFreeIter(pIter); } static void balanceCheckDnodeAccess() { @@ -578,8 +569,6 @@ static void balanceCheckDnodeAccess() { } mnodeDecDnodeRef(pDnode); } - - sdbFreeIter(pIter); } static void balanceProcessBalanceTimer(void *handle, void *tmrId) { @@ -630,6 +619,7 @@ void balanceAsyncNotify() { int32_t balanceInit() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_SCORES, mnodeCancelGetNextDnode); pthread_mutex_init(&tsBalanceMutex, NULL); balanceInitDnodeList(); @@ -667,8 +657,6 @@ int32_t balanceDropDnode(SDnodeObj *pDnode) { mnodeDecDnodeRef(pTempDnode); } - sdbFreeIter(pIter); - if (pDnode->openVnodes > totalFreeVnodes) { mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes); return TSDB_CODE_MND_NO_ENOUGH_DNODES; @@ -780,8 +768,12 @@ void balanceAccquireDnodeList() { SDnodeObj *pDnode = NULL; int32_t dnodeIndex = 0; - while (1) { - if (dnodeIndex >= dnodesNum) break; + while (1) { + if (dnodeIndex >= dnodesNum) { + mnodeCancelGetNextDnode(pIter); + break; + } + pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; if (pDnode->status == TAOS_DN_STATUS_OFFLINE) { @@ -802,8 +794,6 @@ void balanceAccquireDnodeList() { dnodeIndex++; } - sdbFreeIter(pIter); - tsBalanceDnodeListSize = dnodeIndex; } diff --git a/src/mnode/inc/mnodeAcct.h b/src/mnode/inc/mnodeAcct.h index 744a62f948..595dcca413 100644 --- a/src/mnode/inc/mnodeAcct.h +++ b/src/mnode/inc/mnodeAcct.h @@ -27,6 +27,7 @@ void mnodeCleanupAccts(); void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo); void * mnodeGetAcct(char *acctName); void * mnodeGetNextAcct(void *pIter, SAcctObj **pAcct); +void mnodeCancelGetNextAcct(void *pIter); void mnodeIncAcctRef(SAcctObj *pAcct); void mnodeDecAcctRef(SAcctObj *pAcct); void mnodeAddDbToAcct(SAcctObj *pAcct, SDbObj *pDb); diff --git a/src/mnode/inc/mnodeDb.h b/src/mnode/inc/mnodeDb.h index 7cbd08ed92..9354b923d7 100644 --- a/src/mnode/inc/mnodeDb.h +++ b/src/mnode/inc/mnodeDb.h @@ -34,6 +34,7 @@ int64_t mnodeGetDbNum(); SDbObj *mnodeGetDb(char *db); SDbObj *mnodeGetDbByTableId(char *db); void * mnodeGetNextDb(void *pIter, SDbObj **pDb); +void mnodeCancelGetNextDb(void *pIter); void mnodeIncDbRef(SDbObj *pDb); void mnodeDecDbRef(SDbObj *pDb); bool mnodeCheckIsMonitorDB(char *db, char *monitordb); diff --git a/src/mnode/inc/mnodeDnode.h b/src/mnode/inc/mnodeDnode.h index b6ddb7a9bf..8bc29ef9ef 100644 --- a/src/mnode/inc/mnodeDnode.h +++ b/src/mnode/inc/mnodeDnode.h @@ -65,6 +65,7 @@ int32_t mnodeGetDnodesNum(); int32_t mnodeGetOnlinDnodesCpuCoreNum(); int32_t mnodeGetOnlineDnodesNum(); void * mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode); +void mnodeCancelGetNextDnode(void *pIter); void mnodeIncDnodeRef(SDnodeObj *pDnode); void mnodeDecDnodeRef(SDnodeObj *pDnode); void * mnodeGetDnode(int32_t dnodeId); diff --git a/src/mnode/inc/mnodeMnode.h b/src/mnode/inc/mnodeMnode.h index a28a03ea40..10cbcebe22 100644 --- a/src/mnode/inc/mnodeMnode.h +++ b/src/mnode/inc/mnodeMnode.h @@ -38,6 +38,7 @@ void mnodeDropMnodeLocal(int32_t dnodeId); void * mnodeGetMnode(int32_t mnodeId); int32_t mnodeGetMnodesNum(); void * mnodeGetNextMnode(void *pIter, struct SMnodeObj **pMnode); +void mnodeCancelGetNextMnode(void *pIter); void mnodeIncMnodeRef(struct SMnodeObj *pMnode); void mnodeDecMnodeRef(struct SMnodeObj *pMnode); diff --git a/src/mnode/inc/mnodeSdb.h b/src/mnode/inc/mnodeSdb.h index 29d8cf1207..81629ede78 100644 --- a/src/mnode/inc/mnodeSdb.h +++ b/src/mnode/inc/mnodeSdb.h @@ -92,9 +92,9 @@ int32_t sdbDeleteRow(SSdbRow *pRow); int32_t sdbUpdateRow(SSdbRow *pRow); int32_t sdbInsertRowToQueue(SSdbRow *pRow); -void *sdbGetRow(void *pTable, void *key); -void *sdbFetchRow(void *pTable, void *pIter, void **ppRow); -void sdbFreeIter(void *pIter); +void * sdbGetRow(void *pTable, void *key); +void * sdbFetchRow(void *pTable, void *pIter, void **ppRow); +void sdbFreeIter(void *pTable, void *pIter); void sdbIncRef(void *pTable, void *pRow); void sdbDecRef(void *pTable, void *pRow); int64_t sdbGetNumOfRows(void *pTable); diff --git a/src/mnode/inc/mnodeShow.h b/src/mnode/inc/mnodeShow.h index da66e71678..f985fe792d 100644 --- a/src/mnode/inc/mnodeShow.h +++ b/src/mnode/inc/mnodeShow.h @@ -26,8 +26,10 @@ void mnodeCleanUpShow(); typedef int32_t (*SShowMetaFp)(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); typedef int32_t (*SShowRetrieveFp)(SShowObj *pShow, char *data, int32_t rows, void *pConn); +typedef void (*SShowFreeIterFp)(void *pIter); void mnodeAddShowMetaHandle(uint8_t showType, SShowMetaFp fp); void mnodeAddShowRetrieveHandle(uint8_t showType, SShowRetrieveFp fp); +void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp); void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); #ifdef __cplusplus diff --git a/src/mnode/inc/mnodeTable.h b/src/mnode/inc/mnodeTable.h index 7c0077aa60..bf04f26a90 100644 --- a/src/mnode/inc/mnodeTable.h +++ b/src/mnode/inc/mnodeTable.h @@ -31,6 +31,8 @@ void mnodeIncTableRef(void *pTable); void mnodeDecTableRef(void *pTable); void * mnodeGetNextChildTable(void *pIter, SCTableObj **pTable); void * mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable); +void mnodeCancelGetNextChildTable(void *pIter); +void mnodeCancelGetNextSuperTable(void *pIter); void mnodeDropAllChildTables(SDbObj *pDropDb); void mnodeDropAllSuperTables(SDbObj *pDropDb); void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup); diff --git a/src/mnode/inc/mnodeUser.h b/src/mnode/inc/mnodeUser.h index 073460f9d3..156bc7aaeb 100644 --- a/src/mnode/inc/mnodeUser.h +++ b/src/mnode/inc/mnodeUser.h @@ -25,6 +25,7 @@ int32_t mnodeInitUsers(); void mnodeCleanupUsers(); SUserObj *mnodeGetUser(char *name); void * mnodeGetNextUser(void *pIter, SUserObj **pUser); +void mnodeCancelGetNextUser(void *pIter); void mnodeIncUserRef(SUserObj *pUser); void mnodeDecUserRef(SUserObj *pUser); SUserObj *mnodeGetUserFromConn(void *pConn); diff --git a/src/mnode/inc/mnodeVgroup.h b/src/mnode/inc/mnodeVgroup.h index 0e6d9dfde4..ee9ec7ae93 100644 --- a/src/mnode/inc/mnodeVgroup.h +++ b/src/mnode/inc/mnodeVgroup.h @@ -34,6 +34,7 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode); //void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb); void * mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup); +void mnodeCancelGetNextVgroup(void *pIter); void mnodeUpdateVgroup(SVgObj *pVgroup); void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload); void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t openVnodes); diff --git a/src/mnode/src/mnodeAcct.c b/src/mnode/src/mnodeAcct.c index 9fff2f0229..3da889b284 100644 --- a/src/mnode/src/mnodeAcct.c +++ b/src/mnode/src/mnodeAcct.c @@ -144,7 +144,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) { pAcctInfo->numOfTimeSeries += pAcct->acctInfo.numOfTimeSeries; mnodeDecAcctRef(pAcct); } - sdbFreeIter(pIter); SVgObj *pVgroup = NULL; pIter = NULL; @@ -158,7 +157,6 @@ void mnodeGetStatOfAllAcct(SAcctInfo* pAcctInfo) { pAcctInfo->totalPoints += pVgroup->pointsWritten; mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); } void *mnodeGetAcct(char *name) { @@ -169,6 +167,10 @@ void *mnodeGetNextAcct(void *pIter, SAcctObj **pAcct) { return sdbFetchRow(tsAcctSdb, pIter, (void **)pAcct); } +void mnodeCancelGetNextAcct(void *pIter) { + sdbFreeIter(tsAcctSdb, pIter); +} + void mnodeIncAcctRef(SAcctObj *pAcct) { sdbIncRef(tsAcctSdb, pAcct); } diff --git a/src/mnode/src/mnodeCluster.c b/src/mnode/src/mnodeCluster.c index 5be67e4ad9..839407c5b0 100644 --- a/src/mnode/src/mnodeCluster.c +++ b/src/mnode/src/mnodeCluster.c @@ -31,6 +31,7 @@ static int32_t mnodeCreateCluster(); static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static void mnodeCancelGetNextCluster(void *pIter); static int32_t mnodeClusterActionDestroy(SSdbRow *pRow) { tfree(pRow->pObj); @@ -108,6 +109,7 @@ int32_t mnodeInitCluster() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster); mDebug("table:%s, hash is created", desc.name); return TSDB_CODE_SUCCESS; @@ -122,6 +124,10 @@ void *mnodeGetNextCluster(void *pIter, SClusterObj **pCluster) { return sdbFetchRow(tsClusterSdb, pIter, (void **)pCluster); } +void mnodeCancelGetNextCluster(void *pIter) { + sdbFreeIter(tsClusterSdb, pIter); +} + void mnodeIncClusterRef(SClusterObj *pCluster) { sdbIncRef(tsClusterSdb, pCluster); } @@ -167,7 +173,7 @@ void mnodeUpdateClusterId() { } mnodeDecClusterRef(pCluster); - sdbFreeIter(pIter); + mnodeCancelGetNextCluster(pIter); } static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index d121208447..77f3b93eb1 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -171,6 +171,7 @@ int32_t mnodeInitDbs() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_DB, mnodeProcessDropDbMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DB, mnodeGetDbMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DB, mnodeRetrieveDbs); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DB, mnodeCancelGetNextDb); mDebug("table:dbs table is created"); return 0; @@ -180,6 +181,10 @@ void *mnodeGetNextDb(void *pIter, SDbObj **pDb) { return sdbFetchRow(tsDbSdb, pIter, (void **)pDb); } +void mnodeCancelGetNextDb(void *pIter) { + sdbFreeIter(tsDbSdb, pIter); +} + SDbObj *mnodeGetDb(char *db) { return (SDbObj *)sdbGetRow(tsDbSdb, db); } @@ -986,8 +991,8 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { SDbObj *pDb = pMsg->pDb; void *pIter = NULL; - while (1) { - SVgObj *pVgroup = NULL; + SVgObj *pVgroup = NULL; + while (1) { pIter = mnodeGetNextVgroup(pIter, &pVgroup); if (pVgroup == NULL) break; if (pVgroup->pDb == pDb) { @@ -995,7 +1000,6 @@ static int32_t mnodeAlterDbCb(SMnodeMsg *pMsg, int32_t code) { } mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); mDebug("db:%s, all vgroups is altered", pDb->name); mLInfo("db:%s, is alterd by %s", pDb->name, mnodeGetUserFromMsg(pMsg)); @@ -1146,7 +1150,5 @@ void mnodeDropAllDbs(SAcctObj *pAcct) { mnodeDecDbRef(pDb); } - sdbFreeIter(pIter); - mInfo("acct:%s, all dbs:%d is dropped from sdb", pAcct->user, numOfDbs); } diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index f76533c760..0e9b56e459 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -206,6 +206,7 @@ int32_t mnodeInitDnodes() { mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_DNODE, mnodeRetrieveDnodes); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_DNODE, mnodeCancelGetNextDnode); mDebug("table:dnodes table is created"); return 0; @@ -223,6 +224,10 @@ void *mnodeGetNextDnode(void *pIter, SDnodeObj **pDnode) { return sdbFetchRow(tsDnodeSdb, pIter, (void **)pDnode); } +void mnodeCancelGetNextDnode(void *pIter) { + sdbFreeIter(tsDnodeSdb, pIter); +} + int32_t mnodeGetDnodesNum() { return sdbGetNumOfRows(tsDnodeSdb); } @@ -241,8 +246,6 @@ int32_t mnodeGetOnlinDnodesCpuCoreNum() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - if (cpuCores < 2) cpuCores = 2; return cpuCores; } @@ -259,8 +262,6 @@ int32_t mnodeGetOnlineDnodesNum() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - return onlineDnodes; } @@ -276,13 +277,12 @@ void *mnodeGetDnodeByEp(char *ep) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; if (strcmp(ep, pDnode->dnodeEp) == 0) { - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); return pDnode; } mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); return NULL; } @@ -464,7 +464,10 @@ static void mnodeUpdateDnodeEps() { while (1) { pIter = mnodeGetNextDnode(pIter, &pDnode); if (pDnode == NULL) break; - if (dnodesNum >= totalDnodes) break; + if (dnodesNum >= totalDnodes) { + mnodeCancelGetNextDnode(pIter); + break; + } SDnodeEp *pEp = &tsDnodeEps->dnodeEps[dnodesNum]; dnodesNum++; @@ -474,7 +477,6 @@ static void mnodeUpdateDnodeEps() { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); pthread_mutex_unlock(&tsDnodeEpsMutex); } @@ -1100,7 +1102,7 @@ static int32_t mnodeGetVnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC pDnode = mnodeGetDnodeByEp(pShow->payload); } else { void *pIter = mnodeGetNextDnode(NULL, (SDnodeObj **)&pDnode); - sdbFreeIter(pIter); + mnodeCancelGetNextDnode(pIter); } if (pDnode != NULL) { @@ -1148,7 +1150,6 @@ static int32_t mnodeRetrieveVnodes(SShowObj *pShow, char *data, int32_t rows, vo mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); } else { numOfRows = 0; } @@ -1217,8 +1218,6 @@ int32_t balanceAllocVnodes(SVgObj *pVgroup) { mnodeDecDnodeRef(pDnode); } - sdbFreeIter(pIter); - if (pSelDnode == NULL) { mError("failed to alloc vnode to vgroup"); return TSDB_CODE_MND_NO_ENOUGH_DNODES; diff --git a/src/mnode/src/mnodeMnode.c b/src/mnode/src/mnodeMnode.c index 205bfda4b9..8e3ad0a248 100644 --- a/src/mnode/src/mnodeMnode.c +++ b/src/mnode/src/mnodeMnode.c @@ -123,7 +123,7 @@ static int32_t mnodeMnodeActionRestored() { pMnode->role = TAOS_SYNC_ROLE_MASTER; mnodeDecMnodeRef(pMnode); } - sdbFreeIter(pIter); + mnodeCancelGetNextMnode(pIter); } mnodeUpdateMnodeEpSet(); @@ -161,6 +161,7 @@ int32_t mnodeInitMnodes() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MNODE, mnodeGetMnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MNODE, mnodeRetrieveMnodes); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_MNODE, mnodeCancelGetNextMnode); mDebug("table:mnodes table is created"); return TSDB_CODE_SUCCESS; @@ -192,6 +193,10 @@ void *mnodeGetNextMnode(void *pIter, SMnodeObj **pMnode) { return sdbFetchRow(tsMnodeSdb, pIter, (void **)pMnode); } +void mnodeCancelGetNextMnode(void *pIter) { + sdbFreeIter(tsMnodeSdb, pIter); +} + void mnodeUpdateMnodeEpSet() { mInfo("update mnodes epSet, numOfEps:%d ", mnodeGetMnodesNum()); @@ -239,8 +244,6 @@ void mnodeUpdateMnodeEpSet() { tsMnodeEpSetForShell.numOfEps = index; tsMnodeEpSetForPeer.numOfEps = index; - sdbFreeIter(pIter); - mnodeMnodeUnLock(); } diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 12ac64854c..8467adfc17 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -42,6 +42,7 @@ static int32_t mnodeGetQueryMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC static int32_t mnodeRetrieveQueries(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveConns(SShowObj *pShow, char *data, int32_t rows, void *pConn); +static void mnodeCancelGetNextConn(void *pIter); static int32_t mnodeGetStreamMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, void *pConn); static void mnodeFreeConn(void *data); @@ -52,10 +53,13 @@ static int32_t mnodeProcessKillConnectionMsg(SMnodeMsg *pMsg); int32_t mnodeInitProfile() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_QUERIES, mnodeGetQueryMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_QUERIES, mnodeRetrieveQueries); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_QUERIES, mnodeCancelGetNextConn); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONNS, mnodeGetConnsMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONNS, mnodeRetrieveConns); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CONNS, mnodeCancelGetNextConn); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMS, mnodeGetStreamMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMS, mnodeRetrieveStreams); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMS, mnodeCancelGetNextConn); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_QUERY, mnodeProcessKillQueryMsg); mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_KILL_STREAM, mnodeProcessKillStreamMsg); @@ -137,21 +141,15 @@ static void mnodeFreeConn(void *data) { mDebug("connId:%d, is destroyed", pConn->connId); } -static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) { +static void *mnodeGetNextConn(void *pIter, SConnObj **pConn) { *pConn = NULL; - if (pIter == NULL) { - pIter = taosHashCreateIter(tsMnodeConnCache->pHashTable); - } + pIter = taosHashIterate(tsMnodeConnCache->pHashTable, pIter); + if (pIter == NULL) return NULL; - if (!taosHashIterNext(pIter)) { - taosHashDestroyIter(pIter); - return NULL; - } - - SCacheDataNode **pNode = taosHashIterGet(pIter); + SCacheDataNode **pNode = pIter; if (pNode == NULL || *pNode == NULL) { - taosHashDestroyIter(pIter); + taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter); return NULL; } @@ -159,6 +157,10 @@ static void *mnodeGetNextConn(SHashMutableIterator *pIter, SConnObj **pConn) { return pIter; } +static void mnodeCancelGetNextConn(void *pIter) { + taosHashCancelIterate(tsMnodeConnCache->pHashTable, pIter); +} + static int32_t mnodeGetConnsMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { SUserObj *pUser = mnodeGetUserFromConn(pConn); if (pUser == NULL) return 0; diff --git a/src/mnode/src/mnodeSdb.c b/src/mnode/src/mnodeSdb.c index 1728bb7e3b..575845b4f2 100644 --- a/src/mnode/src/mnodeSdb.c +++ b/src/mnode/src/mnodeSdb.c @@ -325,7 +325,6 @@ void sdbUpdateSync(void *pMnodes) { mnodeDecDnodeRef(pDnode); mnodeDecMnodeRef(pMnode); } - sdbFreeIter(pIter); syncCfg.replica = index; mDebug("vgId:1, mnodes info not input, use infos in sdb, numOfMnodes:%d", syncCfg.replica); } else { @@ -756,24 +755,17 @@ int32_t sdbUpdateRow(SSdbRow *pRow) { } } -void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) { +void *sdbFetchRow(void *tparam, void *pIter, void **ppRow) { SSdbTable *pTable = tparam; *ppRow = NULL; if (pTable == NULL) return NULL; - SHashMutableIterator *pIter = pNode; - if (pIter == NULL) { - pIter = taosHashCreateIter(pTable->iHandle); - } + pIter = taosHashIterate(pTable->iHandle, pIter); + if (pIter == NULL) return NULL; - if (!taosHashIterNext(pIter)) { - taosHashDestroyIter(pIter); - return NULL; - } - - void **ppMetaRow = taosHashIterGet(pIter); + void **ppMetaRow = pIter; if (ppMetaRow == NULL) { - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->iHandle, pIter); return NULL; } @@ -783,10 +775,11 @@ void *sdbFetchRow(void *tparam, void *pNode, void **ppRow) { return pIter; } -void sdbFreeIter(void *pIter) { - if (pIter != NULL) { - taosHashDestroyIter(pIter); - } +void sdbFreeIter(void *tparam, void *pIter) { + SSdbTable *pTable = tparam; + if (pTable == NULL || pIter == NULL) return; + + taosHashCancelIterate(pTable->iHandle, pIter); } void *sdbOpenTable(SSdbTableDesc *pDesc) { @@ -827,9 +820,10 @@ void sdbCloseTable(void *handle) { tsSdbMgmt.numOfTables--; tsSdbMgmt.tableList[pTable->id] = NULL; - SHashMutableIterator *pIter = taosHashCreateIter(pTable->iHandle); - while (taosHashIterNext(pIter)) { - void **ppRow = taosHashIterGet(pIter); + void *pIter = taosHashIterate(pTable->iHandle, NULL); + while (pIter) { + void **ppRow = pIter; + pIter = taosHashIterate(pTable->iHandle, pIter); if (ppRow == NULL) continue; SSdbRow row = { @@ -840,7 +834,7 @@ void sdbCloseTable(void *handle) { (*pTable->fpDestroy)(&row); } - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->iHandle, pIter); taosHashCleanup(pTable->iHandle); pthread_mutex_destroy(&pTable->mutex); diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 0d53fa9617..2621441cb5 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -57,6 +57,7 @@ static void *tsMnodeShowCache = NULL; static int32_t tsShowObjIndex = 0; static SShowMetaFp tsMnodeShowMetaFp[TSDB_MGMT_TABLE_MAX] = {0}; static SShowRetrieveFp tsMnodeShowRetrieveFp[TSDB_MGMT_TABLE_MAX] = {0}; +static SShowFreeIterFp tsMnodeShowFreeIterFp[TSDB_MGMT_TABLE_MAX] = {0}; int32_t mnodeInitShow() { mnodeAddReadMsgHandle(TSDB_MSG_TYPE_CM_SHOW, mnodeProcessShowMsg); @@ -85,6 +86,10 @@ void mnodeAddShowRetrieveHandle(uint8_t msgType, SShowRetrieveFp fp) { tsMnodeShowRetrieveFp[msgType] = fp; } +void mnodeAddShowFreeIterHandle(uint8_t msgType, SShowFreeIterFp fp) { + tsMnodeShowFreeIterFp[msgType] = fp; +} + static char *mnodeGetShowType(int32_t showType) { switch (showType) { case TSDB_MGMT_TABLE_ACCT: return "show accounts"; @@ -412,7 +417,9 @@ static void* mnodePutShowObj(SShowObj *pShow) { static void mnodeFreeShowObj(void *data) { SShowObj *pShow = *(SShowObj **)data; - sdbFreeIter(pShow->pIter); + if (tsMnodeShowFreeIterFp[pShow->type] != NULL && pShow->pIter != NULL) { + (*tsMnodeShowFreeIterFp[pShow->type])(pShow->pIter); + } mDebug("%p, show is destroyed, data:%p index:%d", pShow, data, pShow->index); tfree(pShow); diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index 4d5f0808f5..8b07f73c65 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -342,8 +342,7 @@ static int32_t mnodeChildTableActionRestored() { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - + mnodeCancelGetNextChildTable(pIter); return 0; } @@ -602,10 +601,13 @@ int32_t mnodeInitTables() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_TABLE, mnodeGetShowTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_TABLE, mnodeRetrieveShowTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_TABLE, mnodeCancelGetNextChildTable); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_METRIC, mnodeGetShowSuperTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_METRIC, mnodeRetrieveShowSuperTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_METRIC, mnodeCancelGetNextSuperTable); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeGetStreamTableMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeRetrieveStreamTables); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_STREAMTABLES, mnodeCancelGetNextChildTable); return TSDB_CODE_SUCCESS; } @@ -626,14 +628,12 @@ static void *mnodeGetSuperTableByUid(uint64_t uid) { pIter = mnodeGetNextSuperTable(pIter, &pStable); if (pStable == NULL) break; if (pStable->uid == uid) { - sdbFreeIter(pIter); + mnodeCancelGetNextSuperTable(pIter); return pStable; } mnodeDecTableRef(pStable); } - sdbFreeIter(pIter); - return NULL; } @@ -655,10 +655,18 @@ void *mnodeGetNextChildTable(void *pIter, SCTableObj **pTable) { return sdbFetchRow(tsChildTableSdb, pIter, (void **)pTable); } +void mnodeCancelGetNextChildTable(void *pIter) { + sdbFreeIter(tsChildTableSdb, pIter); +} + void *mnodeGetNextSuperTable(void *pIter, SSTableObj **pTable) { return sdbFetchRow(tsSuperTableSdb, pIter, (void **)pTable); } +void mnodeCancelGetNextSuperTable(void *pIter) { + sdbFreeIter(tsSuperTableSdb, pIter); +} + void mnodeIncTableRef(void *p1) { STableObj *pTable = (STableObj *)p1; if (pTable->type == TSDB_SUPER_TABLE) { @@ -914,10 +922,10 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { SSTableObj *pStable = (SSTableObj *)pMsg->pTable; if (pStable->vgHash != NULL /*pStable->numOfTables != 0*/) { - SHashMutableIterator *pIter = taosHashCreateIter(pStable->vgHash); - while (taosHashIterNext(pIter)) { - int32_t *pVgId = taosHashIterGet(pIter); + int32_t *pVgId = taosHashIterate(pStable->vgHash, NULL); + while (pVgId) { SVgObj *pVgroup = mnodeGetVgroup(*pVgId); + pVgId = taosHashIterate(pStable->vgHash, pVgId); if (pVgroup == NULL) break; SDropSTableMsg *pDrop = rpcMallocCont(sizeof(SDropSTableMsg)); @@ -933,7 +941,8 @@ static int32_t mnodeProcessDropSuperTableMsg(SMnodeMsg *pMsg) { dnodeSendMsgToDnode(&epSet, &rpcMsg); mnodeDecVgroupRef(pVgroup); } - taosHashDestroyIter(pIter); + + taosHashCancelIterate(pStable->vgHash, pVgId); mnodeDropAllChildTablesInStable(pStable); } @@ -1430,8 +1439,6 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("db:%s, all super tables:%d is dropped from sdb", pDropDb->name, numOfTables); } @@ -1523,11 +1530,11 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { } else { SVgroupsMsg *pVgroupMsg = (SVgroupsMsg *)msg; - SHashMutableIterator *pIter = taosHashCreateIter(pTable->vgHash); - int32_t vgSize = 0; - while (taosHashIterNext(pIter)) { - int32_t *pVgId = taosHashIterGet(pIter); - SVgObj * pVgroup = mnodeGetVgroup(*pVgId); + int32_t *pVgId = taosHashIterate(pTable->vgHash, NULL); + int32_t vgSize = 0; + while (pVgId) { + SVgObj *pVgroup = mnodeGetVgroup(*pVgId); + pVgId = taosHashIterate(pTable->vgHash, pVgId); if (pVgroup == NULL) continue; pVgroupMsg->vgroups[vgSize].vgId = htonl(pVgroup->vgId); @@ -1547,7 +1554,7 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) { mnodeDecVgroupRef(pVgroup); } - taosHashDestroyIter(pIter); + taosHashCancelIterate(pTable->vgHash, pVgId); mnodeDecTableRef(pTable); pVgroupMsg->numOfVgroups = htonl(vgSize); @@ -2230,8 +2237,6 @@ void mnodeDropAllChildTablesInVgroups(SVgObj *pVgroup) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("vgId:%d, all child tables is dropped from sdb", pVgroup->vgId); } @@ -2263,8 +2268,6 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("db:%s, all child tables:%d is dropped from sdb", pDropDb->name, numOfTables); } @@ -2293,8 +2296,6 @@ static void mnodeDropAllChildTablesInStable(SSTableObj *pStable) { mnodeDecTableRef(pTable); } - sdbFreeIter(pIter); - mInfo("stable:%s, all child tables:%d is dropped from sdb", pStable->info.tableId, numOfTables); } diff --git a/src/mnode/src/mnodeUser.c b/src/mnode/src/mnodeUser.c index dc76d92eb8..aee167631f 100644 --- a/src/mnode/src/mnodeUser.c +++ b/src/mnode/src/mnodeUser.c @@ -123,7 +123,6 @@ static void mnodePrintUserAuth() { } fflush(fp); - sdbFreeIter(pIter); fclose(fp); } @@ -177,6 +176,8 @@ int32_t mnodeInitUsers() { mnodeAddWriteMsgHandle(TSDB_MSG_TYPE_CM_DROP_USER, mnodeProcessDropUserMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_USER, mnodeGetUserMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_USER, mnodeRetrieveUsers); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_USER, mnodeCancelGetNextUser); + mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_AUTH, mnodeProcessAuthMsg); mDebug("table:%s, hash is created", desc.name); @@ -196,6 +197,10 @@ void *mnodeGetNextUser(void *pIter, SUserObj **pUser) { return sdbFetchRow(tsUserSdb, pIter, (void **)pUser); } +void mnodeCancelGetNextUser(void *pIter) { + sdbFreeIter(tsUserSdb, pIter); +} + void mnodeIncUserRef(SUserObj *pUser) { return sdbIncRef(tsUserSdb, pUser); } @@ -574,8 +579,6 @@ void mnodeDropAllUsers(SAcctObj *pAcct) { mnodeDecUserRef(pUser); } - sdbFreeIter(pIter); - mDebug("acct:%s, all users:%d is dropped from sdb", pAcct->user, numOfUsers); } diff --git a/src/mnode/src/mnodeVgroup.c b/src/mnode/src/mnodeVgroup.c index f9a49e5ec2..a8343083aa 100644 --- a/src/mnode/src/mnodeVgroup.c +++ b/src/mnode/src/mnodeVgroup.c @@ -230,6 +230,7 @@ int32_t mnodeInitVgroups() { mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VGROUP, mnodeGetVgroupMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VGROUP, mnodeRetrieveVgroups); + mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_VGROUP, mnodeCancelGetNextVgroup); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_CREATE_VNODE_RSP, mnodeProcessCreateVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_ALTER_VNODE_RSP, mnodeProcessAlterVnodeRsp); mnodeAddPeerRspHandle(TSDB_MSG_TYPE_MD_DROP_VNODE_RSP, mnodeProcessDropVnodeRsp); @@ -304,7 +305,7 @@ void mnodeCheckUnCreatedVgroup(SDnodeObj *pDnode, SVnodeLoad *pVloads, int32_t o mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); + mnodeCancelGetNextVgroup(pIter); } void mnodeUpdateVgroupStatus(SVgObj *pVgroup, SDnodeObj *pDnode, SVnodeLoad *pVload) { @@ -491,6 +492,10 @@ void *mnodeGetNextVgroup(void *pIter, SVgObj **pVgroup) { return sdbFetchRow(tsVgroupSdb, pIter, (void **)pVgroup); } +void mnodeCancelGetNextVgroup(void *pIter) { + sdbFreeIter(tsVgroupSdb, pIter); +} + static int32_t mnodeCreateVgroupFp(SMnodeMsg *pMsg) { SVgObj *pVgroup = pMsg->pVgroup; SDbObj *pDb = pMsg->pDb; @@ -1095,8 +1100,6 @@ void mnodeDropAllDnodeVgroups(SDnodeObj *pDropDnode) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("dnode:%d, all vgroups:%d is dropped from sdb", pDropDnode->dnodeId, numOfVgroups); } @@ -1118,8 +1121,6 @@ void mnodeUpdateAllDbVgroups(SDbObj *pAlterDb) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups is updated in sdb", pAlterDb->name); } #endif @@ -1147,8 +1148,6 @@ void mnodeDropAllDbVgroups(SDbObj *pDropDb) { mnodeDecVgroupRef(pVgroup); } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups:%d is dropped from sdb", pDropDb->name, numOfVgroups); } @@ -1170,7 +1169,5 @@ void mnodeSendDropAllDbVgroupsMsg(SDbObj *pDropDb) { numOfVgroups++; } - sdbFreeIter(pIter); - mInfo("db:%s, all vgroups:%d drop msg is sent to dnode", pDropDb->name, numOfVgroups); } diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 339391369f..74df989090 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -542,10 +542,11 @@ void vnodeBuildStatusMsg(void *param) { void *pIter = taosHashIterate(tsVnodesHash, NULL); while (pIter) { - SVnodeObj **pVnode = pIter; - if (*pVnode) { - vnodeBuildVloadMsg(*pVnode, pStatus); + SVnodeObj **pVnode = pIter; + if (*pVnode) { + vnodeBuildVloadMsg(*pVnode, pStatus); } + pIter = taosHashIterate(tsVnodesHash, pIter); } } diff --git a/tests/script/basicSuite.sim b/tests/script/basicSuite.sim index 61a9d68d26..5e22e02297 100644 --- a/tests/script/basicSuite.sim +++ b/tests/script/basicSuite.sim @@ -14,7 +14,6 @@ run general/table/vgroup.sim run general/user/authority.sim run general/vector/metrics_mix.sim run general/vector/table_field.sim -run general/user/authority.sim run general/tag/set.sim run general/table/delete_writing.sim run general/stable/disk.sim From d0e85228401fe967cb99dc06acc8a65bef573615 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 11:12:58 +0800 Subject: [PATCH 05/41] [TD-2175]: fix the error in result field name and nchar column length --- src/client/src/tscSql.c | 11 ++++++++++- src/query/inc/sql.y | 8 ++++---- src/query/src/sql.c | 11 +++++------ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 020305a0a8..de01776472 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -420,7 +420,16 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { for(int32_t i = 0; i < pFieldInfo->numOfOutput; ++i) { SInternalField* pField = tscFieldInfoGetInternalField(pFieldInfo, i); if (pField->visible) { - f[j++] = pField->field; + f[j] = pField->field; + + // revise the length for binary and nchar fields + if (f[j].type == TSDB_DATA_TYPE_BINARY) { + f[j].bytes -= VARSTR_HEADER_SIZE; + } else if (f[j].type == TSDB_DATA_TYPE_NCHAR) { + f[j].bytes = (f[j].bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE; + } + + j += 1; } } diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 09b1e1592a..e2b3bb6cbf 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -556,11 +556,11 @@ having_opt(A) ::= HAVING expr(X). {A = X;} //limit-offset subclause %type limit_opt {SLimitVal} limit_opt(A) ::= . {A.limit = -1; A.offset = 0;} -limit_opt(A) ::= LIMIT signed(X). {printf("aa1, %d\n", X); A.limit = X; A.offset = 0;} +limit_opt(A) ::= LIMIT signed(X). {A.limit = X; A.offset = 0;} limit_opt(A) ::= LIMIT signed(X) OFFSET signed(Y). - {printf("aa2\n, %d\n", X); A.limit = X; A.offset = Y;} + { A.limit = X; A.offset = Y;} limit_opt(A) ::= LIMIT signed(X) COMMA signed(Y). - {printf("aa3\n, %d\n", X); A.limit = Y; A.offset = X;} + { A.limit = Y; A.offset = X;} %type slimit_opt {SLimitVal} slimit_opt(A) ::= . {A.limit = -1; A.offset = 0;} @@ -581,7 +581,7 @@ where_opt(A) ::= WHERE expr(X). {A = X;} %type expr {tSQLExpr*} %destructor expr {tSQLExprDestroy($$);} -expr(A) ::= LP expr(X) RP. {A = X; } +expr(A) ::= LP(X) expr(Y) RP(Z). {A = Y; A->token.z = X.z; A->token.n = (Z.z - X.z + 1);} expr(A) ::= ID(X). {A = tSQLExprIdValueCreate(&X, TK_ID);} expr(A) ::= ID(X) DOT ID(Y). {X.n += (1+Y.n); A = tSQLExprIdValueCreate(&X, TK_ID);} diff --git a/src/query/src/sql.c b/src/query/src/sql.c index da2c56ee9e..d4e2b7f5f4 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -2628,16 +2628,14 @@ static void yy_reduce( {yymsp[1].minor.yy216.limit = -1; yymsp[1].minor.yy216.offset = 0;} break; case 177: /* limit_opt ::= LIMIT signed */ + case 181: /* slimit_opt ::= SLIMIT signed */ yytestcase(yyruleno==181); {yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} break; case 178: /* limit_opt ::= LIMIT signed OFFSET signed */ -{yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} +{ yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} break; case 179: /* limit_opt ::= LIMIT signed COMMA signed */ -{yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} - break; - case 181: /* slimit_opt ::= SLIMIT signed */ -{yymsp[-1].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-1].minor.yy216.offset = 0;} +{ yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} break; case 182: /* slimit_opt ::= SLIMIT signed SOFFSET signed */ {yymsp[-3].minor.yy216.limit = yymsp[-2].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[0].minor.yy207;} @@ -2646,7 +2644,8 @@ static void yy_reduce( {yymsp[-3].minor.yy216.limit = yymsp[0].minor.yy207; yymsp[-3].minor.yy216.offset = yymsp[-2].minor.yy207;} break; case 186: /* expr ::= LP expr RP */ -{yymsp[-2].minor.yy64 = yymsp[-1].minor.yy64; } +{yylhsminor.yy64 = yymsp[-1].minor.yy64; yylhsminor.yy64->token.z = yymsp[-2].minor.yy0.z; yylhsminor.yy64->token.n = (yymsp[0].minor.yy0.z - yymsp[-2].minor.yy0.z + 1);} + yymsp[-2].minor.yy64 = yylhsminor.yy64; break; case 187: /* expr ::= ID */ {yylhsminor.yy64 = tSQLExprIdValueCreate(&yymsp[0].minor.yy0, TK_ID);} From 98ba80a993556d12fcc8cffa26d70c711445bc23 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 11:13:46 +0800 Subject: [PATCH 06/41] [TD-225] refactor codes. --- src/query/inc/qUtil.h | 2 ++ src/query/src/qExecutor.c | 54 ++++++++++++++++++++++++-------------- src/query/src/qResultbuf.c | 2 +- src/query/src/qUtil.c | 51 +++++++++++++++++------------------ 4 files changed, 63 insertions(+), 46 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 4cd0e60ebe..b2c4b6e2ec 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -80,5 +80,7 @@ void* destroyResultRowPool(SResultRowPool* p); int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); +uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv); + #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d0874c36bc..c44d65e373 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -186,7 +186,7 @@ static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); -static void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); +static void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); @@ -283,8 +283,8 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { } } -static UNUSED_FUNC int32_t getGroupResultId(int32_t groupIndex) { - int32_t base = 20000000; +static int32_t getMergeResultGroupId(int32_t groupIndex) { + int32_t base = 50000000; return base + (groupIndex * 10000); } @@ -1115,7 +1115,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - // not assign result buffer yet, add new result buffer + // not assign result buffer yet, add new result buffer, TODO remove it char* d = pData; int16_t len = bytes; if (type == TSDB_DATA_TYPE_BINARY||type == TSDB_DATA_TYPE_NCHAR) { @@ -1128,7 +1128,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - uint64_t uid = groupIndex; // uid is always set to be 0. + uint64_t uid = groupIndex; SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); if (pResultRow == NULL) { return -1; @@ -1714,7 +1714,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order // if it is group by normal column, do not set output buffer, the output buffer is pResult // fixed output query/multi-output query for normal table if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); } if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { @@ -2936,10 +2936,24 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { assert(size == pGroupResInfo->numOfDataPages); bool done = false; + + //TODO add API for release none-dirty pages +// SPageInfo* prev = NULL; + for (int32_t j = pGroupResInfo->pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); + // release previous buffer pages +// if (prev == NULL) { +// prev = pi; +// } else { +// if (prev->pageId != pi->pageId) { +// releaseResBufPageInfo(pResultBuf, prev); +// prev = pi; +// } +// } + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num); int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId); @@ -3058,7 +3072,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); - pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); + pQInfo->groupResInfo.groupId = getMergeResultGroupId(pQInfo->groupIndex); // todo add windowRes iterator int64_t lastTimestamp = -1; @@ -3339,12 +3353,12 @@ int32_t initResultRow(SResultRow *pResultRow) { return TSDB_CODE_SUCCESS; } -void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { +void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t groupIndex = 0; - int32_t uid = 0; - SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); + int32_t tid = 0; + int64_t uid = getResultInfoUId(pRuntimeEnv); + SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid); for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -3427,7 +3441,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { pQuery->limit.offset -= pQuery->rec.rows; pQuery->rec.rows = 0; - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); // clear the buffer full flag if exists CLEAR_QUERY_STATUS(pQuery, QUERY_RESBUF_FULL); @@ -3792,7 +3806,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } - uint64_t uid = 0; // uid is always set to be 0 + uint64_t uid = getResultInfoUId(pRuntimeEnv); SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); if (pResultRow == NULL) { @@ -4629,10 +4643,10 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo int32_t ps = DEFAULT_PAGE_SIZE; int32_t rowsize = 0; getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - int32_t TWOMB = 1024*1024*2; + int32_t TENMB = 1024*1024*10; if (isSTableQuery && !onlyQueryTags(pRuntimeEnv->pQuery)) { - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4660,7 +4674,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } else if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery) || (!isSTableQuery)) { int32_t numOfResultRows = getInitialPageNum(pQInfo); getIntermediateBufInfo(pRuntimeEnv, &ps, &rowsize); - code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TWOMB, pQInfo); + code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, rowsize, ps, TENMB, pQInfo); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4930,7 +4944,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); if (isPointInterpoQuery(pQuery)) { - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); while (pQInfo->groupIndex < numOfGroups) { @@ -5096,7 +5110,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { return; } - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); resetTimeWindowInfo(pRuntimeEnv, &pRuntimeEnv->windowResInfo); SArray *group = GET_TABLEGROUP(pQInfo, 0); @@ -5456,7 +5470,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); } // skip blocks without load the actual data block from file if no filter condition present @@ -5486,7 +5500,7 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) qDebug("QInfo:%p skip current result, offset:%" PRId64 ", next qrange:%" PRId64 "-%" PRId64, pQInfo, pQuery->limit.offset, pQuery->current->lastKey, pQuery->current->win.ekey); - resetCtxOutputBuf(pRuntimeEnv); + resetDefaultResInfoOutputBuf(pRuntimeEnv); } limitResults(pRuntimeEnv); diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index c5ba551f20..67cb7af074 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -407,7 +407,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f", + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb", pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, pResultBuf->fileSize/1024.0); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index c5317226c7..650b886c36 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -20,6 +20,18 @@ #include "qExecutor.h" #include "qUtil.h" +static int32_t getResultRowKeyInfo(SResultRow* pResult, int16_t type, char** key, int16_t* bytes) { + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + *key = varDataVal(pResult->key); + *bytes = varDataLen(pResult->key); + } else { + *key = (char*) &pResult->win.skey; + *bytes = tDataTypeDesc[type].nSize; + } + + return 0; +} + int32_t getOutputInterResultBufSize(SQuery* pQuery) { int32_t size = 0; @@ -94,12 +106,8 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); assert(num >= 0 && num <= numOfClosed); - int16_t type = pWindowResInfo->type; - STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0. - int64_t uid = id->uid; - if (pRuntimeEnv->groupbyNormalCol) { - uid = 0; - } + int16_t type = pWindowResInfo->type; + int64_t uid = getResultInfoUId(pRuntimeEnv); char *key = NULL; int16_t bytes = -1; @@ -107,16 +115,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t i = 0; i < num; ++i) { SResultRow *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table - - // todo refactor - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - key = varDataVal(pResult->key); - bytes = varDataLen(pResult->key); - } else { - key = (char*) &pResult->win.skey; - bytes = tDataTypeDesc[pWindowResInfo->type].nSize; - } - + getResultRowKeyInfo(pResult, type, &key, &bytes); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); } else { @@ -141,16 +140,9 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SResultRow *pResult = pWindowResInfo->pResult[k]; - - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - key = varDataVal(pResult->key); - bytes = varDataLen(pResult->key); - } else { - key = (char*) &pResult->win.skey; - bytes = tDataTypeDesc[pWindowResInfo->type].nSize; - } - + getResultRowKeyInfo(pResult, type, &key, &bytes); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); + int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); assert(p != NULL); @@ -379,3 +371,12 @@ void* destroyResultRowPool(SResultRowPool* p) { tfree(p); return NULL; } + +uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { + if (!pRuntimeEnv->stableQuery) { + return 0; // for simple table query, the uid is always set to be 0; + } + + STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current); + return id->uid; +} \ No newline at end of file From 227823762cac5dbfa333308d023c6c0a80463e5c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 11:14:27 +0800 Subject: [PATCH 07/41] [TD-2175] add test cases. --- src/client/tests/resultFieldTest.cpp | 138 +++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 src/client/tests/resultFieldTest.cpp diff --git a/src/client/tests/resultFieldTest.cpp b/src/client/tests/resultFieldTest.cpp new file mode 100644 index 0000000000..c917f0ebaf --- /dev/null +++ b/src/client/tests/resultFieldTest.cpp @@ -0,0 +1,138 @@ +#include +#include + +#include "taos.h" + +namespace { +static int64_t start_ts = 1433955661000; +} +/* test parse time function */ +TEST(testCase, result_field_test) { + taos_options(TSDB_OPTION_CONFIGDIR, "~/first/cfg"); + taos_init(); + + TAOS* conn = taos_connect("ubuntu", "root", "taosdata", 0, 0); + if (conn == NULL) { + printf("Failed to connect to DB, reason:%s", taos_errstr(conn)); + exit(-1); + } + + TAOS_RES* res = taos_query(conn, "create database if not exists test"); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + res = taos_query(conn, "use test"); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + res = taos_query(conn, "create table if not exists t1(ts timestamp, k int, a binary(11), b nchar(4))"); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + char sql[512] = {0}; + sprintf(sql, "insert into t1 values(%ld, 99, 'abc', 'test')", start_ts); + + res = taos_query(conn, sql); + ASSERT_EQ(taos_errno(res), 0); + taos_free_result(res); + + res = taos_query(conn, "select count(*), spread(ts)/(1000 * 3600 * 24), first(a), last(b) from t1"); + ASSERT_EQ(taos_num_fields(res), 4); + + TAOS_FIELD* fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_BIGINT); + ASSERT_STREQ(fields[0].name, "count(*)"); + + ASSERT_EQ(fields[1].bytes, 8); + ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_DOUBLE); + ASSERT_STREQ(fields[1].name, "spread(ts)/(1000 * 3600 * 24)"); + + ASSERT_EQ(fields[2].bytes, 11); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); + ASSERT_STREQ(fields[2].name, "first(a)"); + + ASSERT_EQ(fields[3].bytes, 4); + ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[3].name, "last(b)"); + + taos_free_result(res); + + res = taos_query(conn, "select last_row(*) from t1"); + ASSERT_EQ(taos_num_fields(res), 4); + + fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[0].name, "last_row(ts)"); + + ASSERT_EQ(fields[1].bytes, 4); + ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[1].name, "last_row(k)"); + + ASSERT_EQ(fields[2].bytes, 11); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); + ASSERT_STREQ(fields[2].name, "last_row(a)"); + + ASSERT_EQ(fields[3].bytes, 4); + ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[3].name, "last_row(b)"); + + taos_free_result(res); + res = taos_query(conn, "select first(*), last(*) from t1"); + ASSERT_EQ(taos_num_fields(res), 8); + + fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[0].name, "first(ts)"); + + ASSERT_EQ(fields[1].bytes, 4); + ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[1].name, "first(k)"); + + ASSERT_EQ(fields[2].bytes, 11); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_BINARY); + ASSERT_STREQ(fields[2].name, "first(a)"); + + ASSERT_EQ(fields[3].bytes, 4); + ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[3].name, "first(b)"); + + taos_free_result(res); + + res = taos_query(conn, "select first(ts, a, k, k, b, b, ts) from t1"); + ASSERT_EQ(taos_num_fields(res), 7); + + fields = taos_fetch_fields(res); + ASSERT_EQ(fields[0].bytes, 8); + ASSERT_EQ(fields[0].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[0].name, "first(ts)"); + + ASSERT_EQ(fields[1].bytes, 11); + ASSERT_EQ(fields[1].type, TSDB_DATA_TYPE_BINARY); + ASSERT_STREQ(fields[1].name, "first(a)"); + + ASSERT_EQ(fields[2].bytes, 4); + ASSERT_EQ(fields[2].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[2].name, "first(k)"); + + ASSERT_EQ(fields[3].bytes, 4); + ASSERT_EQ(fields[3].type, TSDB_DATA_TYPE_INT); + ASSERT_STREQ(fields[3].name, "first(k)"); + + ASSERT_EQ(fields[4].bytes, 4); + ASSERT_EQ(fields[4].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[4].name, "first(b)"); + + ASSERT_EQ(fields[5].bytes, 4); + ASSERT_EQ(fields[5].type, TSDB_DATA_TYPE_NCHAR); + ASSERT_STREQ(fields[5].name, "first(b)"); + + ASSERT_EQ(fields[6].bytes, 8); + ASSERT_EQ(fields[6].type, TSDB_DATA_TYPE_TIMESTAMP); + ASSERT_STREQ(fields[6].name, "first(ts)"); + + taos_free_result(res); + taos_close(conn); +} From 6a837620f0539d30ebb6affe5a7fa89f3848db93 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 11:15:59 +0800 Subject: [PATCH 08/41] [TD-225] update the sim script. --- .../parser/col_arithmetic_operation.sim | 100 +++----- .../general/parser/col_arithmetic_query.sim | 240 ++++++++++++++++++ 2 files changed, 269 insertions(+), 71 deletions(-) create mode 100644 tests/script/general/parser/col_arithmetic_query.sim diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index d5ba57e6c7..c05229ab65 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -5,6 +5,7 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 0 system sh/exec.sh -n dnode1 -s start sleep 500 sql connect +#========================================= setup environment ================================ $dbPrefix = ca_db $tbPrefix = ca_tb @@ -52,44 +53,39 @@ while $i < $halfTbNum $i = $i + 1 endw -##### select from table -$tb = $tbPrefix . 0 -## TBASE-344 -sql select c1*2 from $tb -if $rows != $rowNum then - return -1 -endi -if $data00 != 0.000000000 then - return -1 -endi -if $data10 != 2.000000000 then - return -1 -endi -if $data20 != 4.000000000 then - return -1 -endi -if $data90 != 18.000000000 then - return -1 -endi +#=================================== above are setup test environment ============================= +run general/parser/col_arithmetic_query.sim -sql select c4*1+1/2 from $tb -if $rows != $rowNum then - return -1 -endi -if $data00 != 0.500000000 then - return -1 -endi -if $data10 != 1.500000000 then - return -1 -endi -if $data90 != 9.500000000 then - return -1 -endi +#======================================= all in files query ======================================= +print ================== restart server to commit data into disk +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 3000 +system sh/exec.sh -n dnode1 -s start -#### illegal operations +print ================== server restart completed +sql connect +sleep 500 + +run general/parser/col_arithmetic_query.sim + +# ================================================================================================ + +print ====================> crash +# sql select spread(ts )/(1000*3600*24) from ca_stb0 interval(1y) + + +sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y) +sql_error select first(ts) - last(ts) from stb interval(1y) +sql_error select top(c1, 2) - last(c1) from stb; +sql_error select stddev(c1) - last(c1) from stb; +sql_error select diff(c1) - last(c1) from stb; +sql_error select first(c7) - last(c7) from stb; +sql_error select first(c8) - last(c8) from stb; +sql_error select first(c9) - last(c9) from stb; sql_error select max(c2*2) from $tb sql_error select max(c1-c2) from $tb +#========================================regression test cases==================================== print =====================> td-1764 sql select sum(c1)/count(*), sum(c1) as b, count(*) as b from $stb interval(1y) if $rows != 1 then @@ -108,42 +104,4 @@ if $data02 != 225000 then return -1 endi -sql select first(c1) - last(c1), first(c1) as b, last(c1) as b, min(c1) - max(c1), spread(c1) from ca_stb0 interval(1y) -if $rows != 1 then - return -1 -endi - -if $data00 != @18-01-01 00:00:00.000@ then - return -1 -endi - -if $data01 != -9.000000000 then - return -1 -endi - -if $data02 != 0 then - return -1 -endi - -if $data03 != 9 then - return -1 -endi - -if $data04 != -9.000000000 then - return -1 -endi - -if $data05 != 9.000000000 then - return -1 -endi - -sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y) -sql_error select first(ts) - last(ts) from stb interval(1y) -sql_error select top(c1, 2) - last(c1) from stb; -sql_error select stddev(c1) - last(c1) from stb; -sql_error select diff(c1) - last(c1) from stb; -sql_error select first(c7) - last(c7) from stb; -sql_error select first(c8) - last(c8) from stb; -sql_error select first(c9) - last(c9) from stb; - system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim new file mode 100644 index 0000000000..95258168ef --- /dev/null +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -0,0 +1,240 @@ +# ======================================= query test cases ======================================== +# select from table + +$tb = $tbPrefix . 0 +## TBASE-344 +sql select c1*2 from $tb +if $rows != $rowNum then + return -1 +endi +if $data00 != 0.000000000 then + return -1 +endi +if $data10 != 2.000000000 then + return -1 +endi +if $data20 != 4.000000000 then + return -1 +endi +if $data90 != 18.000000000 then + return -1 +endi + +# asc/desc order [d.2] +sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc; +sql select (c1 * 2) % 7.9 from $tb order by ts desc; + +sql select c1 * c2 /4 from $tb where ts < and ts > + +# no result return [d.3] +sql select c1 * 91- 7 from $tb where ts<1537146000000 +if $rows != 0 then + return -1 +endi + +# no result return [d.3] +sql select c2 - c2 from $tb where ts>xxx +if $rows != 0 then + return -1 +endi + +# single row result aggregation [d.4] +# not available + +# error cases +sql_error select first(c1,c2) - last(c1,c2) from $tb + +# multi row result aggregation [d.4] +sql select top(c1, 1) - bottom(c1, 1) from $tb +sql select top(c1, 99) - bottom(c1, 99) from $tb + +# all data types [d.6] +sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $tb + + + +# error case, ts/bool/binary/nchar not support arithmetic expression +sql_error select ts+ts from $tb +sql_error select ts+22 from $tb +sql_error select c7*12 from $tb +sql_error select c8/55 from $tb +sql_error select c9+c8 from $tb +sql_error select c7-c8, c9-c8 from $tb +sql_error select ts-c9 from $tb +sql_error select c8+c7, c9+c9+c8+c7/c6 from $tb +sql_error select c1/0, c2/0, c3/0.0 from $tb +sql_error select c1/0.00000000001 from $tb + +# arithmetic expression in join [d.7] + + +# arithmetic expression in union [d.8] + + +# arithmetic expression in group by [d.9] +# in group by tag +# not support for normal table +sql_error select c5*99 from $tb group by t1 + +# in group by column +sql_error select c6-(c6+c3)*12 from $tb group by c3; + + +# limit offset [d.10] +sql select c6 - c6 + 12 from $tb limit 12 offset 99; +sql select c4 / 99.123 from $tb limit 1 offset 9999; + +# slimit/soffset not support for normal table query. [d.11] +sql_error select sum(c1) from $tb slimit 1 soffset 19; + +# fill [d.12] +sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); + +# constant column. [d.13] + + +# column value filter [d.14] + + +# tag filter(not support for normal table). [d.15] +sql_error select c2+99 from $tb where t1=12; + +# multi-field output [d.16] +sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb + +sql select c4*1+1/2 from $tb +if $rows != $rowNum then + return -1 +endi +if $data00 != 0.500000000 then + return -1 +endi +if $data10 != 1.500000000 then + return -1 +endi +if $data90 != 9.500000000 then + return -1 +endi + +# interval query [d.17] +sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s) +sql_error select c7-c9 from $tb interval(2y) + +# aggregation query [d.18] +# see test cases below + +# first/last query [d.19] +# see test cases below + +# multiple retrieve [d.20] +sql select c2-c2 from $tb; + +#======================================= aggregation function arithmetic query cases ================ +# asc/desc order [d.2] +sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc; +sql select (count(c1) * 2) % 7.9 from $stb order by ts desc; + +# all possible function in the arithmetic expressioin +sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), xxxx, from $stb where ts < and ts > + +# no result return [d.3] +sql select first(c1) * 91 - 7, last(c3) from $stb where ts < 1537146000000 +if $rows != 0 then + return -1 +endi + +# no result return [d.3] +sql select sum(c2) - avg(c2) from $tb where ts>xxx +if $rows != 0 then + return -1 +endi + +# single row result aggregation [d.4] +sql select + +# error cases +sql_error select first(c1, c2) - last(c1, c2) from $tb + +# multi row result aggregation [d.4] +sql select top(c1, 1) - bottom(c1, 1) from $tb +sql select top(c1, 99) - bottom(c1, 99) from $tb + +# all data types [d.6] +sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $tb + +# error case, ts/bool/binary/nchar not support arithmetic expression +sql_error select ts+ts from $tb +sql_error select ts+22 from $tb +sql_error select c7*12 from $tb +sql_error select c8/55 from $tb +sql_error select c9+c8 from $tb + +# arithmetic expression in join [d.7] + + +# arithmetic expression in union [d.8] + + +# arithmetic expression in group by [d.9] +# in group by tag +# not support for normal table +sql_error select c5*99 from $tb group by t1 + +# in group by column +sql_error select c6-c6+c3*12 from $tb group by c3; + +sql select first(c6) - last(c6) *12 / count(*) from $tb group by c3; + +# limit offset [d.10] +sql select c6-c6+12 from $tb limit 12 offset 99; +sql select c4/99.123 from $tb limit 1 offset 9999; + +# slimit/soffset not suport for normal table query. [d.11] +sql_error select sum(c1) from $tb slimit 1 soffset 19; + +# fill [d.12] +sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); + +# constant column. [d.13] + + +# column value filter [d.14] + + +# tag filter(not support for normal table). [d.15] +sql_error select c2+99 from $tb where t1=12; + +# multi-field output [d.16] +sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb + +sql select c4*1+1/2 from $tb +if $rows != $rowNum then + return -1 +endi +if $data00 != 0.500000000 then + return -1 +endi +if $data10 != 1.500000000 then + return -1 +endi +if $data90 != 9.500000000 then + return -1 +endi + +# interval query [d.17] +sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s) +sql_error select c7-c9 from $tb interval(2y) + +# aggregation query [d.18] +# see test cases below + +# first/last query [d.19] +# see test cases below + +# multiple retrieve [d.20] +sql select c2-c2 from $tb; + + +sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb + + From 5894dded1ddf64561f6e991b8f92469bde27650a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 12:47:34 +0800 Subject: [PATCH 09/41] [TD-2169]: fix memory leak in handing the group by binary|nchar type data. --- src/query/inc/qUtil.h | 2 +- src/query/src/qUtil.c | 19 +++++++++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b2c4b6e2ec..62bf245a2b 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -26,7 +26,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery); -void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow); +void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type); void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 650b886c36..4b8d7ea930 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -68,6 +68,12 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->pResult == NULL); return; } + + if (pWindowResInfo->type == TSDB_DATA_TYPE_BINARY || pWindowResInfo->type == TSDB_DATA_TYPE_NCHAR) { + for(int32_t i = 0; i < pWindowResInfo->size; ++i) { + tfree(pWindowResInfo->pResult[i]->key); + } + } tfree(pWindowResInfo->pResult); } @@ -81,7 +87,7 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SResultRow *pWindowRes = pWindowResInfo->pResult[i]; - clearResultRow(pRuntimeEnv, pWindowRes); + clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); int32_t groupIndex = 0; int64_t uid = 0; @@ -133,7 +139,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // move the unclosed window in the front of the window list for (int32_t k = remain; k < pWindowResInfo->size; ++k) { SResultRow *pWindowRes = pWindowResInfo->pResult[k]; - clearResultRow(pRuntimeEnv, pWindowRes); + clearResultRow(pRuntimeEnv, pWindowRes, pWindowResInfo->type); } pWindowResInfo->size = remain; @@ -229,7 +235,7 @@ void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { getResultRow(pWindowResInfo, slot)->closed = true; } -void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { +void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16_t type) { if (pWindowRes == NULL) { return; } @@ -253,7 +259,12 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { pWindowRes->pageId = -1; pWindowRes->rowId = -1; pWindowRes->closed = false; - pWindowRes->win = TSWINDOW_INITIALIZER; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + tfree(pWindowRes->key); + } else { + pWindowRes->win = TSWINDOW_INITIALIZER; + } } /** From f55fce821764f91f36e2fb8c23706bead039f8bf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 14:01:10 +0800 Subject: [PATCH 10/41] [TD-2163]: add the version check for query message. --- src/client/src/tscServer.c | 3 ++- src/inc/taosmsg.h | 2 ++ src/query/src/qExecutor.c | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cbc5604a27..eaa00f290b 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -669,6 +669,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { } SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; + tstrncpy(pQueryMsg->version, version, tListLen(pQueryMsg->version)); int32_t numOfTags = (int32_t)taosArrayGetSize(pTableMetaInfo->tagColList); @@ -693,8 +694,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit; pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); - pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); + pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->queryType = htonl(pQueryInfo->type); size_t numOfOutput = tscSqlExprNumOfExprs(pQueryInfo); diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 681fa44929..b77db69c46 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -464,6 +464,8 @@ typedef struct STimeWindow { typedef struct { SMsgHead head; + char version[TSDB_VERSION_LEN]; + STimeWindow window; int32_t numOfTables; int16_t order; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c44d65e373..630a469dd7 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5825,6 +5825,10 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, char **tagCond, char** tbnameCond, SColIndex **groupbyCols, SColumnInfo** tagCols) { int32_t code = TSDB_CODE_SUCCESS; + if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { + return TSDB_CODE_QRY_INVALID_MSG; + } + pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); From 7df48991217b8f487cfdc30eade6a1413e919b48 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 16:27:04 +0800 Subject: [PATCH 11/41] [TD-2180]: fix coverity issues. --- src/client/src/tscPrepare.c | 72 +++++++++---------- src/client/src/tscSQLParser.c | 13 +--- src/client/src/tscServer.c | 6 +- src/client/src/tscUtil.c | 7 +- src/common/src/tvariant.c | 4 +- src/query/src/qExecutor.c | 2 +- src/query/src/qParserImpl.c | 3 +- src/query/src/qResultbuf.c | 7 +- src/query/src/qTsbuf.c | 4 +- .../parser/col_arithmetic_operation.sim | 1 - 10 files changed, 59 insertions(+), 60 deletions(-) diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 68f2ecbf0e..30e0729427 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -258,11 +258,7 @@ static char* normalStmtBuildSql(STscStmt* stmt) { static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { if (bind->is_null != NULL && *(bind->is_null)) { - if (param->type == TSDB_DATA_TYPE_BINARY || param->type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(data + param->offset, param->type); - } else { - setNull(data + param->offset, param->type, param->bytes); - } + setNull(data + param->offset, param->type, param->bytes); return TSDB_CODE_SUCCESS; } @@ -312,13 +308,13 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } memcpy(data + param->offset, &u.v1, sizeof(u.v1)); return TSDB_CODE_SUCCESS; @@ -336,40 +332,40 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_INT: { int32_t v = *(int32_t*)bind->buffer; u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)bind->buffer; u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { int64_t v; - int n,r; + int n, r; r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { + if (r == 1 && n == strlen((const char*)bind->buffer)) { u.v1 = (int8_t)v; if (v >= SCHAR_MIN && v <= SCHAR_MAX) break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } memcpy(data + param->offset, &u.v1, sizeof(u.v1)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_SMALLINT: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: @@ -383,34 +379,34 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_BIGINT: { int64_t v = *(int64_t*)bind->buffer; u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: { int64_t v; - int n,r; + int n, r; r = sscanf((const char*)bind->buffer, "%" PRId64 "%n", &v, &n); - if (r==1 && n==strlen((const char*)bind->buffer)) { + if (r == 1 && n == strlen((const char*)bind->buffer)) { u.v2 = (int16_t)v; if (v >= SHRT_MIN && v <= SHRT_MAX) break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } memcpy(data + param->offset, &u.v2, sizeof(u.v2)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_INT: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: @@ -514,17 +510,17 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } memcpy(data + param->offset, &u.v8, sizeof(u.v8)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_DOUBLE: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_BOOL: @@ -556,15 +552,15 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { break; } return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } case TSDB_DATA_TYPE_TIMESTAMP: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; - } break; + } + } memcpy(data + param->offset, &u.f8, sizeof(u.f8)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_TIMESTAMP: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_TIMESTAMP: { @@ -589,10 +585,10 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { default: { return TSDB_CODE_TSC_INVALID_VALUE; } break; - } break; + }; memcpy(data + param->offset, &u.v8, sizeof(u.v8)); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_BINARY: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_BINARY: { @@ -602,7 +598,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { short size = (short)*bind->length; STR_WITH_SIZE_TO_VARSTR(data + param->offset, bind->buffer, size); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -614,9 +610,9 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { case TSDB_DATA_TYPE_NCHAR: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } - } break; + } case TSDB_DATA_TYPE_NCHAR: { switch (bind->buffer_type) { case TSDB_DATA_TYPE_NCHAR: { @@ -626,7 +622,7 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { } varDataSetLen(data + param->offset, output); return TSDB_CODE_SUCCESS; - } break; + } case TSDB_DATA_TYPE_BOOL: case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_SMALLINT: @@ -638,12 +634,12 @@ static int doBindParam(char* data, SParamInfo* param, TAOS_BIND* bind) { case TSDB_DATA_TYPE_BINARY: default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } - } break; + } default: { return TSDB_CODE_TSC_INVALID_VALUE; - } break; + } } } diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 15d2647c51..f859d2969e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -80,7 +80,6 @@ static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprLis static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); -static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd); @@ -2951,14 +2950,6 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, SArray* pList, SSqlCmd* pCmd) return TSDB_CODE_SUCCESS; } -void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo) { - if (QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - tscFieldInfoUpdateOffset(pQueryInfo); - } else { - tscFieldInfoUpdateOffset(pQueryInfo); - } -} - static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { if (pColumn == NULL) { return NULL; @@ -3537,7 +3528,7 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer if (i == 0) { id = p1->uid; - } else if (id != p1->uid){ + } else if (id != p1->uid) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -6531,7 +6522,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return code; } - setColumnOffsetValueInResultset(pQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); /* * fill options are set at the end position, when all columns are set properly diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index eaa00f290b..15859674ab 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -62,7 +62,7 @@ static void tscSetDnodeEpSet(SSqlObj* pSql, SVgroupInfo* pVgroupInfo) { pEpSet->numOfEps = pVgroupInfo->numOfEps; for(int32_t i = 0; i < pVgroupInfo->numOfEps; ++i) { - strcpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn); + tstrncpy(pEpSet->fqdn[i], pVgroupInfo->epAddr[i].fqdn, tListLen(pEpSet->fqdn[i])); pEpSet->port[i] = pVgroupInfo->epAddr[i].port; if (!hasFqdn) { @@ -2031,7 +2031,8 @@ static void createHBObj(STscObj* pObj) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); if (pQueryInfo == NULL) { - pSql->res.code = terrno; + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + tfree(pSql); return; } @@ -2039,6 +2040,7 @@ static void createHBObj(STscObj* pObj) { pSql->cmd.command = pQueryInfo->command; if (TSDB_CODE_SUCCESS != tscAllocPayload(&(pSql->cmd), TSDB_DEFAULT_PAYLOAD_SIZE)) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tfree(pSql); return; } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7a82bcaaab..a75f6d8296 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -451,14 +451,16 @@ void tscFreeRegisteredSqlObj(void *pSql) { STscObj* pTscObj = (*p)->pTscObj; assert((*p)->self != 0 && (*p)->self == (p)); + + SSqlObj* ptr = *p; tscFreeSqlObj(*p); int32_t ref = T_REF_DEC(pTscObj); assert(ref >= 0); - tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", *p, pTscObj, ref); + tscDebug("%p free sqlObj completed, tscObj:%p ref:%d", ptr, pTscObj, ref); if (ref == 0) { - tscDebug("%p all sqlObj freed, free tscObj:%p", *p, pTscObj); + tscDebug("%p all sqlObj freed, free tscObj:%p", ptr, pTscObj); taosRemoveRef(tscRefId, pTscObj->rid); } } @@ -644,6 +646,7 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff dataBuf->pData = calloc(1, dataBuf->nAllocSize); if (dataBuf->pData == NULL) { tscError("failed to allocated memory, reason:%s", strerror(errno)); + tfree(dataBuf); return TSDB_CODE_TSC_OUT_OF_MEMORY; } diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 6dd0653822..fc00f50a7a 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -171,7 +171,9 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { } } - pDst->nLen = tDataTypeDesc[pDst->nType].nSize; + if (pDst->nType != TSDB_DATA_TYPE_ARRAY) { + pDst->nLen = tDataTypeDesc[pDst->nType].nSize; + } } int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 630a469dd7..3fcf44f057 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6814,7 +6814,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { - size_t sz = read(fd, data, (uint32_t)s); + size_t sz = read(fd, data, (uint32_t) s); if(sz < s) { // todo handle error assert(0); } diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index 0853565fc6..5cd9d3c77a 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -187,7 +187,8 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { pExpr->token.type = pLeft->token.type; } - if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { + if ((pLeft != NULL && pRight != NULL) && + (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) { /* * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond * Otherwise, the time precision is adaptive, determined by the time precision from databases. diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 67cb7af074..dfa8bcd95b 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -119,8 +119,11 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { pg->info.offset = allocatePositionInFile(pResultBuf, size); pResultBuf->nextPos += size; - fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - /*int32_t ret =*/ fwrite(t, 1, size, pResultBuf->file); + int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + assert(ret == 0); + + ret = fwrite(t, 1, size, pResultBuf->file); + assert(ret == size); if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { pResultBuf->fileSize = pg->info.offset + pg->info.length; diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 8a5a87baab..d0c59fe5ef 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -341,8 +341,10 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { pBlock->tag.pz = tp; sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); - } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + UNUSED(sz); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { //TODO check the return value sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); + UNUSED(sz); } sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index c05229ab65..64f41ae11f 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -73,7 +73,6 @@ run general/parser/col_arithmetic_query.sim print ====================> crash # sql select spread(ts )/(1000*3600*24) from ca_stb0 interval(1y) - sql_error select first(c1, c2) - last(c1, c2) from stb interval(1y) sql_error select first(ts) - last(ts) from stb interval(1y) sql_error select top(c1, 2) - last(c1) from stb; From 2783c56f8ab3b33d7858a67ddab01062adf726cb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 18:45:09 +0800 Subject: [PATCH 12/41] [TD-225]update script. --- .../parser/col_arithmetic_operation.sim | 68 ++++++-- .../general/parser/col_arithmetic_query.sim | 147 ++++++++++++++---- 2 files changed, 175 insertions(+), 40 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_operation.sim b/tests/script/general/parser/col_arithmetic_operation.sim index 64f41ae11f..7adae8ef81 100644 --- a/tests/script/general/parser/col_arithmetic_operation.sim +++ b/tests/script/general/parser/col_arithmetic_operation.sim @@ -29,12 +29,41 @@ sql create table $stb (ts timestamp, c1 int, c2 bigint, c3 float, c4 double, c5 $i = 0 $ts = $ts0 $halfTbNum = $tbNum / 2 -while $i < $halfTbNum - $tbId = $i + $halfTbNum - $tb = $tbPrefix . $i - $tb1 = $tbPrefix . $tbId - sql create table $tb using $stb tags( $i ) - sql create table $tb1 using $stb tags( $tbId ) +#while $i < $halfTbNum + $t1 = $i + 1 + $t2 = $i + 2 + $t3 = $i + 3 + $t4 = $i + 4 + + $t5 = $i + $halfTbNum + $t6 = $t5 + 1 + $t7 = $t6 + 1 + $t8 = $t7 + 1 + $t9 = $t8 + 1 + + $tb0 = $tbPrefix . $i + $tb1 = $tbPrefix . $t1 + $tb2 = $tbPrefix . $t2 + $tb3 = $tbPrefix . $t3 + $tb4 = $tbPrefix . $t4 + + $tb5 = $tbPrefix . $t5 + $tb6 = $tbPrefix . $t6 + $tb7 = $tbPrefix . $t7 + $tb8 = $tbPrefix . $t8 + $tb9 = $tbPrefix . $t9 + + sql create table $tb0 using $stb tags( $i ) + sql create table $tb1 using $stb tags( $t1 ) + sql create table $tb2 using $stb tags( $t2 ) + sql create table $tb3 using $stb tags( $t3 ) + sql create table $tb4 using $stb tags( $t4 ) + + sql create table $tb5 using $stb tags( $t5 ) + sql create table $tb6 using $stb tags( $t6 ) + sql create table $tb7 using $stb tags( $t7 ) + sql create table $tb8 using $stb tags( $t8 ) + sql create table $tb9 using $stb tags( $t9 ) $x = 0 while $x < $rowNum @@ -47,11 +76,28 @@ while $i < $halfTbNum $binary = $binary . ' $nchar = 'nchar . $c $nchar = $nchar . ' - sql insert into $tb values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) + sql insert into $tb0 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb1 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb2 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb3 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $tb4 values ( $ts , $c , $c , $c , $c , $c , $c , true, $binary , $nchar ) $x = $x + 1 - endw - $i = $i + 1 -endw + endw + #$i = $i + 1 + + $x = 0 + while $x < $rowNum + $xs = $x * $delta + $ts = $ts0 + $xs + $c = $x / 10 + $c = $c * 10 + $c = $x - $c + $binary = 'binary . $c + $binary = $binary . ' + $nchar = 'nchar . $c + $nchar = $nchar . ' + + sql insert into $tb5 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb6 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb7 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb8 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) $tb9 values ( $ts , NULL , $c , NULL , $c , NULL , $c , NULL, NULL , $nchar ) + $x = $x + 1 + endw + +#endw #=================================== above are setup test environment ============================= run general/parser/col_arithmetic_query.sim @@ -64,7 +110,7 @@ system sh/exec.sh -n dnode1 -s start print ================== server restart completed sql connect -sleep 500 +sleep 500c run general/parser/col_arithmetic_query.sim diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 95258168ef..61a48e4a80 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -1,6 +1,17 @@ # ======================================= query test cases ======================================== # select from table +$dbPrefix = ca_db +$tbPrefix = ca_tb +$stbPrefix = ca_stb +$rowNum = 10000 + +$i = 0 +$db = $dbPrefix . $i + +$db = $dbPrefix . $i +sql use $db + $tb = $tbPrefix . 0 ## TBASE-344 sql select c1*2 from $tb @@ -20,38 +31,100 @@ if $data90 != 18.000000000 then return -1 endi -# asc/desc order [d.2] +# asc/desc order [d.2] ====================================================== sql select c1 *( 2 / 3 ), c1/c1 from $tb order by ts asc; +if $rows != 10000 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +if $data01 != NaN then + return -1 +endi + +if $data10 != 0.666666667 then + return -1 +endi + +if $data11 != 1.000000000 then + return -1 +endi + +if $data90 != 6.000000000 then + return -1 +endi + +if $data91 != 1.000000000 then + return -1 +endi + sql select (c1 * 2) % 7.9 from $tb order by ts desc; +if $rows != 10000 then + return -1 +endi -sql select c1 * c2 /4 from $tb where ts < and ts > +if $data00 != 0.100000000 then + return -1 +endi -# no result return [d.3] -sql select c1 * 91- 7 from $tb where ts<1537146000000 +if $data10 != 2.100000000 then + return -1 +endi + +if $data90 != 6.000000000 then + return -1 +endi + +# [d.3] +sql select c1 * c2 /4 from $tb where ts < 1537166000000 and ts > 1537156000000 +if $rows != 17 then + return -1 +endi + +if $data00 != 12.250000000 then + return -1 +endi + +if $data10 != 16.000000000 then + return -1 +endi + +if $data21 != 20.250000000 then + return -1 +endi + +if $data31 != 0.000000000 then + return -1 +endi + + +# no result return [d.3] ============================================================== +sql select c1 * 91- 7 from $tb where ts < 1537146000000 if $rows != 0 then return -1 endi # no result return [d.3] -sql select c2 - c2 from $tb where ts>xxx +sql select c2 - c2 from $tb where ts > '2018-09-17 12:50:00.000' and ts<'2018-09-17 13:00:00.000' if $rows != 0 then return -1 endi -# single row result aggregation [d.4] +# single row result aggregation [d.4] ================================================= # not available # error cases -sql_error select first(c1,c2) - last(c1,c2) from $tb +# not available # multi row result aggregation [d.4] sql select top(c1, 1) - bottom(c1, 1) from $tb sql select top(c1, 99) - bottom(c1, 99) from $tb -# all data types [d.6] -sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22 from $tb - - +# all data types [d.6] ================================================================ +sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22, c2-c2 from $tb # error case, ts/bool/binary/nchar not support arithmetic expression sql_error select ts+ts from $tb @@ -65,41 +138,57 @@ sql_error select c8+c7, c9+c9+c8+c7/c6 from $tb sql_error select c1/0, c2/0, c3/0.0 from $tb sql_error select c1/0.00000000001 from $tb -# arithmetic expression in join [d.7] +# arithmetic expression in join [d.7]================================================== -# arithmetic expression in union [d.8] +# arithmetic expression in union [d.8]================================================= -# arithmetic expression in group by [d.9] -# in group by tag -# not support for normal table +# arithmetic expression in group by [d.9]============================================== +# in group by tag, not support for normal table sql_error select c5*99 from $tb group by t1 # in group by column sql_error select c6-(c6+c3)*12 from $tb group by c3; -# limit offset [d.10] -sql select c6 - c6 + 12 from $tb limit 12 offset 99; -sql select c4 / 99.123 from $tb limit 1 offset 9999; +# limit offset [d.10]================================================================== +sql select c6 * c1 + 12 from $tb limit 12 offset 99; +if $rows != 12 then + return -1 +endi -# slimit/soffset not support for normal table query. [d.11] +if $data00 != 93.000000000 then + return -1 +endi + +if $data90 != 12.000000000 then + return -1 +endi + +sql select c4 / 99.123 from $tb limit 10 offset 9999; +if $rows != 1 then + return -1 +endi + +if $data00 != + +# slimit/soffset not support for normal table query. [d.11]============================ sql_error select sum(c1) from $tb slimit 1 soffset 19; -# fill [d.12] +# fill [d.12]========================================================================== sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); -# constant column. [d.13] +# constant column. [d.13]============================================================== -# column value filter [d.14] +# column value filter [d.14]=========================================================== -# tag filter(not support for normal table). [d.15] +# tag filter(not support for normal table). [d.15]===================================== sql_error select c2+99 from $tb where t1=12; -# multi-field output [d.16] +# multi-field output [d.16]============================================================ sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb sql select c4*1+1/2 from $tb @@ -116,17 +205,17 @@ if $data90 != 9.500000000 then return -1 endi -# interval query [d.17] +# interval query [d.17]================================================================== sql_error select c2*c2, c3-c3, c4+9 from $tb interval(1s) sql_error select c7-c9 from $tb interval(2y) -# aggregation query [d.18] +# aggregation query [d.18]=============================================================== # see test cases below -# first/last query [d.19] +# first/last query [d.19]=============================================================== # see test cases below -# multiple retrieve [d.20] +# multiple retrieve [d.20]=============================================================== sql select c2-c2 from $tb; #======================================= aggregation function arithmetic query cases ================ From dd9228e06733c5a775e953e01b1bd12e1711a5e4 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 18:51:13 +0800 Subject: [PATCH 13/41] [TD-1859]: restful implementation for taos-jdbc --- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 9 +- .../taosdata/jdbc/rs/RestfulConnection.java | 41 ++++--- .../com/taosdata/jdbc/rs/RestfulDriver.java | 2 +- .../taosdata/jdbc/rs/RestfulStatement.java | 71 ++++++++++-- .../jdbc/utils/SqlSyntaxValidator.java | 29 ++++- .../taosdata/jdbc/rs/RestfulDriverTest.java | 23 ---- .../com/taosdata/jdbc/rs/RestfulJDBCTest.java | 108 ++++++++++++++++++ .../jdbc/utils/SqlSyntaxValidatorTest.java | 29 +++++ 9 files changed, 263 insertions(+), 51 deletions(-) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 51db837c7b..46de54ffb2 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.10 + 2.0.13 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index e7124a0599..3d1f402435 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.8 + 2.0.13 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc @@ -112,6 +112,13 @@ maven-surefire-plugin 2.12.4 + + **/*Test.java + + + **/BatchInsertTest.java + **/FailOverTest.java + true diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java index b82efca3ef..6b0937a9b7 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulConnection.java @@ -15,7 +15,6 @@ public class RestfulConnection implements Connection { private final String database; private final String url; - public RestfulConnection(String host, String port, Properties props, String database, String url) { this.host = host; this.port = Integer.parseInt(port); @@ -28,7 +27,7 @@ public class RestfulConnection implements Connection { public Statement createStatement() throws SQLException { if (isClosed()) throw new SQLException(TSDBConstants.WrapErrMsg("restful TDengine connection is closed.")); - return new RestfulStatement(this, this.database); + return new RestfulStatement(this, database); } @Override @@ -104,22 +103,28 @@ public class RestfulConnection implements Connection { @Override public void setTransactionIsolation(int level) throws SQLException { - + //transaction is not supported + throw new SQLFeatureNotSupportedException("transactions are not supported"); } + /** + * + */ @Override public int getTransactionIsolation() throws SQLException { - return 0; + //Connection.TRANSACTION_NONE specifies that transactions are not supported. + return Connection.TRANSACTION_NONE; } @Override public SQLWarning getWarnings() throws SQLException { + //TODO: getWarnings not implemented return null; } @Override public void clearWarnings() throws SQLException { - + throw new SQLFeatureNotSupportedException("clearWarnings not supported."); } @Override @@ -209,22 +214,26 @@ public class RestfulConnection implements Connection { @Override public Clob createClob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public Blob createBlob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public NClob createNClob() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public SQLXML createSQLXML() throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override @@ -254,12 +263,14 @@ public class RestfulConnection implements Connection { @Override public Array createArrayOf(String typeName, Object[] elements) throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override public Struct createStruct(String typeName, Object[] attributes) throws SQLException { - return null; + //TODO: not supported + throw new SQLFeatureNotSupportedException(); } @Override @@ -289,12 +300,16 @@ public class RestfulConnection implements Connection { @Override public T unwrap(Class iface) throws SQLException { - return null; + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return iface.isInstance(this); } public String getHost() { diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java index a07d6eb2b6..9e87cfa680 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulDriver.java @@ -35,7 +35,7 @@ public class RestfulDriver extends AbstractTaosDriver { Properties props = parseURL(url, info); String host = props.getProperty(TSDBDriver.PROPERTY_KEY_HOST, "localhost"); String port = props.getProperty(TSDBDriver.PROPERTY_KEY_PORT, "6041"); - String database = props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME); + String database = props.containsKey(TSDBDriver.PROPERTY_KEY_DBNAME) ? props.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME) : null; String loginUrl = "http://" + props.getProperty(TSDBDriver.PROPERTY_KEY_HOST) + ":" + props.getProperty(TSDBDriver.PROPERTY_KEY_PORT) + "/rest/login/" diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java index 20510f0135..30b56638d8 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/rs/RestfulStatement.java @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.taosdata.jdbc.TSDBConstants; import com.taosdata.jdbc.rs.util.HttpClientPoolUtil; +import com.taosdata.jdbc.utils.SqlSyntaxValidator; import java.sql.*; import java.util.Arrays; @@ -11,19 +12,23 @@ import java.util.List; public class RestfulStatement implements Statement { - private final String catalog; + private boolean closed; + private String database; private final RestfulConnection conn; - public RestfulStatement(RestfulConnection c, String catalog) { + public RestfulStatement(RestfulConnection c, String database) { this.conn = c; - this.catalog = catalog; + this.database = database; } @Override public ResultSet executeQuery(String sql) throws SQLException { + if (isClosed()) + throw new SQLException("statement already closed"); + if (!SqlSyntaxValidator.isSelectSql(sql)) + throw new SQLException("not a select sql for executeQuery: " + sql); - final String url = "http://" + conn.getHost() + ":"+conn.getPort()+"/rest/sql"; - + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; String result = HttpClientPoolUtil.execute(url, sql); String fields = ""; List words = Arrays.asList(sql.split(" ")); @@ -65,12 +70,29 @@ public class RestfulStatement implements Statement { @Override public int executeUpdate(String sql) throws SQLException { - return 0; + if (isClosed()) + throw new SQLException("statement already closed"); + if (!SqlSyntaxValidator.isValidForExecuteUpdate(sql)) + throw new SQLException("not a valid sql for executeUpdate: " + sql); + + if (this.database == null) + throw new SQLException("Database not specified or available"); + + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); + String result = HttpClientPoolUtil.execute(url, sql); + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + + jsonObject.getString("desc") + "\n" + + "error code: " + jsonObject.getString("code"))); + } + return Integer.parseInt(jsonObject.getString("rows")); } @Override public void close() throws SQLException { - + this.closed = true; } @Override @@ -115,6 +137,7 @@ public class RestfulStatement implements Statement { @Override public SQLWarning getWarnings() throws SQLException { + //TODO: getWarnings not Implemented return null; } @@ -130,7 +153,29 @@ public class RestfulStatement implements Statement { @Override public boolean execute(String sql) throws SQLException { - return false; + if (isClosed()) { + throw new SQLException("Invalid method call on a closed statement."); + } + //如果执行了use操作应该将当前Statement的catalog设置为新的database + if (SqlSyntaxValidator.isUseSql(sql)) { + this.database = sql.trim().replace("use", "").trim(); + } + if (this.database == null) + throw new SQLException("Database not specified or available"); + + final String url = "http://" + conn.getHost() + ":" + conn.getPort() + "/rest/sql"; + // use database + HttpClientPoolUtil.execute(url, "use " + conn.getDatabase()); + // execute sql + String result = HttpClientPoolUtil.execute(url, sql); + // parse result + JSONObject jsonObject = JSON.parseObject(result); + if (jsonObject.getString("status").equals("error")) { + throw new SQLException(TSDBConstants.WrapErrMsg("SQL execution error: " + + jsonObject.getString("desc") + "\n" + + "error code: " + jsonObject.getString("code"))); + } + return true; } @Override @@ -245,7 +290,7 @@ public class RestfulStatement implements Statement { @Override public boolean isClosed() throws SQLException { - return false; + return closed; } @Override @@ -270,11 +315,15 @@ public class RestfulStatement implements Statement { @Override public T unwrap(Class iface) throws SQLException { - return null; + try { + return iface.cast(this); + } catch (ClassCastException cce) { + throw new SQLException("Unable to unwrap to " + iface.toString()); + } } @Override public boolean isWrapperFor(Class iface) throws SQLException { - return false; + return iface.isInstance(this); } } diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java index 066dfad5d5..388c3978be 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/SqlSyntaxValidator.java @@ -22,6 +22,9 @@ import java.sql.SQLException; public class SqlSyntaxValidator { + private static final String[] updateSQL = {"insert", "update", "delete", "create", "alter", "drop", "show", "describe", "use"}; + private static final String[] querySQL = {"select"}; + private TSDBConnection tsdbConnection; public SqlSyntaxValidator(Connection connection) { @@ -34,7 +37,7 @@ public class SqlSyntaxValidator { if (tsdbConnection == null || tsdbConnection.isClosed()) { throw new SQLException("invalid connection"); } else { - TSDBJNIConnector jniConnector = tsdbConnection.getConnection(); + TSDBJNIConnector jniConnector = tsdbConnection.getConnection(); if (jniConnector == null) { throw new SQLException("jniConnector is null"); } else { @@ -43,4 +46,28 @@ public class SqlSyntaxValidator { } return res; } + + public static boolean isValidForExecuteUpdate(String sql) { + for (String prefix : updateSQL) { + if (sql.trim().toLowerCase().startsWith(prefix)) + return true; + } + return false; + } + + public static boolean isUseSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[8]) || sql.trim().toLowerCase().matches("create\\s*database.*") || sql.toLowerCase().toLowerCase().matches("drop\\s*database.*"); + } + + public static boolean isUpdateSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[1]); + } + + public static boolean isInsertSql(String sql) { + return sql.trim().toLowerCase().startsWith(updateSQL[0]); + } + + public static boolean isSelectSql(String sql) { + return sql.trim().toLowerCase().startsWith(querySQL[0]); + } } diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java index 768eceab8e..d07a6a2179 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulDriverTest.java @@ -7,29 +7,6 @@ import java.sql.*; public class RestfulDriverTest { - @Test - public void testCase001() { - try { - Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); - Connection connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/?user=root&password=taosdata"); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery("select * from log.log"); - ResultSetMetaData metaData = resultSet.getMetaData(); - while (resultSet.next()) { - for (int i = 1; i <= metaData.getColumnCount(); i++) { - String column = metaData.getColumnLabel(i); - String value = resultSet.getString(i); - System.out.print(column + ":" + value + "\t"); - } - System.out.println(); - } - statement.close(); - connection.close(); - } catch (SQLException | ClassNotFoundException e) { - e.printStackTrace(); - } - } - @Test public void connect() { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java new file mode 100644 index 0000000000..d13475b96d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/rs/RestfulJDBCTest.java @@ -0,0 +1,108 @@ +package com.taosdata.jdbc.rs; + +import org.junit.*; +import org.junit.runners.MethodSorters; + +import java.sql.*; +import java.util.Random; + +@FixMethodOrder(MethodSorters.NAME_ASCENDING) +public class RestfulJDBCTest { + + private Connection connection; + + @Before + public void before() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.rs.RestfulDriver"); + connection = DriverManager.getConnection("jdbc:TAOS-RS://master:6041/restful_test?user=root&password=taosdata"); + } + + @After + public void after() throws SQLException { + if (connection != null) + connection.close(); + } + + + /** + * 查询所有log.log + **/ + @Test + public void testCase001() { + try { + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("select * from log.log"); + ResultSetMetaData metaData = resultSet.getMetaData(); + while (resultSet.next()) { + for (int i = 1; i <= metaData.getColumnCount(); i++) { + String column = metaData.getColumnLabel(i); + String value = resultSet.getString(i); + System.out.print(column + ":" + value + "\t"); + } + System.out.println(); + } + statement.close(); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /** + * create database + */ + @Test + public void testCase002() { + try (Statement stmt = connection.createStatement()) { + stmt.execute("drop database if exists restful_test"); + stmt.execute("create database if not exists restful_test"); + stmt.execute("use restful_test"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + /** + * create super table + ***/ + @Test + public void testCase003() { + try (Statement stmt = connection.createStatement()) { + stmt.execute("create table weather(ts timestamp, temperature float, humidity int) tags(location nchar(64), groupId int)"); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + @Test + public void testCase004() { + try (Statement stmt = connection.createStatement()) { + for (int i = 1; i <= 100; i++) { + stmt.execute("create table t" + i + " using weather tags('beijing', '" + i + "')"); + } + } catch (SQLException e) { + e.printStackTrace(); + } + } + + private Random random = new Random(System.currentTimeMillis()); + + @Test + public void testCase005() { + try (Statement stmt = connection.createStatement()) { + int rows = 0; + for (int i = 0; i < 10; i++) { + for (int j = 1; j <= 100; j++) { + long currentTimeMillis = System.currentTimeMillis(); + int affectRows = stmt.executeUpdate("insert into t" + j + " values(" + currentTimeMillis + "," + (random.nextFloat() * 50) + "," + random.nextInt(100) + ")"); + Assert.assertEquals(1, affectRows); + rows += affectRows; + } + } + Assert.assertEquals(1000, rows); + } catch (SQLException e) { + e.printStackTrace(); + } + } + + +} diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java new file mode 100644 index 0000000000..25c6c4913d --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -0,0 +1,29 @@ +package com.taosdata.jdbc.utils; + +import com.sun.source.tree.AssertTree; +import org.junit.Assert; +import org.junit.Test; + +public class SqlSyntaxValidatorTest { + + @Test + public void validateSqlSyntax() { + } + + @Test + public void isSelectSQL() { + Assert.assertTrue(SqlSyntaxValidator.isSelectSql("select * from test.weather")); + Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather")); + Assert.assertTrue(SqlSyntaxValidator.isSelectSql(" select * from test.weather ")); + Assert.assertFalse(SqlSyntaxValidator.isSelectSql("insert into test.weather values(now, 1.1, 2)")); + } + + @Test + public void isUseSQL() { + Assert.assertTrue(SqlSyntaxValidator.isUseSql("use database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("create database if not exist test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database test")); + Assert.assertTrue(SqlSyntaxValidator.isUseSql("drop database if exist test")); + } +} \ No newline at end of file From f26fc83be8c8523e140adef265d74529447e27d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 23 Nov 2020 18:59:13 +0800 Subject: [PATCH 14/41] [TD-2193]: fix the offset error in arithmetic expression. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3fcf44f057..2623c74e45 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -947,7 +947,7 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - sas->offset = 0; + sas->offset = pQuery->pos; sas->colList = pQuery->colList; sas->numOfCols = pQuery->numOfCols; sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); From a9e43baa3f8841d468efe1fedb5beda78beeb49d Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:08:03 +0800 Subject: [PATCH 15/41] change --- .../java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java index 25c6c4913d..fb570e16c4 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/utils/SqlSyntaxValidatorTest.java @@ -1,6 +1,5 @@ package com.taosdata.jdbc.utils; -import com.sun.source.tree.AssertTree; import org.junit.Assert; import org.junit.Test; From 0481d9836f33f6e0a1e26e85af7a4bdc5a8ea49d Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:12:07 +0800 Subject: [PATCH 16/41] change --- src/connector/jdbc/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index c565853ab0..701a39b209 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.8-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.13-dist.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) From 5e91e8e307c232e9eaf47615499c6a7d0dc5a936 Mon Sep 17 00:00:00 2001 From: zyyang Date: Mon, 23 Nov 2020 19:28:45 +0800 Subject: [PATCH 17/41] change --- cmake/install.inc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/install.inc b/cmake/install.inc index 9bbcc2cf40..4886e79e73 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.8-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.13-dist.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") From 7c9ce886942275ffbee67d15b80eff42954fe109 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 24 Nov 2020 09:18:55 +0800 Subject: [PATCH 18/41] change --- src/connector/jdbc/deploy-pom.xml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index 46de54ffb2..1dc2625e62 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -49,17 +49,29 @@ - - org.apache.commons - commons-lang3 - 3.5 - junit junit 4.13 test + + + + org.apache.httpcomponents + httpclient + 4.5.8 + + + org.apache.commons + commons-lang3 + 3.9 + + + com.alibaba + fastjson + 1.2.58 + From e3fc92c9f42c078e4f0cd360ab9e3def1d536ad6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 10:24:43 +0800 Subject: [PATCH 19/41] [TD-225] fix bugs. --- src/client/src/tscSQLParser.c | 18 +++++++++++++----- src/query/src/qExecutor.c | 6 +++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f859d2969e..743cb42eb3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1767,10 +1767,10 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return TSDB_CODE_SUCCESS; } -void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) { +void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken, bool multiCols) { if (pItem->aliasName != NULL) { tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); - } else { + } else if (multiCols) { char uname[TSDB_COL_NAME_LEN] = {0}; int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); @@ -1781,6 +1781,9 @@ void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrT snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); tstrncpy(name, tmp, TSDB_COL_NAME_LEN); + } else { // use the user-input result column name + int32_t len = MIN(pItem->pNode->token.n + 1, TSDB_COL_NAME_LEN); + tstrncpy(name, pItem->pNode->token.z, len); } } @@ -2055,7 +2058,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; - setResultColName(name, pItem, cvtFunc.originFuncId, &t); + setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2077,7 +2080,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col char name[TSDB_COL_NAME_LEN] = {0}; SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); - setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo); + + bool multiColOutput = pItem->pNode->pParam->nExpr > 1; + setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo, multiColOutput); if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2119,7 +2124,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col char name[TSDB_COL_NAME_LEN] = {0}; SStrToken t = {.z = pSchema[i].name, .n = (uint32_t)strnlen(pSchema[i].name, TSDB_COL_NAME_LEN)}; - setResultColName(name, pItem, cvtFunc.originFuncId, &t); + setResultColName(name, pItem, cvtFunc.originFuncId, &t, true); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; @@ -4243,6 +4248,9 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE tExprTreeDestroy(&p, NULL); taosArrayDestroy(colList); + if (taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "filter on tag not supported for normal table"); + } } pCondExpr->pTagCond = NULL; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2623c74e45..7d6068ff64 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -947,10 +947,10 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas if (functionId == TSDB_FUNC_ARITHM) { sas->pArithExpr = &pQuery->pExpr1[col]; - sas->offset = pQuery->pos; - sas->colList = pQuery->colList; + sas->offset = (QUERY_IS_ASC_QUERY(pQuery)) ? pQuery->pos : pQuery->pos - (size - 1); + sas->colList = pQuery->colList; sas->numOfCols = pQuery->numOfCols; - sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); + sas->data = calloc(pQuery->numOfCols, POINTER_BYTES); if (sas->data == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); From 3b92e7f49b2ddaadcfb2fd11e879214c8495b17a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 10:29:08 +0800 Subject: [PATCH 20/41] [TD-225] update sim test script. --- .../general/parser/col_arithmetic_query.sim | 146 +++++++++++++++--- 1 file changed, 124 insertions(+), 22 deletions(-) diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 61a48e4a80..95e0ed68d7 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -7,12 +7,12 @@ $stbPrefix = ca_stb $rowNum = 10000 $i = 0 -$db = $dbPrefix . $i - $db = $dbPrefix . $i sql use $db $tb = $tbPrefix . 0 +$stb = $stbPrefix . $i + ## TBASE-344 sql select c1*2 from $tb if $rows != $rowNum then @@ -41,7 +41,8 @@ if $data00 != 0.000000000 then return -1 endi -if $data01 != NaN then +if $data01 != -nan then + print expect -nan, actual: $data01 return -1 endi @@ -67,6 +68,7 @@ if $rows != 10000 then endi if $data00 != 0.100000000 then + print expect 0.100000000, acutal:$data00 return -1 endi @@ -92,15 +94,15 @@ if $data10 != 16.000000000 then return -1 endi -if $data21 != 20.250000000 then +if $data20 != 20.250000000 then + print expect 20.250000000, acutal:$data21 return -1 endi -if $data31 != 0.000000000 then +if $data30 != 0.000000000 then return -1 endi - # no result return [d.3] ============================================================== sql select c1 * 91- 7 from $tb where ts < 1537146000000 if $rows != 0 then @@ -120,11 +122,59 @@ endi # not available # multi row result aggregation [d.4] -sql select top(c1, 1) - bottom(c1, 1) from $tb -sql select top(c1, 99) - bottom(c1, 99) from $tb +sql_error select top(c1, 1) - bottom(c1, 1) from $tb +sql_error select top(c1, 99) - bottom(c1, 99) from $tb +sql_error select top(c1,1) - 88 from $tb # all data types [d.6] ================================================================ -sql select c2-c1, c3/c2, c4*c3, c5%c4, c6+99%22, c2-c2 from $tb +sql select c2-c1*1.1, c3/c2, c4*c3, c5%c4, (c6+c4)%22, c2-c2 from $tb +if $rows != 10000 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +if $data01 != -nan then + return -1 +endi + +if $data02 != 0.000000000 then + return -1 +endi + +if $data03 != 0.000000000 then + return -1 +endi + +if $data04 != 0.000000000 then + return -1 +endi + +if $data05 != 0.000000000 then + return -1 +endi + +if $data90 != -0.900000000 then + return -1 +endi + +if $data91 != 1.000000000 then + return -1 +endi + +if $data92 != 81.000000000 then + return -1 +endi + +if $data93 != 0.000000000 then + return -1 +endi + +if $data94 != 18.000000000 then + return -1 +endi # error case, ts/bool/binary/nchar not support arithmetic expression sql_error select ts+ts from $tb @@ -135,8 +185,6 @@ sql_error select c9+c8 from $tb sql_error select c7-c8, c9-c8 from $tb sql_error select ts-c9 from $tb sql_error select c8+c7, c9+c9+c8+c7/c6 from $tb -sql_error select c1/0, c2/0, c3/0.0 from $tb -sql_error select c1/0.00000000001 from $tb # arithmetic expression in join [d.7]================================================== @@ -162,7 +210,7 @@ if $data00 != 93.000000000 then return -1 endi -if $data90 != 12.000000000 then +if $data90 != 76.000000000 then return -1 endi @@ -171,27 +219,81 @@ if $rows != 1 then return -1 endi -if $data00 != +if $data00 != 0.090796283 then + return -1 +endi # slimit/soffset not support for normal table query. [d.11]============================ sql_error select sum(c1) from $tb slimit 1 soffset 19; # fill [d.12]========================================================================== -sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); +sql_error select c2-c2, c3-c4, c5%c3 from $tb fill(value, 12); # constant column. [d.13]============================================================== +sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb +if $rows != 10000 then + return -1 +endi +if $data00 != 0 then + return -1 +endi + +if $data01 != 0.000000000 then + return -1 +endi + +if $data02 != 12.987654568 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data04 != 1.100000000 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi # column value filter [d.14]=========================================================== +sql select c1, c2+c6, 12.9876545678, 1, 1.1 from $tb where c1<2 +if $rows != 2000 then + return -1 +endi +if $data00 != 0 then + return -1 +endi + +if $data01 != 0.000000000 then + return -1 +endi + +if $data02 != 12.987654568 then + return -1 +endi + +if $data03 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data20 != 0 then + return -1 +endi # tag filter(not support for normal table). [d.15]===================================== sql_error select c2+99 from $tb where t1=12; # multi-field output [d.16]============================================================ -sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb - -sql select c4*1+1/2 from $tb +sql select c4*1+1/2,c4*1+1/2,c4*1+1/2,c4*1+1/2,c4*1+1/2 from $tb if $rows != $rowNum then return -1 endi @@ -216,15 +318,17 @@ sql_error select c7-c9 from $tb interval(2y) # see test cases below # multiple retrieve [d.20]=============================================================== -sql select c2-c2 from $tb; +sql select c2-c2, 911 from $tb #======================================= aggregation function arithmetic query cases ================ # asc/desc order [d.2] sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc; sql select (count(c1) * 2) % 7.9 from $stb order by ts desc; +sql select spread(c1 )/44 from $stb order by ts asc; + # all possible function in the arithmetic expressioin -sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2), xxxx, from $stb where ts < and ts > +sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2) from $stb where ts < and ts > # no result return [d.3] sql select first(c1) * 91 - 7, last(c3) from $stb where ts < 1537146000000 @@ -291,7 +395,7 @@ sql_error select c2-c2, c3-c4, c5%c6 from $tb fill(value, 12); # tag filter(not support for normal table). [d.15] -sql_error select c2+99 from $tb where t1=12; +sql_error select sum(c2)+99 from $tb where t1=12; # multi-field output [d.16] sql select count(*), sum(c1)*avg(c2), avg(c3)*count(c3), sum(c3), sum(c4), first(c7), last(c8), first(c9), first(c7), last(c8) from $tb @@ -325,5 +429,3 @@ sql select c2-c2 from $tb; sql select first(c1)-last(c1), spread(c2), max(c3) - min(c3), avg(c4)*count(c4) from $tb - - From c17d202e0c35c1a25ffcb23ef7424094fa9757d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 11:00:10 +0800 Subject: [PATCH 21/41] [TD-225] fix compiler error. --- src/query/src/qResultbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index dfa8bcd95b..55a7aea53a 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -122,7 +122,7 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); assert(ret == 0); - ret = fwrite(t, 1, size, pResultBuf->file); + ret = (int32_t) fwrite(t, 1, size, pResultBuf->file); assert(ret == size); if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { From c3fd294c0b816a0e52a12ba7b9d178899997ae21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 24 Nov 2020 12:00:17 +0800 Subject: [PATCH 22/41] [TD-225] fix bugs in regression test. --- src/query/inc/qUtil.h | 1 + src/query/src/qExecutor.c | 2 +- src/query/src/qUtil.c | 6 +++ .../general/parser/col_arithmetic_query.sim | 51 ++++++++++++++++++- 4 files changed, 57 insertions(+), 3 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 62bf245a2b..bc23ffadd4 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -81,6 +81,7 @@ int32_t getNumOfAllocatedResultRows(SResultRowPool* p); int32_t getNumOfUsedResultRows(SResultRowPool* p); uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv); +bool isPointInterpoQuery(SQuery *pQuery); #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7d6068ff64..fce25ae119 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1825,7 +1825,7 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) { } // todo refactor with isLastRowQuery -static bool isPointInterpoQuery(SQuery *pQuery) { +bool isPointInterpoQuery(SQuery *pQuery) { for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionID = pQuery->pExpr1[i].base.functionId; if (functionID == TSDB_FUNC_INTERP) { diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 4b8d7ea930..2b4cdef9b0 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -388,6 +388,12 @@ uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) { return 0; // for simple table query, the uid is always set to be 0; } + SQuery* pQuery = pRuntimeEnv->pQuery; + if ((pQuery->checkBuffer == 1 && pQuery->interval.interval == 0) || isPointInterpoQuery(pQuery) || + pRuntimeEnv->groupbyNormalCol) { + return 0; + } + STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current); return id->uid; } \ No newline at end of file diff --git a/tests/script/general/parser/col_arithmetic_query.sim b/tests/script/general/parser/col_arithmetic_query.sim index 95e0ed68d7..408b039144 100644 --- a/tests/script/general/parser/col_arithmetic_query.sim +++ b/tests/script/general/parser/col_arithmetic_query.sim @@ -323,9 +323,56 @@ sql select c2-c2, 911 from $tb #======================================= aggregation function arithmetic query cases ================ # asc/desc order [d.2] sql select first(c1) * ( 2 / 3 ) from $stb order by ts asc; -sql select (count(c1) * 2) % 7.9 from $stb order by ts desc; +if $rows != 1 then + return -1 +endi -sql select spread(c1 )/44 from $stb order by ts asc; +if $data00 != 0.000000000 then + return -1 +endi + +sql select first(c1) * (2/99) from $stb order by ts desc; +if $rows != 1 then + return -1 +endi + +if $data00 != 0.000000000 then + return -1 +endi + +sql select (count(c1) * 2) % 7.9, (count(c1) * 2), ( count(1)*2) from $stb order by ts desc; +if $rows != 1 then + return -1 +endi + +if $data00 != 1.800000000 then + return -1 +endi + +if $data01 != 100000 then + return -1 +endi + +if $data02 != 200000 then + return -1 +endi + +sql select spread( c1 )/44, spread(c1), 0.204545455 * 44 from $stb order by ts asc; +if $rows != 1 then + return -1 +endi + +if $data00 != 0.204545455 then + return -1 +endi + +if $data01 != 9.000000000 then + return -1 +endi + +if $data02 != 9.000000020 then + return -1 +endi # all possible function in the arithmetic expressioin sql select min(c1) * max(c2) /4, sum(c1) * percentile(c2, 20), apercentile(c4, 33) + 52/9, spread(c5)/min(c2) from $stb where ts < and ts > From f9c34c937068a76cde87aa719fb177749846404d Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Tue, 24 Nov 2020 17:27:42 +0800 Subject: [PATCH 23/41] [TD-2125][TD-2173]python connector --- src/connector/python/linux/python2/setup.py | 2 +- .../python/linux/python2/taos/cinterface.py | 115 ++++++++++++---- .../python/linux/python2/taos/cursor.py | 20 ++- src/connector/python/linux/python3/setup.py | 2 +- .../python/linux/python3/taos/cinterface.py | 129 ++++++++++++------ .../python/linux/python3/taos/cursor.py | 26 ++-- src/connector/python/windows/python2/setup.py | 2 +- .../python/windows/python2/taos/cinterface.py | 118 ++++++++++++---- .../python/windows/python2/taos/cursor.py | 22 ++- src/connector/python/windows/python3/setup.py | 2 +- .../python/windows/python3/taos/cinterface.py | 107 ++++++++++++--- .../python/windows/python3/taos/cursor.py | 22 ++- 12 files changed, 428 insertions(+), 139 deletions(-) diff --git a/src/connector/python/linux/python2/setup.py b/src/connector/python/linux/python2/setup.py index 2e4f80b8f0..b3daa98bdc 100644 --- a/src/connector/python/linux/python2/setup.py +++ b/src/connector/python/linux/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python2/taos/cinterface.py b/src/connector/python/linux/python2/taos/cinterface.py index 269326535c..6f0435722f 100644 --- a/src/connector/python/linux/python2/taos/cinterface.py +++ b/src/connector/python/linux/python2/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -227,8 +270,8 @@ class CTaosInterface(object): print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -237,7 +280,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -310,6 +353,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -393,7 +454,7 @@ class CTaosInterface(object): def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result) + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') if __name__ == '__main__': @@ -410,4 +471,4 @@ if __name__ == '__main__': print(data) cinter.freeResult(result) - cinter.close(conn) \ No newline at end of file + cinter.close(conn) diff --git a/src/connector/python/linux/python2/taos/cursor.py b/src/connector/python/linux/python2/taos/cursor.py index bc6670ca77..eee5e39488 100644 --- a/src/connector/python/linux/python2/taos/cursor.py +++ b/src/connector/python/linux/python2/taos/cursor.py @@ -49,7 +49,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock( + block, self._block_rows = CTaosInterface.fetchRow( self._result, self._fields) if self._block_rows == 0: raise StopIteration @@ -190,6 +190,23 @@ class TDengineCursor(object): if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: @@ -203,7 +220,6 @@ class TDengineCursor(object): for i in range(len(self._fields)): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) - def nextset(self): """ """ diff --git a/src/connector/python/linux/python3/setup.py b/src/connector/python/linux/python3/setup.py index 03a49fc1c5..f49ebe2b6d 100644 --- a/src/connector/python/linux/python3/setup.py +++ b/src/connector/python/linux/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/linux/python3/taos/cinterface.py b/src/connector/python/linux/python3/taos/cinterface.py index be5b99d8c1..bbf5a0c714 100644 --- a/src/connector/python/linux/python3/taos/cinterface.py +++ b/src/connector/python/linux/python3/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_long))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -253,7 +296,7 @@ class CTaosInterface(object): raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -308,29 +351,26 @@ class CTaosInterface(object): return fields - # @staticmethod - # def fetchBlock(result, fields): - # pblock = ctypes.c_void_p(0) - # num_of_rows = CTaosInterface.libtaos.taos_fetch_block( - # result, ctypes.byref(pblock)) - # if num_of_rows == 0: - # return None, 0 - - # isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) - # blocks = [None] * len(fields) - # fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) - # fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] - # for i in range(len(fields)): - # data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] - - # if fields[i]['type'] not in _CONVERT_FUNC: - # raise DatabaseError("Invalid data type returned from database") - # print('====================',fieldLen[i]) - # blocks[i] = _CONVERT_FUNC[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) - - # return blocks, abs(num_of_rows) @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -350,6 +390,7 @@ class CTaosInterface(object): else: return None, 0 return blocks, abs(num_of_rows) + @staticmethod def freeResult(result): CTaosInterface.libtaos.taos_free_result(result) diff --git a/src/connector/python/linux/python3/taos/cursor.py b/src/connector/python/linux/python3/taos/cursor.py index eb10bed485..d8184668c8 100644 --- a/src/connector/python/linux/python3/taos/cursor.py +++ b/src/connector/python/linux/python3/taos/cursor.py @@ -5,7 +5,6 @@ import threading # querySeqNum = 0 - class TDengineCursor(object): """Database cursor which is used to manage the context of a fetch operation. @@ -51,7 +50,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock( + block, self._block_rows = CTaosInterface.fetchRow( self._result, self._fields) if self._block_rows == 0: raise StopIteration @@ -196,18 +195,13 @@ class TDengineCursor(object): def fetchall(self): """Fetch all (remaining) rows of a query result, returning them as a sequence of sequences (e.g. a list of tuples). Note that the cursor's arraysize attribute can affect the performance of this operation. """ - # if threading.get_ident() != self._threadId: - # info ="[WARNING] Cursor fetchall:Thread ID not match,creater:"+str(self._threadId)+" caller:"+str(threading.get_ident()) - # raise OperationalError(info) - # print(info) - # return None if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: - block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) errno = CTaosInterface.libtaos.taos_errno(self._result) if errno != 0: raise ProgrammingError(CTaosInterface.errStr(self._result), errno) @@ -218,6 +212,22 @@ class TDengineCursor(object): buffer[i].extend(block[i]) return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchBlock(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) def nextset(self): """ """ diff --git a/src/connector/python/windows/python2/setup.py b/src/connector/python/windows/python2/setup.py index fd82a55650..34cb7a04d7 100644 --- a/src/connector/python/windows/python2/setup.py +++ b/src/connector/python/windows/python2/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python2/taos/cinterface.py b/src/connector/python/windows/python2/taos/cinterface.py index 084d38e41c..d4bf1a0350 100644 --- a/src/connector/python/windows/python2/taos/cinterface.py +++ b/src/connector/python/windows/python2/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -90,9 +90,7 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C nchar row to python row """ assert(nbytes is not None) - - res = [] - + res=[] for i in range(abs(num_of_rows)): try: if num_of_rows >= 0: @@ -103,17 +101,49 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): except ValueError: res.append(None) + return res + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +158,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -225,9 +268,10 @@ class CTaosInterface(object): if connection.value == None: print('connect to TDengine failed') + raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -236,7 +280,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -252,7 +296,7 @@ class CTaosInterface(object): raise AttributeError("sql is expected as a string") # finally: # CTaosInterface.libtaos.close(connection) - + @staticmethod def affectedRows(result): """The affected rows after runing query @@ -309,6 +353,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : @@ -393,7 +455,7 @@ class CTaosInterface(object): def errStr(result): """Return the error styring """ - return CTaosInterface.libtaos.taos_errstr(result) + return CTaosInterface.libtaos.taos_errstr(result).decode('utf-8') if __name__ == '__main__': @@ -410,4 +472,4 @@ if __name__ == '__main__': print(data) cinter.freeResult(result) - cinter.close(conn) \ No newline at end of file + cinter.close(conn) diff --git a/src/connector/python/windows/python2/taos/cursor.py b/src/connector/python/windows/python2/taos/cursor.py index 35846cbe11..71651afee1 100644 --- a/src/connector/python/windows/python2/taos/cursor.py +++ b/src/connector/python/windows/python2/taos/cursor.py @@ -50,7 +50,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -143,7 +143,25 @@ class TDengineCursor(object): """ if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: @@ -178,7 +196,7 @@ class TDengineCursor(object): self._description = None self._rowcount = -1 if self._result is not None: - CTaosInterface.freeResult(self._result) + CTaosInterface.freeResult(self._result) self._result = None self._fields = None self._block = None diff --git a/src/connector/python/windows/python3/setup.py b/src/connector/python/windows/python3/setup.py index 9abdace5a9..c09644d330 100644 --- a/src/connector/python/windows/python3/setup.py +++ b/src/connector/python/windows/python3/setup.py @@ -5,7 +5,7 @@ with open("README.md", "r") as fh: setuptools.setup( name="taos", - version="2.0.0", + version="2.0.2", author="Taosdata Inc.", author_email="support@taosdata.com", description="TDengine python client package", diff --git a/src/connector/python/windows/python3/taos/cinterface.py b/src/connector/python/windows/python3/taos/cinterface.py index 68adf191c9..fb1b65fc15 100644 --- a/src/connector/python/windows/python3/taos/cinterface.py +++ b/src/connector/python/windows/python3/taos/cinterface.py @@ -18,7 +18,7 @@ def _crow_timestamp_to_python(data, num_of_rows, nbytes=None, micro=False): _timestamp_converter = _convert_microsecond_to_datetime if num_of_rows > 0: - return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1])) + return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) else: return list(map(_timestamp_converter, ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)])) @@ -26,7 +26,7 @@ def _crow_bool_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bool row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BOOL_NULL else bool(ele) for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_bool))[:abs(num_of_rows)] ] @@ -34,7 +34,7 @@ def _crow_tinyint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C tinyint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_TINYINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_byte))[:abs(num_of_rows)] ] @@ -42,7 +42,7 @@ def _crow_smallint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C smallint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)][::1]] + return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)]] else: return [ None if ele == FieldType.C_SMALLINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_short))[:abs(num_of_rows)] ] @@ -50,7 +50,7 @@ def _crow_int_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C int row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_INT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_int))[:abs(num_of_rows)] ] @@ -58,7 +58,7 @@ def _crow_bigint_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C bigint row to python row """ if num_of_rows > 0: - return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)][::1] ] + return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] else: return [ None if ele == FieldType.C_BIGINT_NULL else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_longlong))[:abs(num_of_rows)] ] @@ -66,7 +66,7 @@ def _crow_float_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C float row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_float))[:abs(num_of_rows)] ] @@ -74,7 +74,7 @@ def _crow_double_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C double row to python row """ if num_of_rows > 0: - return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)][::1] ] + return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] else: return [ None if math.isnan(ele) else ele for ele in ctypes.cast(data, ctypes.POINTER(ctypes.c_double))[:abs(num_of_rows)] ] @@ -82,7 +82,7 @@ def _crow_binary_to_python(data, num_of_rows, nbytes=None, micro=False): """Function to convert C binary row to python row """ if num_of_rows > 0: - return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)][::1]] + return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] else: return [ None if ele.value[0:1] == FieldType.C_BINARY_NULL else ele.value.decode('utf-8') for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_char * nbytes)))[:abs(num_of_rows)]] @@ -104,16 +104,48 @@ def _crow_nchar_to_python(data, num_of_rows, nbytes=None, micro=False): res.append(None) return res - # if num_of_rows > 0: - # for i in range(abs(num_of_rows)): - # try: - # res.append( (ctypes.cast(data+nbytes*i, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) - # except ValueError: - # res.append(None) - # return res - # # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)][::-1]] - # else: - # return [ele.value for ele in (ctypes.cast(data, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[:abs(num_of_rows)]] + +def _crow_binary_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C binary row to python row + """ + res=[] + if num_of_rows > 0: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + rbyte=ctypes.cast(data+nbytes*i,ctypes.POINTER(ctypes.c_short))[:1].pop() + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode()[0:rbyte] ) + except ValueError: + res.append(None) + return res + +def _crow_nchar_to_python_block(data, num_of_rows, nbytes=None, micro=False): + """Function to convert C nchar row to python row + """ + assert(nbytes is not None) + res=[] + if num_of_rows >= 0: + for i in range(abs(num_of_rows)): + try: + tmpstr = ctypes.c_char_p(data+nbytes*i+2) + res.append( tmpstr.value.decode() ) + except ValueError: + res.append(None) + else: + for i in range(abs(num_of_rows)): + try: + res.append( (ctypes.cast(data+nbytes*i+2, ctypes.POINTER(ctypes.c_wchar * (nbytes//4))))[0].value ) + except ValueError: + res.append(None) + return res _CONVERT_FUNC = { FieldType.C_BOOL: _crow_bool_to_python, @@ -128,6 +160,19 @@ _CONVERT_FUNC = { FieldType.C_NCHAR : _crow_nchar_to_python } +_CONVERT_FUNC_BLOCK = { + FieldType.C_BOOL: _crow_bool_to_python, + FieldType.C_TINYINT : _crow_tinyint_to_python, + FieldType.C_SMALLINT : _crow_smallint_to_python, + FieldType.C_INT : _crow_int_to_python, + FieldType.C_BIGINT : _crow_bigint_to_python, + FieldType.C_FLOAT : _crow_float_to_python, + FieldType.C_DOUBLE : _crow_double_to_python, + FieldType.C_BINARY: _crow_binary_to_python_block, + FieldType.C_TIMESTAMP : _crow_timestamp_to_python, + FieldType.C_NCHAR : _crow_nchar_to_python_block +} + # Corresponding TAOS_FIELD structure in C class TaosField(ctypes.Structure): _fields_ = [('name', ctypes.c_char * 65), @@ -227,8 +272,8 @@ class CTaosInterface(object): print('connect to TDengine failed') raise ConnectionError("connect to TDengine failed") # sys.exit(1) - else: - print('connect to TDengine success') + #else: + # print('connect to TDengine success') return connection @@ -237,7 +282,7 @@ class CTaosInterface(object): '''Close the TDengine handle ''' CTaosInterface.libtaos.taos_close(connection) - print('connection is closed') + #print('connection is closed') @staticmethod def query(connection, sql): @@ -310,6 +355,24 @@ class CTaosInterface(object): @staticmethod def fetchBlock(result, fields): + pblock = ctypes.c_void_p(0) + num_of_rows = CTaosInterface.libtaos.taos_fetch_block( + result, ctypes.byref(pblock)) + if num_of_rows == 0: + return None, 0 + isMicro = (CTaosInterface.libtaos.taos_result_precision(result) == FieldType.C_TIMESTAMP_MICRO) + blocks = [None] * len(fields) + fieldL = CTaosInterface.libtaos.taos_fetch_lengths(result) + fieldLen = [ele for ele in ctypes.cast(fieldL, ctypes.POINTER(ctypes.c_int))[:len(fields)]] + for i in range(len(fields)): + data = ctypes.cast(pblock, ctypes.POINTER(ctypes.c_void_p))[i] + if fields[i]['type'] not in _CONVERT_FUNC_BLOCK: + raise DatabaseError("Invalid data type returned from database") + blocks[i] = _CONVERT_FUNC_BLOCK[fields[i]['type']](data, num_of_rows, fieldLen[i], isMicro) + + return blocks, abs(num_of_rows) + @staticmethod + def fetchRow(result, fields): pblock = ctypes.c_void_p(0) pblock = CTaosInterface.libtaos.taos_fetch_row(result) if pblock : diff --git a/src/connector/python/windows/python3/taos/cursor.py b/src/connector/python/windows/python3/taos/cursor.py index b58b494cad..b813bba357 100644 --- a/src/connector/python/windows/python3/taos/cursor.py +++ b/src/connector/python/windows/python3/taos/cursor.py @@ -51,7 +51,7 @@ class TDengineCursor(object): raise OperationalError("Invalid use of fetch iterator") if self._block_rows <= self._block_iter: - block, self._block_rows = CTaosInterface.fetchBlock(self._result, self._fields) + block, self._block_rows = CTaosInterface.fetchRow(self._result, self._fields) if self._block_rows == 0: raise StopIteration self._block = list(map(tuple, zip(*block))) @@ -144,7 +144,25 @@ class TDengineCursor(object): """ if self._result is None or self._fields is None: raise OperationalError("Invalid use of fetchall") - + + buffer = [[] for i in range(len(self._fields))] + self._rowcount = 0 + while True: + block, num_of_fields = CTaosInterface.fetchRow(self._result, self._fields) + errno = CTaosInterface.libtaos.taos_errno(self._result) + if errno != 0: + raise ProgrammingError(CTaosInterface.errStr(self._result), errno) + if num_of_fields == 0: + break + self._rowcount += num_of_fields + for i in range(len(self._fields)): + buffer[i].extend(block[i]) + return list(map(tuple, zip(*buffer))) + + def fetchall_block(self): + if self._result is None or self._fields is None: + raise OperationalError("Invalid use of fetchall") + buffer = [[] for i in range(len(self._fields))] self._rowcount = 0 while True: From 19dedc226accda1655633aa1dd66d5ed47282445 Mon Sep 17 00:00:00 2001 From: zyyang Date: Tue, 24 Nov 2020 18:34:02 +0800 Subject: [PATCH 24/41] [TD-2132]: add memory leak test case --- .../jdbc/cases/AppMemoryLeakTest.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java new file mode 100644 index 0000000000..8a3dbb2bc1 --- /dev/null +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java @@ -0,0 +1,23 @@ +package com.taosdata.jdbc.cases; + +import org.junit.Test; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +public class AppMemoryLeakTest { + + @Test + public void testAppMemoryLeak() { + try { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + + while (true) { + DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + } + } catch (ClassNotFoundException | SQLException e) { + e.printStackTrace(); + } + } +} From 0e88426c7f864a678095ab761870d55a49cb7eb7 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 11:19:15 +0800 Subject: [PATCH 25/41] [TD-2152] add test case for join query --- tests/pytest/insert/restfulInsert.py | 7 +++++-- tests/pytest/query/filterOtherTypes.py | 18 ++++++------------ tests/pytest/query/queryJoin.py | 2 ++ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index 9fa1f33a24..c39d1cbe48 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -55,9 +55,12 @@ class RestfulInsert: def insertUnlimitedData(self, threadID): print("thread %d started" % threadID) tablesPerThread = int (self.numOfTables / self.numOfThreads) + + count = 0 while True: i = 0 - start = self.ts + start = self.ts + count * self.batchSize + count = count + 1 for i in range(tablesPerThread): tableID = i + threadID * tablesPerThread @@ -65,7 +68,7 @@ class RestfulInsert: data = "insert into %s.%s%d values" % (self.dbname, self.tableNamePerfix, tableID) values = [] for k in range(self.batchSize): - values.append("(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) + values.append("(%d, %d, %d, %d)" % (start + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100))) if(self.outOfOrder == False): for k in range(len(values)): diff --git a/tests/pytest/query/filterOtherTypes.py b/tests/pytest/query/filterOtherTypes.py index 85d5a67bef..f80552138d 100644 --- a/tests/pytest/query/filterOtherTypes.py +++ b/tests/pytest/query/filterOtherTypes.py @@ -376,11 +376,9 @@ class TDTestCase: tdSql.execute("insert into t1 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") tdSql.execute("insert into t2 values(1538548685000, 4) (1538548685001, 5) (1538548685002, 6)") - tdSql.query("select * from t1 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t1 where tag1 like '%g'") - tdSql.query("select * from t2 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t2 where tag1 like '%g'") tdSql.query("select * from meters where tag1 like '%g'") tdSql.checkRows(6) @@ -396,20 +394,16 @@ class TDTestCase: tdSql.execute("insert into t5 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") tdSql.execute("insert into t6 values(1538548685000, 1) (1538548685001, 2) (1538548685002, 3)") - tdSql.query("select * from t3 where tag1 like '%京'") - tdSql.checkRows(3) + tdSql.error("select * from t3 where tag1 like '%京'") - tdSql.query("select * from t4 where tag1 like '%京'") - tdSql.checkRows(3) + tdSql.error("select * from t4 where tag1 like '%京'") tdSql.query("select * from meters1 where tag1 like '%京'") tdSql.checkRows(6) - tdSql.query("select * from t5 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t5 where tag1 like '%g'") - tdSql.query("select * from t6 where tag1 like '%g'") - tdSql.checkRows(3) + tdSql.error("select * from t6 where tag1 like '%g'") tdSql.query("select * from meters1 where tag1 like '%g'") tdSql.checkRows(6) diff --git a/tests/pytest/query/queryJoin.py b/tests/pytest/query/queryJoin.py index 57b7e8868d..59e01615b4 100644 --- a/tests/pytest/query/queryJoin.py +++ b/tests/pytest/query/queryJoin.py @@ -175,6 +175,8 @@ class TDTestCase: tdSql.error("select count(join_mt0.c1), first(join_mt0.c1)-first(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts") tdSql.error("select count(join_mt0.c1), first(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2 order by join_mt0.t1 desc slimit 3") tdSql.error("select count(join_mt0.c1), first(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts interval(10a) group by join_mt0.t1, join_mt0.t2, join_mt1.t1 order by join_mt0.ts desc, join_mt1.ts asc limit 10;") + tdSql.error("select join_mt1.c1,join_mt0.c1 from join_mt1,join_mt0 where join_mt1.ts = join_mt0.ts and join_mt1.t1 = join_mt0.t1 order by t") + def stop(self): tdSql.close() From 092c9de4aa994bd1d15f23e02591407a6bd25edc Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Wed, 25 Nov 2020 15:14:01 +0800 Subject: [PATCH 26/41] [TD-2218]add test case --- tests/pytest/fulltest.sh | 1 + tests/pytest/query/bug2218.py | 54 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 tests/pytest/query/bug2218.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 525fbad6c1..8977d96df4 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -158,6 +158,7 @@ python3 ./test.py -f query/bug1471.py python3 ./test.py -f query/bug1874.py python3 ./test.py -f query/bug1875.py python3 ./test.py -f query/bug1876.py +python3 ./test.py -f query/bug2218.py #stream python3 ./test.py -f stream/metric_1.py diff --git a/tests/pytest/query/bug2218.py b/tests/pytest/query/bug2218.py new file mode 100644 index 0000000000..bb92e5d9ce --- /dev/null +++ b/tests/pytest/query/bug2218.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + print("==========step1") + print("create table && insert data") + + tdSql.execute("create table mt0 (ts timestamp, c1 int, c2 float, c3 bigint, c4 smallint, c5 tinyint, c6 double, c7 bool)") + insertRows = 1000 + t0 = 1604298064000 + tdLog.info("insert %d rows" % (insertRows)) + for i in range(insertRows): + ret = tdSql.execute( + "insert into mt0 values (%d , %d,%d,%d,%d,%d,%d,%d)" % + (t0+i,i%100,i/2,i%100,i%100,i%100,i*1.0,i%2)) + print("==========step2") + print("test col*1*1 desc ") + tdSql.query('select c1,c1*1*1,c2*1*1,c3*1*1,c4*1*1,c5*1*1,c6*1*1 from mt0 order by ts desc limit 2') + tdSql.checkData(0,0,99) + tdSql.checkData(0,1,0.0) + tdSql.checkData(0,2,0.0) + tdSql.checkData(0,3,0.0) + tdSql.checkData(0,4,0.0) + tdSql.checkData(0,5,0.0) + tdSql.checkData(0,6,0.0) + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 3e7c81eafbcd57880ea2f4ac799858b753a02391 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 16:03:43 +0800 Subject: [PATCH 27/41] [TD-2126] add test case for taosdump --- tests/pytest/tools/taosdumpTest.py | 89 ++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tests/pytest/tools/taosdumpTest.py diff --git a/tests/pytest/tools/taosdumpTest.py b/tests/pytest/tools/taosdumpTest.py new file mode 100644 index 0000000000..534a477b34 --- /dev/null +++ b/tests/pytest/tools/taosdumpTest.py @@ -0,0 +1,89 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import os +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + self.ts = 1538548685000 + self.numberOfTables = 10000 + self.numberOfRecords = 100 + + def run(self): + tdSql.prepare() + + tdSql.execute("create table st(ts timestamp, c1 int, c2 nchar(10)) tags(t1 int, t2 binary(10))") + tdSql.execute("create table t1 using st tags(1, 'beijing')") + sql = "insert into t1 values" + currts = self.ts + for i in range(100): + sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) + tdSql.execute(sql) + + tdSql.execute("create table t2 using st tags(2, 'shanghai')") + sql = "insert into t2 values" + currts = self.ts + for i in range(100): + sql += "(%d, %d, 'nchar%d')" % (currts + i, i % 100, i % 100) + tdSql.execute(sql) + + os.system("taosdump --databases db -o /tmp") + + tdSql.execute("drop database db") + tdSql.query("show databases") + tdSql.checkRows(0) + + os.system("taosdump -i /tmp") + + tdSql.query("show databases") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'db') + + tdSql.execute("use db") + tdSql.query("show stables") + tdSql.checkRows(1) + tdSql.checkData(0, 0, 'st') + + tdSql.query("show tables") + tdSql.checkRows(2) + tdSql.checkData(0, 0, 't2') + tdSql.checkData(1, 0, 't1') + + tdSql.query("select * from t1") + tdSql.checkRows(100) + for i in range(100): + tdSql.checkData(i, 1, i) + tdSql.checkData(i, 2, "nchar%d" % i) + + tdSql.query("select * from t2") + tdSql.checkRows(100) + for i in range(100): + tdSql.checkData(i, 1, i) + tdSql.checkData(i, 2, "nchar%d" % i) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 431fc93950be86183926fa2503529810d93ec7fd Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 25 Nov 2020 16:15:22 +0800 Subject: [PATCH 28/41] add test cases into fulltest.sh --- tests/pytest/fulltest.sh | 4 +++- tests/pytest/tools/{lowa.py => lowaTest.py} | 0 tests/pytest/tools/{taosdemo.py => taosdemoTest.py} | 0 3 files changed, 3 insertions(+), 1 deletion(-) rename tests/pytest/tools/{lowa.py => lowaTest.py} (100%) rename tests/pytest/tools/{taosdemo.py => taosdemoTest.py} (100%) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index 525fbad6c1..89b5b0f125 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -205,7 +205,9 @@ python3 test.py -f query/queryInterval.py python3 test.py -f query/queryFillTest.py # tools -python3 test.py -f tools/taosdemo.py +python3 test.py -f tools/taosdemoTest.py +python3 test.py -f tools/taosdumpTest.py +python3 test.py -f tools/lowaTest.py # subscribe python3 test.py -f subscribe/singlemeter.py diff --git a/tests/pytest/tools/lowa.py b/tests/pytest/tools/lowaTest.py similarity index 100% rename from tests/pytest/tools/lowa.py rename to tests/pytest/tools/lowaTest.py diff --git a/tests/pytest/tools/taosdemo.py b/tests/pytest/tools/taosdemoTest.py similarity index 100% rename from tests/pytest/tools/taosdemo.py rename to tests/pytest/tools/taosdemoTest.py From aeed9db84c8b752eabc0577477f78733e33519a8 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Thu, 26 Nov 2020 09:27:17 +0800 Subject: [PATCH 29/41] [TD-2230] use error max len when dump in --- src/kit/taosdump/taosdump.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 88f07ee602..a7258c9724 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -2146,7 +2146,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c char * line = NULL; size_t line_len = 0; - cmd = (char *)malloc(COMMAND_SIZE); + cmd = (char *)malloc(TSDB_MAX_ALLOWED_SQL_LEN); if (cmd == NULL) { fprintf(stderr, "failed to allocate memory\n"); return -1; @@ -2155,7 +2155,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c int lineNo = 0; while ((read_len = getline(&line, &line_len, fp)) != -1) { ++lineNo; - if (read_len >= COMMAND_SIZE) continue; + if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; line[--read_len] = '\0'; //if (read_len == 0 || isCommentLine(line)) { // line starts with # @@ -2176,7 +2176,7 @@ int taosDumpInOneFile(TAOS * taos, FILE* fp, char* fcharset, char* encode, c fprintf(stderr, "error sql: linenu:%d, file:%s\n", lineNo, fileName); } - memset(cmd, 0, COMMAND_SIZE); + memset(cmd, 0, TSDB_MAX_ALLOWED_SQL_LEN); cmd_len = 0; } From 4f2d7a1a83c00d900fda99ee174e8008f89bfc02 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 04:07:38 +0000 Subject: [PATCH 30/41] [TD-2216]: crash in qUtil.c --- src/query/inc/qUtil.h | 2 +- src/query/src/qUtil.c | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index bc23ffadd4..8b84ac0182 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -27,7 +27,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery); void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow, int16_t type); -void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); +void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src, int16_t type); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 2b4cdef9b0..3f56366db8 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -133,7 +133,7 @@ void clearFirstNWindowRes(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]); + copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k], type); } // move the unclosed window in the front of the window list @@ -272,9 +272,15 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes, int16 * since the attribute of "Pos" is bound to each window result when the window result is created in the * disk-based result buffer. */ -void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) { +void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src, int16_t type) { dst->numOfRows = src->numOfRows; - dst->win = src->win; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + dst->key = realloc(dst->key, varDataTLen(src->key)); + varDataCopy(dst->key, src->key); + } else { + dst->win = src->win; + } dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; From e9c83f8251545936fe424a9004ddb4c6003c8612 Mon Sep 17 00:00:00 2001 From: zyyang Date: Thu, 26 Nov 2020 14:33:53 +0800 Subject: [PATCH 31/41] [TD-2132]: add memory leak test case --- .../jdbc/cases/AppMemoryLeakTest.java | 41 ++++++++++++++----- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java index 8a3dbb2bc1..8de2e3b442 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/AppMemoryLeakTest.java @@ -5,19 +5,40 @@ import org.junit.Test; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Statement; public class AppMemoryLeakTest { - @Test - public void testAppMemoryLeak() { - try { - Class.forName("com.taosdata.jdbc.TSDBDriver"); - - while (true) { - DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); - } - } catch (ClassNotFoundException | SQLException e) { - e.printStackTrace(); + @Test(expected = SQLException.class) + public void testCreateTooManyConnection() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int conCnt = 0; + while (true) { + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + System.out.println(conCnt++ + " : " + conn); } } + + @Test + public void testCreateTooManyStatement() throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int stmtCnt = 0; + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + while (true) { + Statement stmt = conn.createStatement(); + System.out.println(++stmtCnt + " : " + stmt); + } + } + + public static void main(String[] args) throws ClassNotFoundException, SQLException { + Class.forName("com.taosdata.jdbc.TSDBDriver"); + int stmtCnt = 0; + Connection conn = DriverManager.getConnection("jdbc:TAOS://localhost:6030/?user=root&password=taosdata"); + while (true) { + Statement stmt = conn.createStatement(); + System.out.println(++stmtCnt + " : " + stmt); + } + } + + } From d3729d273b5b1f200df77b2f1fc137148b2e5ff9 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 26 Nov 2020 15:09:41 +0800 Subject: [PATCH 32/41] add test for column which not exist --- tests/pytest/concurrent_inquiry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/concurrent_inquiry.py b/tests/pytest/concurrent_inquiry.py index 5d1e9a7537..6f4258312d 100644 --- a/tests/pytest/concurrent_inquiry.py +++ b/tests/pytest/concurrent_inquiry.py @@ -146,7 +146,7 @@ class ConcurrentInquiry: col_list=self.stb_stru_list[tbi-1] tag_list=self.stb_tag_list[tbi-1] is_stb=1 - tlist=col_list+tag_list + tlist=col_list+tag_list+['abc'] #增加不存在的域'abc',是否会引起新bug con_rand=random.randint(0,len(condition_list)) func_rand=random.randint(0,len(func_list)) col_rand=random.randint(0,len(col_list)) From f547e29a55e57031d035c77bc685e813ffaad772 Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 07:39:47 +0000 Subject: [PATCH 33/41] [TD-2135]: reduce memory usage --- src/client/inc/tsclient.h | 2 +- src/client/src/tscParseInsert.c | 20 ++++++++++++++------ src/client/src/tscServer.c | 28 +++++++++++++++++++--------- src/client/src/tscUtil.c | 16 ++++++++++++++-- 4 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index ff36cf0f5a..c482375bff 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -282,7 +282,7 @@ typedef struct { int8_t dataSourceType; // load data from file or not int8_t submitSchema; // submit block is built with table schema - STagData tagData; + STagData *pTagData; // NOTE: pTagData->data is used as a variant length array SHashObj *pTableList; // referred table involved in sql SArray *pDataBlocks; // SArray submit data blocks after parsing sql } SSqlCmd; diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a44a158f93..6de8195d73 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -790,9 +790,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { sql += index; tscAllocPayload(pCmd, sizeof(STagData)); - STagData *pTag = &pCmd->tagData; - - memset(pTag, 0, sizeof(STagData)); //the source super table is moved to the secondary position of the pTableMetaInfo list if (pQueryInfo->numOfTables < 2) { @@ -805,7 +802,14 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } + STagData *pTag = realloc(pCmd->pTagData, offsetof(STagData, data)); + if (pTag == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memset(pTag, 0, offsetof(STagData, data)); tstrncpy(pTag->name, pSTableMeterMetaInfo->name, sizeof(pTag->name)); + pCmd->pTagData = pTag; + code = tscGetTableMeta(pSql, pSTableMeterMetaInfo); if (code != TSDB_CODE_SUCCESS) { return code; @@ -934,7 +938,13 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } tdSortKVRowByColIdx(row); - pTag->dataLen = kvRowLen(row); + + pTag = (STagData*)realloc(pCmd->pTagData, offsetof(STagData, data) + kvRowLen(row)); + if (pTag == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + pCmd->pTagData = pTag; + pTag->dataLen = htonl(kvRowLen(row)); kvRowCpy(pTag->data, row); free(row); @@ -945,8 +955,6 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return tscSQLSyntaxErrMsg(pCmd->payload, ") expected", sToken.z); } - pTag->dataLen = htonl(pTag->dataLen); - if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) { return tscInvalidSQLErrMsg(pCmd->payload, "invalid table name", *sqlstr); } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 15859674ab..9735e26ece 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1565,11 +1565,11 @@ int tscBuildTableMetaMsg(SSqlObj *pSql, SSqlInfo *pInfo) { char *pMsg = (char *)pInfoMsg + sizeof(STableInfoMsg); - size_t len = htonl(pCmd->tagData.dataLen); - if (pSql->cmd.autoCreated) { + if (pCmd->autoCreated && pCmd->pTagData != NULL) { + int len = htonl(pCmd->pTagData->dataLen); if (len > 0) { - len += sizeof(pCmd->tagData.name) + sizeof(pCmd->tagData.dataLen); - memcpy(pInfoMsg->tags, &pCmd->tagData, len); + len += sizeof(pCmd->pTagData->name) + sizeof(pCmd->pTagData->dataLen); + memcpy(pInfoMsg->tags, pCmd->pTagData, len); pMsg += len; } } @@ -2239,8 +2239,6 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->signature = pNew; pNew->cmd.command = TSDB_SQL_META; - registerSqlObj(pNew); - tscAddSubqueryInfo(&pNew->cmd); SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); @@ -2248,8 +2246,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { tscError("%p malloc failed for payload to get table meta", pSql); - free(pNew); - + tscFreeSqlObj(pNew); return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2257,12 +2254,25 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf assert(pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); tstrncpy(pNewMeterMetaInfo->name, pTableMetaInfo->name, sizeof(pNewMeterMetaInfo->name)); - memcpy(&pNew->cmd.tagData, &pSql->cmd.tagData, sizeof(pSql->cmd.tagData)); + + if (pSql->cmd.pTagData != NULL) { + int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); + pNew->cmd.pTagData = calloc(1, size); + if (pNew->cmd.pTagData == NULL) { + tscError("%p malloc failed for new tag data to get table meta", pSql); + tscFreeSqlObj(pNew); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); + } + tscDebug("%p new pSqlObj:%p to get tableMeta, auto create:%d", pSql, pNew, pNew->cmd.autoCreated); pNew->fp = tscTableMetaCallBack; pNew->param = pSql; + registerSqlObj(pNew); + int32_t code = tscProcessSql(pNew); if (code == TSDB_CODE_SUCCESS) { code = TSDB_CODE_TSC_ACTION_IN_PROGRESS; // notify upper application that current process need to be terminated diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index a75f6d8296..d22c2588cc 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -408,7 +408,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd, bool removeFromCache) { pCmd->pTableList = NULL; pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); - + tscFreeQueryInfo(pCmd, removeFromCache); } @@ -512,6 +512,8 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscFreeSqlResult(pSql); tscResetSqlCmdObj(pCmd, false); + tfree(pCmd->pTagData); + memset(pCmd->payload, 0, (size_t)pCmd->allocSize); tfree(pCmd->payload); pCmd->allocSize = 0; @@ -1909,7 +1911,17 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm pCmd->command = cmd; pCmd->parseFinished = 1; pCmd->autoCreated = pSql->cmd.autoCreated; - memcpy(&pCmd->tagData, &pSql->cmd.tagData, sizeof(pCmd->tagData)); + + if (pSql->cmd.pTagData != NULL) { + int size = offsetof(STagData, data) + htonl(pSql->cmd.pTagData->dataLen); + pNew->cmd.pTagData = calloc(1, size); + if (pNew->cmd.pTagData == NULL) { + tscError("%p new subquery failed, unable to malloc tag data, tableIndex:%d", pSql, 0); + free(pNew); + return NULL; + } + memcpy(pNew->cmd.pTagData, pSql->cmd.pTagData, size); + } if (tscAddSubqueryInfo(pCmd) != TSDB_CODE_SUCCESS) { tscFreeSqlObj(pNew); From 55ce69e2f0c08aed576b46bf11cff1a7b7bc2d4a Mon Sep 17 00:00:00 2001 From: Bomin Zhang Date: Thu, 26 Nov 2020 08:00:58 +0000 Subject: [PATCH 34/41] [TD-2182]: coverity issues --- src/client/src/tscSub.c | 7 ++++++- src/query/src/qExecutor.c | 6 +++--- src/util/src/tqueue.c | 3 --- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/client/src/tscSub.c b/src/client/src/tscSub.c index 2c81bd7c7c..a782b53e75 100644 --- a/src/client/src/tscSub.c +++ b/src/client/src/tscSub.c @@ -442,7 +442,12 @@ TAOS_RES *taos_consume(TAOS_SUB *tsub) { size_t size = taosArrayGetSize(pSub->progress) * sizeof(STableIdInfo); size += sizeof(SQueryTableMsg) + 4096; - tscAllocPayload(&pSql->cmd, (int)size); + int code = tscAllocPayload(&pSql->cmd, (int)size); + if (code != TSDB_CODE_SUCCESS) { + tscError("failed to alloc payload"); + return NULL; + } + for (int retry = 0; retry < 3; retry++) { tscRemoveFromSqlList(pSql); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index fce25ae119..826858d1dd 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -5836,9 +5836,9 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); - pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; - pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; - pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; + // pQueryMsg->interval.intervalUnit = pQueryMsg->interval.intervalUnit; + // pQueryMsg->interval.slidingUnit = pQueryMsg->interval.slidingUnit; + // pQueryMsg->interval.offsetUnit = pQueryMsg->interval.offsetUnit; pQueryMsg->limit = htobe64(pQueryMsg->limit); pQueryMsg->offset = htobe64(pQueryMsg->offset); diff --git a/src/util/src/tqueue.c b/src/util/src/tqueue.c index d3c17e7626..143ce7c474 100644 --- a/src/util/src/tqueue.c +++ b/src/util/src/tqueue.c @@ -79,15 +79,12 @@ void taosCloseQueue(taos_queue param) { if (queue->qset) taosRemoveFromQset(qset, queue); - pthread_mutex_lock(&queue->mutex); - while (pNode) { pTemp = pNode; pNode = pNode->next; free (pTemp); } - pthread_mutex_unlock(&queue->mutex); pthread_mutex_destroy(&queue->mutex); free(queue); From 651672957d470f715b265c29892f495d62c94da0 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 26 Nov 2020 16:32:10 +0800 Subject: [PATCH 35/41] [TD-2240] update restfulInsert --- tests/pytest/insert/restfulInsert.py | 37 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/pytest/insert/restfulInsert.py b/tests/pytest/insert/restfulInsert.py index c39d1cbe48..e3a963f1d4 100644 --- a/tests/pytest/insert/restfulInsert.py +++ b/tests/pytest/insert/restfulInsert.py @@ -18,10 +18,10 @@ import time import argparse class RestfulInsert: - def __init__(self, host, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder): + def __init__(self, host, startTimestamp, dbname, threads, tables, records, batchSize, tbNamePerfix, outOfOrder): self.header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} self.url = "http://%s:6041/rest/sql" % host - self.ts = 1500000000000 + self.ts = startTimestamp self.dbname = dbname self.numOfThreads = threads self.numOfTables = tables @@ -36,8 +36,10 @@ class RestfulInsert: for i in range(tablesPerThread): tableID = threadID * tablesPerThread name = 'beijing' if tableID % 2 == 0 else 'shanghai' - data = "create table %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) - requests.post(self.url, data, headers = self.header) + data = "create table if not exists %s.%s%d using %s.meters tags(%d, '%s')" % (self.dbname, self.tableNamePerfix, tableID + i, self.dbname, tableID + i, name) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) def insertData(self, threadID): print("thread %d started" % threadID) @@ -50,7 +52,9 @@ class RestfulInsert: values = [] for k in range(self.batchSize): data += "(%d, %d, %d, %d)" % (start + j * self.batchSize + k, random.randint(1, 100), random.randint(1, 100), random.randint(1, 100)) - requests.post(self.url, data, headers = self.header) + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) def insertUnlimitedData(self, threadID): print("thread %d started" % threadID) @@ -76,15 +80,15 @@ class RestfulInsert: else: random.shuffle(values) for k in range(len(values)): - data += values[k] - requests.post(self.url, data, headers = self.header) + data += values[k] + response = requests.post(self.url, data, headers = self.header) + if response.status_code != 200: + print(response.content) - def run(self): - data = "drop database if exists %s" % self.dbname + def run(self): + data = "create database if not exists %s" % self.dbname requests.post(self.url, data, headers = self.header) - data = "create database %s" % self.dbname - requests.post(self.url, data, headers = self.header) - data = "create table %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname + data = "create table if not exists %s.meters(ts timestamp, f1 int, f2 int, f3 int) tags(id int, loc nchar(20))" % self.dbname requests.post(self.url, data, headers = self.header) threads = [] @@ -120,6 +124,13 @@ parser.add_argument( default='127.0.0.1', type=str, help='host name to be connected (default: 127.0.0.1)') +parser.add_argument( + '-S', + '--start-timestamp', + action='store', + default=1500000000000, + type=int, + help='insert data from timestamp (default: 1500000000000)') parser.add_argument( '-d', '--db-name', @@ -169,5 +180,5 @@ parser.add_argument( help='The order of test data (default: False)') args = parser.parse_args() -ri = RestfulInsert(args.host_name, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order) +ri = RestfulInsert(args.host_name, args.start_timestamp, args.db_name, args.number_of_threads, args.number_of_tables, args.number_of_records, args.batch_size, args.table_name_prefix, args.out_of_order) ri.run() \ No newline at end of file From fa25d38dea8ec9e310c85fac1da6aa36c70104ef Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Thu, 26 Nov 2020 16:37:11 +0800 Subject: [PATCH 36/41] full test --- tests/test-all.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/test-all.sh b/tests/test-all.sh index ff47cbfd71..5897978bce 100755 --- a/tests/test-all.sh +++ b/tests/test-all.sh @@ -17,9 +17,9 @@ function runSimCaseOneByOne { echo -e "${GREEN}$case success${NC}" | tee -a out.log || \ echo -e "${RED}$case failed${NC}" | tee -a out.log out_log=`tail -1 out.log ` - if [[ $out_log =~ 'failed' ]];then - exit 8 - fi + # if [[ $out_log =~ 'failed' ]];then + # exit 8 + # fi end_time=`date +%s` echo execution time of $case was `expr $end_time - $start_time`s. | tee -a out.log fi @@ -42,9 +42,9 @@ function runPyCaseOneByOne { echo -e "${RED}$case failed${NC}" | tee -a pytest-out.log end_time=`date +%s` out_log=`tail -1 pytest-out.log ` - if [[ $out_log =~ 'failed' ]];then - exit 8 - fi + # if [[ $out_log =~ 'failed' ]];then + # exit 8 + # fi echo execution time of $case was `expr $end_time - $start_time`s. | tee -a pytest-out.log else $line > /dev/null 2>&1 From 19eccc86ade72ded17cd4c4033b44b275137dc0f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 26 Nov 2020 08:44:20 +0000 Subject: [PATCH 37/41] fix TD-2184 --- src/tsdb/src/tsdbMemTable.c | 4 ++-- src/util/src/tskiplist.c | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 71944c87c6..4c51608b5d 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -871,10 +871,10 @@ static void tsdbFreeRows(STsdbRepo *pRepo, void **rows, int rowCounter) { listNEles(pRepo->mem->bufBlockList), pBufBlock->offset, pBufBlock->remain); if (pBufBlock->offset == 0) { // return the block to buffer pool - tsdbLockRepo(pRepo); + if (tsdbLockRepo(pRepo) < 0) return; SListNode *pNode = tdListPopTail(pRepo->mem->bufBlockList); tdListPrependNode(pBufPool->bufBlockList, pNode); - tsdbUnlockRepo(pRepo); + if (tsdbUnlockRepo(pRepo) < 0) return; } } else { ASSERT(listNEles(pRepo->mem->extraBuffList) > 0); diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index a36f7f0261..20bec16954 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -291,7 +291,10 @@ bool tSkipListIterNext(SSkipListIterator *iter) { iter->next = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0); iter->step++; } else { - if (iter->cur == pSkipList->pHead) return false; + if (iter->cur == pSkipList->pHead) { + tSkipListUnlock(pSkipList); + return false; + } iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0); // a new node is inserted into between iter->cur and iter->next, ignore it From 86479c1f6a42a97d0ef5d880e18565eae1660ded Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 27 Nov 2020 11:33:44 +0800 Subject: [PATCH 38/41] TD-2246 --- tests/test/c/CMakeLists.txt | 7 ++-- tests/test/c/hashIterator.c | 72 +++++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 tests/test/c/hashIterator.c diff --git a/tests/test/c/CMakeLists.txt b/tests/test/c/CMakeLists.txt index 26aa20e647..11480a8ba2 100644 --- a/tests/test/c/CMakeLists.txt +++ b/tests/test/c/CMakeLists.txt @@ -31,8 +31,8 @@ IF (TD_LINUX) #add_executable(createTablePerformance createTablePerformance.c) #target_link_libraries(createTablePerformance taos_static tutil common pthread) - #add_executable(createNormalTable createNormalTable.c) - #target_link_libraries(createNormalTable taos_static tutil common pthread) + add_executable(createNormalTable createNormalTable.c) + target_link_libraries(createNormalTable taos_static tutil common pthread) #add_executable(queryPerformance queryPerformance.c) #target_link_libraries(queryPerformance taos_static tutil common pthread) @@ -45,5 +45,8 @@ IF (TD_LINUX) #add_executable(invalidTableId invalidTableId.c) #target_link_libraries(invalidTableId taos_static tutil common pthread) + + add_executable(hashIterator hashIterator.c) + target_link_libraries(hashIterator taos_static tutil common pthread) ENDIF() diff --git a/tests/test/c/hashIterator.c b/tests/test/c/hashIterator.c new file mode 100644 index 0000000000..cbd8a0895e --- /dev/null +++ b/tests/test/c/hashIterator.c @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "os.h" +#include "taos.h" +#include "tulog.h" +#include "tutil.h" +#include "hash.h" + +typedef struct HashTestRow { + int32_t keySize; + char key[100]; +} HashTestRow; + +int main(int argc, char *argv[]) { + _hash_fn_t hashFp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + void * hashHandle = taosHashInit(100, hashFp, true, HASH_ENTRY_LOCK); + + pPrint("insert 3 rows to hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow row = {0}; + row.keySize = sprintf(row.key, "0.db.st%d", t); + + taosHashPut(hashHandle, row.key, row.keySize, &row, sizeof(HashTestRow)); + } + + pPrint("start iterator"); + HashTestRow *row = taosHashIterate(hashHandle, NULL); + while (row) { + pPrint("drop key:%s", row->key); + taosHashRemove(hashHandle, row->key, row->keySize); + + pPrint("get rows from hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow r = {0}; + r.keySize = sprintf(r.key, "0.db.st%d", t); + + void *result = taosHashGet(hashHandle, r.key, r.keySize); + pPrint("get key:%s result:%p", r.key, result); + } + + //Before getting the next iterator, the object just deleted can be obtained + row = taosHashIterate(hashHandle, row); + } + + pPrint("stop iterator"); + taosHashCancelIterate(hashHandle, row); + + pPrint("get rows from hash"); + for (int32_t t = 0; t < 3; ++t) { + HashTestRow r = {0}; + r.keySize = sprintf(r.key, "0.db.st%d", t); + + void *result = taosHashGet(hashHandle, r.key, r.keySize); + pPrint("get key:%s result:%p", r.key, result); + } + + return 0; +} \ No newline at end of file From 4b09f05290596014d987bcdc8c34d2004814618e Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 27 Nov 2020 07:23:16 +0000 Subject: [PATCH 39/41] add a removed flag in pNode --- src/connector/go | 2 +- src/connector/grafanaplugin | 2 +- src/rpc/src/rpcTcp.c | 15 +++++++++------ src/util/inc/hash.h | 1 + src/util/src/hash.c | 19 +++++++++++++++---- 5 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/connector/go b/src/connector/go index 050667e5b4..8c58c512b6 160000 --- a/src/connector/go +++ b/src/connector/go @@ -1 +1 @@ -Subproject commit 050667e5b4d0eafa5387e4283e713559b421203f +Subproject commit 8c58c512b6acda8bcdfa48fdc7140227b5221766 diff --git a/src/connector/grafanaplugin b/src/connector/grafanaplugin index ec77d9049a..d598db167e 160000 --- a/src/connector/grafanaplugin +++ b/src/connector/grafanaplugin @@ -1 +1 @@ -Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944 +Subproject commit d598db167eb256fe67409b7bb3d0eb7fffc3ff8c diff --git a/src/rpc/src/rpcTcp.c b/src/rpc/src/rpcTcp.c index 7b8cf3cda2..2850046d05 100644 --- a/src/rpc/src/rpcTcp.c +++ b/src/rpc/src/rpcTcp.c @@ -371,10 +371,13 @@ void taosCloseTcpConnection(void *chandle) { int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle) { SFdObj *pFdObj = chandle; - if (pFdObj == NULL || pFdObj->signature != pFdObj) return -1; + SThreadObj *pThreadObj = pFdObj->pThreadObj; - return taosWriteMsg(pFdObj->fd, data, len); + int ret = taosWriteMsg(pFdObj->fd, data, len); + tTrace("%s %p TCP data is sent, FD:%p fd:%d bytes:%d", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, ret); + + return ret; } static void taosReportBrokenLink(SFdObj *pFdObj) { @@ -409,7 +412,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { headLen = taosReadMsg(pFdObj->fd, &rpcHead, sizeof(SRpcHead)); if (headLen != sizeof(SRpcHead)) { - tDebug("%s %p read error, headLen:%d", pThreadObj->label, pFdObj->thandle, headLen); + tDebug("%s %p read error, FD:%p headLen:%d", pThreadObj->label, pFdObj->thandle, pFdObj, headLen); return -1; } @@ -420,7 +423,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("TCP malloc mem:%p size:%d", buffer, size); + tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer); } msg = buffer + tsRpcOverhead; @@ -583,8 +586,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p numOfFds:%d", - pThreadObj->label, pFdObj->thandle, pFdObj, pThreadObj->numOfFds); + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", + pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); } diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index aaff48f4f8..572f1a50c6 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -36,6 +36,7 @@ typedef struct SHashNode { uint32_t keyLen; // length of the key uint32_t dataLen; // length of data int8_t count; // reference count + int8_t removed; // flag to indicate removed char data[]; } SHashNode; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index cd9a9f7d0a..3ac54eedd8 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -76,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -222,7 +222,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) { assert(pNode->hashVal == hashVal); break; } @@ -361,7 +361,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe SHashNode *prevNode = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0) && pNode->removed == 0) break; prevNode = pNode; @@ -372,6 +372,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe code = 0; // it is found pNode->count--; + pNode->removed = 1; if (pNode->count <= 0) { if (prevNode) { prevNode->next = pNode->next; @@ -702,8 +703,13 @@ static void *taosHashReleaseNode(SHashObj *pHashObj, void *p, int *slot) { pNode = pNode->next; } - if (pNode) { + if (pNode) { pNode = pNode->next; + while (pNode) { + if (pNode->removed == 0) break; + pNode = pNode->next; + } + pOld->count--; if (pOld->count <=0) { if (prevNode) { @@ -755,6 +761,11 @@ void *taosHashIterate(SHashObj *pHashObj, void *p) { } pNode = pe->next; + while (pNode) { + if (pNode->removed == 0) break; + pNode = pNode->next; + } + if (pNode) break; if (pHashObj->type == HASH_ENTRY_LOCK) { From 257af62edfc99dca2198eb4dc3fbb00c36d9b043 Mon Sep 17 00:00:00 2001 From: Hui Li Date: Fri, 27 Nov 2020 15:29:12 +0800 Subject: [PATCH 40/41] [TD-2258] taos_fetch_block() resutl printf error --- tests/examples/c/apitest.c | 12 ++++++------ tests/examples/c/subscribe.c | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index be60a88ad7..e08d667d6b 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -79,11 +79,11 @@ static int print_result(TAOS_RES* res, int blockFetch) { if (blockFetch) { int rows = 0; while ((rows = taos_fetch_block(res, &row))) { - for (int i = 0; i < rows; i++) { - char temp[256]; - taos_print_row(temp, row + i, fields, num_fields); - puts(temp); - } + //for (int i = 0; i < rows; i++) { + // char temp[256]; + // taos_print_row(temp, row + i, fields, num_fields); + // puts(temp); + //} nRows += rows; } } else { @@ -498,4 +498,4 @@ int main(int argc, char *argv[]) { taos_close(taos); taos_cleanup(); -} \ No newline at end of file +} diff --git a/tests/examples/c/subscribe.c b/tests/examples/c/subscribe.c index 8368af18f7..cdd8ddaf7f 100644 --- a/tests/examples/c/subscribe.c +++ b/tests/examples/c/subscribe.c @@ -19,10 +19,10 @@ void print_result(TAOS_RES* res, int blockFetch) { if (blockFetch) { nRows = taos_fetch_block(res, &row); - for (int i = 0; i < nRows; i++) { - taos_print_row(buf, row + i, fields, num_fields); - puts(buf); - } + //for (int i = 0; i < nRows; i++) { + // taos_print_row(buf, row + i, fields, num_fields); + // puts(buf); + //} } else { while ((row = taos_fetch_row(res))) { taos_print_row(buf, row, fields, num_fields); From 15691bfb088fd114f3e53c769c2829937e746d63 Mon Sep 17 00:00:00 2001 From: Jeff Tao Date: Fri, 27 Nov 2020 07:41:35 +0000 Subject: [PATCH 41/41] data type compatible with windows --- src/util/inc/hash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 572f1a50c6..b0319d3e13 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -34,7 +34,7 @@ typedef struct SHashNode { struct SHashNode *next; uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key - uint32_t dataLen; // length of data + size_t dataLen; // length of data int8_t count; // reference count int8_t removed; // flag to indicate removed char data[];