xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #10536 from taosdata/feature/3.0_liaohj 

Feature/3.0 liaohj
This commit is contained in:
Haojun Liao 2022-03-04 10:28:28 +08:00 committed by GitHub
parent 448492a70d 4bfece6ee1
commit 0900a492d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 2103 additions and 1127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/test/craft/CMakeLists.txt
View File

@ -1,2 +1,2 @@
add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode craft lz4 uv_a)
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)

2
contrib/test/traft/single_node/CMakeLists.txt
View File

@ -3,4 +3,4 @@ target_sources(singleNode
PRIVATE
"singleNode.c"
)
target_link_libraries(singleNode traft lz4 uv_a)
target_link_libraries(singleNode PUBLIC traft lz4 uv_a)

1
include/os/os.h
View File

@ -44,6 +44,7 @@ extern "C" {
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/utsname.h>
#include <sys/param.h>
#include <unistd.h>
#include <wchar.h>
#include <wctype.h>

67
include/util/tcache.h
View File

@ -40,55 +40,9 @@ typedef struct SCacheStatis {
int64_t refreshCount;
} SCacheStatis;
struct STrashElem;
typedef struct SCacheDataNode {
uint64_t addedTime; // the added time when this element is added or updated into cache
uint64_t lifespan; // life duration when this element should be remove from cache
uint64_t expireTime; // expire time
uint64_t signature;
struct STrashElem *pTNodeHeader; // point to trash node head
uint16_t keySize : 15; // max key size: 32kb
bool inTrashcan : 1; // denote if it is in trash or not
uint32_t size; // allocated size for current SCacheDataNode
T_REF_DECLARE()
char *key;
char data[];
} SCacheDataNode;
typedef struct STrashElem {
struct STrashElem *prev;
struct STrashElem *next;
SCacheDataNode *pData;
} STrashElem;
/*
* to accommodate the old data which has the same key value of new one in hashList
* when an new node is put into cache, if an existed one with the same key:
* 1. if the old one does not be referenced, update it.
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
typedef struct {
int64_t totalSize; // total allocated buffer in this hash table, SCacheObj is not included.
int64_t refreshTime;
STrashElem *pTrash;
char *name;
SCacheStatis statistics;
SHashObj *pHashTable;
__cache_free_fn_t freeFp;
uint32_t numOfElemsInTrash; // number of element in trash
uint8_t deleting; // set the deleting flag to stop refreshing ASAP.
pthread_t refreshWorker;
bool extendLifespan; // auto extend life span when one item is accessed.
int64_t checkTick; // tick used to record the check times of the refresh threads
#if defined(LINUX)
pthread_rwlock_t lock;
#else
pthread_mutex_t lock;
#endif
} SCacheObj;
typedef struct SCacheObj SCacheObj;
typedef struct SCacheIter SCacheIter;
typedef struct STrashElem STrashElem;
/**
* initialize the cache object
@ -141,7 +95,7 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data);
* @param data
* @return
*/
void *taosCacheTransfer(SCacheObj *pCacheObj, void **data);
void *taosCacheTransferData(SCacheObj *pCacheObj, void **data);
/**
* remove data in cache, the data will not be removed immediately.
@ -152,6 +106,13 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data);
*/
void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove);
/**
*
* @param pCacheObj
* @return
*/
size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj);
/**
* move all data node into trash, clear node in trash can if it is not referenced by any clients
* @param handle
@ -184,6 +145,12 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1);
*/
void taosStopCacheRefreshWorker();
SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj);
bool taosCacheIterNext(SCacheIter* pIter);
void* taosCacheIterGetData(const SCacheIter* pIter, size_t* dataLen);
void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* keyLen);
void taosCacheDestroyIter(SCacheIter* pIter);
#ifdef __cplusplus
}
#endif

92
include/util/thash.h
View File

@ -28,11 +28,6 @@ typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_before_fn_t)(void *);
typedef void (*_hash_free_fn_t)(void *);
#define HASH_MAX_CAPACITY (1024 * 1024 * 16)
#define HASH_DEFAULT_LOAD_FACTOR (0.75)
#define HASH_INDEX(v, c) ((v) & ((c)-1))
#define HASH_NODE_EXIST(code) (code == -2)
/**
@ -62,41 +57,17 @@ typedef struct SHashNode {
uint32_t hashVal; // the hash value of key
uint32_t dataLen; // length of data
uint32_t keyLen; // length of the key
uint16_t count; // reference count
uint16_t refCount; // reference count
int8_t removed; // flag to indicate removed
char data[];
} SHashNode;
#define GET_HASH_NODE_KEY(_n) ((char *)(_n) + sizeof(SHashNode) + (_n)->dataLen)
#define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode))
#define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode)))
typedef enum SHashLockTypeE {
HASH_NO_LOCK = 0,
HASH_ENTRY_LOCK = 1,
} SHashLockTypeE;
typedef struct SHashEntry {
int32_t num; // number of elements in current entry
SRWLatch latch; // entry latch
SHashNode *next;
} SHashEntry;
typedef struct SHashObj {
SHashEntry **hashList;
uint32_t capacity; // number of slots
uint32_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function
_hash_free_fn_t freeFp; // hash node free callback function
_equal_fn_t equalFp; // equal function
_hash_before_fn_t callbackFp; // function invoked before return the value to caller
SRWLatch lock; // read-write spin lock
SHashLockTypeE type; // lock type
bool enableUpdate; // enable update
SArray *pMemBlock; // memory block allocated for SHashEntry
} SHashObj;
typedef struct SHashObj SHashObj;
/**
* init the hash table
@ -126,8 +97,6 @@ int32_t taosHashGetSize(const SHashObj *pHashObj);
*/
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size);
int32_t taosHashPutExt(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size, bool *newAdded);
/**
* return the payload data with the specified key
*
@ -146,17 +115,18 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
* @param destBuf
* @return
*/
void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf);
int32_t taosHashGetDup(SHashObj *pHashObj, const void *key, size_t keyLen, void *destBuf);
/**
* Clone the result to interval allocated buffer
*
* @param pHashObj
* @param key
* @param keyLen
* @param destBuf
* @param size
* @return
*/
void *taosHashGetCloneExt(SHashObj *pHashObj, const void *key, size_t keyLen, void (*fp)(void *), void **d, size_t *sz);
int32_t taosHashGetDup_m(SHashObj* pHashObj, const void* key, size_t keyLen, void** destBuf, int32_t* size);
/**
* remove item with the specified key
@ -207,37 +177,13 @@ void *taosHashIterate(SHashObj *pHashObj, void *p);
*/
void taosHashCancelIterate(SHashObj *pHashObj, void *p);
/**
* Get the corresponding key information for a given data in hash table
* @param data
* @return
*/
int32_t taosHashGetKey(void *data, void **key, size_t *keyLen);
/**
* Get the corresponding key information for a given data in hash table, using memcpy
* @param data
* @param dst
* @return
*/
static FORCE_INLINE int32_t taosHashCopyKey(void *data, void *dst) {
if (NULL == data || NULL == dst) {
return -1;
}
SHashNode *node = GET_HASH_PNODE(data);
void *key = GET_HASH_NODE_KEY(node);
memcpy(dst, key, node->keyLen);
return 0;
}
/**
* Get the corresponding data length for a given data in hash table
* @param data
* @return
*/
int32_t taosHashGetDataLen(void *data);
/**
* Get the corresponding key information for a given data in hash table
* @param data
* @param keyLen
* @return
*/
void *taosHashGetKey(void *data, size_t* keyLen);
/**
* return the payload data with the specified key(reference number added)
@ -258,8 +204,20 @@ void *taosHashAcquire(SHashObj *pHashObj, const void *key, size_t keyLen);
*/
void taosHashRelease(SHashObj *pHashObj, void *p);
/**
*
* @param pHashObj
* @param fp
*/
void taosHashSetEqualFp(SHashObj *pHashObj, _equal_fn_t fp);
/**
*
* @param pHashObj
* @param fp
*/
void taosHashSetFreeFp(SHashObj *pHashObj, _hash_free_fn_t fp);
#ifdef __cplusplus
}
#endif

2
include/util/tpagedbuf.h
View File

@ -53,7 +53,7 @@ typedef struct SDiskbasedBufStatis {
* @param handle
* @return
*/
int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id, const char* dir);
/**
*

3
source/client/src/clientHb.c
View File

@ -473,7 +473,8 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) {
free(pAppHbMgr);
return NULL;
}
pAppHbMgr->activeInfo->freeFp = tFreeClientHbReq;
taosHashSetFreeFp(pAppHbMgr->activeInfo, tFreeClientHbReq);
// init getInfoFunc
pAppHbMgr->connInfo = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);

2
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -85,7 +85,7 @@ static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) {
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
taosHashGetClone(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
taosHashGetDup(pMgmt->hash, &vgId, sizeof(int32_t), (void *)&pVnode);
if (pVnode == NULL) {
terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
} else {

65
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -48,7 +48,7 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
static void mndFreeConn(SConnObj *pConn);
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId);
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn);
static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn);
static void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter);
static void mndCancelGetNextConn(SMnode *pMnode, void *pIter);
static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq);
static int32_t mndProcessConnectReq(SMnodeMsg *pReq);
@ -158,27 +158,23 @@ static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) {
taosCacheRelease(pMgmt->cache, (void **)&pConn, false);
}
static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
*pConn = NULL;
pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter);
if (pIter == NULL) return NULL;
SCacheDataNode **pNode = pIter;
if (pNode == NULL || *pNode == NULL) {
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
return NULL;
void *mndGetNextConn(SMnode *pMnode, SCacheIter *pIter) {
SConnObj* pConn = NULL;
bool hasNext = taosCacheIterNext(pIter);
if (hasNext) {
size_t dataLen = 0;
pConn = taosCacheIterGetData(pIter, &dataLen);
} else {
taosCacheDestroyIter(pIter);
}
*pConn = (SConnObj *)((*pNode)->data);
return pIter;
return pConn;
}
static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
if (pIter != NULL) {
taosCacheDestroyIter(pIter);
}
}
static int32_t mndProcessConnectReq(SMnodeMsg *pReq) {
@ -376,8 +372,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
int32_t rspLen = 0;
mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv);
SKv kv1 = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv1);
}
break;
}
@ -386,8 +382,8 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) {
int32_t rspLen = 0;
mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv);
SKv kv1 = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv1);
}
break;
}
@ -638,7 +634,7 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *
pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
}
pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable);
pShow->numOfRows = taosCacheGetNumOfObj(pMgmt->cache);
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbName, mndShowStr(pShow->type));
@ -653,8 +649,13 @@ static int32_t mndRetrieveConns(SMnodeMsg *pReq, SShowObj *pShow, char *data, in
char *pWrite;
char ipStr[TSDB_IPv4ADDR_LEN + 6];
if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->cache);
}
while (numOfRows < rows) {
pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn);
pConn = mndGetNextConn(pMnode, pShow->pIter);
if (pConn == NULL) break;
cols = 0;
@ -823,19 +824,24 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
void *pIter;
char str[TSDB_IPv4ADDR_LEN + 6] = {0};
if (pShow->pIter == NULL) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
pShow->pIter = taosCacheCreateIter(pMgmt->cache);
}
while (numOfRows < rows) {
pIter = mndGetNextConn(pMnode, pShow->pIter, &pConn);
pConn = mndGetNextConn(pMnode, pShow->pIter);
if (pConn == NULL) {
pShow->pIter = pIter;
pShow->pIter = NULL;
break;
}
if (numOfRows + pConn->numOfQueries >= rows) {
mndCancelGetNextConn(pMnode, pIter);
taosCacheDestroyIter(pShow->pIter);
pShow->pIter = NULL;
break;
}
pShow->pIter = pIter;
for (int32_t i = 0; i < pConn->numOfQueries; ++i) {
SQueryDesc *pDesc = pConn->pQueries + i;
cols = 0;
@ -913,6 +919,7 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pReq, SShowObj *pShow, char *data,
}
static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) {
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
taosHashCancelIterate(pMgmt->cache->pHashTable, pIter);
if (pIter != NULL) {
taosCacheDestroyIter(pIter);
}
}

60
source/libs/catalog/src/catalog.c
View File

@ -168,7 +168,7 @@ void ctgDbgShowTableMeta(SCatalog* pCtg, const char *tbName, STableMeta* p) {
ctgDebug("table [%s] meta: type:%d, vgId:%d, uid:%" PRIx64 ",suid:%" PRIx64 ",sv:%d, tv:%d, tagNum:%d, precision:%d, colNum:%d, rowSize:%d",
tbName, p->tableType, p->vgId, p->uid, p->suid, p->sversion, p->tversion, c->numOfTags, c->precision, c->numOfColumns, c->rowSize);
}
int32_t colNum = c->numOfColumns + c->numOfTags;
for (int32_t i = 0; i < colNum; ++i) {
SSchema *s = &p->schema[i];
@ -190,7 +190,7 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
dbCache = (SCtgDBCache *)pIter;
taosHashGetKey(dbCache, (void **)&dbFName, &len);
taosHashGetKey((void **)&dbFName, &len);
int32_t metaNum = dbCache->tbCache.metaCache ? taosHashGetSize(dbCache->tbCache.metaCache) : 0;
int32_t stbNum = dbCache->tbCache.stbCache ? taosHashGetSize(dbCache->tbCache.stbCache) : 0;
@ -204,9 +204,9 @@ void ctgDbgShowDBCache(SCatalog* pCtg, SHashObj *dbHash) {
if (dbCache->vgInfo->vgHash) {
vgNum = taosHashGetSize(dbCache->vgInfo->vgHash);
}
}
}
ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
ctgDebug("[%d] db [%.*s][%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d",
i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum);
pIter = taosHashIterate(dbHash, pIter);
@ -222,7 +222,7 @@ void ctgDbgShowClusterCache(SCatalog* pCtg) {
}
ctgDebug("## cluster %"PRIx64" %p cache Info ##", pCtg->clusterId, pCtg);
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
ctgDebug("db:%d meta:%d stb:%d dbRent:%d stbRent:%d", ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_META_NUM),
ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM));
ctgDbgShowDBCache(pCtg, pCtg->dbCache);
@ -306,9 +306,9 @@ int32_t ctgPushRmDBMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId)
return TSDB_CODE_SUCCESS;
_return:
tfree(action.data);
CTG_RET(code);
CTG_RET(code);
}
@ -336,9 +336,9 @@ int32_t ctgPushRmStbMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId
return TSDB_CODE_SUCCESS;
_return:
tfree(action.data);
CTG_RET(code);
CTG_RET(code);
}
@ -366,9 +366,9 @@ int32_t ctgPushRmTblMsgInQueue(SCatalog* pCtg, const char *dbFName, int64_t dbId
return TSDB_CODE_SUCCESS;
_return:
tfree(action.data);
CTG_RET(code);
CTG_RET(code);
}
@ -657,9 +657,9 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
return TSDB_CODE_SUCCESS;
}
size_t sz = 0;
int32_t sz = 0;
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
STableMeta *tbMeta = taosHashGetCloneExt(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), NULL, (void **)pTableMeta, &sz);
int32_t code = taosHashGetDup_m(dbCache->tbCache.metaCache, pTableName->tname, strlen(pTableName->tname), (void **)pTableMeta, &sz);
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (NULL == *pTableMeta) {
@ -673,8 +673,8 @@ int32_t ctgGetTableMetaFromCache(SCatalog* pCtg, const SName* pTableName, STable
if (dbId) {
*dbId = dbCache->dbId;
}
tbMeta = *pTableMeta;
STableMeta* tbMeta = *pTableMeta;
if (tbMeta->tableType != TSDB_CHILD_TABLE) {
ctgReleaseDBCache(pCtg, dbCache);
@ -1076,7 +1076,7 @@ _return:
}
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t compare) {
int16_t widx = abs(id % mgmt->slotNum);
int16_t widx = labs(id % mgmt->slotNum);
SCtgRentSlot *slot = &mgmt->slots[widx];
int32_t code = 0;
@ -1238,7 +1238,7 @@ void ctgRemoveStbRent(SCatalog* pCtg, SCtgTbMetaCache *cache) {
void *pIter = taosHashIterate(cache->stbCache, NULL);
while (pIter) {
uint64_t *suid = NULL;
taosHashGetKey(pIter, (void **)&suid, NULL);
suid = taosHashGetKey(pIter, NULL);
if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionCompare)) {
ctgDebug("stb removed from rent, suid:%"PRIx64, *suid);
@ -1397,7 +1397,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:%"PRIx64, dbFName, tbName, orig->suid);
ctgMetaRentRemove(&pCtg->stbRent, orig->suid, ctgStbVersionCompare);
}
@ -1437,7 +1437,7 @@ int32_t ctgUpdateTblMeta(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, ui
if (taosHashPut(tbCache->stbCache, &meta->suid, sizeof(meta->suid), &tbMeta, POINTER_BYTES) != 0) {
CTG_UNLOCK(CTG_WRITE, &tbCache->stbLock);
CTG_UNLOCK(CTG_READ, &tbCache->metaLock);
ctgError("taosHashPutExt stable to stable cache failed, suid:%"PRIx64, meta->suid);
ctgError("taosHashPut stable to stable cache failed, suid:%"PRIx64, meta->suid);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
@ -1475,7 +1475,7 @@ int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst) {
int32_t *vgId = NULL;
void *pIter = taosHashIterate(src->vgHash, NULL);
while (pIter) {
taosHashGetKey(pIter, (void **)&vgId, NULL);
vgId = taosHashGetKey(pIter, NULL);
if (taosHashPut((*dst)->vgHash, (void *)vgId, sizeof(int32_t), pIter, sizeof(SVgroupInfo))) {
qError("taosHashPut failed, hashSize:%d", (int32_t)hashSize);
@ -1635,7 +1635,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps,
if (!CTG_FLAG_IS_FORCE_UPDATE(flag)) {
CTG_ERR_JRET(ctgIsTableMetaExistInCache(pCtg, output->dbFName, output->tbName, &exist));
}
if (0 == exist) {
CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCtg, pTrans, pMgmtEps, output->dbFName, output->tbName, &moutput));
@ -1723,9 +1723,9 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
tbType = (*pTableMeta)->tableType;
suid = (*pTableMeta)->suid;
tfree(*pTableMeta);
tfree(*pTableMeta);
}
if (CTG_FLAG_IS_UNKNOWN_STB(flag)) {
CTG_FLAG_SET_STB(flag, tbType);
}
@ -1950,21 +1950,21 @@ int32_t ctgActRemoveTbl(SCtgMetaAction *action) {
ctgDebug("dbId already modified, dbFName:%s, current:%"PRIx64", dbId:%"PRIx64", tbName:%s", msg->dbFName, dbCache->dbId, msg->dbId, msg->tbName);
return TSDB_CODE_SUCCESS;
}
CTG_LOCK(CTG_READ, &dbCache->tbCache.metaLock);
if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) {
if (taosHashRemove(dbCache->tbCache.metaCache, msg->tbName, strlen(msg->tbName))) {
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgError("stb not exist in cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
}
CTG_UNLOCK(CTG_READ, &dbCache->tbCache.metaLock);
ctgInfo("table removed from cache, dbFName:%s, tbName:%s", msg->dbFName, msg->tbName);
_return:
tfree(msg);
CTG_RET(code);
}
@ -2458,7 +2458,7 @@ int32_t catalogGetTableDistVgInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgm
CTG_ERR_JRET(ctgGenerateVgList(pCtg, vgHash, pVgList));
} else {
int32_t vgId = tbMeta->vgId;
if (NULL == taosHashGetClone(vgHash, &vgId, sizeof(vgId), &vgroupInfo)) {
if (taosHashGetDup(vgHash, &vgId, sizeof(vgId), &vgroupInfo) != 0) {
ctgError("table's vgId not found in vgroup list, vgId:%d, tbName:%s", vgId, tNameGetTableName(pTableName));
CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR);
}

2
source/libs/executor/inc/executil.h
View File

@ -68,7 +68,7 @@ typedef struct SResultRow {
} SResultRow;
typedef struct SResultRowInfo {
SResultRow *pCurResult; // current active result row info
SList* pRows;
SResultRow** pResult; // result list
// int16_t type:8; // data type for hash key
int32_t size; // number of result set

41
source/libs/executor/inc/executorimpl.h
View File

@ -233,9 +233,9 @@ typedef struct STaskAttr {
SArray* pUdfInfo; // no need to free
} STaskAttr;
typedef int32_t (*__optr_prepare_fn_t)(void* param);
typedef SSDataBlock* (*__operator_fn_t)(void* param, bool* newgroup);
typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num);
typedef int32_t (*__optr_open_fn_t)(void* param);
typedef SSDataBlock* (*__optr_fn_t)(void* param, bool* newgroup);
typedef void (*__optr_close_fn_t)(void* param, int32_t num);
struct SOperatorInfo;
@ -306,21 +306,21 @@ enum {
};
typedef struct SOperatorInfo {
uint8_t operatorType;
bool blockingOptr; // block operator or not
uint8_t status; // denote if current operator is completed
int32_t numOfOutput; // number of columns of the current operator results
char* name; // name, used to show the query execution plan
void* info; // extension attribution
SExprInfo* pExpr;
STaskRuntimeEnv* pRuntimeEnv; // todo remove it
SExecTaskInfo* pTaskInfo;
uint8_t operatorType;
bool blockingOptr; // block operator or not
uint8_t status; // denote if current operator is completed
int32_t numOfOutput; // number of columns of the current operator results
char* name; // name, used to show the query execution plan
void* info; // extension attribution
SExprInfo* pExpr;
STaskRuntimeEnv* pRuntimeEnv; // todo remove it
SExecTaskInfo* pTaskInfo;
struct SOperatorInfo** pDownstream; // downstram pointer list
int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator
__optr_prepare_fn_t prepareFn;
__operator_fn_t exec;
__optr_cleanup_fn_t cleanupFn;
__optr_open_fn_t openFn;
__optr_fn_t nextDataFn;
__optr_close_fn_t closeFn;
} SOperatorInfo;
typedef struct {
@ -479,9 +479,6 @@ typedef struct SAggOperatorInfo {
typedef struct SProjectOperatorInfo {
SOptrBasicInfo binfo;
int32_t bufCapacity;
uint32_t seed;
SSDataBlock* existDataBlock;
} SProjectOperatorInfo;
@ -615,10 +612,10 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo);
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr,
int32_t numOfOutput);
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
SExprInfo* pExpr, int32_t numOfOutput);
@ -654,8 +651,6 @@ SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOf
SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t numOfDownstream, SArray* pExprInfo, SArray* pOrderVal, SArray* pGroupInfo, SExecTaskInfo* pTaskInfo);
// SSDataBlock* doGlobalAggregate(void* param, bool* newgroup);
// SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup);
// SSDataBlock* doSLimit(void* param, bool* newgroup);
// int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId);

104
source/libs/executor/inc/tsimplehash.h Normal file
View File

@ -0,0 +1,104 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSIMPLEHASH_H
#define TDENGINE_TSIMPLEHASH_H
#include "tarray.h"
#include "tlockfree.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef uint32_t (*_hash_fn_t)(const char *, uint32_t);
typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_free_fn_t)(void *);
typedef struct SSHashObj SSHashObj;
/**
* init the hash table
*
* @param capacity initial capacity of the hash table
* @param fn hash function to generate the hash value
* @return
*/
SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen);
/**
* return the size of hash table
* @param pHashObj
* @return
*/
int32_t tSimpleHashGetSize(const SSHashObj *pHashObj);
/**
* put element into hash table, if the element with the same key exists, update it
* @param pHashObj
* @param key
* @param data
* @return
*/
int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data);
/**
* return the payload data with the specified key
*
* @param pHashObj
* @param key
* @return
*/
void *tSimpleHashGet(SSHashObj *pHashObj, const void *key);
/**
* remove item with the specified key
* @param pHashObj
* @param key
* @param keyLen
*/
int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key);
/**
* Clear the hash table.
* @param pHashObj
*/
void tSimpleHashClear(SSHashObj *pHashObj);
/**
* Clean up hash table and release all allocated resources.
* @param handle
*/
void tSimpleHashCleanup(SSHashObj *pHashObj);
/**
* Get the hash table size
* @param pHashObj
* @return
*/
size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj);
/**
* Get the corresponding key information for a given data in hash table
* @param data
* @param keyLen
* @return
*/
void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSIMPLEHASH_H

2
source/libs/executor/src/executorMain.c
View File

@ -158,7 +158,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes, uint64_t *useconds) {
int64_t st = 0;
st = taosGetTimestampUs();
*pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup);
*pRes = pTaskInfo->pRoot->nextDataFn(pTaskInfo->pRoot, &newgroup);
uint64_t el = (taosGetTimestampUs() - st);
pTaskInfo->cost.elapsedTime += el;

157
source/libs/executor/src/executorimpl.c
View File

@ -4619,7 +4619,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr
getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize);
int32_t TENMB = 1024*1024*10;
int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, "/tmp");
int32_t code = createDiskbasedBuf(&pRuntimeEnv->pResultBuf, ps, TENMB, "", "/tmp");
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -5281,7 +5281,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray*
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = size;
pOperator->exec = doLoadRemoteData;
pOperator->nextDataFn = doLoadRemoteData;
pOperator->pTaskInfo = pTaskInfo;
#if 1
@ -5361,7 +5361,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->exec = doTableScan;
pOperator->nextDataFn = doTableScan;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
@ -5386,7 +5386,7 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntim
pOperator->info = pInfo;
pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doTableScanImpl;
pOperator->nextDataFn = doTableScanImpl;
return pOperator;
}
@ -5410,7 +5410,7 @@ SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRunt
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
// pOperator->numOfOutput = pRuntimeEnv->pQueryAttr->numOfCols;
pOperator->exec = doBlockInfoScan;
pOperator->nextDataFn = doBlockInfoScan;
return pOperator;
}
@ -5452,7 +5452,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void *streamReadHandle, SArray* pExp
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->exec = doStreamBlockScan;
pOperator->nextDataFn = doStreamBlockScan;
pOperator->pTaskInfo = pTaskInfo;
return pOperator;
}
@ -5663,7 +5663,7 @@ SSDataBlock* loadNextDataBlock(void* param) {
SOperatorInfo* pOperator = (SOperatorInfo*) param;
bool newgroup = false;
return pOperator->exec(pOperator, &newgroup);
return pOperator->nextDataFn(pOperator, &newgroup);
}
static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char **buf, int32_t rowIndex) {
@ -5983,8 +5983,8 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t
pOperator->pExpr = exprArrayDup(pExprInfo);
pOperator->pTaskInfo = pTaskInfo;
pOperator->exec = doSortedMerge;
pOperator->cleanupFn = destroySortedMergeOperatorInfo;
pOperator->nextDataFn = doSortedMerge;
pOperator->closeFn = destroySortedMergeOperatorInfo;
code = appendDownstream(pOperator, downstream, numOfDownstream);
if (code != TSDB_CODE_SUCCESS) {
@ -6079,8 +6079,8 @@ SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprI
pOperator->info = pInfo;
pOperator->pTaskInfo = pTaskInfo;
pOperator->exec = doSort;
pOperator->cleanupFn = destroyOrderOperatorInfo;
pOperator->nextDataFn = doSort;
pOperator->closeFn = destroyOrderOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -6105,7 +6105,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6155,7 +6155,7 @@ static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6224,7 +6224,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows);
updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows);
projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);
@ -6241,7 +6241,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
// The downstream exec may change the value of the newgroup, so use a local variable instead.
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6274,7 +6274,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) {
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order);
updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows);
updateOutputBuf(pInfo, &pInfo->capacity, pBlock->info.rows);
projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput);
pRes->info.rows = getNumOfResult(pInfo->pCtx, pOperator->numOfOutput);
@ -6299,7 +6299,7 @@ static SSDataBlock* doLimit(void* param, bool* newgroup) {
SSDataBlock* pBlock = NULL;
while (1) {
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6350,7 +6350,7 @@ static SSDataBlock* doFilter(void* param, bool* newgroup) {
while (1) {
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock *pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
SSDataBlock *pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6393,7 +6393,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6453,7 +6453,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6516,7 +6516,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6571,7 +6571,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6706,7 +6706,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) {
SOperatorInfo* downstream = pOperator->pDownstream[0];
while (1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -6768,7 +6768,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
break;
@ -6821,7 +6821,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = downstream->exec(downstream, newgroup);
SSDataBlock* pBlock = downstream->nextDataFn(downstream, newgroup);
publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
break;
@ -6906,7 +6906,7 @@ static SSDataBlock* doFill(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
SSDataBlock* pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
SSDataBlock* pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (*newgroup) {
@ -6979,8 +6979,8 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) {
return;
}
if (pOperator->cleanupFn != NULL) {
pOperator->cleanupFn(pOperator->info, pOperator->numOfOutput);
if (pOperator->closeFn != NULL) {
pOperator->closeFn(pOperator->info, pOperator->numOfOutput);
}
if (pOperator->pDownstream != NULL) {
@ -7067,8 +7067,8 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pE
pOperator->numOfOutput = taosArrayGetSize(pExprInfo);
pOperator->pTaskInfo = pTaskInfo;
pOperator->exec = doAggregate;
pOperator->cleanupFn = destroyAggOperatorInfo;
pOperator->nextDataFn = doAggregate;
pOperator->closeFn = destroyAggOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7164,38 +7164,34 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray
pOperator->pExpr = exprArrayDup(pExprInfo);
pOperator->numOfOutput = numOfOutput;
pOperator->exec = doMultiTableAggregate;
pOperator->cleanupFn = destroyAggOperatorInfo;
pOperator->nextDataFn = doMultiTableAggregate;
pOperator->closeFn = destroyAggOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
}
SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) {
SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo));
pInfo->seed = rand();
pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity;
int32_t numOfRows = 4096;
pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows);
pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
SOptrBasicInfo* pBInfo = &pInfo->binfo;
pBInfo->pRes = createOutputBuf(pExpr, numOfOutput, pInfo->bufCapacity);
pBInfo->pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pBInfo->rowCellInfoOffset);
initResultRowInfo(&pBInfo->resultRowInfo, 8);
setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MAIN_SCAN);
// initResultRowInfo(&pBInfo->resultRowInfo, 8);
// setDefaultOutputBuf_rv(pBInfo, MAIN_SCAN);
SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo));
pOperator->name = "ProjectOperator";
// pOperator->operatorType = OP_Project;
pOperator->operatorType = OP_Project;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->pExpr = exprArrayDup(pExprInfo);
pOperator->numOfOutput = taosArrayGetSize(pExprInfo);
pOperator->exec = doProjectOperation;
pOperator->cleanupFn = destroyProjectOperatorInfo;
pOperator->nextDataFn = doProjectOperation;
pOperator->closeFn = destroyProjectOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7250,10 +7246,10 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
pOperator->status = OP_IN_EXECUTING;
pOperator->numOfOutput = numOfOutput;
pOperator->pExpr = pExpr;
pOperator->exec = doFilter;
pOperator->nextDataFn = doFilter;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanupFn = destroyConditionOperatorInfo;
pOperator->closeFn = destroyConditionOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7269,7 +7265,7 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
// pOperator->operatorType = OP_Limit;
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->exec = doLimit;
pOperator->nextDataFn = doLimit;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
int32_t code = appendDownstream(pOperator, &downstream, 1);
@ -7277,24 +7273,18 @@ SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorIn
return pOperator;
}
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SInterval* pInterval, SExecTaskInfo* pTaskInfo) {
STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
initAggSup(&pInfo->aggSup, pExprInfo);
// todo:
pInfo->order = TSDB_ORDER_ASC;
pInfo->precision = TSDB_TIME_PRECISION_MICRO;
pInfo->win.skey = INT64_MIN;
pInfo->win.ekey = INT64_MAX;
pInfo->interval.intervalUnit = 's';
pInfo->interval.slidingUnit = 's';
pInfo->interval.interval = 1000;
pInfo->interval.sliding = 1000;
pInfo->win = pTaskInfo->window;
pInfo->interval = *pInterval;
int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, 0, "/tmp/");
int32_t code = createDiskbasedBuf(&pInfo->pResultBuf, 4096, 4096 * 256, pTaskInfo->id.str, "/tmp/");
int32_t numOfOutput = taosArrayGetSize(pExprInfo);
pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize);
pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, pInfo->binfo.capacity);
@ -7309,16 +7299,15 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SArray* pEx
pOperator->pExpr = exprArrayDup(pExprInfo);
pOperator->pTaskInfo = pTaskInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->numOfOutput = taosArrayGetSize(pExprInfo);
pOperator->info = pInfo;
pOperator->exec = doIntervalAgg;
pOperator->cleanupFn = destroyBasicOperatorInfo;
pOperator->nextDataFn = doIntervalAgg;
pOperator->closeFn = destroyBasicOperatorInfo;
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
}
SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) {
STableIntervalOperatorInfo* pInfo = calloc(1, sizeof(STableIntervalOperatorInfo));
@ -7336,8 +7325,8 @@ SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doAllIntervalAgg;
pOperator->cleanupFn = destroyBasicOperatorInfo;
pOperator->nextDataFn = doAllIntervalAgg;
pOperator->closeFn = destroyBasicOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7360,8 +7349,8 @@ SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOper
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doStateWindowAgg;
pOperator->cleanupFn = destroyStateWindowOperatorInfo;
pOperator->nextDataFn = doStateWindowAgg;
pOperator->closeFn = destroyStateWindowOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7385,8 +7374,8 @@ SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doSessionWindowAgg;
pOperator->cleanupFn = destroySWindowOperatorInfo;
pOperator->nextDataFn = doSessionWindowAgg;
pOperator->closeFn = destroySWindowOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7409,8 +7398,8 @@ SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntim
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doSTableIntervalAgg;
pOperator->cleanupFn = destroyBasicOperatorInfo;
pOperator->nextDataFn = doSTableIntervalAgg;
pOperator->closeFn = destroyBasicOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7433,8 +7422,8 @@ SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRun
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doAllSTableIntervalAgg;
pOperator->cleanupFn = destroyBasicOperatorInfo;
pOperator->nextDataFn = doAllSTableIntervalAgg;
pOperator->closeFn = destroyBasicOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
@ -7465,8 +7454,8 @@ SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperator
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = hashGroupbyAggregate;
pOperator->cleanupFn = destroyGroupbyOperatorInfo;
pOperator->nextDataFn = hashGroupbyAggregate;
pOperator->closeFn = destroyGroupbyOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7504,8 +7493,8 @@ SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInf
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = doFill;
pOperator->cleanupFn = destroySFillOperatorInfo;
pOperator->nextDataFn = doFill;
pOperator->closeFn = destroySFillOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7553,7 +7542,7 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI
// pOperator->exec = doSLimit;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanupFn = destroySlimitOperatorInfo;
pOperator->closeFn = destroySlimitOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
@ -7707,11 +7696,11 @@ SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo
pOperator->blockingOptr = false;
pOperator->status = OP_IN_EXECUTING;
pOperator->info = pInfo;
pOperator->exec = doTagScan;
pOperator->nextDataFn = doTagScan;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->cleanupFn = destroyTagScanOperatorInfo;
pOperator->closeFn = destroyTagScanOperatorInfo;
return pOperator;
}
@ -7777,7 +7766,7 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) {
while(1) {
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC);
pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup);
pBlock = pOperator->pDownstream[0]->nextDataFn(pOperator->pDownstream[0], newgroup);
publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_AFTER_OPERATOR_EXEC);
if (pBlock == NULL) {
@ -7849,9 +7838,9 @@ SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperato
pOperator->numOfOutput = numOfOutput;
pOperator->info = pInfo;
pOperator->pRuntimeEnv = pRuntimeEnv;
pOperator->exec = hashDistinct;
pOperator->nextDataFn = hashDistinct;
pOperator->pExpr = pExpr;
pOperator->cleanupFn = destroyDistinctOperatorInfo;
pOperator->closeFn = destroyDistinctOperatorInfo;
int32_t code = appendDownstream(pOperator, &downstream, 1);
return pOperator;

7
source/libs/executor/src/tlinearhash.c
View File

@ -29,8 +29,8 @@ typedef struct SLHashBucket {
typedef struct SLHashObj {
SDiskbasedBuf *pBuf;
_hash_fn_t hashFn;
int32_t tuplesPerPage;
SLHashBucket **pBucket; // entry list
int32_t tuplesPerPage;
int32_t numOfAlloc; // number of allocated bucket ptr slot
int32_t bits; // the number of bits used in hash
int32_t numOfBuckets; // the number of buckets
@ -142,7 +142,7 @@ static void doRemoveFromBucket(SFilePage* pPage, SLHashNode* pNode, SLHashBucket
pBucket->size -= 1;
}
static void doCompressBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) {
static void doTrimBucketPages(SLHashObj *pHashObj, SLHashBucket* pBucket) {
size_t numOfPages = taosArrayGetSize(pBucket->pPageIdList);
if (numOfPages <= 1) {
return;
@ -253,6 +253,7 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_
return NULL;
}
// disable compress when flushing to disk
setBufPageCompressOnDisk(pHashObj->pBuf, false);
/**
@ -367,7 +368,7 @@ int32_t tHashPut(SLHashObj* pHashObj, const void *key, size_t keyLen, void *data
releaseBufPage(pHashObj->pBuf, p);
}
doCompressBucketPages(pHashObj, pBucket);
doTrimBucketPages(pHashObj, pBucket);
}
return TSDB_CODE_SUCCESS;

309
source/libs/executor/src/tsimplehash.c Normal file
View File

@ -0,0 +1,309 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tsimplehash.h"
#include "taoserror.h"
#define SHASH_DEFAULT_LOAD_FACTOR 0.75
#define HASH_MAX_CAPACITY (1024*1024*16)
#define SHASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR)
#define GET_SHASH_NODE_KEY(_n, _dl) ((char*)(_n) + sizeof(SHNode) + (_dl))
#define GET_SHASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHNode))
#define HASH_INDEX(v, c) ((v) & ((c)-1))
#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * SHASH_DEFAULT_LOAD_FACTOR)
#define FREE_HASH_NODE(_n) \
do { \
tfree(_n); \
} while (0);
typedef struct SHNode {
struct SHNode *next;
char data[];
} SHNode;
typedef struct SSHashObj {
SHNode **hashList;
size_t capacity; // number of slots
size_t size; // number of elements in hash table
_hash_fn_t hashFp; // hash function
_equal_fn_t equalFp; // equal function
int32_t keyLen;
int32_t dataLen;
} SSHashObj;
static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
int32_t len = MIN(length, HASH_MAX_CAPACITY);
int32_t i = 4;
while (i < len) i = (i << 1u);
return i;
}
SSHashObj *tSimpleHashInit(size_t capacity, _hash_fn_t fn, size_t keyLen, size_t dataLen) {
ASSERT(fn != NULL);
if (capacity == 0) {
capacity = 4;
}
SSHashObj* pHashObj = (SSHashObj*) calloc(1, sizeof(SSHashObj));
if (pHashObj == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
// the max slots is not defined by user
pHashObj->capacity = taosHashCapacity((int32_t)capacity);
pHashObj->equalFp = memcmp;
pHashObj->hashFp = fn;
ASSERT((pHashObj->capacity & (pHashObj->capacity - 1)) == 0);
pHashObj->keyLen = keyLen;
pHashObj->dataLen = dataLen;
pHashObj->hashList = (SHNode **)calloc(pHashObj->capacity, sizeof(void *));
if (pHashObj->hashList == NULL) {
free(pHashObj);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
return pHashObj;
}
int32_t tSimpleHashGetSize(const SSHashObj *pHashObj) {
if (pHashObj == NULL) {
return 0;
}
return (int32_t)atomic_load_64(&pHashObj->size);
}
static SHNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
SHNode *pNewNode = malloc(sizeof(SHNode) + keyLen + dsize);
if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pNewNode->next = NULL;
memcpy(GET_SHASH_NODE_DATA(pNewNode), pData, dsize);
memcpy(GET_SHASH_NODE_KEY(pNewNode, dsize), key, keyLen);
return pNewNode;
}
void taosHashTableResize(SSHashObj *pHashObj) {
if (!HASH_NEED_RESIZE(pHashObj)) {
return;
}
int32_t newCapacity = (int32_t)(pHashObj->capacity << 1u);
if (newCapacity > HASH_MAX_CAPACITY) {
// uDebug("current capacity:%zu, maximum capacity:%d, no resize applied due to limitation is reached",
// pHashObj->capacity, HASH_MAX_CAPACITY);
return;
}
int64_t st = taosGetTimestampUs();
void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newCapacity);
if (pNewEntryList == NULL) {
// qWarn("hash resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity);
return;
}
size_t inc = newCapacity - pHashObj->capacity;
memset(pNewEntryList + pHashObj->capacity * sizeof(void*), 0, inc);
pHashObj->hashList = pNewEntryList;
pHashObj->capacity = newCapacity;
for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) {
SHNode* pNode = pHashObj->hashList[idx];
SHNode *pNext;
SHNode *pPrev = NULL;
if (pNode == NULL) {
continue;
}
while (pNode != NULL) {
void* key = GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen);
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->dataLen);
int32_t newIdx = HASH_INDEX(hashVal, pHashObj->capacity);
pNext = pNode->next;
if (newIdx != idx) {
if (pPrev == NULL) {
pHashObj->hashList[idx] = pNext;
} else {
pPrev->next = pNext;
}
pNode->next = pHashObj->hashList[newIdx];
pHashObj->hashList[newIdx] = pNode;
} else {
pPrev = pNode;
}
pNode = pNext;
}
}
int64_t et = taosGetTimestampUs();
// uDebug("hash table resize completed, new capacity:%d, load factor:%f, elapsed time:%fms", (int32_t)pHashObj->capacity,
// ((double)pHashObj->size) / pHashObj->capacity, (et - st) / 1000.0);
}
int32_t tSimpleHashPut(SSHashObj *pHashObj, const void *key, const void *data) {
if (pHashObj == NULL || key == NULL) {
return -1;
}
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
// need the resize process, write lock applied
if (SHASH_NEED_RESIZE(pHashObj)) {
taosHashTableResize(pHashObj);
}
int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
SHNode *pNode = pHashObj->hashList[slot];
if (pNode == NULL) {
SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal);
if (pNewNode == NULL) {
return -1;
}
pHashObj->hashList[slot] = pNewNode;
return 0;
}
while (pNode) {
if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
break;
}
pNode = pNode->next;
}
if (pNode == NULL) {
SHNode *pNewNode = doCreateHashNode(key, pHashObj->keyLen, data, pHashObj->size, hashVal);
if (pNewNode == NULL) {
return -1;
}
pNewNode->next = pHashObj->hashList[slot];
pHashObj->hashList[slot] = pNewNode;
atomic_add_fetch_64(&pHashObj->size, 1);
} else { //update data
memcpy(GET_SHASH_NODE_DATA(pNode), data, pHashObj->dataLen);
}
return 0;
}
static FORCE_INLINE SHNode *doSearchInEntryList(SSHashObj *pHashObj, const void *key, int32_t index) {
SHNode *pNode = pHashObj->hashList[index];
while (pNode) {
if ((*(pHashObj->equalFp))(GET_SHASH_NODE_KEY(pNode, pHashObj->dataLen), key, pHashObj->keyLen) == 0) {
break;
}
pNode = pNode->next;
}
return pNode;
}
static FORCE_INLINE bool taosHashTableEmpty(const SSHashObj *pHashObj) {
return tSimpleHashGetSize(pHashObj) == 0;
}
void *tSimpleHashGet(SSHashObj *pHashObj, const void *key) {
if (pHashObj == NULL || taosHashTableEmpty(pHashObj) || key == NULL) {
return NULL;
}
uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)pHashObj->keyLen);
int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
SHNode *pNode = pHashObj->hashList[slot];
if (pNode == NULL) {
return NULL;
}
char *data = NULL;
pNode = doSearchInEntryList(pHashObj, key, slot);
if (pNode != NULL) {
data = GET_SHASH_NODE_DATA(pNode);
}
return data;
}
int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key) {
// todo
}
void tSimpleHashClear(SSHashObj *pHashObj) {
if (pHashObj == NULL) {
return;
}
SHNode *pNode, *pNext;
for (int32_t i = 0; i < pHashObj->capacity; ++i) {
pNode = pHashObj->hashList[i];
if (pNode == NULL) {
continue;
}
while (pNode) {
pNext = pNode->next;
FREE_HASH_NODE(pNode);
pNode = pNext;
}
}
pHashObj->size = 0;
}
void tSimpleHashCleanup(SSHashObj *pHashObj) {
if (pHashObj == NULL) {
return;
}
tSimpleHashClear(pHashObj);
tfree(pHashObj->hashList);
}
size_t tSimpleHashGetMemSize(const SSHashObj *pHashObj) {
if (pHashObj == NULL) {
return 0;
}
return (pHashObj->capacity * sizeof(void *)) + sizeof(SHNode) * tSimpleHashGetSize(pHashObj) + sizeof(SSHashObj);
}
void *tSimpleHashGetKey(const SSHashObj* pHashObj, void *data, size_t* keyLen) {
int32_t offset = offsetof(SHNode, data);
SHNode *node = data - offset;
if (keyLen != NULL) {
*keyLen = pHashObj->keyLen;
}
return GET_SHASH_NODE_KEY(node, pHashObj->dataLen);
}

704
source/libs/executor/test/executorTests.cpp
View File

@ -201,9 +201,9 @@ SOperatorInfo* createDummyOperator(int32_t startVal, int32_t numOfBlocks, int32_
pOperator->name = "dummyInputOpertor4Test";
if (numOfCols == 1) {
pOperator->exec = getDummyBlock;
pOperator->nextDataFn = getDummyBlock;
} else {
pOperator->exec = get2ColsDummyBlock;
pOperator->nextDataFn = get2ColsDummyBlock;
}
SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo));
@ -222,100 +222,600 @@ int main(int argc, char** argv) {
return RUN_ALL_TESTS();
}
#if 0
TEST(testCase, build_executor_tree_Test) {
const char* msg = "{\n"
"\t\"Id\":\t{\n"
"\t\t\"QueryId\":\t1.3108161807422521e+19,\n"
"\t\t\"TemplateId\":\t0,\n"
"\t\t\"SubplanId\":\t0\n"
"\t},\n"
"\t\"Node\":\t{\n"
"\t\t\"Name\":\t\"TableScan\",\n"
"\t\t\"Targets\":\t[{\n"
"\t\t\t\t\"Base\":\t{\n"
"\t\t\t\t\t\"Schema\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\"ColId\":\t5000,\n"
"\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t},\n"
"\t\t\t\t\t\"Columns\":\t[{\n"
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
"\t\t\t\t\t\t\t\"Info\":\t{\n"
"\t\t\t\t\t\t\t\t\"ColId\":\t1,\n"
"\t\t\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t\t\t}\n"
"\t\t\t\t\t\t}],\n"
"\t\t\t\t\t\"InterBytes\":\t0\n"
"\t\t\t\t},\n"
"\t\t\t\t\"Expr\":\t{\n"
"\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\"Column\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\t\t\"ColId\":\t1,\n"
"\t\t\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t\t\t}\n"
"\t\t\t\t}\n"
"\t\t\t}, {\n"
"\t\t\t\t\"Base\":\t{\n"
"\t\t\t\t\t\"Schema\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\"ColId\":\t5001,\n"
"\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t},\n"
"\t\t\t\t\t\"Columns\":\t[{\n"
"\t\t\t\t\t\t\t\"TableId\":\t1,\n"
"\t\t\t\t\t\t\t\"Flag\":\t0,\n"
"\t\t\t\t\t\t\t\"Info\":\t{\n"
"\t\t\t\t\t\t\t\t\"ColId\":\t2,\n"
"\t\t\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t\t\t}\n"
"\t\t\t\t\t\t}],\n"
"\t\t\t\t\t\"InterBytes\":\t0\n"
"\t\t\t\t},\n"
"\t\t\t\t\"Expr\":\t{\n"
"\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\"Column\":\t{\n"
"\t\t\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\t\t\"ColId\":\t2,\n"
"\t\t\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t\t\t}\n"
"\t\t\t\t}\n"
"\t\t\t}],\n"
"\t\t\"InputSchema\":\t[{\n"
"\t\t\t\t\"Type\":\t9,\n"
"\t\t\t\t\"ColId\":\t5000,\n"
"\t\t\t\t\"Bytes\":\t8\n"
"\t\t\t}, {\n"
"\t\t\t\t\"Type\":\t4,\n"
"\t\t\t\t\"ColId\":\t5001,\n"
"\t\t\t\t\"Bytes\":\t4\n"
"\t\t\t}],\n"
"\t\t\"TableScan\":\t{\n"
"\t\t\t\"TableId\":\t1,\n"
"\t\t\t\"TableType\":\t2,\n"
"\t\t\t\"Flag\":\t0,\n"
"\t\t\t\"Window\":\t{\n"
"\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n"
"\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n"
"\t\t\t}\n"
"\t\t}\n"
"\t},\n"
"\t\"DataSink\":\t{\n"
"\t\t\"Name\":\t\"Dispatch\",\n"
"\t\t\"Dispatch\":\t{\n"
"\t\t}\n"
"\t}\n"
" \"Type\": \"33\",\n"
" \"Name\": \"PhysiProject\",\n"
" \"PhysiProject\": {\n"
" \"OutputDataBlockDesc\": {\n"
" \"Type\": \"19\",\n"
" \"Name\": \"TupleDesc\",\n"
" \"TupleDesc\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"Slots\": [\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"0\",\n"
" \"DataType\": {\n"
" \"Type\": \"9\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"1\",\n"
" \"DataType\": {\n"
" \"Type\": \"4\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"4\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"2\",\n"
" \"DataType\": {\n"
" \"Type\": \"8\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"20\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"3\",\n"
" \"DataType\": {\n"
" \"Type\": \"5\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"4\",\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"5\",\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": false\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" },\n"
" \"Children\": [\n"
" {\n"
" \"Type\": \"30\",\n"
" \"Name\": \"PhysiTableScan\",\n"
" \"PhysiTableScan\": {\n"
" \"OutputDataBlockDesc\": {\n"
" \"Type\": \"19\",\n"
" \"Name\": \"TupleDesc\",\n"
" \"TupleDesc\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"Slots\": [\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"0\",\n"
" \"DataType\": {\n"
" \"Type\": \"9\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"1\",\n"
" \"DataType\": {\n"
" \"Type\": \"4\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"4\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"2\",\n"
" \"DataType\": {\n"
" \"Type\": \"8\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"20\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"3\",\n"
" \"DataType\": {\n"
" \"Type\": \"5\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"4\",\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"20\",\n"
" \"Name\": \"SlotDesc\",\n"
" \"SlotDesc\": {\n"
" \"SlotId\": \"5\",\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"Reserve\": false,\n"
" \"Output\": true\n"
" }\n"
" }\n"
" ]\n"
" }\n"
" },\n"
" \"ScanCols\": [\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"9\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"ts\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"1\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"ts\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"1\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"4\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"4\"\n"
" },\n"
" \"AliasName\": \"c1\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"2\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c1\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"2\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"8\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"20\"\n"
" },\n"
" \"AliasName\": \"c2\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"3\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c2\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"3\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"5\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c3\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"4\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c3\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"4\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c4\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"5\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c4\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"5\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c5\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"6\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c5\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" }\n"
" ],\n"
" \"TableId\": \"1\",\n"
" \"TableType\": \"3\",\n"
" \"ScanOrder\": \"1\",\n"
" \"ScanCount\": \"1\",\n"
" \"ReverseScanCount\": \"0\",\n"
" \"ScanFlag\": \"0\",\n"
" \"StartKey\": \"-9223372036854775808\",\n"
" \"EndKey\": \"9223372036854775807\"\n"
" }\n"
" }\n"
" ],\n"
" \"Projections\": [\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"0\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"9\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"ts\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"1\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"ts\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"0\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"1\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"4\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"4\"\n"
" },\n"
" \"AliasName\": \"c1\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"2\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c1\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"1\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"2\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"8\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"20\"\n"
" },\n"
" \"AliasName\": \"c2\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"3\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c2\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"2\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"3\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"5\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c3\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"4\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c3\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"3\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"4\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c4\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"5\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c4\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"4\"\n"
" }\n"
" }\n"
" }\n"
" },\n"
" {\n"
" \"Type\": \"18\",\n"
" \"Name\": \"Target\",\n"
" \"Target\": {\n"
" \"DataBlockId\": \"1\",\n"
" \"SlotId\": \"5\",\n"
" \"Expr\": {\n"
" \"Type\": \"1\",\n"
" \"Name\": \"Column\",\n"
" \"Column\": {\n"
" \"DataType\": {\n"
" \"Type\": \"7\",\n"
" \"Precision\": \"0\",\n"
" \"Scale\": \"0\",\n"
" \"Bytes\": \"8\"\n"
" },\n"
" \"AliasName\": \"c5\",\n"
" \"TableId\": \"0\",\n"
" \"ColId\": \"6\",\n"
" \"ColType\": \"1\",\n"
" \"DbName\": \"test\",\n"
" \"TableName\": \"t1\",\n"
" \"TableAlias\": \"t1\",\n"
" \"ColName\": \"c5\",\n"
" \"DataBlockId\": \"0\",\n"
" \"SlotId\": \"5\"\n"
" }\n"
" }\n"
" }\n"
" }\n"
" ]\n"
" }\n"
"}";
SExecTaskInfo* pTaskInfo = nullptr;
DataSinkHandle sinkHandle = nullptr;
SReadHandle handle = {.reader = reinterpret_cast<void*>(0x1), .meta = reinterpret_cast<void*>(0x1)};
// int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
SSubplan* plan = NULL;
qStringToSubplan(msg, &plan);
int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle);
}
TEST(testCase, inMem_sort_Test) {
@ -334,10 +834,10 @@ TEST(testCase, inMem_sort_Test) {
exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
taosArrayPush(pExprInfo, &exp1);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal, NULL);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 5, 1000, data_asc, 1), pExprInfo, pOrderVal, NULL);
bool newgroup = false;
SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup);
SSDataBlock* pRes = pOperator->nextDataFn(pOperator, &newgroup);
SColumnInfoData* pCol1 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 0));
SColumnInfoData* pCol2 = static_cast<SColumnInfoData*>(taosArrayGet(pRes->pDataBlock, 1));
@ -400,7 +900,7 @@ TEST(testCase, external_sort_Test) {
exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1");
// taosArrayPush(pExprInfo, &exp1);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal, NULL);
SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(10000, 1500, 1000, data_desc, 1), pExprInfo, pOrderVal, NULL);
bool newgroup = false;
SSDataBlock* pRes = NULL;
@ -412,7 +912,7 @@ TEST(testCase, external_sort_Test) {
while(1) {
int64_t s = taosGetTimestampUs();
pRes = pOperator->exec(pOperator, &newgroup);
pRes = pOperator->nextDataFn(pOperator, &newgroup);
int64_t e = taosGetTimestampUs();
if (t++ == 1) {
@ -435,7 +935,7 @@ TEST(testCase, external_sort_Test) {
int64_t s2 = taosGetTimestampUs();
printf("total:%ld\n", s2 - s1);
pOperator->cleanupFn(pOperator->info, 2);
pOperator->closeFn(pOperator->info, 2);
tfree(exp);
tfree(exp1);
taosArrayDestroy(pExprInfo);
@ -469,7 +969,7 @@ TEST(testCase, sorted_merge_Test) {
int32_t numOfSources = 10;
SOperatorInfo** plist = (SOperatorInfo**) calloc(numOfSources, sizeof(void*));
for(int32_t i = 0; i < numOfSources; ++i) {
plist[i] = createDummyOperator(1, 1, 1, data_asc);
plist[i] = createDummyOperator(1, 1, 1, data_asc, 1);
}
SOperatorInfo* pOperator = createSortedMergeOperatorInfo(plist, numOfSources, pExprInfo, pOrderVal, NULL, NULL);
@ -484,7 +984,7 @@ TEST(testCase, sorted_merge_Test) {
while(1) {
int64_t s = taosGetTimestampUs();
pRes = pOperator->exec(pOperator, &newgroup);
pRes = pOperator->nextDataFn(pOperator, &newgroup);
int64_t e = taosGetTimestampUs();
if (t++ == 1) {
@ -507,15 +1007,13 @@ TEST(testCase, sorted_merge_Test) {
int64_t s2 = taosGetTimestampUs();
printf("total:%ld\n", s2 - s1);
pOperator->cleanupFn(pOperator->info, 2);
pOperator->closeFn(pOperator->info, 2);
tfree(exp);
tfree(exp1);
taosArrayDestroy(pExprInfo);
taosArrayDestroy(pOrderVal);
}
#endif
TEST(testCase, time_interval_Operator_Test) {
srand(time(NULL));
@ -548,7 +1046,11 @@ TEST(testCase, time_interval_Operator_Test) {
SOperatorInfo* p = createDummyOperator(1, 1, 2000, data_asc, 2);
SExecTaskInfo ti = {0};
SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &ti);
SInterval interval = {0};
interval.sliding = interval.interval = 1000;
interval.slidingUnit = interval.intervalUnit = 'a';
SOperatorInfo* pOperator = createIntervalOperatorInfo(p, pExprInfo, &interval, &ti);
bool newgroup = false;
SSDataBlock* pRes = NULL;
@ -560,7 +1062,7 @@ TEST(testCase, time_interval_Operator_Test) {
while(1) {
int64_t s = taosGetTimestampUs();
pRes = pOperator->exec(pOperator, &newgroup);
pRes = pOperator->nextDataFn(pOperator, &newgroup);
int64_t e = taosGetTimestampUs();
if (t++ == 1) {
@ -583,7 +1085,7 @@ TEST(testCase, time_interval_Operator_Test) {
int64_t s2 = taosGetTimestampUs();
printf("total:%ld\n", s2 - s1);
pOperator->cleanupFn(pOperator->info, 2);
pOperator->closeFn(pOperator->info, 2);
tfree(exp);
tfree(exp1);
taosArrayDestroy(pExprInfo);

2
source/libs/function/src/tpercentile.c
View File

@ -255,7 +255,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval,
resetSlotInfo(pBucket);
int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, "/tmp");
int32_t ret = createDiskbasedBuf(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, "1", "/tmp");
if (ret != 0) {
tMemBucketDestroy(pBucket);
return NULL;

2
source/libs/parser/src/insertParser.c
View File

@ -153,7 +153,7 @@ static int32_t buildOutput(SInsertParseContext* pCxt) {
if (NULL == dst) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
taosHashGetClone(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
taosHashGetDup(pCxt->pVgroupsHashObj, (const char*)&src->vgId, sizeof(src->vgId), &dst->vg);
dst->numOfTables = src->numOfTables;
dst->size = src->size;
TSWAP(dst->pData, src->pData, char*);

618
source/util/src/tcache.c
View File

@ -15,11 +15,99 @@
#define _DEFAULT_SOURCE
#include "tcache.h"
#include "taoserror.h"
#include "tlog.h"
#include "ttimer.h"
#include "tutil.h"
static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) {
#define CACHE_MAX_CAPACITY 1024*1024*16
#define CACHE_DEFAULT_CAPACITY 1024*4
static pthread_t cacheRefreshWorker = {0};
static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT;
static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER;
static SArray *pCacheArrayList = NULL;
static bool stopRefreshWorker = false;
static bool refreshWorkerNormalStopped = false;
static bool refreshWorkerUnexpectedStopped = false;
typedef struct SCacheNode {
uint64_t addedTime; // the added time when this element is added or updated into cache
uint64_t lifespan; // life duration when this element should be remove from cache
int64_t expireTime; // expire time
uint64_t signature;
struct STrashElem *pTNodeHeader; // point to trash node head
uint16_t keyLen: 15; // max key size: 32kb
bool inTrashcan : 1; // denote if it is in trash or not
uint32_t size; // allocated size for current SCacheNode
uint32_t dataLen;
T_REF_DECLARE()
struct SCacheNode *pNext;
char *key;
char *data;
} SCacheNode;
typedef struct SCacheEntry {
int32_t num; // number of elements in current entry
SRWLatch latch; // entry latch
SCacheNode *next;
} SCacheEntry;
typedef struct STrashElem {
struct STrashElem *prev;
struct STrashElem *next;
SCacheNode *pData;
} STrashElem;
typedef struct SCacheIter {
SCacheObj *pCacheObj;
SCacheNode **pCurrent;
int32_t entryIndex;
int32_t index;
int32_t numOfObj;
} SCacheIter;
/*
* to accommodate the old data which has the same key value of new one in hashList
* when an new node is put into cache, if an existed one with the same key:
* 1. if the old one does not be referenced, update it.
* 2. otherwise, move the old one to pTrash, addedTime the new one.
*
* when the node in pTrash does not be referenced, it will be release at the expired expiredTime
*/
struct SCacheObj {
int64_t sizeInBytes; // total allocated buffer in this hash table, SCacheObj is not included.
int64_t refreshTime;
char *name;
SCacheStatis statistics;
SCacheEntry *pEntryList;
size_t capacity; // number of slots
size_t numOfElems; // number of elements in cache
_hash_fn_t hashFp; // hash function
__cache_free_fn_t freeFp;
uint32_t numOfElemsInTrash; // number of element in trash
STrashElem *pTrash;
uint8_t deleting; // set the deleting flag to stop refreshing ASAP.
pthread_t refreshWorker;
bool extendLifespan; // auto extend life span when one item is accessed.
int64_t checkTick; // tick used to record the check times of the refresh threads
#if defined(LINUX)
pthread_rwlock_t lock;
#else
pthread_mutex_t lock;
#endif
};
typedef struct SCacheObjTravSup {
SCacheObj *pCacheObj;
int64_t time;
__cache_trav_fn_t fp;
void *param1;
} SCacheObjTravSup;
static FORCE_INLINE void __trashcan_wr_lock(SCacheObj *pCacheObj) {
#if defined(LINUX)
pthread_rwlock_wrlock(&pCacheObj->lock);
#else
@ -27,7 +115,7 @@ static FORCE_INLINE void __cache_wr_lock(SCacheObj *pCacheObj) {
#endif
}
static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) {
static FORCE_INLINE void __trashcan_unlock(SCacheObj *pCacheObj) {
#if defined(LINUX)
pthread_rwlock_unlock(&pCacheObj->lock);
#else
@ -35,7 +123,7 @@ static FORCE_INLINE void __cache_unlock(SCacheObj *pCacheObj) {
#endif
}
static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) {
static FORCE_INLINE int32_t __trashcan_lock_init(SCacheObj *pCacheObj) {
#if defined(LINUX)
return pthread_rwlock_init(&pCacheObj->lock, NULL);
#else
@ -43,7 +131,7 @@ static FORCE_INLINE int32_t __cache_lock_init(SCacheObj *pCacheObj) {
#endif
}
static FORCE_INLINE void __cache_lock_destroy(SCacheObj *pCacheObj) {
static FORCE_INLINE void __trashcan_lock_destroy(SCacheObj *pCacheObj) {
#if defined(LINUX)
pthread_rwlock_destroy(&pCacheObj->lock);
#else
@ -63,14 +151,6 @@ static void doCleanupDataCache(SCacheObj *pCacheObj);
*/
static void *taosCacheTimedRefresh(void *handle);
static pthread_t cacheRefreshWorker = {0};
static pthread_once_t cacheThreadInit = PTHREAD_ONCE_INIT;
static pthread_mutex_t guard = PTHREAD_MUTEX_INITIALIZER;
static SArray *pCacheArrayList = NULL;
static bool stopRefreshWorker = false;
static bool refreshWorkerNormalStopped = false;
static bool refreshWorkerUnexpectedStopped = false;
static void doInitRefreshThread(void) {
pCacheArrayList = taosArrayInit(4, POINTER_BYTES);
@ -99,9 +179,9 @@ pthread_t doRegisterCacheObj(SCacheObj *pCacheObj) {
* in pData. Pointer copy causes memory access error.
* @param size size of block
* @param lifespan total survial expiredTime from now
* @return SCacheDataNode
* @return SCacheNode
*/
static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
static SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size,
uint64_t duration);
/**
@ -110,7 +190,7 @@ static SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const
* @param pCacheObj Cache object
* @param pNode Cache slot object
*/
static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode);
static void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode);
/**
* remove nodes in trash with refCount == 0 in cache
@ -126,18 +206,16 @@ static void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force);
* @param pCacheObj cache object
* @param pNode data node
*/
static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
static FORCE_INLINE void taosCacheReleaseNode(SCacheObj *pCacheObj, SCacheNode *pNode) {
if (pNode->signature != (uint64_t)pNode) {
uError("key:%s, %p data is invalid, or has been released", pNode->key, pNode);
return;
}
atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size);
int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable);
assert(size > 0);
atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size);
uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, total num:%d size:%" PRId64 "bytes",
pCacheObj->name, pNode->key, pNode->data, pNode->size, size - 1, pCacheObj->totalSize);
pCacheObj->name, pNode->key, pNode->data, pNode->size, (int)pCacheObj->numOfElems - 1, pCacheObj->sizeInBytes);
if (pCacheObj->freeFp) {
pCacheObj->freeFp(pNode->data);
@ -181,6 +259,97 @@ static FORCE_INLINE void doDestroyTrashcanElem(SCacheObj *pCacheObj, STrashElem
free(pElem);
}
static void pushfrontNodeInEntryList(SCacheEntry *pEntry, SCacheNode *pNode) {
assert(pNode != NULL && pEntry != NULL);
pNode->pNext = pEntry->next;
pEntry->next = pNode;
pEntry->num += 1;
}
static void removeNodeInEntryList(SCacheEntry* pe, SCacheNode* prev, SCacheNode* pNode) {
if (prev == NULL) {
ASSERT(pe->next == pNode);
pe->next = pNode->pNext;
} else {
prev->pNext = pNode->pNext;
}
pNode->pNext = NULL;
pe->num -= 1;
}
static FORCE_INLINE SCacheEntry* doFindEntry(SCacheObj* pCacheObj, const void* key, size_t keyLen) {
uint32_t hashVal = (*pCacheObj->hashFp)(key, keyLen);
int32_t slot = hashVal % pCacheObj->capacity;
return &pCacheObj->pEntryList[slot];
}
static FORCE_INLINE SCacheNode *
doSearchInEntryList(SCacheEntry *pe, const void *key, size_t keyLen, SCacheNode** prev) {
SCacheNode *pNode = pe->next;
while (pNode) {
if ((pNode->keyLen == keyLen) && memcmp(pNode->key, key, keyLen) == 0) {
break;
}
*prev = pNode;
pNode = pNode->pNext;
}
return pNode;
}
static bool doRemoveExpiredFn(void *param, SCacheNode* pNode) {
SCacheObjTravSup *ps = (SCacheObjTravSup *)param;
SCacheObj *pCacheObj = ps->pCacheObj;
if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) {
taosCacheReleaseNode(pCacheObj, pNode);
// this node should be remove from hash table
return false;
}
if (ps->fp) {
(ps->fp)(pNode->data, ps->param1);
}
// do not remove element in hash table
return true;
}
static bool doRemoveNodeFn(void *param, SCacheNode *pNode) {
SCacheObjTravSup *ps = (SCacheObjTravSup *)param;
SCacheObj *pCacheObj = ps->pCacheObj;
if (T_REF_VAL_GET(pNode) == 0) {
taosCacheReleaseNode(pCacheObj, pNode);
} else { // do add to trashcan
taosAddToTrashcan(pCacheObj, pNode);
}
// this node should be remove from hash table
return false;
}
static FORCE_INLINE int32_t getCacheCapacity(int32_t length) {
int32_t len = 0;
if (length < CACHE_DEFAULT_CAPACITY) {
len = CACHE_DEFAULT_CAPACITY;
return len;
} else if (length > CACHE_MAX_CAPACITY) {
len = CACHE_MAX_CAPACITY;
return len;
}
len = CACHE_DEFAULT_CAPACITY;
while (len < length && len < CACHE_MAX_CAPACITY) {
len = (len << 1u);
}
return len > CACHE_MAX_CAPACITY? CACHE_MAX_CAPACITY:len;
}
SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool extendLifespan, __cache_free_fn_t fn,
const char *cacheName) {
const int32_t SLEEP_DURATION = 500; // 500 ms
@ -195,39 +364,42 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
return NULL;
}
pCacheObj->pHashTable = taosHashInit(4096, taosGetDefaultHashFunction(keyType), false, HASH_ENTRY_LOCK);
pCacheObj->name = strdup(cacheName);
if (pCacheObj->pHashTable == NULL) {
// TODO add the auto extend procedure
pCacheObj->capacity = 4096;
pCacheObj->pEntryList = calloc(pCacheObj->capacity, sizeof(SCacheEntry));
if (pCacheObj->pEntryList == NULL) {
free(pCacheObj);
uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL;
}
// set free cache node callback function
pCacheObj->freeFp = fn;
pCacheObj->refreshTime = refreshTimeInSeconds * 1000;
pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION;
pCacheObj->hashFp = taosGetDefaultHashFunction(keyType);
pCacheObj->freeFp = fn;
pCacheObj->refreshTime = refreshTimeInSeconds * 1000;
pCacheObj->checkTick = pCacheObj->refreshTime / SLEEP_DURATION;
pCacheObj->extendLifespan = extendLifespan; // the TTL after the last access
if (__cache_lock_init(pCacheObj) != 0) {
taosHashCleanup(pCacheObj->pHashTable);
if (__trashcan_lock_init(pCacheObj) != 0) {
tfree(pCacheObj->pEntryList);
free(pCacheObj);
uError("failed to init lock, reason:%s", strerror(errno));
return NULL;
}
pCacheObj->name = strdup(cacheName);
doRegisterCacheObj(pCacheObj);
return pCacheObj;
}
void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize,
int32_t durationMS) {
if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) {
if (pCacheObj == NULL || pCacheObj->pEntryList == NULL || pCacheObj->deleting == 1) {
return NULL;
}
SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS);
SCacheNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS);
if (pNode1 == NULL) {
uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key);
return NULL;
@ -235,86 +407,77 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
T_REF_INC(pNode1);
int32_t succ = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *));
if (succ == 0) {
atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size);
SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen);
taosWLockLatch(&pe->latch);
SCacheNode *prev = NULL;
SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev);
if (pNode == NULL) {
pushfrontNodeInEntryList(pe, pNode1);
atomic_add_fetch_64(&pCacheObj->numOfElems, 1);
atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size);
uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes",
pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime,
(int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize);
", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes",
pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems,
pCacheObj->sizeInBytes, (int64_t)dataSize);
} else { // duplicated key exists
while (1) {
SCacheDataNode *p = NULL;
// int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, key, keyLen, (void*) &p, sizeof(void*));
int32_t ret = taosHashRemove(pCacheObj->pHashTable, key, keyLen);
// move current node to trashcan
removeNodeInEntryList(pe, prev, pNode);
// add to trashcan
if (ret == 0) {
if (T_REF_VAL_GET(p) == 0) {
if (pCacheObj->freeFp) {
pCacheObj->freeFp(p->data);
}
atomic_sub_fetch_64(&pCacheObj->totalSize, p->size);
tfree(p);
} else {
taosAddToTrashcan(pCacheObj, p);
uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, p->data);
}
if (T_REF_VAL_GET(pNode) == 0) {
if (pCacheObj->freeFp) {
pCacheObj->freeFp(pNode->data);
}
assert(T_REF_VAL_GET(pNode1) == 1);
ret = taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *));
if (ret == 0) {
atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size);
uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
", totalNum:%d totalSize:%" PRId64 "bytes size:%" PRId64 "bytes",
pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime,
(int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize);
return pNode1->data;
} else {
// failed, try again
}
atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size);
tfree(pNode);
} else {
taosAddToTrashcan(pCacheObj, pNode);
uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pNode->data);
}
pushfrontNodeInEntryList(pe, pNode1);
atomic_add_fetch_64(&pCacheObj->sizeInBytes, pNode1->size);
uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64
", totalNum:%d sizeInBytes:%" PRId64 "bytes size:%" PRId64 "bytes",
pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime, (int32_t)pCacheObj->numOfElems,
pCacheObj->sizeInBytes, (int64_t)dataSize);
}
taosWUnLockLatch(&pe->latch);
return pNode1->data;
}
static void incRefFn(void *ptNode) {
assert(ptNode != NULL);
SCacheDataNode **p = (SCacheDataNode **)ptNode;
assert(T_REF_VAL_GET(*p) >= 0);
int32_t ret = T_REF_INC(*p);
assert(ret > 0);
}
void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) {
if (pCacheObj == NULL || pCacheObj->deleting == 1) {
return NULL;
}
if (taosHashGetSize(pCacheObj->pHashTable) == 0) {
if (pCacheObj->numOfElems == 0) {
atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1);
return NULL;
}
SCacheDataNode *ptNode = NULL;
taosHashGetClone(pCacheObj->pHashTable, key, keyLen, &ptNode);
// taosHashGetClone(pCacheObj->pHashTable, key, keyLen, incRefFn, &ptNode);
SCacheNode *prev = NULL;
SCacheEntry *pe = doFindEntry(pCacheObj, key, keyLen);
void *pData = (ptNode != NULL) ? ptNode->data : NULL;
taosRLockLatch(&pe->latch);
SCacheNode* pNode = doSearchInEntryList(pe, key, keyLen, &prev);
if (pNode != NULL) {
int32_t ref = T_REF_INC(pNode);
ASSERT(ref > 0);
}
taosRUnLockLatch(&pe->latch);
void *pData = (pNode != NULL) ? pNode->data : NULL;
if (pData != NULL) {
atomic_add_fetch_32(&pCacheObj->statistics.hitCount, 1);
uDebug("cache:%s, key:%p, %p is retrieved from cache, refcnt:%d", pCacheObj->name, key, pData,
T_REF_VAL_GET(ptNode));
T_REF_VAL_GET(pNode));
} else {
atomic_add_fetch_32(&pCacheObj->statistics.missCount, 1);
uDebug("cache:%s, key:%p, not in cache, retrieved failed", pCacheObj->name, key);
@ -327,9 +490,7 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen
void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
if (pCacheObj == NULL || data == NULL) return NULL;
size_t offset = offsetof(SCacheDataNode, data);
SCacheDataNode *ptNode = (SCacheDataNode *)((char *)data - offset);
SCacheNode *ptNode = (SCacheNode *)((char *)data - sizeof(SCacheNode));
if (ptNode->signature != (uint64_t)ptNode) {
uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode);
return NULL;
@ -343,24 +504,20 @@ void *taosCacheAcquireByData(SCacheObj *pCacheObj, void *data) {
return data;
}
void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) {
void *taosCacheTransferData(SCacheObj *pCacheObj, void **data) {
if (pCacheObj == NULL || data == NULL || (*data) == NULL) return NULL;
size_t offset = offsetof(SCacheDataNode, data);
SCacheDataNode *ptNode = (SCacheDataNode *)((char *)(*data) - offset);
SCacheNode *ptNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode));
if (ptNode->signature != (uint64_t)ptNode) {
uError("cache:%s, key: %p the data from cache is invalid", pCacheObj->name, ptNode);
return NULL;
}
assert(T_REF_VAL_GET(ptNode) >= 1);
char *d = *data;
// clear its reference to old area
*data = NULL;
return d;
}
@ -378,9 +535,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
// therefore the check for the empty of both the hash table and the trashcan has a race condition.
// It happens when there is only one object in the cache, and two threads which has referenced this object
// start to free the it simultaneously [TD-1569].
size_t offset = offsetof(SCacheDataNode, data);
SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset);
SCacheNode *pNode = (SCacheNode *)((char *)(*data) - sizeof(SCacheNode));
if (pNode->signature != (uint64_t)pNode) {
uError("cache:%s, %p, release invalid cache data", pCacheObj->name, pNode);
return;
@ -419,9 +574,9 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
// destroyed by refresh worker if decrease ref count before removing it from linked-list.
assert(pNode->pTNodeHeader->pData == pNode);
__cache_wr_lock(pCacheObj);
__trashcan_wr_lock(pCacheObj);
doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader);
__cache_unlock(pCacheObj);
__trashcan_unlock(pCacheObj);
ref = T_REF_DEC(pNode);
assert(ref == 0);
@ -434,36 +589,37 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
} else {
// NOTE: remove it from hash in the first place, otherwise, the pNode may have been released by other thread
// when reaches here.
SCacheDataNode *p = NULL;
int32_t ret = taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize);
// int32_t ret = taosHashRemoveWithData(pCacheObj->pHashTable, pNode->key, pNode->keySize, &p, sizeof(void
// *));
SCacheNode * prev = NULL;
SCacheEntry *pe = doFindEntry(pCacheObj, pNode->key, pNode->keyLen);
taosWLockLatch(&pe->latch);
ref = T_REF_DEC(pNode);
// successfully remove from hash table, if failed, this node must have been move to trash already, do nothing.
// note that the remove operation can be executed only once.
if (ret == 0) {
SCacheNode *p = doSearchInEntryList(pe, pNode->key, pNode->keyLen, &prev);
if (p != NULL) {
// successfully remove from hash table, if failed, this node must have been move to trash already, do nothing.
// note that the remove operation can be executed only once.
if (p != pNode) {
uDebug(
"cache:%s, key:%p, successfully removed a new entry:%p, refcnt:%d, prev entry:%p has been removed by "
"others already",
pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data);
"cache:%s, key:%p, a new entry:%p found, refcnt:%d, prev entry:%p, refcnt:%d has been removed by "
"others already, prev must in trashcan",
pCacheObj->name, pNode->key, p->data, T_REF_VAL_GET(p), pNode->data, T_REF_VAL_GET(pNode));
assert(p->pTNodeHeader == NULL);
taosAddToTrashcan(pCacheObj, p);
assert(p->pTNodeHeader == NULL && pNode->pTNodeHeader != NULL);
} else {
removeNodeInEntryList(pe, prev, p);
uDebug("cache:%s, key:%p, %p successfully removed from hash table, refcnt:%d", pCacheObj->name, pNode->key,
pNode->data, ref);
if (ref > 0) {
assert(pNode->pTNodeHeader == NULL);
taosAddToTrashcan(pCacheObj, pNode);
} else { // ref == 0
atomic_sub_fetch_64(&pCacheObj->totalSize, pNode->size);
atomic_sub_fetch_64(&pCacheObj->sizeInBytes, pNode->size);
int32_t size = (int32_t)taosHashGetSize(pCacheObj->pHashTable);
int32_t size = (int32_t)pCacheObj->numOfElems;
uDebug("cache:%s, key:%p, %p is destroyed from cache, size:%dbytes, totalNum:%d size:%" PRId64 "bytes",
pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->totalSize);
pCacheObj->name, pNode->key, pNode->data, pNode->size, size, pCacheObj->sizeInBytes);
if (pCacheObj->freeFp) {
pCacheObj->freeFp(pNode->data);
@ -472,6 +628,8 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
free(pNode);
}
}
taosWUnLockLatch(&pe->latch);
} else {
uDebug("cache:%s, key:%p, %p has been removed from hash table by others already, refcnt:%d", pCacheObj->name,
pNode->key, pNode->data, ref);
@ -483,59 +641,43 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
char *key = pNode->key;
char *p = pNode->data;
// int32_t ref = T_REF_VAL_GET(pNode);
//
// if (ref == 1 && inTrashcan) {
// // If it is the last ref, remove it from trashcan linked-list first, and then destroy it.Otherwise, it may
// be
// // destroyed by refresh worker if decrease ref count before removing it from linked-list.
// assert(pNode->pTNodeHeader->pData == pNode);
//
// __cache_wr_lock(pCacheObj);
// doRemoveElemInTrashcan(pCacheObj, pNode->pTNodeHeader);
// __cache_unlock(pCacheObj);
//
// ref = T_REF_DEC(pNode);
// assert(ref == 0);
//
// doDestroyTrashcanElem(pCacheObj, pNode->pTNodeHeader);
// } else {
// ref = T_REF_DEC(pNode);
// assert(ref >= 0);
// }
int32_t ref = T_REF_DEC(pNode);
uDebug("cache:%s, key:%p, %p released, refcnt:%d, data in trashcan:%d", pCacheObj->name, key, p, ref, inTrashcan);
}
}
typedef struct SHashTravSupp {
SCacheObj *pCacheObj;
int64_t time;
__cache_trav_fn_t fp;
void *param1;
} SHashTravSupp;
void doTraverseElems(SCacheObj* pCacheObj, bool (*fp)(void *param, SCacheNode* pNode), SCacheObjTravSup* pSup) {
int32_t numOfEntries = (int32_t)pCacheObj->capacity;
for (int32_t i = 0; i < numOfEntries; ++i) {
SCacheEntry *pEntry = &pCacheObj->pEntryList[i];
if (pEntry->num == 0) {
continue;
}
static bool travHashTableEmptyFn(void *param, void *data) {
SHashTravSupp *ps = (SHashTravSupp *)param;
SCacheObj *pCacheObj = ps->pCacheObj;
taosWLockLatch(&pEntry->latch);
SCacheDataNode *pNode = *(SCacheDataNode **)data;
SCacheNode *pNode = pEntry->next;
while (pNode != NULL) {
SCacheNode *next = pNode->pNext;
if (T_REF_VAL_GET(pNode) == 0) {
taosCacheReleaseNode(pCacheObj, pNode);
} else { // do add to trashcan
taosAddToTrashcan(pCacheObj, pNode);
if (fp(pSup, pNode)) {
pNode = pNode->pNext;
} else {
pEntry->next = next;
pEntry->num -= 1;
atomic_sub_fetch_64(&pCacheObj->numOfElems, 1);
pNode = next;
}
}
taosWUnLockLatch(&pEntry->latch);
}
// this node should be remove from hash table
return false;
}
void taosCacheEmpty(SCacheObj *pCacheObj) {
SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup);
void taosCacheEmpty(SCacheObj* pCacheObj) {
SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
doTraverseElems(pCacheObj, doRemoveNodeFn, &sup);
taosTrashcanEmpty(pCacheObj, false);
}
@ -558,38 +700,41 @@ void taosCacheCleanup(SCacheObj *pCacheObj) {
doCleanupDataCache(pCacheObj);
}
SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) {
size_t totalSize = size + sizeof(SCacheDataNode) + keyLen;
SCacheNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) {
size_t sizeInBytes = size + sizeof(SCacheNode) + keyLen;
SCacheDataNode *pNewNode = calloc(1, totalSize);
SCacheNode *pNewNode = calloc(1, sizeInBytes);
if (pNewNode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL;
}
pNewNode->data = (char*)pNewNode + sizeof(SCacheNode);
pNewNode->dataLen = size;
memcpy(pNewNode->data, pData, size);
pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + size;
pNewNode->keySize = (uint16_t)keyLen;
pNewNode->key = (char *)pNewNode + sizeof(SCacheNode) + size;
pNewNode->keyLen = (uint16_t)keyLen;
memcpy(pNewNode->key, key, keyLen);
pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
pNewNode->lifespan = duration;
pNewNode->addedTime = (uint64_t)taosGetTimestampMs();
pNewNode->lifespan = duration;
pNewNode->expireTime = pNewNode->addedTime + pNewNode->lifespan;
pNewNode->signature = (uint64_t)pNewNode;
pNewNode->size = (uint32_t)totalSize;
pNewNode->signature = (uint64_t)pNewNode;
pNewNode->size = (uint32_t)sizeInBytes;
return pNewNode;
}
void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheNode *pNode) {
if (pNode->inTrashcan) { /* node is already in trash */
assert(pNode->pTNodeHeader != NULL && pNode->pTNodeHeader->pData == pNode);
return;
}
__cache_wr_lock(pCacheObj);
__trashcan_wr_lock(pCacheObj);
STrashElem *pElem = calloc(1, sizeof(STrashElem));
pElem->pData = pNode;
pElem->prev = NULL;
@ -604,14 +749,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
pCacheObj->pTrash = pElem;
pCacheObj->numOfElemsInTrash++;
__cache_unlock(pCacheObj);
__trashcan_unlock(pCacheObj);
uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key,
pNode->data, pElem, pCacheObj->numOfElemsInTrash);
}
void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
__cache_wr_lock(pCacheObj);
__trashcan_wr_lock(pCacheObj);
if (pCacheObj->numOfElemsInTrash == 0) {
if (pCacheObj->pTrash != NULL) {
@ -620,7 +765,7 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
pCacheObj->numOfElemsInTrash);
}
__cache_unlock(pCacheObj);
__trashcan_unlock(pCacheObj);
return;
}
@ -645,49 +790,28 @@ void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {
}
}
__cache_unlock(pCacheObj);
__trashcan_unlock(pCacheObj);
}
void doCleanupDataCache(SCacheObj *pCacheObj) {
// SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableEmptyFn, &sup);
SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = NULL, .time = taosGetTimestampMs()};
doTraverseElems(pCacheObj, doRemoveNodeFn, &sup);
// todo memory leak if there are object with refcount greater than 0 in hash table?
taosHashCleanup(pCacheObj->pHashTable);
taosTrashcanEmpty(pCacheObj, true);
__cache_lock_destroy(pCacheObj);
__trashcan_lock_destroy(pCacheObj);
tfree(pCacheObj->pEntryList);
tfree(pCacheObj->name);
memset(pCacheObj, 0, sizeof(SCacheObj));
free(pCacheObj);
}
bool travHashTableFn(void *param, void *data) {
SHashTravSupp *ps = (SHashTravSupp *)param;
SCacheObj *pCacheObj = ps->pCacheObj;
SCacheDataNode *pNode = *(SCacheDataNode **)data;
if ((int64_t)pNode->expireTime < ps->time && T_REF_VAL_GET(pNode) <= 0) {
taosCacheReleaseNode(pCacheObj, pNode);
// this node should be remove from hash table
return false;
}
if (ps->fp) {
(ps->fp)(pNode->data, ps->param1);
}
// do not remove element in hash table
return true;
}
static void doCacheRefresh(SCacheObj *pCacheObj, int64_t time, __cache_trav_fn_t fp, void *param1) {
assert(pCacheObj != NULL);
SHashTravSupp sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1};
// taosHashCondTraverse(pCacheObj->pHashTable, travHashTableFn, &sup);
SCacheObjTravSup sup = {.pCacheObj = pCacheObj, .fp = fp, .time = time, .param1 = param1};
doTraverseElems(pCacheObj, doRemoveExpiredFn, &sup);
}
void taosCacheRefreshWorkerUnexpectedStopped(void) {
@ -746,7 +870,7 @@ void *taosCacheTimedRefresh(void *handle) {
continue;
}
size_t elemInHash = taosHashGetSize(pCacheObj->pHashTable);
size_t elemInHash = pCacheObj->numOfElems;
if (elemInHash + pCacheObj->numOfElemsInTrash == 0) {
continue;
}
@ -784,4 +908,94 @@ void taosCacheRefresh(SCacheObj *pCacheObj, __cache_trav_fn_t fp, void *param1)
doCacheRefresh(pCacheObj, now, fp, param1);
}
void taosStopCacheRefreshWorker(void) { stopRefreshWorker = true; }
void taosStopCacheRefreshWorker(void) {
stopRefreshWorker = true;
}
size_t taosCacheGetNumOfObj(const SCacheObj* pCacheObj) {
return pCacheObj->numOfElems + pCacheObj->numOfElemsInTrash;
}
SCacheIter* taosCacheCreateIter(const SCacheObj* pCacheObj) {
ASSERT(pCacheObj != NULL);
SCacheIter* pIter = calloc(1, sizeof(SCacheIter));
pIter->pCacheObj = (SCacheObj*) pCacheObj;
pIter->entryIndex = -1;
pIter->index = -1;
return pIter;
}
bool taosCacheIterNext(SCacheIter* pIter) {
SCacheObj* pCacheObj = pIter->pCacheObj;
if (pIter->index + 1 >= pIter->numOfObj) {
if (pIter->entryIndex + 1 >= pCacheObj->capacity) {
return false;
}
// release the reference for all objects in the snapshot
for(int32_t i = 0; i < pIter->numOfObj; ++i) {
char* p= pIter->pCurrent[i]->data;
taosCacheRelease(pCacheObj, (void**) &p, false);
pIter->pCurrent[i] = NULL;
}
while(1) {
SCacheEntry *pEntry = &pCacheObj->pEntryList[++pIter->entryIndex];
taosRLockLatch(&pEntry->latch);
if (pEntry->num == 0) {
taosRUnLockLatch(&pEntry->latch);
continue;
}
if (pIter->numOfObj < pEntry->num) {
char *tmp = realloc(pIter->pCurrent, pEntry->num * POINTER_BYTES);
if (tmp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosRUnLockLatch(&pEntry->latch);
return false;
}
pIter->pCurrent = (SCacheNode **)tmp;
}
SCacheNode* pNode = pEntry->next;
for (int32_t i = 0; i < pEntry->num; ++i) {
ASSERT(pNode != NULL);
pIter->pCurrent[i] = pNode;
int32_t ref = T_REF_INC(pIter->pCurrent[i]);
ASSERT(ref >= 1);
pNode = pNode->pNext;
}
pIter->numOfObj = pEntry->num;
taosRUnLockLatch(&pEntry->latch);
pIter->index = -1;
break;
}
}
pIter->index += 1;
return true;
}
void* taosCacheIterGetData(const SCacheIter* pIter, size_t* len) {
SCacheNode* pNode = pIter->pCurrent[pIter->index];
*len = pNode->dataLen;
return pNode->data;
}
void* taosCacheIterGetKey(const SCacheIter* pIter, size_t* len) {
SCacheNode* pNode = pIter->pCurrent[pIter->index];
*len = pNode->keyLen;
return pNode->key;
}
void taosCacheDestroyIter(SCacheIter* pIter) {
tfree(pIter->pCurrent);
tfree(pIter);
}

782
source/util/src/thash.c
View File

File diff suppressed because it is too large Load Diff

43
source/util/src/tpagedbuf.c
View File

@ -42,8 +42,8 @@ struct SDiskbasedBuf {
bool comp; // compressed before flushed to disk
uint64_t nextPos; // next page flush position
uint64_t qId; // for debug purpose
bool printStatis; // Print statistics info when closing this buffer.
char* id; // for debug purpose
bool printStatis; // Print statistics info when closing this buffer.
SDiskbasedBufStatis statis;
};
@ -269,11 +269,12 @@ static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pag
SPageInfo* ppi = malloc(sizeof(SPageInfo));
ppi->pageId = pageId;
ppi->pData = NULL;
ppi->pData = NULL;
ppi->offset = -1;
ppi->length = -1;
ppi->used = true;
ppi->pn = NULL;
ppi->used = true;
ppi->pn = NULL;
ppi->dirty = false;
return *(SPageInfo**)taosArrayPush(list, &ppi);
}
@ -356,7 +357,7 @@ static SPageInfo* getPageInfoFromPayload(void* page) {
return ppi;
}
int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId,
int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, const char* id,
const char* dir) {
*pBuf = calloc(1, sizeof(SDiskbasedBuf));
@ -366,13 +367,13 @@ int32_t createDiskbasedBuf(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMem
}
pPBuf->pageSize = pagesize;
pPBuf->numOfPages = 0; // all pages are in buffer in the first place
pPBuf->numOfPages = 0; // all pages are in buffer in the first place
pPBuf->totalBufSize = 0;
pPBuf->inMemPages = inMemBufSize / pagesize; // maximum allowed pages, it is a soft limit.
pPBuf->allocateId = -1;
pPBuf->comp = true;
pPBuf->pFile = NULL;
pPBuf->qId = qId;
pPBuf->comp = true;
pPBuf->pFile = NULL;
pPBuf->id = strdup(id);
pPBuf->fileSize = 0;
pPBuf->pFree = taosArrayInit(4, sizeof(SFreeListItem));
pPBuf->freePgList = tdListNew(POINTER_BYTES);
@ -471,7 +472,7 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
return (void*)(GET_DATA_PAYLOAD(*pi));
} else { // not in memory
assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0);
assert((*pi)->pData == NULL && (*pi)->pn == NULL && (((*pi)->length >= 0 && (*pi)->offset >= 0) || ((*pi)->length == -1 && (*pi)->offset == -1)));
char* availablePage = NULL;
if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) {
@ -493,9 +494,12 @@ void* getBufPage(SDiskbasedBuf* pBuf, int32_t id) {
lruListPushFront(pBuf->lruList, *pi);
(*pi)->used = true;
int32_t code = loadPageFromDisk(pBuf, *pi);
if (code != 0) {
return NULL;
// some data has been flushed to disk, and needs to be loaded into buffer again.
if ((*pi)->length > 0 && (*pi)->offset >= 0) {
int32_t code = loadPageFromDisk(pBuf, *pi);
if (code != 0) {
return NULL;
}
}
return (void*)(GET_DATA_PAYLOAD(*pi));
@ -540,13 +544,13 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
if (pBuf->pFile != NULL) {
uDebug(
"Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page "
"size:%.2f Kb, %" PRIx64 "\n",
"size:%.2f Kb, %s\n",
pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId);
listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->id);
taosCloseFile(&pBuf->pFile);
} else {
uDebug("Paged buffer closed, total:%.2f Kb, no file created, %" PRIx64, pBuf->totalBufSize / 1024.0, pBuf->qId);
uDebug("Paged buffer closed, total:%.2f Kb, no file created, %s", pBuf->totalBufSize / 1024.0, pBuf->id);
}
// print the statistics information
@ -584,6 +588,7 @@ void destroyDiskbasedBuf(SDiskbasedBuf* pBuf) {
taosHashCleanup(pBuf->groupSet);
taosHashCleanup(pBuf->all);
tfree(pBuf->id);
tfree(pBuf->assistBuf);
tfree(pBuf);
}
@ -639,9 +644,9 @@ void dBufPrintStatis(const SDiskbasedBuf* pBuf) {
printf(
"Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f "
"Kb, %" PRIx64 "\n",
"Kb, %s\n",
pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0,
listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId);
listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->id);
printf(
"Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n",

5
source/util/test/arrayTest.cpp
View File

@ -43,6 +43,9 @@ static void remove_batch_test() {
taosArrayPush(delList, &a);
taosArrayRemoveBatch(pa, (const int32_t*) TARRAY_GET_START(delList), taosArrayGetSize(delList));
EXPECT_EQ(taosArrayGetSize(pa), 17);
taosArrayDestroy(pa);
taosArrayDestroy(delList);
}
} // namespace
@ -79,4 +82,6 @@ TEST(arrayTest, array_search_test) {
}
}
taosArrayDestroy(pa);
}

24
source/util/test/cacheTest.cpp
View File

@ -6,7 +6,7 @@
#include "tcache.h"
// test cache
TEST(testCase, client_cache_test) {
TEST(cacheTest, client_cache_test) {
const int32_t REFRESH_TIME_IN_SEC = 2;
SCacheObj* tscMetaCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, 0, NULL, "test");
@ -92,7 +92,7 @@ TEST(testCase, client_cache_test) {
taosCacheCleanup(tscMetaCache);
}
TEST(testCase, cache_resize_test) {
TEST(cacheTest, cache_iter_test) {
const int32_t REFRESH_TIME_IN_SEC = 2;
auto* pCache = taosCacheInit(TSDB_DATA_TYPE_BINARY, REFRESH_TIME_IN_SEC, false, NULL, "test");
@ -107,6 +107,7 @@ TEST(testCase, cache_resize_test) {
int32_t len = sprintf(key, "abc_%7d", i);
taosCachePut(pCache, key, strlen(key), data, len, 3600);
}
uint64_t endTime = taosGetTimestampUs();
printf("add %d object cost:%" PRIu64 " us, avg:%f us\n", num, endTime - startTime, (endTime-startTime)/(double)num);
@ -120,5 +121,22 @@ TEST(testCase, cache_resize_test) {
endTime = taosGetTimestampUs();
printf("retrieve %d object cost:%" PRIu64 " us,avg:%f\n", num, endTime - startTime, (endTime - startTime)/(double)num);
int32_t count = 0;
SCacheIter* pIter = taosCacheCreateIter(pCache);
while(taosCacheIterNext(pIter)) {
size_t keyLen = 0;
size_t dataLen = 0;
char* key1 = static_cast<char*>(taosCacheIterGetKey(pIter, &keyLen));
char* data1 = static_cast<char*>(taosCacheIterGetData(pIter, &dataLen));
// char d[256] = {0};
// memcpy(d, data1, dataLen);
// char k[256] = {0};
// memcpy(k, key1, keyLen);
}
ASSERT_EQ(count, num);
taosCacheCleanup(pCache);
}
}

8
source/util/test/encodeTest.cpp
View File

@ -201,8 +201,8 @@ TEST(td_encode_test, encode_decode_cstr) {
}
}
delete buf;
delete cstr;
delete[] buf;
delete[] cstr;
}
typedef struct {
@ -354,7 +354,7 @@ static int32_t tSFinalReq_v2_decode(SCoder *pCoder, SFinalReq_v2 *ps2) {
tEndDecode(pCoder);
return 0;
}
#if 0
TEST(td_encode_test, compound_struct_encode_test) {
SCoder encoder, decoder;
uint8_t * buf1;
@ -436,5 +436,5 @@ TEST(td_encode_test, compound_struct_encode_test) {
GTEST_ASSERT_EQ(dreq21.v_b, req2.v_b);
tCoderClear(&decoder);
}
#endif
#pragma GCC diagnostic pop

9
source/util/test/hashTest.cpp
View File

@ -106,7 +106,7 @@ void noLockPerformanceTest() {
ASSERT_EQ(taosHashGetSize(hashTable), 0);
char key[128] = {0};
int32_t num = 5000000;
int32_t num = 5000;
int64_t st = taosGetTimestampUs();
@ -186,10 +186,15 @@ void acquireRleaseTest() {
printf("%s,expect:%s", pdata->p, str3);
ASSERT_TRUE(strcmp(pdata->p, str3) == 0);
tfree(pdata->p);
taosHashRelease(hashTable, pdata);
num = taosHashGetSize(hashTable);
ASSERT_EQ(num, 1);
taosHashCleanup(hashTable);
tfree(data.p);
}
}

113
source/util/test/pageBufferTest.cpp
View File

@ -12,145 +12,150 @@
namespace {
// simple test
void simpleTest() {
SDiskbasedBuf* pResultBuf = NULL;
int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4096, 1, "/tmp/");
SDiskbasedBuf* pBuf = NULL;
int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4096, "", "/tmp/");
int32_t pageId = 0;
int32_t groupId = 0;
SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
ASSERT_TRUE(pBufPage != NULL);
ASSERT_EQ(getTotalBufSize(pResultBuf), 1024);
ASSERT_EQ(getTotalBufSize(pBuf), 1024);
SIDList list = getDataBufPagesIdList(pResultBuf, groupId);
SIDList list = getDataBufPagesIdList(pBuf, groupId);
ASSERT_EQ(taosArrayGetSize(list), 1);
ASSERT_EQ(getNumOfBufGroupId(pResultBuf), 1);
ASSERT_EQ(getNumOfBufGroupId(pBuf), 1);
releaseBufPage(pResultBuf, pBufPage);
releaseBufPage(pBuf, pBufPage);
SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* t = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t == pBufPage1);
SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t1 == pBufPage2);
SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t2 == pBufPage3);
SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t3 == pBufPage4);
SFilePage* pBufPage5 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
releaseBufPage(pBuf, pBufPage2);
SFilePage* pBufPage5 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t4 == pBufPage5);
destroyDiskbasedBuf(pResultBuf);
destroyDiskbasedBuf(pBuf);
}
void writeDownTest() {
SDiskbasedBuf* pResultBuf = NULL;
int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/");
SDiskbasedBuf* pBuf = NULL;
int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/");
int32_t pageId = 0;
int32_t writePageId = 0;
int32_t groupId = 0;
int32_t nx = 12345;
SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
ASSERT_TRUE(pBufPage != NULL);
*(int32_t*)(pBufPage->data) = nx;
writePageId = pageId;
releaseBufPage(pResultBuf, pBufPage);
setBufPageDirty(pBufPage, true);
releaseBufPage(pBuf, pBufPage);
SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t1 == pBufPage1);
ASSERT_TRUE(pageId == 1);
SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t2 == pBufPage2);
ASSERT_TRUE(pageId == 2);
SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t3 == pBufPage3);
ASSERT_TRUE(pageId == 3);
SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t4 == pBufPage4);
ASSERT_TRUE(pageId == 4);
releaseBufPage(pResultBuf, t4);
releaseBufPage(pBuf, t4);
// flush the written page to disk, and read it out again
SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pResultBuf, writePageId));
SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pBuf, writePageId));
ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
SArray* pa = getDataBufPagesIdList(pBuf, groupId);
ASSERT_EQ(taosArrayGetSize(pa), 5);
destroyDiskbasedBuf(pResultBuf);
destroyDiskbasedBuf(pBuf);
}
void recyclePageTest() {
SDiskbasedBuf* pResultBuf = NULL;
int32_t ret = createDiskbasedBuf(&pResultBuf, 1024, 4*1024, 1, "/tmp/");
SDiskbasedBuf* pBuf = NULL;
int32_t ret = createDiskbasedBuf(&pBuf, 1024, 4*1024, "1", "/tmp/");
int32_t pageId = 0;
int32_t writePageId = 0;
int32_t groupId = 0;
int32_t nx = 12345;
SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* pBufPage = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
ASSERT_TRUE(pBufPage != NULL);
releaseBufPage(pResultBuf, pBufPage);
releaseBufPage(pBuf, pBufPage);
SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage1 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t1 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t1 == pBufPage1);
ASSERT_TRUE(pageId == 1);
SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage2 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t2 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t2 == pBufPage2);
ASSERT_TRUE(pageId == 2);
SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage3 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t3 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t3 == pBufPage3);
ASSERT_TRUE(pageId == 3);
SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage4 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t4 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t4 == pBufPage4);
ASSERT_TRUE(pageId == 4);
releaseBufPage(pResultBuf, t4);
releaseBufPage(pBuf, t4);
SFilePage* pBufPage5 = static_cast<SFilePage*>(getNewBufPage(pResultBuf, groupId, &pageId));
SFilePage* t5 = static_cast<SFilePage*>(getBufPage(pResultBuf, pageId));
SFilePage* pBufPage5 = static_cast<SFilePage*>(getNewBufPage(pBuf, groupId, &pageId));
SFilePage* t5 = static_cast<SFilePage*>(getBufPage(pBuf, pageId));
ASSERT_TRUE(t5 == pBufPage5);
ASSERT_TRUE(pageId == 5);
releaseBufPage(pBuf, t5);
// flush the written page to disk, and read it out again
SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pResultBuf, writePageId));
SFilePage* pBufPagex = static_cast<SFilePage*>(getBufPage(pBuf, writePageId));
*(int32_t*)(pBufPagex->data) = nx;
writePageId = pageId; // update the data
releaseBufPage(pResultBuf, pBufPagex);
releaseBufPage(pBuf, pBufPagex);
SFilePage* pBufPagex1 = static_cast<SFilePage*>(getBufPage(pResultBuf, 1));
SFilePage* pBufPagex1 = static_cast<SFilePage*>(getBufPage(pBuf, 1));
SArray* pa = getDataBufPagesIdList(pResultBuf, groupId);
SArray* pa = getDataBufPagesIdList(pBuf, groupId);
ASSERT_EQ(taosArrayGetSize(pa), 6);
destroyDiskbasedBuf(pResultBuf);
destroyDiskbasedBuf(pBuf);
}
} // namespace

2
tests

@ -1 +1 @@
Subproject commit 08ed39f0a5fcbbfb5a630b945ab3d1998d4b4136
Subproject commit 904e6f0e152e8fe61edfe0a0a9ae497cfde2a72c