xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

[td-225] refactor codes

This commit is contained in:
Haojun Liao 2020-07-30 23:47:06 +08:00
parent ae244c9e84
commit c2ce0fc4f8
4 changed files with 113 additions and 158 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/util/inc/hash.h
View File

@ -29,16 +29,12 @@ extern "C" {
typedef void (*_hash_free_fn_t)(void *param); typedef void (*_hash_free_fn_t)(void *param);
typedef struct SHashNode { typedef struct SHashNode {
char *key; char *key;
// union { struct SHashNode *prev;
struct SHashNode * prev;
// struct SHashEntry *prev1;
// };
//
struct SHashNode *next; struct SHashNode *next;
uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash
uint32_t keyLen; // length of the key uint32_t keyLen; // length of the key
char data[]; char *data;
} SHashNode; } SHashNode;
typedef struct SHashObj { typedef struct SHashObj {
@ -109,6 +105,8 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen);
*/ */
void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen);
void taosHashRemoveNode();
/** /**
* clean up hash table * clean up hash table
* @param handle * @param handle

127
src/util/src/hash.c
View File

@ -19,6 +19,8 @@
#include "tulog.h" #include "tulog.h"
#include "tutil.h" #include "tutil.h"
#define HASH_NEED_RESIZE(_h) ((_h)->size >= (_h)->capacity * HASH_DEFAULT_LOAD_FACTOR)
static FORCE_INLINE void __wr_lock(void *lock) { static FORCE_INLINE void __wr_lock(void *lock) {
if (lock == NULL) { if (lock == NULL) {
return; return;
@ -95,29 +97,19 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) {
* @param hashVal hash value by hash function * @param hashVal hash value by hash function
* @return * @return
*/ */
FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t *hashVal) { FORCE_INLINE SHashNode *doGetNodeFromHashTable(SHashObj *pHashObj, const void *key, uint32_t keyLen, uint32_t hashVal) {
uint32_t hash = (*pHashObj->hashFp)(key, keyLen); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
int32_t slot = HASH_INDEX(hash, pHashObj->capacity);
SHashNode *pNode = pHashObj->hashList[slot]; SHashNode *pNode = pHashObj->hashList[slot];
while (pNode) { while (pNode) {
if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) {
assert(pNode->hashVal == hashVal);
break; break;
} }
pNode = pNode->next; pNode = pNode->next;
} }
if (pNode) {
assert(HASH_INDEX(pNode->hashVal, pHashObj->capacity) == slot);
}
// return the calculated hash value, to avoid calculating it again in other functions
if (hashVal != NULL) {
*hashVal = hash;
}
return pNode; return pNode;
} }
@ -148,7 +140,13 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p
* @param dsize size of actual data * @param dsize size of actual data
* @return hash node * @return hash node
*/ */
static SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize); static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) {
assert(pNode->keyLen == pNewNode->keyLen);
SWAP(pNode->key, pNewNode->key, void*);
SWAP(pNode->data, pNewNode->data, void*);
return pNewNode;
}
/** /**
* insert the hash node at the front of the linked list * insert the hash node at the front of the linked list
@ -217,58 +215,43 @@ size_t taosHashGetSize(const SHashObj *pHashObj) {
} }
int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) { int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size) {
__wr_lock(pHashObj->lock); uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen);
SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal);
if (pNewNode == NULL) {
return -1;
}
uint32_t hashVal = 0; __wr_lock(pHashObj->lock);
SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, &hashVal); SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal);
if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table if (pNode == NULL) { // no data in hash table with the specified key, add it into hash table
taosHashTableResize(pHashObj);
SHashNode *pNewNode = doCreateHashNode(key, keyLen, data, size, hashVal); if (HASH_NEED_RESIZE(pHashObj)) {
if (pNewNode == NULL) { taosHashTableResize(pHashObj);
__unlock(pHashObj->lock);
return -1;
} }
doAddToHashTable(pHashObj, pNewNode); doAddToHashTable(pHashObj, pNewNode);
__unlock(pHashObj->lock);
} else { } else {
SHashNode *pNewNode = doUpdateHashNode(pNode, key, keyLen, data, size); doUpdateHashNode(pNode, pNewNode);
if (pNewNode == NULL) { __unlock(pHashObj->lock);
__unlock(pHashObj->lock);
return -1;
}
if (pNewNode->prev) { tfree(pNewNode->data)
pNewNode->prev->next = pNewNode; tfree(pNewNode);
} else {
int32_t slot = HASH_INDEX(pNewNode->hashVal, pHashObj->capacity);
assert(pHashObj->hashList[slot] == pNode);
pHashObj->hashList[slot] = pNewNode;
}
if (pNewNode->next) {
(pNewNode->next)->prev = pNewNode;
}
} }
__unlock(pHashObj->lock);
return 0; return 0;
} }
void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) { void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen);
__rd_lock(pHashObj->lock); __rd_lock(pHashObj->lock);
SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal);
uint32_t hashVal = 0;
SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, &hashVal);
__unlock(pHashObj->lock); __unlock(pHashObj->lock);
if (pNode != NULL) { if (pNode) {
assert(pNode->hashVal == hashVal); assert(pNode->hashVal == hashVal);
return pNode->data; return pNode->data;
} else { } else {
return NULL; return NULL;
@ -276,10 +259,10 @@ void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen) {
} }
void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
__wr_lock(pHashObj->lock); uint32_t hashVal = (*pHashObj->hashFp)(key, keyLen);
uint32_t val = 0; __wr_lock(pHashObj->lock);
SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, &val); SHashNode *pNode = doGetNodeFromHashTable(pHashObj, key, keyLen, hashVal);
if (pNode == NULL) { if (pNode == NULL) {
__unlock(pHashObj->lock); __unlock(pHashObj->lock);
return; return;
@ -287,7 +270,7 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
SHashNode *pNext = pNode->next; SHashNode *pNext = pNode->next;
if (pNode->prev == NULL) { if (pNode->prev == NULL) {
int32_t slot = HASH_INDEX(val, pHashObj->capacity); int32_t slot = HASH_INDEX(hashVal, pHashObj->capacity);
assert(pHashObj->hashList[slot] == pNode); assert(pHashObj->hashList[slot] == pNode);
pHashObj->hashList[slot] = pNext; pHashObj->hashList[slot] = pNext;
@ -299,13 +282,13 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) {
pNext->prev = pNode->prev; pNext->prev = pNode->prev;
} }
pHashObj->size--; pHashObj->size -= 1;
__unlock(pHashObj->lock);
pNode->next = NULL; pNode->next = NULL;
pNode->prev = NULL; pNode->prev = NULL;
tfree(pNode); tfree(pNode);
__unlock(pHashObj->lock);
} }
void taosHashCleanup(SHashObj *pHashObj) { void taosHashCleanup(SHashObj *pHashObj) {
@ -341,14 +324,6 @@ void taosHashCleanup(SHashObj *pHashObj) {
free(pHashObj); free(pHashObj);
} }
void taosHashSetFreecb(SHashObj *pHashObj, _hash_free_fn_t freeFp) {
if (pHashObj == NULL || freeFp == NULL) {
return;
}
pHashObj->freeFp = freeFp;
}
SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) { SHashMutableIterator *taosHashCreateIter(SHashObj *pHashObj) {
SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator)); SHashMutableIterator *pIter = calloc(1, sizeof(SHashMutableIterator));
if (pIter == NULL) { if (pIter == NULL) {
@ -528,38 +503,20 @@ void taosHashTableResize(SHashObj *pHashObj) {
} }
SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) {
size_t totalSize = dsize + sizeof(SHashNode) + keyLen; SHashNode *pNewNode = calloc(1, sizeof(SHashNode));
SHashNode *pNewNode = calloc(1, totalSize);
if (pNewNode == NULL) { if (pNewNode == NULL) {
uError("failed to allocate memory, reason:%s", strerror(errno)); uError("failed to allocate memory, reason:%s", strerror(errno));
return NULL; return NULL;
} }
memcpy(pNewNode->data, pData, dsize);
pNewNode->key = pNewNode->data + dsize;
memcpy(pNewNode->key, key, keyLen);
pNewNode->keyLen = keyLen;
pNewNode->hashVal = hashVal;
return pNewNode;
}
SHashNode *doUpdateHashNode(SHashNode *pNode, const void *key, size_t keyLen, const void *pData, size_t dsize) { pNewNode->data = malloc(dsize + keyLen);
size_t size = dsize + sizeof(SHashNode) + keyLen;
SHashNode *pNewNode = (SHashNode *)realloc(pNode, size);
if (pNewNode == NULL) {
return NULL;
}
memcpy(pNewNode->data, pData, dsize); memcpy(pNewNode->data, pData, dsize);
pNewNode->key = pNewNode->data + dsize; pNewNode->key = pNewNode->data + dsize;
assert(memcmp(pNewNode->key, key, keyLen) == 0 && keyLen == pNewNode->keyLen);
memcpy(pNewNode->key, key, keyLen); memcpy(pNewNode->key, key, keyLen);
pNewNode->keyLen = keyLen;
pNewNode->hashVal = hashVal;
return pNewNode; return pNewNode;
} }

130
src/util/src/tcache.c
View File

@ -147,49 +147,18 @@ static FORCE_INLINE void taosCacheMoveToTrash(SCacheObj *pCacheObj, SCacheDataNo
* @param dataSize * @param dataSize
* @return * @return
*/ */
static SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode *pNode, const char *key, int32_t keyLen, static UNUSED_FUNC SCacheDataNode *taosUpdateCacheImpl(SCacheObj *pCacheObj, SCacheDataNode* pNode, SCacheDataNode* pNewNode,
const void *pData, uint32_t dataSize, uint64_t duration) { const char *key, int32_t keyLen) {
SCacheDataNode *pNewNode = NULL;
// only a node is not referenced by any other object, in-place update it // only a node is not referenced by any other object, in-place update it
if (T_REF_VAL_GET(pNode) == 0) { if (T_REF_VAL_GET(pNode) > 0) {
size_t newSize = sizeof(SCacheDataNode) + dataSize + keyLen + 1;
pNewNode = (SCacheDataNode *)realloc(pNode, newSize);
if (pNewNode == NULL) {
return NULL;
}
memset(pNewNode, 0, newSize);
pNewNode->signature = (uint64_t)pNewNode;
memcpy(pNewNode->data, pData, dataSize);
pNewNode->key = (char *)pNewNode + sizeof(SCacheDataNode) + dataSize;
pNewNode->keySize = keyLen;
memcpy(pNewNode->key, key, keyLen);
// update the timestamp information for updated key/value
pNewNode->addedTime = taosGetTimestampMs();
pNewNode->lifespan = duration;
T_REF_INC(pNewNode);
// the address of this node may be changed, so the prev and next element should update the corresponding pointer
taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *));
} else {
taosCacheMoveToTrash(pCacheObj, pNode); taosCacheMoveToTrash(pCacheObj, pNode);
pNewNode = taosCreateCacheNode(key, keyLen, pData, dataSize, duration);
if (pNewNode == NULL) {
return NULL;
}
T_REF_INC(pNewNode);
// addedTime new element to hashtable
taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *));
} }
T_REF_INC(pNewNode);
// addedTime new element to hashtable
taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNewNode, sizeof(void *));
return pNewNode; return pNewNode;
} }
@ -238,7 +207,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
return NULL; return NULL;
} }
pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), false); pCacheObj->pHashTable = taosHashInit(128, taosGetDefaultHashFunction(keyType), true);
pCacheObj->name = strdup(cacheName); pCacheObj->name = strdup(cacheName);
if (pCacheObj->pHashTable == NULL) { if (pCacheObj->pHashTable == NULL) {
free(pCacheObj); free(pCacheObj);
@ -270,36 +239,59 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
} }
void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) {
SCacheDataNode *pNode; // SCacheDataNode *pNode = NULL;
if (pCacheObj == NULL || pCacheObj->pHashTable == NULL) { if (pCacheObj == NULL || pCacheObj->pHashTable == NULL) {
return NULL; return NULL;
} }
SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, duration);
if (pNode1 == NULL) {
uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key);
return NULL;
}
__cache_wr_lock(pCacheObj); __cache_wr_lock(pCacheObj);
SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); SCacheDataNode **pt = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen);
SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL; SCacheDataNode * pOld = (pt != NULL) ? (*pt) : NULL;
if (pOld == NULL) { // do addedTime to cache if (pOld == NULL) { // do addedTime to cache
pNode = taosAddToCacheImpl(pCacheObj, key, keyLen, pData, dataSize, duration * 1000L); T_REF_INC(pNode1);
if (NULL != pNode) { taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *));
pCacheObj->totalSize += pNode->size; __cache_unlock(pCacheObj);
uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64 atomic_add_fetch_64(&pCacheObj->totalSize, pNode1->size);
"bytes size:%" PRId64 "bytes", uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64
pCacheObj->name, key, pNode->data, pNode->addedTime, pNode->expireTime, "bytes size:%" PRId64 "bytes",
(int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize); pCacheObj->name, key, pNode1->data, pNode1->addedTime, pNode1->expireTime,
} else { (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize);
uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key);
}
} else { // old data exists, update the node } else { // old data exists, update the node
pNode = taosUpdateCacheImpl(pCacheObj, pOld, key, keyLen, pData, dataSize, duration * 1000L);
uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode->data, pOld); bool addToTrashcan = false;
if (T_REF_VAL_GET(pOld) > 0) {
// todo removed by node, instead of by key
taosHashRemove(pCacheObj->pHashTable, pOld->key, pOld->keySize);
} else {
addToTrashcan = true;
}
T_REF_INC(pNode1);
// addedTime new element to hashtable
taosHashPut(pCacheObj->pHashTable, key, keyLen, &pNode1, sizeof(void *));
__cache_unlock(pCacheObj);
// todo add trashcan lock
if (addToTrashcan) {
taosAddToTrash(pCacheObj, pOld);
} else {
tfree(pOld);
}
uDebug("cache:%s, key:%p, %p exist in cache, updated old:%p", pCacheObj->name, key, pNode1->data, pOld);
} }
__cache_unlock(pCacheObj); return pNode1->data;
return (pNode != NULL) ? pNode->data : NULL;
} }
void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) { void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen) {
@ -310,7 +302,6 @@ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen
void *pData = NULL; void *pData = NULL;
__cache_rd_lock(pCacheObj); __cache_rd_lock(pCacheObj);
SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen); SCacheDataNode **ptNode = (SCacheDataNode **)taosHashGet(pCacheObj->pHashTable, key, keyLen);
int32_t ref = 0; int32_t ref = 0;
@ -439,21 +430,33 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) {
* that tries to do the same thing. * that tries to do the same thing.
*/ */
if (pNode->inTrashCan) { if (pNode->inTrashCan) {
__cache_unlock(pCacheObj);
if (ref == 0) { if (ref == 0) {
assert(pNode->pTNodeHeader->pData == pNode); assert(pNode->pTNodeHeader->pData == pNode);
taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader); taosRemoveFromTrashCan(pCacheObj, pNode->pTNodeHeader);
} }
} else { } else {
taosHashRemove(pCacheObj->pHashTable, pNode->key, pNode->keySize);
__cache_unlock(pCacheObj);
if (ref > 0) { if (ref > 0) {
assert(pNode->pTNodeHeader == NULL); assert(pNode->pTNodeHeader == NULL);
taosCacheMoveToTrash(pCacheObj, pNode);
// todo trashcan lock
taosAddToTrash(pCacheObj, pNode);
} else { } else {
taosCacheReleaseNode(pCacheObj, pNode); // taosCacheReleaseNode(pCacheObj, pNode);
atomic_fetch_sub_ptr(&pCacheObj->totalSize, pNode->size);
uDebug("cache:%s, key:%p, %p is destroyed from cache, totalNum:%d totalSize:%" PRId64 "bytes size:%dbytes",
pCacheObj->name, pNode->key, pNode->data, (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize,
pNode->size);
if (pCacheObj->freeFp) pCacheObj->freeFp(pNode->data);
free(pNode);
} }
} }
__cache_unlock(pCacheObj);
} else { } else {
// NOTE: once refcount is decrease, pNode may be freed by other thread immediately. // NOTE: once refcount is decrease, pNode may be freed by other thread immediately.
int32_t ref = T_REF_DEC(pNode); int32_t ref = T_REF_DEC(pNode);
@ -497,8 +500,7 @@ void taosCacheCleanup(SCacheObj *pCacheObj) {
doCleanupDataCache(pCacheObj); doCleanupDataCache(pCacheObj);
} }
SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, SCacheDataNode *taosCreateCacheNode(const char *key, size_t keyLen, const char *pData, size_t size, uint64_t duration) {
uint64_t duration) {
size_t totalSize = size + sizeof(SCacheDataNode) + keyLen; size_t totalSize = size + sizeof(SCacheDataNode) + keyLen;
SCacheDataNode *pNewNode = calloc(1, totalSize); SCacheDataNode *pNewNode = calloc(1, totalSize);

2
src/vnode/src/vnodeRead.c
View File

@ -207,7 +207,6 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle); vDebug("vgId:%d, QInfo:%p, start to build result rsp after query paused, %p", pVnode->vgId, *handle, pReadMsg->rpcMsg.handle);
code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle); code = vnodeDumpQueryResult(&pReadMsg->rspRet, pVnode, *handle, &freehandle);
freehandle = false;
// todo test the error code case // todo test the error code case
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
code = TSDB_CODE_QRY_HAS_RSP; code = TSDB_CODE_QRY_HAS_RSP;
@ -267,7 +266,6 @@ static int32_t vnodeProcessFetchMsg(SVnodeObj *pVnode, SReadMsg *pReadMsg) {
} }
code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle); code = vnodeDumpQueryResult(pRet, pVnode, *handle, &freeHandle);
freeHandle = false;
} }
qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle); qReleaseQInfo(pVnode->qMgmt, (void**) &handle, freeHandle);