xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix invalid write in sdb

This commit is contained in:
Shengliang Guan 2021-12-28 22:35:23 -08:00
parent 86e7dd3a61
commit 97f7018f5b
10 changed files with 122 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
include/dnode/mnode/sdb/sdb.h
View File

@ -25,7 +25,7 @@ extern "C" {
#define SDB_GET_INT64(pData, pRow, dataPos, val) \
{ \
if (sdbGetRawInt64(pRaw, dataPos, val) != 0) { \
sdbFreeRow(pRow); \
tfree(pRow); \
return NULL; \
} \
dataPos += sizeof(int64_t); \
@ -34,7 +34,7 @@ extern "C" {
#define SDB_GET_INT32(pData, pRow, dataPos, val) \
{ \
if (sdbGetRawInt32(pRaw, dataPos, val) != 0) { \
sdbFreeRow(pRow); \
tfree(pRow); \
return NULL; \
} \
dataPos += sizeof(int32_t); \
@ -43,7 +43,7 @@ extern "C" {
#define SDB_GET_INT16(pData, pRow, dataPos, val) \
{ \
if (sdbGetRawInt16(pRaw, dataPos, val) != 0) { \
sdbFreeRow(pRow); \
tfree(pRow); \
return NULL; \
} \
dataPos += sizeof(int16_t); \
@ -52,7 +52,7 @@ extern "C" {
#define SDB_GET_INT8(pData, pRow, dataPos, val) \
{ \
if (sdbGetRawInt8(pRaw, dataPos, val) != 0) { \
sdbFreeRow(pRow); \
tfree(pRow); \
return NULL; \
} \
dataPos += sizeof(int8_t); \
@ -61,7 +61,7 @@ extern "C" {
#define SDB_GET_BINARY(pRaw, pRow, dataPos, val, valLen) \
{ \
if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \
sdbFreeRow(pRow); \
tfree(pRow); \
return NULL; \
} \
dataPos += valLen; \
@ -71,7 +71,7 @@ extern "C" {
{ \
char val[valLen] = {0}; \
if (sdbGetRawBinary(pRaw, dataPos, val, valLen) != 0) { \
sdbFreeRow(pRow); \
tfree(pRow); \
return NULL; \
} \
dataPos += valLen; \
@ -325,7 +325,7 @@ int32_t sdbGetRawSoftVer(SSdbRaw *pRaw, int8_t *sver);
int32_t sdbGetRawTotalSize(SSdbRaw *pRaw);
SSdbRow *sdbAllocRow(int32_t objSize);
void sdbFreeRow(SSdbRow *pRow);
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow);
void *sdbGetRowObj(SSdbRow *pRow);
#ifdef __cplusplus

10
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -118,17 +118,17 @@ static SConnObj *mndCreateConn(SMnode *pMnode, SRpcConnInfo *pInfo, int32_t pid,
SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000);
if (pConn == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("conn:%d, data:%p failed to put into cache since %s, user:%s", connId, pConn, pInfo->user, terrstr());
mError("conn:%d, failed to put into cache since %s, user:%s", connId, pInfo->user, terrstr());
return NULL;
} else {
mTrace("conn:%d, data:%p created, user:%s", pConn->id, pConn, pInfo->user);
mTrace("conn:%d, is created, data:%p user:%s", pConn->id, pConn, pInfo->user);
return pConn;
}
}
static void mndFreeConn(SConnObj *pConn) {
tfree(pConn->pQueries);
mTrace("conn:%d, data:%p destroyed", pConn->id, pConn);
mTrace("conn:%d, is destroyed, data:%p", pConn->id, pConn);
}
static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) {
@ -143,13 +143,13 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId) {
int32_t keepTime = pMnode->cfg.shellActivityTimer * 3;
pConn->lastAccessTimeMs = keepTime * 1000 + (uint64_t)taosGetTimestampMs();
mTrace("conn:%d, data:%p acquired from cache", pConn->id, pConn);
mTrace("conn:%d, acquired from cache, data:%p", pConn->id, pConn);
return pConn;
}
static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) {
if (pConn == NULL) return;
mTrace("conn:%d, data:%p released from cache", pConn->id, pConn);
mTrace("conn:%d, released from cache, data:%p", pConn->id, pConn);
SProfileMgmt *pMgmt = &pMnode->profileMgmt;
taosCacheRelease(pMgmt->cache, (void **)&pConn, false);

62
source/dnode/mnode/impl/src/mndShow.c
View File

@ -55,32 +55,25 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) {
int32_t showId = atomic_add_fetch_32(&pMgmt->showId, 1);
if (showId == 0) atomic_add_fetch_32(&pMgmt->showId, 1);
int32_t size = sizeof(SShowObj) + pMsg->payloadLen;
SShowObj *pShow = calloc(1, size);
if (pShow != NULL) {
pShow->id = showId;
pShow->pMnode = pMnode;
pShow->type = pMsg->type;
pShow->payloadLen = pMsg->payloadLen;
memcpy(pShow->db, pMsg->db, TSDB_DB_FNAME_LEN);
memcpy(pShow->payload, pMsg->payload, pMsg->payloadLen);
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("failed to process show-meta msg:%s since %s", mndShowStr(pMsg->type), terrstr());
return NULL;
}
int32_t size = sizeof(SShowObj) + pMsg->payloadLen;
SShowObj showObj = {0};
showObj.id = showId;
showObj.pMnode = pMnode;
showObj.type = pMsg->type;
showObj.payloadLen = pMsg->payloadLen;
memcpy(showObj.db, pMsg->db, TSDB_DB_FNAME_LEN);
memcpy(showObj.payload, pMsg->payload, pMsg->payloadLen);
int32_t keepTime = pMnode->cfg.shellActivityTimer * 6 * 1000;
SShowObj *pShowRet = taosCachePut(pMgmt->cache, &showId, sizeof(int32_t), pShow, size, keepTime);
free(pShow);
if (pShowRet == NULL) {
SShowObj *pShow = taosCachePut(pMgmt->cache, &showId, sizeof(int32_t), &showObj, size, keepTime);
if (pShow == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:%d, failed to put into cache since %s", showId, terrstr());
return NULL;
} else {
mTrace("show:%d, data:%p created", showId, pShowRet);
return pShowRet;
}
mTrace("show:%d, is created, data:%p", showId, pShow);
return pShow;
}
static void mndFreeShowObj(SShowObj *pShow) {
@ -94,7 +87,7 @@ static void mndFreeShowObj(SShowObj *pShow) {
}
}
mTrace("show:%d, data:%p destroyed", pShow->id, pShow);
mTrace("show:%d, is destroyed, data:%p", pShow->id, pShow);
}
static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) {
@ -106,14 +99,14 @@ static SShowObj *mndAcquireShowObj(SMnode *pMnode, int32_t showId) {
return NULL;
}
mTrace("show:%d, data:%p acquired from cache", pShow->id, pShow);
mTrace("show:%d, acquired from cache, data:%p", pShow->id, pShow);
return pShow;
}
static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) {
if (pShow == NULL) return;
mTrace("show:%d, data:%p released from cache, force:%d", pShow->id, pShow, forceRemove);
mTrace("show:%d, released from cache, data:%p force:%d", pShow->id, pShow, forceRemove);
// A bug in tcache.c
forceRemove = 0;
@ -158,8 +151,8 @@ static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) {
}
int32_t code = (*metaFp)(pMnodeMsg, pShow, &pRsp->tableMeta);
mDebug("show:%d, data:%p get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow,
pShow->numOfRows, pShow->numOfColumns, mndShowStr(type), tstrerror(code));
mDebug("show:%d, get meta finished, numOfRows:%d cols:%d type:%s result:%s", pShow->id, pShow->numOfRows,
pShow->numOfColumns, mndShowStr(type), tstrerror(code));
if (code == TSDB_CODE_SUCCESS) {
pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns;
@ -195,16 +188,15 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
if (retrieveFp == NULL) {
mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
mError("show:%d, data:%p failed to retrieve data since %s", pShow->id, pShow, terrstr());
mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr());
return -1;
}
mDebug("show:%d, data:%p start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow,
pShow->numOfReads, pShow->numOfRows, mndShowStr(pShow->type));
mDebug("show:%d, start retrieve data, numOfReads:%d numOfRows:%d type:%s", pShow->id, pShow->numOfReads,
pShow->numOfRows, mndShowStr(pShow->type));
if (mndCheckRetrieveFinished(pShow)) {
mDebug("show:%d, data:%p read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow, pShow->numOfReads,
pShow->numOfRows);
mDebug("show:%d, read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows);
pShow->numOfReads = pShow->numOfRows;
}
@ -227,7 +219,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
if (pRsp == NULL) {
mndReleaseShowObj(pShow, false);
terrno = TSDB_CODE_OUT_OF_MEMORY;
mError("show:%d, data:%p failed to retrieve data since %s", pShow->id, pShow, terrstr());
mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr());
return -1;
}
@ -236,7 +228,7 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead);
}
mDebug("show:%d, data:%p stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, pShow, rowsRead, rowsToRead);
mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead);
pRsp->numOfRows = htonl(rowsRead);
pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision
@ -246,10 +238,10 @@ static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) {
if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) {
pRsp->completed = 1;
mDebug("show:%d, data:%p retrieve completed", pShow->id, pShow);
mDebug("show:%d, retrieve completed", pShow->id);
mndReleaseShowObj(pShow, true);
} else {
mDebug("show:%d, data:%p retrieve not completed yet", pShow->id, pShow);
mDebug("show:%d, retrieve not completed yet", pShow->id);
mndReleaseShowObj(pShow, false);
}

14
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -294,18 +294,18 @@ TRANS_DECODE_OVER:
return NULL;
}
mTrace("trans:%d, decode from raw:%p", pTrans->id, pRaw);
mTrace("trans:%d, decode from raw:%p, data:%p", pTrans->id, pRaw, pTrans);
return pRow;
}
static int32_t mndTransActionInsert(SSdb *pSdb, STrans *pTrans) {
pTrans->stage = TRN_STAGE_PREPARE;
mTrace("trans:%d, perform insert action", pTrans->id);
mTrace("trans:%d, perform insert action, data:%p", pTrans->id, pTrans);
return 0;
}
static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
mTrace("trans:%d, perform delete action", pTrans->id);
mTrace("trans:%d, perform delete action, data:%p", pTrans->id, pTrans);
mndTransDropLogs(pTrans->redoLogs);
mndTransDropLogs(pTrans->undoLogs);
@ -317,7 +317,7 @@ static int32_t mndTransActionDelete(SSdb *pSdb, STrans *pTrans) {
}
static int32_t mndTransActionUpdate(SSdb *pSdb, STrans *pOldTrans, STrans *pNewTrans) {
mTrace("trans:%d, perform update action", pOldTrans->id);
mTrace("trans:%d, perform update action, data:%p", pOldTrans->id, pOldTrans);
pOldTrans->stage = pNewTrans->stage;
return 0;
}
@ -362,14 +362,14 @@ STrans *mndTransCreate(SMnode *pMnode, ETrnPolicy policy, SRpcMsg *pMsg) {
return NULL;
}
mDebug("trans:%d, is created", pTrans->id);
mDebug("trans:%d, is created, data:%p", pTrans->id, pTrans);
return pTrans;
}
static void mndTransDropLogs(SArray *pArray) {
for (int32_t i = 0; i < pArray->size; ++i) {
SSdbRaw *pRaw = taosArrayGetP(pArray, i);
tfree(pRaw);
sdbFreeRaw(pRaw);
}
taosArrayDestroy(pArray);
@ -391,7 +391,7 @@ void mndTransDrop(STrans *pTrans) {
mndTransDropActions(pTrans->redoActions);
mndTransDropActions(pTrans->undoActions);
// mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
mDebug("trans:%d, is dropped, data:%p", pTrans->id, pTrans);
tfree(pTrans);
}

1
source/dnode/mnode/sdb/inc/sdbInt.h
View File

@ -72,6 +72,7 @@ typedef struct SSdb {
} SSdb;
int32_t sdbWriteFile(SSdb *pSdb);
void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper);
#ifdef __cplusplus
}

8
source/dnode/mnode/sdb/src/sdb.c
View File

@ -80,16 +80,12 @@ void sdbCleanup(SSdb *pSdb) {
SHashObj *hash = pSdb->hashObjs[i];
if (hash == NULL) continue;
SdbDeleteFp deleteFp = pSdb->deleteFps[i];
SSdbRow **ppRow = taosHashIterate(hash, NULL);
SSdbRow **ppRow = taosHashIterate(hash, NULL);
while (ppRow != NULL) {
SSdbRow *pRow = *ppRow;
if (pRow == NULL) continue;
if (deleteFp != NULL) {
(*deleteFp)(pSdb, pRow->pObj);
}
sdbFreeRow(pRow);
sdbFreeRow(pSdb, pRow);
ppRow = taosHashIterate(hash, ppRow);
}
}

79
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -16,6 +16,50 @@
#define _DEFAULT_SOURCE
#include "sdbInt.h"
static const char *sdbTableName(ESdbType type) {
switch (type) {
case SDB_TRANS:
return "trans";
case SDB_CLUSTER:
return "cluster";
case SDB_MNODE:
return "mnode";
case SDB_DNODE:
return "dnode";
case SDB_USER:
return "user";
case SDB_AUTH:
return "auth";
case SDB_ACCT:
return "acct";
case SDB_TOPIC:
return "topic";
case SDB_VGROUP:
return "vgId";
case SDB_STB:
return "stb";
case SDB_DB:
return "db";
case SDB_FUNC:
return "func";
default:
return "undefine";
}
}
void sdbPrintOper(SSdb *pSdb, SSdbRow *pRow, const char *oper) {
EKeyType keyType = pSdb->keyTypes[pRow->type];
if (keyType == SDB_KEY_BINARY) {
mTrace("%s:%s, refCount:%d oper:%s", sdbTableName(pRow->type), (char *)pRow->pObj, pRow->refCount, oper);
} else if (keyType == SDB_KEY_INT32) {
mTrace("%s:%d, refCount:%d oper:%s", sdbTableName(pRow->type), *(int32_t *)pRow->pObj, pRow->refCount, oper);
} else if (keyType == SDB_KEY_INT64) {
mTrace("%s:%" PRId64 ", refCount:%d oper:%s", sdbTableName(pRow->type), *(int64_t *)pRow->pObj, pRow->refCount, oper);
} else {
}
}
static SHashObj *sdbGetHash(SSdb *pSdb, int32_t type) {
if (type >= SDB_MAX || type <= SDB_START) {
terrno = TSDB_CODE_SDB_INVALID_TABLE_TYPE;
@ -55,17 +99,18 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SSdbRow *pOldRow = taosHashGet(hash, pRow->pObj, keySize);
if (pOldRow != NULL) {
taosWUnLockLatch(pLock);
sdbFreeRow(pRow);
sdbFreeRow(pSdb, pRow);
terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
return terrno;
}
pRow->refCount = 1;
pRow->refCount = 0;
pRow->status = pRaw->status;
sdbPrintOper(pSdb, pRow, "insertRow");
if (taosHashPut(hash, pRow->pObj, keySize, &pRow, sizeof(void *)) != 0) {
taosWUnLockLatch(pLock);
sdbFreeRow(pRow);
sdbFreeRow(pSdb, pRow);
terrno = TSDB_CODE_SDB_OBJ_ALREADY_THERE;
return terrno;
}
@ -83,7 +128,7 @@ static int32_t sdbInsertRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosWLockLatch(pLock);
taosHashRemove(hash, pRow->pObj, keySize);
taosWUnLockLatch(pLock);
sdbFreeRow(pRow);
sdbFreeRow(pSdb, pRow);
terrno = code;
return terrno;
}
@ -113,7 +158,7 @@ static int32_t sdbUpdateRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
code = (*updateFp)(pSdb, pOldRow->pObj, pNewRow->pObj);
}
sdbFreeRow(pNewRow);
sdbFreeRow(pSdb, pNewRow);
return code;
}
@ -123,14 +168,10 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
SRWLatch *pLock = &pSdb->locks[pRow->type];
taosWLockLatch(pLock);
// remove attached object such as trans
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
if (deleteFp != NULL) (*deleteFp)(pSdb, pRow->pObj);
SSdbRow **ppOldRow = taosHashGet(hash, pRow->pObj, keySize);
if (ppOldRow == NULL || *ppOldRow == NULL) {
taosWUnLockLatch(pLock);
sdbFreeRow(pRow);
sdbFreeRow(pSdb, pRow);
terrno = TSDB_CODE_SDB_OBJ_NOT_THERE;
return terrno;
}
@ -140,8 +181,8 @@ static int32_t sdbDeleteRow(SSdb *pSdb, SHashObj *hash, SSdbRaw *pRaw, SSdbRow *
taosHashRemove(hash, pOldRow->pObj, keySize);
taosWUnLockLatch(pLock);
sdbRelease(pSdb, pOldRow->pObj);
sdbFreeRow(pRow);
// sdbRelease(pSdb, pOldRow->pObj);
sdbFreeRow(pSdb, pRow);
return code;
}
@ -206,6 +247,7 @@ void *sdbAcquire(SSdb *pSdb, ESdbType type, void *pKey) {
case SDB_STATUS_UPDATING:
atomic_add_fetch_32(&pRow->refCount, 1);
pRet = pRow->pObj;
sdbPrintOper(pSdb, pRow, "acquireRow");
break;
case SDB_STATUS_CREATING:
terrno = TSDB_CODE_SDB_OBJ_CREATING;
@ -232,13 +274,9 @@ void sdbRelease(SSdb *pSdb, void *pObj) {
taosRLockLatch(pLock);
int32_t ref = atomic_sub_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "releaseRow");
if (ref <= 0 && pRow->status == SDB_STATUS_DROPPED) {
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
if (deleteFp != NULL) {
(*deleteFp)(pSdb, pRow->pObj);
}
sdbFreeRow(pRow);
sdbFreeRow(pSdb, pRow);
}
taosRUnLockLatch(pLock);
@ -255,9 +293,9 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
if (pIter != NULL) {
SSdbRow *pLastRow = *(SSdbRow **)pIter;
int32_t ref = atomic_sub_fetch_32(&pLastRow->refCount, 1);
int32_t ref = atomic_load_32(&pLastRow->refCount);
if (ref <= 0 && pLastRow->status == SDB_STATUS_DROPPED) {
sdbFreeRow(pLastRow);
sdbFreeRow(pSdb, pLastRow);
}
}
@ -270,6 +308,7 @@ void *sdbFetch(SSdb *pSdb, ESdbType type, void *pIter, void **ppObj) {
}
atomic_add_fetch_32(&pRow->refCount, 1);
sdbPrintOper(pSdb, pRow, "fetchRow");
*ppObj = pRow->pObj;
break;
}

4
source/dnode/mnode/sdb/src/sdbRaw.c
View File

@ -27,12 +27,12 @@ SSdbRaw *sdbAllocRaw(ESdbType type, int8_t sver, int32_t dataLen) {
pRaw->sver = sver;
pRaw->dataLen = dataLen;
// mTrace("raw:%p, is created, len:%d", pRaw, dataLen);
mTrace("raw:%p, is created, len:%d", pRaw, dataLen);
return pRaw;
}
void sdbFreeRaw(SSdbRaw *pRaw) {
// mTrace("raw:%p, is freed", pRaw);
mTrace("raw:%p, is freed", pRaw);
free(pRaw);
}

11
source/dnode/mnode/sdb/src/sdbRow.c
View File

@ -35,4 +35,13 @@ void *sdbGetRowObj(SSdbRow *pRow) {
return pRow->pObj;
}
void sdbFreeRow(SSdbRow *pRow) { tfree(pRow); }
void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow) {
// remove attached object such as trans
SdbDeleteFp deleteFp = pSdb->deleteFps[pRow->type];
if (deleteFp != NULL) {
(*deleteFp)(pSdb, pRow->pObj);
}
sdbPrintOper(pSdb, pRow, "freeRow");
tfree(pRow);
}

4
tests/script/sh/exec.sh
View File

@ -17,7 +17,7 @@ OS_TYPE=`$UNAME_BIN`
NODE_NAME=
EXEC_OPTON=
CLEAR_OPTION="false"
while getopts "n:s:u:x:ct" arg
while getopts "n:s:u:x:cv" arg
do
case $arg in
n)
@ -29,7 +29,7 @@ do
c)
CLEAR_OPTION="clear"
;;
t)
v)
SHELL_OPTION="true"
;;
u)