xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/dnode3

This commit is contained in:
Shengliang Guan 2022-01-05 20:22:36 -08:00
parent db2dbe5f51 cf0ecbae0e
commit d2238d38d4
41 changed files with 1274 additions and 930 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

223
Jenkinsfile vendored
View File

@ -110,227 +110,12 @@ pipeline {
cd ${WKC}/tests cd ${WKC}/tests
./test-all.sh b1fq ./test-all.sh b1fq
''' '''
sh'''
cd ${WKC}/debug
ctest
'''
} }
} }
// stage('Parallel test stage') {
// skip defaultCheckout
// options { skipDefaultCheckout() }
// when {
// allOf{
// changeRequest()
// }
// }
// parallel {
// stage('python_1_s1') {
// agent{label " slave1 || slave11 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p1
// // date'''
// // }
// }
// }
// stage('python_2_s5') {
// agent{label " slave5 || slave15 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p2
// // date'''
// // }
// }
// }
// stage('python_3_s6') {
// agent{label " slave6 || slave16 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p3
// // date'''
// // }
// }
// }
// stage('test_b1_s2') {
// agent{label " slave2 || slave12 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // rm -rf /var/lib/taos/*
// // rm -rf /var/log/taos/*
// // nohup taosd >/dev/null &
// // sleep 10
// // '''
// // sh '''
// // cd ${WKC}/tests/examples/nodejs
// // npm install td2.0-connector > /dev/null 2>&1
// // node nodejsChecker.js host=localhost
// // node test1970.js
// // cd ${WKC}/tests/connectorTest/nodejsTest/nanosupport
// // npm install td2.0-connector > /dev/null 2>&1
// // node nanosecondTest.js
// // '''
// // sh '''
// // cd ${WKC}/tests/examples/C#/taosdemo
// // mcs -out:taosdemo *.cs > /dev/null 2>&1
// // echo '' |./taosdemo -c /etc/taos
// // cd ${WKC}/tests/connectorTest/C#Test/nanosupport
// // mcs -out:nano *.cs > /dev/null 2>&1
// // echo '' |./nano
// // '''
// // sh '''
// // cd ${WKC}/tests/gotest
// // bash batchtest.sh
// // '''
// // sh '''
// // cd ${WKC}/tests
// // ./test-all.sh b1fq
// // date'''
// // }
// }
// }
// stage('test_crash_gen_s3') {
// agent{label " slave3 || slave13 "}
// steps {
// pre_test()
// // timeout(time: 60, unit: 'MINUTES'){
// // sh '''
// // cd ${WKC}/tests/pytest
// // ./crash_gen.sh -a -p -t 4 -s 2000
// // '''
// // }
// // timeout(time: 60, unit: 'MINUTES'){
// // // sh '''
// // // cd ${WKC}/tests/pytest
// // // rm -rf /var/lib/taos/*
// // // rm -rf /var/log/taos/*
// // // ./handle_crash_gen_val_log.sh
// // // '''
// // sh '''
// // cd ${WKC}/tests/pytest
// // rm -rf /var/lib/taos/*
// // rm -rf /var/log/taos/*
// // ./handle_taosd_val_log.sh
// // '''
// // }
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b2fq
// // date
// // '''
// // }
// }
// }
// stage('test_valgrind_s4') {
// agent{label " slave4 || slave14 "}
// steps {
// pre_test()
// // catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// // sh '''
// // cd ${WKC}/tests/pytest
// // ./valgrind-test.sh 2>&1 > mem-error-out.log
// // ./handle_val_log.sh
// // '''
// // }
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b3fq
// // date'''
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh full example
// // date'''
// // }
// }
// }
// stage('test_b4_s7') {
// agent{label " slave7 || slave17 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b4fq
// // cd ${WKC}/tests
// // ./test-all.sh p4
// // cd ${WKC}/tests
// // ./test-all.sh full jdbc
// // cd ${WKC}/tests
// // ./test-all.sh full unit
// // date'''
// // }
// }
// }
// stage('test_b5_s8') {
// agent{label " slave8 || slave18 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b5fq
// // date'''
// // }
// }
// }
// stage('test_b6_s9') {
// agent{label " slave9 || slave19 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b6fq
// // date'''
// // }
// }
// }
// stage('test_b7_s10') {
// agent{label " slave10 || slave20 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b7fq
// // date'''
// // }
// }
// }
// }
// }
} }
post { post {
success { success {

96
include/common/tmsg.h
View File

@ -312,14 +312,23 @@ typedef struct SEpSet {
} SEpSet; } SEpSet;
static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) {
if (buf == NULL) return sizeof(SEpSet); int tlen = 0;
memcpy(buf, pEp, sizeof(SEpSet)); tlen += taosEncodeFixedI8(buf, pEp->inUse);
// TODO: endian conversion tlen += taosEncodeFixedI8(buf, pEp->numOfEps);
return sizeof(SEpSet); for (int i = 0; i < TSDB_MAX_REPLICA; i++) {
tlen += taosEncodeFixedU16(buf, pEp->port[i]);
tlen += taosEncodeString(buf, pEp->fqdn[i]);
}
return tlen;
} }
static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEpSet) { static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) {
memcpy(pEpSet, buf, sizeof(SEpSet)); buf = taosDecodeFixedI8(buf, &pEp->inUse);
buf = taosDecodeFixedI8(buf, &pEp->numOfEps);
for (int i = 0; i < TSDB_MAX_REPLICA; i++) {
buf = taosDecodeFixedU16(buf, &pEp->port[i]);
buf = taosDecodeStringTo(buf, pEp->fqdn[i]);
}
return buf; return buf;
} }
@ -1083,8 +1092,8 @@ typedef struct {
static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) { static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) {
int tlen = 0; int tlen = 0;
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeFixedI8(buf, pReq->igExists); tlen += taosEncodeFixedI8(buf, pReq->igExists);
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeString(buf, pReq->physicalPlan); tlen += taosEncodeString(buf, pReq->physicalPlan);
tlen += taosEncodeString(buf, pReq->logicalPlan); tlen += taosEncodeString(buf, pReq->logicalPlan);
return tlen; return tlen;
@ -1114,41 +1123,62 @@ static FORCE_INLINE void* tDeserializeSCMCreateTopicRsp(void* buf, SCMCreateTopi
} }
typedef struct { typedef struct {
char* topicName; int32_t topicNum;
char* consumerGroup;
int64_t consumerId; int64_t consumerId;
char* consumerGroup;
char* topicName[];
} SCMSubscribeReq; } SCMSubscribeReq;
static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { static FORCE_INLINE int tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
int tlen = 0; int tlen = 0;
tlen += taosEncodeString(buf, pReq->topicName); tlen += taosEncodeFixedI32(buf, pReq->topicNum);
tlen += taosEncodeString(buf, pReq->consumerGroup);
tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeString(buf, pReq->consumerGroup);
for(int i = 0; i < pReq->topicNum; i++) {
tlen += taosEncodeString(buf, pReq->topicName[i]);
}
return tlen; return tlen;
} }
static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
buf = taosDecodeString(buf, &pReq->topicName); buf = taosDecodeFixedI32(buf, &pReq->topicNum);
buf = taosDecodeString(buf, &pReq->consumerGroup);
buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeString(buf, &pReq->consumerGroup);
for(int i = 0; i < pReq->topicNum; i++) {
buf = taosDecodeString(buf, &pReq->topicName[i]);
}
return buf; return buf;
} }
typedef struct { typedef struct SMqSubTopic {
int32_t vgId; int32_t vgId;
SEpSet pEpSet; int64_t topicId;
SEpSet epSet;
} SMqSubTopic;
typedef struct {
int32_t topicNum;
SMqSubTopic topics[];
} SCMSubscribeRsp; } SCMSubscribeRsp;
static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) { static FORCE_INLINE int tSerializeSCMSubscribeRsp(void** buf, const SCMSubscribeRsp* pRsp) {
int tlen = 0; int tlen = 0;
tlen += taosEncodeFixedI32(buf, pRsp->vgId); tlen += taosEncodeFixedI32(buf, pRsp->topicNum);
tlen += taosEncodeSEpSet(buf, &pRsp->pEpSet); for(int i = 0; i < pRsp->topicNum; i++) {
tlen += taosEncodeFixedI32(buf, pRsp->topics[i].vgId);
tlen += taosEncodeFixedI64(buf, pRsp->topics[i].topicId);
tlen += taosEncodeSEpSet(buf, &pRsp->topics[i].epSet);
}
return tlen; return tlen;
} }
static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) { static FORCE_INLINE void* tDeserializeSCMSubscribeRsp(void* buf, SCMSubscribeRsp* pRsp) {
buf = taosDecodeFixedI32(buf, &pRsp->vgId); buf = taosDecodeFixedI32(buf, &pRsp->topicNum);
buf = taosDecodeSEpSet(buf, &pRsp->pEpSet); for(int i = 0; i < pRsp->topicNum; i++) {
buf = taosDecodeFixedI32(buf, &pRsp->topics[i].vgId);
buf = taosDecodeFixedI64(buf, &pRsp->topics[i].topicId);
buf = taosDecodeSEpSet(buf, &pRsp->topics[i].epSet);
}
return buf; return buf;
} }
@ -1157,10 +1187,36 @@ typedef struct {
int64_t consumerId; int64_t consumerId;
int64_t consumerGroupId; int64_t consumerGroupId;
int64_t offset; int64_t offset;
char* sql;
char* logicalPlan;
char* physicalPlan;
} SMVSubscribeReq; } SMVSubscribeReq;
static FORCE_INLINE int tSerializeSMVSubscribeReq(void** buf, SMVSubscribeReq* pReq) {
int tlen = 0;
tlen += taosEncodeFixedI64(buf, pReq->topicId);
tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeFixedI64(buf, pReq->consumerGroupId);
tlen += taosEncodeFixedI64(buf, pReq->offset);
tlen += taosEncodeString(buf, pReq->sql);
tlen += taosEncodeString(buf, pReq->logicalPlan);
tlen += taosEncodeString(buf, pReq->physicalPlan);
return tlen;
}
static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq* pReq) {
buf = taosDecodeFixedI64(buf, &pReq->topicId);
buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeFixedI64(buf, &pReq->consumerGroupId);
buf = taosDecodeFixedI64(buf, &pReq->offset);
buf = taosDecodeString(buf, &pReq->sql);
buf = taosDecodeString(buf, &pReq->logicalPlan);
buf = taosDecodeString(buf, &pReq->physicalPlan);
return buf;
}
typedef struct { typedef struct {
int64_t newOffset; int64_t status;
} SMVSubscribeRsp; } SMVSubscribeRsp;
typedef struct { typedef struct {

1
include/common/tmsgdef.h
View File

@ -159,6 +159,7 @@ enum {
TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_QUERY, "vnode-mq-query", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_CONNECT, "vnode-mq-connect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_DISCONNECT, "vnode-mq-disconnect", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CONN, "vnode-mq-set-conn", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MQ_SET_CUR, "vnode-mq-set-cur", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_RES_READY, "vnode-res-ready", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TASKS_STATUS, "vnode-tasks-status", NULL, NULL)

19
include/libs/catalog/catalog.h
View File

@ -50,6 +50,7 @@ typedef struct SCatalogCfg {
uint32_t maxDBCacheNum; uint32_t maxDBCacheNum;
} SCatalogCfg; } SCatalogCfg;
int32_t catalogInit(SCatalogCfg *cfg); int32_t catalogInit(SCatalogCfg *cfg);
/** /**
@ -87,15 +88,28 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB
*/ */
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
/**
* Get a super table's meta data.
* @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name)
* @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code
*/
int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta);
/** /**
* Force renew a table's local cached meta data. * Force renew a table's local cached meta data.
* @param pCatalog (input, got with catalogGetHandle) * @param pCatalog (input, got with catalogGetHandle)
* @param pTransporter (input, rpc object) * @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name, NOT including db name)
* @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure)
* @return error code * @return error code
*/ */
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName); int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable);
/** /**
* Force renew a table's local cached meta data and get the new one. * Force renew a table's local cached meta data and get the new one.
@ -104,9 +118,10 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void * pTransporter, co
* @param pMgmtEps (input, mnode EPs) * @param pMgmtEps (input, mnode EPs)
* @param pTableName (input, table name, NOT including db name) * @param pTableName (input, table name, NOT including db name)
* @param pTableMeta(output, table meta data, NEED to free it by calller) * @param pTableMeta(output, table meta data, NEED to free it by calller)
* @param isSTable (input, is super table or not, 1:supposed to be stable, 0: supposed not to be stable, -1:not sure)
* @return error code * @return error code
*/ */
int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta); int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable);
/** /**

2
include/libs/parser/parser.h
View File

@ -43,7 +43,7 @@ int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery);
bool qIsDdlQuery(const SQueryNode* pQuery); bool qIsDdlQuery(const SQueryNode* pQuery);
void qDestroyQuery(SQueryNode* pQuery); void qDestroyQuery(SQueryNode* pQueryNode);
/** /**
* Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that

2
include/util/taoserror.h
View File

@ -401,6 +401,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal") #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) //"Unexpected generic error in wal")
#define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted") #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) //"WAL file is corrupted")
#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) //"WAL size exceeds limit") #define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) //"WAL size exceeds limit")
#define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) //"WAL invalid version")
#define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) //"WAL out of memory")
// tfs // tfs
#define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory") #define TSDB_CODE_FS_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x2200) //"tfs out of memory")

1
include/util/tdef.h
View File

@ -177,6 +177,7 @@ do { \
#define TSDB_TYPE_STR_MAX_LEN 32 #define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN) #define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN #define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN
#define TSDB_CONSUMER_GROUP_LEN 192
#define TSDB_COL_NAME_LEN 65 #define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 #define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE #define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE

16
source/client/src/clientEnv.c
View File

@ -180,6 +180,14 @@ void* createRequest(STscObj* pObj, __taos_async_fn_t fp, void* param, int32_t ty
return pRequest; return pRequest;
} }
static void doFreeReqResultInfo(SReqResultInfo* pResInfo) {
tfree(pResInfo->pRspMsg);
tfree(pResInfo->length);
tfree(pResInfo->row);
tfree(pResInfo->pCol);
tfree(pResInfo->fields);
}
static void doDestroyRequest(void* p) { static void doDestroyRequest(void* p) {
assert(p != NULL); assert(p != NULL);
SRequestObj* pRequest = (SRequestObj*)p; SRequestObj* pRequest = (SRequestObj*)p;
@ -190,7 +198,7 @@ static void doDestroyRequest(void* p) {
tfree(pRequest->sqlstr); tfree(pRequest->sqlstr);
tfree(pRequest->pInfo); tfree(pRequest->pInfo);
tfree(pRequest->body.resInfo.pRspMsg); doFreeReqResultInfo(&pRequest->body.resInfo);
deregisterRequest(pRequest); deregisterRequest(pRequest);
tfree(pRequest); tfree(pRequest);
@ -415,7 +423,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) {
*+------------+-----+-----------+---------------+ *+------------+-----+-----------+---------------+
*| uid|localIp| PId | timestamp | serial number | *| uid|localIp| PId | timestamp | serial number |
*+------------+-----+-----------+---------------+ *+------------+-----+-----------+---------------+
*| 16bit |12bit|20bit |16bit | *| 12bit |12bit|24bit |16bit |
*+------------+-----+-----------+---------------+ *+------------+-----+-----------+---------------+
* @return * @return
*/ */
@ -435,11 +443,11 @@ uint64_t generateRequestId() {
} }
} }
int64_t ts = taosGetTimestampUs(); int64_t ts = taosGetTimestampMs();
uint64_t pid = taosGetPId(); uint64_t pid = taosGetPId();
int32_t val = atomic_add_fetch_32(&requestSerialId, 1); int32_t val = atomic_add_fetch_32(&requestSerialId, 1);
uint64_t id = ((hashId & 0xFFFF) << 48) | ((pid & 0x0FFF) << 36) | ((ts & 0xFFFFF) << 16) | (val & 0xFFFF); uint64_t id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF);
return id; return id;
} }

39
source/client/src/clientImpl.c
View File

@ -192,24 +192,32 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
} }
tsem_wait(&pRequest->body.rspSem); tsem_wait(&pRequest->body.rspSem);
destroySendMsgInfo(pSendMsg);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQuery, SQueryDag** pDag) { int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) {
pRequest->type = pQuery->type; pRequest->type = pQueryNode->type;
return qCreateQueryDag(pQuery, pDag, pRequest->requestId); return qCreateQueryDag(pQueryNode, pDag, pRequest->requestId);
} }
int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) {
if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) {
SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf};
int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, pJob, &res); int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, pJob, &res);
if (code != TSDB_CODE_SUCCESS) {
// handle error and retry
} else {
if (*pJob != NULL) {
scheduleFreeJob(*pJob);
}
}
pRequest->affectedRows = res.numOfRows; pRequest->affectedRows = res.numOfRows;
return res.code; return res.code;
} }
return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob); return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL /*todo appInfo.xxx*/, pDag, pJob);
} }
TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) {
@ -364,8 +372,6 @@ STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, con
asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem); tsem_wait(&pRequest->body.rspSem);
destroySendMsgInfo(body);
if (pRequest->code != TSDB_CODE_SUCCESS) { if (pRequest->code != TSDB_CODE_SUCCESS) {
const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno); const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno);
printf("failed to connect to server, reason: %s\n\n", errorMsg); printf("failed to connect to server, reason: %s\n\n", errorMsg);
@ -456,17 +462,21 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) {
taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId); taosReleaseRef(clientReqRefPool, pSendInfo->requestObjRefId);
} }
SDataBuf buf = {.len = pMsg->contLen}; SDataBuf buf = {.len = pMsg->contLen, .pData = NULL};
buf.pData = calloc(1, pMsg->contLen);
if (buf.pData == NULL) { if (pMsg->contLen > 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; buf.pData = calloc(1, pMsg->contLen);
pMsg->code = TSDB_CODE_OUT_OF_MEMORY; if (buf.pData == NULL) {
} else { terrno = TSDB_CODE_OUT_OF_MEMORY;
memcpy(buf.pData, pMsg->pCont, pMsg->contLen); pMsg->code = TSDB_CODE_OUT_OF_MEMORY;
} else {
memcpy(buf.pData, pMsg->pCont, pMsg->contLen);
}
} }
pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); pSendInfo->fp(pSendInfo->param, &buf, pMsg->code);
rpcFreeCont(pMsg->pCont); rpcFreeCont(pMsg->pCont);
destroySendMsgInfo(pSendInfo);
} }
TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) { TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port) {
@ -539,7 +549,6 @@ void* doFetchRow(SRequestObj* pRequest) {
asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body); asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem); tsem_wait(&pRequest->body.rspSem);
destroySendMsgInfo(body);
pResultInfo->current = 0; pResultInfo->current = 0;
if (pResultInfo->numOfRows <= pResultInfo->current) { if (pResultInfo->numOfRows <= pResultInfo->current) {

7
source/client/src/clientMsgHandler.c
View File

@ -30,6 +30,7 @@ int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param; SRequestObj* pRequest = param;
setErrno(pRequest, code); setErrno(pRequest, code);
free(pMsg->pData);
sem_post(&pRequest->body.rspSem); sem_post(&pRequest->body.rspSem);
return code; return code;
} }
@ -37,6 +38,7 @@ int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) {
int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param; SRequestObj* pRequest = param;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
free(pMsg->pData);
setErrno(pRequest, code); setErrno(pRequest, code);
sem_post(&pRequest->body.rspSem); sem_post(&pRequest->body.rspSem);
return code; return code;
@ -73,6 +75,7 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId,
pTscObj->pAppInfo->numOfConns); pTscObj->pAppInfo->numOfConns);
free(pMsg->pData);
sem_post(&pRequest->body.rspSem); sem_post(&pRequest->body.rspSem);
return 0; return 0;
} }
@ -238,6 +241,7 @@ int32_t processRetrieveVndRsp(void* param, const SDataBuf* pMsg, int32_t code) {
int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { int32_t processCreateDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
// todo rsp with the vnode id list // todo rsp with the vnode id list
SRequestObj* pRequest = param; SRequestObj* pRequest = param;
free(pMsg->pData);
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
} }
@ -245,6 +249,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SRequestObj* pRequest = param; SRequestObj* pRequest = param;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
free(pMsg->pData);
setErrno(pRequest, code); setErrno(pRequest, code);
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
return code; return code;
@ -258,6 +263,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
tNameGetDbName(&name, db); tNameGetDbName(&name, db);
setConnectionDB(pRequest->pTscObj, db); setConnectionDB(pRequest->pTscObj, db);
free(pMsg->pData);
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);
return 0; return 0;
} }
@ -266,6 +272,7 @@ int32_t processCreateTableRsp(void* param, const SDataBuf* pMsg, int32_t code) {
assert(pMsg != NULL && param != NULL); assert(pMsg != NULL && param != NULL);
SRequestObj* pRequest = param; SRequestObj* pRequest = param;
free(pMsg->pData);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
setErrno(pRequest, code); setErrno(pRequest, code);
tsem_post(&pRequest->body.rspSem); tsem_post(&pRequest->body.rspSem);

191
source/client/test/clientTests.cpp
View File

@ -112,6 +112,7 @@ TEST(testCase, show_user_Test) {
printf("%s\n", str); printf("%s\n", str);
} }
taos_free_result(pRes);
taos_close(pConn); taos_close(pConn);
} }
@ -130,7 +131,7 @@ TEST(testCase, drop_user_Test) {
TEST(testCase, show_db_Test) { TEST(testCase, show_db_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "show databases"); TAOS_RES* pRes = taos_query(pConn, "show databases");
TAOS_ROW pRow = NULL; TAOS_ROW pRow = NULL;
@ -170,62 +171,62 @@ TEST(testCase, create_db_Test) {
} }
taos_close(pConn); taos_close(pConn);
} }
//
//TEST(testCase, create_dnode_Test) { TEST(testCase, create_dnode_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000");
// if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); printf("error in create dnode, reason:%s\n", taos_errstr(pRes));
// } }
// taos_free_result(pRes); taos_free_result(pRes);
//
// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000");
// if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); printf("failed to create dnode, reason:%s\n", taos_errstr(pRes));
// } }
// taos_free_result(pRes); taos_free_result(pRes);
//
// taos_close(pConn); taos_close(pConn);
//} }
//
//TEST(testCase, drop_dnode_Test) { TEST(testCase, drop_dnode_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); TAOS_RES* pRes = taos_query(pConn, "drop dnode 2");
// if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); printf("error in drop dnode, reason:%s\n", taos_errstr(pRes));
// } }
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// ASSERT_TRUE(pFields == NULL); ASSERT_TRUE(pFields == NULL);
//
// int32_t numOfFields = taos_num_fields(pRes); int32_t numOfFields = taos_num_fields(pRes);
// ASSERT_EQ(numOfFields, 0); ASSERT_EQ(numOfFields, 0);
//
// taos_free_result(pRes); taos_free_result(pRes);
// taos_close(pConn); taos_close(pConn);
//} }
//
//TEST(testCase, use_db_test) { TEST(testCase, use_db_test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); assert(pConn != NULL);
//
// TAOS_RES* pRes = taos_query(pConn, "use abc1"); TAOS_RES* pRes = taos_query(pConn, "use abc1");
// if (taos_errno(pRes) != 0) { if (taos_errno(pRes) != 0) {
// printf("error in use db, reason:%s\n", taos_errstr(pRes)); printf("error in use db, reason:%s\n", taos_errstr(pRes));
// } }
//
// TAOS_FIELD* pFields = taos_fetch_fields(pRes); TAOS_FIELD* pFields = taos_fetch_fields(pRes);
// ASSERT_TRUE(pFields == NULL); ASSERT_TRUE(pFields == NULL);
//
// int32_t numOfFields = taos_num_fields(pRes); int32_t numOfFields = taos_num_fields(pRes);
// ASSERT_EQ(numOfFields, 0); ASSERT_EQ(numOfFields, 0);
//
// taos_close(pConn); taos_close(pConn);
//} }
//TEST(testCase, drop_db_test) { //TEST(testCase, drop_db_test) {
//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); //// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
@ -280,19 +281,19 @@ TEST(testCase, create_db_Test) {
taos_close(pConn); taos_close(pConn);
} }
//TEST(testCase, create_table_Test) { TEST(testCase, create_table_Test) {
// // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// // assert(pConn != NULL); assert(pConn != NULL);
// //
// // TAOS_RES* pRes = taos_query(pConn, "use abc1"); TAOS_RES* pRes = taos_query(pConn, "use abc1");
// // taos_free_result(pRes); taos_free_result(pRes);
// //
// // pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)");
// // taos_free_result(pRes); taos_free_result(pRes);
// //
// // taos_close(pConn); taos_close(pConn);
//} }
//
//TEST(testCase, create_ctable_Test) { //TEST(testCase, create_ctable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); // assert(pConn != NULL);
@ -303,12 +304,12 @@ TEST(testCase, create_db_Test) {
// } // }
// taos_free_result(pRes); // taos_free_result(pRes);
// //
//// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); // pRes = taos_query(pConn, "create table tm0 using st1 tags(1)");
//// if (taos_errno(pRes) != 0) { // if (taos_errno(pRes) != 0) {
//// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); // printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes));
//// } // }
//// //
//// taos_free_result(pRes); // taos_free_result(pRes);
// taos_close(pConn); // taos_close(pConn);
//} //}
// //
@ -342,7 +343,7 @@ TEST(testCase, create_db_Test) {
// taos_free_result(pRes); // taos_free_result(pRes);
// taos_close(pConn); // taos_close(pConn);
//} //}
//
//TEST(testCase, show_vgroup_Test) { //TEST(testCase, show_vgroup_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); // assert(pConn != NULL);
@ -375,7 +376,7 @@ TEST(testCase, create_db_Test) {
// //
// taos_close(pConn); // taos_close(pConn);
//} //}
//
//TEST(testCase, drop_stable_Test) { //TEST(testCase, drop_stable_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL); // assert(pConn != NULL);
@ -468,6 +469,13 @@ TEST(testCase, create_multiple_tables) {
ASSERT_NE(pConn, nullptr); ASSERT_NE(pConn, nullptr);
TAOS_RES* pRes = taos_query(pConn, "use abc1"); TAOS_RES* pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("failed to use db, reason:%s", taos_errstr(pRes));
taos_free_result(pRes);
taos_close(pConn);
return;
}
taos_free_result(pRes); taos_free_result(pRes);
pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); pRes = taos_query(pConn, "create table t_2 using st1 tags(1)");
@ -497,15 +505,15 @@ TEST(testCase, create_multiple_tables) {
taos_free_result(pRes); taos_free_result(pRes);
// for(int32_t i = 0; i < 10000; ++i) { for(int32_t i = 0; i < 1000; ++i) {
// char sql[512] = {0}; char sql[512] = {0};
// snprintf(sql, tListLen(sql), "create table t_x_%d using st1 tags(2)", i); snprintf(sql, tListLen(sql), "create table t_x_%d using st1 tags(2)", i);
// TAOS_RES* pres = taos_query(pConn, sql); TAOS_RES* pres = taos_query(pConn, sql);
// if (taos_errno(pres) != 0) { if (taos_errno(pres) != 0) {
// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres));
// } }
// taos_free_result(pres); taos_free_result(pres);
// } }
taos_close(pConn); taos_close(pConn);
} }
@ -513,14 +521,17 @@ TEST(testCase, create_multiple_tables) {
TEST(testCase, generated_request_id_test) { TEST(testCase, generated_request_id_test) {
SHashObj *phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); SHashObj *phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK);
for(int32_t i = 0; i < 1000000; ++i) { for(int32_t i = 0; i < 50000; ++i) {
uint64_t v = generateRequestId(); uint64_t v = generateRequestId();
void* result = taosHashGet(phash, &v, sizeof(v)); void* result = taosHashGet(phash, &v, sizeof(v));
ASSERT_EQ(result, nullptr); if (result != nullptr) {
printf("0x%lx, index:%d\n", v, i);
}
assert(result == nullptr);
taosHashPut(phash, &v, sizeof(v), NULL, 0); taosHashPut(phash, &v, sizeof(v), NULL, 0);
} }
taosHashClear(phash); taosHashCleanup(phash);
} }
//TEST(testCase, projection_query_tables) { //TEST(testCase, projection_query_tables) {

7
source/dnode/mnode/impl/inc/mndConsumer.h
View File

@ -25,11 +25,8 @@ extern "C" {
int32_t mndInitConsumer(SMnode *pMnode); int32_t mndInitConsumer(SMnode *pMnode);
void mndCleanupConsumer(SMnode *pMnode); void mndCleanupConsumer(SMnode *pMnode);
SConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId); SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId);
void mndReleaseConsumer(SMnode *pMnode, SConsumerObj *pConsumer); void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer);
SCGroupObj *mndAcquireCGroup(SMnode *pMnode, char *consumerGroup);
void mndReleaseCGroup(SMnode *pMnode, SCGroupObj *pCGroup);
#ifdef __cplusplus #ifdef __cplusplus
} }

134
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -22,6 +22,7 @@
#include "sync.h" #include "sync.h"
#include "tmsg.h" #include "tmsg.h"
#include "thash.h" #include "thash.h"
#include "tlist.h"
#include "tlog.h" #include "tlog.h"
#include "trpc.h" #include "trpc.h"
#include "ttimer.h" #include "ttimer.h"
@ -307,46 +308,117 @@ typedef struct {
char payload[]; char payload[];
} SShowObj; } SShowObj;
typedef struct { #if 0
char name[TSDB_TOPIC_FNAME_LEN]; typedef struct SConsumerObj {
char db[TSDB_DB_FNAME_LEN];
int64_t createTime;
int64_t updateTime;
uint64_t uid;
uint64_t dbUid;
int32_t version;
SRWLatch lock;
int32_t execLen;
void* executor;
int32_t sqlLen;
char* sql;
char* logicalPlan;
char* physicalPlan;
} STopicObj;
typedef struct {
char name[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int64_t createTime;
int64_t updateTime;
uint64_t uid; uint64_t uid;
int64_t createTime;
int64_t updateTime;
//uint64_t dbUid; //uint64_t dbUid;
int32_t version; int32_t version;
SRWLatch lock; SRWLatch lock;
SArray* topics;
} SConsumerObj; } SConsumerObj;
typedef struct { typedef struct SMqTopicConsumer {
char name[TSDB_TOPIC_FNAME_LEN]; int64_t consumerId;
char db[TSDB_DB_FNAME_LEN]; SList* topicList;
int64_t createTime; } SMqTopicConsumer;
int64_t updateTime; #endif
typedef struct SMqCGroup {
char name[TSDB_CONSUMER_GROUP_LEN];
int32_t status; // 0 - uninitialized, 1 - wait rebalance, 2- normal
SList *consumerIds; // SList<int64_t>
SList *idleVGroups; // SList<int32_t>
} SMqCGroup;
typedef struct SMqTopicObj {
char name[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int64_t createTime;
int64_t updateTime;
uint64_t uid;
uint64_t dbUid;
int32_t version;
SRWLatch lock;
int32_t sqlLen;
char *sql;
char *logicalPlan;
char *physicalPlan;
SHashObj *cgroups; // SHashObj<SMqCGroup>
} SMqTopicObj;
// TODO: add cache and change name to id
typedef struct SMqConsumerTopic {
char name[TSDB_TOPIC_FNAME_LEN];
SList *vgroups; // SList<int32_t>
} SMqConsumerTopic;
typedef struct SMqConsumerObj {
SRWLatch lock;
int64_t consumerId;
char cgroup[TSDB_CONSUMER_GROUP_LEN];
SArray *topics; // SArray<SMqConsumerTopic>
} SMqConsumerObj;
typedef struct SMqSubConsumerObj {
int64_t consumerUid; // if -1, unassigned
SList *vgId; // SList<int32_t>
} SMqSubConsumerObj;
typedef struct SMqSubCGroupObj {
char name[TSDB_CONSUMER_GROUP_LEN];
SList *consumers; // SList<SMqConsumerObj>
} SMqSubCGroupObj;
typedef struct SMqSubTopicObj {
char name[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int64_t createTime;
int64_t updateTime;
int64_t uid;
int64_t dbUid;
int32_t version;
SRWLatch lock;
int32_t sqlLen;
char *sql;
char *logicalPlan;
char *physicalPlan;
SList *cgroups; // SList<SMqSubCGroupObj>
} SMqSubTopicObj;
typedef struct SMqConsumerSubObj {
int64_t topicUid;
SList *vgIds; // SList<int64_t>
} SMqConsumerSubObj;
typedef struct SMqConsumerHbObj {
int64_t consumerId;
SList *consumerSubs; // SList<SMqConsumerSubObj>
} SMqConsumerHbObj;
typedef struct SMqVGroupSubObj {
int64_t topicUid;
SList *consumerIds; // SList<int64_t>
} SMqVGroupSubObj;
typedef struct SMqVGroupHbObj {
int64_t vgId;
SList *vgSubs; // SList<SMqVGroupSubObj>
} SMqVGroupHbObj;
#if 0
typedef struct SCGroupObj {
char name[TSDB_TOPIC_FNAME_LEN];
int64_t createTime;
int64_t updateTime;
uint64_t uid; uint64_t uid;
//uint64_t dbUid; //uint64_t dbUid;
int32_t version; int32_t version;
SRWLatch lock; SRWLatch lock;
SList* consumerIds;
} SCGroupObj; } SCGroupObj;
#endif
typedef struct SMnodeMsg { typedef struct SMnodeMsg {
char user[TSDB_USER_LEN]; char user[TSDB_USER_LEN];

7
source/dnode/mnode/impl/inc/mndTopic.h
View File

@ -25,8 +25,11 @@ extern "C" {
int32_t mndInitTopic(SMnode *pMnode); int32_t mndInitTopic(SMnode *pMnode);
void mndCleanupTopic(SMnode *pMnode); void mndCleanupTopic(SMnode *pMnode);
STopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName); SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName);
void mndReleaseTopic(SMnode *pMnode, STopicObj *pTopic); void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic);
SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic);
SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw);
#ifdef __cplusplus #ifdef __cplusplus
} }

370
source/dnode/mnode/impl/src/mndConsumer.c
View File

@ -24,16 +24,17 @@
#include "mndTrans.h" #include "mndTrans.h"
#include "mndUser.h" #include "mndUser.h"
#include "mndVgroup.h" #include "mndVgroup.h"
#include "tcompare.h"
#include "tname.h" #include "tname.h"
#define MND_CONSUMER_VER_NUMBER 1 #define MND_CONSUMER_VER_NUMBER 1
#define MND_CONSUMER_RESERVE_SIZE 64 #define MND_CONSUMER_RESERVE_SIZE 64
static SSdbRaw *mndConsumerActionEncode(SConsumerObj *pConsumer); static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer);
static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw); static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw);
static int32_t mndConsumerActionInsert(SSdb *pSdb, SConsumerObj *pConsumer); static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionDelete(SSdb *pSdb, SConsumerObj *pConsumer); static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer);
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SConsumerObj *pConsumer, SConsumerObj *pNewConsumer); static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pConsumer, SMqConsumerObj *pNewConsumer);
static int32_t mndProcessCreateConsumerMsg(SMnodeMsg *pMsg); static int32_t mndProcessCreateConsumerMsg(SMnodeMsg *pMsg);
static int32_t mndProcessDropConsumerMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropConsumerMsg(SMnodeMsg *pMsg);
static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg); static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg);
@ -57,8 +58,8 @@ int32_t mndInitConsumer(SMnode *pMnode) {
.deleteFp = (SdbDeleteFp)mndConsumerActionDelete}; .deleteFp = (SdbDeleteFp)mndConsumerActionDelete};
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq); mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE, mndProcessSubscribeReq);
mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp); /*mndSetMsgHandle(pMnode, TDMT_MND_SUBSCRIBE_RSP, mndProcessSubscribeRsp);*/
mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq); /*mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE, mndProcessSubscribeInternalReq);*/
mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp); mndSetMsgHandle(pMnode, TDMT_VND_SUBSCRIBE_RSP, mndProcessSubscribeInternalRsp);
return sdbSetTable(pMnode->pSdb, table); return sdbSetTable(pMnode->pSdb, table);
@ -66,33 +67,52 @@ int32_t mndInitConsumer(SMnode *pMnode) {
void mndCleanupConsumer(SMnode *pMnode) {} void mndCleanupConsumer(SMnode *pMnode) {}
static SSdbRaw *mndConsumerActionEncode(SConsumerObj *pConsumer) { static void *mndBuildMqVGroupSetReq(SMnode *pMnode, char *topicName, int32_t vgId, int64_t consumerId, char *cgroup) {
terrno = TSDB_CODE_OUT_OF_MEMORY; return 0;
}
int32_t size = sizeof(SConsumerObj) + MND_CONSUMER_RESERVE_SIZE; static SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) {
int32_t size = sizeof(SMqConsumerObj) + MND_CONSUMER_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size); SSdbRaw *pRaw = sdbAllocRaw(SDB_CONSUMER, MND_CONSUMER_VER_NUMBER, size);
if (pRaw == NULL) goto CM_ENCODE_OVER; if (pRaw == NULL) goto CM_ENCODE_OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pConsumer->name, TSDB_TABLE_FNAME_LEN, CM_ENCODE_OVER) int32_t topicNum = taosArrayGetSize(pConsumer->topics);
SDB_SET_BINARY(pRaw, dataPos, pConsumer->db, TSDB_DB_FNAME_LEN, CM_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pConsumer->consumerId, CM_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pConsumer->createTime, CM_ENCODE_OVER) int32_t len = strlen(pConsumer->cgroup);
SDB_SET_INT64(pRaw, dataPos, pConsumer->updateTime, CM_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER);
SDB_SET_INT64(pRaw, dataPos, pConsumer->uid, CM_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CM_ENCODE_OVER);
/*SDB_SET_INT64(pRaw, dataPos, pConsumer->dbUid);*/ SDB_SET_INT32(pRaw, dataPos, topicNum, CM_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pConsumer->version, CM_ENCODE_OVER) for (int i = 0; i < topicNum; i++) {
int32_t len;
SMqConsumerTopic *pConsumerTopic = taosArrayGet(pConsumer->topics, i);
len = strlen(pConsumerTopic->name);
SDB_SET_INT32(pRaw, dataPos, len, CM_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pConsumerTopic->name, len, CM_ENCODE_OVER);
int vgSize;
if (pConsumerTopic->vgroups == NULL) {
vgSize = 0;
} else {
vgSize = listNEles(pConsumerTopic->vgroups);
}
SDB_SET_INT32(pRaw, dataPos, vgSize, CM_ENCODE_OVER);
for (int j = 0; j < vgSize; j++) {
// SList* head;
/*SDB_SET_INT64(pRaw, dataPos, 0[> change to list item <]);*/
}
}
SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER) SDB_SET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CM_ENCODE_OVER);
SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, CM_ENCODE_OVER);
CM_ENCODE_OVER: CM_ENCODE_OVER:
if (terrno != 0) { if (terrno != 0) {
mError("consumer:%s, failed to encode to raw:%p since %s", pConsumer->name, pRaw, terrstr()); mError("consumer:%ld, failed to encode to raw:%p since %s", pConsumer->consumerId, pRaw, terrstr());
sdbFreeRaw(pRaw); sdbFreeRaw(pRaw);
return NULL; return NULL;
} }
mTrace("consumer:%s, encode to raw:%p, row:%p", pConsumer->name, pRaw, pConsumer); mTrace("consumer:%ld, encode to raw:%p, row:%p", pConsumer->consumerId, pRaw, pConsumer);
return pRaw; return pRaw;
} }
@ -107,68 +127,77 @@ static SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) {
goto CONSUME_DECODE_OVER; goto CONSUME_DECODE_OVER;
} }
int32_t size = sizeof(SConsumerObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); int32_t size = sizeof(SMqConsumerObj);
SSdbRow *pRow = sdbAllocRow(size); SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto CONSUME_DECODE_OVER; if (pRow == NULL) goto CONSUME_DECODE_OVER;
SConsumerObj *pConsumer = sdbGetRowObj(pRow); SMqConsumerObj *pConsumer = sdbGetRowObj(pRow);
if (pConsumer == NULL) goto CONSUME_DECODE_OVER; if (pConsumer == NULL) goto CONSUME_DECODE_OVER;
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, dataPos, pConsumer->name, TSDB_TABLE_FNAME_LEN, CONSUME_DECODE_OVER) SDB_GET_INT64(pRaw, dataPos, &pConsumer->consumerId, CONSUME_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pConsumer->db, TSDB_DB_FNAME_LEN, CONSUME_DECODE_OVER) int32_t len, topicNum;
SDB_GET_INT64(pRaw, dataPos, &pConsumer->createTime, CONSUME_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &len, CONSUME_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pConsumer->updateTime, CONSUME_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pConsumer->cgroup, len, CONSUME_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pConsumer->uid, CONSUME_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &topicNum, CONSUME_DECODE_OVER);
/*SDB_GET_INT64(pRaw, pRow, dataPos, &pConsumer->dbUid);*/ for (int i = 0; i < topicNum; i++) {
SDB_GET_INT32(pRaw, dataPos, &pConsumer->version, CONSUME_DECODE_OVER) int32_t topicLen;
SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE, CONSUME_DECODE_OVER) SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic));
terrno = 0; if (pConsumerTopic == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
// TODO
return NULL;
}
/*pConsumerTopic->vgroups = taosArrayInit(topicNum, sizeof(SMqConsumerTopic));*/
SDB_GET_INT32(pRaw, dataPos, &topicLen, CONSUME_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pConsumerTopic->name, topicLen, CONSUME_DECODE_OVER);
int32_t vgSize;
SDB_GET_INT32(pRaw, dataPos, &vgSize, CONSUME_DECODE_OVER);
}
CONSUME_DECODE_OVER: CONSUME_DECODE_OVER:
if (terrno != 0) { if (terrno != 0) {
mError("consumer:%s, failed to decode from raw:%p since %s", pConsumer->name, pRaw, terrstr()); mError("consumer:%ld, failed to decode from raw:%p since %s", pConsumer->consumerId, pRaw, terrstr());
tfree(pRow); tfree(pRow);
return NULL; return NULL;
} }
mTrace("consumer:%s, decode from raw:%p, row:%p", pConsumer->name, pRaw, pConsumer); /*SDB_GET_RESERVE(pRaw, dataPos, MND_CONSUMER_RESERVE_SIZE);*/
return pRow; return pRow;
} }
static int32_t mndConsumerActionInsert(SSdb *pSdb, SConsumerObj *pConsumer) { static int32_t mndConsumerActionInsert(SSdb *pSdb, SMqConsumerObj *pConsumer) {
mTrace("consumer:%s, perform insert action, row:%p", pConsumer->name, pConsumer); mTrace("consumer:%ld, perform insert action", pConsumer->consumerId);
return 0; return 0;
} }
static int32_t mndConsumerActionDelete(SSdb *pSdb, SConsumerObj *pConsumer) { static int32_t mndConsumerActionDelete(SSdb *pSdb, SMqConsumerObj *pConsumer) {
mTrace("consumer:%s, perform delete action, row:%p", pConsumer->name, pConsumer); mTrace("consumer:%ld, perform delete action", pConsumer->consumerId);
return 0; return 0;
} }
static int32_t mndConsumerActionUpdate(SSdb *pSdb, SConsumerObj *pOldConsumer, SConsumerObj *pNewConsumer) { static int32_t mndConsumerActionUpdate(SSdb *pSdb, SMqConsumerObj *pOldConsumer, SMqConsumerObj *pNewConsumer) {
mTrace("consumer:%s, perform update action, old_row:%p new_row:%p", pOldConsumer->name, pOldConsumer, pNewConsumer); mTrace("consumer:%ld, perform update action", pOldConsumer->consumerId);
atomic_exchange_32(&pOldConsumer->updateTime, pNewConsumer->updateTime);
atomic_exchange_32(&pOldConsumer->version, pNewConsumer->version);
taosWLockLatch(&pOldConsumer->lock);
// TODO handle update // TODO handle update
/*taosWLockLatch(&pOldConsumer->lock);*/
/*taosWUnLockLatch(&pOldConsumer->lock);*/
taosWUnLockLatch(&pOldConsumer->lock);
return 0; return 0;
} }
SConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId) { SMqConsumerObj *mndAcquireConsumer(SMnode *pMnode, int32_t consumerId) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId); SMqConsumerObj *pConsumer = sdbAcquire(pSdb, SDB_CONSUMER, &consumerId);
if (pConsumer == NULL) { if (pConsumer == NULL) {
/*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/ /*terrno = TSDB_CODE_MND_CONSUMER_NOT_EXIST;*/
} }
return pConsumer; return pConsumer;
} }
void mndReleaseConsumer(SMnode *pMnode, SConsumerObj *pConsumer) { void mndReleaseConsumer(SMnode *pMnode, SMqConsumerObj *pConsumer) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pConsumer); sdbRelease(pSdb, pConsumer);
} }
@ -178,23 +207,185 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) {
char *msgStr = pMsg->rpcMsg.pCont; char *msgStr = pMsg->rpcMsg.pCont;
SCMSubscribeReq *pSubscribe; SCMSubscribeReq *pSubscribe;
tDeserializeSCMSubscribeReq(msgStr, pSubscribe); tDeserializeSCMSubscribeReq(msgStr, pSubscribe);
// add consumerGroupId -> list<consumerId> to sdb int64_t consumerId = pSubscribe->consumerId;
// add consumerId -> list<consumer> to sdb char *consumerGroup = pSubscribe->consumerGroup;
// add consumer -> list<consumerId> to sdb
SArray *newSub = NULL;
int newTopicNum = pSubscribe->topicNum;
if (newTopicNum) {
newSub = taosArrayInit(newTopicNum, sizeof(SMqConsumerTopic));
}
for (int i = 0; i < newTopicNum; i++) {
char *topic = pSubscribe->topicName[i];
SMqConsumerTopic *pConsumerTopic = malloc(sizeof(SMqConsumerTopic));
if (pConsumerTopic == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
// TODO: free
return -1;
}
pConsumerTopic->vgroups = tdListNew(sizeof(int64_t));
taosArrayPush(newSub, pConsumerTopic);
free(pConsumerTopic);
}
taosArraySortString(newSub, taosArrayCompareString);
SArray *oldSub = NULL;
int oldTopicNum = 0;
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
// create consumer
pConsumer = malloc(sizeof(SMqConsumerObj));
if (pConsumer == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
strcpy(pConsumer->cgroup, pSubscribe->consumerGroup);
} else {
oldSub = pConsumer->topics;
oldTopicNum = taosArrayGetSize(oldSub);
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, &pMsg->rpcMsg);
if (pTrans == NULL) {
return -1;
}
int i = 0, j = 0;
while (i < newTopicNum || j < oldTopicNum) {
SMqConsumerTopic *pOldTopic = NULL;
SMqConsumerTopic *pNewTopic = NULL;
if (i >= newTopicNum) {
// encode unset topic msg to all vnodes related to that topic
pOldTopic = taosArrayGet(oldSub, j);
j++;
} else if (j >= oldTopicNum) {
pNewTopic = taosArrayGet(newSub, i);
} else {
pNewTopic = taosArrayGet(newSub, i);
pOldTopic = taosArrayGet(oldSub, j);
char *newName = pNewTopic->name;
char *oldName = pOldTopic->name;
int comp = compareLenPrefixedStr(newName, oldName);
if (comp == 0) {
// do nothing
pOldTopic = pNewTopic = NULL;
i++;
j++;
continue;
} else if (comp < 0) {
pOldTopic = NULL;
i++;
} else {
pNewTopic = NULL;
j++;
}
}
if (pOldTopic != NULL) {
ASSERT(pNewTopic == NULL);
char *oldTopicName = pOldTopic->name;
SList *vgroups = pOldTopic->vgroups;
SListIter iter;
tdListInitIter(vgroups, &iter, TD_LIST_FORWARD);
SListNode *pn;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, oldTopicName);
ASSERT(pTopic != NULL);
SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup));
while ((pn = tdListNext(&iter)) != NULL) {
int32_t vgId = *(int64_t *)pn->data;
SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId);
// TODO release
if (pVgObj == NULL) {
// TODO handle error
continue;
}
// acquire and get epset
void *pMqVgSetReq =
mndBuildMqVGroupSetReq(pMnode, oldTopicName, vgId, pSubscribe->consumerId, pSubscribe->consumerGroup);
// TODO:serialize
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
STransAction action = {0};
action.epSet = mndGetVgroupEpset(pMnode, pVgObj);
action.pCont = pMqVgSetReq;
action.contLen = 0; // TODO
action.msgType = TDMT_VND_MQ_SET_CONN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMqVgSetReq);
mndTransDrop(pTrans);
// TODO free
return -1;
}
}
taosHashRemove(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup));
} else if (pNewTopic != NULL) {
ASSERT(pOldTopic == NULL);
char *newTopicName = pNewTopic->name;
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName);
ASSERT(pTopic != NULL);
SMqCGroup *pGroup = taosHashGet(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup));
if (pGroup == NULL) {
// add new group
pGroup = malloc(sizeof(SMqCGroup));
if (pGroup == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pGroup->consumerIds = tdListNew(sizeof(int64_t));
if (pGroup->consumerIds == NULL) {
free(pGroup);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pGroup->status = 0;
// add into cgroups
taosHashPut(pTopic->cgroups, pSubscribe->consumerGroup, strlen(pSubscribe->consumerGroup), pGroup,
sizeof(SMqCGroup));
}
// put the consumer into list
// rebalance will be triggered by timer
tdListAppend(pGroup->consumerIds, &pSubscribe->consumerId);
SSdbRaw *pTopicRaw = mndTopicActionEncode(pTopic);
sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY);
// TODO: error handling
mndTransAppendRedolog(pTrans, pTopicRaw);
} else {
ASSERT(0);
}
}
// destroy old sub
taosArrayDestroy(oldSub);
// put new sub into consumerobj
pConsumer->topics = newSub;
// persist consumerObj
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
// TODO: error handling
mndTransAppendRedolog(pTrans, pConsumerRaw);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
// TODO: free memory
mndTransDrop(pTrans);
return 0; return 0;
} }
static int32_t mndProcessSubscribeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessSubscribeInternalReq(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; } static int32_t mndProcessSubscribeInternalRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessDropConsumerInRsp(SMnodeMsg *pMsg) {
mndTransProcessRsp(pMsg);
return 0;
}
static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) { static int32_t mndProcessConsumerMetaMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
STableInfoMsg *pInfo = pMsg->rpcMsg.pCont; STableInfoMsg *pInfo = pMsg->rpcMsg.pCont;
@ -272,13 +463,11 @@ static int32_t mndGetNumOfConsumers(SMnode *pMnode, char *dbName, int32_t *pNumO
int32_t numOfConsumers = 0; int32_t numOfConsumers = 0;
void *pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
SConsumerObj *pConsumer = NULL; SMqConsumerObj *pConsumer = NULL;
pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer); pIter = sdbFetch(pSdb, SDB_CONSUMER, pIter, (void **)&pConsumer);
if (pIter == NULL) break; if (pIter == NULL) break;
if (strcmp(pConsumer->db, dbName) == 0) { numOfConsumers++;
numOfConsumers++;
}
sdbRelease(pSdb, pConsumer); sdbRelease(pSdb, pConsumer);
} }
@ -337,57 +526,6 @@ static int32_t mndGetConsumerMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMs
return 0; return 0;
} }
static int32_t mndRetrieveConsumer(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) {
SMnode *pMnode = pMsg->pMnode;
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
SConsumerObj *pConsumer = NULL;
int32_t cols = 0;
char *pWrite;
char prefix[64] = {0};
tstrncpy(prefix, pShow->db, 64);
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = (int32_t)strlen(prefix);
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_CONSUMER, pShow->pIter, (void **)&pConsumer);
if (pShow->pIter == NULL) break;
if (strncmp(pConsumer->name, prefix, prefixLen) != 0) {
sdbRelease(pSdb, pConsumer);
continue;
}
cols = 0;
char consumerName[TSDB_TABLE_NAME_LEN] = {0};
tstrncpy(consumerName, pConsumer->name + prefixLen, TSDB_TABLE_NAME_LEN);
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
STR_TO_VARSTR(pWrite, consumerName);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pConsumer->createTime;
cols++;
/*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/
/**(int32_t *)pWrite = pConsumer->numOfColumns;*/
/*cols++;*/
/*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/
/**(int32_t *)pWrite = pConsumer->numOfTags;*/
/*cols++;*/
numOfRows++;
sdbRelease(pSdb, pConsumer);
}
pShow->numOfReads += numOfRows;
mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
return numOfRows;
}
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);

151
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -14,6 +14,7 @@
*/ */
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "mndTopic.h"
#include "mndDb.h" #include "mndDb.h"
#include "mndDnode.h" #include "mndDnode.h"
#include "mndMnode.h" #include "mndMnode.h"
@ -27,18 +28,16 @@
#define MND_TOPIC_VER_NUMBER 1 #define MND_TOPIC_VER_NUMBER 1
#define MND_TOPIC_RESERVE_SIZE 64 #define MND_TOPIC_RESERVE_SIZE 64
static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic); static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic);
static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw); static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic);
static int32_t mndTopicActionInsert(SSdb *pSdb, STopicObj *pTopic); static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic);
static int32_t mndTopicActionDelete(SSdb *pSdb, STopicObj *pTopic); static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg);
static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pTopic, STopicObj *pNewTopic); static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg);
static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg); static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg);
static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg); static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg);
static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg); static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta);
static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg); static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter);
int32_t mndInitTopic(SMnode *pMnode) { int32_t mndInitTopic(SMnode *pMnode) {
SSdbTable table = {.sdbType = SDB_TOPIC, SSdbTable table = {.sdbType = SDB_TOPIC,
@ -58,45 +57,31 @@ int32_t mndInitTopic(SMnode *pMnode) {
void mndCleanupTopic(SMnode *pMnode) {} void mndCleanupTopic(SMnode *pMnode) {}
static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic) { SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SMqTopicObj) + MND_TOPIC_RESERVE_SIZE;
int32_t size = sizeof(STopicObj) + MND_TOPIC_RESERVE_SIZE;
SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size);
if (pRaw == NULL) goto TOPIC_ENCODE_OVER; if (pRaw == NULL) goto WTF;
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, TOPIC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, WTF);
SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, WTF);
SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, TOPIC_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pTopic->createTime, WTF);
SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, TOPIC_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime, WTF);
SDB_SET_INT64(pRaw, dataPos, pTopic->uid, TOPIC_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pTopic->uid, WTF);
SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER) SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, WTF);
SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pTopic->version, WTF);
SDB_SET_INT32(pRaw, dataPos, pTopic->execLen, TOPIC_ENCODE_OVER) SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, WTF);
SDB_SET_BINARY(pRaw, dataPos, pTopic->executor, pTopic->execLen, TOPIC_ENCODE_OVER) SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, WTF);
SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER)
SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER)
SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_ENCODE_OVER) SDB_SET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, WTF);
SDB_SET_DATALEN(pRaw, dataPos, TOPIC_ENCODE_OVER) SDB_SET_DATALEN(pRaw, dataPos, WTF);
terrno = 0; WTF:
TOPIC_ENCODE_OVER:
if (terrno != 0) {
mError("topic:%s, failed to encode to raw:%p since %s", pTopic->name, pRaw, terrstr());
sdbFreeRaw(pRaw);
return NULL;
}
mTrace("topic:%s, encode to raw:%p, row:%p", pTopic->name, pRaw, pTopic);
return pRaw; return pRaw;
} }
static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
int8_t sver = 0; int8_t sver = 0;
if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto TOPIC_DECODE_OVER;
@ -105,25 +90,28 @@ static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
goto TOPIC_DECODE_OVER; goto TOPIC_DECODE_OVER;
} }
int32_t size = sizeof(STopicObj) + TSDB_MAX_COLUMNS * sizeof(SSchema); int32_t size = sizeof(SMqTopicObj);
SSdbRow *pRow = sdbAllocRow(size); SSdbRow *pRow = sdbAllocRow(size);
if (pRow == NULL) goto TOPIC_DECODE_OVER; if (pRow == NULL) goto TOPIC_DECODE_OVER;
STopicObj *pTopic = sdbGetRowObj(pRow); SMqTopicObj *pTopic = sdbGetRowObj(pRow);
if (pTopic == NULL) goto TOPIC_DECODE_OVER; if (pTopic == NULL) goto TOPIC_DECODE_OVER;
int32_t len;
int32_t dataPos = 0; int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->createTime, TOPIC_DECODE_OVER) SDB_GET_INT64(pRaw, dataPos, &pTopic->createTime, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER) SDB_GET_INT64(pRaw, dataPos, &pTopic->updateTime, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER) SDB_GET_INT64(pRaw, dataPos, &pTopic->uid, TOPIC_DECODE_OVER);
SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER) SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->execLen, TOPIC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->executor, pTopic->execLen, TOPIC_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER) SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER) SDB_GET_BINARY(pRaw, dataPos, pTopic->logicalPlan, len, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER);
SDB_GET_BINARY(pRaw, dataPos, pTopic->physicalPlan, len, TOPIC_DECODE_OVER);
SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER) SDB_GET_RESERVE(pRaw, dataPos, MND_TOPIC_RESERVE_SIZE, TOPIC_DECODE_OVER)
@ -140,18 +128,18 @@ TOPIC_DECODE_OVER:
return pRow; return pRow;
} }
static int32_t mndTopicActionInsert(SSdb *pSdb, STopicObj *pTopic) { static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic) {
mTrace("topic:%s, perform insert action, row:%p", pTopic->name, pTopic); mTrace("topic:%s, perform insert action", pTopic->name);
return 0; return 0;
} }
static int32_t mndTopicActionDelete(SSdb *pSdb, STopicObj *pTopic) { static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic) {
mTrace("topic:%s, perform delete action, row:%p", pTopic->name, pTopic); mTrace("topic:%s, perform delete action", pTopic->name);
return 0; return 0;
} }
static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pOldTopic, STopicObj *pNewTopic) { static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopicObj *pNewTopic) {
mTrace("topic:%s, perform update action, old_row:%p new_row:%p", pOldTopic->name, pOldTopic, pNewTopic); mTrace("topic:%s, perform update action", pOldTopic->name);
atomic_exchange_32(&pOldTopic->updateTime, pNewTopic->updateTime); atomic_exchange_32(&pOldTopic->updateTime, pNewTopic->updateTime);
atomic_exchange_32(&pOldTopic->version, pNewTopic->version); atomic_exchange_32(&pOldTopic->version, pNewTopic->version);
@ -163,16 +151,16 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, STopicObj *pOldTopic, STopicObj
return 0; return 0;
} }
STopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) { SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
STopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName);
if (pTopic == NULL) { if (pTopic == NULL) {
terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST;
} }
return pTopic; return pTopic;
} }
void mndReleaseTopic(SMnode *pMnode, STopicObj *pTopic) { void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pTopic); sdbRelease(pSdb, pTopic);
} }
@ -187,7 +175,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) {
return mndAcquireDb(pMnode, db); return mndAcquireDb(pMnode, db);
} }
static SDDropTopicMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, STopicObj *pTopic) { static SDDropTopicMsg *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) {
int32_t contLen = sizeof(SDDropTopicMsg); int32_t contLen = sizeof(SDDropTopicMsg);
SDDropTopicMsg *pDrop = calloc(1, contLen); SDDropTopicMsg *pDrop = calloc(1, contLen);
@ -210,7 +198,7 @@ static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *pCreate) {
} }
static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) { static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) {
STopicObj topicObj = {0}; SMqTopicObj topicObj = {0};
tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN); tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN); tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
topicObj.createTime = taosGetTimestampMs(); topicObj.createTime = taosGetTimestampMs();
@ -222,6 +210,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq
SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj);
if (pTopicRaw == NULL) return -1; if (pTopicRaw == NULL) return -1;
if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1; if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1;
// TODO: replace with trans to support recovery
return sdbWrite(pMnode->pSdb, pTopicRaw); return sdbWrite(pMnode->pSdb, pTopicRaw);
} }
@ -238,7 +227,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) {
return -1; return -1;
} }
STopicObj *pTopic = mndAcquireTopic(pMnode, pCreate->name); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pCreate->name);
if (pTopic != NULL) { if (pTopic != NULL) {
sdbRelease(pMnode->pSdb, pTopic); sdbRelease(pMnode->pSdb, pTopic);
if (pCreate->igExists) { if (pCreate->igExists) {
@ -270,7 +259,7 @@ static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS; return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} }
static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, STopicObj *pTopic) { return 0; } static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic) { return 0; }
static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) { static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
@ -278,7 +267,7 @@ static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) {
mDebug("topic:%s, start to drop", pDrop->name); mDebug("topic:%s, start to drop", pDrop->name);
STopicObj *pTopic = mndAcquireTopic(pMnode, pDrop->name); SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pDrop->name);
if (pTopic == NULL) { if (pTopic == NULL) {
if (pDrop->igNotExists) { if (pDrop->igNotExists) {
mDebug("topic:%s, not exist, ignore not exist is set", pDrop->name); mDebug("topic:%s, not exist, ignore not exist is set", pDrop->name);
@ -384,13 +373,11 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo
int32_t numOfTopics = 0; int32_t numOfTopics = 0;
void *pIter = NULL; void *pIter = NULL;
while (1) { while (1) {
STopicObj *pTopic = NULL; SMqTopicObj *pTopic = NULL;
pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic);
if (pIter == NULL) break; if (pIter == NULL) break;
if (strcmp(pTopic->db, dbName) == 0) { numOfTopics++;
numOfTopics++;
}
sdbRelease(pSdb, pTopic); sdbRelease(pSdb, pTopic);
} }
@ -463,13 +450,13 @@ static void mndExtractTableName(char *tableId, char *name) {
} }
static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) {
SMnode *pMnode = pMsg->pMnode; SMnode *pMnode = pMsg->pMnode;
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0; int32_t numOfRows = 0;
STopicObj *pTopic = NULL; SMqTopicObj *pTopic = NULL;
int32_t cols = 0; int32_t cols = 0;
char *pWrite; char *pWrite;
char prefix[64] = {0}; char prefix[64] = {0};
tstrncpy(prefix, pShow->db, 64); tstrncpy(prefix, pShow->db, 64);
strcat(prefix, TS_PATH_DELIMITER); strcat(prefix, TS_PATH_DELIMITER);

8
source/libs/catalog/inc/catalogInt.h
View File

@ -64,6 +64,14 @@ typedef struct SCatalogMgmt {
typedef uint32_t (*tableNameHashFp)(const char *, uint32_t); typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
#define CTG_IS_STABLE(isSTable) (1 == (isSTable))
#define CTG_IS_NOT_STABLE(isSTable) (0 == (isSTable))
#define CTG_IS_UNKNOWN_STABLE(isSTable) ((isSTable) < 0)
#define CTG_SET_STABLE(isSTable, tbType) do { (isSTable) = ((tbType) == TSDB_SUPER_TABLE) ? 1 : ((tbType) > TSDB_SUPER_TABLE ? 0 : -1); } while (0)
#define CTG_TBTYPE_MATCH(isSTable, tbType) (CTG_IS_UNKNOWN_STABLE(isSTable) || (CTG_IS_STABLE(isSTable) && (tbType) == TSDB_SUPER_TABLE) || (CTG_IS_NOT_STABLE(isSTable) && (tbType) != TSDB_SUPER_TABLE))
#define CTG_TABLE_NOT_EXIST(code) (code == TSDB_CODE_TDB_INVALID_TABLE_ID)
#define ctgFatal(...) do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgFatal(...) do { if (ctgDebugFlag & DEBUG_FATAL) { taosPrintLog("CTG FATAL ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgError(...) do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgError(...) do { if (ctgDebugFlag & DEBUG_ERROR) { taosPrintLog("CTG ERROR ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgWarn(...) do { if (ctgDebugFlag & DEBUG_WARN) { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }} while(0) #define ctgWarn(...) do { if (ctgDebugFlag & DEBUG_WARN) { taosPrintLog("CTG WARN ", ctgDebugFlag, __VA_ARGS__); }} while(0)

165
source/libs/catalog/src/catalog.c
View File

@ -106,6 +106,8 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN
*exist = 1; *exist = 1;
tbMeta = *pTableMeta;
if (tbMeta->tableType != TSDB_CHILD_TABLE) { if (tbMeta->tableType != TSDB_CHILD_TABLE) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -143,6 +145,29 @@ int32_t ctgGetTableMetaFromCache(struct SCatalog* pCatalog, const SName* pTableN
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetTableTypeFromCache(struct SCatalog* pCatalog, const SName* pTableName, int32_t *tbType) {
if (NULL == pCatalog->tableCache.cache) {
return TSDB_CODE_SUCCESS;
}
char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
size_t sz = 0;
STableMeta *pTableMeta = NULL;
taosHashGetCloneExt(pCatalog->tableCache.cache, tbFullName, strlen(tbFullName), NULL, (void **)&pTableMeta, &sz);
if (NULL == pTableMeta) {
return TSDB_CODE_SUCCESS;
}
*tbType = pTableMeta->tableType;
return TSDB_CODE_SUCCESS;
}
void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) { void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) {
epSet->inUse = 0; epSet->inUse = 0;
epSet->numOfEps = vgroupInfo->numOfEps; epSet->numOfEps = vgroupInfo->numOfEps;
@ -153,14 +178,7 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) {
} }
} }
int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) { int32_t ctgGetTableMetaFromMnodeImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, char* tbFullName, STableMetaOutput* output) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == output) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName}; SBuildTableMetaInput bInput = {.vgId = 0, .dbName = NULL, .tableFullName = tbFullName};
char *msg = NULL; char *msg = NULL;
SEpSet *pVnodeEpSet = NULL; SEpSet *pVnodeEpSet = NULL;
@ -179,6 +197,12 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) {
output->metaNum = 0;
ctgDebug("tbmeta:%s not exist in mnode", tbFullName);
return TSDB_CODE_SUCCESS;
}
ctgError("error rsp for table meta, code:%x", rpcRsp.code); ctgError("error rsp for table meta, code:%x", rpcRsp.code);
CTG_ERR_RET(rpcRsp.code); CTG_ERR_RET(rpcRsp.code);
} }
@ -188,6 +212,13 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, STableMetaOutput* output) {
char tbFullName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTableName, tbFullName);
return ctgGetTableMetaFromMnodeImpl(pCatalog, pRpc, pMgmtEps, tbFullName, output);
}
int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) { int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == vgroupInfo || NULL == output) {
@ -197,7 +228,7 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE
char dbFullName[TSDB_DB_FNAME_LEN]; char dbFullName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFullName); tNameGetFullDbName(pTableName, dbFullName);
SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = pTableName->tname}; SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbName = dbFullName, .tableFullName = (char *)pTableName->tname};
char *msg = NULL; char *msg = NULL;
SEpSet *pVnodeEpSet = NULL; SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0; int32_t msgLen = 0;
@ -214,10 +245,15 @@ int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SE
SEpSet epSet; SEpSet epSet;
ctgGenEpSet(&epSet, vgroupInfo); ctgGenEpSet(&epSet, vgroupInfo);
rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp); rpcSendRecv(pRpc, &epSet, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) { if (TSDB_CODE_SUCCESS != rpcRsp.code) {
if (CTG_TABLE_NOT_EXIST(rpcRsp.code)) {
output->metaNum = 0;
ctgDebug("tbmeta:%s not exist in vnode", pTableName->tname);
return TSDB_CODE_SUCCESS;
}
ctgError("error rsp for table meta, code:%x", rpcRsp.code); ctgError("error rsp for table meta, code:%x", rpcRsp.code);
CTG_ERR_RET(rpcRsp.code); CTG_ERR_RET(rpcRsp.code);
} }
@ -322,7 +358,7 @@ _return:
CTG_RET(TSDB_CODE_SUCCESS); CTG_RET(TSDB_CODE_SUCCESS);
} }
int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta) { int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, bool forceUpdate, STableMeta** pTableMeta, int32_t isSTable) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) { if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pTableName || NULL == pTableMeta) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
@ -332,12 +368,18 @@ int32_t ctgGetTableMetaImpl(struct SCatalog* pCatalog, void *pRpc, const SEpSet*
if (!forceUpdate) { if (!forceUpdate) {
CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist));
if (exist) { if (exist && CTG_TBTYPE_MATCH(isSTable, (*pTableMeta)->tableType)) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} else if (CTG_IS_UNKNOWN_STABLE(isSTable)) {
int32_t tbType = 0;
CTG_ERR_RET(ctgGetTableTypeFromCache(pCatalog, pTableName, &tbType));
CTG_SET_STABLE(isSTable, tbType);
} }
CTG_ERR_RET(catalogRenewTableMeta(pCatalog, pRpc, pMgmtEps, pTableName)); CTG_ERR_RET(ctgRenewTableMetaImpl(pCatalog, pRpc, pMgmtEps, pTableName, isSTable));
CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist)); CTG_ERR_RET(ctgGetTableMetaFromCache(pCatalog, pTableName, pTableMeta, &exist));
@ -364,19 +406,27 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out
} }
if (NULL == pCatalog->tableCache.cache) { if (NULL == pCatalog->tableCache.cache) {
pCatalog->tableCache.cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.cache) { if (NULL == cache) {
ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
} }
if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.cache, NULL, cache)) {
taosHashCleanup(cache);
}
} }
if (NULL == pCatalog->tableCache.stableCache) { if (NULL == pCatalog->tableCache.stableCache) {
pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.stableCache) { if (NULL == cache) {
ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum); ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
} }
if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->tableCache.stableCache, NULL, cache)) {
taosHashCleanup(cache);
}
} }
if (output->metaNum == 2) { if (output->metaNum == 2) {
@ -481,6 +531,50 @@ int32_t ctgValidateAndRemoveDb(struct SCatalog* pCatalog, const char* dbName, SD
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t ctgRenewTableMetaImpl(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) {
if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
SVgroupInfo vgroupInfo = {0};
int32_t code = 0;
CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo));
STableMetaOutput voutput = {0};
STableMetaOutput moutput = {0};
STableMetaOutput *output = &voutput;
if (CTG_IS_STABLE(isSTable)) {
CTG_ERR_JRET(ctgGetTableMetaFromMnode(pCatalog, pTransporter, pMgmtEps, pTableName, &moutput));
if (0 == moutput.metaNum) {
CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput));
} else {
output = &moutput;
}
} else {
CTG_ERR_JRET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &voutput));
if (voutput.metaNum > 0 && TSDB_SUPER_TABLE == voutput.tbMeta->tableType) {
CTG_ERR_JRET(ctgGetTableMetaFromMnodeImpl(pCatalog, pTransporter, pMgmtEps, voutput.tbFname, &moutput));
tfree(voutput.tbMeta);
voutput.tbMeta = moutput.tbMeta;
moutput.tbMeta = NULL;
}
}
CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, output));
_return:
tfree(voutput.tbMeta);
tfree(moutput.tbMeta);
CTG_RET(code);
}
int32_t catalogInit(SCatalogCfg *cfg) { int32_t catalogInit(SCatalogCfg *cfg) {
if (ctgMgmt.pCluster) { if (ctgMgmt.pCluster) {
@ -643,11 +737,15 @@ int32_t catalogUpdateDBVgroup(struct SCatalog* pCatalog, const char* dbName, SDB
} }
if (NULL == pCatalog->dbCache.cache) { if (NULL == pCatalog->dbCache.cache) {
pCatalog->dbCache.cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); SHashObj *cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->dbCache.cache) { if (NULL == cache) {
ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER); ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR); CTG_ERR_JRET(TSDB_CODE_CTG_MEM_ERROR);
} }
if (NULL != atomic_val_compare_exchange_ptr(&pCatalog->dbCache.cache, NULL, cache)) {
taosHashCleanup(cache);
}
} else { } else {
CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo)); CTG_ERR_JRET(ctgValidateAndRemoveDb(pCatalog, dbName, dbInfo));
} }
@ -672,34 +770,23 @@ _return:
} }
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta); return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, -1);
} }
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName) { int32_t catalogGetSTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, false, pTableMeta, 1);
}
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, int32_t isSTable) {
if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) { if (NULL == pCatalog || NULL == pTransporter || NULL == pMgmtEps || NULL == pTableName) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
} }
SVgroupInfo vgroupInfo = {0}; return ctgRenewTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, isSTable);
int32_t code = 0;
CTG_ERR_RET(catalogGetTableHashVgroup(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo));
STableMetaOutput output = {0};
CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pTransporter, pMgmtEps, pTableName, &vgroupInfo, &output));
//CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pTableName, &output));
CTG_ERR_JRET(ctgUpdateTableMetaCache(pCatalog, &output));
_return:
tfree(output.tbMeta);
CTG_RET(code);
} }
int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta) { int32_t catalogRenewAndGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const SName* pTableName, STableMeta** pTableMeta, int32_t isSTable) {
return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta); return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pTableName, true, pTableMeta, isSTable);
} }
int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) { int32_t catalogGetTableDistVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const SName* pTableName, SArray** pVgroupList) {

7
source/libs/catalog/test/catalogTests.cpp
View File

@ -45,7 +45,7 @@ void ctgTestSetPrepareSTableMeta();
bool ctgTestStop = false; bool ctgTestStop = false;
bool ctgTestEnableSleep = false; bool ctgTestEnableSleep = false;
bool ctgTestDeadLoop = true; bool ctgTestDeadLoop = false;
int32_t ctgTestCurrentVgVersion = 0; int32_t ctgTestCurrentVgVersion = 0;
int32_t ctgTestVgVersion = 1; int32_t ctgTestVgVersion = 1;
@ -600,7 +600,6 @@ void *ctgTestSetCtableMetaThread(void *param) {
} }
#if 0
TEST(tableMeta, normalTable) { TEST(tableMeta, normalTable) {
struct SCatalog* pCtg = NULL; struct SCatalog* pCtg = NULL;
@ -768,7 +767,7 @@ TEST(tableMeta, superTableCase) {
ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); ASSERT_EQ(tableMeta->tableInfo.rowSize, 12);
tableMeta = NULL; tableMeta = NULL;
code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta); code = catalogRenewAndGetTableMeta(pCtg, mockPointer, (const SEpSet *)mockPointer, &n, &tableMeta, 0);
ASSERT_EQ(code, 0); ASSERT_EQ(code, 0);
ASSERT_EQ(tableMeta->vgId, 9); ASSERT_EQ(tableMeta->vgId, 9);
ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE); ASSERT_EQ(tableMeta->tableType, TSDB_CHILD_TABLE);
@ -999,8 +998,6 @@ TEST(multiThread, getSetDbVgroupCase) {
catalogDestroy(); catalogDestroy();
} }
#endif
TEST(multiThread, ctableMeta) { TEST(multiThread, ctableMeta) {
struct SCatalog* pCtg = NULL; struct SCatalog* pCtg = NULL;

3
source/libs/index/inc/index_cache.h
View File

@ -40,11 +40,12 @@ typedef struct IndexCache {
SIndex* index; SIndex* index;
char* colName; char* colName;
int32_t version; int32_t version;
int32_t nTerm; int64_t occupiedMem;
int8_t type; int8_t type;
uint64_t suid; uint64_t suid;
pthread_mutex_t mtx; pthread_mutex_t mtx;
pthread_cond_t finished;
} IndexCache; } IndexCache;
#define CACHE_VERSION(cache) atomic_load_32(&cache->version) #define CACHE_VERSION(cache) atomic_load_32(&cache->version)

24
source/libs/index/src/index.c
View File

@ -399,6 +399,8 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) {
if (sIdx == NULL) { return -1; } if (sIdx == NULL) { return -1; }
indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid); indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
int64_t st = taosGetTimestampUs();
IndexCache* pCache = (IndexCache*)cache; IndexCache* pCache = (IndexCache*)cache;
TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName);
if (pReader == NULL) { indexWarn("empty tfile reader found"); } if (pReader == NULL) { indexWarn("empty tfile reader found"); }
@ -452,16 +454,13 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) {
while (tn == true) { while (tn == true) {
IterateValue* tv = tfileIter->getValue(tfileIter); IterateValue* tv = tfileIter->getValue(tfileIter);
TFileValue* tfv = tfileValueCreate(tv->colVal); TFileValue* tfv = tfileValueCreate(tv->colVal);
if (tv->val == NULL) {
// HO
printf("NO....");
}
taosArrayAddAll(tfv->tableId, tv->val); taosArrayAddAll(tfv->tableId, tv->val);
indexMergeSameKey(result, tfv); indexMergeSameKey(result, tfv);
tn = tfileIter->next(tfileIter); tn = tfileIter->next(tfileIter);
} }
int ret = indexGenTFile(sIdx, pCache, result); int ret = indexGenTFile(sIdx, pCache, result);
indexDestroyTempResult(result); indexDestroyTempResult(result);
indexCacheDestroyImm(pCache); indexCacheDestroyImm(pCache);
indexCacheIteratorDestroy(cacheIter); indexCacheIteratorDestroy(cacheIter);
@ -469,7 +468,14 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) {
tfileReaderUnRef(pReader); tfileReaderUnRef(pReader);
indexCacheUnRef(pCache); indexCacheUnRef(pCache);
return 0;
int64_t cost = taosGetTimestampUs() - st;
if (ret != 0) {
indexError("failed to merge, time cost: %" PRId64 "ms", cost / 1000);
} else {
indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000);
}
return ret;
} }
void iterateValueDestroy(IterateValue* value, bool destroy) { void iterateValueDestroy(IterateValue* value, bool destroy) {
if (destroy) { if (destroy) {
@ -502,8 +508,7 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
if (reader == NULL) { goto END; } if (reader == NULL) { goto END; }
TFileHeader* header = &reader->header; TFileHeader* header = &reader->header;
ICacheKey key = { ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)};
.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType};
pthread_mutex_lock(&sIdx->mtx); pthread_mutex_lock(&sIdx->mtx);
IndexTFile* ifile = (IndexTFile*)sIdx->tindex; IndexTFile* ifile = (IndexTFile*)sIdx->tindex;
@ -511,7 +516,10 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
pthread_mutex_unlock(&sIdx->mtx); pthread_mutex_unlock(&sIdx->mtx);
return ret; return ret;
END: END:
tfileWriterClose(tw); if (tw != NULL) {
writerCtxDestroy(tw->ctx, true);
free(tw);
}
return -1; return -1;
} }

36
source/libs/index/src/index_cache.c
View File

@ -21,10 +21,8 @@
#define MAX_INDEX_KEY_LEN 256 // test only, change later #define MAX_INDEX_KEY_LEN 256 // test only, change later
#define MEM_TERM_LIMIT 10 * 10000 #define MEM_TERM_LIMIT 10 * 10000
// ref index_cache.h:22 #define MEM_THRESHOLD 1024 * 1024 * 2
//#define CACHE_KEY_LEN(p) \ #define MEM_ESTIMATE_RADIO 1.5
// (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) +
// sizeof(p->operType))
static void indexMemRef(MemTable* tbl); static void indexMemRef(MemTable* tbl);
static void indexMemUnRef(MemTable* tbl); static void indexMemUnRef(MemTable* tbl);
@ -54,7 +52,11 @@ IndexCache* indexCacheCreate(SIndex* idx, uint64_t suid, const char* colName, in
cache->index = idx; cache->index = idx;
cache->version = 0; cache->version = 0;
cache->suid = suid; cache->suid = suid;
cache->occupiedMem = 0;
pthread_mutex_init(&cache->mtx, NULL); pthread_mutex_init(&cache->mtx, NULL);
pthread_cond_init(&cache->finished, NULL);
indexCacheRef(cache); indexCacheRef(cache);
return cache; return cache;
} }
@ -125,6 +127,7 @@ void indexCacheDestroyImm(IndexCache* cache) {
pthread_mutex_lock(&cache->mtx); pthread_mutex_lock(&cache->mtx);
tbl = cache->imm; tbl = cache->imm;
cache->imm = NULL; // or throw int bg thread cache->imm = NULL; // or throw int bg thread
pthread_cond_broadcast(&cache->finished);
pthread_mutex_unlock(&cache->mtx); pthread_mutex_unlock(&cache->mtx);
indexMemUnRef(tbl); indexMemUnRef(tbl);
@ -137,6 +140,9 @@ void indexCacheDestroy(void* cache) {
indexMemUnRef(pCache->imm); indexMemUnRef(pCache->imm);
free(pCache->colName); free(pCache->colName);
pthread_mutex_destroy(&pCache->mtx);
pthread_cond_destroy(&pCache->finished);
free(pCache); free(pCache);
} }
@ -177,19 +183,19 @@ int indexCacheSchedToMerge(IndexCache* pCache) {
} }
static void indexCacheMakeRoomForWrite(IndexCache* cache) { static void indexCacheMakeRoomForWrite(IndexCache* cache) {
while (true) { while (true) {
if (cache->nTerm < MEM_TERM_LIMIT) { if (cache->occupiedMem * MEM_ESTIMATE_RADIO < MEM_THRESHOLD) {
cache->nTerm += 1;
break; break;
} else if (cache->imm != NULL) { } else if (cache->imm != NULL) {
// TODO: wake up by condition variable // TODO: wake up by condition variable
pthread_mutex_unlock(&cache->mtx); // pthread_mutex_unlock(&cache->mtx);
taosMsleep(50); pthread_cond_wait(&cache->finished, &cache->mtx);
pthread_mutex_lock(&cache->mtx); // taosMsleep(50);
// pthread_mutex_lock(&cache->mtx);
} else { } else {
indexCacheRef(cache); indexCacheRef(cache);
cache->imm = cache->mem; cache->imm = cache->mem;
cache->mem = indexInternalCacheCreate(cache->type); cache->mem = indexInternalCacheCreate(cache->type);
cache->nTerm = 1; cache->occupiedMem = 0;
// sched to merge // sched to merge
// unref cache in bgwork // unref cache in bgwork
indexCacheSchedToMerge(cache); indexCacheSchedToMerge(cache);
@ -215,8 +221,9 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
ct->operaType = term->operType; ct->operaType = term->operType;
// ugly code, refactor later // ugly code, refactor later
int64_t estimate = sizeof(ct) + strlen(ct->colVal);
pthread_mutex_lock(&pCache->mtx); pthread_mutex_lock(&pCache->mtx);
pCache->occupiedMem += estimate;
indexCacheMakeRoomForWrite(pCache); indexCacheMakeRoomForWrite(pCache);
MemTable* tbl = pCache->mem; MemTable* tbl = pCache->mem;
indexMemRef(tbl); indexMemRef(tbl);
@ -275,14 +282,12 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV
SIndexTerm* term = query->term; SIndexTerm* term = query->term;
EIndexQueryType qtype = query->qType; EIndexQueryType qtype = query->qType;
CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)}; CacheTerm ct = {.colVal = term->colVal, .version = atomic_load_32(&pCache->version)};
// indexCacheDebug(pCache);
int ret = indexQueryMem(mem, &ct, qtype, result, s); int ret = indexQueryMem(mem, &ct, qtype, result, s);
if (ret == 0 && *s != kTypeDeletion) { if (ret == 0 && *s != kTypeDeletion) {
// continue search in imm // continue search in imm
ret = indexQueryMem(imm, &ct, qtype, result, s); ret = indexQueryMem(imm, &ct, qtype, result, s);
} }
// cacheTermDestroy(ct);
indexMemUnRef(mem); indexMemUnRef(mem);
indexMemUnRef(imm); indexMemUnRef(imm);
@ -339,7 +344,7 @@ static int32_t compareKey(const void* l, const void* r) {
static MemTable* indexInternalCacheCreate(int8_t type) { static MemTable* indexInternalCacheCreate(int8_t type) {
MemTable* tbl = calloc(1, sizeof(MemTable)); MemTable* tbl = calloc(1, sizeof(MemTable));
indexMemRef(tbl); indexMemRef(tbl);
if (type == TSDB_DATA_TYPE_BINARY) { if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) {
tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey); tbl->mem = tSkipListCreate(MAX_SKIP_LIST_LEVEL, type, MAX_INDEX_KEY_LEN, compareKey, SL_ALLOW_DUP_KEY, getIndexKey);
} }
return tbl; return tbl;
@ -354,9 +359,6 @@ static bool indexCacheIteratorNext(Iterate* itera) {
SSkipListIterator* iter = itera->iter; SSkipListIterator* iter = itera->iter;
if (iter == NULL) { return false; } if (iter == NULL) { return false; }
IterateValue* iv = &itera->val; IterateValue* iv = &itera->val;
if (iv->colVal != NULL && iv->val != NULL) {
// indexError("value in cache: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
}
iterateValueDestroy(iv, false); iterateValueDestroy(iv, false);
bool next = tSkipListIterNext(iter); bool next = tSkipListIterNext(iter);

56
source/libs/index/src/index_tfile.c
View File

@ -61,9 +61,6 @@ TFileCache* tfileCacheCreate(const char* path) {
tcache->capacity = 64; tcache->capacity = 64;
SArray* files = tfileGetFileList(path); SArray* files = tfileGetFileList(path);
uint64_t suid;
int32_t colId, version;
for (size_t i = 0; i < taosArrayGetSize(files); i++) { for (size_t i = 0; i < taosArrayGetSize(files); i++) {
char* file = taosArrayGetP(files, i); char* file = taosArrayGetP(files, i);
@ -76,10 +73,9 @@ TFileCache* tfileCacheCreate(const char* path) {
TFileReader* reader = tfileReaderCreate(wc); TFileReader* reader = tfileReaderCreate(wc);
if (reader == NULL) { goto End; } if (reader == NULL) { goto End; }
TFileHeader* header = &reader->header; TFileHeader* header = &reader->header;
ICacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName)};
char buf[128] = {0}; char buf[128] = {0};
ICacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName)};
int32_t sz = indexSerialCacheKey(&key, buf); int32_t sz = indexSerialCacheKey(&key, buf);
assert(sz < sizeof(buf)); assert(sz < sizeof(buf));
taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*)); taosHashPut(tcache->tableCache, buf, sz, &reader, sizeof(void*));
@ -212,24 +208,13 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
tfileGenFileFullName(fullname, path, suid, colName, version); tfileGenFileFullName(fullname, path, suid, colName, version);
WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
// indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size);
if (wc == NULL) { return NULL; } if (wc == NULL) { return NULL; }
TFileReader* reader = tfileReaderCreate(wc); TFileReader* reader = tfileReaderCreate(wc);
return reader; return reader;
} }
TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) { TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
// char pathBuf[128] = {0};
// sprintf(pathBuf, "%s/% " PRIu64 "-%d-%d.tindex", path, suid, colId, version);
// TFileHeader header = {.suid = suid, .version = version, .colName = {0}, colType = colType};
// memcpy(header.colName, );
// char buf[TFILE_HADER_PRE_SIZE];
// int len = TFILE_HADER_PRE_SIZE;
// if (len != ctx->write(ctx, buf, len)) {
// indexError("index: %" PRIu64 " failed to write header info", header->suid);
// return NULL;
//}
TFileWriter* tw = calloc(1, sizeof(TFileWriter)); TFileWriter* tw = calloc(1, sizeof(TFileWriter));
if (tw == NULL) { if (tw == NULL) {
indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid); indexError("index: %" PRIu64 " failed to alloc TFilerWriter", header->suid);
@ -278,34 +263,14 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
// check buf has enough space or not // check buf has enough space or not
int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz); int32_t ttsz = TF_TABLE_TATOAL_SIZE(tbsz);
// if (offset + ttsz >= bufLimit) {
// // batch write
// indexInfo("offset: %d, ttsz: %d", offset, ttsz);
// // std::cout << "offset: " << offset << std::endl;
// // std::cout << "ttsz:" << ttsz < < < std::endl;
// tw->ctx->write(tw->ctx, buf, offset);
// offset = 0;
// memset(buf, 0, bufLimit);
// p = buf;
//}
// if (ttsz >= bufLimit) {
//}
char* buf = calloc(1, ttsz * sizeof(char)); char* buf = calloc(1, ttsz * sizeof(char));
char* p = buf; char* p = buf;
tfileSerialTableIdsToBuf(p, v->tableId); tfileSerialTableIdsToBuf(p, v->tableId);
tw->ctx->write(tw->ctx, buf, ttsz); tw->ctx->write(tw->ctx, buf, ttsz);
// offset += ttsz;
// p = buf + offset;
// set up value offset
v->offset = tw->offset; v->offset = tw->offset;
tw->offset += ttsz; tw->offset += ttsz;
free(buf); free(buf);
} }
// if (offset != 0) {
// write reversed data in buf to tindex
// tw->ctx->write(tw->ctx, buf, offset);
//}
// tfree(buf);
tw->fb = fstBuilderCreate(tw->ctx, 0); tw->fb = fstBuilderCreate(tw->ctx, 0);
if (tw->fb == NULL) { if (tw->fb == NULL) {
@ -381,9 +346,6 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
} }
static bool tfileIteratorNext(Iterate* iiter) { static bool tfileIteratorNext(Iterate* iiter) {
IterateValue* iv = &iiter->val; IterateValue* iv = &iiter->val;
if (iv->colVal != NULL && iv->val != NULL) {
// indexError("value in fst: colVal: %s, size: %d", iv->colVal, (int)taosArrayGetSize(iv->val));
}
iterateValueDestroy(iv, false); iterateValueDestroy(iv, false);
char* colVal = NULL; char* colVal = NULL;
@ -643,15 +605,9 @@ static void tfileDestroyFileName(void* elem) {
free(p); free(p);
} }
static int tfileCompare(const void* a, const void* b) { static int tfileCompare(const void* a, const void* b) {
const char* aName = *(char**)a; const char* as = *(char**)a;
const char* bName = *(char**)b; const char* bs = *(char**)b;
return strcmp(as, bs);
size_t aLen = strlen(aName);
size_t bLen = strlen(bName);
int ret = strncmp(aName, bName, aLen > bLen ? aLen : bLen);
if (ret == 0) { return ret; }
return ret < 0 ? -1 : 1;
} }
static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version) { static int tfileParseFileName(const char* filename, uint64_t* suid, char* col, int* version) {

6
source/libs/index/test/indexTests.cc
View File

@ -28,7 +28,7 @@
#include "tutil.h" #include "tutil.h"
using namespace std; using namespace std;
#define NUM_OF_THREAD 5 #define NUM_OF_THREAD 10
class DebugInfo { class DebugInfo {
public: public:
@ -882,8 +882,8 @@ static void single_write_and_search(IndexObj* idx) {
static void multi_write_and_search(IndexObj* idx) { static void multi_write_and_search(IndexObj* idx) {
int target = idx->SearchOne("tag1", "Hello"); int target = idx->SearchOne("tag1", "Hello");
target = idx->SearchOne("tag2", "Test"); target = idx->SearchOne("tag2", "Test");
idx->WriteMultiMillonData("tag1", "Hello", 100 * 10000); idx->WriteMultiMillonData("tag1", "hello world test", 100 * 10000);
idx->WriteMultiMillonData("tag2", "Test", 100 * 10000); idx->WriteMultiMillonData("tag2", "world test nothing", 100 * 10000);
} }
TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) { TEST_F(IndexEnv2, testIndex_serarch_cache_and_tfile) {
std::string path = "/tmp/cache_and_tfile"; std::string path = "/tmp/cache_and_tfile";

2
source/libs/parser/inc/astToMsg.h
View File

@ -10,7 +10,7 @@ SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, in
SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen);
SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen); SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen);
SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf);
SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); SCreateStbMsg* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf);
SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf);
SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf);
SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf);

8
source/libs/parser/inc/parserInt.h
View File

@ -38,14 +38,6 @@ typedef struct SMsgBuf {
char *buf; char *buf;
} SMsgBuf; } SMsgBuf;
// create table operation type
enum TSQL_CREATE_TABLE_TYPE {
TSQL_CREATE_TABLE = 0x1,
TSQL_CREATE_STABLE = 0x2,
TSQL_CREATE_CTABLE = 0x3,
TSQL_CREATE_STREAM = 0x4,
};
void clearTableMetaInfo(STableMetaInfo* pTableMetaInfo); void clearTableMetaInfo(STableMetaInfo* pTableMetaInfo);
void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t id); void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t id);

16
source/libs/parser/inc/sql.y
View File

@ -370,7 +370,7 @@ create_table_list(A) ::= create_from_stable(Z). {
pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo));
taosArrayPush(pCreateTable->childTableInfo, &Z); taosArrayPush(pCreateTable->childTableInfo, &Z);
pCreateTable->type = TSQL_CREATE_CTABLE; pCreateTable->type = TSDB_SQL_CREATE_TABLE;
A = pCreateTable; A = pCreateTable;
} }
@ -381,7 +381,7 @@ create_table_list(A) ::= create_table_list(X) create_from_stable(Z). {
%type create_table_args{SCreateTableSql*} %type create_table_args{SCreateTableSql*}
create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. { create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. {
A = tSetCreateTableInfo(X, NULL, NULL, TSQL_CREATE_TABLE); A = tSetCreateTableInfo(X, NULL, NULL, TSDB_SQL_CREATE_TABLE);
setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE);
V.n += Z.n; V.n += Z.n;
@ -391,7 +391,7 @@ create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP. {
// create super table // create super table
%type create_stable_args{SCreateTableSql*} %type create_stable_args{SCreateTableSql*}
create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. { create_stable_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) LP columnlist(X) RP TAGS LP columnlist(Y) RP. {
A = tSetCreateTableInfo(X, Y, NULL, TSQL_CREATE_STABLE); A = tSetCreateTableInfo(X, Y, NULL, TSDB_SQL_CREATE_STABLE);
setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_STABLE); setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_STABLE);
V.n += Z.n; V.n += Z.n;
@ -421,11 +421,11 @@ tagNamelist(A) ::= ids(X). {A = taosArrayInit(4, sizeof(STo
// create stream // create stream
// create table table_name as select count(*) from super_table_name interval(time) // create table table_name as select count(*) from super_table_name interval(time)
create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). { create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). {
A = tSetCreateTableInfo(NULL, NULL, S, TSQL_CREATE_STREAM); // A = tSetCreateTableInfo(NULL, NULL, S, TSQL_CREATE_STREAM);
setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE); // setSqlInfo(pInfo, A, NULL, TSDB_SQL_CREATE_TABLE);
//
V.n += Z.n; // V.n += Z.n;
setCreatedTableName(pInfo, &V, &U); // setCreatedTableName(pInfo, &V, &U);
} }
%type column{SField} %type column{SField}

18
source/libs/parser/src/astGenerator.c
View File

@ -579,25 +579,21 @@ SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSe
SCreateTableSql *pCreate = calloc(1, sizeof(SCreateTableSql)); SCreateTableSql *pCreate = calloc(1, sizeof(SCreateTableSql));
switch (type) { switch (type) {
case TSQL_CREATE_TABLE: { case TSDB_SQL_CREATE_TABLE: {
pCreate->colInfo.pColumns = pCols; pCreate->colInfo.pColumns = pCols;
assert(pTags == NULL); assert(pTags == NULL);
break; break;
} }
case TSQL_CREATE_STABLE: { case TSDB_SQL_CREATE_STABLE: {
pCreate->colInfo.pColumns = pCols; pCreate->colInfo.pColumns = pCols;
pCreate->colInfo.pTagColumns = pTags; pCreate->colInfo.pTagColumns = pTags;
assert(pTags != NULL && pCols != NULL); assert(pTags != NULL && pCols != NULL);
break; break;
} }
case TSQL_CREATE_STREAM: { // case TSQL_CREATE_STREAM: {
pCreate->pSelect = pSelect; // pCreate->pSelect = pSelect;
break; // break;
} // }
case TSQL_CREATE_CTABLE: {
assert(0);
}
default: default:
assert(false); assert(false);
@ -785,7 +781,7 @@ void destroySqlInfo(SSqlInfo *pInfo) {
taosArrayDestroy(pInfo->funcs); taosArrayDestroy(pInfo->funcs);
if (pInfo->type == TSDB_SQL_SELECT) { if (pInfo->type == TSDB_SQL_SELECT) {
destroyAllSqlNode(&pInfo->sub); destroyAllSqlNode(&pInfo->sub);
} else if (pInfo->type == TSDB_SQL_CREATE_STABLE) { } else if (pInfo->type == TSDB_SQL_CREATE_STABLE || pInfo->type == TSDB_SQL_CREATE_TABLE) {
pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo); pInfo->pCreateTableInfo = destroyCreateTableSql(pInfo->pCreateTableInfo);
} else if (pInfo->type == TSDB_SQL_ALTER_TABLE) { } else if (pInfo->type == TSDB_SQL_ALTER_TABLE) {
taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeItem); taosArrayDestroyEx(pInfo->pAlterInfo->varList, freeItem);

29
source/libs/parser/src/astToMsg.c
View File

@ -230,7 +230,7 @@ SCreateDbMsg* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCt
return pCreateMsg; return pCreateMsg;
} }
SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { SCreateStbMsg* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) {
SSchema* pSchema; SSchema* pSchema;
int32_t numOfTags = 0; int32_t numOfTags = 0;
@ -239,16 +239,16 @@ SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* le
numOfTags = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pTagColumns); numOfTags = (int32_t) taosArrayGetSize(pCreateTableSql->colInfo.pTagColumns);
} }
SCreateStbMsg* pCreateTableMsg = (SCreateStbMsg*)calloc(1, sizeof(SCreateStbMsg) + (numOfCols + numOfTags) * sizeof(SSchema)); SCreateStbMsg* pCreateStbMsg = (SCreateStbMsg*)calloc(1, sizeof(SCreateStbMsg) + (numOfCols + numOfTags) * sizeof(SSchema));
char* pMsg = NULL; char* pMsg = NULL;
#if 0
int32_t tableType = pCreateTableSql->type; int32_t tableType = pCreateTableSql->type;
if (tableType != TSQL_CREATE_TABLE && tableType != TSQL_CREATE_STABLE) { // create by using super table, tags value if (tableType != TSQL_CREATE_TABLE && tableType != TSQL_CREATE_STABLE) { // create by using super table, tags value
#if 0
SArray* list = pInfo->pCreateTableInfo->childTableInfo; SArray* list = pInfo->pCreateTableInfo->childTableInfo;
int32_t numOfTables = (int32_t)taosArrayGetSize(list); int32_t numOfTables = (int32_t)taosArrayGetSize(list);
pCreateTableMsg->numOfTables = htonl(numOfTables); pCreateStbMsg->numOfTables = htonl(numOfTables);
pMsg = (char*)pCreateMsg; pMsg = (char*)pCreateMsg;
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
@ -268,25 +268,27 @@ SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* le
int32_t len = (int32_t)(pMsg - (char*)pCreate); int32_t len = (int32_t)(pMsg - (char*)pCreate);
pCreate->len = htonl(len); pCreate->len = htonl(len);
} }
} else {
#endif #endif
} else { // create (super) table // create (super) table
SName n = {0}; SName n = {0};
int32_t code = createSName(&n, &pCreateTableSql->name, pParseCtx, pMsgBuf); int32_t code = createSName(&n, &pCreateTableSql->name, pParseCtx, pMsgBuf);
if (code != 0) { if (code != 0) {
return NULL; return NULL;
} }
code = tNameExtractFullName(&n, pCreateTableMsg->name); code = tNameExtractFullName(&n, pCreateStbMsg->name);
if (code != 0) { if (code != 0) {
buildInvalidOperationMsg(pMsgBuf, "invalid table name or database not specified"); buildInvalidOperationMsg(pMsgBuf, "invalid table name or database not specified");
return NULL; return NULL;
} }
pCreateTableMsg->igExists = pCreateTableSql->existCheck ? 1 : 0; pCreateStbMsg->igExists = pCreateTableSql->existCheck ? 1 : 0;
pCreateTableMsg->numOfColumns = htonl(numOfCols); pCreateStbMsg->numOfColumns = htonl(numOfCols);
pCreateTableMsg->numOfTags = htonl(numOfTags); pCreateStbMsg->numOfTags = htonl(numOfTags);
pSchema = (SSchema*) pCreateTableMsg->pSchema; pSchema = (SSchema*)pCreateStbMsg->pSchema;
for (int i = 0; i < numOfCols; ++i) { for (int i = 0; i < numOfCols; ++i) {
SField* pField = taosArrayGet(pCreateTableSql->colInfo.pColumns, i); SField* pField = taosArrayGet(pCreateTableSql->colInfo.pColumns, i);
pSchema->type = pField->type; pSchema->type = pField->type;
@ -306,12 +308,11 @@ SCreateStbMsg* buildCreateTableMsg(SCreateTableSql* pCreateTableSql, int32_t* le
} }
pMsg = (char*)pSchema; pMsg = (char*)pSchema;
}
int32_t msgLen = (int32_t)(pMsg - (char*)pCreateTableMsg); int32_t msgLen = (int32_t)(pMsg - (char*)pCreateStbMsg);
*len = msgLen; *len = msgLen;
return pCreateTableMsg; return pCreateStbMsg;
} }
SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) {
@ -330,7 +331,7 @@ SDropStbMsg* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* p
assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_TABLE_NAME_T); assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_TABLE_NAME_T);
pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; pDropTableMsg->igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0;
*len = sizeof(SDropTableMsg); *len = sizeof(SDropStbMsg);
return pDropTableMsg; return pDropTableMsg;
} }

277
source/libs/parser/src/dCDAstProcess.c
View File

@ -1,10 +1,12 @@
#include <astGenerator.h>
#include <tmsg.h> #include <tmsg.h>
#include <ttime.h>
#include "astToMsg.h" #include "astToMsg.h"
#include "parserInt.h" #include "parserInt.h"
#include "parserUtil.h" #include "parserUtil.h"
#include "queryInfoUtil.h" #include "queryInfoUtil.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h"
#include "ttime.h"
/* is contained in pFieldList or not */ /* is contained in pFieldList or not */
static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { static bool has(SArray* pFieldList, int32_t startIndex, const char* name) {
@ -248,16 +250,10 @@ static int32_t validateTableColumns(SArray* pFieldList, int32_t maxRowLength, in
} }
static int32_t validateTableColumnInfo(SArray* pFieldList, SMsgBuf* pMsgBuf) { static int32_t validateTableColumnInfo(SArray* pFieldList, SMsgBuf* pMsgBuf) {
assert(pFieldList != NULL); assert(pFieldList != NULL && pMsgBuf != NULL);
const char* msg1 = "first column must be timestamp"; const char* msg1 = "first column must be timestamp";
const char* msg2 = "row length exceeds max length"; const char* msg2 = "illegal number of columns";
const char* msg3 = "duplicated column names";
const char* msg4 = "invalid data type";
const char* msg5 = "invalid binary/nchar column length";
const char* msg6 = "invalid column name";
const char* msg7 = "too many columns";
const char* msg8 = "illegal number of columns";
// first column must be timestamp // first column must be timestamp
SField* pField = taosArrayGet(pFieldList, 0); SField* pField = taosArrayGet(pFieldList, 0);
@ -268,7 +264,7 @@ static int32_t validateTableColumnInfo(SArray* pFieldList, SMsgBuf* pMsgBuf) {
// number of fields no less than 2 // number of fields no less than 2
size_t numOfCols = taosArrayGetSize(pFieldList); size_t numOfCols = taosArrayGetSize(pFieldList);
if (numOfCols <= 1) { if (numOfCols <= 1) {
return buildInvalidOperationMsg(pMsgBuf, msg8); return buildInvalidOperationMsg(pMsgBuf, msg2);
} }
return validateTableColumns(pFieldList, TSDB_MAX_BYTES_PER_ROW, TSDB_MAX_COLUMNS, pMsgBuf); return validateTableColumns(pFieldList, TSDB_MAX_BYTES_PER_ROW, TSDB_MAX_COLUMNS, pMsgBuf);
@ -297,11 +293,9 @@ static int32_t validateTagParams(SArray* pTagsList, SArray* pFieldList, SMsgBuf*
return validateTableColumns(pFieldList, TSDB_MAX_TAGS_LEN, TSDB_MAX_TAGS, pMsgBuf); return validateTableColumns(pFieldList, TSDB_MAX_TAGS_LEN, TSDB_MAX_TAGS, pMsgBuf);
} }
int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) { int32_t doCheckForCreateTable(SCreateTableSql* pCreateTable, SMsgBuf* pMsgBuf) {
const char* msg1 = "invalid table name"; const char* msg1 = "invalid table name";
SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
SArray* pFieldList = pCreateTable->colInfo.pColumns; SArray* pFieldList = pCreateTable->colInfo.pColumns;
SArray* pTagList = pCreateTable->colInfo.pTagColumns; SArray* pTagList = pCreateTable->colInfo.pTagColumns;
assert(pFieldList != NULL); assert(pFieldList != NULL);
@ -326,6 +320,8 @@ typedef struct SVgroupTablesBatch {
SVgroupInfo info; SVgroupInfo info;
} SVgroupTablesBatch; } SVgroupTablesBatch;
static SArray* doSerializeVgroupCreateTableInfo(SHashObj* pVgroupHashmap);
static int32_t doParseSerializeTagValue(SSchema* pTagSchema, int32_t numOfInputTag, SKVRowBuilder* pKvRowBuilder, static int32_t doParseSerializeTagValue(SSchema* pTagSchema, int32_t numOfInputTag, SKVRowBuilder* pKvRowBuilder,
SArray* pTagValList, int32_t tsPrecision, SMsgBuf* pMsgBuf) { SArray* pTagValList, int32_t tsPrecision, SMsgBuf* pMsgBuf) {
const char* msg1 = "illegal value or data overflow"; const char* msg1 = "illegal value or data overflow";
@ -351,14 +347,52 @@ static int32_t doParseSerializeTagValue(SSchema* pTagSchema, int32_t numOfInputT
return code; return code;
} }
int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { static void addCreateTbReqIntoVgroup(SHashObj* pVgroupHashmap, const SName* pTableName, SKVRow row, uint64_t suid, SVgroupInfo* pVgInfo) {
struct SVCreateTbReq req = {0};
req.type = TD_CHILD_TABLE;
req.name = strdup(tNameGetTableName(pTableName));
req.ctbCfg.suid = suid;
req.ctbCfg.pTag = row;
SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
if (pTableBatch == NULL) {
SVgroupTablesBatch tBatch = {0};
tBatch.info = *pVgInfo;
tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
taosArrayPush(tBatch.req.pArray, &req);
taosHashPut(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &tBatch, sizeof(tBatch));
} else { // add to the correct vgroup
assert(pVgInfo->vgId == pTableBatch->info.vgId);
taosArrayPush(pTableBatch->req.pArray, &req);
}
}
static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) {
size_t size = taosArrayGetSize(pTbBatch->req.pArray);
for(int32_t i = 0; i < size; ++i) {
SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i);
tfree(pTableReq->name);
if (pTableReq->type == TSDB_NORMAL_TABLE) {
tfree(pTableReq->ntbCfg.pSchema);
} else if (pTableReq->type == TSDB_CHILD_TABLE) {
tfree(pTableReq->ctbCfg.pTag);
} else {
assert(0);
}
}
taosArrayDestroy(pTbBatch->req.pArray);
}
static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) {
const char* msg1 = "invalid table name"; const char* msg1 = "invalid table name";
const char* msg2 = "tags number not matched"; const char* msg2 = "tags number not matched";
const char* msg3 = "tag value too long"; const char* msg3 = "tag value too long";
const char* msg4 = "illegal value or data overflow"; const char* msg4 = "illegal value or data overflow";
SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); SHashObj* pVgroupHashmap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
// super table name, create table by using dst // super table name, create table by using dst
@ -378,6 +412,11 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
return code; return code;
} }
SKVRowBuilder kvRowBuilder = {0};
if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SArray* pValList = pCreateTableInfo->pTagVals; SArray* pValList = pCreateTableInfo->pTagVals;
size_t numOfInputTag = taosArrayGetSize(pValList); size_t numOfInputTag = taosArrayGetSize(pValList);
@ -393,26 +432,22 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta); SSchema* pTagSchema = getTableTagSchema(pSuperTableMeta);
STableComInfo tinfo = getTableInfo(pSuperTableMeta); STableComInfo tinfo = getTableInfo(pSuperTableMeta);
SKVRowBuilder kvRowBuilder = {0};
if (tdInitKVRowBuilder(&kvRowBuilder) < 0) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SArray* pNameList = NULL; SArray* pNameList = NULL;
size_t nameSize = 0; size_t numOfBoundTags = 0;
int32_t schemaSize = getNumOfTags(pSuperTableMeta); int32_t schemaSize = getNumOfTags(pSuperTableMeta);
if (pCreateTableInfo->pTagNames) { if (pCreateTableInfo->pTagNames) {
pNameList = pCreateTableInfo->pTagNames; pNameList = pCreateTableInfo->pTagNames;
nameSize = taosArrayGetSize(pNameList); numOfBoundTags = taosArrayGetSize(pNameList);
if (numOfInputTag != nameSize || schemaSize < numOfInputTag) { if (numOfInputTag != numOfBoundTags || schemaSize < numOfInputTag) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
tfree(pSuperTableMeta);
return buildInvalidOperationMsg(pMsgBuf, msg2); return buildInvalidOperationMsg(pMsgBuf, msg2);
} }
bool findColumnIndex = false; bool findColumnIndex = false;
for (int32_t i = 0; i < nameSize; ++i) { for (int32_t i = 0; i < numOfBoundTags; ++i) {
SToken* sToken = taosArrayGet(pNameList, i); SToken* sToken = taosArrayGet(pNameList, i);
char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // create tmp buf to avoid alter orginal sqlstr char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0}; // create tmp buf to avoid alter orginal sqlstr
@ -440,6 +475,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) { if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
if (pItem->pVar.nLen > pSchema->bytes) { if (pItem->pVar.nLen > pSchema->bytes) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
tfree(pSuperTableMeta);
return buildInvalidOperationMsg(pMsgBuf, msg3); return buildInvalidOperationMsg(pMsgBuf, msg3);
} }
} else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { } else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
@ -460,12 +496,14 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
int16_t len = varDataTLen(tagVal); int16_t len = varDataTLen(tagVal);
if (len > pSchema->bytes) { if (len > pSchema->bytes) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
tfree(pSuperTableMeta);
return buildInvalidOperationMsg(pMsgBuf, msg3); return buildInvalidOperationMsg(pMsgBuf, msg3);
} }
} }
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
tfree(pSuperTableMeta);
return buildInvalidOperationMsg(pMsgBuf, msg4); return buildInvalidOperationMsg(pMsgBuf, msg4);
} }
@ -484,11 +522,14 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
} else { } else {
if (schemaSize != numOfInputTag) { if (schemaSize != numOfInputTag) {
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
tfree(pSuperTableMeta);
return buildInvalidOperationMsg(pMsgBuf, msg2); return buildInvalidOperationMsg(pMsgBuf, msg2);
} }
code = doParseSerializeTagValue(pTagSchema, numOfInputTag, &kvRowBuilder, pValList, tinfo.precision, pMsgBuf); code = doParseSerializeTagValue(pTagSchema, numOfInputTag, &kvRowBuilder, pValList, tinfo.precision, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tdDestroyKVRowBuilder(&kvRowBuilder);
tfree(pSuperTableMeta);
return code; return code;
} }
} }
@ -496,6 +537,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder); SKVRow row = tdGetKVRowFromBuilder(&kvRowBuilder);
tdDestroyKVRowBuilder(&kvRowBuilder); tdDestroyKVRowBuilder(&kvRowBuilder);
if (row == NULL) { if (row == NULL) {
tfree(pSuperTableMeta);
return TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY;
} }
@ -503,6 +545,84 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
SName tableName = {0}; SName tableName = {0};
code = createSName(&tableName, &pCreateTableInfo->name, pCtx, pMsgBuf); code = createSName(&tableName, &pCreateTableInfo->name, pCtx, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) {
tfree(pSuperTableMeta);
return code;
}
// Find a appropriate vgroup to accommodate this table , according to the table name
SVgroupInfo info = {0};
catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &tableName, &info);
addCreateTbReqIntoVgroup(pVgroupHashmap, &tableName, row, pSuperTableMeta->uid, &info);
tfree(pSuperTableMeta);
}
*pBufArray = doSerializeVgroupCreateTableInfo(pVgroupHashmap);
taosHashCleanup(pVgroupHashmap);
return TSDB_CODE_SUCCESS;
}
static int32_t serializeVgroupTablesBatchImpl(SVgroupTablesBatch* pTbBatch, SArray* pBufArray) {
int tlen = sizeof(SMsgHead) + tSVCreateTbBatchReqSerialize(NULL, &(pTbBatch->req));
void* buf = malloc(tlen);
if (buf == NULL) {
// TODO: handle error
}
((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId);
((SMsgHead*)buf)->contLen = htonl(tlen);
void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tSVCreateTbBatchReqSerialize(&pBuf, &(pTbBatch->req));
SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks));
pVgData->vg = pTbBatch->info;
pVgData->pData = buf;
pVgData->size = tlen;
pVgData->numOfTables = (int32_t) taosArrayGetSize(pTbBatch->req.pArray);
taosArrayPush(pBufArray, &pVgData);
}
static int32_t doBuildSingleTableBatchReq(SName* pTableName, SArray* pColumns, SVgroupInfo* pVgroupInfo, SVgroupTablesBatch* pBatch) {
struct SVCreateTbReq req = {0};
req.type = TD_NORMAL_TABLE;
req.name = strdup(tNameGetTableName(pTableName));
req.ntbCfg.nCols = taosArrayGetSize(pColumns);
int32_t num = req.ntbCfg.nCols;
req.ntbCfg.pSchema = calloc(num, sizeof(SSchema));
for(int32_t i = 0; i < num; ++i) {
SSchema* pSchema = taosArrayGet(pColumns, i);
memcpy(&req.ntbCfg.pSchema[i], pSchema, sizeof(SSchema));
}
pBatch->info = *pVgroupInfo;
pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq));
if (pBatch->req.pArray == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
taosArrayPush(pBatch->req.pArray, &req);
return TSDB_CODE_SUCCESS;
}
int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) {
SArray* pBufArray = NULL;
// it is a sql statement to create a normal table
if (pCreateTable->childTableInfo == NULL) {
assert(taosArrayGetSize(pCreateTable->colInfo.pColumns) > 0 && pCreateTable->colInfo.pTagColumns == NULL);
int32_t code = doCheckForCreateTable(pCreateTable, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SName tableName = {0};
code = createSName(&tableName, &pCreateTable->name, pCtx, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
@ -510,56 +630,24 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
SVgroupInfo info = {0}; SVgroupInfo info = {0};
catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &tableName, &info); catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, &tableName, &info);
struct SVCreateTbReq req = {0}; SVgroupTablesBatch tbatch = {0};
req.type = TD_CHILD_TABLE; code = doBuildSingleTableBatchReq(&tableName, pCreateTable->colInfo.pColumns, &info, &tbatch);
req.name = strdup(tNameGetTableName(&tableName)); if (code != TSDB_CODE_SUCCESS) {
req.ctbCfg.suid = pSuperTableMeta->uid; return code;
req.ctbCfg.pTag = row;
SVgroupTablesBatch* pTableBatch = taosHashGet(pVgroupHashmap, &info.vgId, sizeof(info.vgId));
if (pTableBatch == NULL) {
SVgroupTablesBatch tBatch = {0};
tBatch.info = info;
tBatch.req.pArray = taosArrayInit(4, sizeof(struct SVCreateTbReq));
taosArrayPush(tBatch.req.pArray, &req);
taosHashPut(pVgroupHashmap, &info.vgId, sizeof(info.vgId), &tBatch, sizeof(tBatch));
} else { // add to the correct vgroup
assert(info.vgId == pTableBatch->info.vgId);
taosArrayPush(pTableBatch->req.pArray, &req);
} }
pBufArray = taosArrayInit(1, POINTER_BYTES);
if (pBufArray == NULL) {
return TSDB_CODE_QRY_OUT_OF_MEMORY;
}
serializeVgroupTablesBatchImpl(&tbatch, pBufArray);
destroyCreateTbReqBatch(&tbatch);
} else { // it is a child table, created according to a super table
doCheckAndBuildCreateCTableReq(pCreateTable, pCtx, pMsgBuf, &pBufArray);
} }
// TODO: serialize and
SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*));
SVgroupTablesBatch* pTbBatch = NULL;
do {
pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch);
if (pTbBatch == NULL) break;
int tlen = sizeof(SMsgHead) + tSVCreateTbBatchReqSerialize(NULL, &(pTbBatch->req));
void* buf = malloc(tlen);
if (buf == NULL) {
// TODO: handle error
}
((SMsgHead*)buf)->vgId = htonl(pTbBatch->info.vgId);
((SMsgHead*)buf)->contLen = htonl(tlen);
void* pBuf = POINTER_SHIFT(buf, sizeof(SMsgHead));
tSVCreateTbBatchReqSerialize(&pBuf, &(pTbBatch->req));
SVgDataBlocks* pVgData = calloc(1, sizeof(SVgDataBlocks));
pVgData->vg = pTbBatch->info;
pVgData->pData = buf;
pVgData->size = tlen;
pVgData->numOfTables = (int32_t) taosArrayGetSize(pTbBatch->req.pArray);
taosArrayPush(pBufArray, &pVgData);
} while (true);
SVnodeModifOpStmtInfo* pStmtInfo = calloc(1, sizeof(SVnodeModifOpStmtInfo)); SVnodeModifOpStmtInfo* pStmtInfo = calloc(1, sizeof(SVnodeModifOpStmtInfo));
pStmtInfo->nodeType = TSDB_SQL_CREATE_TABLE; pStmtInfo->nodeType = TSDB_SQL_CREATE_TABLE;
pStmtInfo->pDataBlocks = pBufArray; pStmtInfo->pDataBlocks = pBufArray;
@ -570,6 +658,23 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SMsgBuf* p
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SArray* doSerializeVgroupCreateTableInfo(SHashObj* pVgroupHashmap) {
SArray* pBufArray = taosArrayInit(taosHashGetSize(pVgroupHashmap), sizeof(void*));
SVgroupTablesBatch* pTbBatch = NULL;
do {
pTbBatch = taosHashIterate(pVgroupHashmap, pTbBatch);
if (pTbBatch == NULL) {
break;
}
/*int32_t code = */serializeVgroupTablesBatchImpl(pTbBatch, pBufArray);
destroyCreateTbReqBatch(pTbBatch);
} while (true);
return pBufArray;
}
SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) {
int32_t code = 0; int32_t code = 0;
@ -776,21 +881,13 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c
case TSDB_SQL_CREATE_STABLE: { case TSDB_SQL_CREATE_STABLE: {
SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
assert(pCreateTable->type != TSQL_CREATE_CTABLE); if ((code = doCheckForCreateTable(pCreateTable, pMsgBuf)) != TSDB_CODE_SUCCESS) {
terrno = code;
if (pCreateTable->type == TSQL_CREATE_TABLE || pCreateTable->type == TSQL_CREATE_STABLE) { goto _error;
if ((code = doCheckForCreateTable(pInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) {
terrno = code;
goto _error;
}
pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE) ? TDMT_VND_CREATE_TABLE : TDMT_MND_CREATE_STB;
} else if (pCreateTable->type == TSQL_CREATE_STREAM) {
// if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) {
// return code;
} }
pDcl->pMsg = (char*)buildCreateStbMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
pDcl->msgType = TDMT_MND_CREATE_STB;
break; break;
} }
@ -838,19 +935,19 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c
SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) {
SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo;
assert(pCreateTable->type == TSQL_CREATE_CTABLE); assert(pCreateTable->type == TSDB_SQL_CREATE_TABLE);
SMsgBuf m = {.buf = msgBuf, .len = msgBufLen}; SMsgBuf m = {.buf = msgBuf, .len = msgBufLen};
SMsgBuf* pMsgBuf = &m; SMsgBuf* pMsgBuf = &m;
SVnodeModifOpStmtInfo* pInsertStmt = NULL; SVnodeModifOpStmtInfo* pModifSqlStmt = NULL;
int32_t msgLen = 0; int32_t msgLen = 0;
int32_t code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, (char**) &pInsertStmt, &msgLen); int32_t code = doCheckAndBuildCreateTableReq(pCreateTable, pCtx, pMsgBuf, (char**) &pModifSqlStmt, &msgLen);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
tfree(pInsertStmt); tfree(pModifSqlStmt);
return NULL; return NULL;
} }
return pInsertStmt; return pModifSqlStmt;
} }

28
source/libs/parser/src/parser.c
View File

@ -44,21 +44,21 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
} }
if (!isDqlSqlStatement(&info)) { if (!isDqlSqlStatement(&info)) {
bool toVnode = false; // bool toVnode = false;
if (info.type == TSDB_SQL_CREATE_TABLE) { if (info.type == TSDB_SQL_CREATE_TABLE) {
SCreateTableSql* pCreateSql = info.pCreateTableInfo; // SCreateTableSql* pCreateSql = info.pCreateTableInfo;
if (pCreateSql->type == TSQL_CREATE_CTABLE || pCreateSql->type == TSQL_CREATE_TABLE) { // if (pCreateSql->type == TSQL_CREATE_CTABLE || pCreateSql->type == TSQL_CREATE_TABLE) {
toVnode = true; // toVnode = true;
} // }
} // }
if (toVnode) { // if (toVnode) {
SVnodeModifOpStmtInfo *pInsertInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen);
if (pInsertInfo == NULL) { if (pModifStmtInfo == NULL) {
return terrno; return terrno;
} }
*pQuery = (SQueryNode*) pInsertInfo; *pQuery = (SQueryNode*)pModifStmtInfo;
} else { } else {
SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen);
if (pDcl == NULL) { if (pDcl == NULL) {
@ -240,6 +240,10 @@ void qParserCleanupMetaRequestInfo(SCatalogReq* pMetaReq) {
taosArrayDestroy(pMetaReq->pUdf); taosArrayDestroy(pMetaReq->pUdf);
} }
void qDestroyQuery(SQueryNode* pQuery) { void qDestroyQuery(SQueryNode* pQueryNode) {
// todo if (nodeType(pQueryNode) == TSDB_SQL_INSERT || nodeType(pQueryNode) == TSDB_SQL_CREATE_TABLE) {
SVnodeModifOpStmtInfo* pModifInfo = (SVnodeModifOpStmtInfo*)pQueryNode;
taosArrayDestroy(pModifInfo->pDataBlocks);
}
tfree(pQueryNode);
} }

16
source/libs/parser/src/sql.c
View File

@ -2642,7 +2642,7 @@ static void yy_reduce(
pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo)); pCreateTable->childTableInfo = taosArrayInit(4, sizeof(SCreatedTableInfo));
taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy150); taosArrayPush(pCreateTable->childTableInfo, &yymsp[0].minor.yy150);
pCreateTable->type = TSQL_CREATE_CTABLE; pCreateTable->type = TSDB_SQL_CREATE_TABLE;
yylhsminor.yy326 = pCreateTable; yylhsminor.yy326 = pCreateTable;
} }
yymsp[0].minor.yy326 = yylhsminor.yy326; yymsp[0].minor.yy326 = yylhsminor.yy326;
@ -2656,7 +2656,7 @@ static void yy_reduce(
break; break;
case 140: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */ case 140: /* create_table_args ::= ifnotexists ids cpxName LP columnlist RP */
{ {
yylhsminor.yy326 = tSetCreateTableInfo(yymsp[-1].minor.yy165, NULL, NULL, TSQL_CREATE_TABLE); yylhsminor.yy326 = tSetCreateTableInfo(yymsp[-1].minor.yy165, NULL, NULL, TSDB_SQL_CREATE_TABLE);
setSqlInfo(pInfo, yylhsminor.yy326, NULL, TSDB_SQL_CREATE_TABLE); setSqlInfo(pInfo, yylhsminor.yy326, NULL, TSDB_SQL_CREATE_TABLE);
yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n; yymsp[-4].minor.yy0.n += yymsp[-3].minor.yy0.n;
@ -2666,7 +2666,7 @@ static void yy_reduce(
break; break;
case 141: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */ case 141: /* create_stable_args ::= ifnotexists ids cpxName LP columnlist RP TAGS LP columnlist RP */
{ {
yylhsminor.yy326 = tSetCreateTableInfo(yymsp[-5].minor.yy165, yymsp[-1].minor.yy165, NULL, TSQL_CREATE_STABLE); yylhsminor.yy326 = tSetCreateTableInfo(yymsp[-5].minor.yy165, yymsp[-1].minor.yy165, NULL, TSDB_SQL_CREATE_STABLE);
setSqlInfo(pInfo, yylhsminor.yy326, NULL, TSDB_SQL_CREATE_STABLE); setSqlInfo(pInfo, yylhsminor.yy326, NULL, TSDB_SQL_CREATE_STABLE);
yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n; yymsp[-8].minor.yy0.n += yymsp[-7].minor.yy0.n;
@ -2700,11 +2700,11 @@ static void yy_reduce(
break; break;
case 146: /* create_table_args ::= ifnotexists ids cpxName AS select */ case 146: /* create_table_args ::= ifnotexists ids cpxName AS select */
{ {
yylhsminor.yy326 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy278, TSQL_CREATE_STREAM); // yylhsminor.yy326 = tSetCreateTableInfo(NULL, NULL, yymsp[0].minor.yy278, TSQL_CREATE_STREAM);
setSqlInfo(pInfo, yylhsminor.yy326, NULL, TSDB_SQL_CREATE_TABLE); // setSqlInfo(pInfo, yylhsminor.yy326, NULL, TSDB_SQL_CREATE_TABLE);
//
yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n; // yymsp[-3].minor.yy0.n += yymsp[-2].minor.yy0.n;
setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0); // setCreatedTableName(pInfo, &yymsp[-3].minor.yy0, &yymsp[-4].minor.yy0);
} }
yymsp[-4].minor.yy326 = yylhsminor.yy326; yymsp[-4].minor.yy326 = yylhsminor.yy326;
break; break;

12
source/libs/planner/src/logicPlan.c
View File

@ -392,6 +392,18 @@ SArray* createQueryPlanImpl(const SQueryStmtInfo* pQueryInfo) {
} }
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) { static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
if (pQueryNode->info.type == QNODE_MODIFY) {
SDataPayloadInfo* pInfo = pQueryNode->pExtInfo;
size_t size = taosArrayGetSize(pInfo->payload);
for (int32_t i = 0; i < size; ++i) {
SVgDataBlocks* pBlock = taosArrayGetP(pInfo->payload, i);
tfree(pBlock);
}
taosArrayDestroy(pInfo->payload);
}
tfree(pQueryNode->pExtInfo); tfree(pQueryNode->pExtInfo);
tfree(pQueryNode->pSchema); tfree(pQueryNode->pSchema);
tfree(pQueryNode->info.name); tfree(pQueryNode->info.name);

8
source/libs/planner/src/physicalPlan.c
View File

@ -305,10 +305,10 @@ static void splitModificationOpSubPlan(SPlanContext* pCxt, SQueryPlanNode* pPlan
SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i); SVgDataBlocks* blocks = (SVgDataBlocks*)taosArrayGetP(pPayload->payload, i);
vgroupInfoToEpSet(&blocks->vg, &subplan->execNode); vgroupInfoToEpSet(&blocks->vg, &subplan->execNode);
subplan->pDataSink = createDataInserter(pCxt, blocks); subplan->pDataSink = createDataInserter(pCxt, blocks);
subplan->pNode = NULL; subplan->pNode = NULL;
subplan->type = QUERY_TYPE_MODIFY; subplan->type = QUERY_TYPE_MODIFY;
subplan->msgType = pPayload->msgType; subplan->msgType = pPayload->msgType;
subplan->id.queryId = pCxt->pDag->queryId; subplan->id.queryId = pCxt->pDag->queryId;
RECOVERY_CURRENT_SUBPLAN(pCxt); RECOVERY_CURRENT_SUBPLAN(pCxt);

36
source/libs/planner/src/planner.c
View File

@ -16,12 +16,44 @@
#include "parser.h" #include "parser.h"
#include "plannerInt.h" #include "plannerInt.h"
static void destroyDataSinkNode(SDataSink* pSinkNode) {
if (pSinkNode == NULL) {
return;
}
tfree(pSinkNode);
}
void qDestroySubplan(SSubplan* pSubplan) { void qDestroySubplan(SSubplan* pSubplan) {
// todo if (pSubplan == NULL) {
return;
}
taosArrayDestroy(pSubplan->pChildren);
taosArrayDestroy(pSubplan->pParents);
destroyDataSinkNode(pSubplan->pDataSink);
// todo destroy pNode
tfree(pSubplan);
} }
void qDestroyQueryDag(struct SQueryDag* pDag) { void qDestroyQueryDag(struct SQueryDag* pDag) {
// todo if (pDag == NULL) {
return;
}
size_t size = taosArrayGetSize(pDag->pSubplans);
for(size_t i = 0; i < size; ++i) {
SArray* pa = taosArrayGetP(pDag->pSubplans, i);
size_t t = taosArrayGetSize(pa);
for(int32_t j = 0; j < t; ++j) {
SSubplan* pSubplan = taosArrayGetP(pa, j);
qDestroySubplan(pSubplan);
}
taosArrayDestroy(pa);
}
taosArrayDestroy(pDag->pSubplans);
tfree(pDag);
} }
int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) {

30
source/libs/scheduler/inc/schedulerInt.h
View File

@ -82,12 +82,11 @@ typedef struct SSchJobAttr {
} SSchJobAttr; } SSchJobAttr;
typedef struct SSchJob { typedef struct SSchJob {
uint64_t queryId; uint64_t queryId;
int32_t levelNum; int32_t levelNum;
int32_t levelIdx; int32_t levelIdx;
int8_t status; int8_t status;
SSchJobAttr attr; SSchJobAttr attr;
SQueryProfileSummary summary;
SEpSet dataSrcEps; SEpSet dataSrcEps;
SEpAddr resEp; SEpAddr resEp;
void *transport; void *transport;
@ -95,18 +94,20 @@ typedef struct SSchJob {
tsem_t rspSem; tsem_t rspSem;
int32_t userFetch; int32_t userFetch;
int32_t remoteFetch; int32_t remoteFetch;
SSchTask *fetchTask; SSchTask *fetchTask;
int32_t errCode; int32_t errCode;
void *res; void *res;
int32_t resNumOfRows; int32_t resNumOfRows;
SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask* SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask*
SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask*
SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask*
SArray *levels; // Element is SQueryLevel, starting from 0. SArray *levels; // Element is SQueryLevel, starting from 0. SArray<SSchLevel>
SArray *subPlans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. SArray *subPlans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0. SArray<void*>
SQueryProfileSummary summary;
} SSchJob; } SSchJob;
#define SCH_HAS_QNODE_IN_CLUSTER(type) (false) //TODO CLUSTER TYPE #define SCH_HAS_QNODE_IN_CLUSTER(type) (false) //TODO CLUSTER TYPE
@ -114,8 +115,9 @@ typedef struct SSchJob {
#define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN) #define SCH_IS_DATA_SRC_TASK(task) ((task)->plan->type == QUERY_TYPE_SCAN)
#define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY) #define SCH_TASK_NEED_WAIT_ALL(task) ((task)->plan->type == QUERY_TYPE_MODIFY)
#define SCH_JOB_ERR_LOG(param, ...) qError("QID:%"PRIx64 param, job->queryId, __VA_ARGS__) #define SCH_JOB_ELOG(param, ...) qError("QID:% "PRIx64 param, job->queryId, __VA_ARGS__)
#define SCH_TASK_ERR_LOG(param, ...) qError("QID:%"PRIx64",TID:%"PRIx64 param, job->queryId, task->taskId, __VA_ARGS__) #define SCH_TASK_ELOG(param, ...) qError("QID:%"PRIx64",TID:% "PRIx64 param, job->queryId, task->taskId, __VA_ARGS__)
#define SCH_TASK_DLOG(param, ...) qDebug("QID:%"PRIx64",TID:% "PRIx64 param, job->queryId, task->taskId, __VA_ARGS__)
#define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0) #define SCH_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define SCH_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)

116
source/libs/scheduler/src/scheduler.c
View File

@ -18,8 +18,7 @@
#include "query.h" #include "query.h"
#include "catalog.h" #include "catalog.h"
SSchedulerMgmt schMgmt = {0}; static SSchedulerMgmt schMgmt = {0};
int32_t schBuildTaskRalation(SSchJob *job, SHashObj *planToTask) { int32_t schBuildTaskRalation(SSchJob *job, SHashObj *planToTask) {
for (int32_t i = 0; i < job->levelNum; ++i) { for (int32_t i = 0; i < job->levelNum; ++i) {
@ -93,11 +92,30 @@ int32_t schBuildTaskRalation(SSchJob *job, SHashObj *planToTask) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static SSchTask initTask(SSchJob* pJob, SSubplan* plan, SSchLevel *pLevel) {
SSchTask task = {0};
if (plan->type == QUERY_TYPE_MODIFY) {
pJob->attr.needFetch = false;
} else {
pJob->attr.queryJob = true;
}
int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) { task.plan = plan;
task.level = pLevel;
task.status = JOB_TASK_STATUS_NOT_START;
task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1);
return task;
}
static void cleanupTask(SSchTask* pTask) {
taosArrayDestroy(pTask->condidateAddrs);
}
int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *pJob) {
int32_t code = 0; int32_t code = 0;
job->queryId = dag->queryId; pJob->queryId = dag->queryId;
if (dag->numOfSubplans <= 0) { if (dag->numOfSubplans <= 0) {
qError("invalid subplan num:%d", dag->numOfSubplans); qError("invalid subplan num:%d", dag->numOfSubplans);
@ -116,19 +134,19 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
job->levels = taosArrayInit(levelNum, sizeof(SSchLevel)); pJob->levels = taosArrayInit(levelNum, sizeof(SSchLevel));
if (NULL == job->levels) { if (NULL == pJob->levels) {
qError("taosArrayInit %d failed", levelNum); qError("taosArrayInit %d failed", levelNum);
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
//?? //??
job->attr.needFetch = true; pJob->attr.needFetch = true;
job->levelNum = levelNum; pJob->levelNum = levelNum;
job->levelIdx = levelNum - 1; pJob->levelIdx = levelNum - 1;
job->subPlans = dag->pSubplans; pJob->subPlans = dag->pSubplans;
SSchLevel level = {0}; SSchLevel level = {0};
SArray *levelPlans = NULL; SArray *levelPlans = NULL;
@ -138,12 +156,12 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) {
level.status = JOB_TASK_STATUS_NOT_START; level.status = JOB_TASK_STATUS_NOT_START;
for (int32_t i = 0; i < levelNum; ++i) { for (int32_t i = 0; i < levelNum; ++i) {
if (NULL == taosArrayPush(job->levels, &level)) { if (NULL == taosArrayPush(pJob->levels, &level)) {
qError("taosArrayPush failed"); qError("taosArrayPush failed");
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
pLevel = taosArrayGet(job->levels, i); pLevel = taosArrayGet(pJob->levels, i);
pLevel->level = i; pLevel->level = i;
levelPlans = taosArrayGetP(dag->pSubplans, i); levelPlans = taosArrayGetP(dag->pSubplans, i);
@ -168,20 +186,13 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) {
for (int32_t n = 0; n < levelPlanNum; ++n) { for (int32_t n = 0; n < levelPlanNum; ++n) {
SSubplan *plan = taosArrayGetP(levelPlans, n); SSubplan *plan = taosArrayGetP(levelPlans, n);
SSchTask task = {0};
if (plan->type == QUERY_TYPE_MODIFY) { if (plan->type == QUERY_TYPE_MODIFY) {
job->attr.needFetch = false; pJob->attr.needFetch = false;
} else { } else {
job->attr.queryJob = true; pJob->attr.queryJob = true;
} }
SSchTask task = initTask(pJob, plan, pLevel);
task.taskId = atomic_add_fetch_64(&schMgmt.taskId, 1);
task.plan = plan;
task.level = pLevel;
task.status = JOB_TASK_STATUS_NOT_START;
void *p = taosArrayPush(pLevel->subTasks, &task); void *p = taosArrayPush(pLevel->subTasks, &task);
if (NULL == p) { if (NULL == p) {
qError("taosArrayPush failed"); qError("taosArrayPush failed");
@ -193,10 +204,9 @@ int32_t schValidateAndBuildJob(SQueryDag *dag, SSchJob *job) {
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
} }
} }
SCH_ERR_JRET(schBuildTaskRalation(job, planToTask)); SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask));
if (planToTask) { if (planToTask) {
taosHashCleanup(planToTask); taosHashCleanup(planToTask);
@ -270,6 +280,8 @@ int32_t schPushTaskToExecList(SSchJob *job, SSchTask *task) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SCH_TASK_DLOG("push to %s list", "execTasks");
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -284,6 +296,8 @@ int32_t schMoveTaskToSuccList(SSchJob *job, SSchTask *task, bool *moved) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SCH_TASK_DLOG("push to %s list", "succTasks");
*moved = true; *moved = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -299,6 +313,8 @@ int32_t schMoveTaskToFailList(SSchJob *job, SSchTask *task, bool *moved) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
} }
SCH_TASK_DLOG("push to %s list", "failTasks");
*moved = true; *moved = true;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
@ -372,7 +388,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *job, SSchTask *task) {
SCH_ERR_RET(schMoveTaskToSuccList(job, task, &moved)); SCH_ERR_RET(schMoveTaskToSuccList(job, task, &moved));
if (!moved) { if (!moved) {
SCH_TASK_ERR_LOG(" task may already moved, status:%d", task->status); SCH_TASK_ELOG("task may already moved, status:%d", task->status);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -447,11 +463,11 @@ int32_t schProcessOnTaskFailure(SSchJob *job, SSchTask *task, int32_t errCode) {
SCH_ERR_RET(schTaskCheckAndSetRetry(job, task, errCode, &needRetry)); SCH_ERR_RET(schTaskCheckAndSetRetry(job, task, errCode, &needRetry));
if (!needRetry) { if (!needRetry) {
SCH_TASK_ERR_LOG("task failed[%x], no more retry", errCode); SCH_TASK_ELOG("task failed[%x], no more retry", errCode);
SCH_ERR_RET(schMoveTaskToFailList(job, task, &moved)); SCH_ERR_RET(schMoveTaskToFailList(job, task, &moved));
if (!moved) { if (!moved) {
SCH_TASK_ERR_LOG("task may already moved, status:%d", task->status); SCH_TASK_ELOG("task may already moved, status:%d", task->status);
} }
if (SCH_TASK_NEED_WAIT_ALL(task)) { if (SCH_TASK_NEED_WAIT_ALL(task)) {
@ -491,7 +507,6 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms
goto _task_error; goto _task_error;
} }
} }
break; break;
} }
case TDMT_VND_SUBMIT_RSP: { case TDMT_VND_SUBMIT_RSP: {
@ -568,19 +583,25 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in
int32_t code = 0; int32_t code = 0;
SSchCallbackParam *pParam = (SSchCallbackParam *)param; SSchCallbackParam *pParam = (SSchCallbackParam *)param;
SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); SSchJob **pjob = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId));
if (NULL == job || NULL == (*job)) { if (NULL == pjob || NULL == (*pjob)) {
qError("taosHashGet queryId:%"PRIx64" not exist", pParam->queryId); qError("taosHashGet queryId:%"PRIx64" not exist", pParam->queryId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
SSchTask **task = taosHashGet((*job)->execTasks, &pParam->taskId, sizeof(pParam->taskId)); SSchJob *job = *pjob;
if (NULL == task || NULL == (*task)) {
SSchTask **ptask = taosHashGet(job->execTasks, &pParam->taskId, sizeof(pParam->taskId));
if (NULL == ptask || NULL == (*ptask)) {
qError("taosHashGet taskId:%"PRIx64" not exist", pParam->taskId); qError("taosHashGet taskId:%"PRIx64" not exist", pParam->taskId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
schProcessRspMsg(*job, *task, msgType, pMsg->pData, pMsg->len, rspCode); SSchTask *task = *ptask;
SCH_TASK_DLOG("Got msg:%d, rspCode:%d", msgType, rspCode);
schProcessRspMsg(job, task, msgType, pMsg->pData, pMsg->len, rspCode);
_return: _return:
tfree(param); tfree(param);
@ -810,7 +831,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task) {
SCH_ERR_RET(schSetTaskCondidateAddrs(job, task)); SCH_ERR_RET(schSetTaskCondidateAddrs(job, task));
if (NULL == task->condidateAddrs || taosArrayGetSize(task->condidateAddrs) <= 0) { if (NULL == task->condidateAddrs || taosArrayGetSize(task->condidateAddrs) <= 0) {
SCH_TASK_ERR_LOG("no valid condidate node for task:%"PRIx64, task->taskId); SCH_TASK_ELOG("no valid condidate node for task:%"PRIx64, task->taskId);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
@ -839,7 +860,6 @@ void schDropJobAllTasks(SSchJob *job) {
void *pIter = taosHashIterate(job->succTasks, NULL); void *pIter = taosHashIterate(job->succTasks, NULL);
while (pIter) { while (pIter) {
SSchTask *task = *(SSchTask **)pIter; SSchTask *task = *(SSchTask **)pIter;
schBuildAndSendMsg(job, task, TDMT_VND_DROP_TASK); schBuildAndSendMsg(job, task, TDMT_VND_DROP_TASK);
pIter = taosHashIterate(job->succTasks, pIter); pIter = taosHashIterate(job->succTasks, pIter);
@ -933,16 +953,15 @@ int32_t scheduleExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag,
code = taosHashPut(schMgmt.jobs, &job->queryId, sizeof(job->queryId), &job, POINTER_BYTES); code = taosHashPut(schMgmt.jobs, &job->queryId, sizeof(job->queryId), &job, POINTER_BYTES);
if (0 != code) { if (0 != code) {
if (HASH_NODE_EXIST(code)) { if (HASH_NODE_EXIST(code)) {
qError("taosHashPut queryId:%"PRIx64" already exist", job->queryId); qError("taosHashPut queryId:0x%"PRIx64" already exist", job->queryId);
SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
} else { } else {
qError("taosHashPut queryId:%"PRIx64" failed", job->queryId); qError("taosHashPut queryId:0x%"PRIx64" failed", job->queryId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
} }
} }
job->status = JOB_TASK_STATUS_NOT_START; job->status = JOB_TASK_STATUS_NOT_START;
SCH_ERR_JRET(schLaunchJob(job)); SCH_ERR_JRET(schLaunchJob(job));
*(SSchJob **)pJob = job; *(SSchJob **)pJob = job;
@ -954,7 +973,6 @@ int32_t scheduleExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag,
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
_return: _return:
*(SSchJob **)pJob = NULL; *(SSchJob **)pJob = NULL;
scheduleFreeJob(job); scheduleFreeJob(job);
@ -1064,7 +1082,25 @@ void scheduleFreeJob(void *pJob) {
schDropJobAllTasks(job); schDropJobAllTasks(job);
} }
//TODO free job job->subPlans = NULL; // it is a reference to pDag->pSubplans
int32_t numOfLevels = taosArrayGetSize(job->levels);
for(int32_t i = 0; i < numOfLevels; ++i) {
SSchLevel *pLevel = taosArrayGet(job->levels, i);
int32_t numOfTasks = taosArrayGetSize(pLevel->subTasks);
for(int32_t j = 0; j < numOfTasks; ++j) {
SSchTask* pTask = taosArrayGet(pLevel->subTasks, j);
cleanupTask(pTask);
}
taosArrayDestroy(pLevel->subTasks);
}
taosHashCleanup(job->execTasks);
taosHashCleanup(job->failTasks);
taosHashCleanup(job->succTasks);
taosArrayDestroy(job->levels);
tfree(job);
} }
void schedulerDestroy(void) { void schedulerDestroy(void) {

3
source/libs/wal/src/walMeta.c
View File

@ -261,15 +261,18 @@ int walLoadMeta(SWal* pWal) {
memset(buf, 0, size + 5); memset(buf, 0, size + 5);
int tfd = tfOpenRead(fnameStr); int tfd = tfOpenRead(fnameStr);
if (tfRead(tfd, buf, size) != size) { if (tfRead(tfd, buf, size) != size) {
tfClose(tfd);
free(buf); free(buf);
return -1; return -1;
} }
// load into fileInfoSet // load into fileInfoSet
int code = walMetaDeserialize(pWal, buf); int code = walMetaDeserialize(pWal, buf);
if (code != 0) { if (code != 0) {
tfClose(tfd);
free(buf); free(buf);
return -1; return -1;
} }
tfClose(tfd);
free(buf); free(buf);
return 0; return 0;
} }

13
source/libs/wal/src/walRead.c
View File

@ -15,6 +15,7 @@
#include "tfile.h" #include "tfile.h"
#include "walInt.h" #include "walInt.h"
#include "taoserror.h"
SWalReadHandle *walOpenReadHandle(SWal *pWal) { SWalReadHandle *walOpenReadHandle(SWal *pWal) {
SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle)); SWalReadHandle *pRead = malloc(sizeof(SWalReadHandle));
@ -32,6 +33,7 @@ SWalReadHandle *walOpenReadHandle(SWal *pWal) {
pRead->status = 0; pRead->status = 0;
pRead->pHead = malloc(sizeof(SWalHead)); pRead->pHead = malloc(sizeof(SWalHead));
if (pRead->pHead == NULL) { if (pRead->pHead == NULL) {
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
free(pRead); free(pRead);
return NULL; return NULL;
} }
@ -57,16 +59,19 @@ static int32_t walReadSeekFilePos(SWalReadHandle *pRead, int64_t fileFirstVer, i
int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry); int64_t offset = (ver - fileFirstVer) * sizeof(SWalIdxEntry);
code = tfLseek(idxTfd, offset, SEEK_SET); code = tfLseek(idxTfd, offset, SEEK_SET);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
SWalIdxEntry entry; SWalIdxEntry entry;
if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) { if (tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)) != sizeof(SWalIdxEntry)) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; return -1;
} }
// TODO:deserialize // TODO:deserialize
ASSERT(entry.ver == ver); ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_SET); code = tfLseek(logTfd, entry.offset, SEEK_SET);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
return code; return code;
@ -81,6 +86,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) {
walBuildLogName(pRead->pWal, fileFirstVer, fnameStr); walBuildLogName(pRead->pWal, fileFirstVer, fnameStr);
int64_t logTfd = tfOpenRead(fnameStr); int64_t logTfd = tfOpenRead(fnameStr);
if (logTfd < 0) { if (logTfd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
@ -102,6 +108,7 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
return 0; return 0;
} }
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1; return -1;
} }
if (ver < pWal->vers.snapshotVer) { if (ver < pWal->vers.snapshotVer) {
@ -115,7 +122,6 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
if (pRead->curFileFirstVer != pRet->firstVer) { if (pRead->curFileFirstVer != pRet->firstVer) {
code = walReadChangeFile(pRead, pRet->firstVer); code = walReadChangeFile(pRead, pRet->firstVer);
if (code < 0) { if (code < 0) {
// TODO: set error flag
return -1; return -1;
} }
} }
@ -134,7 +140,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
// TODO: check wal life // TODO: check wal life
if (pRead->curVersion != ver) { if (pRead->curVersion != ver) {
code = walReadSeekVer(pRead, ver); code = walReadSeekVer(pRead, ver);
if (code != 0) { if (code < 0) {
return -1; return -1;
} }
} }
@ -147,11 +153,13 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
} }
code = walValidHeadCksum(pRead->pHead); code = walValidHeadCksum(pRead->pHead);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; return -1;
} }
if (pRead->capacity < pRead->pHead->head.len) { if (pRead->capacity < pRead->pHead->head.len) {
void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len); void *ptr = realloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.len);
if (ptr == NULL) { if (ptr == NULL) {
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
return -1; return -1;
} }
pRead->pHead = ptr; pRead->pHead = ptr;
@ -165,6 +173,7 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
code = walValidBodyCksum(pRead->pHead); code = walValidBodyCksum(pRead->pHead);
if (code != 0) { if (code != 0) {
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1; return -1;
} }
pRead->curVersion++; pRead->curVersion++;

9
source/libs/wal/src/walSeek.c
View File

@ -30,17 +30,20 @@ static int walSeekFilePos(SWal* pWal, int64_t ver) {
int64_t idxOff = walGetVerIdxOffset(pWal, ver); int64_t idxOff = walGetVerIdxOffset(pWal, ver);
code = tfLseek(idxTfd, idxOff, SEEK_SET); code = tfLseek(idxTfd, idxOff, SEEK_SET);
if (code != 0) { if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
SWalIdxEntry entry; SWalIdxEntry entry;
// TODO:deserialize // TODO:deserialize
code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry)); code = tfRead(idxTfd, &entry, sizeof(SWalIdxEntry));
if (code != 0) { if (code != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
ASSERT(entry.ver == ver); ASSERT(entry.ver == ver);
code = tfLseek(logTfd, entry.offset, SEEK_CUR); code = tfLseek(logTfd, entry.offset, SEEK_CUR);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
return code; return code;
@ -56,11 +59,13 @@ int walChangeFileToLast(SWal* pWal) {
walBuildIdxName(pWal, fileFirstVer, fnameStr); walBuildIdxName(pWal, fileFirstVer, fnameStr);
idxTfd = tfOpenReadWrite(fnameStr); idxTfd = tfOpenReadWrite(fnameStr);
if (idxTfd < 0) { if (idxTfd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
walBuildLogName(pWal, fileFirstVer, fnameStr); walBuildLogName(pWal, fileFirstVer, fnameStr);
logTfd = tfOpenReadWrite(fnameStr); logTfd = tfOpenReadWrite(fnameStr);
if (logTfd < 0) { if (logTfd < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
// switch file // switch file
@ -76,11 +81,12 @@ int walChangeFile(SWal* pWal, int64_t ver) {
code = tfClose(pWal->writeLogTfd); code = tfClose(pWal->writeLogTfd);
if (code != 0) { if (code != 0) {
// TODO // TODO
terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
code = tfClose(pWal->writeIdxTfd); code = tfClose(pWal->writeIdxTfd);
if (code != 0) { if (code != 0) {
// TODO terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
} }
SWalFileInfo tmpInfo; SWalFileInfo tmpInfo;
@ -113,6 +119,7 @@ int walSeekVer(SWal* pWal, int64_t ver) {
return 0; return 0;
} }
if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1; return -1;
} }
if (ver < pWal->vers.snapshotVer) { if (ver < pWal->vers.snapshotVer) {

2
source/libs/wal/src/walWrite.c
View File

@ -25,6 +25,7 @@ int32_t walCommit(SWal *pWal, int64_t ver) {
ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer); ASSERT(pWal->vers.commitVer >= pWal->vers.snapshotVer);
ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer); ASSERT(pWal->vers.commitVer <= pWal->vers.lastVer);
if (ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) { if (ver < pWal->vers.commitVer || ver > pWal->vers.lastVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1; return -1;
} }
pWal->vers.commitVer = ver; pWal->vers.commitVer = ver;
@ -38,6 +39,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
return 0; return 0;
} }
if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) { if (ver > pWal->vers.lastVer || ver < pWal->vers.commitVer) {
terrno = TSDB_CODE_WAL_INVALID_VER;
return -1; return -1;
} }
pthread_mutex_lock(&pWal->mutex); pthread_mutex_lock(&pWal->mutex);