xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2022-02-23 10:53:27 +08:00
parent 1ce00b1b87 c06aaf01fa
commit b5982c5c3d
82 changed files with 2824 additions and 546 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmake/cmake.options
View File

@ -47,13 +47,13 @@ option(
option( option(
BUILD_WITH_UV BUILD_WITH_UV
"If build with libuv" "If build with libuv"
OFF ON
) )
option( option(
BUILD_WITH_UV_TRANS BUILD_WITH_UV_TRANS
"If build with libuv_trans " "If build with libuv_trans "
OFF ON
) )
option( option(

2
include/libs/nodes/nodes.h
View File

@ -132,7 +132,7 @@ bool nodesEqualNode(const SNode* a, const SNode* b);
SNode* nodesCloneNode(const SNode* pNode); SNode* nodesCloneNode(const SNode* pNode);
SNodeList* nodesCloneList(const SNodeList* pList); SNodeList* nodesCloneList(const SNodeList* pList);
int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen);
int32_t nodesStringToNode(const char* pStr, SNode** pNode); int32_t nodesStringToNode(const char* pStr, SNode** pNode);
#ifdef __cplusplus #ifdef __cplusplus

11
include/libs/sync/sync.h
View File

@ -137,22 +137,21 @@ typedef struct {
} SSyncInfo; } SSyncInfo;
// will be defined in syncInt.h, here just for complie struct SSyncNode;
typedef struct SSyncNode { typedef struct SSyncNode SSyncNode;
} SSyncNode;
int32_t syncInit(); int32_t syncInit();
void syncCleanUp(); void syncCleanUp();
int64_t syncStart(const SSyncInfo*); int64_t syncStart(const SSyncInfo* pSyncInfo);
void syncStop(int64_t rid); void syncStop(int64_t rid);
int32_t syncReconfig(int64_t rid, const SSyncCfg*); int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg);
// int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak); // int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pBuf, bool isWeak);
int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak); int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak);
ESyncState syncGetMyRole(int64_t rid); ESyncState syncGetMyRole(int64_t rid);
void syncGetNodesRole(int64_t rid, SNodesRole*); void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole);
extern int32_t sDebugFlag; extern int32_t sDebugFlag;

1
include/libs/transport/trpc.h
View File

@ -64,6 +64,7 @@ typedef struct SRpcInit {
int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS
int idleTime; // milliseconds, 0 means idle timer is disabled int idleTime; // milliseconds, 0 means idle timer is disabled
bool noPool; // create conn pool or not
// the following is for client app ecurity only // the following is for client app ecurity only
char *user; // user name char *user; // user name
char spi; // security parameter index char spi; // security parameter index

2
include/util/tdef.h
View File

@ -335,6 +335,8 @@ do { \
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type #define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode #define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default #define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default

2
include/util/tjson.h
View File

@ -30,6 +30,7 @@ void tjsonDelete(SJson* pJson);
SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName); SJson* tjsonAddArrayToObject(SJson* pJson, const char* pName);
int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number); int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t number);
int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number);
int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal); int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal);
int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem); int32_t tjsonAddItemToObject(SJson* pJson, const char* pName, SJson* pItem);
int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem); int32_t tjsonAddItemToArray(SJson* pJson, SJson* pItem);
@ -42,6 +43,7 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj);
typedef int32_t (*FFromJson)(const SJson* pJson, void* pObj); typedef int32_t (*FFromJson)(const SJson* pJson, void* pObj);
char* tjsonToString(const SJson* pJson); char* tjsonToString(const SJson* pJson);
char* tjsonToUnformattedString(const SJson* pJson);
#ifdef __cplusplus #ifdef __cplusplus
} }

6
source/client/src/tmq.c
View File

@ -75,7 +75,7 @@ struct tmq_message_t {
SMqConsumeRsp rsp; SMqConsumeRsp rsp;
}; };
typedef struct SMqClientVg { typedef struct {
// statistics // statistics
int64_t pollCnt; int64_t pollCnt;
// offset // offset
@ -86,7 +86,7 @@ typedef struct SMqClientVg {
SEpSet epSet; SEpSet epSet;
} SMqClientVg; } SMqClientVg;
typedef struct SMqClientTopic { typedef struct {
// subscribe info // subscribe info
int32_t sqlLen; int32_t sqlLen;
char* sql; char* sql;
@ -779,11 +779,9 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) {
param->pVg = pVg; param->pVg = pVg;
tsem_init(&param->rspSem, 0, 0); tsem_init(&param->rspSem, 0, 0);
SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME); SRequestObj* pRequest = createRequest(tmq->pTscObj, NULL, NULL, TDMT_VND_CONSUME);
pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq), .handle = NULL}; pRequest->body.requestMsg = (SDataBuf){.pData = pReq, .len = sizeof(SMqConsumeReq), .handle = NULL};
SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest); SMsgSendInfo* sendInfo = buildMsgInfoImpl(pRequest);
sendInfo->requestObjRefId = 0; sendInfo->requestObjRefId = 0;
sendInfo->param = param; sendInfo->param = param;

2
source/common/src/tmsg.c
View File

@ -34,6 +34,7 @@ int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
} }
pIter->totalLen = pMsg->length; pIter->totalLen = pMsg->length;
ASSERT(pIter->totalLen > 0);
pIter->len = 0; pIter->len = 0;
pIter->pMsg = pMsg; pIter->pMsg = pMsg;
if (pMsg->length <= sizeof(SSubmitReq)) { if (pMsg->length <= sizeof(SSubmitReq)) {
@ -52,6 +53,7 @@ int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) {
} else { } else {
SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len);
pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen); pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen);
ASSERT(pIter->len > 0);
} }
if (pIter->len > pIter->totalLen) { if (pIter->len > pIter->totalLen) {

1
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -193,6 +193,7 @@ static int32_t dndInitClient(SDnode *pDnode) {
rpcInit.ckey = INTERNAL_CKEY; rpcInit.ckey = INTERNAL_CKEY;
rpcInit.spi = 1; rpcInit.spi = 1;
rpcInit.parent = pDnode; rpcInit.parent = pDnode;
rpcInit.noPool = true;
char pass[TSDB_PASSWORD_LEN + 1] = {0}; char pass[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass); taosEncryptPass_c((uint8_t *)(INTERNAL_SECRET), strlen(INTERNAL_SECRET), pass);

5
source/dnode/mgmt/impl/test/sut/inc/client.h
View File

@ -21,13 +21,18 @@ class TestClient {
bool Init(const char* user, const char* pass, const char* fqdn, uint16_t port); bool Init(const char* user, const char* pass, const char* fqdn, uint16_t port);
void Cleanup(); void Cleanup();
void DoInit();
SRpcMsg* SendReq(SRpcMsg* pReq); SRpcMsg* SendReq(SRpcMsg* pReq);
void SetRpcRsp(SRpcMsg* pRsp); void SetRpcRsp(SRpcMsg* pRsp);
tsem_t* GetSem(); tsem_t* GetSem();
void Restart();
private: private:
char fqdn[TSDB_FQDN_LEN]; char fqdn[TSDB_FQDN_LEN];
uint16_t port; uint16_t port;
char user[128];
char pass[128];
void* clientRpc; void* clientRpc;
SRpcMsg* pRsp; SRpcMsg* pRsp;
tsem_t sem; tsem_t sem;

3
source/dnode/mgmt/impl/test/sut/inc/sut.h
View File

@ -20,10 +20,10 @@
#include "os.h" #include "os.h"
#include "dnode.h" #include "dnode.h"
#include "tmsg.h"
#include "tconfig.h" #include "tconfig.h"
#include "tdataformat.h" #include "tdataformat.h"
#include "tglobal.h" #include "tglobal.h"
#include "tmsg.h"
#include "tnote.h" #include "tnote.h"
#include "trpc.h" #include "trpc.h"
#include "tthread.h" #include "tthread.h"
@ -39,6 +39,7 @@ class Testbase {
void Restart(); void Restart();
void ServerStop(); void ServerStop();
void ServerStart(); void ServerStart();
void ClientRestart();
SRpcMsg* SendReq(tmsg_t msgType, void* pCont, int32_t contLen); SRpcMsg* SendReq(tmsg_t msgType, void* pCont, int32_t contLen);
private: private:

36
source/dnode/mgmt/impl/test/sut/src/client.cpp
View File

@ -13,33 +13,41 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "tep.h"
#include "sut.h" #include "sut.h"
#include "tep.h"
static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) { static void processClientRsp(void* parent, SRpcMsg* pRsp, SEpSet* pEpSet) {
TestClient* client = (TestClient*)parent; TestClient* client = (TestClient*)parent;
client->SetRpcRsp(pRsp); client->SetRpcRsp(pRsp);
uInfo("response:%s from dnode, code:0x%x", TMSG_INFO(pRsp->msgType), pRsp->code); uInfo("x response:%s from dnode, code:0x%x, msgSize: %d", TMSG_INFO(pRsp->msgType), pRsp->code, pRsp->contLen);
tsem_post(client->GetSem()); tsem_post(client->GetSem());
} }
void TestClient::SetRpcRsp(SRpcMsg* pRsp) { this->pRsp = pRsp; }; void TestClient::SetRpcRsp(SRpcMsg* rsp) {
if (this->pRsp) {
free(this->pRsp);
}
this->pRsp = (SRpcMsg*)calloc(1, sizeof(SRpcMsg));
this->pRsp->msgType = rsp->msgType;
this->pRsp->code = rsp->code;
this->pRsp->pCont = rsp->pCont;
this->pRsp->contLen = rsp->contLen;
};
tsem_t* TestClient::GetSem() { return &sem; } tsem_t* TestClient::GetSem() { return &sem; }
bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) { void TestClient::DoInit() {
char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0}; char secretEncrypt[TSDB_PASSWORD_LEN + 1] = {0};
taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt); taosEncryptPass_c((uint8_t*)pass, strlen(pass), secretEncrypt);
SRpcInit rpcInit; SRpcInit rpcInit;
memset(&rpcInit, 0, sizeof(rpcInit)); memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.label = (char*)"DND-C"; rpcInit.label = (char*)"shell";
rpcInit.numOfThreads = 1; rpcInit.numOfThreads = 1;
rpcInit.cfp = processClientRsp; rpcInit.cfp = processClientRsp;
rpcInit.sessions = 1024; rpcInit.sessions = 1024;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
rpcInit.idleTime = 30 * 1000; rpcInit.idleTime = 30 * 1000;
rpcInit.user = (char*)user; rpcInit.user = (char*)this->user;
rpcInit.ckey = (char*)"key"; rpcInit.ckey = (char*)"key";
rpcInit.parent = this; rpcInit.parent = this;
rpcInit.secret = (char*)secretEncrypt; rpcInit.secret = (char*)secretEncrypt;
@ -47,11 +55,16 @@ bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint
clientRpc = rpcOpen(&rpcInit); clientRpc = rpcOpen(&rpcInit);
ASSERT(clientRpc); ASSERT(clientRpc);
tsem_init(&this->sem, 0, 0);
}
tsem_init(&sem, 0, 0); bool TestClient::Init(const char* user, const char* pass, const char* fqdn, uint16_t port) {
strcpy(this->fqdn, fqdn); strcpy(this->fqdn, fqdn);
strcpy(this->user, user);
strcpy(this->pass, pass);
this->port = port; this->port = port;
this->pRsp = NULL;
this->DoInit();
return true; return true;
} }
@ -60,11 +73,16 @@ void TestClient::Cleanup() {
rpcClose(clientRpc); rpcClose(clientRpc);
} }
void TestClient::Restart() {
this->Cleanup();
this->DoInit();
}
SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) { SRpcMsg* TestClient::SendReq(SRpcMsg* pReq) {
SEpSet epSet = {0}; SEpSet epSet = {0};
addEpIntoEpSet(&epSet, fqdn, port); addEpIntoEpSet(&epSet, fqdn, port);
rpcSendRequest(clientRpc, &epSet, pReq, NULL); rpcSendRequest(clientRpc, &epSet, pReq, NULL);
tsem_wait(&sem); tsem_wait(&sem);
uInfo("y response:%s from dnode, code:0x%x, msgSize: %d", TMSG_INFO(pRsp->msgType), pRsp->code, pRsp->contLen);
return pRsp; return pRsp;
} }

15
source/dnode/mgmt/impl/test/sut/src/sut.cpp
View File

@ -21,9 +21,9 @@ void Testbase::InitLog(const char* path) {
mDebugFlag = 143; mDebugFlag = 143;
cDebugFlag = 0; cDebugFlag = 0;
jniDebugFlag = 0; jniDebugFlag = 0;
tmrDebugFlag = 0; tmrDebugFlag = 143;
uDebugFlag = 0; uDebugFlag = 143;
rpcDebugFlag = 0; rpcDebugFlag = 143;
qDebugFlag = 0; qDebugFlag = 0;
wDebugFlag = 0; wDebugFlag = 0;
sDebugFlag = 0; sDebugFlag = 0;
@ -66,16 +66,21 @@ void Testbase::Init(const char* path, int16_t port) {
void Testbase::Cleanup() { void Testbase::Cleanup() {
tFreeSTableMetaRsp(&metaRsp); tFreeSTableMetaRsp(&metaRsp);
server.Stop();
client.Cleanup(); client.Cleanup();
taosMsleep(10);
server.Stop();
dndCleanup(); dndCleanup();
} }
void Testbase::Restart() { server.Restart(); } void Testbase::Restart() {
server.Restart();
client.Restart();
}
void Testbase::ServerStop() { server.Stop(); } void Testbase::ServerStop() { server.Stop(); }
void Testbase::ServerStart() { server.DoStart(); } void Testbase::ServerStart() { server.DoStart(); }
void Testbase::ClientRestart() { client.Restart(); }
SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) {
SRpcMsg rpcMsg = {0}; SRpcMsg rpcMsg = {0};

3
source/dnode/mnode/impl/src/mnode.c
View File

@ -404,7 +404,8 @@ SMnodeMsg *mndInitMsg(SMnode *pMnode, SRpcMsg *pRpcMsg) {
return NULL; return NULL;
} }
if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER && pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) { if (pRpcMsg->msgType != TDMT_MND_TRANS && pRpcMsg->msgType != TDMT_MND_MQ_TIMER &&
pRpcMsg->msgType != TDMT_MND_MQ_DO_REBALANCE) {
SRpcConnInfo connInfo = {0}; SRpcConnInfo connInfo = {0};
if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) { if ((pRpcMsg->msgType & 1U) && rpcGetConnInfo(pRpcMsg->handle, &connInfo) != 0) {
taosFreeQitem(pMsg); taosFreeQitem(pMsg);

5
source/dnode/mnode/impl/test/qnode/qnode.cpp
View File

@ -190,6 +190,9 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) {
tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq);
server2.Stop(); server2.Stop();
taosMsleep(1000);
// test.ClientRestart();
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_QNODE, pReq, contLen);
ASSERT_NE(pRsp, nullptr); ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL); ASSERT_EQ(pRsp->code, TSDB_CODE_RPC_NETWORK_UNAVAIL);
@ -226,6 +229,7 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) {
{ {
// server start, wait until the rollback finished // server start, wait until the rollback finished
server2.DoStart(); server2.DoStart();
test.ClientRestart();
taosMsleep(1000); taosMsleep(1000);
int32_t retry = 0; int32_t retry = 0;
@ -248,7 +252,6 @@ TEST_F(MndTestQnode, 03_Create_Qnode_Rollback) {
ASSERT_NE(retry, retryMax); ASSERT_NE(retry, retryMax);
} }
} }
TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) { TEST_F(MndTestQnode, 04_Drop_Qnode_Rollback) {
{ {
// send message first, then dnode2 crash, result is returned, and rollback is started // send message first, then dnode2 crash, result is returned, and rollback is started

3
source/dnode/mnode/impl/test/trans/trans.cpp
View File

@ -46,8 +46,10 @@ class MndTestTrans : public ::testing::Test {
free(buffer); free(buffer);
taosFsyncFile(fd); taosFsyncFile(fd);
taosCloseFile(fd); taosCloseFile(fd);
taosMsleep(1000);
test.ServerStart(); test.ServerStart();
test.ClientRestart();
} }
static Testbase test; static Testbase test;
@ -284,6 +286,7 @@ TEST_F(MndTestTrans, 03_Create_Qnode2_Crash) {
} }
} }
// create db // create db
// partial create stb // partial create stb
// drop db failed // drop db failed

1
source/dnode/mnode/impl/test/user/user.cpp
View File

@ -617,6 +617,7 @@ TEST_F(MndTestUser, 06_Create_Drop_Alter_User) {
// restart // restart
test.Restart(); test.Restart();
taosMsleep(1000);
test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, ""); test.SendShowMetaReq(TSDB_MGMT_TABLE_USER, "");
CHECK_META("show users", 4); CHECK_META("show users", 4);

2
source/dnode/vnode/inc/vnode.h
View File

@ -226,7 +226,7 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S
return 0; return 0;
} }
void tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver);
bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlock(STqReadHandle *pHandle);
int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo);
// return SArray<SColumnInfoData> // return SArray<SColumnInfoData>

33
source/dnode/vnode/src/tq/tqRead.c
View File

@ -31,13 +31,28 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) {
return pReadHandle; return pReadHandle;
} }
void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t ver) { int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t ver) {
pReadHandle->pMsg = pMsg; pReadHandle->pMsg = pMsg;
pMsg->length = htonl(pMsg->length); pMsg->length = htonl(pMsg->length);
pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); pMsg->numOfBlocks = htonl(pMsg->numOfBlocks);
tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter);
if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
while (true) {
if (tGetSubmitMsgNext(&pReadHandle->msgIter, &pReadHandle->pBlock) < 0) return -1;
if (pReadHandle->pBlock == NULL) break;
pReadHandle->pBlock->uid = htobe64(pReadHandle->pBlock->uid);
pReadHandle->pBlock->tid = htonl(pReadHandle->pBlock->tid);
pReadHandle->pBlock->sversion = htonl(pReadHandle->pBlock->sversion);
pReadHandle->pBlock->dataLen = htonl(pReadHandle->pBlock->dataLen);
pReadHandle->pBlock->schemaLen = htonl(pReadHandle->pBlock->schemaLen);
pReadHandle->pBlock->numOfRows = htons(pReadHandle->pBlock->numOfRows);
}
if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1;
pReadHandle->ver = ver; pReadHandle->ver = ver;
memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter));
return 0;
} }
bool tqNextDataBlock(STqReadHandle* pHandle) { bool tqNextDataBlock(STqReadHandle* pHandle) {
@ -47,19 +62,19 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
} }
if (pHandle->pBlock == NULL) return false; if (pHandle->pBlock == NULL) return false;
pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); /*pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid);*/
/*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/
ASSERT(pHandle->tbIdHash); ASSERT(pHandle->tbIdHash);
void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t));
if (ret != NULL) { if (ret != NULL) {
/*printf("retrieve one tb %ld\n", pHandle->pBlock->uid);*/ /*printf("retrieve one tb %ld\n", pHandle->pBlock->uid);*/
pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); /*pHandle->pBlock->tid = htonl(pHandle->pBlock->tid);*/
pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); /*pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion);*/
pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); /*pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen);*/
pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); /*pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen);*/
pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); /*pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows);*/
return true; return true;
} else { /*} else {*/
/*printf("skip one tb %ld\n", pHandle->pBlock->uid);*/ /*printf("skip one tb %ld\n", pHandle->pBlock->uid);*/
} }
} }

2
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -126,7 +126,7 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) {
goto _exit; goto _exit;
} }
metaRsp.dbId = htobe64(pVnode->config.dbId); metaRsp.dbId = pVnode->config.dbId;
memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName));
strcpy(metaRsp.tbName, infoReq.tbName); strcpy(metaRsp.tbName, infoReq.tbName);
if (pTbCfg->type == META_CHILD_TABLE) { if (pTbCfg->type == META_CHILD_TABLE) {

6
source/libs/catalog/src/catalog.c
View File

@ -1556,6 +1556,7 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
STableMetaOutput *output = NULL; STableMetaOutput *output = NULL;
while (true) {
CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output)); CTG_ERR_JRET(ctgRefreshTblMeta(pCtg, pRpc, pMgmtEps, pTableName, isSTable, &output));
if (CTG_IS_META_TABLE(output->metaType)) { if (CTG_IS_META_TABLE(output->metaType)) {
@ -1584,11 +1585,14 @@ int32_t ctgGetTableMeta(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, cons
CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist)); CTG_ERR_JRET(ctgGetTableMetaFromCache(pCtg, &stbName, pTableMeta, &exist));
if (0 == exist) { if (0 == exist) {
ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname); ctgDebug("stb no longer exist, dbFName:%s, tbName:%s", output->dbFName, pTableName->tname);
CTG_ERR_JRET(TSDB_CODE_VND_TB_NOT_EXIST); continue;
} }
memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta)); memcpy(*pTableMeta, &output->ctbMeta, sizeof(output->ctbMeta));
break;
}
_return: _return:
tfree(output); tfree(output);

7
source/libs/executor/src/executor.c
View File

@ -14,9 +14,9 @@
*/ */
#include "executor.h" #include "executor.h"
#include "tq.h"
#include "executorimpl.h" #include "executorimpl.h"
#include "planner.h" #include "planner.h"
#include "tq.h"
static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) { static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id) {
ASSERT(pOperator != NULL); ASSERT(pOperator != NULL);
@ -34,7 +34,10 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, char* id)
return doSetStreamBlock(pOperator->pDownstream[0], input, id); return doSetStreamBlock(pOperator->pDownstream[0], input, id);
} else { } else {
SStreamBlockScanInfo* pInfo = pOperator->info; SStreamBlockScanInfo* pInfo = pOperator->info;
tqReadHandleSetMsg(pInfo->readerHandle, input, 0); if (tqReadHandleSetMsg(pInfo->readerHandle, input, 0) < 0) {
qError("submit msg error while set stream msg, %s" PRIx64, id);
return TSDB_CODE_QRY_APP_ERROR;
}
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
} }

56
source/libs/index/src/index.c
View File

@ -66,7 +66,9 @@ static void indexMergeSameKey(SArray* result, TFileValue* tv);
int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
pthread_once(&isInit, indexInit); pthread_once(&isInit, indexInit);
SIndex* sIdx = calloc(1, sizeof(SIndex)); SIndex* sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) { return -1; } if (sIdx == NULL) {
return -1;
}
#ifdef USE_LUCENE #ifdef USE_LUCENE
index_t* index = index_open(path); index_t* index = index_open(path);
@ -76,7 +78,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
#ifdef USE_INVERTED_INDEX #ifdef USE_INVERTED_INDEX
// sIdx->cache = (void*)indexCacheCreate(sIdx); // sIdx->cache = (void*)indexCacheCreate(sIdx);
sIdx->tindex = indexTFileCreate(path); sIdx->tindex = indexTFileCreate(path);
if (sIdx->tindex == NULL) { goto END; } if (sIdx->tindex == NULL) {
goto END;
}
sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); sIdx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
sIdx->cVersion = 1; sIdx->cVersion = 1;
@ -87,7 +91,9 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
#endif #endif
END: END:
if (sIdx != NULL) { indexClose(sIdx); } if (sIdx != NULL) {
indexClose(sIdx);
}
*index = NULL; *index = NULL;
return -1; return -1;
@ -103,7 +109,9 @@ void indexClose(SIndex* sIdx) {
void* iter = taosHashIterate(sIdx->colObj, NULL); void* iter = taosHashIterate(sIdx->colObj, NULL);
while (iter) { while (iter) {
IndexCache** pCache = iter; IndexCache** pCache = iter;
if (*pCache) { indexCacheUnRef(*pCache); } if (*pCache) {
indexCacheUnRef(*pCache);
}
iter = taosHashIterate(sIdx->colObj, iter); iter = taosHashIterate(sIdx->colObj, iter);
} }
taosHashCleanup(sIdx->colObj); taosHashCleanup(sIdx->colObj);
@ -161,7 +169,9 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
IndexCache** cache = taosHashGet(index->colObj, buf, sz); IndexCache** cache = taosHashGet(index->colObj, buf, sz);
assert(*cache != NULL); assert(*cache != NULL);
int ret = indexCachePut(*cache, p, uid); int ret = indexCachePut(*cache, p, uid);
if (ret != 0) { return ret; } if (ret != 0) {
return ret;
}
} }
#endif #endif
@ -191,7 +201,9 @@ int indexSearch(SIndex* index, SIndexMultiTermQuery* multiQuerys, SArray* result
int tsz = 0; int tsz = 0;
index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz); index_multi_search(index->index, (const char**)fields, (const char**)keys, types, nQuery, opera, &tResult, &tsz);
for (int i = 0; i < tsz; i++) { taosArrayPush(result, &tResult[i]); } for (int i = 0; i < tsz; i++) {
taosArrayPush(result, &tResult[i]);
}
for (int i = 0; i < nQuery; i++) { for (int i = 0; i < nQuery; i++) {
free(fields[i]); free(fields[i]);
@ -248,7 +260,9 @@ void indexOptsDestroy(SIndexOpts* opts) {
*/ */
SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) { SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery)); SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) { return NULL; } if (p == NULL) {
return NULL;
}
p->opera = opera; p->opera = opera;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery)); p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p; return p;
@ -270,7 +284,9 @@ int indexMultiTermQueryAdd(SIndexMultiTermQuery* pQuery, SIndexTerm* term, EInde
SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName, SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char* colName,
int32_t nColName, const char* colVal, int32_t nColVal) { int32_t nColName, const char* colVal, int32_t nColVal) {
SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm))); SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) { return NULL; } if (t == NULL) {
return NULL;
}
t->suid = suid; t->suid = suid;
t->operType = oper; t->operType = oper;
@ -343,7 +359,9 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
return 0; return 0;
} }
static void indexInterResultsDestroy(SArray* results) { static void indexInterResultsDestroy(SArray* results) {
if (results == NULL) { return; } if (results == NULL) {
return;
}
size_t sz = taosArrayGetSize(results); size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
@ -419,18 +437,24 @@ static void indexDestroyTempResult(SArray* result) {
taosArrayDestroy(result); taosArrayDestroy(result);
} }
int indexFlushCacheToTFile(SIndex* sIdx, void* cache) { int indexFlushCacheToTFile(SIndex* sIdx, void* cache) {
if (sIdx == NULL) { return -1; } if (sIdx == NULL) {
return -1;
}
indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid); indexInfo("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
int64_t st = taosGetTimestampUs(); int64_t st = taosGetTimestampUs();
IndexCache* pCache = (IndexCache*)cache; IndexCache* pCache = (IndexCache*)cache;
TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName); TFileReader* pReader = tfileGetReaderByCol(sIdx->tindex, pCache->suid, pCache->colName);
if (pReader == NULL) { indexWarn("empty tfile reader found"); } if (pReader == NULL) {
indexWarn("empty tfile reader found");
}
// handle flush // handle flush
Iterate* cacheIter = indexCacheIteratorCreate(pCache); Iterate* cacheIter = indexCacheIteratorCreate(pCache);
Iterate* tfileIter = tfileIteratorCreate(pReader); Iterate* tfileIter = tfileIteratorCreate(pReader);
if (tfileIter == NULL) { indexWarn("empty tfile reader iterator"); } if (tfileIter == NULL) {
indexWarn("empty tfile reader iterator");
}
SArray* result = taosArrayInit(1024, sizeof(void*)); SArray* result = taosArrayInit(1024, sizeof(void*));
@ -484,7 +508,9 @@ void iterateValueDestroy(IterateValue* value, bool destroy) {
taosArrayDestroy(value->val); taosArrayDestroy(value->val);
value->val = NULL; value->val = NULL;
} else { } else {
if (value->val != NULL) { taosArrayClear(value->val); } if (value->val != NULL) {
taosArrayClear(value->val);
}
} }
free(value->colVal); free(value->colVal);
value->colVal = NULL; value->colVal = NULL;
@ -507,7 +533,9 @@ static int indexGenTFile(SIndex* sIdx, IndexCache* cache, SArray* batch) {
tfileWriterClose(tw); tfileWriterClose(tw);
TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName); TFileReader* reader = tfileReaderOpen(sIdx->path, cache->suid, version, cache->colName);
if (reader == NULL) { return -1; } if (reader == NULL) {
return -1;
}
TFileHeader* header = &reader->header; TFileHeader* header = &reader->header;
ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)}; ICacheKey key = {.suid = cache->suid, .colName = header->colName, .nColName = strlen(header->colName)};

69
source/libs/index/src/index_cache.c
View File

@ -119,13 +119,17 @@ void indexCacheDestroySkiplist(SSkipList* slt) {
tSkipListDestroy(slt); tSkipListDestroy(slt);
} }
void indexCacheDestroyImm(IndexCache* cache) { void indexCacheDestroyImm(IndexCache* cache) {
if (cache == NULL) { return; } if (cache == NULL) {
return;
}
MemTable* tbl = NULL; MemTable* tbl = NULL;
pthread_mutex_lock(&cache->mtx); pthread_mutex_lock(&cache->mtx);
tbl = cache->imm; tbl = cache->imm;
cache->imm = NULL; // or throw int bg thread cache->imm = NULL; // or throw int bg thread
pthread_cond_broadcast(&cache->finished); pthread_cond_broadcast(&cache->finished);
pthread_mutex_unlock(&cache->mtx); pthread_mutex_unlock(&cache->mtx);
indexMemUnRef(tbl); indexMemUnRef(tbl);
@ -133,7 +137,9 @@ void indexCacheDestroyImm(IndexCache* cache) {
} }
void indexCacheDestroy(void* cache) { void indexCacheDestroy(void* cache) {
IndexCache* pCache = cache; IndexCache* pCache = cache;
if (pCache == NULL) { return; } if (pCache == NULL) {
return;
}
indexMemUnRef(pCache->mem); indexMemUnRef(pCache->mem);
indexMemUnRef(pCache->imm); indexMemUnRef(pCache->imm);
free(pCache->colName); free(pCache->colName);
@ -146,7 +152,9 @@ void indexCacheDestroy(void* cache) {
Iterate* indexCacheIteratorCreate(IndexCache* cache) { Iterate* indexCacheIteratorCreate(IndexCache* cache) {
Iterate* iiter = calloc(1, sizeof(Iterate)); Iterate* iiter = calloc(1, sizeof(Iterate));
if (iiter == NULL) { return NULL; } if (iiter == NULL) {
return NULL;
}
pthread_mutex_lock(&cache->mtx); pthread_mutex_lock(&cache->mtx);
@ -164,7 +172,9 @@ Iterate* indexCacheIteratorCreate(IndexCache* cache) {
return iiter; return iiter;
} }
void indexCacheIteratorDestroy(Iterate* iter) { void indexCacheIteratorDestroy(Iterate* iter) {
if (iter == NULL) { return; } if (iter == NULL) {
return;
}
tSkipListDestroyIter(iter->iter); tSkipListDestroyIter(iter->iter);
iterateValueDestroy(&iter->val, true); iterateValueDestroy(&iter->val, true);
free(iter); free(iter);
@ -186,9 +196,6 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
} else if (cache->imm != NULL) { } else if (cache->imm != NULL) {
// TODO: wake up by condition variable // TODO: wake up by condition variable
pthread_cond_wait(&cache->finished, &cache->mtx); pthread_cond_wait(&cache->finished, &cache->mtx);
// pthread_mutex_unlock(&cache->mtx);
// taosMsleep(50);
// pthread_mutex_lock(&cache->mtx);
} else { } else {
indexCacheRef(cache); indexCacheRef(cache);
cache->imm = cache->mem; cache->imm = cache->mem;
@ -202,13 +209,17 @@ static void indexCacheMakeRoomForWrite(IndexCache* cache) {
} }
int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) { int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
if (cache == NULL) { return -1; } if (cache == NULL) {
return -1;
}
IndexCache* pCache = cache; IndexCache* pCache = cache;
indexCacheRef(pCache); indexCacheRef(pCache);
// encode data // encode data
CacheTerm* ct = calloc(1, sizeof(CacheTerm)); CacheTerm* ct = calloc(1, sizeof(CacheTerm));
if (cache == NULL) { return -1; } if (cache == NULL) {
return -1;
}
// set up key // set up key
ct->colType = term->colType; ct->colType = term->colType;
ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1)); ct->colVal = (char*)calloc(1, sizeof(char) * (term->nColVal + 1));
@ -240,7 +251,9 @@ int indexCacheDel(void* cache, const char* fieldValue, int32_t fvlen, uint64_t u
} }
static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) { static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SArray* result, STermValueType* s) {
if (mem == NULL) { return 0; } if (mem == NULL) {
return 0;
}
char* key = indexCacheTermGet(ct); char* key = indexCacheTermGet(ct);
SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC); SSkipListIterator* iter = tSkipListCreateIterFromVal(mem->mem, key, TSDB_DATA_TYPE_BINARY, TSDB_ORDER_ASC);
@ -266,7 +279,9 @@ static int indexQueryMem(MemTable* mem, CacheTerm* ct, EIndexQueryType qtype, SA
return 0; return 0;
} }
int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) { int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermValueType* s) {
if (cache == NULL) { return 0; } if (cache == NULL) {
return 0;
}
IndexCache* pCache = cache; IndexCache* pCache = cache;
MemTable *mem = NULL, *imm = NULL; MemTable *mem = NULL, *imm = NULL;
@ -294,23 +309,33 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, SArray* result, STermV
} }
void indexCacheRef(IndexCache* cache) { void indexCacheRef(IndexCache* cache) {
if (cache == NULL) { return; } if (cache == NULL) {
return;
}
int ref = T_REF_INC(cache); int ref = T_REF_INC(cache);
UNUSED(ref); UNUSED(ref);
} }
void indexCacheUnRef(IndexCache* cache) { void indexCacheUnRef(IndexCache* cache) {
if (cache == NULL) { return; } if (cache == NULL) {
return;
}
int ref = T_REF_DEC(cache); int ref = T_REF_DEC(cache);
if (ref == 0) { indexCacheDestroy(cache); } if (ref == 0) {
indexCacheDestroy(cache);
}
} }
void indexMemRef(MemTable* tbl) { void indexMemRef(MemTable* tbl) {
if (tbl == NULL) { return; } if (tbl == NULL) {
return;
}
int ref = T_REF_INC(tbl); int ref = T_REF_INC(tbl);
UNUSED(ref); UNUSED(ref);
} }
void indexMemUnRef(MemTable* tbl) { void indexMemUnRef(MemTable* tbl) {
if (tbl == NULL) { return; } if (tbl == NULL) {
return;
}
int ref = T_REF_DEC(tbl); int ref = T_REF_DEC(tbl);
if (ref == 0) { if (ref == 0) {
SSkipList* slt = tbl->mem; SSkipList* slt = tbl->mem;
@ -320,7 +345,9 @@ void indexMemUnRef(MemTable* tbl) {
} }
static void indexCacheTermDestroy(CacheTerm* ct) { static void indexCacheTermDestroy(CacheTerm* ct) {
if (ct == NULL) { return; } if (ct == NULL) {
return;
}
free(ct->colVal); free(ct->colVal);
free(ct); free(ct);
} }
@ -333,7 +360,9 @@ static int32_t indexCacheTermCompare(const void* l, const void* r) {
CacheTerm* rt = (CacheTerm*)r; CacheTerm* rt = (CacheTerm*)r;
// compare colVal // compare colVal
int32_t cmp = strcmp(lt->colVal, rt->colVal); int32_t cmp = strcmp(lt->colVal, rt->colVal);
if (cmp == 0) { return rt->version - lt->version; } if (cmp == 0) {
return rt->version - lt->version;
}
return cmp; return cmp;
} }
@ -354,7 +383,9 @@ static void doMergeWork(SSchedMsg* msg) {
} }
static bool indexCacheIteratorNext(Iterate* itera) { static bool indexCacheIteratorNext(Iterate* itera) {
SSkipListIterator* iter = itera->iter; SSkipListIterator* iter = itera->iter;
if (iter == NULL) { return false; } if (iter == NULL) {
return false;
}
IterateValue* iv = &itera->val; IterateValue* iv = &itera->val;
iterateValueDestroy(iv, false); iterateValueDestroy(iv, false);

198
source/libs/index/src/index_fst.c
View File

@ -31,20 +31,24 @@ static uint8_t fstPackDetla(FstCountingWriter* wrt, CompiledAddr nodeAddr, Compi
FstUnFinishedNodes* fstUnFinishedNodesCreate() { FstUnFinishedNodes* fstUnFinishedNodesCreate() {
FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes)); FstUnFinishedNodes* nodes = malloc(sizeof(FstUnFinishedNodes));
if (nodes == NULL) { return NULL; } if (nodes == NULL) {
return NULL;
}
nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished)); nodes->stack = (SArray*)taosArrayInit(64, sizeof(FstBuilderNodeUnfinished));
fstUnFinishedNodesPushEmpty(nodes, false); fstUnFinishedNodesPushEmpty(nodes, false);
return nodes; return nodes;
} }
void unFinishedNodeDestroyElem(void* elem) { static void unFinishedNodeDestroyElem(void* elem) {
FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem;
fstBuilderNodeDestroy(b->node); fstBuilderNodeDestroy(b->node);
free(b->last); free(b->last);
b->last = NULL; b->last = NULL;
} }
void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) { void fstUnFinishedNodesDestroy(FstUnFinishedNodes* nodes) {
if (nodes == NULL) { return; } if (nodes == NULL) {
return;
}
taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem); taosArrayDestroyEx(nodes->stack, unFinishedNodeDestroyElem);
free(nodes); free(nodes);
@ -92,7 +96,9 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes* nodes, CompiledAddr add
} }
void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) { void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes* nodes, FstSlice bs, Output out) {
FstSlice* s = &bs; FstSlice* s = &bs;
if (fstSliceIsEmpty(s)) { return; } if (fstSliceIsEmpty(s)) {
return;
}
size_t sz = taosArrayGetSize(nodes->stack) - 1; size_t sz = taosArrayGetSize(nodes->stack) - 1;
FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz); FstBuilderNodeUnfinished* un = taosArrayGet(nodes->stack, sz);
assert(un->last == NULL); assert(un->last == NULL);
@ -172,7 +178,9 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes* node,
FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) {
FstState fs = {.state = EmptyFinal, .val = 0}; FstState fs = {.state = EmptyFinal, .val = 0};
if (addr == EMPTY_ADDRESS) { return fs; } if (addr == EMPTY_ADDRESS) {
return fs;
}
uint8_t* data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
uint8_t v = data[addr]; uint8_t v = data[addr];
@ -229,7 +237,9 @@ void fstStateCompileForOneTrans(FstCountingWriter* w, CompiledAddr addr, FstTran
fstStateSetCommInput(&st, trn->inp); fstStateSetCommInput(&st, trn->inp);
bool null = false; bool null = false;
uint8_t inp = fstStateCommInput(&st, &null); uint8_t inp = fstStateCommInput(&st, &null);
if (null == true) { fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp)); } if (null == true) {
fstCountingWriterWrite(w, (char*)&trn->inp, sizeof(trn->inp));
}
fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val)); fstCountingWriterWrite(w, (char*)(&(st.val)), sizeof(st.val));
return; return;
} }
@ -263,7 +273,9 @@ void fstStateCompileForAnyTrans(FstCountingWriter* w, CompiledAddr addr, FstBuil
fstStateSetStateNtrans(&st, (uint8_t)sz); fstStateSetStateNtrans(&st, (uint8_t)sz);
if (anyOuts) { if (anyOuts) {
if (FST_BUILDER_NODE_IS_FINAL(node)) { fstCountingWriterPackUintIn(w, node->finalOutput, oSize); } if (FST_BUILDER_NODE_IS_FINAL(node)) {
fstCountingWriterPackUintIn(w, node->finalOutput, oSize);
}
for (int32_t i = sz - 1; i >= 0; i--) { for (int32_t i = sz - 1; i >= 0; i--) {
FstTransition* t = taosArrayGet(node->trans, i); FstTransition* t = taosArrayGet(node->trans, i);
fstCountingWriterPackUintIn(w, t->out, oSize); fstCountingWriterPackUintIn(w, t->out, oSize);
@ -428,7 +440,9 @@ Output fstStateOutput(FstState* s, FstNode* node) {
assert(s->state == OneTrans); assert(s->state == OneTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) { return 0; } if (oSizes == 0) {
return 0;
}
FstSlice* slice = &node->data; FstSlice* slice = &node->data;
uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes); uint8_t tSizes = FST_GET_TRANSITION_PACK_SIZE(node->sizes);
@ -440,7 +454,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(node->sizes);
if (oSizes == 0) { return 0; } if (oSizes == 0) {
return 0;
}
FstSlice* slice = &node->data; FstSlice* slice = &node->data;
uint8_t* data = fstSliceData(slice, NULL); uint8_t* data = fstSliceData(slice, NULL);
uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size uint64_t at = node->start - fstStateNtransLen(s) - 1 // pack size
@ -453,7 +469,9 @@ Output fstStateOutputForAnyTrans(FstState* s, FstNode* node, uint64_t i) {
void fstStateSetFinalState(FstState* s, bool yes) { void fstStateSetFinalState(FstState* s, bool yes) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
if (yes) { s->val |= 0b01000000; } if (yes) {
s->val |= 0b01000000;
}
return; return;
} }
bool fstStateIsFinalState(FstState* s) { bool fstStateIsFinalState(FstState* s) {
@ -463,7 +481,9 @@ bool fstStateIsFinalState(FstState* s) {
void fstStateSetStateNtrans(FstState* s, uint8_t n) { void fstStateSetStateNtrans(FstState* s, uint8_t n) {
assert(s->state == AnyTrans); assert(s->state == AnyTrans);
if (n <= 0b00111111) { s->val = (s->val & 0b11000000) | n; } if (n <= 0b00111111) {
s->val = (s->val & 0b11000000) | n;
}
return; return;
} }
// state_ntrans // state_ntrans
@ -495,7 +515,9 @@ uint64_t fstStateNtransLen(FstState* s) {
uint64_t fstStateNtrans(FstState* s, FstSlice* slice) { uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
bool null = false; bool null = false;
uint8_t n = fstStateStateNtrans(s, &null); uint8_t n = fstStateStateNtrans(s, &null);
if (null != true) { return n; } if (null != true) {
return n;
}
int32_t len; int32_t len;
uint8_t* data = fstSliceData(slice, &len); uint8_t* data = fstSliceData(slice, &len);
n = data[len - 2]; n = data[len - 2];
@ -505,7 +527,9 @@ uint64_t fstStateNtrans(FstState* s, FstSlice* slice) {
} }
Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) { Output fstStateFinalOutput(FstState* s, uint64_t version, FstSlice* slice, PackSizes sizes, uint64_t nTrans) {
uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes); uint8_t oSizes = FST_GET_OUTPUT_PACK_SIZE(sizes);
if (oSizes == 0 || !fstStateIsFinalState(s)) { return 0; } if (oSizes == 0 || !fstStateIsFinalState(s)) {
return 0;
}
uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size uint64_t at = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1 // pack size
- fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes;
@ -522,7 +546,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
uint8_t* data = fstSliceData(slice, &dlen); uint8_t* data = fstSliceData(slice, &dlen);
uint64_t i = data[at + b]; uint64_t i = data[at + b];
// uint64_t i = slice->data[slice->start + at + b]; // uint64_t i = slice->data[slice->start + at + b];
if (i >= node->nTrans) { *null = true; } if (i >= node->nTrans) {
*null = true;
}
return i; return i;
} else { } else {
uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size uint64_t start = node->start - fstStateNtransLen(s) - 1 // pack size
@ -539,7 +565,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
return node->nTrans - i - 1; // bug return node->nTrans - i - 1; // bug
} }
} }
if (i == len) { *null = true; } if (i == len) {
*null = true;
}
fstSliceDestroy(&t); fstSliceDestroy(&t);
} }
} }
@ -548,7 +576,9 @@ uint64_t fstStateFindInput(FstState* s, FstNode* node, uint8_t b, bool* null) {
FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) { FstNode* fstNodeCreate(int64_t version, CompiledAddr addr, FstSlice* slice) {
FstNode* n = (FstNode*)malloc(sizeof(FstNode)); FstNode* n = (FstNode*)malloc(sizeof(FstNode));
if (n == NULL) { return NULL; } if (n == NULL) {
return NULL;
}
FstState st = fstStateCreateFrom(slice, addr); FstState st = fstStateCreateFrom(slice, addr);
@ -614,7 +644,9 @@ void fstNodeDestroy(FstNode* node) {
} }
FstTransitions* fstNodeTransitions(FstNode* node) { FstTransitions* fstNodeTransitions(FstNode* node) {
FstTransitions* t = malloc(sizeof(FstTransitions)); FstTransitions* t = malloc(sizeof(FstTransitions));
if (NULL == t) { return NULL; } if (NULL == t) {
return NULL;
}
FstRange range = {.start = 0, .end = FST_NODE_LEN(node)}; FstRange range = {.start = 0, .end = FST_NODE_LEN(node)};
t->range = range; t->range = range;
t->node = node; t->node = node;
@ -721,7 +753,9 @@ bool fstBuilderNodeCompileTo(FstBuilderNode* b, FstCountingWriter* wrt, Compiled
FstBuilder* fstBuilderCreate(void* w, FstType ty) { FstBuilder* fstBuilderCreate(void* w, FstType ty) {
FstBuilder* b = malloc(sizeof(FstBuilder)); FstBuilder* b = malloc(sizeof(FstBuilder));
if (NULL == b) { return b; } if (NULL == b) {
return b;
}
b->wrt = fstCountingWriterCreate(w); b->wrt = fstCountingWriterCreate(w);
b->unfinished = fstUnFinishedNodesCreate(); b->unfinished = fstUnFinishedNodesCreate();
@ -735,15 +769,17 @@ FstBuilder* fstBuilderCreate(void* w, FstType ty) {
taosEncodeFixedU64(&pBuf64, VERSION); taosEncodeFixedU64(&pBuf64, VERSION);
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
memset(buf64, 0, sizeof(buf64));
pBuf64 = buf64; pBuf64 = buf64;
memset(buf64, 0, sizeof(buf64));
taosEncodeFixedU64(&pBuf64, ty); taosEncodeFixedU64(&pBuf64, ty);
fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64));
return b; return b;
} }
void fstBuilderDestroy(FstBuilder* b) { void fstBuilderDestroy(FstBuilder* b) {
if (b == NULL) { return; } if (b == NULL) {
return;
}
fstCountingWriterDestroy(b->wrt); fstCountingWriterDestroy(b->wrt);
fstUnFinishedNodesDestroy(b->unfinished); fstUnFinishedNodesDestroy(b->unfinished);
@ -830,6 +866,7 @@ void fstBuilderCompileFrom(FstBuilder* b, uint64_t istate) {
fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); fstUnFinishedNodesTopLastFreeze(b->unfinished, addr);
return; return;
} }
CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) { CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) {
if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) { if (FST_BUILDER_NODE_IS_FINAL(bn) && FST_BUILDER_NODE_TRANS_ISEMPTY(bn) && FST_BUILDER_NODE_FINALOUTPUT_ISZERO(bn)) {
return EMPTY_ADDRESS; return EMPTY_ADDRESS;
@ -844,7 +881,9 @@ CompiledAddr fstBuilderCompile(FstBuilder* b, FstBuilderNode* bn) {
fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr); fstBuilderNodeCompileTo(bn, b->wrt, b->lastAddr, startAddr);
b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1); b->lastAddr = (CompiledAddr)(FST_WRITER_COUNT(b->wrt) - 1);
if (entry->state == NOTFOUND) { FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr); } if (entry->state == NOTFOUND) {
FST_REGISTRY_CELL_INSERT(entry->cell, b->lastAddr);
}
fstRegistryEntryDestroy(entry); fstRegistryEntryDestroy(entry);
return b->lastAddr; return b->lastAddr;
@ -887,7 +926,9 @@ FstSlice fstNodeAsSlice(FstNode* node) {
FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) { FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
FstLastTransition* trn = malloc(sizeof(FstLastTransition)); FstLastTransition* trn = malloc(sizeof(FstLastTransition));
if (trn == NULL) { return NULL; } if (trn == NULL) {
return NULL;
}
trn->inp = inp; trn->inp = inp;
trn->out = out; trn->out = out;
@ -897,7 +938,9 @@ FstLastTransition* fstLastTransitionCreate(uint8_t inp, Output out) {
void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); } void fstLastTransitionDestroy(FstLastTransition* trn) { free(trn); }
void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) { void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, CompiledAddr addr) {
FstLastTransition* trn = unNode->last; FstLastTransition* trn = unNode->last;
if (trn == NULL) { return; } if (trn == NULL) {
return;
}
FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr}; FstTransition t = {.inp = trn->inp, .out = trn->out, .addr = addr};
taosArrayPush(unNode->node->trans, &t); taosArrayPush(unNode->node->trans, &t);
fstLastTransitionDestroy(trn); fstLastTransitionDestroy(trn);
@ -906,27 +949,35 @@ void fstBuilderNodeUnfinishedLastCompiled(FstBuilderNodeUnfinished* unNode, Comp
} }
void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) { void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished* unNode, Output out) {
if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) { unNode->node->finalOutput += out; } if (FST_BUILDER_NODE_IS_FINAL(unNode->node)) {
unNode->node->finalOutput += out;
}
size_t sz = taosArrayGetSize(unNode->node->trans); size_t sz = taosArrayGetSize(unNode->node->trans);
for (size_t i = 0; i < sz; i++) { for (size_t i = 0; i < sz; i++) {
FstTransition* trn = taosArrayGet(unNode->node->trans, i); FstTransition* trn = taosArrayGet(unNode->node->trans, i);
trn->out += out; trn->out += out;
} }
if (unNode->last) { unNode->last->out += out; } if (unNode->last) {
unNode->last->out += out;
}
return; return;
} }
Fst* fstCreate(FstSlice* slice) { Fst* fstCreate(FstSlice* slice) {
int32_t slen; int32_t slen;
char* buf = fstSliceData(slice, &slen); char* buf = fstSliceData(slice, &slen);
if (slen < 36) { return NULL; } if (slen < 36) {
return NULL;
}
uint64_t len = slen; uint64_t len = slen;
uint64_t skip = 0; uint64_t skip = 0;
uint64_t version; uint64_t version;
taosDecodeFixedU64(buf, &version); taosDecodeFixedU64(buf, &version);
skip += sizeof(version); skip += sizeof(version);
if (version == 0 || version > VERSION) { return NULL; } if (version == 0 || version > VERSION) {
return NULL;
}
uint64_t type; uint64_t type;
taosDecodeFixedU64(buf + skip, &type); taosDecodeFixedU64(buf + skip, &type);
@ -949,10 +1000,14 @@ Fst* fstCreate(FstSlice* slice) {
taosDecodeFixedU64(buf + len, &fstLen); taosDecodeFixedU64(buf + len, &fstLen);
// TODO(validate root addr) // TODO(validate root addr)
Fst* fst = (Fst*)calloc(1, sizeof(Fst)); Fst* fst = (Fst*)calloc(1, sizeof(Fst));
if (fst == NULL) { return NULL; } if (fst == NULL) {
return NULL;
}
fst->meta = (FstMeta*)malloc(sizeof(FstMeta)); fst->meta = (FstMeta*)malloc(sizeof(FstMeta));
if (NULL == fst->meta) { goto FST_CREAT_FAILED; } if (NULL == fst->meta) {
goto FST_CREAT_FAILED;
}
fst->meta->version = version; fst->meta->version = version;
fst->meta->rootAddr = rootAddr; fst->meta->rootAddr = rootAddr;
@ -983,7 +1038,7 @@ void fstDestroy(Fst* fst) {
bool fstGet(Fst* fst, FstSlice* b, Output* out) { bool fstGet(Fst* fst, FstSlice* b, Output* out) {
// dec lock range // dec lock range
pthread_mutex_lock(&fst->mtx); // pthread_mutex_lock(&fst->mtx);
FstNode* root = fstGetRoot(fst); FstNode* root = fstGetRoot(fst);
Output tOut = 0; Output tOut = 0;
int32_t len; int32_t len;
@ -996,7 +1051,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
uint8_t inp = data[i]; uint8_t inp = data[i];
Output res = 0; Output res = 0;
if (false == fstNodeFindInput(root, inp, &res)) { if (false == fstNodeFindInput(root, inp, &res)) {
pthread_mutex_unlock(&fst->mtx); // pthread_mutex_unlock(&fst->mtx);
return false; return false;
} }
@ -1007,7 +1062,7 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
taosArrayPush(nodes, &root); taosArrayPush(nodes, &root);
} }
if (!FST_NODE_IS_FINAL(root)) { if (!FST_NODE_IS_FINAL(root)) {
pthread_mutex_unlock(&fst->mtx); // pthread_mutex_unlock(&fst->mtx);
return false; return false;
} else { } else {
tOut = tOut + FST_NODE_FINAL_OUTPUT(root); tOut = tOut + FST_NODE_FINAL_OUTPUT(root);
@ -1018,8 +1073,8 @@ bool fstGet(Fst* fst, FstSlice* b, Output* out) {
fstNodeDestroy(*node); fstNodeDestroy(*node);
} }
taosArrayDestroy(nodes); taosArrayDestroy(nodes);
fst->root = NULL; // fst->root = NULL;
pthread_mutex_unlock(&fst->mtx); // pthread_mutex_unlock(&fst->mtx);
*out = tOut; *out = tOut;
return true; return true;
} }
@ -1028,7 +1083,9 @@ FstStreamBuilder* fstSearch(Fst* fst, AutomationCtx* ctx) {
return fstStreamBuilderCreate(fst, ctx); return fstStreamBuilderCreate(fst, ctx);
} }
StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) { StreamWithState* streamBuilderIntoStream(FstStreamBuilder* sb) {
if (sb == NULL) { return NULL; } if (sb == NULL) {
return NULL;
}
return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max); return streamWithStateCreate(sb->fst, sb->aut, sb->min, sb->max);
} }
FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) { FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
@ -1039,15 +1096,6 @@ FstStreamWithStateBuilder* fstSearchWithState(Fst* fst, AutomationCtx* ctx) {
FstNode* fstGetRoot(Fst* fst) { FstNode* fstGetRoot(Fst* fst) {
CompiledAddr rAddr = fstGetRootAddr(fst); CompiledAddr rAddr = fstGetRootAddr(fst);
return fstGetNode(fst, rAddr); return fstGetNode(fst, rAddr);
// pthread_mutex_lock(&fst->mtx);
// if (fst->root != NULL) {
// // pthread_mutex_unlock(&fst->mtx);
// return fst->root;
//}
// CompiledAddr rAddr = fstGetRootAddr(fst);
// fst->root = fstGetNode(fst, rAddr);
//// pthread_mutex_unlock(&fst->mtx);
// return fst->root;
} }
FstNode* fstGetNode(Fst* fst, CompiledAddr addr) { FstNode* fstGetNode(Fst* fst, CompiledAddr addr) {
@ -1074,14 +1122,18 @@ bool fstVerify(Fst* fst) {
uint32_t len, checkSum = fst->meta->checkSum; uint32_t len, checkSum = fst->meta->checkSum;
uint8_t* data = fstSliceData(fst->data, &len); uint8_t* data = fstSliceData(fst->data, &len);
TSCKSUM initSum = 0; TSCKSUM initSum = 0;
if (!taosCheckChecksumWhole(data, len)) { return false; } if (!taosCheckChecksumWhole(data, len)) {
return false;
}
return true; return true;
} }
// data bound function // data bound function
FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) { FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice* data) {
FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData)); FstBoundWithData* b = calloc(1, sizeof(FstBoundWithData));
if (b == NULL) { return NULL; } if (b == NULL) {
return NULL;
}
if (data != NULL) { if (data != NULL) {
b->data = fstSliceCopy(data, data->start, data->end); b->data = fstSliceCopy(data, data->start, data->end);
@ -1118,7 +1170,9 @@ void fstBoundDestroy(FstBoundWithData* bound) { free(bound); }
StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min, StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstBoundWithData* min,
FstBoundWithData* max) { FstBoundWithData* max) {
StreamWithState* sws = calloc(1, sizeof(StreamWithState)); StreamWithState* sws = calloc(1, sizeof(StreamWithState));
if (sws == NULL) { return NULL; } if (sws == NULL) {
return NULL;
}
sws->fst = fst; sws->fst = fst;
sws->aut = automation; sws->aut = automation;
@ -1134,7 +1188,9 @@ StreamWithState* streamWithStateCreate(Fst* fst, AutomationCtx* automation, FstB
return sws; return sws;
} }
void streamWithStateDestroy(StreamWithState* sws) { void streamWithStateDestroy(StreamWithState* sws) {
if (sws == NULL) { return; } if (sws == NULL) {
return;
}
taosArrayDestroy(sws->inp); taosArrayDestroy(sws->inp);
taosArrayDestroyEx(sws->stack, streamStateDestroy); taosArrayDestroyEx(sws->stack, streamStateDestroy);
@ -1200,7 +1256,9 @@ bool streamWithStateSeekMin(StreamWithState* sws, FstBoundWithData* min) {
uint64_t i = 0; uint64_t i = 0;
for (i = trans->range.start; i < trans->range.end; i++) { for (i = trans->range.start; i < trans->range.end; i++) {
FstTransition trn; FstTransition trn;
if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) { break; } if (fstNodeGetTransitionAt(node, i, &trn) && trn.inp > b) {
break;
}
} }
StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState}; StreamState s = {.node = node, .trans = i, .out = {.null = false, .out = out}, .autState = autState};
@ -1248,7 +1306,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
while (taosArrayGetSize(sws->stack) > 0) { while (taosArrayGetSize(sws->stack) > 0) {
StreamState* p = (StreamState*)taosArrayPop(sws->stack); StreamState* p = (StreamState*)taosArrayPop(sws->stack);
if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) { if (p->trans >= FST_NODE_LEN(p->node) || !automFuncs[aut->type].canMatch(aut, p->autState)) {
if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) { taosArrayPop(sws->inp); } if (FST_NODE_ADDR(p->node) != fstGetRootAddr(sws->fst)) {
taosArrayPop(sws->inp);
}
streamStateDestroy(p); streamStateDestroy(p);
continue; continue;
} }
@ -1267,7 +1327,9 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
if (FST_NODE_IS_FINAL(nextNode)) { if (FST_NODE_IS_FINAL(nextNode)) {
// void *eofState = sws->aut->acceptEof(nextState); // void *eofState = sws->aut->acceptEof(nextState);
void* eofState = automFuncs[aut->type].acceptEof(aut, nextState); void* eofState = automFuncs[aut->type].acceptEof(aut, nextState);
if (eofState != NULL) { isMatch = automFuncs[aut->type].isMatch(aut, eofState); } if (eofState != NULL) {
isMatch = automFuncs[aut->type].isMatch(aut, eofState);
}
} }
StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState}; StreamState s1 = {.node = p->node, .trans = p->trans + 1, .out = p->out, .autState = p->autState};
taosArrayPush(sws->stack, &s1); taosArrayPush(sws->stack, &s1);
@ -1277,24 +1339,26 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
size_t isz = taosArrayGetSize(sws->inp); size_t isz = taosArrayGetSize(sws->inp);
uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t)); uint8_t* buf = (uint8_t*)malloc(isz * sizeof(uint8_t));
for (uint32_t i = 0; i < isz; i++) { buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i); } for (uint32_t i = 0; i < isz; i++) {
buf[i] = *(uint8_t*)taosArrayGet(sws->inp, i);
}
FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp)); FstSlice slice = fstSliceCreate(buf, taosArrayGetSize(sws->inp));
if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { if (fstBoundWithDataExceededBy(sws->endAt, &slice)) {
taosArrayDestroyEx(sws->stack, streamStateDestroy); taosArrayDestroyEx(sws->stack, streamStateDestroy);
sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState)); sws->stack = (SArray*)taosArrayInit(256, sizeof(StreamState));
free(buf); tfree(buf);
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
return NULL; return NULL;
} }
if (FST_NODE_IS_FINAL(nextNode) && isMatch) { if (FST_NODE_IS_FINAL(nextNode) && isMatch) {
FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)};
StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState); StreamWithStateResult* result = swsResultCreate(&slice, fOutput, tState);
free(buf); tfree(buf);
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
taosArrayDestroy(nodes); taosArrayDestroy(nodes);
return result; return result;
} }
free(buf); tfree(buf);
fstSliceDestroy(&slice); fstSliceDestroy(&slice);
} }
for (size_t i = 0; i < taosArrayGetSize(nodes); i++) { for (size_t i = 0; i < taosArrayGetSize(nodes); i++) {
@ -1307,16 +1371,19 @@ StreamWithStateResult* streamWithStateNextWith(StreamWithState* sws, StreamCallb
StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) { StreamWithStateResult* swsResultCreate(FstSlice* data, FstOutput fOut, void* state) {
StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult)); StreamWithStateResult* result = calloc(1, sizeof(StreamWithStateResult));
if (result == NULL) { return NULL; } if (result == NULL) {
return NULL;
}
result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1);
result->out = fOut; result->out = fOut;
result->state = state; result->state = state;
return result; return result;
} }
void swsResultDestroy(StreamWithStateResult* result) { void swsResultDestroy(StreamWithStateResult* result) {
if (NULL == result) { return; } if (NULL == result) {
return;
}
fstSliceDestroy(&result->data); fstSliceDestroy(&result->data);
startWithStateValueDestroy(result->state); startWithStateValueDestroy(result->state);
@ -1324,16 +1391,18 @@ void swsResultDestroy(StreamWithStateResult* result) {
} }
void streamStateDestroy(void* s) { void streamStateDestroy(void* s) {
if (NULL == s) { return; } if (NULL == s) {
return;
}
StreamState* ss = (StreamState*)s; StreamState* ss = (StreamState*)s;
fstNodeDestroy(ss->node); fstNodeDestroy(ss->node);
// free(s->autoState);
} }
FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) { FstStreamBuilder* fstStreamBuilderCreate(Fst* fst, AutomationCtx* aut) {
FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder)); FstStreamBuilder* b = calloc(1, sizeof(FstStreamBuilder));
if (NULL == b) { return NULL; } if (NULL == b) {
return NULL;
}
b->fst = fst; b->fst = fst;
b->aut = aut; b->aut = aut;
@ -1349,8 +1418,9 @@ void fstStreamBuilderDestroy(FstStreamBuilder* b) {
free(b); free(b);
} }
FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) { FstStreamBuilder* fstStreamBuilderRange(FstStreamBuilder* b, FstSlice* val, RangeType type) {
if (b == NULL) { return NULL; } if (b == NULL) {
return NULL;
}
if (type == GE) { if (type == GE) {
b->min->type = Included; b->min->type = Included;
fstSliceDestroy(&(b->min->data)); fstSliceDestroy(&(b->min->data));

24
source/libs/index/src/index_fst_automation.c
View File

@ -17,7 +17,9 @@
StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) { StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueType ty, void* val) {
StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; } if (nsv == NULL) {
return NULL;
}
nsv->kind = kind; nsv->kind = kind;
nsv->type = ty; nsv->type = ty;
@ -35,7 +37,9 @@ StartWithStateValue* startWithStateValueCreate(StartWithStateKind kind, ValueTyp
} }
void startWithStateValueDestroy(void* val) { void startWithStateValueDestroy(void* val) {
StartWithStateValue* sv = (StartWithStateValue*)val; StartWithStateValue* sv = (StartWithStateValue*)val;
if (sv == NULL) { return; } if (sv == NULL) {
return;
}
if (sv->type == FST_INT) { if (sv->type == FST_INT) {
// //
@ -48,7 +52,9 @@ void startWithStateValueDestroy(void* val) {
} }
StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) { StartWithStateValue* startWithStateValueDump(StartWithStateValue* sv) {
StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue)); StartWithStateValue* nsv = calloc(1, sizeof(StartWithStateValue));
if (nsv == NULL) { return NULL; } if (nsv == NULL) {
return NULL;
}
nsv->kind = sv->kind; nsv->kind = sv->kind;
nsv->type = sv->type; nsv->type = sv->type;
@ -88,10 +94,14 @@ static bool prefixCanMatch(AutomationCtx* ctx, void* sv) {
static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; } static bool prefixWillAlwaysMatch(AutomationCtx* ctx, void* state) { return true; }
static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) { static void* prefixAccept(AutomationCtx* ctx, void* state, uint8_t byte) {
StartWithStateValue* ssv = (StartWithStateValue*)state; StartWithStateValue* ssv = (StartWithStateValue*)state;
if (ssv == NULL || ctx == NULL) { return NULL; } if (ssv == NULL || ctx == NULL) {
return NULL;
}
char* data = ctx->data; char* data = ctx->data;
if (ssv->kind == Done) { return startWithStateValueCreate(Done, FST_INT, &ssv->val); } if (ssv->kind == Done) {
return startWithStateValueCreate(Done, FST_INT, &ssv->val);
}
if ((strlen(data) > ssv->val) && data[ssv->val] == byte) { if ((strlen(data) > ssv->val) && data[ssv->val] == byte) {
int val = ssv->val + 1; int val = ssv->val + 1;
@ -128,7 +138,9 @@ AutomationFunc automFuncs[] = {
AutomationCtx* automCtxCreate(void* data, AutomationType atype) { AutomationCtx* automCtxCreate(void* data, AutomationType atype) {
AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx)); AutomationCtx* ctx = calloc(1, sizeof(AutomationCtx));
if (ctx == NULL) { return NULL; } if (ctx == NULL) {
return NULL;
}
StartWithStateValue* sv = NULL; StartWithStateValue* sv = NULL;
if (atype == AUTOMATION_ALWAYS) { if (atype == AUTOMATION_ALWAYS) {

25
source/libs/index/src/index_fst_util.c
View File

@ -29,18 +29,6 @@ const uint64_t VERSION = 3;
const uint64_t TRANS_INDEX_THRESHOLD = 32; const uint64_t TRANS_INDEX_THRESHOLD = 32;
// uint8_t commonInput(uint8_t idx) {
// if (idx == 0) { return -1; }
// else {
// return COMMON_INPUTS_INV[idx - 1];
// }
//}
//
// uint8_t commonIdx(uint8_t v, uint8_t max) {
// uint8_t v = ((uint16_t)tCOMMON_INPUTS[v] + 1)%256;
// return v > max ? 0: v;
//}
uint8_t packSize(uint64_t n) { uint8_t packSize(uint64_t n) {
if (n < (1u << 8)) { if (n < (1u << 8)) {
return 1; return 1;
@ -103,9 +91,6 @@ FstSlice fstSliceCreate(uint8_t* data, uint64_t len) {
FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) { FstSlice fstSliceCopy(FstSlice* s, int32_t start, int32_t end) {
FstString* str = s->str; FstString* str = s->str;
str->ref++; str->ref++;
// uint8_t *buf = fstSliceData(s, &alen);
// start = buf + start - (buf - s->start);
// end = buf + end - (buf - s->start);
FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start};
return t; return t;
@ -130,19 +115,19 @@ FstSlice fstSliceDeepCopy(FstSlice* s, int32_t start, int32_t end) {
ans.end = tlen - 1; ans.end = tlen - 1;
return ans; return ans;
} }
bool fstSliceIsEmpty(FstSlice* s) { bool fstSliceIsEmpty(FstSlice* s) { return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; }
return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0;
}
uint8_t* fstSliceData(FstSlice* s, int32_t* size) { uint8_t* fstSliceData(FstSlice* s, int32_t* size) {
FstString* str = s->str; FstString* str = s->str;
if (size != NULL) { *size = s->end - s->start + 1; } if (size != NULL) {
*size = s->end - s->start + 1;
}
return str->data + s->start; return str->data + s->start;
} }
void fstSliceDestroy(FstSlice* s) { void fstSliceDestroy(FstSlice* s) {
FstString* str = s->str; FstString* str = s->str;
str->ref--; str->ref--;
if (str->ref <= 0) { if (str->ref == 0) {
free(str->data); free(str->data);
free(str); free(str);
s->str = NULL; s->str = NULL;

126
source/libs/index/src/index_tfile.c
View File

@ -13,8 +13,6 @@ p *
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
//#include <sys/types.h>
//#include <dirent.h>
#include "index_tfile.h" #include "index_tfile.h"
#include "index.h" #include "index.h"
#include "index_fst.h" #include "index_fst.h"
@ -61,7 +59,9 @@ static void tfileGenFileFullName(char* fullname, const char* path, uint64_t s
TFileCache* tfileCacheCreate(const char* path) { TFileCache* tfileCacheCreate(const char* path) {
TFileCache* tcache = calloc(1, sizeof(TFileCache)); TFileCache* tcache = calloc(1, sizeof(TFileCache));
if (tcache == NULL) { return NULL; } if (tcache == NULL) {
return NULL;
}
tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); tcache->tableCache = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
tcache->capacity = 64; tcache->capacity = 64;
@ -98,7 +98,9 @@ End:
return NULL; return NULL;
} }
void tfileCacheDestroy(TFileCache* tcache) { void tfileCacheDestroy(TFileCache* tcache) {
if (tcache == NULL) { return; } if (tcache == NULL) {
return;
}
// free table cache // free table cache
TFileReader** reader = taosHashIterate(tcache->tableCache, NULL); TFileReader** reader = taosHashIterate(tcache->tableCache, NULL);
@ -119,7 +121,9 @@ TFileReader* tfileCacheGet(TFileCache* tcache, ICacheKey* key) {
int32_t sz = indexSerialCacheKey(key, buf); int32_t sz = indexSerialCacheKey(key, buf);
assert(sz < sizeof(buf)); assert(sz < sizeof(buf));
TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz); TFileReader** reader = taosHashGet(tcache->tableCache, buf, sz);
if (reader == NULL) { return NULL; } if (reader == NULL) {
return NULL;
}
tfileReaderRef(*reader); tfileReaderRef(*reader);
return *reader; return *reader;
@ -142,7 +146,9 @@ void tfileCachePut(TFileCache* tcache, ICacheKey* key, TFileReader* reader) {
} }
TFileReader* tfileReaderCreate(WriterCtx* ctx) { TFileReader* tfileReaderCreate(WriterCtx* ctx) {
TFileReader* reader = calloc(1, sizeof(TFileReader)); TFileReader* reader = calloc(1, sizeof(TFileReader));
if (reader == NULL) { return NULL; } if (reader == NULL) {
return NULL;
}
reader->ctx = ctx; reader->ctx = ctx;
@ -169,7 +175,9 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) {
return reader; return reader;
} }
void tfileReaderDestroy(TFileReader* reader) { void tfileReaderDestroy(TFileReader* reader) {
if (reader == NULL) { return; } if (reader == NULL) {
return;
}
// T_REF_INC(reader); // T_REF_INC(reader);
fstDestroy(reader->fst); fstDestroy(reader->fst);
writerCtxDestroy(reader->ctx, reader->remove); writerCtxDestroy(reader->ctx, reader->remove);
@ -209,7 +217,9 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int32_t version, const c
tfileGenFileFullName(fullname, path, suid, colName, version); tfileGenFileFullName(fullname, path, suid, colName, version);
// indexInfo("open write file name %s", fullname); // indexInfo("open write file name %s", fullname);
WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64); WriterCtx* wcx = writerCtxCreate(TFile, fullname, false, 1024 * 1024 * 64);
if (wcx == NULL) { return NULL; } if (wcx == NULL) {
return NULL;
}
TFileHeader tfh = {0}; TFileHeader tfh = {0};
tfh.suid = suid; tfh.suid = suid;
@ -225,7 +235,9 @@ TFileReader* tfileReaderOpen(char* path, uint64_t suid, int32_t version, const c
WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024); WriterCtx* wc = writerCtxCreate(TFile, fullname, true, 1024 * 1024 * 1024);
indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size); indexInfo("open read file name:%s, size: %d", wc->file.buf, wc->file.size);
if (wc == NULL) { return NULL; } if (wc == NULL) {
return NULL;
}
TFileReader* reader = tfileReaderCreate(wc); TFileReader* reader = tfileReaderCreate(wc);
return reader; return reader;
@ -316,19 +328,25 @@ int tfileWriterPut(TFileWriter* tw, void* data, bool order) {
return 0; return 0;
} }
void tfileWriterClose(TFileWriter* tw) { void tfileWriterClose(TFileWriter* tw) {
if (tw == NULL) { return; } if (tw == NULL) {
return;
}
writerCtxDestroy(tw->ctx, false); writerCtxDestroy(tw->ctx, false);
free(tw); free(tw);
} }
void tfileWriterDestroy(TFileWriter* tw) { void tfileWriterDestroy(TFileWriter* tw) {
if (tw == NULL) { return; } if (tw == NULL) {
return;
}
writerCtxDestroy(tw->ctx, false); writerCtxDestroy(tw->ctx, false);
free(tw); free(tw);
} }
IndexTFile* indexTFileCreate(const char* path) { IndexTFile* indexTFileCreate(const char* path) {
TFileCache* cache = tfileCacheCreate(path); TFileCache* cache = tfileCacheCreate(path);
if (cache == NULL) { return NULL; } if (cache == NULL) {
return NULL;
}
IndexTFile* tfile = calloc(1, sizeof(IndexTFile)); IndexTFile* tfile = calloc(1, sizeof(IndexTFile));
if (tfile == NULL) { if (tfile == NULL) {
@ -340,21 +358,27 @@ IndexTFile* indexTFileCreate(const char* path) {
return tfile; return tfile;
} }
void indexTFileDestroy(IndexTFile* tfile) { void indexTFileDestroy(IndexTFile* tfile) {
if (tfile == NULL) { return; } if (tfile == NULL) {
return;
}
tfileCacheDestroy(tfile->cache); tfileCacheDestroy(tfile->cache);
free(tfile); free(tfile);
} }
int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) { int indexTFileSearch(void* tfile, SIndexTermQuery* query, SArray* result) {
int ret = -1; int ret = -1;
if (tfile == NULL) { return ret; } if (tfile == NULL) {
return ret;
}
IndexTFile* pTfile = (IndexTFile*)tfile; IndexTFile* pTfile = (IndexTFile*)tfile;
SIndexTerm* term = query->term; SIndexTerm* term = query->term;
ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName}; ICacheKey key = {.suid = term->suid, .colType = term->colType, .colName = term->colName, .nColName = term->nColName};
TFileReader* reader = tfileCacheGet(pTfile->cache, &key); TFileReader* reader = tfileCacheGet(pTfile->cache, &key);
if (reader == NULL) { return 0; } if (reader == NULL) {
return 0;
}
return tfileReaderSearch(reader, query, result); return tfileReaderSearch(reader, query, result);
} }
@ -373,7 +397,9 @@ static bool tfileIteratorNext(Iterate* iiter) {
TFileFstIter* tIter = iiter->iter; TFileFstIter* tIter = iiter->iter;
StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL); StreamWithStateResult* rt = streamWithStateNextWith(tIter->st, NULL);
if (rt == NULL) { return false; } if (rt == NULL) {
return false;
}
int32_t sz = 0; int32_t sz = 0;
char* ch = (char*)fstSliceData(&rt->data, &sz); char* ch = (char*)fstSliceData(&rt->data, &sz);
@ -383,7 +409,9 @@ static bool tfileIteratorNext(Iterate* iiter) {
offset = (uint64_t)(rt->out.out); offset = (uint64_t)(rt->out.out);
swsResultDestroy(rt); swsResultDestroy(rt);
// set up iterate value // set up iterate value
if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) { return false; } if (tfileReaderLoadTableIds(tIter->rdr, offset, iv->val) != 0) {
return false;
}
iv->colVal = colVal; iv->colVal = colVal;
return true; return true;
@ -394,7 +422,9 @@ static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; }
static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) { static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter)); TFileFstIter* tIter = calloc(1, sizeof(TFileFstIter));
if (tIter == NULL) { return NULL; } if (tIter == NULL) {
return NULL;
}
tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS); tIter->ctx = automCtxCreate(NULL, AUTOMATION_ALWAYS);
tIter->fb = fstSearch(reader->fst, tIter->ctx); tIter->fb = fstSearch(reader->fst, tIter->ctx);
@ -404,7 +434,9 @@ static TFileFstIter* tfileFstIteratorCreate(TFileReader* reader) {
} }
Iterate* tfileIteratorCreate(TFileReader* reader) { Iterate* tfileIteratorCreate(TFileReader* reader) {
if (reader == NULL) { return NULL; } if (reader == NULL) {
return NULL;
}
Iterate* iter = calloc(1, sizeof(Iterate)); Iterate* iter = calloc(1, sizeof(Iterate));
iter->iter = tfileFstIteratorCreate(reader); iter->iter = tfileFstIteratorCreate(reader);
@ -419,7 +451,9 @@ Iterate* tfileIteratorCreate(TFileReader* reader) {
return iter; return iter;
} }
void tfileIteratorDestroy(Iterate* iter) { void tfileIteratorDestroy(Iterate* iter) {
if (iter == NULL) { return; } if (iter == NULL) {
return;
}
IterateValue* iv = &iter->val; IterateValue* iv = &iter->val;
iterateValueDestroy(iv, true); iterateValueDestroy(iv, true);
@ -434,7 +468,9 @@ void tfileIteratorDestroy(Iterate* iter) {
} }
TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) { TFileReader* tfileGetReaderByCol(IndexTFile* tf, uint64_t suid, char* colName) {
if (tf == NULL) { return NULL; } if (tf == NULL) {
return NULL;
}
ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)}; ICacheKey key = {.suid = suid, .colType = TSDB_DATA_TYPE_BINARY, .colName = colName, .nColName = strlen(colName)};
return tfileCacheGet(tf->cache, &key); return tfileCacheGet(tf->cache, &key);
} }
@ -446,7 +482,9 @@ static int tfileUidCompare(const void* a, const void* b) {
} }
static int tfileStrCompare(const void* a, const void* b) { static int tfileStrCompare(const void* a, const void* b) {
int ret = strcmp((char*)a, (char*)b); int ret = strcmp((char*)a, (char*)b);
if (ret == 0) { return ret; } if (ret == 0) {
return ret;
}
return ret < 0 ? -1 : 1; return ret < 0 ? -1 : 1;
} }
@ -461,13 +499,17 @@ static int tfileValueCompare(const void* a, const void* b, const void* param) {
TFileValue* tfileValueCreate(char* val) { TFileValue* tfileValueCreate(char* val) {
TFileValue* tf = calloc(1, sizeof(TFileValue)); TFileValue* tf = calloc(1, sizeof(TFileValue));
if (tf == NULL) { return NULL; } if (tf == NULL) {
return NULL;
}
tf->colVal = tstrdup(val); tf->colVal = tstrdup(val);
tf->tableId = taosArrayInit(32, sizeof(uint64_t)); tf->tableId = taosArrayInit(32, sizeof(uint64_t));
return tf; return tf;
} }
int tfileValuePush(TFileValue* tf, uint64_t val) { int tfileValuePush(TFileValue* tf, uint64_t val) {
if (tf == NULL) { return -1; } if (tf == NULL) {
return -1;
}
taosArrayPush(tf->tableId, &val); taosArrayPush(tf->tableId, &val);
return 0; return 0;
} }
@ -489,7 +531,9 @@ static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) {
int32_t fstOffset = offset + sizeof(tw->header.fstOffset); int32_t fstOffset = offset + sizeof(tw->header.fstOffset);
tw->header.fstOffset = fstOffset; tw->header.fstOffset = fstOffset;
if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; } if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) {
return -1;
}
indexInfo("tfile write fst offset: %d", tw->ctx->size(tw->ctx)); indexInfo("tfile write fst offset: %d", tw->ctx->size(tw->ctx));
tw->offset += sizeof(fstOffset); tw->offset += sizeof(fstOffset);
return 0; return 0;
@ -502,7 +546,9 @@ static int tfileWriteHeader(TFileWriter* writer) {
indexInfo("tfile pre write header size: %d", writer->ctx->size(writer->ctx)); indexInfo("tfile pre write header size: %d", writer->ctx->size(writer->ctx));
int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf)); int nwrite = writer->ctx->write(writer->ctx, buf, sizeof(buf));
if (sizeof(buf) != nwrite) { return -1; } if (sizeof(buf) != nwrite) {
return -1;
}
indexInfo("tfile after write header size: %d", writer->ctx->size(writer->ctx)); indexInfo("tfile after write header size: %d", writer->ctx->size(writer->ctx));
writer->offset = nwrite; writer->offset = nwrite;
@ -556,7 +602,9 @@ static int tfileReaderLoadFst(TFileReader* reader) {
static int FST_MAX_SIZE = 64 * 1024 * 1024; static int FST_MAX_SIZE = 64 * 1024 * 1024;
char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE); char* buf = calloc(1, sizeof(char) * FST_MAX_SIZE);
if (buf == NULL) { return -1; } if (buf == NULL) {
return -1;
}
WriterCtx* ctx = reader->ctx; WriterCtx* ctx = reader->ctx;
int size = ctx->size(ctx); int size = ctx->size(ctx);
@ -586,12 +634,16 @@ static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray*
int32_t total = sizeof(uint64_t) * nid; int32_t total = sizeof(uint64_t) * nid;
char* buf = calloc(1, total); char* buf = calloc(1, total);
if (buf == NULL) { return -1; } if (buf == NULL) {
return -1;
}
nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid)); nread = ctx->readFrom(ctx, buf, total, offset + sizeof(nid));
assert(total == nread); assert(total == nread);
for (int32_t i = 0; i < nid; i++) { taosArrayPush(result, (uint64_t*)buf + i); } for (int32_t i = 0; i < nid; i++) {
taosArrayPush(result, (uint64_t*)buf + i);
}
free(buf); free(buf);
return 0; return 0;
} }
@ -615,13 +667,17 @@ static int tfileReaderVerify(TFileReader* reader) {
} }
void tfileReaderRef(TFileReader* reader) { void tfileReaderRef(TFileReader* reader) {
if (reader == NULL) { return; } if (reader == NULL) {
return;
}
int ref = T_REF_INC(reader); int ref = T_REF_INC(reader);
UNUSED(ref); UNUSED(ref);
} }
void tfileReaderUnRef(TFileReader* reader) { void tfileReaderUnRef(TFileReader* reader) {
if (reader == NULL) { return; } if (reader == NULL) {
return;
}
int ref = T_REF_DEC(reader); int ref = T_REF_DEC(reader);
if (ref == 0) { if (ref == 0) {
// do nothing // do nothing
@ -637,11 +693,15 @@ static SArray* tfileGetFileList(const char* path) {
uint32_t version; uint32_t version;
DIR* dir = opendir(path); DIR* dir = opendir(path);
if (NULL == dir) { return NULL; } if (NULL == dir) {
return NULL;
}
struct dirent* entry; struct dirent* entry;
while ((entry = readdir(dir)) != NULL) { while ((entry = readdir(dir)) != NULL) {
char* file = entry->d_name; char* file = entry->d_name;
if (0 != tfileParseFileName(file, &suid, buf, &version)) { continue; } if (0 != tfileParseFileName(file, &suid, buf, &version)) {
continue;
}
size_t len = strlen(path) + 1 + strlen(file) + 1; size_t len = strlen(path) + 1 + strlen(file) + 1;
char* buf = calloc(1, len); char* buf = calloc(1, len);

20
source/libs/index/test/CMakeLists.txt
View File

@ -1,5 +1,7 @@
add_executable(indexTest "") add_executable(indexTest "")
add_executable(fstTest "") add_executable(fstTest "")
add_executable(fstUT "")
target_sources(indexTest target_sources(indexTest
PRIVATE PRIVATE
"indexTests.cc" "indexTests.cc"
@ -8,6 +10,11 @@ target_sources(fstTest
PRIVATE PRIVATE
"fstTest.cc" "fstTest.cc"
) )
target_sources(fstUT
PRIVATE
"fstUT.cc"
)
target_include_directories ( indexTest target_include_directories ( indexTest
PUBLIC PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_SOURCE_DIR}/include/libs/index"
@ -18,6 +25,12 @@ target_include_directories ( fstTest
"${CMAKE_SOURCE_DIR}/include/libs/index" "${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc" "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
) )
target_include_directories ( fstUT
PUBLIC
"${CMAKE_SOURCE_DIR}/include/libs/index"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
target_link_libraries (indexTest target_link_libraries (indexTest
os os
util util
@ -32,6 +45,13 @@ target_link_libraries (fstTest
gtest_main gtest_main
index index
) )
target_link_libraries (fstUT
os
util
common
gtest_main
index
)
#add_test( #add_test(

11
source/libs/index/test/fstTest.cc
View File

@ -58,7 +58,9 @@ class FstReadMemory {
bool init() { bool init() {
char* buf = (char*)calloc(1, sizeof(char) * _size); char* buf = (char*)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size); int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
if (nRead <= 0) { return false; } if (nRead <= 0) {
return false;
}
_size = nRead; _size = nRead;
_s = fstSliceCreate((uint8_t*)buf, _size); _s = fstSliceCreate((uint8_t*)buf, _size);
_fst = fstCreate(&_s); _fst = fstCreate(&_s);
@ -97,7 +99,8 @@ class FstReadMemory {
printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out)); printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
swsResultDestroy(rt); swsResultDestroy(rt);
} }
for (size_t i = 0; i < result.size(); i++) {} for (size_t i = 0; i < result.size(); i++) {
}
std::cout << std::endl; std::cout << std::endl;
return true; return true;
} }
@ -173,7 +176,9 @@ void checkMillonWriteAndReadOfFst() {
delete fw; delete fw;
FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024); FstReadMemory* fr = new FstReadMemory(1024 * 64 * 1024);
if (fr->init()) { printf("success to init fst read"); } if (fr->init()) {
printf("success to init fst read");
}
Performance_fstReadRecords(fr); Performance_fstReadRecords(fr);
tfCleanup(); tfCleanup();

232
source/libs/index/test/fstUT.cc Normal file
View File

@ -0,0 +1,232 @@
#include <gtest/gtest.h>
#include <algorithm>
#include <iostream>
#include <string>
#include <thread>
#include <vector>
#include "index.h"
#include "indexInt.h"
#include "index_cache.h"
#include "index_fst.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "tglobal.h"
#include "tskiplist.h"
#include "tutil.h"
#include "ulog.h"
static std::string dir = "/tmp/index";
static char indexlog[PATH_MAX] = {0};
static char tindex[PATH_MAX] = {0};
static char tindexDir[PATH_MAX] = {0};
static void EnvInit() {
tfInit();
std::string path = dir;
taosRemoveDir(path.c_str());
taosMkDir(path.c_str());
// init log file
snprintf(indexlog, PATH_MAX, "%s/tindex.idx", path.c_str());
if (taosInitLog(indexlog, tsNumOfLogLines, 1) != 0) {
printf("failed to init log");
}
// init index file
memset(tindex, 0, sizeof(tindex));
snprintf(tindex, PATH_MAX, "%s/tindex.idx", path.c_str());
}
static void EnvCleanup() {}
class FstWriter {
public:
FstWriter() {
_wc = writerCtxCreate(TFile, tindex, false, 64 * 1024 * 1024);
_b = fstBuilderCreate(_wc, 0);
}
bool Put(const std::string& key, uint64_t val) {
// char buf[128] = {0};
// int len = 0;
// taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
// FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstBuilderInsert(_b, skey, val);
fstSliceDestroy(&skey);
return ok;
}
~FstWriter() {
fstBuilderFinish(_b);
fstBuilderDestroy(_b);
writerCtxDestroy(_wc, false);
}
private:
FstBuilder* _b;
WriterCtx* _wc;
};
class FstReadMemory {
public:
FstReadMemory(size_t size) {
_wc = writerCtxCreate(TFile, tindex, true, 64 * 1024);
_w = fstCountingWriterCreate(_wc);
_size = size;
memset((void*)&_s, 0, sizeof(_s));
}
bool init() {
char* buf = (char*)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
if (nRead <= 0) {
return false;
}
_size = nRead;
_s = fstSliceCreate((uint8_t*)buf, _size);
_fst = fstCreate(&_s);
free(buf);
return _fst != NULL;
}
bool Get(const std::string& key, uint64_t* val) {
// char buf[128] = {0};
// int len = 0;
// taosMbsToUcs4(key.c_str(), key.size(), buf, 128, &len);
// FstSlice skey = fstSliceCreate((uint8_t*)buf, len);
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstGet(_fst, &skey, val);
fstSliceDestroy(&skey);
return ok;
}
bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) {
int64_t s = taosGetTimestampUs();
bool ok = this->Get(key, val);
int64_t e = taosGetTimestampUs();
*elapse = e - s;
return ok;
}
// add later
bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
FstStreamBuilder* sb = fstSearch(_fst, ctx);
StreamWithState* st = streamBuilderIntoStream(sb);
StreamWithStateResult* rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
// result.push_back((uint64_t)(rt->out.out));
FstSlice* s = &rt->data;
int32_t sz = 0;
char* ch = (char*)fstSliceData(s, &sz);
std::string key(ch, sz);
printf("key: %s, val: %" PRIu64 "\n", key.c_str(), (uint64_t)(rt->out.out));
swsResultDestroy(rt);
}
for (size_t i = 0; i < result.size(); i++) {
}
std::cout << std::endl;
return true;
}
bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
int64_t s = taosGetTimestampUs();
bool ok = this->Search(ctx, result);
int64_t e = taosGetTimestampUs();
return ok;
}
~FstReadMemory() {
fstCountingWriterDestroy(_w);
fstDestroy(_fst);
fstSliceDestroy(&_s);
writerCtxDestroy(_wc, false);
tfCleanup();
}
private:
FstCountingWriter* _w;
Fst* _fst;
FstSlice _s;
WriterCtx* _wc;
size_t _size;
};
class FstWriterEnv : public ::testing::Test {
protected:
virtual void SetUp() { fw = new FstWriter(); }
virtual void TearDown() { delete fw; }
FstWriter* fw = NULL;
};
class FstReadEnv : public ::testing::Test {
protected:
virtual void SetUp() { fr = new FstReadMemory(1024); }
virtual void TearDown() { delete fr; }
FstReadMemory* fr = NULL;
};
class TFst {
public:
void CreateWriter() { fw = new FstWriter; }
void ReCreateWriter() {
if (fw != NULL) delete fw;
fw = new FstWriter;
}
void DestroyWriter() {
if (fw != NULL) delete fw;
}
void CreateReader() {
fr = new FstReadMemory(1024);
fr->init();
}
void ReCreateReader() {
if (fr != NULL) delete fr;
fr = new FstReadMemory(1024);
}
void DestroyReader() {
delete fr;
fr = NULL;
}
bool Put(const std::string& k, uint64_t v) {
if (fw == NULL) {
return false;
}
return fw->Put(k, v);
}
bool Get(const std::string& k, uint64_t* v) {
if (fr == NULL) {
return false;
}
return fr->Get(k, v);
}
private:
FstWriter* fw;
FstReadMemory* fr;
};
class FstEnv : public ::testing::Test {
protected:
virtual void SetUp() {
EnvInit();
fst = new TFst;
}
virtual void TearDown() { delete fst; }
TFst* fst;
};
TEST_F(FstEnv, writeNormal) {
fst->CreateWriter();
std::string str("aa");
for (int i = 0; i < 10; i++) {
str[0] = 'a' + i;
str.resize(2);
assert(fst->Put(str, i) == true);
}
// order failed
assert(fst->Put("aa", 1) == false);
fst->DestroyWriter();
fst->CreateReader();
uint64_t val;
assert(fst->Get("a", &val) == false);
assert(fst->Get("aa", &val) == true);
assert(val == 0);
}

26
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -15,6 +15,7 @@
#include "querynodes.h" #include "querynodes.h"
#include "taos.h" #include "taos.h"
#include "taoserror.h"
#define COPY_SCALAR_FIELD(fldname) \ #define COPY_SCALAR_FIELD(fldname) \
do { \ do { \
@ -57,14 +58,13 @@ static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) {
} }
static void exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { static void exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) {
COPY_SCALAR_FIELD(type);
dataTypeCopy(&pSrc->resType, &pDst->resType); dataTypeCopy(&pSrc->resType, &pDst->resType);
COPY_CHAR_ARRAY_FIELD(aliasName); COPY_CHAR_ARRAY_FIELD(aliasName);
// COPY_NODE_LIST_FIELD(pAssociationList); // COPY_NODE_LIST_FIELD(pAssociationList);
} }
static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
COPY_SCALAR_FIELD(colId); COPY_SCALAR_FIELD(colId);
COPY_SCALAR_FIELD(colType); COPY_SCALAR_FIELD(colType);
COPY_CHAR_ARRAY_FIELD(dbName); COPY_CHAR_ARRAY_FIELD(dbName);
@ -76,7 +76,7 @@ static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
} }
static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
COPY_CHAR_POINT_FIELD(literal); COPY_CHAR_POINT_FIELD(literal);
COPY_SCALAR_FIELD(isDuration); COPY_SCALAR_FIELD(isDuration);
switch (pSrc->node.resType.type) { switch (pSrc->node.resType.type) {
@ -119,7 +119,7 @@ static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
} }
static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) {
exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
COPY_SCALAR_FIELD(opType); COPY_SCALAR_FIELD(opType);
COPY_NODE_FIELD(pLeft); COPY_NODE_FIELD(pLeft);
COPY_NODE_FIELD(pRight); COPY_NODE_FIELD(pRight);
@ -127,14 +127,14 @@ static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) {
} }
static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) { static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) {
exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
COPY_SCALAR_FIELD(condType); COPY_SCALAR_FIELD(condType);
COPY_NODE_LIST_FIELD(pParameterList); COPY_NODE_LIST_FIELD(pParameterList);
return (SNode*)pDst; return (SNode*)pDst;
} }
static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); exprNodeCopy((const SExprNode*)pSrc, (SExprNode*)pDst);
COPY_CHAR_ARRAY_FIELD(functionName); COPY_CHAR_ARRAY_FIELD(functionName);
COPY_SCALAR_FIELD(funcId); COPY_SCALAR_FIELD(funcId);
COPY_SCALAR_FIELD(funcType); COPY_SCALAR_FIELD(funcType);
@ -142,12 +142,19 @@ static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) {
return (SNode*)pDst; return (SNode*)pDst;
} }
static SNode* groupingSetNodeCopy(const SGroupingSetNode* pSrc, SGroupingSetNode* pDst) {
COPY_SCALAR_FIELD(groupingSetType);
COPY_NODE_LIST_FIELD(pParameterList);
return (SNode*)pDst;
}
SNode* nodesCloneNode(const SNode* pNode) { SNode* nodesCloneNode(const SNode* pNode) {
if (NULL == pNode) { if (NULL == pNode) {
return NULL; return NULL;
} }
SNode* pDst = nodesMakeNode(nodeType(pNode)); SNode* pDst = nodesMakeNode(nodeType(pNode));
if (NULL == pDst) { if (NULL == pDst) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
switch (nodeType(pNode)) { switch (nodeType(pNode)) {
@ -164,7 +171,9 @@ SNode* nodesCloneNode(const SNode* pNode) {
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_TEMP_TABLE:
case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_JOIN_TABLE:
break;
case QUERY_NODE_GROUPING_SET: case QUERY_NODE_GROUPING_SET:
return groupingSetNodeCopy((const SGroupingSetNode*)pNode, (SGroupingSetNode*)pDst);
case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_ORDER_BY_EXPR:
case QUERY_NODE_LIMIT: case QUERY_NODE_LIMIT:
default: default:
@ -174,8 +183,13 @@ SNode* nodesCloneNode(const SNode* pNode) {
} }
SNodeList* nodesCloneList(const SNodeList* pList) { SNodeList* nodesCloneList(const SNodeList* pList) {
if (NULL == pList) {
return NULL;
}
SNodeList* pDst = nodesMakeList(); SNodeList* pDst = nodesMakeList();
if (NULL == pDst) { if (NULL == pDst) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL; return NULL;
} }
SNode* pNode; SNode* pNode;

322
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -26,27 +26,47 @@ static char* nodeName(ENodeType type) {
case QUERY_NODE_COLUMN: case QUERY_NODE_COLUMN:
return "Column"; return "Column";
case QUERY_NODE_VALUE: case QUERY_NODE_VALUE:
return "Value";
case QUERY_NODE_OPERATOR: case QUERY_NODE_OPERATOR:
return "Operator";
case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_LOGIC_CONDITION:
return "LogicCondition";
case QUERY_NODE_FUNCTION: case QUERY_NODE_FUNCTION:
return "Function";
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
return "RealTable";
case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_TEMP_TABLE:
return "TempTable";
case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_JOIN_TABLE:
return "JoinTable";
case QUERY_NODE_GROUPING_SET: case QUERY_NODE_GROUPING_SET:
return "GroupingSet";
case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_ORDER_BY_EXPR:
return "OrderByExpr";
case QUERY_NODE_LIMIT: case QUERY_NODE_LIMIT:
return "Limit";
case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_STATE_WINDOW:
return "StateWindow";
case QUERY_NODE_SESSION_WINDOW: case QUERY_NODE_SESSION_WINDOW:
return "SessionWinow";
case QUERY_NODE_INTERVAL_WINDOW: case QUERY_NODE_INTERVAL_WINDOW:
return "IntervalWindow";
case QUERY_NODE_NODE_LIST: case QUERY_NODE_NODE_LIST:
return "NodeList";
case QUERY_NODE_FILL: case QUERY_NODE_FILL:
return "Fill";
case QUERY_NODE_COLUMN_REF: case QUERY_NODE_COLUMN_REF:
return "ColumnRef";
case QUERY_NODE_TARGET: case QUERY_NODE_TARGET:
return "Target";
case QUERY_NODE_RAW_EXPR: case QUERY_NODE_RAW_EXPR:
return "RawExpr";
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
return "SetOperator";
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return "SelectStmt";
case QUERY_NODE_SHOW_STMT: case QUERY_NODE_SHOW_STMT:
break; return "ShowStmt";
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
return "LogicScan"; return "LogicScan";
case QUERY_NODE_LOGIC_PLAN_JOIN: case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -119,7 +139,7 @@ static int32_t logicPlanNodeToJson(const void* pObj, SJson* pJson) {
static const char* jkScanLogicPlanScanCols = "ScanCols"; static const char* jkScanLogicPlanScanCols = "ScanCols";
static const char* jkScanLogicPlanTableMeta = "TableMeta"; static const char* jkScanLogicPlanTableMeta = "TableMeta";
static int32_t logicScanToJson(const void* pObj, SJson* pJson) { static int32_t logicScanNodeToJson(const void* pObj, SJson* pJson) {
const SScanLogicNode* pNode = (const SScanLogicNode*)pObj; const SScanLogicNode* pNode = (const SScanLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson); int32_t code = logicPlanNodeToJson(pObj, pJson);
@ -135,7 +155,7 @@ static int32_t logicScanToJson(const void* pObj, SJson* pJson) {
static const char* jkProjectLogicPlanProjections = "Projections"; static const char* jkProjectLogicPlanProjections = "Projections";
static int32_t logicProjectToJson(const void* pObj, SJson* pJson) { static int32_t logicProjectNodeToJson(const void* pObj, SJson* pJson) {
const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj; const SProjectLogicNode* pNode = (const SProjectLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson); int32_t code = logicPlanNodeToJson(pObj, pJson);
@ -149,7 +169,7 @@ static int32_t logicProjectToJson(const void* pObj, SJson* pJson) {
static const char* jkJoinLogicPlanJoinType = "JoinType"; static const char* jkJoinLogicPlanJoinType = "JoinType";
static const char* jkJoinLogicPlanOnConditions = "OnConditions"; static const char* jkJoinLogicPlanOnConditions = "OnConditions";
static int32_t logicJoinToJson(const void* pObj, SJson* pJson) { static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj; const SJoinLogicNode* pNode = (const SJoinLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson); int32_t code = logicPlanNodeToJson(pObj, pJson);
@ -163,14 +183,14 @@ static int32_t logicJoinToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static int32_t logicFilterToJson(const void* pObj, SJson* pJson) { static int32_t logicFilterNodeToJson(const void* pObj, SJson* pJson) {
return logicPlanNodeToJson(pObj, pJson); return logicPlanNodeToJson(pObj, pJson);
} }
static const char* jkAggLogicPlanGroupKeys = "GroupKeys"; static const char* jkAggLogicPlanGroupKeys = "GroupKeys";
static const char* jkAggLogicPlanAggFuncs = "AggFuncs"; static const char* jkAggLogicPlanAggFuncs = "AggFuncs";
static int32_t logicAggToJson(const void* pObj, SJson* pJson) { static int32_t logicAggNodeToJson(const void* pObj, SJson* pJson) {
const SAggLogicNode* pNode = (const SAggLogicNode*)pObj; const SAggLogicNode* pNode = (const SAggLogicNode*)pObj;
int32_t code = logicPlanNodeToJson(pObj, pJson); int32_t code = logicPlanNodeToJson(pObj, pJson);
@ -184,17 +204,291 @@ static int32_t logicAggToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
static const char* jkDataTypeType = "Type";
static const char* jkDataTypePrecision = "Precision";
static const char* jkDataTypeScale = "Scale";
static const char* jkDataTypeDataBytes = "Bytes";
static int32_t dataTypeToJson(const void* pObj, SJson* pJson) {
const SDataType* pNode = (const SDataType*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkDataTypeType, pNode->type);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDataTypePrecision, pNode->precision);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDataTypeScale, pNode->scale);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkDataTypeDataBytes, pNode->bytes);
}
return code;
}
static const char* jkExprDataType = "DataType";
static const char* jkExprAliasName = "AliasName";
static int32_t exprNodeToJson(const void* pObj, SJson* pJson) {
const SExprNode* pNode = (const SExprNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkExprDataType, dataTypeToJson, &pNode->resType);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkExprAliasName, pNode->aliasName);
}
return code;
}
static const char* jkColumnTableId = "TableId";
static const char* jkColumnColId = "ColId";
static const char* jkColumnColType = "ColType";
static const char* jkColumnDbName = "DbName";
static const char* jkColumnTableName = "TableName";
static const char* jkColumnTableAlias = "TableAlias";
static const char* jkColumnColName = "ColName";
static int32_t columnNodeToJson(const void* pObj, SJson* pJson) {
const SColumnNode* pNode = (const SColumnNode*)pObj;
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnTableId, pNode->tableId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnColId, pNode->colId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkColumnColType, pNode->colType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkColumnDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkColumnTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkColumnTableAlias, pNode->tableAlias);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkColumnColName, pNode->colName);
}
return code;
}
// typedef struct SValueNode {
// SExprNode node; // QUERY_NODE_VALUE
// char* ;
// bool ;
// union {
// bool b;
// int64_t i;
// uint64_t u;
// double d;
// char* p;
// } datum;
// } SValueNode;
static const char* jkValueLiteral = "Literal";
static const char* jkValueDuration = "Duration";
static const char* jkValueDatum = "Datum";
static int32_t valueNodeToJson(const void* pObj, SJson* pJson) {
const SValueNode* pNode = (const SValueNode*)pObj;
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->isDuration);
}
switch (pNode->node.resType.type) {
case TSDB_DATA_TYPE_NULL:
break;
case TSDB_DATA_TYPE_BOOL:
code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.b);
break;
case TSDB_DATA_TYPE_TINYINT:
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_INT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_TIMESTAMP:
code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.i);
break;
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT:
code = tjsonAddIntegerToObject(pJson, jkValueDuration, pNode->datum.u);
break;
case TSDB_DATA_TYPE_FLOAT:
case TSDB_DATA_TYPE_DOUBLE:
code = tjsonAddDoubleToObject(pJson, jkValueDuration, pNode->datum.d);
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_VARCHAR:
case TSDB_DATA_TYPE_VARBINARY:
code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->datum.p);
break;
case TSDB_DATA_TYPE_JSON:
case TSDB_DATA_TYPE_DECIMAL:
case TSDB_DATA_TYPE_BLOB:
// todo
default:
break;
}
return code;
}
static const char* jkOperatorType = "OpType";
static const char* jkOperatorLeft = "Left";
static const char* jkOperatorRight = "Right";
static int32_t operatorNodeToJson(const void* pObj, SJson* pJson) {
const SOperatorNode* pNode = (const SOperatorNode*)pObj;
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkOperatorType, pNode->opType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkOperatorLeft, nodeToJson, pNode->pLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkOperatorRight, nodeToJson, pNode->pRight);
}
return code;
}
static const char* jkLogicCondType = "CondType";
static const char* jkLogicCondParameters = "Parameters";
static int32_t logicConditionNodeToJson(const void* pObj, SJson* pJson) {
const SLogicConditionNode* pNode = (const SLogicConditionNode*)pObj;
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkLogicCondType, pNode->condType);
}
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkLogicCondParameters, nodeToJson, pNode->pParameterList);
}
return code;
}
static const char* jkFunctionName = "Name";
static const char* jkFunctionId = "Id";
static const char* jkFunctionType = "Type";
static const char* jkFunctionParameter = "Parameters";
static int32_t functionNodeToJson(const void* pObj, SJson* pJson) {
const SFunctionNode* pNode = (const SFunctionNode*)pObj;
int32_t code = exprNodeToJson(pObj, pJson);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkFunctionName, pNode->functionName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkFunctionId, pNode->funcId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkFunctionType, pNode->funcType);
}
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkFunctionParameter, nodeToJson, pNode->pParameterList);
}
return code;
}
static const char* jkGroupingSetType = "GroupingSetType";
static const char* jkGroupingSetParameter = "Parameters";
static int32_t groupingSetNodeToJson(const void* pObj, SJson* pJson) {
const SGroupingSetNode* pNode = (const SGroupingSetNode*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkGroupingSetType, pNode->groupingSetType);
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkGroupingSetParameter, nodeToJson, pNode->pParameterList);
}
return code;
}
static const char* jkSelectStmtDistinct = "Distinct";
static const char* jkSelectStmtProjections = "Projections";
static const char* jkSelectStmtFrom = "From";
static const char* jkSelectStmtWhere = "Where";
static const char* jkSelectStmtPartitionBy = "PartitionBy";
static const char* jkSelectStmtWindow = "Window";
static const char* jkSelectStmtGroupBy = "GroupBy";
static const char* jkSelectStmtHaving = "Having";
static const char* jkSelectStmtOrderBy = "OrderBy";
static const char* jkSelectStmtLimit = "Limit";
static const char* jkSelectStmtSlimit = "Slimit";
static int32_t selectStmtTojson(const void* pObj, SJson* pJson) {
const SSelectStmt* pNode = (const SSelectStmt*)pObj;
int32_t code = tjsonAddIntegerToObject(pJson, jkSelectStmtDistinct, pNode->isDistinct);
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkSelectStmtProjections, nodeToJson, pNode->pProjectionList);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSelectStmtFrom, nodeToJson, pNode->pFromTable);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSelectStmtWhere, nodeToJson, pNode->pWhere);
}
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkSelectStmtPartitionBy, nodeToJson, pNode->pPartitionByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSelectStmtWindow, nodeToJson, pNode->pWindow);
}
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkSelectStmtGroupBy, nodeToJson, pNode->pGroupByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSelectStmtHaving, nodeToJson, pNode->pHaving);
}
if (TSDB_CODE_SUCCESS == code) {
code = addNodeList(pJson, jkSelectStmtOrderBy, nodeToJson, pNode->pOrderByList);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSelectStmtLimit, nodeToJson, pNode->pLimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkSelectStmtSlimit, nodeToJson, pNode->pSlimit);
}
return code;
}
static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
switch (nodeType(pObj)) { switch (nodeType(pObj)) {
case QUERY_NODE_COLUMN: case QUERY_NODE_COLUMN:
return columnNodeToJson(pObj, pJson);
case QUERY_NODE_VALUE: case QUERY_NODE_VALUE:
return valueNodeToJson(pObj, pJson);
case QUERY_NODE_OPERATOR: case QUERY_NODE_OPERATOR:
return operatorNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_CONDITION: case QUERY_NODE_LOGIC_CONDITION:
return logicConditionNodeToJson(pObj, pJson);
case QUERY_NODE_FUNCTION: case QUERY_NODE_FUNCTION:
return functionNodeToJson(pObj, pJson);
case QUERY_NODE_REAL_TABLE: case QUERY_NODE_REAL_TABLE:
case QUERY_NODE_TEMP_TABLE: case QUERY_NODE_TEMP_TABLE:
case QUERY_NODE_JOIN_TABLE: case QUERY_NODE_JOIN_TABLE:
break;
case QUERY_NODE_GROUPING_SET: case QUERY_NODE_GROUPING_SET:
return groupingSetNodeToJson(pObj, pJson);
case QUERY_NODE_ORDER_BY_EXPR: case QUERY_NODE_ORDER_BY_EXPR:
case QUERY_NODE_LIMIT: case QUERY_NODE_LIMIT:
case QUERY_NODE_STATE_WINDOW: case QUERY_NODE_STATE_WINDOW:
@ -206,19 +500,21 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
case QUERY_NODE_TARGET: case QUERY_NODE_TARGET:
case QUERY_NODE_RAW_EXPR: case QUERY_NODE_RAW_EXPR:
case QUERY_NODE_SET_OPERATOR: case QUERY_NODE_SET_OPERATOR:
break;
case QUERY_NODE_SELECT_STMT: case QUERY_NODE_SELECT_STMT:
return selectStmtTojson(pObj, pJson);
case QUERY_NODE_SHOW_STMT: case QUERY_NODE_SHOW_STMT:
break; break;
case QUERY_NODE_LOGIC_PLAN_SCAN: case QUERY_NODE_LOGIC_PLAN_SCAN:
return logicScanToJson(pObj, pJson); return logicScanNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_JOIN: case QUERY_NODE_LOGIC_PLAN_JOIN:
return logicJoinToJson(pObj, pJson); return logicJoinNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_FILTER: case QUERY_NODE_LOGIC_PLAN_FILTER:
return logicFilterToJson(pObj, pJson); return logicFilterNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_AGG: case QUERY_NODE_LOGIC_PLAN_AGG:
return logicAggToJson(pObj, pJson); return logicAggNodeToJson(pObj, pJson);
case QUERY_NODE_LOGIC_PLAN_PROJECT: case QUERY_NODE_LOGIC_PLAN_PROJECT:
return logicProjectToJson(pObj, pJson); return logicProjectNodeToJson(pObj, pJson);
default: default:
break; break;
} }
@ -238,7 +534,7 @@ static int32_t nodeToJson(const void* pObj, SJson* pJson) {
return code; return code;
} }
int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { int32_t nodesNodeToString(const SNode* pNode, bool format, char** pStr, int32_t* pLen) {
if (NULL == pNode || NULL == pStr || NULL == pLen) { if (NULL == pNode || NULL == pStr || NULL == pLen) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -255,7 +551,7 @@ int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) {
return code; return code;
} }
*pStr = tjsonToString(pJson); *pStr = format ? tjsonToString(pJson) : tjsonToUnformattedString(pJson);
tjsonDelete(pJson); tjsonDelete(pJson);
*pLen = strlen(*pStr) + 1; *pLen = strlen(*pStr) + 1;

15
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -135,11 +135,22 @@ int32_t nodesListAppend(SNodeList* pList, SNode* pNode) {
} }
int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) { int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc) {
if (NULL == pTarget || NULL == pSrc) {
return TSDB_CODE_SUCCESS;
}
if (NULL == pTarget->pHead) {
pTarget->pHead = pSrc->pHead;
} else {
pTarget->pTail->pNext = pSrc->pHead; pTarget->pTail->pNext = pSrc->pHead;
if (NULL != pSrc->pHead) {
pSrc->pHead->pPrev = pTarget->pTail; pSrc->pHead->pPrev = pTarget->pTail;
}
}
pTarget->pTail = pSrc->pTail; pTarget->pTail = pSrc->pTail;
pTarget->length += pSrc->length; pTarget->length += pSrc->length;
tfree(pSrc); tfree(pSrc);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -323,6 +334,10 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
nodesDestroyList(cxt.pCols); nodesDestroyList(cxt.pCols);
return cxt.errCode; return cxt.errCode;
} }
if (0 == LIST_LENGTH(cxt.pCols)) {
nodesDestroyList(cxt.pCols);
cxt.pCols = NULL;
}
*pCols = cxt.pCols; *pCols = cxt.pCols;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

8
source/libs/parser/src/parserImpl.c
View File

@ -349,7 +349,7 @@ static SNodeList* getProjectList(SNode* pNode) {
return NULL; return NULL;
} }
static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, SColumnNode* pCol) { static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* pColSchema, bool isTag, SColumnNode* pCol) {
strcpy(pCol->dbName, pTable->table.dbName); strcpy(pCol->dbName, pTable->table.dbName);
strcpy(pCol->tableAlias, pTable->table.tableAlias); strcpy(pCol->tableAlias, pTable->table.tableAlias);
strcpy(pCol->tableName, pTable->table.tableName); strcpy(pCol->tableName, pTable->table.tableName);
@ -359,7 +359,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p
} }
pCol->tableId = pTable->pMeta->uid; pCol->tableId = pTable->pMeta->uid;
pCol->colId = pColSchema->colId; pCol->colId = pColSchema->colId;
// pCol->colType = pColSchema->type; pCol->colType = isTag ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN;
pCol->node.resType.type = pColSchema->type; pCol->node.resType.type = pColSchema->type;
pCol->node.resType.bytes = pColSchema->bytes; pCol->node.resType.bytes = pColSchema->bytes;
} }
@ -383,7 +383,7 @@ static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode
if (NULL == pCol) { if (NULL == pCol) {
return generateSyntaxErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY); return generateSyntaxErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
} }
setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, pCol); setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i < pMeta->tableInfo.numOfTags), pCol);
nodesListAppend(pList, (SNode*)pCol); nodesListAppend(pList, (SNode*)pCol);
} }
} else { } else {
@ -408,7 +408,7 @@ static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) {
int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns;
for (int32_t i = 0; i < nums; ++i) { for (int32_t i = 0; i < nums; ++i) {
if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) { if (0 == strcmp(pCol->colName, pMeta->schema[i].name)) {
setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, pCol); setColumnInfoBySchema((SRealTableNode*)pTable, pMeta->schema + i, (i < pMeta->tableInfo.numOfTags), pCol);
found = true; found = true;
break; break;
} }

4
source/libs/parser/test/mockCatalog.cpp
View File

@ -29,9 +29,9 @@
namespace { namespace {
void generateTestT1(MockCatalogService* mcs) { void generateTestT1(MockCatalogService* mcs) {
ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3) ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 4)
.setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP) .setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
.addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20); .addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20).addColumn("c3", TSDB_DATA_TYPE_BIGINT);
builder.done(); builder.done();
} }

117
source/libs/planner/src/plannerImpl.c
View File

@ -18,20 +18,20 @@
#define CHECK_ALLOC(p, res) \ #define CHECK_ALLOC(p, res) \
do { \ do { \
if (NULL == p) { \ if (NULL == (p)) { \
printf("%s : %d\n", __FUNCTION__, __LINE__); \ printf("%s : %d\n", __FUNCTION__, __LINE__); \
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \ pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; \
return res; \ return (res); \
} \ } \
} while (0) } while (0)
#define CHECK_CODE(exec, res) \ #define CHECK_CODE(exec, res) \
do { \ do { \
int32_t code = exec; \ int32_t code = (exec); \
if (TSDB_CODE_SUCCESS != code) { \ if (TSDB_CODE_SUCCESS != code) { \
printf("%s : %d\n", __FUNCTION__, __LINE__); \ printf("%s : %d\n", __FUNCTION__, __LINE__); \
pCxt->errCode = code; \ pCxt->errCode = code; \
return res; \ return (res); \
} \ } \
} while (0) } while (0)
@ -83,14 +83,30 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
typedef struct SNameExprCxt {
int32_t planNodeId;
int32_t rewriteId;
} SNameExprCxt;
static EDealRes doNameExpr(SNode* pNode, void* pContext) {
switch (nodeType(pNode)) {
case QUERY_NODE_OPERATOR:
case QUERY_NODE_LOGIC_CONDITION:
case QUERY_NODE_FUNCTION: {
SNameExprCxt* pCxt = (SNameExprCxt*)pContext;
sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", pCxt->planNodeId, pCxt->rewriteId++);
return DEAL_RES_IGNORE_CHILD;
}
default:
break;
}
return DEAL_RES_CONTINUE;
}
static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) { static int32_t rewriteExpr(int32_t planNodeId, int32_t rewriteId, SNodeList* pExprs, SSelectStmt* pSelect, ESqlClause clause) {
SNode* pNode; SNameExprCxt nameCxt = { .planNodeId = planNodeId, .rewriteId = rewriteId };
FOREACH(pNode, pExprs) { nodesWalkList(pExprs, doNameExpr, &nameCxt);
if (QUERY_NODE_COLUMN == nodeType(pNode) || QUERY_NODE_VALUE == nodeType(pNode)) {
continue;
}
sprintf(((SExprNode*)pNode)->aliasName, "#expr_%d_%d", planNodeId, rewriteId);
}
SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs }; SRewriteExprCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pExprs = pExprs };
nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt); nodesRewriteSelectStmt(pSelect, clause, doRewriteExpr, &cxt);
return cxt.errCode; return cxt.errCode;
@ -135,12 +151,16 @@ static SLogicNode* createScanLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect,
// set columns to scan // set columns to scan
SNodeList* pCols = NULL; SNodeList* pCols = NULL;
CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan); CHECK_CODE(nodesCollectColumns(pSelect, SQL_CLAUSE_FROM, pRealTable->table.tableAlias, &pCols), (SLogicNode*)pScan);
if (NULL != pCols) {
pScan->pScanCols = nodesCloneList(pCols); pScan->pScanCols = nodesCloneList(pCols);
CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan); CHECK_ALLOC(pScan->pScanCols, (SLogicNode*)pScan);
}
// set output // set output
if (NULL != pCols) {
pScan->node.pTargets = nodesCloneList(pCols); pScan->node.pTargets = nodesCloneList(pCols);
CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan); CHECK_ALLOC(pScan->node.pTargets, (SLogicNode*)pScan);
}
return (SLogicNode*)pScan; return (SLogicNode*)pScan;
} }
@ -173,8 +193,10 @@ static SLogicNode* createJoinLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect,
CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin); CHECK_CODE(nodesListAppend(pJoin->node.pChildren, (SNode*)pRight), (SLogicNode*)pJoin);
// set on conditions // set on conditions
if (NULL != pJoinTable->pOnCond) {
pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond); pJoin->pOnConditions = nodesCloneNode(pJoinTable->pOnCond);
CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin); CHECK_ALLOC(pJoin->pOnConditions, (SLogicNode*)pJoin);
}
// set the output // set the output
pJoin->node.pTargets = nodesCloneList(pLeft->pTargets); pJoin->node.pTargets = nodesCloneList(pLeft->pTargets);
@ -220,36 +242,58 @@ static SLogicNode* createWhereFilterLogicNode(SPlanContext* pCxt, SLogicNode* pC
return (SLogicNode*)pFilter; return (SLogicNode*)pFilter;
} }
static SNodeList* createColumnByRewriteExps(SPlanContext* pCxt, SNodeList* pExprs) { typedef struct SCreateColumnCxt {
SNodeList* pList = nodesMakeList(); int32_t errCode;
CHECK_ALLOC(pList, NULL); SNodeList* pList;
SNode* pNode; } SCreateColumnCxt;
FOREACH(pNode, pExprs) {
if (QUERY_NODE_VALUE == nodeType(pNode)) { static EDealRes doCreateColumn(SNode* pNode, void* pContext) {
continue; SCreateColumnCxt* pCxt = (SCreateColumnCxt*)pContext;
} else if (QUERY_NODE_COLUMN == nodeType(pNode)) { switch (nodeType(pNode)) {
case QUERY_NODE_COLUMN: {
SNode* pCol = nodesCloneNode(pNode); SNode* pCol = nodesCloneNode(pNode);
if (NULL == pCol) { if (NULL == pCol || TSDB_CODE_SUCCESS != nodesListAppend(pCxt->pList, pCol)) {
goto error; pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
} }
if (TSDB_CODE_SUCCESS != nodesListAppend(pList, pCol)) { return DEAL_RES_IGNORE_CHILD;
goto error;
} }
} else { case QUERY_NODE_OPERATOR:
case QUERY_NODE_LOGIC_CONDITION:
case QUERY_NODE_FUNCTION: {
SExprNode* pExpr = (SExprNode*)pNode; SExprNode* pExpr = (SExprNode*)pNode;
SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == pCol) { if (NULL == pCol) {
goto error; pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
} }
pCol->node.resType = pExpr->resType; pCol->node.resType = pExpr->resType;
strcpy(pCol->colName, pExpr->aliasName); strcpy(pCol->colName, pExpr->aliasName);
if (TSDB_CODE_SUCCESS != nodesListAppend(pCxt->pList, (SNode*)pCol)) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR;
} }
return DEAL_RES_IGNORE_CHILD;
} }
return pList; default:
error: break;
nodesDestroyList(pList); }
return DEAL_RES_CONTINUE;
}
static SNodeList* createColumnByRewriteExps(SPlanContext* pCxt, SNodeList* pExprs) {
SCreateColumnCxt cxt = { .errCode = TSDB_CODE_SUCCESS, .pList = nodesMakeList() };
if (NULL == cxt.pList) {
return NULL; return NULL;
} }
nodesWalkList(pExprs, doCreateColumn, &cxt);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pList);
return NULL;
}
return cxt.pList;
}
static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) { static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect) {
SNodeList* pAggFuncs = NULL; SNodeList* pAggFuncs = NULL;
@ -263,24 +307,37 @@ static SLogicNode* createAggLogicNode(SPlanContext* pCxt, SSelectStmt* pSelect)
pAgg->node.id = pCxt->planNodeId++; pAgg->node.id = pCxt->planNodeId++;
// set grouyp keys, agg funcs and having conditions // set grouyp keys, agg funcs and having conditions
if (NULL != pSelect->pGroupByList) {
pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList);
CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg); CHECK_ALLOC(pAgg->pGroupKeys, (SLogicNode*)pAgg);
}
if (NULL != pAggFuncs) {
pAgg->pAggFuncs = nodesCloneList(pAggFuncs); pAgg->pAggFuncs = nodesCloneList(pAggFuncs);
CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg); CHECK_ALLOC(pAgg->pAggFuncs, (SLogicNode*)pAgg);
}
// rewrite the expression in subsequent clauses // rewrite the expression in subsequent clauses
CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); CHECK_CODE(rewriteExpr(pAgg->node.id, 1, pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg);
CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg); CHECK_CODE(rewriteExpr(pAgg->node.id, 1 + LIST_LENGTH(pAgg->pGroupKeys), pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY), (SLogicNode*)pAgg);
if (NULL != pSelect->pHaving) {
pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving);
CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg); CHECK_ALLOC(pAgg->node.pConditions, (SLogicNode*)pAgg);
}
// set the output // set the output
pAgg->node.pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys); pAgg->node.pTargets = nodesMakeList();
CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg); CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg);
if (NULL != pAgg->pGroupKeys) {
SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pGroupKeys);
CHECK_ALLOC(pAgg->node.pTargets, (SLogicNode*)pAgg);
nodesListAppendList(pAgg->node.pTargets, pTargets);
}
if (NULL != pAgg->pAggFuncs) {
SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs); SNodeList* pTargets = createColumnByRewriteExps(pCxt, pAgg->pAggFuncs);
CHECK_ALLOC(pTargets, (SLogicNode*)pAgg); CHECK_ALLOC(pTargets, (SLogicNode*)pAgg);
nodesListAppendList(pAgg->node.pTargets, pTargets); nodesListAppendList(pAgg->node.pTargets, pTargets);
}
return (SLogicNode*)pAgg; return (SLogicNode*)pAgg;
} }
@ -344,7 +401,7 @@ static SLogicNode* createQueryLogicNode(SPlanContext* pCxt, SNode* pStmt) {
} }
int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode) { int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode) {
SPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 0 }; SPlanContext cxt = { .errCode = TSDB_CODE_SUCCESS, .planNodeId = 1 };
SLogicNode* pRoot = createQueryLogicNode(&cxt, pNode); SLogicNode* pRoot = createQueryLogicNode(&cxt, pNode);
if (TSDB_CODE_SUCCESS != cxt.errCode) { if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyNode((SNode*)pRoot); nodesDestroyNode((SNode*)pRoot);

55
source/libs/planner/test/newPlannerTest.cpp
View File

@ -42,26 +42,28 @@ protected:
bool run() { bool run() {
int32_t code = parser(&cxt_, &query_); int32_t code = parser(&cxt_, &query_);
// cout << "parser return " << code << endl;
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] parser code:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl; cout << "sql:[" << cxt_.pSql << "] parser code:" << code << ", strerror:" << tstrerror(code) << ", msg:" << errMagBuf_ << endl;
return false; return false;
} }
const string syntaxTreeStr = toString(query_.pRoot, false);
SLogicNode* pLogicPlan = nullptr; SLogicNode* pLogicPlan = nullptr;
code = createLogicPlan(query_.pRoot, &pLogicPlan); code = createLogicPlan(query_.pRoot, &pLogicPlan);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] plan code:" << tstrerror(code) << endl; cout << "sql:[" << cxt_.pSql << "] plan code:" << code << ", strerror:" << tstrerror(code) << endl;
return false; return false;
} }
char* pStr = NULL;
int32_t len = 0; cout << "sql : [" << cxt_.pSql << "]" << endl;
code = nodesNodeToString((const SNode*)pLogicPlan, &pStr, &len); cout << "syntax test : " << endl;
if (code != TSDB_CODE_SUCCESS) { cout << syntaxTreeStr << endl;
cout << "sql:[" << cxt_.pSql << "] toString code:" << tstrerror(code) << endl; // cout << "logic plan : " << endl;
return false; // cout << toString((const SNode*)pLogicPlan) << endl;
} cout << "unformatted logic plan : " << endl;
cout << "logic plan : " << endl; cout << toString((const SNode*)pLogicPlan, false) << endl;
cout << pStr << endl;
return true; return true;
} }
@ -75,6 +77,19 @@ private:
cxt_.msgLen = max_err_len; cxt_.msgLen = max_err_len;
} }
string toString(const SNode* pRoot, bool format = true) {
char* pStr = NULL;
int32_t len = 0;
int32_t code = nodesNodeToString(pRoot, format, &pStr, &len);
if (code != TSDB_CODE_SUCCESS) {
cout << "sql:[" << cxt_.pSql << "] toString code:" << code << ", strerror:" << tstrerror(code) << endl;
return string();
}
string str(pStr);
tfree(pStr);
return str;
}
string acctId_; string acctId_;
string db_; string db_;
char errMagBuf_[max_err_len]; char errMagBuf_[max_err_len];
@ -89,3 +104,19 @@ TEST_F(NewPlannerTest, simple) {
bind("SELECT * FROM t1"); bind("SELECT * FROM t1");
ASSERT_TRUE(run()); ASSERT_TRUE(run());
} }
TEST_F(NewPlannerTest, groupBy) {
setDatabase("root", "test");
bind("SELECT count(*) FROM t1");
ASSERT_TRUE(run());
bind("SELECT c1, count(*) FROM t1 GROUP BY c1");
ASSERT_TRUE(run());
bind("SELECT c1 + c3, c1 + count(*) FROM t1 where c2 = 'abc' GROUP BY c1, c3");
ASSERT_TRUE(run());
bind("SELECT c1 + c3, count(*) FROM t1 where concat(c2, 'wwww') = 'abcwww' GROUP BY c1 + c3");
ASSERT_TRUE(run());
}

1
source/libs/sync/CMakeLists.txt
View File

@ -1,4 +1,3 @@
aux_source_directory(src SYNC_SRC) aux_source_directory(src SYNC_SRC)
add_library(sync ${SYNC_SRC}) add_library(sync ${SYNC_SRC})

38
source/libs/sync/inc/syncAppendEntries.h Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_APPEND_ENTRIES_H
#define _TD_LIBS_SYNC_APPEND_ENTRIES_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg);
void onAppendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_APPEND_ENTRIES_H*/

36
source/libs/sync/inc/syncAppendEntriesReply.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H
#define _TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_APPEND_ENTRIES_REPLY_H*/

32
source/libs/sync/inc/syncElection.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_ELECTION_H
#define _TD_LIBS_SYNC_ELECTION_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "taosdef.h"
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_ELECTION_H*/

79
source/libs/sync/inc/syncInt.h Normal file
View File

@ -0,0 +1,79 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_INT_H
#define _TD_LIBS_SYNC_INT_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "taosdef.h"
#define sFatal(...) \
{ \
if (sDebugFlag & DEBUG_FATAL) { \
taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); \
} \
}
#define sError(...) \
{ \
if (sDebugFlag & DEBUG_ERROR) { \
taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); \
} \
}
#define sWarn(...) \
{ \
if (sDebugFlag & DEBUG_WARN) { \
taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); \
} \
}
#define sInfo(...) \
{ \
if (sDebugFlag & DEBUG_INFO) { \
taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \
} \
}
#define sDebug(...) \
{ \
if (sDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \
} \
}
#define sTrace(...) \
{ \
if (sDebugFlag & DEBUG_TRACE) { \
taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); \
} \
}
typedef struct SSyncNode {
char path[TSDB_FILENAME_LEN];
int8_t replica;
int8_t quorum;
int8_t selfIndex;
uint32_t vgId;
int32_t refCount;
int64_t rid;
} SSyncNode;
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_INT_H*/

105
source/libs/sync/inc/syncMessage.h Normal file
View File

@ -0,0 +1,105 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_MESSAGE_H
#define _TD_LIBS_SYNC_MESSAGE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "sync.h"
#include "syncRaftEntry.h"
#include "taosdef.h"
typedef enum ESyncMessageType {
SYNC_PING = 0,
SYNC_PING_REPLY,
SYNC_CLIENT_REQUEST,
SYNC_CLIENT_REQUEST_REPLY,
SYNC_REQUEST_VOTE,
SYNC_REQUEST_VOTE_REPLY,
SYNC_APPEND_ENTRIES,
SYNC_APPEND_ENTRIES_REPLY,
} ESyncMessageType;
typedef struct SyncPing {
ESyncMessageType msgType;
const SSyncBuffer *pData;
} SyncPing;
typedef struct SyncPingReply {
ESyncMessageType msgType;
const SSyncBuffer *pData;
} SyncPingReply;
typedef struct SyncClientRequest {
ESyncMessageType msgType;
const SSyncBuffer *pData;
int64_t seqNum;
bool isWeak;
} SyncClientRequest;
typedef struct SyncClientRequestReply {
ESyncMessageType msgType;
int32_t errCode;
const SSyncBuffer *pErrMsg;
const SSyncBuffer *pLeaderHint;
} SyncClientRequestReply;
typedef struct SyncRequestVote {
ESyncMessageType msgType;
SyncTerm currentTerm;
SyncNodeId nodeId;
SyncGroupId vgId;
SyncIndex lastLogIndex;
SyncTerm lastLogTerm;
} SyncRequestVote;
typedef struct SyncRequestVoteReply {
ESyncMessageType msgType;
SyncTerm currentTerm;
SyncNodeId nodeId;
SyncGroupId vgId;
bool voteGranted;
} SyncRequestVoteReply;
typedef struct SyncAppendEntries {
ESyncMessageType msgType;
SyncTerm currentTerm;
SyncNodeId nodeId;
SyncIndex prevLogIndex;
SyncTerm prevLogTerm;
int32_t entryCount;
SSyncRaftEntry * logEntries;
SyncIndex commitIndex;
} SyncAppendEntries;
typedef struct SyncAppendEntriesReply {
ESyncMessageType msgType;
SyncTerm currentTerm;
SyncNodeId nodeId;
bool success;
SyncIndex matchIndex;
} SyncAppendEntriesReply;
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_MESSAGE_H*/

35
source/libs/sync/inc/syncOnMessage.h Normal file
View File

@ -0,0 +1,35 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_ON_MESSAGE_H
#define _TD_LIBS_SYNC_ON_MESSAGE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncRaft.h"
#include "taosdef.h"
void onMessage(SRaft *pRaft, void *pMsg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_ON_MESSAGE_H*/

47
source/libs/sync/inc/syncRaft.h Normal file
View File

@ -0,0 +1,47 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_RAFT_H
#define _TD_LIBS_SYNC_RAFT_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "sync.h"
#include "syncMessage.h"
#include "taosdef.h"
typedef struct SRaftId {
SyncNodeId nodeId;
SyncGroupId vgId;
} SRaftId;
typedef struct SRaft {
SRaftId id;
} SRaft;
int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak);
static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_RAFT_H*/

40
source/libs/sync/inc/syncRaftEntry.h Normal file
View File

@ -0,0 +1,40 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_TPL_H
#define _TD_LIBS_TPL_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "sync.h"
#include "taosdef.h"
typedef struct SSyncRaftEntry {
SyncTerm term;
SyncIndex index;
SSyncBuffer data;
int8_t flag;
} SSyncRaftEntry;
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_TPL_H*/

53
source/libs/sync/inc/syncRaftLog.h Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_RAFT_LOG_H
#define _TD_LIBS_SYNC_RAFT_LOG_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "sync.h"
#include "taosdef.h"
int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf);
// get one log entry, user need to free pBuf->data
int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf);
// update log store commit index with "index"
int32_t raftLogUpdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index);
// truncate log with index, entries after the given index (>index) will be deleted
int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index);
// return commit index of log
SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore);
// return index of last entry
SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore);
// return term of last entry
SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_RAFT_LOG_H*/

42
source/libs/sync/inc/syncRaftStore.h Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_RAFT_STORE_H
#define _TD_LIBS_SYNC_RAFT_STORE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "sync.h"
#include "syncRaft.h"
#include "taosdef.h"
int32_t currentTerm(SyncTerm *pCurrentTerm);
int32_t persistCurrentTerm(SyncTerm currentTerm);
int32_t voteFor(SRaftId *pRaftId);
int32_t persistVoteFor(SRaftId *pRaftId);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_RAFT_STORE_H*/

32
source/libs/sync/inc/syncReplication.h Normal file
View File

@ -0,0 +1,32 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_REPLICATION_H
#define _TD_LIBS_SYNC_REPLICATION_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "taosdef.h"
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_REPLICATION_H*/

38
source/libs/sync/inc/syncRequestVote.h Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_REQUEST_VOTE_H
#define _TD_LIBS_SYNC_REQUEST_VOTE_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg);
void onRequestVote(SRaft *pRaft, const SyncRequestVote *pMsg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_REQUEST_VOTE_H*/

36
source/libs/sync/inc/syncRequestVoteReply.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H
#define _TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_REQUEST_VOTE_REPLY_H*/

38
source/libs/sync/inc/syncSnapshot.h Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_SNAPSHOT_H
#define _TD_LIBS_SYNC_SNAPSHOT_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "sync.h"
#include "syncRaft.h"
#include "taosdef.h"
int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_SNAPSHOT_H*/

36
source/libs/sync/inc/syncTimeout.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_LIBS_SYNC_TIMEOUT_H
#define _TD_LIBS_SYNC_TIMEOUT_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include "syncMessage.h"
#include "syncRaft.h"
#include "taosdef.h"
void onTimeout(SRaft *pRaft, void *pMsg);
#ifdef __cplusplus
}
#endif
#endif /*_TD_LIBS_SYNC_TIMEOUT_H*/

1
source/libs/sync/src/sync.c
View File

@ -1 +0,0 @@
#include "sync.h"

117
source/libs/sync/src/syncAppendEntries.c Normal file
View File

@ -0,0 +1,117 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncAppendEntries.h"
#include "sync.h"
void appendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) {
// TLA+ Spec
//AppendEntries(i, j) ==
// /\ i /= j
// /\ state[i] = Leader
// /\ LET prevLogIndex == nextIndex[i][j] - 1
// prevLogTerm == IF prevLogIndex > 0 THEN
// log[i][prevLogIndex].term
// ELSE
// 0
// \* Send up to 1 entry, constrained by the end of the log.
// lastEntry == Min({Len(log[i]), nextIndex[i][j]})
// entries == SubSeq(log[i], nextIndex[i][j], lastEntry)
// IN Send([mtype |-> AppendEntriesRequest,
// mterm |-> currentTerm[i],
// mprevLogIndex |-> prevLogIndex,
// mprevLogTerm |-> prevLogTerm,
// mentries |-> entries,
// \* mlog is used as a history variable for the proof.
// \* It would not exist in a real implementation.
// mlog |-> log[i],
// mcommitIndex |-> Min({commitIndex[i], lastEntry}),
// msource |-> i,
// mdest |-> j])
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
}
void onAppendEntries(SRaft *pRaft, const SyncAppendEntries *pMsg) {
// TLA+ Spec
//HandleAppendEntriesRequest(i, j, m) ==
// LET logOk == \/ m.mprevLogIndex = 0
// \/ /\ m.mprevLogIndex > 0
// /\ m.mprevLogIndex <= Len(log[i])
// /\ m.mprevLogTerm = log[i][m.mprevLogIndex].term
// IN /\ m.mterm <= currentTerm[i]
// /\ \/ /\ \* reject request
// \/ m.mterm < currentTerm[i]
// \/ /\ m.mterm = currentTerm[i]
// /\ state[i] = Follower
// /\ \lnot logOk
// /\ Reply([mtype |-> AppendEntriesResponse,
// mterm |-> currentTerm[i],
// msuccess |-> FALSE,
// mmatchIndex |-> 0,
// msource |-> i,
// mdest |-> j],
// m)
// /\ UNCHANGED <<serverVars, logVars>>
// \/ \* return to follower state
// /\ m.mterm = currentTerm[i]
// /\ state[i] = Candidate
// /\ state' = [state EXCEPT ![i] = Follower]
// /\ UNCHANGED <<currentTerm, votedFor, logVars, messages>>
// \/ \* accept request
// /\ m.mterm = currentTerm[i]
// /\ state[i] = Follower
// /\ logOk
// /\ LET index == m.mprevLogIndex + 1
// IN \/ \* already done with request
// /\ \/ m.mentries = << >>
// \/ /\ m.mentries /= << >>
// /\ Len(log[i]) >= index
// /\ log[i][index].term = m.mentries[1].term
// \* This could make our commitIndex decrease (for
// \* example if we process an old, duplicated request),
// \* but that doesn't really affect anything.
// /\ commitIndex' = [commitIndex EXCEPT ![i] =
// m.mcommitIndex]
// /\ Reply([mtype |-> AppendEntriesResponse,
// mterm |-> currentTerm[i],
// msuccess |-> TRUE,
// mmatchIndex |-> m.mprevLogIndex +
// Len(m.mentries),
// msource |-> i,
// mdest |-> j],
// m)
// /\ UNCHANGED <<serverVars, log>>
// \/ \* conflict: remove 1 entry
// /\ m.mentries /= << >>
// /\ Len(log[i]) >= index
// /\ log[i][index].term /= m.mentries[1].term
// /\ LET new == [index2 \in 1..(Len(log[i]) - 1) |->
// log[i][index2]]
// IN log' = [log EXCEPT ![i] = new]
// /\ UNCHANGED <<serverVars, commitIndex, messages>>
// \/ \* no conflict: append entry
// /\ m.mentries /= << >>
// /\ Len(log[i]) = m.mprevLogIndex
// /\ log' = [log EXCEPT ![i] =
// Append(log[i], m.mentries[1])]
// /\ UNCHANGED <<serverVars, commitIndex, messages>>
// /\ UNCHANGED <<candidateVars, leaderVars>>
//
}

34
source/libs/sync/src/syncAppendEntriesReply.c Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncAppendEntriesReply.h"
#include "sync.h"
void onAppendEntriesReply(SRaft *pRaft, const SyncAppendEntriesReply *pMsg) {
// TLA+ Spec
//HandleAppendEntriesResponse(i, j, m) ==
// /\ m.mterm = currentTerm[i]
// /\ \/ /\ m.msuccess \* successful
// /\ nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1]
// /\ matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex]
// \/ /\ \lnot m.msuccess \* not successful
// /\ nextIndex' = [nextIndex EXCEPT ![i][j] =
// Max({nextIndex[i][j] - 1, 1})]
// /\ UNCHANGED <<matchIndex>>
// /\ Discard(m)
// /\ UNCHANGED <<serverVars, candidateVars, logVars, elections>>
}

16
source/libs/sync/src/syncElection.c Normal file
View File

@ -0,0 +1,16 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sync.h"

34
source/libs/sync/src/syncMain.c Normal file
View File

@ -0,0 +1,34 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <stdint.h>
#include "sync.h"
#include "syncInt.h"
int32_t syncInit() { return 0; }
void syncCleanUp() {}
int64_t syncStart(const SSyncInfo* pSyncInfo) { return 0; }
void syncStop(int64_t rid) {}
int32_t syncReconfig(int64_t rid, const SSyncCfg* pSyncCfg) { return 0; }
int32_t syncForwardToPeer(int64_t rid, const SSyncBuffer* pBuf, bool isWeak) { return 0; }
ESyncState syncGetMyRole(int64_t rid) { return TAOS_SYNC_STATE_LEADER; }
void syncGetNodesRole(int64_t rid, SNodesRole* pNodeRole) {}

20
source/libs/sync/src/syncMessage.c Normal file
View File

@ -0,0 +1,20 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncMessage.h"
#include "sync.h"
#include "syncRaft.h"
void onMessage(SRaft *pRaft, void *pMsg) {}

16
source/libs/sync/src/syncOnMessage.c Normal file
View File

@ -0,0 +1,16 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sync.h"

21
source/libs/sync/src/syncRaft.c Normal file
View File

@ -0,0 +1,21 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncRaft.h"
#include "sync.h"
int32_t raftPropose(SRaft* pRaft, const SSyncBuffer* pBuf, bool isWeak) { return 0; }
static int raftSendMsg(SRaftId destRaftId, const void* pMsg, const SRaft* pRaft) { return 0; }

16
source/libs/sync/src/syncRaftEntry.c Normal file
View File

@ -0,0 +1,16 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sync.h"

37
source/libs/sync/src/syncRaftLog.c Normal file
View File

@ -0,0 +1,37 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncRaftLog.h"
#include "sync.h"
int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncBuffer* pBuf) { return 0; }
// get one log entry, user need to free pBuf->data
int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncBuffer* pBuf) { return 0; }
// update log store commit index with "index"
int32_t raftLogupdateCommitIndex(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; }
// truncate log with index, entries after the given index (>index) will be deleted
int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex index) { return 0; }
// return commit index of log
SyncIndex raftLogGetCommitIndex(struct SSyncLogStore* pLogStore) { return 0; }
// return index of last entry
SyncIndex raftLogGetLastIndex(struct SSyncLogStore* pLogStore) { return 0; }
// return term of last entry
SyncTerm raftLogGetLastTerm(struct SSyncLogStore* pLogStore) { return 0; }

25
source/libs/sync/src/syncRaftStore.c Normal file
View File

@ -0,0 +1,25 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncRaftStore.h"
#include "sync.h"
int32_t currentTerm(SyncTerm *pCurrentTerm) { return 0; }
int32_t persistCurrentTerm(SyncTerm currentTerm) { return 0; }
int32_t voteFor(SRaftId *pRaftId) { return 0; }
int32_t persistVoteFor(SRaftId *pRaftId) { return 0; }

16
source/libs/sync/src/syncReplication.c Normal file
View File

@ -0,0 +1,16 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "sync.h"

59
source/libs/sync/src/syncRequestVote.c Normal file
View File

@ -0,0 +1,59 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncRequestVote.h"
#include "sync.h"
void requestVote(SRaft *pRaft, const SyncRequestVote *pMsg) {
// TLA+ Spec
//RequestVote(i, j) ==
// /\ state[i] = Candidate
// /\ j \notin votesResponded[i]
// /\ Send([mtype |-> RequestVoteRequest,
// mterm |-> currentTerm[i],
// mlastLogTerm |-> LastTerm(log[i]),
// mlastLogIndex |-> Len(log[i]),
// msource |-> i,
// mdest |-> j])
// /\ UNCHANGED <<serverVars, candidateVars, leaderVars, logVars>>
}
void onRequestVote(SRaft *pRaft, const SyncRequestVote *pMsg) {
// TLA+ Spec
//HandleRequestVoteRequest(i, j, m) ==
// LET logOk == \/ m.mlastLogTerm > LastTerm(log[i])
// \/ /\ m.mlastLogTerm = LastTerm(log[i])
// /\ m.mlastLogIndex >= Len(log[i])
// grant == /\ m.mterm = currentTerm[i]
// /\ logOk
// /\ votedFor[i] \in {Nil, j}
// IN /\ m.mterm <= currentTerm[i]
// /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j]
// \/ ~grant /\ UNCHANGED votedFor
// /\ Reply([mtype |-> RequestVoteResponse,
// mterm |-> currentTerm[i],
// mvoteGranted |-> grant,
// \* mlog is used just for the `elections' history variable for
// \* the proof. It would not exist in a real implementation.
// mlog |-> log[i],
// msource |-> i,
// mdest |-> j],
// m)
// /\ UNCHANGED <<state, currentTerm, candidateVars, leaderVars, logVars>>
}

38
source/libs/sync/src/syncRequestVoteReply.c Normal file
View File

@ -0,0 +1,38 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncRequestVoteReply.h"
#include "sync.h"
void onRequestVoteReply(SRaft *pRaft, const SyncRequestVoteReply *pMsg) {
// TLA+ Spec
//HandleRequestVoteResponse(i, j, m) ==
// \* This tallies votes even when the current state is not Candidate, but
// \* they won't be looked at, so it doesn't matter.
// /\ m.mterm = currentTerm[i]
// /\ votesResponded' = [votesResponded EXCEPT ![i] =
// votesResponded[i] \cup {j}]
// /\ \/ /\ m.mvoteGranted
// /\ votesGranted' = [votesGranted EXCEPT ![i] =
// votesGranted[i] \cup {j}]
// /\ voterLog' = [voterLog EXCEPT ![i] =
// voterLog[i] @@ (j :> m.mlog)]
// \/ /\ ~m.mvoteGranted
// /\ UNCHANGED <<votesGranted, voterLog>>
// /\ Discard(m)
// /\ UNCHANGED <<serverVars, votedFor, leaderVars, logVars>>
}

22
source/libs/sync/src/syncSnapshot.c Normal file
View File

@ -0,0 +1,22 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncSnapshot.h"
#include "sync.h"
#include "syncRaft.h"
int32_t takeSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; }
int32_t restoreSnapshot(SSyncFSM *pFsm, SSnapshot *pSnapshot) { return 0; }

19
source/libs/sync/src/syncTimeout.c Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "syncTimeout.h"
#include "sync.h"
void onTimeout(SRaft *pRaft, void *pMsg) {}

7
source/libs/sync/test/syncTest.cpp Normal file
View File

@ -0,0 +1,7 @@
#include <stdio.h>
int main() {
printf("test \n");
return 0;
}

3
source/libs/transport/inc/transComm.h
View File

@ -217,7 +217,7 @@ typedef struct SConnBuffer {
char* buf; char* buf;
int len; int len;
int cap; int cap;
int left; int total;
} SConnBuffer; } SConnBuffer;
typedef void (*AsyncCB)(uv_async_t* handle); typedef void (*AsyncCB)(uv_async_t* handle);
@ -242,6 +242,7 @@ int transInitBuffer(SConnBuffer* buf);
int transClearBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf);
int transDestroyBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf);
int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf);
bool transReadComplete(SConnBuffer* connBuf);
// int transPackMsg(SRpcMsg *rpcMsg, bool sercured, bool auth, char **msg, int32_t *msgLen); // int transPackMsg(SRpcMsg *rpcMsg, bool sercured, bool auth, char **msg, int32_t *msgLen);

1
source/libs/transport/inc/transportInt.h
View File

@ -56,6 +56,7 @@ typedef struct {
int8_t connType; int8_t connType;
int64_t index; int64_t index;
char label[TSDB_LABEL_LEN]; char label[TSDB_LABEL_LEN];
bool noPool; // pool or not
char user[TSDB_UNI_LEN]; // meter ID char user[TSDB_UNI_LEN]; // meter ID
char spi; // security parameter index char spi; // security parameter index

1
source/libs/transport/src/rpcMain.c
View File

@ -64,6 +64,7 @@ typedef struct {
void (*cfp)(void *parent, SRpcMsg *, SEpSet *); void (*cfp)(void *parent, SRpcMsg *, SEpSet *);
int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey);
bool noPool;
int32_t refCount; int32_t refCount;
void * parent; void * parent;
void * idPool; // handle to ID pool void * idPool; // handle to ID pool

4
source/libs/transport/src/trans.c
View File

@ -27,7 +27,7 @@ void* rpcOpen(const SRpcInit* pInit) {
return NULL; return NULL;
} }
if (pInit->label) { if (pInit->label) {
tstrncpy(pRpc->label, pInit->label, strlen(pInit->label)); tstrncpy(pRpc->label, pInit->label, strlen(pInit->label) + 1);
} }
pRpc->cfp = pInit->cfp; pRpc->cfp = pInit->cfp;
if (pInit->connType == TAOS_CONN_SERVER) { if (pInit->connType == TAOS_CONN_SERVER) {
@ -35,6 +35,8 @@ void* rpcOpen(const SRpcInit* pInit) {
} else { } else {
pRpc->numOfThreads = pInit->numOfThreads; pRpc->numOfThreads = pInit->numOfThreads;
} }
pRpc->noPool = pInit->noPool;
pRpc->connType = pInit->connType; pRpc->connType = pInit->connType;
pRpc->idleTime = pInit->idleTime; pRpc->idleTime = pInit->idleTime;
pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc);

63
source/libs/transport/src/transCli.c
View File

@ -84,8 +84,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* co
// register timer in each thread to clear expire conn // register timer in each thread to clear expire conn
static void clientTimeoutCb(uv_timer_t* handle); static void clientTimeoutCb(uv_timer_t* handle);
// check whether already read complete packet from server
static bool clientReadComplete(SConnBuffer* pBuf);
// alloc buf for read // alloc buf for read
static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
// callback after read nbytes from socket // callback after read nbytes from socket
@ -126,6 +124,9 @@ static void clientHandleResp(SCliConn* conn) {
pHead->code = htonl(pHead->code); pHead->code = htonl(pHead->code);
pHead->msgLen = htonl(pHead->msgLen); pHead->msgLen = htonl(pHead->msgLen);
// buf's mem alread translated to rpcMsg.pCont
transClearBuffer(&conn->readBuf);
SRpcMsg rpcMsg; SRpcMsg rpcMsg;
rpcMsg.contLen = transContLenFromMsg(pHead->msgLen); rpcMsg.contLen = transContLenFromMsg(pHead->msgLen);
rpcMsg.pCont = transContFromHead((char*)pHead); rpcMsg.pCont = transContFromHead((char*)pHead);
@ -134,15 +135,15 @@ static void clientHandleResp(SCliConn* conn) {
rpcMsg.ahandle = pCtx->ahandle; rpcMsg.ahandle = pCtx->ahandle;
if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP || if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP ||
rpcMsg.msgType == TDMT_VND_RES_READY) { rpcMsg.msgType == TDMT_VND_RES_READY_RSP) {
rpcMsg.handle = conn; rpcMsg.handle = conn;
conn->persist = 1; conn->persist = 1;
tDebug("client conn %p persist by app", conn); tDebug("client conn %p persist by app", conn);
} }
tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType), tDebug("%s client conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pRpc->label, conn,
inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr), TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port),
ntohs(conn->locaddr.sin_port)); inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port), rpcMsg.contLen);
conn->secured = pHead->secured; conn->secured = pHead->secured;
if (conn->push != NULL && conn->ctnRdCnt != 0) { if (conn->push != NULL && conn->ctnRdCnt != 0) {
@ -150,27 +151,27 @@ static void clientHandleResp(SCliConn* conn) {
conn->push = NULL; conn->push = NULL;
} else { } else {
if (pCtx->pSem == NULL) { if (pCtx->pSem == NULL) {
tTrace("client conn %p handle resp", conn); tTrace("%s client conn %p handle resp", pRpc->label, conn);
(pRpc->cfp)(pRpc->parent, &rpcMsg, NULL); (pRpc->cfp)(pRpc->parent, &rpcMsg, NULL);
} else { } else {
tTrace("client conn(sync) %p handle resp", conn); tTrace("%s client conn(sync) %p handle resp", pRpc->label, conn);
memcpy((char*)pCtx->pRsp, (char*)&rpcMsg, sizeof(rpcMsg)); memcpy((char*)pCtx->pRsp, (char*)&rpcMsg, sizeof(rpcMsg));
tsem_post(pCtx->pSem); tsem_post(pCtx->pSem);
} }
} }
conn->ctnRdCnt += 1; conn->ctnRdCnt += 1;
// buf's mem alread translated to rpcMsg.pCont
transClearBuffer(&conn->readBuf);
uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb); uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb);
SCliThrdObj* pThrd = conn->hostThrd; SCliThrdObj* pThrd = conn->hostThrd;
// user owns conn->persist = 1 // user owns conn->persist = 1
if (conn->push == NULL && conn->persist == 0) { if (conn->push == NULL && conn->persist == 0) {
if (pRpc->noPool == true) {
} else {
addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn);
} }
}
destroyCmsg(conn->data); destroyCmsg(conn->data);
conn->data = NULL; conn->data = NULL;
// start thread's timer of conn pool if not active // start thread's timer of conn pool if not active
@ -184,7 +185,6 @@ static void clientHandleExcept(SCliConn* pConn) {
clientConnDestroy(pConn, true); clientConnDestroy(pConn, true);
return; return;
} }
tTrace("client conn %p start to destroy", pConn);
SCliMsg* pMsg = pConn->data; SCliMsg* pMsg = pConn->data;
tmsg_t msgType = TDMT_MND_CONNECT; tmsg_t msgType = TDMT_MND_CONNECT;
@ -213,6 +213,7 @@ static void clientHandleExcept(SCliConn* pConn) {
} }
pConn->push = NULL; pConn->push = NULL;
} }
tTrace("%s client conn %p start to destroy", pCtx->pTransInst->label, pConn);
if (pConn->push == NULL) { if (pConn->push == NULL) {
destroyCmsg(pConn->data); destroyCmsg(pConn->data);
pConn->data = NULL; pConn->data = NULL;
@ -226,7 +227,7 @@ static void clientTimeoutCb(uv_timer_t* handle) {
SCliThrdObj* pThrd = handle->data; SCliThrdObj* pThrd = handle->data;
SRpcInfo* pRpc = pThrd->pTransInst; SRpcInfo* pRpc = pThrd->pTransInst;
int64_t currentTime = pThrd->nextTimeout; int64_t currentTime = pThrd->nextTimeout;
tTrace("client conn timeout, try to remove expire conn from conn pool"); tTrace("%s, client conn timeout, try to remove expire conn from conn pool", pRpc->label);
SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL);
while (p != NULL) { while (p != NULL) {
@ -306,23 +307,6 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) {
assert(plist != NULL); assert(plist != NULL);
QUEUE_PUSH(&plist->conn, &conn->conn); QUEUE_PUSH(&plist->conn, &conn->conn);
} }
static bool clientReadComplete(SConnBuffer* data) {
STransMsgHead head;
int32_t headLen = sizeof(head);
if (data->len >= headLen) {
memcpy((char*)&head, data->buf, headLen);
int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen);
if (msgLen > data->len) {
data->left = msgLen - data->len;
return false;
} else if (msgLen == data->len) {
data->left = 0;
return true;
}
} else {
return false;
}
}
static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { static void clientAllocBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SCliConn* conn = handle->data; SCliConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf; SConnBuffer* pBuf = &conn->readBuf;
@ -337,8 +321,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
SConnBuffer* pBuf = &conn->readBuf; SConnBuffer* pBuf = &conn->readBuf;
if (nread > 0) { if (nread > 0) {
pBuf->len += nread; pBuf->len += nread;
if (clientReadComplete(pBuf)) { if (transReadComplete(pBuf)) {
uv_read_stop((uv_stream_t*)conn->stream);
tTrace("client conn %p read complete", conn); tTrace("client conn %p read complete", conn);
clientHandleResp(conn); clientHandleResp(conn);
} else { } else {
@ -346,6 +329,10 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
} }
return; return;
} }
if (nread == UV_EOF) {
tError("client conn %p read error: %s", conn, uv_err_name(nread));
clientHandleExcept(conn);
}
assert(nread <= 0); assert(nread <= 0);
if (nread == 0) { if (nread == 0) {
// ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb // ref http://docs.libuv.org/en/v1.x/stream.html?highlight=uv_read_start#c.uv_read_cb
@ -353,7 +340,7 @@ static void clientReadCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf
// read(2). // read(2).
return; return;
} }
if (nread < 0 || nread == UV_EOF) { if (nread < 0) {
tError("client conn %p read error: %s", conn, uv_err_name(nread)); tError("client conn %p read error: %s", conn, uv_err_name(nread));
clientHandleExcept(conn); clientHandleExcept(conn);
} }
@ -467,6 +454,7 @@ static void clientConnCb(uv_connect_t* req, int status) {
static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) {
tDebug("client work thread %p start to quit", pThrd); tDebug("client work thread %p start to quit", pThrd);
destroyCmsg(pMsg); destroyCmsg(pMsg);
destroyConnPool(pThrd->pool);
// transDestroyAsyncPool(pThr) uv_close((uv_handle_t*)pThrd->cliAsync, NULL); // transDestroyAsyncPool(pThr) uv_close((uv_handle_t*)pThrd->cliAsync, NULL);
uv_timer_stop(pThrd->timer); uv_timer_stop(pThrd->timer);
pThrd->quit = true; pThrd->quit = true;
@ -483,7 +471,10 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
SCliConn* conn = NULL; SCliConn* conn = NULL;
if (pMsg->msg.handle == NULL) { if (pMsg->msg.handle == NULL) {
if (pCtx->pTransInst->noPool == true) {
} else {
conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port);
}
if (conn != NULL) { if (conn != NULL) {
tTrace("client conn %p get from conn pool", conn); tTrace("client conn %p get from conn pool", conn);
} }
@ -512,7 +503,11 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) {
conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t));
uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream)); uv_tcp_init(pThrd->loop, (uv_tcp_t*)(conn->stream));
conn->stream->data = conn; conn->stream->data = conn;
uv_tcp_nodelay((uv_tcp_t*)conn->stream, 1);
int ret = uv_tcp_keepalive((uv_tcp_t*)conn->stream, 1, 1);
if (ret) {
tTrace("client conn %p failed to set keepalive, %s", conn, uv_err_name(ret));
}
// write req handle // write req handle
conn->writeReq = malloc(sizeof(uv_write_t)); conn->writeReq = malloc(sizeof(uv_write_t));
conn->writeReq->data = conn; conn->writeReq->data = conn;

27
source/libs/transport/src/transComm.c
View File

@ -205,6 +205,7 @@ int transInitBuffer(SConnBuffer* buf) {
} }
int transClearBuffer(SConnBuffer* buf) { int transClearBuffer(SConnBuffer* buf) {
memset(buf, 0, sizeof(*buf)); memset(buf, 0, sizeof(*buf));
buf->total = -1;
return 0; return 0;
} }
int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) {
@ -214,32 +215,38 @@ int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) {
* |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user
* info--->| * info--->|
*/ */
static const int CAPACITY = 1024; static const int CAPACITY = sizeof(STransMsgHead);
SConnBuffer* p = connBuf; SConnBuffer* p = connBuf;
if (p->cap == 0) { if (p->cap == 0) {
p->buf = (char*)calloc(CAPACITY, sizeof(char)); p->buf = (char*)calloc(CAPACITY, sizeof(char));
p->len = 0; p->len = 0;
p->cap = CAPACITY; p->cap = CAPACITY;
p->left = -1; p->total = -1;
uvBuf->base = p->buf; uvBuf->base = p->buf;
uvBuf->len = CAPACITY; uvBuf->len = CAPACITY;
} else { } else {
if (p->len >= p->cap) { p->cap = p->total;
if (p->left == -1) {
p->cap *= 2;
p->buf = realloc(p->buf, p->cap); p->buf = realloc(p->buf, p->cap);
} else if (p->len + p->left > p->cap) {
p->cap = p->len + p->left;
p->buf = realloc(p->buf, p->len + p->left);
}
}
uvBuf->base = p->buf + p->len; uvBuf->base = p->buf + p->len;
uvBuf->len = p->cap - p->len; uvBuf->len = p->cap - p->len;
} }
return 0; return 0;
} }
// check whether already read complete
bool transReadComplete(SConnBuffer* connBuf) {
if (connBuf->total == -1 && connBuf->len >= sizeof(STransMsgHead)) {
STransMsgHead head;
memcpy((char*)&head, connBuf->buf, sizeof(head));
int32_t msgLen = (int32_t)htonl(head.msgLen);
connBuf->total = msgLen;
}
if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) {
return true;
}
return false;
}
int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {} int transPackMsg(STransMsgHead* msgHead, bool sercured, bool auth) {}
int transUnpackMsg(STransMsgHead* msgHead) {} int transUnpackMsg(STransMsgHead* msgHead) {}

118
source/libs/transport/src/transSrv.c
View File

@ -61,6 +61,7 @@ typedef struct SWorkThrdObj {
SAsyncPool* asyncPool; SAsyncPool* asyncPool;
// uv_async_t* workerAsync; // // uv_async_t* workerAsync; //
queue msg; queue msg;
queue conn;
pthread_mutex_t msgMtx; pthread_mutex_t msgMtx;
void* pTransInst; void* pTransInst;
} SWorkThrdObj; } SWorkThrdObj;
@ -95,6 +96,7 @@ static void uvOnAcceptCb(uv_stream_t* stream, int status);
static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); static void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf);
static void uvWorkerAsyncCb(uv_async_t* handle); static void uvWorkerAsyncCb(uv_async_t* handle);
static void uvAcceptAsyncCb(uv_async_t* handle); static void uvAcceptAsyncCb(uv_async_t* handle);
static void uvShutDownCb(uv_shutdown_t* req, int status);
static void uvStartSendRespInternal(SSrvMsg* smsg); static void uvStartSendRespInternal(SSrvMsg* smsg);
static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb); static void uvPrepareSendData(SSrvMsg* msg, uv_buf_t* wb);
@ -102,8 +104,7 @@ static void uvStartSendResp(SSrvMsg* msg);
static void destroySmsg(SSrvMsg* smsg); static void destroySmsg(SSrvMsg* smsg);
// check whether already read complete packet // check whether already read complete packet
static bool readComplete(SConnBuffer* buf); static SSrvConn* createConn(void* hThrd);
static SSrvConn* createConn();
static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/); static void destroyConn(SSrvConn* conn, bool clear /*clear handle or not*/);
static void uvDestroyConn(uv_handle_t* handle); static void uvDestroyConn(uv_handle_t* handle);
@ -117,51 +118,11 @@ static bool addHandleToWorkloop(void* arg);
static bool addHandleToAcceptloop(void* arg); static bool addHandleToAcceptloop(void* arg);
void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { void uvAllocReadBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
/*
* formate of data buffer:
* |<--------------------------data from socket------------------------------->|
* |<------STransMsgHead------->|<-------------------other data--------------->|
*/
SSrvConn* conn = handle->data; SSrvConn* conn = handle->data;
SConnBuffer* pBuf = &conn->readBuf; SConnBuffer* pBuf = &conn->readBuf;
transAllocBuffer(pBuf, buf); transAllocBuffer(pBuf, buf);
} }
// check data read from socket complete or not
//
static bool readComplete(SConnBuffer* data) {
// TODO(yihao): handle pipeline later
STransMsgHead head;
int32_t headLen = sizeof(head);
if (data->len >= headLen) {
memcpy((char*)&head, data->buf, headLen);
int32_t msgLen = (int32_t)htonl((uint32_t)head.msgLen);
if (msgLen > data->len) {
data->left = msgLen - data->len;
return false;
} else if (msgLen == data->len) {
return true;
} else if (msgLen < data->len) {
return false;
// handle other packet later
}
} else {
return false;
}
}
// static void uvDoProcess(SRecvInfo* pRecv) {
// // impl later
// STransMsgHead* pHead = (STransMsgHead*)pRecv->msg;
// SRpcInfo* pRpc = (SRpcInfo*)pRecv->shandle;
// SSrvConn* pConn = pRecv->thandle;
// tDump(pRecv->msg, pRecv->msgLen);
// terrno = 0;
// // SRpcReqContext* pContest;
//
// // do auth and check
//}
static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) { static int uvAuthMsg(SSrvConn* pConn, char* msg, int len) {
STransMsgHead* pHead = (STransMsgHead*)msg; STransMsgHead* pHead = (STransMsgHead*)msg;
@ -241,7 +202,7 @@ static void uvHandleReq(SSrvConn* pConn) {
} }
pConn->inType = pHead->msgType; pConn->inType = pHead->msgType;
assert(transIsReq(pHead->msgType)); // assert(transIsReq(pHead->msgType));
SRpcInfo* pRpc = (SRpcInfo*)p->shandle; SRpcInfo* pRpc = (SRpcInfo*)p->shandle;
pHead->code = htonl(pHead->code); pHead->code = htonl(pHead->code);
@ -266,9 +227,9 @@ static void uvHandleReq(SSrvConn* pConn) {
transClearBuffer(&pConn->readBuf); transClearBuffer(&pConn->readBuf);
pConn->ref++; pConn->ref++;
tDebug("server conn %p %s received from %s:%d, local info: %s:%d", pConn, TMSG_INFO(rpcMsg.msgType), tDebug("server conn %p %s received from %s:%d, local info: %s:%d, msg size: %d", pConn, TMSG_INFO(rpcMsg.msgType),
inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr),
ntohs(pConn->locaddr.sin_port)); ntohs(pConn->locaddr.sin_port), rpcMsg.contLen);
(*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL); (*(pRpc->cfp))(pRpc->parent, &rpcMsg, NULL);
// uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0); // uv_timer_start(pConn->pTimer, uvHandleActivityTimeout, pRpc->idleTime * 10000, 0);
// auth // auth
@ -282,7 +243,7 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
if (nread > 0) { if (nread > 0) {
pBuf->len += nread; pBuf->len += nread;
tTrace("server conn %p read summary, total read: %d, current read: %d", conn, pBuf->len, (int)nread); tTrace("server conn %p read summary, total read: %d, current read: %d", conn, pBuf->len, (int)nread);
if (readComplete(pBuf)) { if (transReadComplete(pBuf)) {
tTrace("server conn %p alread read complete packet", conn); tTrace("server conn %p alread read complete packet", conn);
uvHandleReq(conn); uvHandleReq(conn);
} else { } else {
@ -290,6 +251,14 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
} }
return; return;
} }
if (nread == UV_EOF) {
tError("server conn %p read error: %s", conn, uv_err_name(nread));
if (conn->ref > 1) {
conn->ref++; // ref > 1 signed that write is in progress
}
destroyConn(conn, true);
return;
}
if (nread == 0) { if (nread == 0) {
return; return;
} }
@ -302,8 +271,8 @@ void uvOnReadCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) {
} }
} }
void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { void uvAllocConnBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
buf->base = malloc(sizeof(char));
buf->len = 2; buf->len = 2;
buf->base = calloc(1, sizeof(char) * buf->len);
} }
void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnTimeoutCb(uv_timer_t* handle) {
@ -386,6 +355,7 @@ static void uvStartSendRespInternal(SSrvMsg* smsg) {
static void uvStartSendResp(SSrvMsg* smsg) { static void uvStartSendResp(SSrvMsg* smsg) {
// impl // impl
SSrvConn* pConn = smsg->pConn; SSrvConn* pConn = smsg->pConn;
pConn->ref--; //
if (taosArrayGetSize(pConn->srvMsgs) > 0) { if (taosArrayGetSize(pConn->srvMsgs) > 0) {
tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr), tDebug("server conn %p push data to client %s:%d, local info: %s:%d", pConn, inet_ntoa(pConn->addr.sin_addr),
ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port)); ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), ntohs(pConn->locaddr.sin_port));
@ -403,6 +373,16 @@ static void destroySmsg(SSrvMsg* smsg) {
transFreeMsg(smsg->msg.pCont); transFreeMsg(smsg->msg.pCont);
free(smsg); free(smsg);
} }
static void destroyAllConn(SWorkThrdObj* pThrd) {
while (!QUEUE_IS_EMPTY(&pThrd->conn)) {
queue* h = QUEUE_HEAD(&pThrd->conn);
QUEUE_REMOVE(h);
QUEUE_INIT(h);
SSrvConn* c = QUEUE_DATA(h, SSrvConn, queue);
destroyConn(c, true);
}
}
void uvWorkerAsyncCb(uv_async_t* handle) { void uvWorkerAsyncCb(uv_async_t* handle) {
SAsyncItem* item = handle->data; SAsyncItem* item = handle->data;
SWorkThrdObj* pThrd = item->pThrd; SWorkThrdObj* pThrd = item->pThrd;
@ -424,8 +404,11 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
continue; continue;
} }
if (msg->pConn == NULL) { if (msg->pConn == NULL) {
//
free(msg); free(msg);
destroyAllConn(pThrd);
uv_loop_close(pThrd->loop);
uv_stop(pThrd->loop); uv_stop(pThrd->loop);
} else { } else {
uvStartSendResp(msg); uvStartSendResp(msg);
@ -439,9 +422,16 @@ void uvWorkerAsyncCb(uv_async_t* handle) {
} }
static void uvAcceptAsyncCb(uv_async_t* async) { static void uvAcceptAsyncCb(uv_async_t* async) {
SServerObj* srv = async->data; SServerObj* srv = async->data;
uv_close((uv_handle_t*)&srv->server, NULL);
uv_stop(srv->loop); uv_stop(srv->loop);
} }
static void uvShutDownCb(uv_shutdown_t* req, int status) {
tDebug("conn failed to shut down: %s", uv_err_name(status));
uv_close((uv_handle_t*)req->handle, uvDestroyConn);
free(req);
}
void uvOnAcceptCb(uv_stream_t* stream, int status) { void uvOnAcceptCb(uv_stream_t* stream, int status) {
if (status == -1) { if (status == -1) {
return; return;
@ -491,7 +481,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
uv_handle_type pending = uv_pipe_pending_type(pipe); uv_handle_type pending = uv_pipe_pending_type(pipe);
assert(pending == UV_TCP); assert(pending == UV_TCP);
SSrvConn* pConn = createConn(); SSrvConn* pConn = createConn(pThrd);
pConn->pTransInst = pThrd->pTransInst; pConn->pTransInst = pThrd->pTransInst;
/* init conn timer*/ /* init conn timer*/
@ -507,6 +497,9 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) {
uv_tcp_init(pThrd->loop, pConn->pTcp); uv_tcp_init(pThrd->loop, pConn->pTcp);
pConn->pTcp->data = pConn; pConn->pTcp->data = pConn;
// uv_tcp_nodelay(pConn->pTcp, 1);
// uv_tcp_keepalive(pConn->pTcp, 1, 1);
// init write request, just // init write request, just
pConn->pWriter = calloc(1, sizeof(uv_write_t)); pConn->pWriter = calloc(1, sizeof(uv_write_t));
pConn->pWriter->data = pConn; pConn->pWriter->data = pConn;
@ -560,6 +553,9 @@ static bool addHandleToWorkloop(void* arg) {
QUEUE_INIT(&pThrd->msg); QUEUE_INIT(&pThrd->msg);
pthread_mutex_init(&pThrd->msgMtx, NULL); pthread_mutex_init(&pThrd->msgMtx, NULL);
// conn set
QUEUE_INIT(&pThrd->conn);
pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb); pThrd->asyncPool = transCreateAsyncPool(pThrd->loop, 4, pThrd, uvWorkerAsyncCb);
uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb);
return true; return true;
@ -598,8 +594,13 @@ void* workerThread(void* arg) {
uv_run(pThrd->loop, UV_RUN_DEFAULT); uv_run(pThrd->loop, UV_RUN_DEFAULT);
} }
static SSrvConn* createConn() { static SSrvConn* createConn(void* hThrd) {
SWorkThrdObj* pThrd = hThrd;
SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn)); SSrvConn* pConn = (SSrvConn*)calloc(1, sizeof(SSrvConn));
QUEUE_INIT(&pConn->queue);
QUEUE_PUSH(&pThrd->conn, &pConn->queue);
pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); // pConn->srvMsgs = taosArrayInit(2, sizeof(void*)); //
tTrace("conn %p created", pConn); tTrace("conn %p created", pConn);
++pConn->ref; ++pConn->ref;
@ -610,7 +611,7 @@ static void destroyConn(SSrvConn* conn, bool clear) {
if (conn == NULL) { if (conn == NULL) {
return; return;
} }
tTrace("server conn %p try to destroy", conn); tTrace("server conn %p try to destroy, ref: %d", conn, conn->ref);
if (--conn->ref > 0) { if (--conn->ref > 0) {
return; return;
} }
@ -621,20 +622,23 @@ static void destroyConn(SSrvConn* conn, bool clear) {
destroySmsg(msg); destroySmsg(msg);
} }
taosArrayDestroy(conn->srvMsgs); taosArrayDestroy(conn->srvMsgs);
QUEUE_REMOVE(&conn->queue);
// destroySmsg(conn->pSrvMsg);
// conn->pSrvMsg = NULL;
if (clear) { if (clear) {
uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); tTrace("try to destroy conn %p", conn);
uv_tcp_close_reset(conn->pTcp, uvDestroyConn);
// uv_shutdown_t* req = malloc(sizeof(uv_shutdown_t));
// uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb);
// uv_unref((uv_handle_t*)conn->pTcp);
// uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn);
} }
} }
static void uvDestroyConn(uv_handle_t* handle) { static void uvDestroyConn(uv_handle_t* handle) {
SSrvConn* conn = handle->data; SSrvConn* conn = handle->data;
tDebug("server conn %p destroy", conn); tDebug("server conn %p destroy", conn);
uv_timer_stop(conn->pTimer); uv_timer_stop(conn->pTimer);
free(conn->pTimer); // free(conn->pTimer);
// free(conn->pTcp); free(conn->pTcp);
free(conn->pWriter); free(conn->pWriter);
free(conn); free(conn);
} }

206
source/libs/transport/test/transUT.cc
View File

@ -16,69 +16,168 @@
#include <cstdio> #include <cstdio>
#include <cstring> #include <cstring>
#include "tep.h" #include "tep.h"
#include "tglobal.h"
#include "trpc.h" #include "trpc.h"
#include "ulog.h"
using namespace std; using namespace std;
class TransObj {
public:
TransObj() {
const char *label = "APP"; const char *label = "APP";
const char *secret = "secret"; const char *secret = "secret";
const char *user = "user"; const char *user = "user";
const char *ckey = "ckey"; const char *ckey = "ckey";
class Server;
int port = 7000;
// server process
static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
// client process;
static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet);
class Client {
public:
void Init(int nThread) {
memset(&rpcInit, 0, sizeof(rpcInit)); memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = 0; rpcInit.localPort = 0;
rpcInit.label = (char *)label; rpcInit.label = (char *)label;
rpcInit.numOfThreads = 5; rpcInit.numOfThreads = nThread;
rpcInit.cfp = NULL; rpcInit.cfp = processResp;
rpcInit.sessions = 100;
rpcInit.idleTime = 100;
rpcInit.user = (char *)user; rpcInit.user = (char *)user;
rpcInit.secret = (char *)secret; rpcInit.secret = (char *)secret;
rpcInit.ckey = (char *)ckey; rpcInit.ckey = (char *)ckey;
rpcInit.spi = 1; rpcInit.spi = 1;
} rpcInit.parent = this;
bool startCli() {
trans = NULL;
rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.connType = TAOS_CONN_CLIENT;
trans = rpcOpen(&rpcInit); this->transCli = rpcOpen(&rpcInit);
return trans != NULL ? true : false; tsem_init(&this->sem, 0, 0);
} }
bool startSrv() { void SetResp(SRpcMsg *pMsg) {
trans = NULL; // set up resp;
rpcInit.connType = TAOS_CONN_SERVER; this->resp = *pMsg;
trans = rpcOpen(&rpcInit); }
return trans != NULL ? true : false; SRpcMsg *Resp() { return &this->resp; }
void Restart() {
rpcClose(this->transCli);
this->transCli = rpcOpen(&rpcInit);
} }
bool sendAndRecv() { void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) {
SEpSet epSet = {0}; SEpSet epSet = {0};
epSet.inUse = 0; epSet.inUse = 0;
addEpIntoEpSet(&epSet, "192.168.1.1", 7000); addEpIntoEpSet(&epSet, "127.0.0.1", 7000);
addEpIntoEpSet(&epSet, "192.168.0.1", 7000);
if (trans == NULL) { rpcSendRequest(this->transCli, &epSet, req, NULL);
return false; SemWait();
*resp = this->resp;
} }
SRpcMsg rpcMsg = {0}, reqMsg = {0}; void SemWait() { tsem_wait(&this->sem); }
reqMsg.pCont = rpcMallocCont(10); void SemPost() { tsem_post(&this->sem); }
reqMsg.contLen = 10; void Reset() {}
reqMsg.ahandle = NULL;
rpcSendRecv(trans, &epSet, &reqMsg, &rpcMsg); ~Client() {
int code = rpcMsg.code; if (this->transCli) rpcClose(this->transCli);
std::cout << tstrerror(code) << std::endl;
return true;
}
bool stop() {
rpcClose(trans);
trans = NULL;
return true;
} }
private: private:
void * trans; tsem_t sem;
SRpcInit rpcInit; SRpcInit rpcInit;
void * transCli;
SRpcMsg resp;
};
class Server {
public:
Server() {
memset(&rpcInit, 0, sizeof(rpcInit));
rpcInit.localPort = port;
rpcInit.label = (char *)label;
rpcInit.numOfThreads = 5;
rpcInit.cfp = processReq;
rpcInit.user = (char *)user;
rpcInit.secret = (char *)secret;
rpcInit.ckey = (char *)ckey;
rpcInit.spi = 1;
rpcInit.connType = TAOS_CONN_SERVER;
}
void Start() {
this->transSrv = rpcOpen(&this->rpcInit);
taosMsleep(1000);
}
void Stop() {
if (this->transSrv == NULL) return;
rpcClose(this->transSrv);
this->transSrv = NULL;
}
void Restart() {
this->Stop();
this->Start();
}
~Server() {
if (this->transSrv) rpcClose(this->transSrv);
this->transSrv = NULL;
}
private:
SRpcInit rpcInit;
void * transSrv;
};
static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = rpcMallocCont(100);
rpcMsg.contLen = 100;
rpcMsg.handle = pMsg->handle;
rpcMsg.code = 0;
rpcSendResponse(&rpcMsg);
}
// client process;
static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
Client *client = (Client *)parent;
client->SetResp(pMsg);
client->SemPost();
}
class TransObj {
public:
TransObj() {
dDebugFlag = 143;
vDebugFlag = 0;
mDebugFlag = 143;
cDebugFlag = 0;
jniDebugFlag = 0;
tmrDebugFlag = 143;
uDebugFlag = 143;
rpcDebugFlag = 143;
qDebugFlag = 0;
wDebugFlag = 0;
sDebugFlag = 0;
tsdbDebugFlag = 0;
cqDebugFlag = 0;
tscEmbeddedInUtil = 1;
tsAsyncLog = 0;
std::string path = "/tmp/transport";
taosRemoveDir(path.c_str());
taosMkDir(path.c_str());
char temp[PATH_MAX];
snprintf(temp, PATH_MAX, "%s/taosdlog", path.c_str());
if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) {
printf("failed to init log file\n");
}
cli = new Client;
cli->Init(1);
srv = new Server;
srv->Start();
}
void RestartCli() { cli->Restart(); }
void StopSrv() { srv->Stop(); }
void RestartSrv() { srv->Restart(); }
void cliSendAndRecv(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecv(req, resp); }
~TransObj() {
delete cli;
delete srv;
}
private:
Client *cli;
Server *srv;
}; };
class TransEnv : public ::testing::Test { class TransEnv : public ::testing::Test {
protected: protected:
@ -93,11 +192,34 @@ class TransEnv : public ::testing::Test {
TransObj *tr = NULL; TransObj *tr = NULL;
}; };
TEST_F(TransEnv, test_start_stop) {
assert(tr->startCli());
assert(tr->sendAndRecv());
assert(tr->stop());
assert(tr->startSrv()); // TEST_F(TransEnv, 01sendAndRec) {
assert(tr->stop()); // for (int i = 0; i < 1; i++) {
// SRpcMsg req = {0}, resp = {0};
// req.msgType = 0;
// req.pCont = rpcMallocCont(10);
// req.contLen = 10;
// tr->cliSendAndRecv(&req, &resp);
// assert(resp.code == 0);
// }
//}
TEST_F(TransEnv, 02StopServer) {
for (int i = 0; i < 1; i++) {
SRpcMsg req = {0}, resp = {0};
req.msgType = 0;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->cliSendAndRecv(&req, &resp);
assert(resp.code == 0);
}
SRpcMsg req = {0}, resp = {0};
req.msgType = 1;
req.pCont = rpcMallocCont(10);
req.contLen = 10;
tr->StopSrv();
// tr->RestartSrv();
tr->cliSendAndRecv(&req, &resp);
assert(resp.code != 0);
} }

14
source/os/src/osTimer.c
View File

@ -22,11 +22,10 @@
* windows implementation * windows implementation
*/ */
#include <Windows.h>
#include <Mmsystem.h> #include <Mmsystem.h>
#include <stdio.h> #include <Windows.h>
#include <stdint.h> #include <stdint.h>
#include <stdio.h>
#pragma warning(disable : 4244) #pragma warning(disable : 4244)
@ -50,9 +49,7 @@ int taosInitTimer(win_timer_f callback, int ms) {
return 0; return 0;
} }
void taosUninitTimer() { void taosUninitTimer() { timeKillEvent(timerId); }
timeKillEvent(timerId);
}
#elif defined(_TD_DARWIN_64) #elif defined(_TD_DARWIN_64)
@ -60,8 +57,8 @@ void taosUninitTimer() {
* darwin implementation * darwin implementation
*/ */
#include <sys/syscall.h>
#include <sys/event.h> #include <sys/event.h>
#include <sys/syscall.h>
#include <unistd.h> #include <unistd.h>
static void (*timer_callback)(int); static void (*timer_callback)(int);
@ -93,6 +90,8 @@ static void* timer_routine(void *arg) {
int taosInitTimer(void (*callback)(int), int ms) { int taosInitTimer(void (*callback)(int), int ms) {
int r = 0; int r = 0;
timer_kq = -1;
timer_stop = 0;
timer_ms = ms; timer_ms = ms;
timer_callback = callback; timer_callback = callback;
@ -203,6 +202,7 @@ static void *taosProcessAlarmSignal(void *tharg) {
} }
int taosInitTimer(void (*callback)(int), int ms) { int taosInitTimer(void (*callback)(int), int ms) {
stopTimer = false;
pthread_attr_t tattr; pthread_attr_t tattr;
pthread_attr_init(&tattr); pthread_attr_init(&tattr);
int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback); int code = pthread_create(&timerThread, &tattr, taosProcessAlarmSignal, callback);

8
source/util/src/tjson.c
View File

@ -32,6 +32,10 @@ int32_t tjsonAddIntegerToObject(SJson* pJson, const char* pName, const uint64_t
return tjsonAddStringToObject(pJson, pName, tmp); return tjsonAddStringToObject(pJson, pName, tmp);
} }
int32_t tjsonAddDoubleToObject(SJson* pJson, const char* pName, const double number) {
return (NULL == cJSON_AddNumberToObject((cJSON*)pJson, pName, number) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
}
int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) { int32_t tjsonAddStringToObject(SJson* pJson, const char* pName, const char* pVal) {
return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS); return (NULL == cJSON_AddStringToObject((cJSON*)pJson, pName, pVal) ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
} }
@ -74,3 +78,7 @@ int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj) {
char* tjsonToString(const SJson* pJson) { char* tjsonToString(const SJson* pJson) {
return cJSON_Print((cJSON*)pJson); return cJSON_Print((cJSON*)pJson);
} }
char* tjsonToUnformattedString(const SJson* pJson) {
return cJSON_PrintUnformatted((cJSON*)pJson);
}

53
source/util/src/ttimer.c
View File

@ -13,19 +13,49 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "ttimer.h"
#include "os.h" #include "os.h"
#include "taoserror.h"
#include "tlog.h" #include "tlog.h"
#include "tsched.h" #include "tsched.h"
#include "ttimer.h"
#include "tutil.h" #include "tutil.h"
#include "taoserror.h"
#define tmrFatal(...) { if (tmrDebugFlag & DEBUG_FATAL) { taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); }} #define tmrFatal(...) \
#define tmrError(...) { if (tmrDebugFlag & DEBUG_ERROR) { taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); }} { \
#define tmrWarn(...) { if (tmrDebugFlag & DEBUG_WARN) { taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); }} if (tmrDebugFlag & DEBUG_FATAL) { \
#define tmrInfo(...) { if (tmrDebugFlag & DEBUG_INFO) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} taosPrintLog("TMR FATAL ", tmrDebugFlag, __VA_ARGS__); \
#define tmrDebug(...) { if (tmrDebugFlag & DEBUG_DEBUG) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} } \
#define tmrTrace(...) { if (tmrDebugFlag & DEBUG_TRACE) { taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); }} }
#define tmrError(...) \
{ \
if (tmrDebugFlag & DEBUG_ERROR) { \
taosPrintLog("TMR ERROR ", tmrDebugFlag, __VA_ARGS__); \
} \
}
#define tmrWarn(...) \
{ \
if (tmrDebugFlag & DEBUG_WARN) { \
taosPrintLog("TMR WARN ", tmrDebugFlag, __VA_ARGS__); \
} \
}
#define tmrInfo(...) \
{ \
if (tmrDebugFlag & DEBUG_INFO) { \
taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \
} \
}
#define tmrDebug(...) \
{ \
if (tmrDebugFlag & DEBUG_DEBUG) { \
taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \
} \
}
#define tmrTrace(...) \
{ \
if (tmrDebugFlag & DEBUG_TRACE) { \
taosPrintLog("TMR ", tmrDebugFlag, __VA_ARGS__); \
} \
}
#define TIMER_STATE_WAITING 0 #define TIMER_STATE_WAITING 0
#define TIMER_STATE_EXPIRED 1 #define TIMER_STATE_EXPIRED 1
@ -491,6 +521,8 @@ static void taosTmrModuleInit(void) {
return; return;
} }
memset(&timerMap, 0, sizeof(timerMap));
for (uint32_t i = 0; i < tsMaxTmrCtrl - 1; ++i) { for (uint32_t i = 0; i < tsMaxTmrCtrl - 1; ++i) {
tmr_ctrl_t* ctrl = tmrCtrls + i; tmr_ctrl_t* ctrl = tmrCtrls + i;
ctrl->next = ctrl + 1; ctrl->next = ctrl + 1;
@ -570,6 +602,7 @@ void taosTmrCleanUp(void* handle) {
unusedTmrCtrl = ctrl; unusedTmrCtrl = ctrl;
pthread_mutex_unlock(&tmrCtrlMutex); pthread_mutex_unlock(&tmrCtrlMutex);
tmrDebug("time controller's tmr ctrl size: %d", numOfTmrCtrl);
if (numOfTmrCtrl <= 0) { if (numOfTmrCtrl <= 0) {
taosUninitTimer(); taosUninitTimer();
@ -595,6 +628,8 @@ void taosTmrCleanUp(void* handle) {
free(timerMap.slots); free(timerMap.slots);
free(tmrCtrls); free(tmrCtrls);
tmrDebug("timer module is cleaned up"); tmrCtrls = NULL;
unusedTmrCtrl = NULL;
tmrModuleInit = PTHREAD_ONCE_INIT; // to support restart
} }
} }