xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/qnode

This commit is contained in:
dapan1121 2021-12-24 09:41:41 +08:00
parent fddd11ed47 1d68b746e6
commit bbada5a8a6
40 changed files with 2025 additions and 653 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.clang-format
View File

@ -86,5 +86,6 @@ SpacesInSquareBrackets: false
Standard: Auto
TabWidth: 8
UseTab: Never
AlignConsecutiveDeclarations: true
...

1
contrib/test/craft/clear.sh
View File

@ -1,3 +1,4 @@
#!/bin/bash
rm -rf 127.0.0.1*
rm -rf ./data

34
contrib/test/craft/common.h Normal file
View File

@ -0,0 +1,34 @@
#ifndef TDENGINE_COMMON_H
#define TDENGINE_COMMON_H
#ifdef __cplusplus
extern "C" {
#endif
#include <stdint.h>
#define MAX_PEERS 10
#define COMMAND_LEN 512
#define TOKEN_LEN 128
#define DIR_LEN 256
#define HOST_LEN 64
#define ADDRESS_LEN (HOST_LEN + 16)
typedef struct {
char host[HOST_LEN];
uint32_t port;
} Addr;
typedef struct {
Addr me;
Addr peers[MAX_PEERS];
int peersCount;
char dir[DIR_LEN];
char dataDir[DIR_LEN + HOST_LEN * 2];
} SRaftServerConfig;
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_COMMON_H

367
contrib/test/craft/raftMain.c Normal file
View File

@ -0,0 +1,367 @@
#include <stdio.h>
#include <unistd.h>
#include <pthread.h>
#include <string.h>
#include <assert.h>
#include <getopt.h>
#include <time.h>
#include <stdlib.h>
#include <getopt.h>
#include <raft.h>
#include <raft/uv.h>
#include "raftServer.h"
#include "common.h"
const char *exe_name;
void parseAddr(const char *addr, char *host, int len, uint32_t *port) {
char* tmp = (char*)malloc(strlen(addr) + 1);
strcpy(tmp, addr);
char* context;
char* separator = ":";
char* token = strtok_r(tmp, separator, &context);
if (token) {
snprintf(host, len, "%s", token);
}
token = strtok_r(NULL, separator, &context);
if (token) {
sscanf(token, "%u", port);
}
free(tmp);
}
// only parse 3 tokens
int parseCommand(const char* str, char* token1, char* token2, char* token3, int len)
{
char* tmp = (char*)malloc(strlen(str) + 1);
strcpy(tmp, str);
char* context;
char* separator = " ";
int n = 0;
char* token = strtok_r(tmp, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token1, token, len);
n++;
}
token = strtok_r(NULL, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token2, token, len);
n++;
}
token = strtok_r(NULL, separator, &context);
if (!token) {
goto ret;
}
if (strcmp(token, "") != 0) {
strncpy(token3, token, len);
n++;
}
ret:
return n;
free(tmp);
}
void *startServerFunc(void *param) {
SRaftServer *pServer = (SRaftServer*)param;
int32_t r = raftServerStart(pServer);
assert(r == 0);
return NULL;
}
// Console ---------------------------------
const char* state2String(unsigned short state) {
if (state == RAFT_UNAVAILABLE) {
return "RAFT_UNAVAILABLE";
} else if (state == RAFT_FOLLOWER) {
return "RAFT_FOLLOWER";
} else if (state == RAFT_CANDIDATE) {
return "RAFT_CANDIDATE";
} else if (state == RAFT_LEADER) {
return "RAFT_LEADER";
}
return "UNKNOWN_RAFT_STATE";
}
void printRaftConfiguration(struct raft_configuration *c) {
printf("configuration: \n");
for (int i = 0; i < c->n; ++i) {
printf("%llu -- %d -- %s\n", c->servers->id, c->servers->role, c->servers->address);
}
}
void printRaftState(struct raft *r) {
printf("----Raft State: -----------\n");
printf("my_id: %llu \n", r->id);
printf("address: %s \n", r->address);
printf("current_term: %llu \n", r->current_term);
printf("voted_for: %llu \n", r->voted_for);
printf("role: %s \n", state2String(r->state));
printf("commit_index: %llu \n", r->commit_index);
printf("last_applied: %llu \n", r->last_applied);
printf("last_stored: %llu \n", r->last_stored);
/*
printf("configuration_index: %llu \n", r->configuration_index);
printf("configuration_uncommitted_index: %llu \n", r->configuration_uncommitted_index);
printRaftConfiguration(&r->configuration);
*/
printf("----------------------------\n");
}
void putValueCb(struct raft_apply *req, int status, void *result) {
raft_free(req);
struct raft *r = req->data;
if (status != 0) {
printf("putValueCb: %s \n", raft_errmsg(r));
} else {
printf("putValueCb: %s \n", "ok");
}
}
void putValue(struct raft *r, const char *value) {
struct raft_buffer buf;
buf.len = TOKEN_LEN;;
buf.base = raft_malloc(buf.len);
snprintf(buf.base, buf.len, "%s", value);
struct raft_apply *req = raft_malloc(sizeof(struct raft_apply));
req->data = r;
int ret = raft_apply(r, req, &buf, 1, putValueCb);
if (ret == 0) {
printf("put %s \n", (char*)buf.base);
} else {
printf("put error: %s \n", raft_errmsg(r));
}
}
void getValue(const char *key) {
char *ptr = getKV(key);
if (ptr) {
printf("get value: [%s] \n", ptr);
} else {
printf("value not found for key: [%s] \n", key);
}
}
void console(SRaftServer *pRaftServer) {
while (1) {
char cmd_buf[COMMAND_LEN];
memset(cmd_buf, 0, sizeof(cmd_buf));
char *ret = fgets(cmd_buf, COMMAND_LEN, stdin);
if (!ret) {
exit(-1);
}
int pos = strlen(cmd_buf);
if(cmd_buf[pos - 1] == '\n') {
cmd_buf[pos - 1] = '\0';
}
if (strncmp(cmd_buf, "", COMMAND_LEN) == 0) {
continue;
}
char cmd[TOKEN_LEN];
memset(cmd, 0, sizeof(cmd));
char param1[TOKEN_LEN];
memset(param1, 0, sizeof(param1));
char param2[TOKEN_LEN];
memset(param2, 0, sizeof(param2));
parseCommand(cmd_buf, cmd, param1, param2, TOKEN_LEN);
if (strcmp(cmd, "addnode") == 0) {
printf("not support \n");
/*
char host[HOST_LEN];
uint32_t port;
parseAddr(param1, host, HOST_LEN, &port);
uint64_t rid = raftId(host, port);
struct raft_change *req = raft_malloc(sizeof(*req));
int r = raft_add(&pRaftServer->raft, req, rid, param1, NULL);
if (r != 0) {
printf("raft_add: %s \n", raft_errmsg(&pRaftServer->raft));
}
printf("add node: %lu %s \n", rid, param1);
struct raft_change *req2 = raft_malloc(sizeof(*req2));
r = raft_assign(&pRaftServer->raft, req2, rid, RAFT_VOTER, NULL);
if (r != 0) {
printf("raft_assign: %s \n", raft_errmsg(&pRaftServer->raft));
}
*/
} else if (strcmp(cmd, "dropnode") == 0) {
printf("not support \n");
} else if (strcmp(cmd, "put") == 0) {
char buf[256];
snprintf(buf, sizeof(buf), "%s--%s", param1, param2);
putValue(&pRaftServer->raft, buf);
} else if (strcmp(cmd, "get") == 0) {
getValue(param1);
} else if (strcmp(cmd, "state") == 0) {
printRaftState(&pRaftServer->raft);
} else if (strcmp(cmd, "snapshot") == 0) {
printf("not support \n");
} else if (strcmp(cmd, "help") == 0) {
printf("addnode \"127.0.0.1:8888\" \n");
printf("dropnode \"127.0.0.1:8888\" \n");
printf("put key value \n");
printf("get key \n");
printf("state \n");
} else {
printf("unknown command: [%s], type \"help\" to see help \n", cmd);
}
//printf("cmd_buf: [%s] \n", cmd_buf);
}
}
void *startConsoleFunc(void *param) {
SRaftServer *pServer = (SRaftServer*)param;
console(pServer);
return NULL;
}
// Config ---------------------------------
void usage() {
printf("\nusage: \n");
printf("%s --me=127.0.0.1:10000 --dir=./data \n", exe_name);
printf("\n");
printf("%s --me=127.0.0.1:10000 --peers=127.0.0.1:10001,127.0.0.1:10002 --dir=./data \n", exe_name);
printf("%s --me=127.0.0.1:10001 --peers=127.0.0.1:10000,127.0.0.1:10002 --dir=./data \n", exe_name);
printf("%s --me=127.0.0.1:10002 --peers=127.0.0.1:10000,127.0.0.1:10001 --dir=./data \n", exe_name);
printf("\n");
}
void parseConf(int argc, char **argv, SRaftServerConfig *pConf) {
memset(pConf, 0, sizeof(*pConf));
int option_index, option_value;
option_index = 0;
static struct option long_options[] = {
{"help", no_argument, NULL, 'h'},
{"peers", required_argument, NULL, 'p'},
{"me", required_argument, NULL, 'm'},
{"dir", required_argument, NULL, 'd'},
{NULL, 0, NULL, 0}
};
while ((option_value = getopt_long(argc, argv, "hp:m:d:", long_options, &option_index)) != -1) {
switch (option_value) {
case 'm': {
parseAddr(optarg, pConf->me.host, sizeof(pConf->me.host), &pConf->me.port);
break;
}
case 'p': {
char tokens[MAX_PEERS][MAX_TOKEN_LEN];
int peerCount = splitString(optarg, ",", tokens, MAX_PEERS);
pConf->peersCount = peerCount;
for (int i = 0; i < peerCount; ++i) {
Addr *pAddr = &pConf->peers[i];
parseAddr(tokens[i], pAddr->host, sizeof(pAddr->host), &pAddr->port);
}
break;
}
case 'd': {
snprintf(pConf->dir, sizeof(pConf->dir), "%s", optarg);
break;
}
case 'h': {
usage();
exit(-1);
}
default: {
usage();
exit(-1);
}
}
}
snprintf(pConf->dataDir, sizeof(pConf->dataDir), "%s/%s:%u", pConf->dir, pConf->me.host, pConf->me.port);
}
void printConf(SRaftServerConfig *pConf) {
printf("\nconf: \n");
printf("me: %s:%u \n", pConf->me.host, pConf->me.port);
printf("peersCount: %d \n", pConf->peersCount);
for (int i = 0; i < pConf->peersCount; ++i) {
Addr *pAddr = &pConf->peers[i];
printf("peer%d: %s:%u \n", i, pAddr->host, pAddr->port);
}
printf("dataDir: %s \n\n", pConf->dataDir);
}
int main(int argc, char **argv) {
srand(time(NULL));
int32_t ret;
exe_name = argv[0];
if (argc < 3) {
usage();
exit(-1);
}
SRaftServerConfig conf;
parseConf(argc, argv, &conf);
printConf(&conf);
char cmd_buf[COMMAND_LEN];
snprintf(cmd_buf, sizeof(cmd_buf), "mkdir -p %s", conf.dataDir);
system(cmd_buf);
struct raft_fsm fsm;
initFsm(&fsm);
SRaftServer raftServer;
ret = raftServerInit(&raftServer, &conf, &fsm);
assert(ret == 0);
pthread_t tidRaftServer;
pthread_create(&tidRaftServer, NULL, startServerFunc, &raftServer);
pthread_t tidConsole;
pthread_create(&tidConsole, NULL, startConsoleFunc, &raftServer);
while (1) {
sleep(10);
}
return 0;
}

156
contrib/test/craft/raftServer.c Normal file
View File

@ -0,0 +1,156 @@
#include <stdlib.h>
#include "common.h"
#include "raftServer.h"
char *keys;
char *values;
void initStore() {
keys = malloc(MAX_RECORD_COUNT * MAX_KV_LEN);
values = malloc(MAX_RECORD_COUNT * MAX_KV_LEN);
writeIndex = 0;
}
void destroyStore() {
free(keys);
free(values);
}
void putKV(const char *key, const char *value) {
if (writeIndex < MAX_RECORD_COUNT) {
strncpy(&keys[writeIndex], key, MAX_KV_LEN);
strncpy(&values[writeIndex], value, MAX_KV_LEN);
writeIndex++;
}
}
char *getKV(const char *key) {
for (int i = 0; i < MAX_RECORD_COUNT; ++i) {
if (strcmp(&keys[i], key) == 0) {
return &values[i];
}
}
return NULL;
}
int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr)
{
if (n_arr <= 0) {
return -1;
}
char* tmp = (char*)malloc(strlen(str) + 1);
strcpy(tmp, str);
char* context;
int n = 0;
char* token = strtok_r(tmp, separator, &context);
if (!token) {
goto ret;
}
strncpy(arr[n], token, MAX_TOKEN_LEN);
n++;
while (1) {
token = strtok_r(NULL, separator, &context);
if (!token || n >= n_arr) {
goto ret;
}
strncpy(arr[n], token, MAX_TOKEN_LEN);
n++;
}
ret:
free(tmp);
return n;
}
uint64_t raftId(const char *host, uint32_t port) {
uint32_t host_uint32 = (uint32_t)inet_addr(host);
assert(host_uint32 != (uint32_t)-1);
uint64_t code = ((uint64_t)host_uint32) << 32 | port;
return code;
}
int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm) {
int ret;
snprintf(pRaftServer->host, sizeof(pRaftServer->host), "%s", pConf->me.host);
pRaftServer->port = pConf->me.port;
snprintf(pRaftServer->address, sizeof(pRaftServer->address), "%s:%u", pRaftServer->host, pRaftServer->port);
strncpy(pRaftServer->dir, pConf->dataDir, sizeof(pRaftServer->dir));
pRaftServer->raftId = raftId(pRaftServer->host, pRaftServer->port);
pRaftServer->fsm = pFsm;
ret = uv_loop_init(&pRaftServer->loop);
if (!ret) {
fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft));
}
ret = raft_uv_tcp_init(&pRaftServer->transport, &pRaftServer->loop);
if (!ret) {
fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft));
}
ret = raft_uv_init(&pRaftServer->io, &pRaftServer->loop, pRaftServer->dir, &pRaftServer->transport);
if (!ret) {
fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft));
}
ret = raft_init(&pRaftServer->raft, &pRaftServer->io, pRaftServer->fsm, pRaftServer->raftId, pRaftServer->address);
if (!ret) {
fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft));
}
struct raft_configuration conf;
raft_configuration_init(&conf);
raft_configuration_add(&conf, pRaftServer->raftId, pRaftServer->address, RAFT_VOTER);
printf("add myself: %llu - %s \n", pRaftServer->raftId, pRaftServer->address);
for (int i = 0; i < pConf->peersCount; ++i) {
const Addr *pAddr = &pConf->peers[i];
raft_id rid = raftId(pAddr->host, pAddr->port);
char addrBuf[ADDRESS_LEN];
snprintf(addrBuf, sizeof(addrBuf), "%s:%u", pAddr->host, pAddr->port);
raft_configuration_add(&conf, rid, addrBuf, RAFT_VOTER);
printf("add peers: %llu - %s \n", rid, addrBuf);
}
raft_bootstrap(&pRaftServer->raft, &conf);
return 0;
}
int32_t raftServerStart(SRaftServer *pRaftServer) {
int ret;
ret = raft_start(&pRaftServer->raft);
if (!ret) {
fprintf(stderr, "%s \n", raft_errmsg(&pRaftServer->raft));
}
uv_run(&pRaftServer->loop, UV_RUN_DEFAULT);
}
void raftServerClose(SRaftServer *pRaftServer) {
}
int fsmApplyCb(struct raft_fsm *pFsm, const struct raft_buffer *buf, void **result) {
char *msg = (char*)buf->base;
printf("fsm apply: %s \n", msg);
char arr[2][MAX_TOKEN_LEN];
splitString(msg, "--", arr, 2);
putKV(arr[0], arr[1]);
return 0;
}
int32_t initFsm(struct raft_fsm *fsm) {
initStore();
fsm->apply = fsmApplyCb;
return 0;
}

61
contrib/test/craft/raftServer.h Normal file
View File

@ -0,0 +1,61 @@
#ifndef TDENGINE_RAFT_SERVER_H
#define TDENGINE_RAFT_SERVER_H
#ifdef __cplusplus
extern "C" {
#endif
#include <netinet/in.h>
#include <arpa/inet.h>
#include <assert.h>
#include <string.h>
#include "raft.h"
#include "raft/uv.h"
#include "common.h"
// simulate a db store, just for test
#define MAX_KV_LEN 100
#define MAX_RECORD_COUNT 500
char *keys;
char *values;
int writeIndex;
void initStore();
void destroyStore();
void putKV(const char *key, const char *value);
char *getKV(const char *key);
typedef struct {
char dir[DIR_LEN + HOST_LEN * 2]; /* Data dir of UV I/O backend */
char host[HOST_LEN];
uint32_t port;
char address[ADDRESS_LEN]; /* Raft instance address */
raft_id raftId; /* For vote */
struct raft_fsm *fsm; /* Sample application FSM */
struct raft raft; /* Raft instance */
struct raft_io io; /* UV I/O backend */
struct uv_loop_s loop; /* UV loop */
struct raft_uv_transport transport; /* UV I/O backend transport */
} SRaftServer;
#define MAX_TOKEN_LEN 32
int splitString(const char* str, char* separator, char (*arr)[MAX_TOKEN_LEN], int n_arr);
uint64_t raftId(const char *host, uint32_t port);
int32_t raftServerInit(SRaftServer *pRaftServer, const SRaftServerConfig *pConf, struct raft_fsm *pFsm);
int32_t raftServerStart(SRaftServer *pRaftServer);
void raftServerClose(SRaftServer *pRaftServer);
int initFsm(struct raft_fsm *fsm);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_RAFT_SERVER_H

4
include/common/taosmsg.h
View File

@ -918,18 +918,15 @@ typedef struct SShowRsp {
typedef struct {
char ep[TSDB_EP_LEN]; // end point, hostname:port
int32_t reserve[8];
} SCreateDnodeMsg;
typedef struct {
int32_t dnodeId;
int32_t reserve[8];
} SDropDnodeMsg;
typedef struct {
int32_t dnodeId;
char config[TSDB_DNODE_CONFIG_LEN];
int32_t reserve[8];
} SCfgDnodeMsg;
typedef struct {
@ -938,7 +935,6 @@ typedef struct {
typedef struct {
int32_t dnodeId;
int8_t align[3];
int8_t replica;
SReplica replicas[TSDB_MAX_REPLICA];
} SCreateMnodeInMsg, SAlterMnodeInMsg;

7
include/libs/parser/parsenodes.h
View File

@ -160,6 +160,13 @@ typedef struct SInsertStmtInfo {
const char* sql; // current sql statement position
} SInsertStmtInfo;
typedef struct SDclStmtInfo {
int16_t nodeType;
int16_t msgType;
char* pMsg;
int32_t msgLen;
} SDclStmtInfo;
#ifdef __cplusplus
}
#endif

21
include/libs/parser/parser.h
View File

@ -22,20 +22,11 @@ extern "C" {
#include "parsenodes.h"
/**
* True will be returned if the input sql string is insert, false otherwise.
* @param pStr sql string
* @param length length of the sql string
* @return
*/
bool qIsInsertSql(const char* pStr, size_t length);
typedef struct SParseContext {
SParseBasicCtx ctx;
void *pRpc;
struct SCatalog *pCatalog;
const SEpSet *pEpSet;
int64_t id; // query id, generated by uuid generator
int8_t schemaAttached; // denote if submit block is built with table schema or not
const char *pSql; // sql string
size_t sqlLen; // length of the sql string
@ -51,17 +42,9 @@ typedef struct SParseContext {
* @param msg extended error message if exists.
* @return error code
*/
int32_t qParseQuerySql(const char* pStr, size_t length, SParseBasicCtx* pParseCtx, int32_t* type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen);
int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery);
/**
* Parse the insert sql statement.
* @param pStr sql string
* @param length length of the sql string
* @param id operator id, generated by uuid generator.
* @param msg extended error message if exists to help avoid the problem in sql statement.
* @return data in binary format to submit to vnode directly.
*/
int32_t qParseInsertSql(SParseContext* pContext, struct SInsertStmtInfo** pInfo);
bool qIsDclQuery(const SQueryNode* pQuery);
/**
* Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that

106
source/client/src/clientImpl.c
View File

@ -144,72 +144,66 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
tscDebugL("0x%"PRIx64" SQL: %s", pRequest->requestId, pRequest->sqlstr);
int32_t code = 0;
if (qIsInsertSql(pRequest->sqlstr, sqlLen)) {
// todo add
} else {
int32_t type = 0;
void* output = NULL;
int32_t outputLen = 0;
SParseContext cxt = {
.ctx = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj)},
.pSql = pRequest->sqlstr,
.sqlLen = sqlLen,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE
};
SQueryNode* pQuery = NULL;
int32_t code = qParseQuerySql(&cxt, &pQuery);
if (qIsDclQuery(pQuery)) {
SDclStmtInfo* pDcl = (SDclStmtInfo*)pQuery;
pRequest->type = pDcl->msgType;
pRequest->body.requestMsg = (SReqMsgInfo){.pMsg = pDcl->pMsg, .len = pDcl->msgLen};
SParseBasicCtx c = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj)};
code = qParseQuerySql(pRequest->sqlstr, sqlLen, &c, &type, &output, &outputLen, pRequest->msgBuf, ERROR_MSG_BUF_DEFAULT_SIZE);
if (type == TSDB_MSG_TYPE_CREATE_USER || type == TSDB_MSG_TYPE_SHOW || type == TSDB_MSG_TYPE_DROP_USER ||
type == TSDB_MSG_TYPE_DROP_ACCT || type == TSDB_MSG_TYPE_CREATE_DB || type == TSDB_MSG_TYPE_CREATE_ACCT ||
type == TSDB_MSG_TYPE_CREATE_TABLE || type == TSDB_MSG_TYPE_CREATE_STB || type == TSDB_MSG_TYPE_USE_DB ||
type == TSDB_MSG_TYPE_DROP_DB || type == TSDB_MSG_TYPE_DROP_STB) {
pRequest->type = type;
pRequest->body.requestMsg = (SReqMsgInfo){.pMsg = output, .len = outputLen};
SRequestMsgBody body = buildRequestMsgImpl(pRequest);
SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
SRequestMsgBody body = buildRequestMsgImpl(pRequest);
SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
if (pDcl->msgType == TSDB_MSG_TYPE_CREATE_TABLE) {
struct SCatalog* pCatalog = NULL;
if (type == TSDB_MSG_TYPE_CREATE_TABLE) {
struct SCatalog* pCatalog = NULL;
char buf[12] = {0};
sprintf(buf, "%d", pTscObj->pAppInfo->clusterId);
code = catalogGetHandle(buf, &pCatalog);
if (code != 0) {
pRequest->code = code;
return pRequest;
}
SCreateTableMsg* pMsg = body.msgInfo.pMsg;
SName t = {0};
tNameFromString(&t, pMsg->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE);
char db[TSDB_DB_NAME_LEN + TS_PATH_DELIMITER_LEN + TSDB_ACCT_ID_LEN] = {0};
tNameGetFullDbName(&t, db);
SVgroupInfo info = {0};
catalogGetTableHashVgroup(pCatalog, pTscObj->pTransporter, pEpSet, db, tNameGetTableName(&t), &info);
int64_t transporterId = 0;
SEpSet ep = {0};
ep.inUse = info.inUse;
ep.numOfEps = info.numOfEps;
for(int32_t i = 0; i < ep.numOfEps; ++i) {
ep.port[i] = info.epAddr[i].port;
tstrncpy(ep.fqdn[i], info.epAddr[i].fqdn, tListLen(ep.fqdn[i]));
}
sendMsgToServer(pTscObj->pTransporter, &ep, &body, &transporterId);
} else {
int64_t transporterId = 0;
sendMsgToServer(pTscObj->pTransporter, pEpSet, &body, &transporterId);
char buf[12] = {0};
sprintf(buf, "%d", pTscObj->pAppInfo->clusterId);
code = catalogGetHandle(buf, &pCatalog);
if (code != 0) {
pRequest->code = code;
return pRequest;
}
tsem_wait(&pRequest->body.rspSem);
destroyRequestMsgBody(&body);
SCreateTableMsg* pMsg = body.msgInfo.pMsg;
SName t = {0};
tNameFromString(&t, pMsg->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE);
char db[TSDB_DB_NAME_LEN + TS_PATH_DELIMITER_LEN + TSDB_ACCT_ID_LEN] = {0};
tNameGetFullDbName(&t, db);
SVgroupInfo info = {0};
catalogGetTableHashVgroup(pCatalog, pTscObj->pTransporter, pEpSet, db, tNameGetTableName(&t), &info);
int64_t transporterId = 0;
SEpSet ep = {0};
ep.inUse = info.inUse;
ep.numOfEps = info.numOfEps;
for(int32_t i = 0; i < ep.numOfEps; ++i) {
ep.port[i] = info.epAddr[i].port;
tstrncpy(ep.fqdn[i], info.epAddr[i].fqdn, tListLen(ep.fqdn[i]));
}
sendMsgToServer(pTscObj->pTransporter, &ep, &body, &transporterId);
} else {
assert(0);
int64_t transporterId = 0;
sendMsgToServer(pTscObj->pTransporter, pEpSet, &body, &transporterId);
}
tfree(c.db);
tsem_wait(&pRequest->body.rspSem);
destroyRequestMsgBody(&body);
}
tfree(cxt.ctx.db);
if (code != TSDB_CODE_SUCCESS) {
pRequest->code = code;
return pRequest;

47
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -349,7 +349,7 @@ static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) {
SReplica *pReplica = &pOption->replicas[0];
pReplica->id = 1;
pReplica->port = pDnode->opt.serverPort;
tstrncpy(pReplica->fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN);
memcpy(pReplica->fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN);
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
pMgmt->selfIndex = pOption->selfIndex;
@ -376,7 +376,7 @@ static int32_t dndBuildMnodeOptionFromMsg(SDnode *pDnode, SMnodeOpt *pOption, SC
SReplica *pReplica = &pOption->replicas[i];
pReplica->id = pMsg->replicas[i].id;
pReplica->port = pMsg->replicas[i].port;
tstrncpy(pReplica->fqdn, pMsg->replicas[i].fqdn, TSDB_FQDN_LEN);
memcpy(pReplica->fqdn, pMsg->replicas[i].fqdn, TSDB_FQDN_LEN);
if (pReplica->id == pOption->dnodeId) {
pOption->selfIndex = i;
}
@ -479,9 +479,11 @@ static int32_t dndDropMnode(SDnode *pDnode) {
return -1;
}
dndReleaseMnode(pDnode, pMnode);
dndStopMnodeWorker(pDnode);
dndWriteMnodeFile(pDnode);
mndClose(pMnode);
pMgmt->pMnode = NULL;
mndDestroy(pDnode->dir.mnode);
return 0;
@ -499,7 +501,7 @@ static SCreateMnodeInMsg *dndParseCreateMnodeMsg(SRpcMsg *pRpcMsg) {
}
static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SCreateMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont);
SCreateMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_MNODE_ID_INVALID;
@ -515,18 +517,23 @@ static int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
}
static int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SAlterMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg->pCont);
SAlterMnodeInMsg *pMsg = dndParseCreateMnodeMsg(pRpcMsg);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_MNODE_ID_INVALID;
return -1;
} else {
SMnodeOpt option = {0};
if (dndBuildMnodeOptionFromMsg(pDnode, &option, pMsg) != 0) {
return -1;
}
return dndAlterMnode(pDnode, &option);
}
SMnodeOpt option = {0};
if (dndBuildMnodeOptionFromMsg(pDnode, &option, pMsg) != 0) {
return -1;
}
if (dndAlterMnode(pDnode, &option) != 0) {
return -1;
}
return dndWriteMnodeFile(pDnode);
}
static int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
@ -555,16 +562,17 @@ static void dndProcessMnodeMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) {
code = dndProcessDropMnodeReq(pDnode, pMsg);
break;
default:
code = TSDB_CODE_MSG_NOT_PROCESSED;
terrno = TSDB_CODE_MSG_NOT_PROCESSED;
code = -1;
break;
}
if (pMsg->msgType & 1u) {
if (code != 0) code = terrno;
SRpcMsg rsp = {.code = code, .handle = pMsg->handle};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
taosFreeQitem(pMsg);
}
@ -625,8 +633,6 @@ static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
}
static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg) {
assert(pQueue);
SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg);
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -647,15 +653,18 @@ void dndProcessMnodeMgmtMsg(SDnode *pDnode, SRpcMsg *pRpcMsg, SEpSet *pEpSet) {
SMnode *pMnode = dndAcquireMnode(pDnode);
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
if (pMsg != NULL) *pMsg = *pRpcMsg;
if (pMsg == NULL || taosWriteQitem(pMgmt->pMgmtQ, pMsg) != 0) {
if (pRpcMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY};
rpcSendResponse(&rsp);
}
rpcFreeCont(pRpcMsg->pCont);
pRpcMsg->pCont = NULL;
taosFreeQitem(pMsg);
}
dndReleaseMnode(pDnode, pMnode);
}
void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
@ -894,6 +903,11 @@ int32_t dndInitMnode(SDnode *pDnode) {
return -1;
}
if (dndAllocMnodeMgmtQueue(pDnode) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
char path[PATH_MAX];
snprintf(path, PATH_MAX, "%s/mnode.json", pDnode->dir.dnode);
pMgmt->file = strdup(path);
@ -935,8 +949,9 @@ void dndCleanupMnode(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
dInfo("dnode-mnode start to clean up");
dndStopMnodeWorker(pDnode);
if (pMgmt->pMnode) dndStopMnodeWorker(pDnode);
dndCleanupMnodeMgmtWorker(pDnode);
dndFreeMnodeMgmtQueue(pDnode);
tfree(pMgmt->file);
mndClose(pMgmt->pMnode);
dInfo("dnode-mnode is cleaned up");

4
source/dnode/mgmt/impl/src/dndTransport.c
View File

@ -140,7 +140,7 @@ static void dndProcessResponse(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
DndMsgFp fp = pMgmt->msgFp[msgType];
if (fp != NULL) {
(*fp)(pDnode, pMsg, pEpSet);
dTrace("RPC %p, rsp:%s is processed, code:0x%0X", pMsg->handle, taosMsg[msgType], pMsg->code & 0XFFFF);
dTrace("RPC %p, rsp:%s is processed, code:0x%x", pMsg->handle, taosMsg[msgType], pMsg->code & 0XFFFF);
} else {
dError("RPC %p, rsp:%s not processed", pMsg->handle, taosMsg[msgType]);
rpcFreeCont(pMsg->pCont);
@ -188,7 +188,7 @@ static void dndProcessRequest(void *param, SRpcMsg *pMsg, SEpSet *pEpSet) {
int32_t msgType = pMsg->msgType;
if (msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
dTrace("RPC %p, network test req, app:%p will be processed", pMsg->handle, pMsg->ahandle);
dTrace("RPC %p, network test req, app:%p will be processed, code:0x%x", pMsg->handle, pMsg->ahandle, pMsg->code);
dndProcessDnodeReq(pDnode, pMsg, pEpSet);
return;
}

2
source/dnode/mgmt/impl/test/CMakeLists.txt
View File

@ -7,7 +7,7 @@ add_subdirectory(cluster)
add_subdirectory(db)
add_subdirectory(dnode)
# add_subdirectory(func)
# add_subdirectory(mnode)
add_subdirectory(mnode)
add_subdirectory(profile)
add_subdirectory(show)
add_subdirectory(stb)

11
source/dnode/mgmt/impl/test/mnode/CMakeLists.txt Normal file
View File

@ -0,0 +1,11 @@
aux_source_directory(. MTEST_SRC)
add_executable(dnode_test_mnode ${MTEST_SRC})
target_link_libraries(
dnode_test_mnode
PUBLIC sut
)
add_test(
NAME dnode_test_mnode
COMMAND dnode_test_mnode
)

300
source/dnode/mgmt/impl/test/mnode/mnode.cpp Normal file
View File

@ -0,0 +1,300 @@
/**
* @file dnode.cpp
* @author slguan (slguan@taosdata.com)
* @brief DNODE module dnode-msg tests
* @version 0.1
* @date 2021-12-15
*
* @copyright Copyright (c) 2021
*
*/
#include "base.h"
class DndTestMnode : public ::testing::Test {
public:
void SetUp() override {}
void TearDown() override {}
public:
static void SetUpTestSuite() {
test.Init("/tmp/dnode_test_mnode1", 9061);
const char* fqdn = "localhost";
const char* firstEp = "localhost:9061";
server2.Start("/tmp/dnode_test_mnode2", fqdn, 9062, firstEp);
server3.Start("/tmp/dnode_test_mnode3", fqdn, 9063, firstEp);
server4.Start("/tmp/dnode_test_mnode4", fqdn, 9064, firstEp);
server5.Start("/tmp/dnode_test_mnode5", fqdn, 9065, firstEp);
taosMsleep(300);
}
static void TearDownTestSuite() {
server2.Stop();
server3.Stop();
server4.Stop();
server5.Stop();
test.Cleanup();
}
static Testbase test;
static TestServer server2;
static TestServer server3;
static TestServer server4;
static TestServer server5;
};
Testbase DndTestMnode::test;
TestServer DndTestMnode::server2;
TestServer DndTestMnode::server3;
TestServer DndTestMnode::server4;
TestServer DndTestMnode::server5;
TEST_F(DndTestMnode, 01_ShowDnode) {
test.SendShowMetaMsg(TSDB_MGMT_TABLE_MNODE, "");
CHECK_META("show mnodes", 5);
CHECK_SCHEMA(0, TSDB_DATA_TYPE_SMALLINT, 2, "id");
CHECK_SCHEMA(1, TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN + VARSTR_HEADER_SIZE, "endpoint");
CHECK_SCHEMA(2, TSDB_DATA_TYPE_BINARY, 12 + VARSTR_HEADER_SIZE, "role");
CHECK_SCHEMA(3, TSDB_DATA_TYPE_TIMESTAMP, 8, "role_time");
CHECK_SCHEMA(4, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time");
test.SendShowRetrieveMsg();
EXPECT_EQ(test.GetShowRows(), 1);
CheckInt16(1);
CheckBinary("localhost:9061", TSDB_EP_LEN);
CheckBinary("master", 12);
CheckInt64(0);
CheckTimestamp();
}
TEST_F(DndTestMnode, 02_Create_Mnode_Invalid_Id) {
{
int32_t contLen = sizeof(SCreateMnodeMsg);
SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen);
pReq->dnodeId = htonl(1);
SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_MNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, TSDB_CODE_MND_MNODE_ALREADY_EXIST);
}
}
TEST_F(DndTestMnode, 03_Create_Mnode_Invalid_Id) {
{
int32_t contLen = sizeof(SCreateMnodeMsg);
SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen);
pReq->dnodeId = htonl(2);
SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_MNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, TSDB_CODE_MND_DNODE_NOT_EXIST);
}
}
TEST_F(DndTestMnode, 04_Create_Mnode) {
{
// create dnode
int32_t contLen = sizeof(SCreateDnodeMsg);
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
strcpy(pReq->ep, "localhost:9062");
SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
taosMsleep(1300);
test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "");
test.SendShowRetrieveMsg();
EXPECT_EQ(test.GetShowRows(), 2);
}
{
// create mnode
int32_t contLen = sizeof(SCreateMnodeMsg);
SCreateMnodeMsg* pReq = (SCreateMnodeMsg*)rpcMallocCont(contLen);
pReq->dnodeId = htonl(2);
SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_MNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
test.SendShowMetaMsg(TSDB_MGMT_TABLE_MNODE, "");
test.SendShowRetrieveMsg();
EXPECT_EQ(test.GetShowRows(), 2);
CheckInt16(1);
CheckInt16(2);
CheckBinary("localhost:9061", TSDB_EP_LEN);
CheckBinary("localhost:9062", TSDB_EP_LEN);
CheckBinary("master", 12);
CheckBinary("slave", 12);
CheckInt64(0);
CheckInt64(0);
CheckTimestamp();
CheckTimestamp();
}
{
// drop mnode
int32_t contLen = sizeof(SDropMnodeMsg);
SDropMnodeMsg* pReq = (SDropMnodeMsg*)rpcMallocCont(contLen);
pReq->dnodeId = htonl(2);
SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_MNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);
ASSERT_EQ(pMsg->code, 0);
test.SendShowMetaMsg(TSDB_MGMT_TABLE_MNODE, "");
test.SendShowRetrieveMsg();
EXPECT_EQ(test.GetShowRows(), 1);
CheckInt16(1);
CheckBinary("localhost:9061", TSDB_EP_LEN);
CheckBinary("master", 12);
CheckInt64(0);
CheckTimestamp();
}
}
// {
// int32_t contLen = sizeof(SDropDnodeMsg);
// SDropDnodeMsg* pReq = (SDropDnodeMsg*)rpcMallocCont(contLen);
// pReq->dnodeId = htonl(2);
// SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_DROP_DNODE, pReq, contLen);
// ASSERT_NE(pMsg, nullptr);
// ASSERT_EQ(pMsg->code, 0);
// }
// test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "");
// CHECK_META("show dnodes", 7);
// test.SendShowRetrieveMsg();
// EXPECT_EQ(test.GetShowRows(), 1);
// CheckInt16(1);
// CheckBinary("localhost:9061", TSDB_EP_LEN);
// CheckInt16(0);
// CheckInt16(1);
// CheckBinary("ready", 10);
// CheckTimestamp();
// CheckBinary("", 24);
// {
// int32_t contLen = sizeof(SCreateDnodeMsg);
// SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
// strcpy(pReq->ep, "localhost:9063");
// SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen);
// ASSERT_NE(pMsg, nullptr);
// ASSERT_EQ(pMsg->code, 0);
// }
// {
// int32_t contLen = sizeof(SCreateDnodeMsg);
// SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
// strcpy(pReq->ep, "localhost:9064");
// SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen);
// ASSERT_NE(pMsg, nullptr);
// ASSERT_EQ(pMsg->code, 0);
// }
// {
// int32_t contLen = sizeof(SCreateDnodeMsg);
// SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
// strcpy(pReq->ep, "localhost:9065");
// SRpcMsg* pMsg = test.SendMsg(TSDB_MSG_TYPE_CREATE_DNODE, pReq, contLen);
// ASSERT_NE(pMsg, nullptr);
// ASSERT_EQ(pMsg->code, 0);
// }
// taosMsleep(1300);
// test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "");
// CHECK_META("show dnodes", 7);
// test.SendShowRetrieveMsg();
// EXPECT_EQ(test.GetShowRows(), 4);
// CheckInt16(1);
// CheckInt16(3);
// CheckInt16(4);
// CheckInt16(5);
// CheckBinary("localhost:9061", TSDB_EP_LEN);
// CheckBinary("localhost:9063", TSDB_EP_LEN);
// CheckBinary("localhost:9064", TSDB_EP_LEN);
// CheckBinary("localhost:9065", TSDB_EP_LEN);
// CheckInt16(0);
// CheckInt16(0);
// CheckInt16(0);
// CheckInt16(0);
// CheckInt16(1);
// CheckInt16(1);
// CheckInt16(1);
// CheckInt16(1);
// CheckBinary("ready", 10);
// CheckBinary("ready", 10);
// CheckBinary("ready", 10);
// CheckBinary("ready", 10);
// CheckTimestamp();
// CheckTimestamp();
// CheckTimestamp();
// CheckTimestamp();
// CheckBinary("", 24);
// CheckBinary("", 24);
// CheckBinary("", 24);
// CheckBinary("", 24);
// // restart
// uInfo("stop all server");
// test.Restart();
// server2.Restart();
// server3.Restart();
// server4.Restart();
// server5.Restart();
// taosMsleep(1300);
// test.SendShowMetaMsg(TSDB_MGMT_TABLE_DNODE, "");
// CHECK_META("show dnodes", 7);
// test.SendShowRetrieveMsg();
// EXPECT_EQ(test.GetShowRows(), 4);
// CheckInt16(1);
// CheckInt16(3);
// CheckInt16(4);
// CheckInt16(5);
// CheckBinary("localhost:9061", TSDB_EP_LEN);
// CheckBinary("localhost:9063", TSDB_EP_LEN);
// CheckBinary("localhost:9064", TSDB_EP_LEN);
// CheckBinary("localhost:9065", TSDB_EP_LEN);
// CheckInt16(0);
// CheckInt16(0);
// CheckInt16(0);
// CheckInt16(0);
// CheckInt16(1);
// CheckInt16(1);
// CheckInt16(1);
// CheckInt16(1);
// CheckBinary("ready", 10);
// CheckBinary("ready", 10);
// CheckBinary("ready", 10);
// CheckBinary("ready", 10);
// CheckTimestamp();
// CheckTimestamp();
// CheckTimestamp();
// CheckTimestamp();
// CheckBinary("", 24);
// CheckBinary("", 24);
// CheckBinary("", 24);
// CheckBinary("", 24);
// }

5
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -290,11 +290,6 @@ typedef struct SMnodeMsg {
char db[TSDB_FULL_DB_NAME_LEN];
int32_t acctId;
SMnode *pMnode;
int16_t received;
int16_t successed;
int16_t expected;
int16_t retry;
int32_t code;
int64_t createdTime;
SRpcMsg rpcMsg;
int32_t contLen;

1
source/dnode/mnode/impl/inc/mndMnode.h
View File

@ -27,6 +27,7 @@ void mndCleanupMnode(SMnode *pMnode);
bool mndIsMnode(SMnode *pMnode, int32_t dnodeId);
void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet);
char *mndGetRoleStr(int32_t role);
void mndUpdateMnodeRole(SMnode *pMnode);
#ifdef __cplusplus
}

8
source/dnode/mnode/impl/src/mndDb.c
View File

@ -828,9 +828,9 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SSyncDbMsg *pSync = pMsg->rpcMsg.pCont;
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
SDbObj *pDb = mndAcquireDb(pMnode, pSync->db);
if (pDb == NULL) {
mError("db:%s, failed to process sync db msg since %s", pMsg->db, terrstr());
mError("db:%s, failed to process sync db msg since %s", pSync->db, terrstr());
return -1;
}
@ -841,9 +841,9 @@ static int32_t mndProcessSyncDbMsg(SMnodeMsg *pMsg) {
static int32_t mndProcessCompactDbMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SCompactDbMsg *pCompact = pMsg->rpcMsg.pCont;
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
SDbObj *pDb = mndAcquireDb(pMnode, pCompact->db);
if (pDb == NULL) {
mError("db:%s, failed to process compact db msg since %s", pMsg->db, terrstr());
mError("db:%s, failed to process compact db msg since %s", pCompact->db, terrstr());
return -1;
}

534
source/dnode/mnode/impl/src/mndMnode.c
View File

@ -23,14 +23,15 @@
#define TSDB_MNODE_RESERVE_SIZE 64
static int32_t mndCreateDefaultMnode(SMnode *pMnode);
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj);
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj);
static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw);
static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pMnodeObj);
static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj);
static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj);
static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj);
static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj *pNewMnode);
static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg);
static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg);
static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pMsg);
static int32_t mndProcessDropMnodeReq(SMnodeMsg *pMsg);
static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pMsg);
static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pMsg);
static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pMsg);
static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta);
static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows);
@ -46,9 +47,10 @@ int32_t mndInitMnode(SMnode *pMnode) {
.updateFp = (SdbUpdateFp)mndMnodeActionUpdate,
.deleteFp = (SdbDeleteFp)mndMnodeActionDelete};
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE, mndProcessCreateMnodeMsg);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE, mndProcessDropMnodeMsg);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE, mndProcessCreateMnodeReq);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE, mndProcessDropMnodeReq);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_MNODE_IN_RSP, mndProcessCreateMnodeRsp);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_MNODE_IN_RSP, mndProcessAlterMnodeRsp);
mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_MNODE_IN_RSP, mndProcessDropMnodeRsp);
mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_MNODE, mndGetMnodeMeta);
@ -69,9 +71,9 @@ static SMnodeObj *mndAcquireMnode(SMnode *pMnode, int32_t mnodeId) {
return pObj;
}
static void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pMnodeObj) {
static void mndReleaseMnode(SMnode *pMnode, SMnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
sdbRelease(pSdb, pMnodeObj);
sdbRelease(pSdb, pObj);
}
char *mndGetRoleStr(int32_t showType) {
@ -87,6 +89,24 @@ char *mndGetRoleStr(int32_t showType) {
}
}
void mndUpdateMnodeRole(SMnode *pMnode) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
while (1) {
SMnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
if (pObj->id == 1) {
pObj->role = TAOS_SYNC_STATE_LEADER;
} else {
pObj->role = TAOS_SYNC_STATE_CANDIDATE;
}
sdbRelease(pSdb, pObj);
}
}
static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
SMnodeObj mnodeObj = {0};
mnodeObj.id = 1;
@ -101,14 +121,14 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) {
return sdbWrite(pMnode->pSdb, pRaw);
}
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pMnodeObj) {
static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) {
SSdbRaw *pRaw = sdbAllocRaw(SDB_MNODE, TSDB_MNODE_VER_NUMBER, sizeof(SMnodeObj) + TSDB_MNODE_RESERVE_SIZE);
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
SDB_SET_INT32(pRaw, dataPos, pMnodeObj->id);
SDB_SET_INT64(pRaw, dataPos, pMnodeObj->createdTime)
SDB_SET_INT64(pRaw, dataPos, pMnodeObj->updateTime)
SDB_SET_INT32(pRaw, dataPos, pObj->id);
SDB_SET_INT64(pRaw, dataPos, pObj->createdTime)
SDB_SET_INT64(pRaw, dataPos, pObj->updateTime)
SDB_SET_RESERVE(pRaw, dataPos, TSDB_MNODE_RESERVE_SIZE)
return pRaw;
@ -125,42 +145,38 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) {
}
SSdbRow *pRow = sdbAllocRow(sizeof(SMnodeObj));
SMnodeObj *pMnodeObj = sdbGetRowObj(pRow);
if (pMnodeObj == NULL) return NULL;
SMnodeObj *pObj = sdbGetRowObj(pRow);
if (pObj == NULL) return NULL;
int32_t dataPos = 0;
SDB_GET_INT32(pRaw, pRow, dataPos, &pMnodeObj->id)
SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pMnodeObj->updateTime)
SDB_GET_INT32(pRaw, pRow, dataPos, &pObj->id)
SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pObj->updateTime)
SDB_GET_RESERVE(pRaw, pRow, dataPos, TSDB_MNODE_RESERVE_SIZE)
return pRow;
}
static void mnodeResetMnode(SMnodeObj *pMnodeObj) {
pMnodeObj->role = TAOS_SYNC_STATE_FOLLOWER;
pMnodeObj->roleTerm = 0;
pMnodeObj->roleTime = 0;
}
static void mnodeResetMnode(SMnodeObj *pObj) { pObj->role = TAOS_SYNC_STATE_FOLLOWER; }
static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pMnodeObj) {
mTrace("mnode:%d, perform insert action", pMnodeObj->id);
pMnodeObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pMnodeObj->id);
if (pMnodeObj->pDnode == NULL) {
static int32_t mndMnodeActionInsert(SSdb *pSdb, SMnodeObj *pObj) {
mTrace("mnode:%d, perform insert action", pObj->id);
pObj->pDnode = sdbAcquire(pSdb, SDB_DNODE, &pObj->id);
if (pObj->pDnode == NULL) {
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
mError("mnode:%d, failed to perform insert action since %s", pMnodeObj->id, terrstr());
mError("mnode:%d, failed to perform insert action since %s", pObj->id, terrstr());
return -1;
}
mnodeResetMnode(pMnodeObj);
mnodeResetMnode(pObj);
return 0;
}
static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj) {
mTrace("mnode:%d, perform delete action", pMnodeObj->id);
if (pMnodeObj->pDnode != NULL) {
sdbRelease(pSdb, pMnodeObj->pDnode);
pMnodeObj->pDnode = NULL;
static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) {
mTrace("mnode:%d, perform delete action", pObj->id);
if (pObj->pDnode != NULL) {
sdbRelease(pSdb, pObj->pDnode);
pObj->pDnode = NULL;
}
return 0;
@ -168,8 +184,6 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pMnodeObj) {
static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj *pNewMnode) {
mTrace("mnode:%d, perform update action", pOldMnode->id);
pOldMnode->id = pNewMnode->id;
pOldMnode->createdTime = pNewMnode->createdTime;
pOldMnode->updateTime = pNewMnode->updateTime;
return 0;
}
@ -177,12 +191,12 @@ static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOldMnode, SMnodeObj
bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) {
SSdb *pSdb = pMnode->pSdb;
SMnodeObj *pMnodeObj = sdbAcquire(pSdb, SDB_MNODE, &dnodeId);
if (pMnodeObj == NULL) {
SMnodeObj *pObj = sdbAcquire(pSdb, SDB_MNODE, &dnodeId);
if (pObj == NULL) {
return false;
}
sdbRelease(pSdb, pMnodeObj);
sdbRelease(pSdb, pObj);
return true;
}
@ -193,14 +207,14 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
void *pIter = NULL;
while (1) {
SMnodeObj *pMnodeObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMnodeObj);
SMnodeObj *pObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
if (pMnodeObj->pDnode == NULL) break;
if (pObj->pDnode == NULL) break;
pEpSet->port[pEpSet->numOfEps] = htons(pMnodeObj->pDnode->port);
tstrncpy(pEpSet->fqdn[pEpSet->numOfEps], pMnodeObj->pDnode->fqdn, TSDB_FQDN_LEN);
if (pMnodeObj->role == TAOS_SYNC_STATE_LEADER) {
pEpSet->port[pEpSet->numOfEps] = htons(pObj->pDnode->port);
memcpy(pEpSet->fqdn[pEpSet->numOfEps], pObj->pDnode->fqdn, TSDB_FQDN_LEN);
if (pObj->role == TAOS_SYNC_STATE_LEADER) {
pEpSet->inUse = pEpSet->numOfEps;
}
@ -208,54 +222,153 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {
}
}
static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateMnodeMsg *pCreate) {
SMnodeObj mnodeObj = {0};
mnodeObj.id = 1; // todo
mnodeObj.createdTime = taosGetTimestampMs();
mnodeObj.updateTime = mnodeObj.createdTime;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("dnode:%d, failed to create since %s", pCreate->dnodeId, terrstr());
return -1;
}
mDebug("trans:%d, used to create dnode:%d", pTrans->id, pCreate->dnodeId);
SSdbRaw *pRedoRaw = mndMnodeActionEncode(&mnodeObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING);
SSdbRaw *pUndoRaw = mndMnodeActionEncode(&mnodeObj);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED);
SSdbRaw *pCommitRaw = mndMnodeActionEncode(&mnodeObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
static int32_t mndSetCreateMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1;
return 0;
}
static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) {
static int32_t mndSetCreateMnodeUndoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
SSdbRaw *pUndoRaw = mndMnodeActionEncode(pObj);
if (pUndoRaw == NULL) return -1;
if (mndTransAppendUndolog(pTrans, pUndoRaw) != 0) return -1;
if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
}
static int32_t mndSetCreateMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1;
return 0;
}
static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int32_t numOfReplicas = 0;
SCreateMnodeInMsg createMsg = {0};
while (1) {
SMnodeObj *pMObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
SReplica *pReplica = &createMsg.replicas[numOfReplicas];
pReplica->id = htonl(pMObj->id);
pReplica->port = htons(pMObj->pDnode->port);
memcpy(pReplica->fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
numOfReplicas++;
sdbRelease(pSdb, pMObj);
}
SReplica *pReplica = &createMsg.replicas[numOfReplicas];
pReplica->id = htonl(pDnode->id);
pReplica->port = htons(pDnode->port);
memcpy(pReplica->fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
numOfReplicas++;
createMsg.replica = numOfReplicas;
while (1) {
SMnodeObj *pMObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
STransAction action = {0};
SAlterMnodeInMsg *pMsg = malloc(sizeof(SAlterMnodeInMsg));
if (pMsg == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pMObj);
return -1;
}
memcpy(pMsg, &createMsg, sizeof(SAlterMnodeInMsg));
pMsg->dnodeId = htonl(pMObj->id);
action.epSet = mndGetDnodeEpset(pMObj->pDnode);
action.pCont = pMsg;
action.contLen = sizeof(SAlterMnodeInMsg);
action.msgType = TSDB_MSG_TYPE_ALTER_MNODE_IN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMsg);
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pMObj);
return -1;
}
sdbRelease(pSdb, pMObj);
}
{
STransAction action = {0};
action.epSet = mndGetDnodeEpset(pDnode);
SCreateMnodeInMsg *pMsg = malloc(sizeof(SCreateMnodeInMsg));
if (pMsg == NULL) return -1;
memcpy(pMsg, &createMsg, sizeof(SAlterMnodeInMsg));
pMsg->dnodeId = htonl(pObj->id);
action.epSet = mndGetDnodeEpset(pDnode);
action.pCont = pMsg;
action.contLen = sizeof(SCreateMnodeInMsg);
action.msgType = TSDB_MSG_TYPE_CREATE_MNODE_IN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMsg);
return -1;
}
}
return 0;
}
static int32_t mndCreateMnode(SMnode *pMnode, SMnodeMsg *pMsg, SDnodeObj *pDnode, SCreateMnodeMsg *pCreate) {
SMnodeObj mnodeObj = {0};
mnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_MNODE);
mnodeObj.createdTime = taosGetTimestampMs();
mnodeObj.updateTime = mnodeObj.createdTime;
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("mnode:%d, failed to create since %s", pCreate->dnodeId, terrstr());
goto CREATE_MNODE_OVER;
}
mDebug("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId);
if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto CREATE_MNODE_OVER;
}
if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto CREATE_MNODE_OVER;
}
if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto CREATE_MNODE_OVER;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
goto CREATE_MNODE_OVER;
}
code = 0;
CREATE_MNODE_OVER:
mndTransDrop(pTrans);
return code;
}
static int32_t mndProcessCreateMnodeReq(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SCreateMnodeMsg *pCreate = pMsg->rpcMsg.pCont;
@ -263,22 +376,23 @@ static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) {
mDebug("mnode:%d, start to create", pCreate->dnodeId);
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId);
if (pDnode == NULL) {
mError("mnode:%d, dnode not exist", pDnode->id);
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
return -1;
}
mndReleaseDnode(pMnode, pDnode);
SMnodeObj *pMnodeObj = mndAcquireMnode(pMnode, pCreate->dnodeId);
if (pMnodeObj != NULL) {
mError("mnode:%d, mnode already exist", pMnodeObj->id);
SMnodeObj *pObj = mndAcquireMnode(pMnode, pCreate->dnodeId);
if (pObj != NULL) {
mndReleaseMnode(pMnode, pObj);
mError("mnode:%d, mnode already exist", pObj->id);
terrno = TSDB_CODE_MND_MNODE_ALREADY_EXIST;
return -1;
}
int32_t code = mndCreateMnode(pMnode, pMsg, pCreate);
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pCreate->dnodeId);
if (pDnode == NULL) {
mError("mnode:%d, dnode not exist", pCreate->dnodeId);
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
return -1;
}
int32_t code = mndCreateMnode(pMnode, pMsg, pDnode, pCreate);
mndReleaseDnode(pMnode, pDnode);
if (code != 0) {
mError("mnode:%d, failed to create since %s", pCreate->dnodeId, terrstr());
@ -288,49 +402,140 @@ static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pMsg, SMnodeObj *pMnodeObj) {
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("mnode:%d, failed to drop since %s", pMnodeObj->id, terrstr());
return -1;
}
mDebug("trans:%d, used to drop user:%d", pTrans->id, pMnodeObj->id);
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pMnodeObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING);
SSdbRaw *pUndoRaw = mndMnodeActionEncode(pMnodeObj);
if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) {
mError("trans:%d, failed to append undo log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY);
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pMnodeObj);
if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) {
mError("trans:%d, failed to append commit log since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
return -1;
}
mndTransDrop(pTrans);
static int32_t mndSetDropMnodeRedoLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
SSdbRaw *pRedoRaw = mndMnodeActionEncode(pObj);
if (pRedoRaw == NULL) return -1;
if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1;
if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1;
return 0;
}
static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) {
static int32_t mndSetDropMnodeCommitLogs(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj) {
SSdbRaw *pCommitRaw = mndMnodeActionEncode(pObj);
if (pCommitRaw == NULL) return -1;
if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1;
if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1;
return 0;
}
static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) {
SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL;
int32_t numOfReplicas = 0;
SAlterMnodeInMsg alterMsg = {0};
while (1) {
SMnodeObj *pMObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
if (pMObj->id != pObj->id) {
SReplica *pReplica = &alterMsg.replicas[numOfReplicas];
pReplica->id = htonl(pMObj->id);
pReplica->port = htons(pMObj->pDnode->port);
memcpy(pReplica->fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN);
numOfReplicas++;
}
sdbRelease(pSdb, pMObj);
}
alterMsg.replica = numOfReplicas;
while (1) {
SMnodeObj *pMObj = NULL;
pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj);
if (pIter == NULL) break;
if (pMObj->id != pObj->id) {
STransAction action = {0};
SAlterMnodeInMsg *pMsg = malloc(sizeof(SAlterMnodeInMsg));
if (pMsg == NULL) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pMObj);
return -1;
}
memcpy(pMsg, &alterMsg, sizeof(SAlterMnodeInMsg));
pMsg->dnodeId = htonl(pMObj->id);
action.epSet = mndGetDnodeEpset(pMObj->pDnode);
action.pCont = pMsg;
action.contLen = sizeof(SAlterMnodeInMsg);
action.msgType = TSDB_MSG_TYPE_ALTER_MNODE_IN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMsg);
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pMObj);
return -1;
}
}
sdbRelease(pSdb, pMObj);
}
{
STransAction action = {0};
action.epSet = mndGetDnodeEpset(pDnode);
SDropMnodeInMsg *pMsg = malloc(sizeof(SDropMnodeInMsg));
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pMsg->dnodeId = htonl(pObj->id);
action.epSet = mndGetDnodeEpset(pDnode);
action.pCont = pMsg;
action.contLen = sizeof(SDropMnodeInMsg);
action.msgType = TSDB_MSG_TYPE_DROP_MNODE_IN;
if (mndTransAppendRedoAction(pTrans, &action) != 0) {
free(pMsg);
return -1;
}
}
return 0;
}
static int32_t mndDropMnode(SMnode *pMnode, SMnodeMsg *pMsg, SMnodeObj *pObj) {
int32_t code = -1;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, pMsg->rpcMsg.handle);
if (pTrans == NULL) {
mError("mnode:%d, failed to drop since %s", pObj->id, terrstr());
goto DROP_MNODE_OVER;
}
mDebug("trans:%d, used to drop mnode:%d", pTrans->id, pObj->id);
if (mndSetDropMnodeRedoLogs(pMnode, pTrans, pObj) != 0) {
mError("trans:%d, failed to set redo log since %s", pTrans->id, terrstr());
goto DROP_MNODE_OVER;
}
if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) {
mError("trans:%d, failed to set commit log since %s", pTrans->id, terrstr());
goto DROP_MNODE_OVER;
}
if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj) != 0) {
mError("trans:%d, failed to set redo actions since %s", pTrans->id, terrstr());
goto DROP_MNODE_OVER;
}
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
goto DROP_MNODE_OVER;
}
code = 0;
DROP_MNODE_OVER:
mndTransDrop(pTrans);
return code;
}
static int32_t mndProcessDropMnodeReq(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SDropMnodeMsg *pDrop = pMsg->rpcMsg.pCont;
pDrop->dnodeId = htonl(pDrop->dnodeId);
@ -343,14 +548,14 @@ static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) {
return -1;
}
SMnodeObj *pMnodeObj = mndAcquireMnode(pMnode, pDrop->dnodeId);
if (pMnodeObj == NULL) {
SMnodeObj *pObj = mndAcquireMnode(pMnode, pDrop->dnodeId);
if (pObj == NULL) {
mError("mnode:%d, not exist", pDrop->dnodeId);
terrno = TSDB_CODE_MND_DNODE_NOT_EXIST;
return -1;
}
int32_t code = mndDropMnode(pMnode, pMsg, pMnodeObj);
int32_t code = mndDropMnode(pMnode, pMsg, pObj);
if (code != 0) {
mError("mnode:%d, failed to drop since %s", pMnode->dnodeId, terrstr());
@ -361,9 +566,20 @@ static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessCreateMnodeRsp(SMnodeMsg *pMsg) {
mndTransHandleActionRsp(pMsg);
return 0;
}
static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pMsg) { return 0; }
static int32_t mndProcessAlterMnodeRsp(SMnodeMsg *pMsg) {
mndTransHandleActionRsp(pMsg);
return 0;
}
static int32_t mndProcessDropMnodeRsp(SMnodeMsg *pMsg) {
mndTransHandleActionRsp(pMsg);
return 0;
}
static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) {
SMnode *pMnode = pMsg->pMnode;
@ -414,6 +630,7 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *
pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
strcpy(pMeta->tbFname, mndShowStr(pShow->type));
mndUpdateMnodeRole(pMnode);
return 0;
}
@ -422,46 +639,39 @@ static int32_t mndRetrieveMnodes(SMnodeMsg *pMsg, SShowObj *pShow, char *data, i
SSdb *pSdb = pMnode->pSdb;
int32_t numOfRows = 0;
int32_t cols = 0;
SMnodeObj *pMnodeObj = NULL;
SMnodeObj *pObj = NULL;
char *pWrite;
while (numOfRows < rows) {
pShow->pIter = sdbFetch(pSdb, SDB_MNODE, pShow->pIter, (void **)&pMnodeObj);
pShow->pIter = sdbFetch(pSdb, SDB_MNODE, pShow->pIter, (void **)&pObj);
if (pShow->pIter == NULL) break;
cols = 0;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int16_t *)pWrite = pMnodeObj->id;
*(int16_t *)pWrite = pObj->id;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
SDnodeObj *pDnode = mndAcquireDnode(pMnode, pMnodeObj->id);
if (pDnode != NULL) {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->ep, pShow->bytes[cols]);
} else {
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, "invalid ep", pShow->bytes[cols]);
}
mndReleaseDnode(pMnode, pDnode);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pObj->pDnode->ep, pShow->bytes[cols]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
char *roles = mndGetRoleStr(pMnodeObj->role);
char *roles = mndGetRoleStr(pObj->role);
STR_WITH_MAXSIZE_TO_VARSTR(pWrite, roles, pShow->bytes[cols]);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pMnodeObj->roleTime;
*(int64_t *)pWrite = pObj->roleTime;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int64_t *)pWrite = pMnodeObj->createdTime;
*(int64_t *)pWrite = pObj->createdTime;
cols++;
numOfRows++;
sdbRelease(pSdb, pMnodeObj);
sdbRelease(pSdb, pObj);
}
mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);

5
source/dnode/mnode/impl/src/mndSync.c
View File

@ -33,4 +33,7 @@ int32_t mndSyncPropose(SMnode *pMnode, SSdbRaw *pRaw) {
return code;
}
bool mndIsMaster(SMnode *pMnode) { return true; }
bool mndIsMaster(SMnode *pMnode) {
// pMnode->role = TAOS_SYNC_STATE_LEADER;
return true;
}

7
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -622,10 +622,10 @@ void mndTransHandleActionRsp(SMnodeMsg *pMsg) {
STransAction *pAction = taosArrayGet(pArray, action);
if (pAction != NULL) {
pAction->msgReceived = 1;
pAction->errCode = pMsg->code;
pAction->errCode = pMsg->rpcMsg.code;
}
mDebug("trans:%d, action:%d response is received, code:0x%x", transId, action, pMsg->code);
mDebug("trans:%d, action:%d response is received, code:0x%x", transId, action, pMsg->rpcMsg.code);
mndTransExecute(pMnode, pTrans);
HANDLE_ACTION_RSP_OVER:
@ -696,7 +696,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
for (int32_t action = 0; action < numOfActions; ++action) {
STransAction *pAction = taosArrayGet(pArray, action);
if (pAction == NULL) continue;
if (pAction->msgSent) continue;
if (pAction->msgReceived && pAction->errCode == 0) continue;
int64_t signature = pTrans->id;
signature = (signature << 32);
@ -736,6 +736,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA
terrno = errorCode;
return errorCode;
} else {
mDebug("trans:%d, %d of %d actions executed, code:0x%x", pTrans->id, numOfReceivedMsgs, numOfActions, errorCode);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
}

2
source/dnode/mnode/impl/src/mnode.c
View File

@ -178,8 +178,10 @@ static int32_t mndExecSteps(SMnode *pMnode) {
// (*pMnode->reportProgress)(pStep->name, "start initialize");
if ((*pStep->initFp)(pMnode) != 0) {
int32_t code = terrno;
mError("step:%s exec failed since %s, start to cleanup", pStep->name, terrstr());
mndCleanupSteps(pMnode, pos);
terrno = code;
return -1;
} else {
mDebug("step:%s is initialized", pStep->name);

35
source/dnode/mnode/sdb/src/sdb.c
View File

@ -16,6 +16,8 @@
#define _DEFAULT_SOURCE
#include "sdbInt.h"
static int32_t sdbCreateDir(SSdb *pSdb);
SSdb *sdbInit(SSdbOpt *pOption) {
mDebug("start to init sdb in %s", pOption->path);
@ -40,6 +42,11 @@ SSdb *sdbInit(SSdbOpt *pOption) {
return NULL;
}
if (sdbCreateDir(pSdb) != 0) {
sdbCleanup(pSdb);
return NULL;
}
for (ESdbType i = 0; i < SDB_MAX; ++i) {
taosInitRWLatch(&pSdb->locks[i]);
}
@ -53,8 +60,8 @@ void sdbCleanup(SSdb *pSdb) {
mDebug("start to cleanup sdb");
// if (pSdb->curVer != pSdb->lastCommitVer) {
mDebug("write sdb file for curVer:% " PRId64 " and lastVer:%" PRId64, pSdb->curVer, pSdb->lastCommitVer);
sdbWriteFile(pSdb);
mDebug("write sdb file for curVer:% " PRId64 " and lastVer:%" PRId64, pSdb->curVer, pSdb->lastCommitVer);
sdbWriteFile(pSdb);
// }
if (pSdb->currDir != NULL) {
@ -133,4 +140,26 @@ int32_t sdbSetTable(SSdb *pSdb, SSdbTable table) {
mDebug("sdb table:%d is initialized", sdbType);
return 0;
}
}
static int32_t sdbCreateDir(SSdb *pSdb) {
if (taosMkDir(pSdb->currDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->currDir, terrstr());
return -1;
}
if (taosMkDir(pSdb->syncDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->syncDir, terrstr());
return -1;
}
if (taosMkDir(pSdb->tmpDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->tmpDir, terrstr());
return -1;
}
return 0;
}

28
source/dnode/mnode/sdb/src/sdbFile.c
View File

@ -17,28 +17,6 @@
#include "sdbInt.h"
#include "tchecksum.h"
static int32_t sdbCreateDir(SSdb *pSdb) {
if (taosMkDir(pSdb->currDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->currDir, terrstr());
return -1;
}
if (taosMkDir(pSdb->syncDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->syncDir, terrstr());
return -1;
}
if (taosMkDir(pSdb->tmpDir) != 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to create dir:%s since %s", pSdb->tmpDir, terrstr());
return -1;
}
return 0;
}
static int32_t sdbRunDeployFp(SSdb *pSdb) {
mDebug("start to deploy sdb");
@ -77,7 +55,7 @@ int32_t sdbReadFile(SSdb *pSdb) {
free(pRaw);
terrno = TAOS_SYSTEM_ERROR(errno);
mError("failed to read file:%s since %s", file, terrstr());
return -1;
return 0;
}
while (1) {
@ -225,10 +203,6 @@ int32_t sdbWriteFile(SSdb *pSdb) {
}
int32_t sdbDeploy(SSdb *pSdb) {
if (sdbCreateDir(pSdb) != 0) {
return -1;
}
if (sdbRunDeployFp(pSdb) != 0) {
return -1;
}

7
source/libs/index/inc/index_tfile.h
View File

@ -42,7 +42,12 @@ typedef struct TFileHeader {
#define TFILE_HEADER_SIZE (sizeof(TFileHeader))
#define TFILE_HEADER_NO_FST (TFILE_HEADER_SIZE - sizeof(int32_t))
//#define TFILE_HADER_PRE_SIZE (sizeof(uint64_t) + sizeof(int32_t) + sizeof(int32_t))
typedef struct TFileValue {
char* colVal; // null terminated
SArray* tableId;
int32_t offset;
} TFileValue;
typedef struct TFileCacheKey {
uint64_t suid;

30
source/libs/index/src/index.c
View File

@ -41,7 +41,7 @@ static pthread_once_t isInit = PTHREAD_ONCE_INIT;
static void indexInit();
static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* term, SArray** result);
static int indexMergeCacheIntoTindex(SIndex* sIdx);
static int indexFlushCacheToTindex(SIndex* sIdx);
static void indexInterResultsDestroy(SArray* results);
static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType, SArray* finalResult);
@ -49,9 +49,7 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oTyp
int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) {
pthread_once(&isInit, indexInit);
SIndex* sIdx = calloc(1, sizeof(SIndex));
if (sIdx == NULL) {
return -1;
}
if (sIdx == NULL) { return -1; }
#ifdef USE_LUCENE
index_t* index = index_open(path);
@ -131,9 +129,7 @@ int indexPut(SIndex* index, SIndexMultiTerm* fVals, uint64_t uid) {
int32_t colId = fi->colId;
int32_t version = index->cVersion;
int ret = indexCachePut(index->cache, p, colId, version, uid);
if (ret != 0) {
return ret;
}
if (ret != 0) { return ret; }
}
#endif
@ -221,9 +217,7 @@ void indexOptsDestroy(SIndexOpts* opts){
SIndexMultiTermQuery* indexMultiTermQueryCreate(EIndexOperatorType opera) {
SIndexMultiTermQuery* p = (SIndexMultiTermQuery*)malloc(sizeof(SIndexMultiTermQuery));
if (p == NULL) {
return NULL;
}
if (p == NULL) { return NULL; }
p->opera = opera;
p->query = taosArrayInit(4, sizeof(SIndexTermQuery));
return p;
@ -250,9 +244,7 @@ SIndexTerm* indexTermCreate(int64_t suid,
const char* colVal,
int32_t nColVal) {
SIndexTerm* t = (SIndexTerm*)calloc(1, (sizeof(SIndexTerm)));
if (t == NULL) {
return NULL;
}
if (t == NULL) { return NULL; }
t->suid = suid;
t->operType = oper;
@ -332,9 +324,7 @@ static int indexTermSearch(SIndex* sIdx, SIndexTermQuery* query, SArray** result
return 0;
}
static void indexInterResultsDestroy(SArray* results) {
if (results == NULL) {
return;
}
if (results == NULL) { return; }
size_t sz = taosArrayGetSize(results);
for (size_t i = 0; i < sz; i++) {
@ -363,10 +353,10 @@ static int indexMergeFinalResults(SArray* interResults, EIndexOperatorType oType
}
return 0;
}
static int indexMergeCacheIntoTindex(SIndex* sIdx) {
if (sIdx == NULL) {
return -1;
}
static int indexFlushCacheToTindex(SIndex* sIdx) {
if (sIdx == NULL) { return -1; }
indexWarn("suid %" PRIu64 " merge cache into tindex", sIdx->suid);
return 0;
}

3
source/libs/index/src/index_cache.c
View File

@ -151,8 +151,7 @@ int indexCacheSearch(void* cache, SIndexTermQuery* query, int16_t colId, int32_t
EIndexQueryType qtype = query->qType;
int32_t keyLen = CACHE_KEY_LEN(term);
char* buf = calloc(1, keyLen);
char* buf = calloc(1, keyLen);
if (qtype == QUERY_TERM) {
//
} else if (qtype == QUERY_PREFIX) {

5
source/libs/index/src/index_fst_counting_writer.c
View File

@ -69,9 +69,9 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
// ugly code, refactor later
ctx->file.readOnly = readOnly;
if (readOnly == false) {
ctx->file.fd = tfOpenCreateWriteAppend(tmpFile);
ctx->file.fd = tfOpenCreateWriteAppend(path);
} else {
ctx->file.fd = tfOpenReadWrite(tmpFile);
ctx->file.fd = tfOpenReadWrite(path);
}
if (ctx->file.fd < 0) {
indexError("open file error %d", errno);
@ -93,6 +93,7 @@ WriterCtx* writerCtxCreate(WriterType type, const char* path, bool readOnly, int
END:
if (ctx->type == TMemory) { free(ctx->mem.buf); }
free(ctx);
return NULL;
}
void writerCtxDestroy(WriterCtx* ctx) {
if (ctx->type == TMemory) {

102
source/libs/index/src/index_tfile.c
View File

@ -25,12 +25,7 @@
#define TF_TABLE_TATOAL_SIZE(sz) (sizeof(sz) + sz * sizeof(uint64_t))
typedef struct TFileValue {
char* colVal; // null terminated
SArray* tableId;
int32_t offset;
} TFileValue;
static int tfileStrCompare(const void* a, const void* b);
static int tfileValueCompare(const void* a, const void* b, const void* param);
static void tfileSerialTableIdsToBuf(char* buf, SArray* tableIds);
@ -38,17 +33,18 @@ static int tfileWriteHeader(TFileWriter* writer);
static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset);
static int tfileWriteData(TFileWriter* write, TFileValue* tval);
static int tfileReadLoadHeader(TFileReader* reader);
static int tfileReadLoadFst(TFileReader* reader);
static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result);
static void tfileReadRef(TFileReader* reader);
static void tfileReadUnRef(TFileReader* reader);
static int tfileReaderLoadHeader(TFileReader* reader);
static int tfileReaderLoadFst(TFileReader* reader);
static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result);
static void tfileReaderRef(TFileReader* reader);
static void tfileReaderUnRef(TFileReader* reader);
static int tfileGetFileList(const char* path, SArray* result);
static int tfileRmExpireFile(SArray* result);
static void tfileDestroyFileName(void* elem);
static int tfileCompare(const void* a, const void* b);
static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version);
static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version);
static void tfileSerialCacheKey(TFileCacheKey* key, char* buf);
TFileCache* tfileCacheCreate(const char* path) {
@ -74,23 +70,12 @@ TFileCache* tfileCacheCreate(const char* path) {
WriterCtx* wc = writerCtxCreate(TFile, file, true, 1024 * 64);
if (wc == NULL) {
indexError("failed to open index: %s", file);
indexError("failed to open index:%s", file);
goto End;
}
TFileReader* reader = tfileReaderCreate(wc);
if (0 != tfileReadLoadHeader(reader)) {
tfileReaderDestroy(reader);
indexError("failed to load index header, index file: %s", file);
goto End;
}
if (0 != tfileReadLoadFst(reader)) {
tfileReaderDestroy(reader);
indexError("failed to load index fst, index file: %s", file);
goto End;
}
tfileReadRef(reader);
tfileReaderRef(reader);
// loader fst and validate it
TFileHeader* header = &reader->header;
TFileCacheKey key = {.suid = header->suid, .colName = header->colName, .nColName = strlen(header->colName), .colType = header->colType};
@ -115,7 +100,7 @@ void tfileCacheDestroy(TFileCache* tcache) {
TFileReader* p = *reader;
indexInfo("drop table cache suid: %" PRIu64 ", colName: %s, colType: %d", p->header.suid, p->header.colName, p->header.colType);
tfileReadUnRef(p);
tfileReaderUnRef(p);
reader = taosHashIterate(tcache->tableCache, reader);
}
taosHashCleanup(tcache->tableCache);
@ -127,7 +112,7 @@ TFileReader* tfileCacheGet(TFileCache* tcache, TFileCacheKey* key) {
tfileSerialCacheKey(key, buf);
TFileReader* reader = taosHashGet(tcache->tableCache, buf, strlen(buf));
tfileReadRef(reader);
tfileReaderRef(reader);
return reader;
}
@ -139,10 +124,10 @@ void tfileCachePut(TFileCache* tcache, TFileCacheKey* key, TFileReader* reader)
if (*p != NULL) {
TFileReader* oldReader = *p;
taosHashRemove(tcache->tableCache, buf, strlen(buf));
tfileReadUnRef(oldReader);
tfileReaderUnRef(oldReader);
}
tfileReadRef(reader);
tfileReaderRef(reader);
taosHashPut(tcache->tableCache, buf, strlen(buf), &reader, sizeof(void*));
return;
}
@ -153,6 +138,19 @@ TFileReader* tfileReaderCreate(WriterCtx* ctx) {
// T_REF_INC(reader);
reader->ctx = ctx;
if (0 != tfileReaderLoadHeader(reader)) {
tfileReaderDestroy(reader);
indexError("failed to load index header, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName);
return NULL;
}
if (0 != tfileReaderLoadFst(reader)) {
tfileReaderDestroy(reader);
indexError("failed to load index fst, suid: %" PRIu64 ", colName: %s", reader->header.suid, reader->header.colName);
return NULL;
}
return reader;
}
void tfileReaderDestroy(TFileReader* reader) {
@ -174,7 +172,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul
FstSlice key = fstSliceCreate(term->colVal, term->nColVal);
if (fstGet(reader->fst, &key, &offset)) {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, found table info in tindex", term->suid, term->colName, term->colVal);
ret = tfileReadLoadTableIds(reader, offset, result);
ret = tfileReaderLoadTableIds(reader, offset, result);
} else {
indexInfo("index: %" PRIu64 ", col: %s, colVal: %s, not found table info in tindex", term->suid, term->colName, term->colVal);
}
@ -185,7 +183,7 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SArray* resul
} else {
// handle later
}
tfileReadUnRef(reader);
tfileReaderUnRef(reader);
return ret;
}
@ -214,11 +212,18 @@ TFileWriter* tfileWriterCreate(WriterCtx* ctx, TFileHeader* header) {
int tfileWriterPut(TFileWriter* tw, void* data) {
// sort by coltype and write to tindex
__compar_fn_t fn = getComparFunc(tw->header.colType, 0);
__compar_fn_t fn;
int8_t colType = tw->header.colType;
if (colType == TSDB_DATA_TYPE_BINARY || colType == TSDB_DATA_TYPE_NCHAR) {
fn = tfileStrCompare;
} else {
fn = getComparFunc(colType, 0);
}
taosArraySortPWithExt((SArray*)(data), tfileValueCompare, &fn);
int32_t bufLimit = 4096, offset = 0;
char* buf = calloc(1, sizeof(bufLimit));
char* buf = calloc(1, sizeof(char) * bufLimit);
char* p = buf;
int32_t sz = taosArrayGetSize((SArray*)data);
int32_t fstOffset = tw->offset;
@ -259,6 +264,11 @@ int tfileWriterPut(TFileWriter* tw, void* data) {
}
tfree(buf);
tw->fb = fstBuilderCreate(tw->ctx, 0);
if (tw->fb == NULL) {
tfileWriterDestroy(tw);
return -1;
}
// write fst
for (size_t i = 0; i < sz; i++) {
// TODO, fst batch write later
@ -267,6 +277,9 @@ int tfileWriterPut(TFileWriter* tw, void* data) {
//
}
}
fstBuilderFinish(tw->fb);
fstBuilderDestroy(tw->fb);
tw->fb = NULL;
return 0;
}
void tfileWriterDestroy(TFileWriter* tw) {
@ -305,6 +318,12 @@ int indexTFilePut(void* tfile, SIndexTerm* term, uint64_t uid) {
return 0;
}
static int tfileStrCompare(const void* a, const void* b) {
int ret = strcmp((char*)a, (char*)b);
if (ret == 0) { return ret; }
return ret < 0 ? -1 : 1;
}
static int tfileValueCompare(const void* a, const void* b, const void* param) {
__compar_fn_t fn = *(__compar_fn_t*)param;
@ -326,6 +345,7 @@ static int tfileWriteFstOffset(TFileWriter* tw, int32_t offset) {
int32_t fstOffset = offset + sizeof(tw->header.fstOffset);
tw->header.fstOffset = fstOffset;
if (sizeof(fstOffset) != tw->ctx->write(tw->ctx, (char*)&fstOffset, sizeof(fstOffset))) { return -1; }
tw->offset += sizeof(fstOffset);
return 0;
}
static int tfileWriteHeader(TFileWriter* writer) {
@ -355,16 +375,16 @@ static int tfileWriteData(TFileWriter* write, TFileValue* tval) {
}
return 0;
}
static int tfileReadLoadHeader(TFileReader* reader) {
static int tfileReaderLoadHeader(TFileReader* reader) {
// TODO simple tfile header later
char buf[TFILE_HEADER_SIZE] = {0};
int64_t nread = reader->ctx->read(reader->ctx, buf, sizeof(buf));
int64_t nread = reader->ctx->readFrom(reader->ctx, buf, sizeof(buf), 0);
assert(nread == sizeof(buf));
memcpy(&reader->header, buf, sizeof(buf));
return 0;
}
static int tfileReadLoadFst(TFileReader* reader) {
static int tfileReaderLoadFst(TFileReader* reader) {
// current load fst into memory, refactor it later
static int FST_MAX_SIZE = 16 * 1024;
@ -381,9 +401,9 @@ static int tfileReadLoadFst(TFileReader* reader) {
free(buf);
fstSliceDestroy(&st);
return reader->fst == NULL ? 0 : -1;
return reader->fst != NULL ? 0 : -1;
}
static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
static int tfileReaderLoadTableIds(TFileReader* reader, int32_t offset, SArray* result) {
int32_t nid;
WriterCtx* ctx = reader->ctx;
@ -403,12 +423,12 @@ static int tfileReadLoadTableIds(TFileReader* reader, int32_t offset, SArray* re
free(buf);
return 0;
}
static void tfileReadRef(TFileReader* reader) {
static void tfileReaderRef(TFileReader* reader) {
int ref = T_REF_INC(reader);
UNUSED(ref);
}
static void tfileReadUnRef(TFileReader* reader) {
static void tfileReaderUnRef(TFileReader* reader) {
int ref = T_REF_DEC(reader);
if (ref == 0) { tfileReaderDestroy(reader); }
}
@ -445,6 +465,10 @@ static int tfileCompare(const void* a, const void* b) {
return strncmp(aName, bName, aLen > bLen ? aLen : bLen);
}
// tfile name suid-colId-version.tindex
static void tfileGenFileName(char* filename, uint64_t suid, int colId, int version) {
sprintf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version);
return;
}
static int tfileParseFileName(const char* filename, uint64_t* suid, int* colId, int* version) {
if (3 == sscanf(filename, "%" PRIu64 "-%d-%d.tindex", suid, colId, version)) {
// read suid & colid & version success

605
source/libs/index/test/indexTests.cc
View File

@ -2,8 +2,7 @@
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
* it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
@ -13,141 +12,141 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <gtest/gtest.h>
#include <string>
#include <iostream>
#include <string>
#include "index.h"
#include "tutil.h"
#include "indexInt.h"
#include "index_fst.h"
#include "index_fst_util.h"
#include "index_fst_counting_writer.h"
#include "index_fst_util.h"
#include "index_tfile.h"
#include "tutil.h"
using namespace std;
class FstWriter {
public:
FstWriter() {
_wc = writerCtxCreate(TFile, "/tmp/tindex", false, 0);
_b = fstBuilderCreate(NULL, 0);
}
bool Put(const std::string &key, uint64_t val) {
FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size());
bool ok = fstBuilderInsert(_b, skey, val);
fstSliceDestroy(&skey);
return ok;
}
~FstWriter() {
fstBuilderFinish(_b);
fstBuilderDestroy(_b);
public:
FstWriter() {
_wc = writerCtxCreate(TFile, "/tmp/tindex", false, 64 * 1024 * 1024);
_b = fstBuilderCreate(NULL, 0);
}
bool Put(const std::string& key, uint64_t val) {
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstBuilderInsert(_b, skey, val);
fstSliceDestroy(&skey);
return ok;
}
~FstWriter() {
fstBuilderFinish(_b);
fstBuilderDestroy(_b);
writerCtxDestroy(_wc);
}
private:
FstBuilder *_b;
WriterCtx *_wc;
writerCtxDestroy(_wc);
}
private:
FstBuilder* _b;
WriterCtx* _wc;
};
class FstReadMemory {
public:
FstReadMemory(size_t size) {
_wc = writerCtxCreate(TFile, "/tmp/tindex", true, 0);
_w = fstCountingWriterCreate(_wc);
_size = size;
memset((void *)&_s, 0, sizeof(_s));
}
bool init() {
char *buf = (char *)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t *)buf, _size);
if (nRead <= 0) { return false; }
_size = nRead;
_s = fstSliceCreate((uint8_t *)buf, _size);
_fst = fstCreate(&_s);
free(buf);
return _fst != NULL;
}
bool Get(const std::string &key, uint64_t *val) {
FstSlice skey = fstSliceCreate((uint8_t *)key.c_str(), key.size());
bool ok = fstGet(_fst, &skey, val);
fstSliceDestroy(&skey);
return ok;
}
bool GetWithTimeCostUs(const std::string &key, uint64_t *val, uint64_t *elapse) {
int64_t s = taosGetTimestampUs();
bool ok = this->Get(key, val);
int64_t e = taosGetTimestampUs();
*elapse = e - s;
return ok;
}
// add later
bool Search(AutomationCtx *ctx, std::vector<uint64_t> &result) {
FstStreamBuilder *sb = fstSearch(_fst, ctx);
StreamWithState *st = streamBuilderIntoStream(sb);
StreamWithStateResult *rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
result.push_back((uint64_t)(rt->out.out));
}
return true;
}
bool SearchWithTimeCostUs(AutomationCtx *ctx, std::vector<uint64_t> &result) {
int64_t s = taosGetTimestampUs();
bool ok = this->Search(ctx, result);
int64_t e = taosGetTimestampUs();
return ok;
}
~FstReadMemory() {
public:
FstReadMemory(size_t size) {
_wc = writerCtxCreate(TFile, "/tmp/tindex", true, 64 * 1024);
_w = fstCountingWriterCreate(_wc);
_size = size;
memset((void*)&_s, 0, sizeof(_s));
}
bool init() {
char* buf = (char*)calloc(1, sizeof(char) * _size);
int nRead = fstCountingWriterRead(_w, (uint8_t*)buf, _size);
if (nRead <= 0) { return false; }
_size = nRead;
_s = fstSliceCreate((uint8_t*)buf, _size);
_fst = fstCreate(&_s);
free(buf);
return _fst != NULL;
}
bool Get(const std::string& key, uint64_t* val) {
FstSlice skey = fstSliceCreate((uint8_t*)key.c_str(), key.size());
bool ok = fstGet(_fst, &skey, val);
fstSliceDestroy(&skey);
return ok;
}
bool GetWithTimeCostUs(const std::string& key, uint64_t* val, uint64_t* elapse) {
int64_t s = taosGetTimestampUs();
bool ok = this->Get(key, val);
int64_t e = taosGetTimestampUs();
*elapse = e - s;
return ok;
}
// add later
bool Search(AutomationCtx* ctx, std::vector<uint64_t>& result) {
FstStreamBuilder* sb = fstSearch(_fst, ctx);
StreamWithState* st = streamBuilderIntoStream(sb);
StreamWithStateResult* rt = NULL;
while ((rt = streamWithStateNextWith(st, NULL)) != NULL) {
result.push_back((uint64_t)(rt->out.out));
}
return true;
}
bool SearchWithTimeCostUs(AutomationCtx* ctx, std::vector<uint64_t>& result) {
int64_t s = taosGetTimestampUs();
bool ok = this->Search(ctx, result);
int64_t e = taosGetTimestampUs();
return ok;
}
~FstReadMemory() {
fstCountingWriterDestroy(_w);
fstDestroy(_fst);
fstSliceDestroy(&_s);
writerCtxDestroy(_wc);
}
private:
FstCountingWriter *_w;
Fst *_fst;
FstSlice _s;
WriterCtx *_wc;
size_t _size;
};
}
//TEST(IndexTest, index_create_test) {
private:
FstCountingWriter* _w;
Fst* _fst;
FstSlice _s;
WriterCtx* _wc;
size_t _size;
};
// TEST(IndexTest, index_create_test) {
// SIndexOpts *opts = indexOptsCreate();
// SIndex *index = indexOpen(opts, "./test");
// if (index == NULL) {
// std::cout << "index open failed" << std::endl;
// std::cout << "index open failed" << std::endl;
// }
//
//
// // write
//
// // write
// for (int i = 0; i < 100000; i++) {
// SIndexMultiTerm* terms = indexMultiTermCreate();
// std::string val = "field";
// std::string val = "field";
//
// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size());
//
// val.append(std::to_string(i));
// val.append(std::to_string(i));
// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size());
//
// val.insert(0, std::to_string(i));
// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size());
//
// val.append("const");
// val.append("const");
// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size());
//
//
//
// indexPut(index, terms, i);
// indexMultiTermDestroy(terms);
// }
//
// }
//
//
// // query
// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
//
// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST);
//
// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX);
// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM);
//
// SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
// SArray *result = (SArray *)taosArrayInit(10, sizeof(int));
// indexSearch(index, multiQuery, result);
//
// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl;
@ -155,25 +154,24 @@ class FstReadMemory {
// int *v = (int *)taosArrayGet(result, i);
// std::cout << "value --->" << *v << std::endl;
// }
// // add more test case
// // add more test case
// indexMultiTermQueryDestroy(multiQuery);
//
// indexOptsDestroy(opts);
// indexClose(index);
// indexOptsDestroy(opts);
// indexClose(index);
// //
//}
#define L 100
#define M 100
#define N 100
int Performance_fstWriteRecords(FstWriter *b) {
std::string str("aa");
int Performance_fstWriteRecords(FstWriter* b) {
std::string str("aa");
for (int i = 0; i < L; i++) {
str[0] = 'a' + i;
str.resize(2);
for(int j = 0; j < M; j++) {
str.resize(2);
for (int j = 0; j < M; j++) {
str[1] = 'a' + j;
str.resize(2);
for (int k = 0; k < N; k++) {
@ -181,87 +179,86 @@ int Performance_fstWriteRecords(FstWriter *b) {
b->Put(str, k);
printf("(%d, %d, %d, %s)\n", i, j, k, str.c_str());
}
}
}
}
return L * M * N;
}
void Performance_fstReadRecords(FstReadMemory *m) {
void Performance_fstReadRecords(FstReadMemory* m) {
std::string str("aa");
for (int i = 0; i < M; i++) {
str[0] = 'a' + i;
str.resize(2);
for(int j = 0; j < N; j++) {
str.resize(2);
for (int j = 0; j < N; j++) {
str[1] = 'a' + j;
str.resize(2);
for (int k = 0; k < L; k++) {
str.push_back('a');
uint64_t val, cost;
uint64_t val, cost;
if (m->GetWithTimeCostUs(str, &val, &cost)) {
printf("succes to get kv(%s, %" PRId64"), cost: %" PRId64"\n", str.c_str(), val, cost);
printf("succes to get kv(%s, %" PRId64 "), cost: %" PRId64 "\n", str.c_str(), val, cost);
} else {
printf("failed to get key: %s\n", str.c_str());
}
}
}
}
}
}
void checkFstPerf() {
FstWriter *fw = new FstWriter;
int64_t s = taosGetTimestampUs();
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int num = Performance_fstWriteRecords(fw);
int num = Performance_fstWriteRecords(fw);
int64_t e = taosGetTimestampUs();
printf("write %d record cost %" PRId64"us\n", num, e - s);
printf("write %d record cost %" PRId64 "us\n", num, e - s);
delete fw;
FstReadMemory *m = new FstReadMemory(1024 * 64);
if (m->init()) {
printf("success to init fst read");
}
Performance_fstReadRecords(m);
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init()) { printf("success to init fst read"); }
Performance_fstReadRecords(m);
delete m;
}
}
void checkFstPrefixSearch() {
FstWriter *fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
FstWriter* fw = new FstWriter;
int64_t s = taosGetTimestampUs();
int count = 2;
std::string key("ab");
for (int i = 0; i < count; i++) {
key[1] = key[1] + i;
fw->Put(key, i);
key[1] = key[1] + i;
fw->Put(key, i);
}
int64_t e = taosGetTimestampUs();
std::cout << "insert data count : " << count << "elapas time: " << e - s << std::endl;
delete fw;
FstReadMemory *m = new FstReadMemory(1024 * 64);
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
// prefix search
// prefix search
std::vector<uint64_t> result;
AutomationCtx *ctx = automCtxCreate((void *)"ab", AUTOMATION_PREFIX);
m->Search(ctx, result);
assert(result.size() == count);
AutomationCtx* ctx = automCtxCreate((void*)"ab", AUTOMATION_PREFIX);
m->Search(ctx, result);
assert(result.size() == count);
for (int i = 0; i < result.size(); i++) {
assert(result[i] == i); // check result
assert(result[i] == i); // check result
}
free(ctx);
delete m;
}
}
void validateFst() {
int val = 100;
int count = 100;
FstWriter *fw = new FstWriter;
// write
int val = 100;
int count = 100;
FstWriter* fw = new FstWriter;
// write
{
std::string key("ab");
for (int i = 0; i < count; i++) {
@ -272,99 +269,255 @@ void validateFst() {
delete fw;
// read
FstReadMemory *m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
FstReadMemory* m = new FstReadMemory(1024 * 64);
if (m->init() == false) {
std::cout << "init readMemory failed" << std::endl;
delete m;
return;
}
{
std::string key("ab");
uint64_t out;
if (m->Get(key, &out)) {
printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
} else {
printf("failed to get(%s)\n", key.c_str());
}
for (int i = 0; i < count; i++) {
key.push_back('a' + i);
if (m->Get(key, &out) ) {
assert(val - i == out);
printf("success to get (%s, %" PRId64")\n", key.c_str(), out);
} else {
printf("failed to get(%s)\n", key.c_str());
std::string key("ab");
uint64_t out;
if (m->Get(key, &out)) {
printf("success to get (%s, %" PRId64 ")\n", key.c_str(), out);
} else {
printf("failed to get(%s)\n", key.c_str());
}
}
}
for (int i = 0; i < count; i++) {
key.push_back('a' + i);
if (m->Get(key, &out)) {
assert(val - i == out);
printf("success to get (%s, %" PRId64 ")\n", key.c_str(), out);
} else {
printf("failed to get(%s)\n", key.c_str());
}
}
}
delete m;
}
class IndexEnv : public ::testing::Test {
protected:
virtual void SetUp() {
taosRemoveDir(path);
opts = indexOptsCreate();
int ret = indexOpen(opts, path, &index);
assert(ret == 0);
}
virtual void TearDown() {
indexClose(index);
indexOptsDestroy(opts);
}
const char *path = "/tmp/tindex";
SIndexOpts *opts;
SIndex *index;
};
TEST_F(IndexEnv, testPut) {
// single index column
{
std::string colName("tag1"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
SIndexMultiTerm *terms = indexMultiTermCreate();
indexMultiTermAdd(terms, term);
for (size_t i = 0; i < 100; i++) {
int tableId = i;
int ret = indexPut(index, terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
// multi index column
{
SIndexMultiTerm *terms = indexMultiTermCreate();
{
std::string colName("tag1"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
}
{
std::string colName("tag2"), colVal("Hello world");
SIndexTerm *term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
indexMultiTermAdd(terms, term);
}
for (int i = 0; i < 100; i++) {
int tableId = i;
int ret = indexPut(index, terms, tableId);
assert(ret == 0);
}
indexMultiTermDestroy(terms);
}
//
}
TEST_F(IndexEnv, testDel) {
}
class IndexEnv : public ::testing::Test {
protected:
virtual void SetUp() {
taosRemoveDir(path);
opts = indexOptsCreate();
int ret = indexOpen(opts, path, &index);
assert(ret == 0);
}
virtual void TearDown() {
indexClose(index);
indexOptsDestroy(opts);
}
const char* path = "/tmp/tindex";
SIndexOpts* opts;
SIndex* index;
};
// TEST_F(IndexEnv, testPut) {
// // single index column
// {
// std::string colName("tag1"), colVal("Hello world");
// SIndexTerm* term = indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(),
// colVal.size()); SIndexMultiTerm* terms = indexMultiTermCreate(); indexMultiTermAdd(terms, term);
//
// for (size_t i = 0; i < 100; i++) {
// int tableId = i;
// int ret = indexPut(index, terms, tableId);
// assert(ret == 0);
// }
// indexMultiTermDestroy(terms);
// }
// // multi index column
// {
// SIndexMultiTerm* terms = indexMultiTermCreate();
// {
// std::string colName("tag1"), colVal("Hello world");
// SIndexTerm* term =
// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
// indexMultiTermAdd(terms, term);
// }
// {
// std::string colName("tag2"), colVal("Hello world");
// SIndexTerm* term =
// indexTermCreate(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
// indexMultiTermAdd(terms, term);
// }
//
// for (int i = 0; i < 100; i++) {
// int tableId = i;
// int ret = indexPut(index, terms, tableId);
// assert(ret == 0);
// }
// indexMultiTermDestroy(terms);
// }
// //
//}
class TFileObj {
public:
TFileObj(const std::string& path = "/tmp/tindex", const std::string& colName = "voltage") : path_(path), colName_(colName) {
colId_ = 10;
// Do Nothing
//
}
int Put(SArray* tv) {
if (reader_ != NULL) {
tfileReaderDestroy(reader_);
reader_ = NULL;
}
if (writer_ == NULL) { InitWriter(); }
return tfileWriterPut(writer_, tv);
}
bool InitWriter() {
TFileHeader header;
header.suid = 1;
header.version = 1;
memcpy(header.colName, colName_.c_str(), colName_.size());
header.colType = TSDB_DATA_TYPE_BINARY;
std::string path(path_);
int colId = 2;
char buf[64] = {0};
sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId_, header.version);
path.append("/").append(buf);
fileName_ = path;
WriterCtx* ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024);
writer_ = tfileWriterCreate(ctx, &header);
return writer_ != NULL ? true : false;
}
bool InitReader() {
WriterCtx* ctx = writerCtxCreate(TFile, fileName_.c_str(), true, 64 * 1024 * 1024);
reader_ = tfileReaderCreate(ctx);
return reader_ != NULL ? true : false;
}
int Get(SIndexTermQuery* query, SArray* result) {
if (writer_ != NULL) {
tfileWriterDestroy(writer_);
writer_ = NULL;
}
if (reader_ == NULL && InitReader()) {
//
//
}
return tfileReaderSearch(reader_, query, result);
}
~TFileObj() {
if (writer_) { tfileWriterDestroy(writer_); }
if (reader_) { tfileReaderDestroy(reader_); }
}
private:
std::string path_;
std::string colName_;
std::string fileName_;
TFileWriter* writer_;
TFileReader* reader_;
int colId_;
};
class IndexTFileEnv : public ::testing::Test {
protected:
virtual void SetUp() {
taosRemoveDir(dir.c_str());
taosMkDir(dir.c_str());
tfInit();
fObj = new TFileObj(dir, colName);
// std::string colName("voltage");
// header.suid = 1;
// header.version = 1;
// memcpy(header.colName, colName.c_str(), colName.size());
// header.colType = TSDB_DATA_TYPE_BINARY;
// std::string path(dir);
// int colId = 2;
// char buf[64] = {0};
// sprintf(buf, "%" PRIu64 "-%d-%d.tindex", header.suid, colId, header.version);
// path.append("/").append(buf);
// ctx = writerCtxCreate(TFile, path.c_str(), false, 64 * 1024 * 1024);
// twrite = tfileWriterCreate(ctx, &header);
}
virtual void TearDown() {
// indexClose(index);
// indexeptsDestroy(opts);
delete fObj;
tfCleanup();
// tfileWriterDestroy(twrite);
}
TFileObj* fObj;
std::string dir = "/tmp/tindex";
std::string colName = "voltage";
int coldId = 2;
int version = 1;
int colType = TSDB_DATA_TYPE_BINARY;
// WriterCtx* ctx = NULL;
// TFileHeader header;
// TFileWriter* twrite = NULL;
};
// static TFileWriter* genTFileWriter(const char* path, TFileHeader* header) {
// char buf[128] = {0};
// WriterCtx* ctx = writerCtxCreate(TFile, path, false, )
//}
static TFileValue* genTFileValue(const char* val) {
TFileValue* tv = (TFileValue*)calloc(1, sizeof(TFileValue));
int32_t vlen = strlen(val) + 1;
tv->colVal = (char*)calloc(1, vlen);
memcpy(tv->colVal, val, vlen);
tv->tableId = (SArray*)taosArrayInit(1, sizeof(uint64_t));
for (size_t i = 0; i < 10; i++) {
uint64_t v = i;
taosArrayPush(tv->tableId, &v);
}
return tv;
}
static void destroyTFileValue(void* val) {
TFileValue* tv = (TFileValue*)val;
free(tv->colVal);
taosArrayDestroy(tv->tableId);
free(tv);
}
TEST_F(IndexTFileEnv, test_tfile_write) {
TFileValue* v1 = genTFileValue("c");
TFileValue* v2 = genTFileValue("a");
TFileValue* v3 = genTFileValue("b");
TFileValue* v4 = genTFileValue("d");
SArray* data = (SArray*)taosArrayInit(4, sizeof(void*));
taosArrayPush(data, &v1);
taosArrayPush(data, &v2);
taosArrayPush(data, &v3);
taosArrayPush(data, &v4);
fObj->Put(data);
for (size_t i = 0; i < taosArrayGetSize(data); i++) {
destroyTFileValue(taosArrayGetP(data, i));
}
taosArrayDestroy(data);
std::string colName("voltage");
std::string colVal("b");
SIndexTerm* term = indexTermCreate(1, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), colVal.c_str(), colVal.size());
SIndexTermQuery query = {.term = term, .qType = QUERY_TERM};
SArray* result = (SArray*)taosArrayInit(1, sizeof(uint64_t));
fObj->Get(&query, result);
assert(taosArrayGetSize(result) == 10);
indexTermDestroy(term);
// tfileWriterDestroy(twrite);
}

2
source/libs/parser/inc/parserInt.h
View File

@ -68,7 +68,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQ
* @param type
* @return
*/
int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen);
int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf, int32_t msgBufLen);
/**
* Evaluate the numeric and timestamp arithmetic expression in the WHERE clause.

49
source/libs/parser/src/astValidate.c
View File

@ -4306,7 +4306,7 @@ int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) {
return TSDB_CODE_SUCCESS;
}
int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, int32_t* type, char* msgBuf, int32_t msgBufLen) {
int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStmtInfo* pDcl, char* msgBuf, int32_t msgBufLen) {
int32_t code = 0;
SMsgBuf m = {.buf = msgBuf, .len = msgBufLen};
@ -4357,8 +4357,8 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void**
}
}
*output = buildUserManipulationMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen);
*type = (pInfo->type == TSDB_SQL_CREATE_USER)? TSDB_MSG_TYPE_CREATE_USER:TSDB_MSG_TYPE_ALTER_USER;
pDcl->pMsg = (char*)buildUserManipulationMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen);
pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_USER)? TSDB_MSG_TYPE_CREATE_USER:TSDB_MSG_TYPE_ALTER_USER;
break;
}
@ -4394,21 +4394,21 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void**
}
}
*output = buildAcctManipulationMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen);
*type = (pInfo->type == TSDB_SQL_CREATE_ACCT)? TSDB_MSG_TYPE_CREATE_ACCT:TSDB_MSG_TYPE_ALTER_ACCT;
pDcl->pMsg = (char*)buildAcctManipulationMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen);
pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_ACCT)? TSDB_MSG_TYPE_CREATE_ACCT:TSDB_MSG_TYPE_ALTER_ACCT;
break;
}
case TSDB_SQL_DROP_ACCT:
case TSDB_SQL_DROP_USER: {
*output = buildDropUserMsg(pInfo, outputLen, pCtx->requestId, msgBuf, msgBufLen);
*type = (pInfo->type == TSDB_SQL_DROP_ACCT)? TSDB_MSG_TYPE_DROP_ACCT:TSDB_MSG_TYPE_DROP_USER;
pDcl->pMsg = (char*)buildDropUserMsg(pInfo, &pDcl->msgLen, pCtx->requestId, msgBuf, msgBufLen);
pDcl->msgType = (pInfo->type == TSDB_SQL_DROP_ACCT)? TSDB_MSG_TYPE_DROP_ACCT:TSDB_MSG_TYPE_DROP_USER;
break;
}
case TSDB_SQL_SHOW: {
code = setShowInfo(&pInfo->pMiscInfo->showOpt, pCtx, output, outputLen, pMsgBuf);
*type = TSDB_MSG_TYPE_SHOW;
code = setShowInfo(&pInfo->pMiscInfo->showOpt, pCtx, (void**)&pDcl->pMsg, &pDcl->msgLen, pMsgBuf);
pDcl->msgType = TSDB_MSG_TYPE_SHOW;
break;
}
@ -4429,9 +4429,9 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void**
SUseDbMsg *pUseDbMsg = (SUseDbMsg *) calloc(1, sizeof(SUseDbMsg));
tNameExtractFullName(&n, pUseDbMsg->db);
*output = pUseDbMsg;
*outputLen = sizeof(SUseDbMsg);
*type = TSDB_MSG_TYPE_USE_DB;
pDcl->pMsg = (char*)pUseDbMsg;
pDcl->msgLen = sizeof(SUseDbMsg);
pDcl->msgType = TSDB_MSG_TYPE_USE_DB;
break;
}
@ -4457,9 +4457,11 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void**
return TSDB_CODE_TSC_INVALID_OPERATION;
}
*output = pCreateMsg;
*outputLen = sizeof(SCreateDbMsg);
*type = (pInfo->type == TSDB_SQL_CREATE_DB)? TSDB_MSG_TYPE_CREATE_DB:TSDB_MSG_TYPE_ALTER_DB;
strncpy(pCreateMsg->db, token.z, token.n);
pDcl->pMsg = (char*)pCreateMsg;
pDcl->msgLen = sizeof(SCreateDbMsg);
pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB)? TSDB_MSG_TYPE_CREATE_DB:TSDB_MSG_TYPE_ALTER_DB;
break;
}
@ -4481,9 +4483,9 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void**
pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0;
assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_DB_NAME_T);
*type = TSDB_MSG_TYPE_DROP_DB;
*outputLen = sizeof(SDropDbMsg);
*output = pDropDbMsg;
pDcl->msgType = TSDB_MSG_TYPE_DROP_DB;
pDcl->msgLen = sizeof(SDropDbMsg);
pDcl->pMsg = (char*)pDropDbMsg;
return TSDB_CODE_SUCCESS;
}
@ -4494,9 +4496,8 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void**
if ((code = doCheckForCreateTable(pInfo, pMsgBuf)) != TSDB_CODE_SUCCESS) {
return code;
}
*output = buildCreateTableMsg(pCreateTable, outputLen, pCtx, pMsgBuf);
*type = (pCreateTable->type == TSQL_CREATE_TABLE)? TSDB_MSG_TYPE_CREATE_TABLE:TSDB_MSG_TYPE_CREATE_STB;
pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE)? TSDB_MSG_TYPE_CREATE_TABLE:TSDB_MSG_TYPE_CREATE_STB;
} else if (pCreateTable->type == TSQL_CREATE_CTABLE) {
// if ((code = doCheckForCreateFromStable(pSql, pInfo)) != TSDB_CODE_SUCCESS) {
// return code;
@ -4511,12 +4512,12 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, void**
}
case TSDB_SQL_DROP_TABLE: {
*output = buildDropTableMsg(pInfo, outputLen, pCtx, pMsgBuf);
if (output == NULL) {
pDcl->pMsg = (char*)buildDropTableMsg(pInfo, &pDcl->msgLen, pCtx, pMsgBuf);
if (pDcl->pMsg == NULL) {
return terrno;
}
*type = TSDB_MSG_TYPE_DROP_STB;
pDcl->msgType = TSDB_MSG_TYPE_DROP_STB;
return TSDB_CODE_SUCCESS;
break;
}

1
source/libs/parser/src/insertParser.c
View File

@ -909,6 +909,7 @@ int32_t parseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
}
*pInfo = context.pOutput;
context.pOutput->nodeType = TSDB_SQL_INSERT;
context.pOutput->schemaAttache = pContext->schemaAttached;
context.pOutput->payloadType = PAYLOAD_TYPE_KV;

34
source/libs/parser/src/parser.c
View File

@ -20,7 +20,7 @@
#include "function.h"
#include "insertParser.h"
bool qIsInsertSql(const char* pStr, size_t length) {
bool isInsertSql(const char* pStr, size_t length) {
int32_t index = 0;
do {
@ -31,18 +31,28 @@ bool qIsInsertSql(const char* pStr, size_t length) {
} while (1);
}
int32_t qParseQuerySql(const char* pStr, size_t length, SParseBasicCtx* pParseCtx, int32_t *type, void** pOutput, int32_t* outputLen, char* msg, int32_t msgLen) {
SSqlInfo info = doGenerateAST(pStr);
bool qIsDclQuery(const SQueryNode* pQuery) {
return TSDB_SQL_INSERT != pQuery->type && TSDB_SQL_SELECT != pQuery->type;
}
int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
SSqlInfo info = doGenerateAST(pCxt->pSql);
if (!info.valid) {
strncpy(msg, info.msg, msgLen);
strncpy(pCxt->pMsg, info.msg, pCxt->msgLen);
terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
return terrno;
}
if (!isDqlSqlStatement(&info)) {
int32_t code = qParserValidateDclSqlNode(&info, pParseCtx, pOutput, outputLen, type, msg, msgLen);
SDclStmtInfo* pDcl = calloc(1, sizeof(SQueryStmtInfo));
if (NULL == pDcl) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code.
return terrno;
}
pDcl->nodeType = info.type;
int32_t code = qParserValidateDclSqlNode(&info, &pCxt->ctx, pDcl, pCxt->pMsg, pCxt->msgLen);
if (code == TSDB_CODE_SUCCESS) {
// do nothing
*pQuery = (SQueryNode*)pDcl;
}
} else {
SQueryStmtInfo* pQueryInfo = calloc(1, sizeof(SQueryStmtInfo));
@ -53,9 +63,9 @@ int32_t qParseQuerySql(const char* pStr, size_t length, SParseBasicCtx* pParseCt
struct SCatalog* pCatalog = NULL;
int32_t code = catalogGetHandle(NULL, &pCatalog);
code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, pParseCtx->requestId, msg, msgLen);
code = qParserValidateSqlNode(pCatalog, &info, pQueryInfo, pCxt->ctx.requestId, pCxt->pMsg, pCxt->msgLen);
if (code == TSDB_CODE_SUCCESS) {
*pOutput = pQueryInfo;
*pQuery = (SQueryNode*)pQueryInfo;
}
}
@ -63,8 +73,12 @@ int32_t qParseQuerySql(const char* pStr, size_t length, SParseBasicCtx* pParseCt
return TSDB_CODE_SUCCESS;
}
int32_t qParseInsertSql(SParseContext* pContext, SInsertStmtInfo** pInfo) {
return parseInsertSql(pContext, pInfo);
int32_t qParseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) {
if (isInsertSql(pCxt->pSql, pCxt->sqlLen)) {
return parseInsertSql(pCxt, (SInsertStmtInfo**)pQuery);
} else {
return parseQuerySql(pCxt, pQuery);
}
}
int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) {

8
source/libs/parser/test/mockCatalog.cpp
View File

@ -23,20 +23,20 @@
namespace {
void generateTestT1(MockCatalogService* mcs) {
ITableBuilder& builder = mcs->createTableBuilder("root.test", "t1", TSDB_NORMAL_TABLE, 3)
ITableBuilder& builder = mcs->createTableBuilder("test", "t1", TSDB_NORMAL_TABLE, 3)
.setPrecision(TSDB_TIME_PRECISION_MILLI).setVgid(1).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
.addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20);
builder.done();
}
void generateTestST1(MockCatalogService* mcs) {
ITableBuilder& builder = mcs->createTableBuilder("root.test", "st1", TSDB_SUPER_TABLE, 3, 2)
ITableBuilder& builder = mcs->createTableBuilder("test", "st1", TSDB_SUPER_TABLE, 3, 2)
.setPrecision(TSDB_TIME_PRECISION_MILLI).addColumn("ts", TSDB_DATA_TYPE_TIMESTAMP)
.addTag("tag1", TSDB_DATA_TYPE_INT).addTag("tag2", TSDB_DATA_TYPE_BINARY, 20)
.addColumn("c1", TSDB_DATA_TYPE_INT).addColumn("c2", TSDB_DATA_TYPE_BINARY, 20);
builder.done();
mcs->createSubTable("root.test", "st1", "st1s1", 1);
mcs->createSubTable("root.test", "st1", "st1s2", 2);
mcs->createSubTable("test", "st1", "st1s1", 1);
mcs->createSubTable("test", "st1", "st1s2", 2);
}
}

14
source/libs/parser/test/mockCatalogService.cpp
View File

@ -94,9 +94,9 @@ public:
return 0;
}
int32_t catalogGetTableMeta(const char* pDBName, const char* pTableName, STableMeta** pTableMeta) const {
int32_t catalogGetTableMeta(const char* pDbFullName, const char* pTableName, STableMeta** pTableMeta) const {
std::unique_ptr<STableMeta> table;
int32_t code = copyTableSchemaMeta(pDBName, pTableName, &table);
int32_t code = copyTableSchemaMeta(toDbname(pDbFullName), pTableName, &table);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
@ -104,7 +104,7 @@ public:
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetTableHashVgroup(const char* pDBName, const char* pTableName, SVgroupInfo* vgInfo) const {
int32_t catalogGetTableHashVgroup(const char* pDbFullName, const char* pTableName, SVgroupInfo* vgInfo) const {
// todo
return 0;
}
@ -195,6 +195,14 @@ private:
typedef std::map<std::string, std::shared_ptr<MockTableMeta>> TableMetaCache;
typedef std::map<std::string, TableMetaCache> DbMetaCache;
std::string toDbname(const std::string& dbFullName) const {
std::string::size_type n = dbFullName.find(".");
if (n == std::string::npos) {
return dbFullName;
}
return dbFullName.substr(n + 1);
}
std::string ttToString(int8_t tableType) const {
switch (tableType) {
case TSDB_SUPER_TABLE:

14
source/libs/parser/test/parserTests.cpp
View File

@ -714,12 +714,9 @@ TEST(testCase, show_user_Test) {
SSqlInfo info1 = doGenerateAST(sql1);
ASSERT_EQ(info1.valid, true);
void* output = NULL;
int32_t type = 0;
int32_t len = 0;
SDclStmtInfo output;
SParseBasicCtx ct= {.db= "abc", .acctId = 1, .requestId = 1};
int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, &len, &type, msg, buf.len);
int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len);
ASSERT_EQ(code, 0);
// convert the show command to be the select query
@ -738,12 +735,9 @@ TEST(testCase, create_user_Test) {
ASSERT_EQ(info1.valid, true);
ASSERT_EQ(isDclSqlStatement(&info1), true);
void* output = NULL;
int32_t type = 0;
int32_t len = 0;
SDclStmtInfo output;
SParseBasicCtx ct= {.db= "abc", .acctId = 1, .requestId = 1};
int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, &len, &type, msg, buf.len);
int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len);
ASSERT_EQ(code, 0);
destroySqlInfo(&info1);

2
source/libs/planner/src/physicalPlanJson.c
View File

@ -845,7 +845,7 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) {
return TSDB_CODE_FAILED;
}
*str = cJSON_Print(json);
*len = strlen(*str);
*len = strlen(*str) + 1;
return TSDB_CODE_SUCCESS;
}

53
source/libs/planner/test/phyPlanTests.cpp
View File

@ -33,10 +33,6 @@ protected:
void pushScan(const string& db, const string& table, int32_t scanOp) {
shared_ptr<MockTableMeta> meta = mockCatalogService->getTableMeta(db, table);
EXPECT_TRUE(meta);
// typedef struct SQueryPlanNode {
// SArray *pExpr; // the query functions or sql aggregations
// int32_t numOfExpr; // number of result columns, which is also the number of pExprs
// } SQueryPlanNode;
unique_ptr<SQueryPlanNode> scan((SQueryPlanNode*)calloc(1, sizeof(SQueryPlanNode)));
scan->info.type = scanOp;
scan->numOfCols = meta->schema->tableInfo.numOfColumns;
@ -54,6 +50,21 @@ protected:
return code;
}
int32_t run(const string& db, const string& sql) {
SParseContext cxt;
buildParseContext(db, sql, &cxt);
SQueryNode* query;
int32_t code = qParseQuerySql(&cxt, &query);
if (TSDB_CODE_SUCCESS != code) {
cout << "error no:" << code << ", msg:" << cxt.pMsg << endl;
return code;
}
SQueryDag* dag = nullptr;
code = qCreateQueryDag(query, nullptr, &dag);
dag_.reset(dag);
return code;
}
void explain() {
size_t level = taosArrayGetSize(dag_->pSubplans);
for (size_t i = 0; i < level; ++i) {
@ -64,7 +75,8 @@ protected:
std::cout << "no " << j << ":" << std::endl;
int32_t len = 0;
char* str = nullptr;
ASSERT_EQ (TSDB_CODE_SUCCESS, qSubPlanToString((const SSubplan*)taosArrayGetP(subplans, j), &str, &len));
ASSERT_EQ(TSDB_CODE_SUCCESS, qSubPlanToString((const SSubplan*)taosArrayGetP(subplans, j), &str, &len));
std::cout << "len:" << len << std::endl;
std::cout << str << std::endl;
free(str);
}
@ -108,6 +120,25 @@ private:
return info;
}
void buildParseContext(const string& db, const string& sql, SParseContext* pCxt) {
static string _db;
static string _sql;
static const int32_t _msgMaxLen = 4096;
static char _msg[_msgMaxLen];
_db = db;
_sql = sql;
memset(_msg, 0, _msgMaxLen);
pCxt->ctx.acctId = 1;
pCxt->ctx.db = _db.c_str();
pCxt->ctx.requestId = 1;
pCxt->pSql = _sql.c_str();
pCxt->sqlLen = _sql.length();
pCxt->pMsg = _msg;
pCxt->msgLen = _msgMaxLen;
}
shared_ptr<MockTableMeta> meta_;
unique_ptr<SQueryPlanNode> logicPlan_;
unique_ptr<SQueryDag> dag_;
@ -115,7 +146,7 @@ private:
// select * from table
TEST_F(PhyPlanTest, tableScanTest) {
pushScan("root.test", "t1", QNODE_TABLESCAN);
pushScan("test", "t1", QNODE_TABLESCAN);
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
explain();
SQueryDag* dag = reslut();
@ -124,9 +155,17 @@ TEST_F(PhyPlanTest, tableScanTest) {
// select * from supertable
TEST_F(PhyPlanTest, superTableScanTest) {
pushScan("root.test", "st1", QNODE_TABLESCAN);
pushScan("test", "st1", QNODE_TABLESCAN);
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
explain();
SQueryDag* dag = reslut();
// todo check
}
// insert into t values(...)
TEST_F(PhyPlanTest, insertTest) {
ASSERT_EQ(run("test", "insert into t1 values (now, 1, \"beijing\")"), TSDB_CODE_SUCCESS);
explain();
SQueryDag* dag = reslut();
// todo check
}

2
source/util/src/tworker.c
View File

@ -38,6 +38,7 @@ int32_t tWorkerInit(SWorkerPool *pool) {
void tWorkerCleanup(SWorkerPool *pool) {
for (int i = 0; i < pool->max; ++i) {
SWorker *worker = pool->workers + i;
if (worker == NULL) continue;
if (taosCheckPthreadValid(worker->thread)) {
taosQsetThreadResume(pool->qset);
}
@ -45,6 +46,7 @@ void tWorkerCleanup(SWorkerPool *pool) {
for (int i = 0; i < pool->max; ++i) {
SWorker *worker = pool->workers + i;
if (worker == NULL) continue;
if (taosCheckPthreadValid(worker->thread)) {
pthread_join(worker->thread, NULL);
}