From d1445018324332a59fdf4e2564b858610bc7356a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 30 Nov 2021 15:42:13 +0800 Subject: [PATCH 01/18] update fst core struct --- source/libs/index/src/index_fst.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 315253d907..1d6e26e612 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1265,7 +1265,7 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { - FstOutput fOutput = {.null = false, out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; + FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; return swsResultCreate(&slice, fOutput , tState); } } From d46ca756bf327841be6e7ab3f9c7f639d32cd985 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Dec 2021 00:24:12 +0800 Subject: [PATCH 02/18] refactor builder struct --- source/libs/index/inc/index_fst.h | 26 ++-- source/libs/index/inc/index_fst_util.h | 25 ++-- source/libs/index/src/index_fst.c | 141 ++++++++++++++----- source/libs/index/src/index_fst_automation.c | 1 + source/libs/index/src/index_fst_util.c | 102 ++++++++++---- 5 files changed, 216 insertions(+), 79 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index bcb7070755..fe669c8567 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -34,6 +34,7 @@ typedef struct FstRange { } FstRange; +typedef enum {GE, GT, LE, LT} RangeType; typedef enum { OneTransNext, OneTrans, AnyTrans, EmptyFinal} State; typedef enum {Ordered, OutOfOrdered, DuplicateKey} OrderType; @@ -169,11 +170,6 @@ uint64_t fstStateFindInput(FstState *state, FstNode *node, uint8_t b, bool *null - - - - - #define FST_STATE_ONE_TRNAS_NEXT(node) (node->state.state == OneTransNext) #define FST_STATE_ONE_TRNAS(node) (node->state.state == OneTrans) #define FST_STATE_ANY_TRANS(node) (node->state.state == AnyTrans) @@ -273,6 +269,8 @@ FstNode* fstGetRoot(Fst *fst); FstType fstGetType(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst); + + Output fstEmptyFinalOutput(Fst *fst, bool *null); bool fstVerify(Fst *fst); @@ -280,10 +278,6 @@ bool fstVerify(Fst *fst); //refactor this function bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); - - - - typedef struct StreamState { FstNode *node; uint64_t trans; @@ -316,4 +310,18 @@ StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoun void streamWithStateDestroy(StreamWithState *sws); bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min); StreamWithStateResult* streamWithStateNextWith(StreamWithState *sws, StreamCallback callback); + +typedef struct FstStreamBuilder { + Fst *fst; + Automation *aut; + FstBoundWithData *min; + FstBoundWithData *max; +} FstStreamBuilder; + +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut); +// set up bound range +// refator, simple code by marco + +FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); + #endif diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index ff0946063d..a3cef493eb 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -67,20 +67,27 @@ uint8_t packDeltaSize(CompiledAddr nodeAddr, CompiledAddr transAddr); CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr); +typedef struct FstString { + uint8_t *data; + uint32_t len; + int32_t ref; +} FstString; typedef struct FstSlice { - uint8_t *data; - uint64_t dLen; - int32_t start; - int32_t end; + FstString *str; + int32_t start; + int32_t end; } FstSlice; -FstSlice fstSliceCopy(FstSlice *slice, int32_t start, int32_t end); -FstSlice fstSliceCreate(uint8_t *data, uint64_t dLen); -bool fstSliceEmpty(FstSlice *slice); -int fstSliceCompare(FstSlice *a, FstSlice *b); +FstSlice fstSliceCreate(uint8_t *data, uint64_t len); +FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); +FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); +bool fstSliceEmpty(FstSlice *s); +int fstSliceCompare(FstSlice *s1, FstSlice *s2); +void fstSliceDestroy(FstSlice *s); +uint8_t *fstSliceData(FstSlice *s, int32_t *sz); -#define FST_SLICE_LEN(s) ((s)->end - (s)->start + 1) +#define FST_SLICE_LEN(s) (s->end - s->start + 1) #endif diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 1d6e26e612..f7ed124f43 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -92,7 +92,7 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr add } void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output out) { FstSlice *s = &bs; - if (s->data == NULL || s->dLen == 0 || s->start > s->end) { + if (fstSliceEmpty(s)) { return; } size_t sz = taosArrayGetSize(nodes->stack) - 1; @@ -104,9 +104,11 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); //trn->inp = s->data[s->start]; //trn->out = out; - un->last = fstLastTransitionCreate(s->data[s->start], out); + int32_t len = 0; + uint8_t *data = fstSliceData(s, &len); + un->last = fstLastTransitionCreate(data[0], out); - for (uint64_t i = s->start; i <= s->end; i++) { + for (uint64_t i = 0; i < len; i++) { FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; @@ -115,7 +117,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); //trn->inp = s->data[i]; //trn->out = out; - FstLastTransition *trn = fstLastTransitionCreate(s->data[i], out); + FstLastTransition *trn = fstLastTransitionCreate(data[i], out); FstBuilderNodeUnfinished un = {.node = n, .last = trn}; taosArrayPush(nodes->stack, &un); @@ -133,7 +135,8 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) uint64_t count = 0; for (size_t i = 0; i < ssz && i < lsz; i++) { FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); - if (un->last->inp == s->data[s->start + i]) { + uint8_t *data = fstSliceData(s, NULL); + if (un->last->inp == data[i]) { count++; } else { break; @@ -153,7 +156,8 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, FstLastTransition *t = un->last; uint64_t addPrefix = 0; - if (t && t->inp == s->data[s->start + i]) { + uint8_t *data = fstSliceData(s, NULL); + if (t && t->inp == data[i]) { uint64_t commPrefix = MIN(t->out, *out); uint64_t tAddPrefix = t->out - commPrefix; (*out) = (*out) - commPrefix; @@ -176,7 +180,9 @@ FstState fstStateCreateFrom(FstSlice* slice, CompiledAddr addr) { if (addr == EMPTY_ADDRESS) { return fs; } - uint8_t v = slice->data[addr]; + + uint8_t *data = fstSliceData(slice, NULL); + uint8_t v = data[addr]; uint8_t t = (v & 0b11000000) >> 6; if (t == 0b11) { fs.state = OneTransNext; @@ -376,7 +382,8 @@ uint8_t fstStateInput(FstState *s, FstNode *node) { FstSlice *slice = &node->data; bool null = false; uint8_t inp = fstStateCommInput(s, &null); - return null == false ? inp : slice->data[slice->start - 1]; + uint8_t *data = fstSliceData(slice, NULL); + return null == false ? inp : data[-1]; } uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { assert(s->state == AnyTrans); @@ -388,7 +395,9 @@ uint8_t fstStateInputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { - fstStateTransIndexSize(s, node->version, node->nTrans) - i - 1; // the output size - return slice->data[at]; + + uint8_t *data = fstSliceData(slice, NULL); + return data[at]; } // trans_addr @@ -406,7 +415,8 @@ CompiledAddr fstStateTransAddr(FstState *s, FstNode *node) { - tSizes; // refactor error logic - return unpackDelta(slice->data + slice->start + i, tSizes, node->end); + uint8_t *data = fstSliceData(slice, NULL); + return unpackDelta(data +i, tSizes, node->end); } } CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i) { @@ -421,7 +431,8 @@ CompiledAddr fstStateTransAddrForAnyTrans(FstState *s, FstNode *node, uint64_t i - node->nTrans - (i * tSizes) - tSizes; - return unpackDelta(slice->data + slice->start + at, tSizes, node->end); + uint8_t *data = fstSliceData(slice, NULL); + return unpackDelta(data + at, tSizes, node->end); } // sizes @@ -434,7 +445,8 @@ PackSizes fstStateSizes(FstState *s, FstSlice *slice) { i = FST_SLICE_LEN(slice) - 1 - fstStateNtransLen(s) - 1; } - return (PackSizes)(slice->data[slice->start + i]); + uint8_t *data = fstSliceData(slice, NULL); + return (PackSizes)(*(data +i)); } // Output Output fstStateOutput(FstState *s, FstNode *node) { @@ -452,7 +464,8 @@ Output fstStateOutput(FstState *s, FstNode *node) { - 1 - tSizes - oSizes; - return unpackUint64(slice->data + slice->start + i, oSizes); + uint8_t *data = fstSliceData(slice, NULL); + return unpackUint64(data + i, oSizes); } Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { @@ -469,7 +482,9 @@ Output fstStateOutputForAnyTrans(FstState *s, FstNode *node, uint64_t i) { - fstStateTotalTransSize(s, node->version, node->sizes, node->nTrans) - (i * oSizes) - oSizes; - return unpackUint64(slice->data + slice->start + at, oSizes); + + uint8_t *data = fstSliceData(slice, NULL); + return unpackUint64(data + at, oSizes); } // anyTrans specify function @@ -523,7 +538,10 @@ uint64_t fstStateNtrans(FstState *s, FstSlice *slice) { if (null != true) { return n; } - n = slice->data[slice->end - 1]; // data[data.len() - 2] + int32_t len; + uint8_t *data = fstSliceData(slice, &len); + n = data[len - 2]; + //n = data[slice->end - 1]; // data[data.len() - 2] return n == 1 ? 256: n; // // "1" is never a normal legal value here, because if there, // is only 1 transition, then it is encoded in the state byte } Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, PackSizes sizes, uint64_t nTrans) { @@ -538,7 +556,8 @@ Output fstStateFinalOutput(FstState *s, uint64_t version, FstSlice *slice, Pack - fstStateTotalTransSize(s, version, sizes, nTrans) - (nTrans * oSizes) - oSizes; - return unpackUint64(slice->data + slice->start + at, (uint8_t)oSizes); + uint8_t *data = fstSliceData(slice, NULL); + return unpackUint64(data + at, (uint8_t)oSizes); } uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { @@ -549,7 +568,10 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { - fstStateNtransLen(s) - 1 // pack size - fstStateTransIndexSize(s, node->version, node->nTrans); - uint64_t i = slice->data[slice->start + at + b]; + int32_t dlen = 0; + uint8_t *data = fstSliceData(slice, &dlen); + uint64_t i = data[at + b]; + //uint64_t i = slice->data[slice->start + at + b]; if (i >= node->nTrans) { *null = true; } @@ -561,8 +583,13 @@ uint64_t fstStateFindInput(FstState *s, FstNode *node, uint8_t b, bool *null) { - node->nTrans; uint64_t end = start + node->nTrans; uint64_t len = end - start; + int32_t dlen = 0; + uint8_t *data = fstSliceData(slice, &dlen); for(int i = 0; i < len; i++) { - uint8_t v = slice->data[slice->start + i]; + //uint8_t v = slice->data[slice->start + i]; + ////slice->data[slice->start + i]; + uint8_t v = data[i]; + if (v == b) { return node->nTrans - i - 1; // bug } @@ -888,7 +915,7 @@ void fstBuilderFinish(FstBuilder *b) { FstSlice fstNodeAsSlice(FstNode *node) { FstSlice *slice = &node->data; - FstSlice s = fstSliceCopy(slice, slice->end, slice->dLen - 1); + FstSlice s = fstSliceCopy(slice, slice->end, FST_SLICE_LEN(slice) - 1); return s; } @@ -929,12 +956,13 @@ void fstBuilderNodeUnfinishedAddOutputPrefix(FstBuilderNodeUnfinished *unNode, O } Fst* fstCreate(FstSlice *slice) { - char *buf = slice->data; - uint64_t skip = 0; - uint64_t len = slice->dLen; - if (len < 36) { + int32_t slen; + char *buf = fstSliceData(slice, &slen); + if (slen < 36) { return NULL; } + uint64_t len = slen; + uint64_t skip = 0; uint64_t version; taosDecodeFixedU64(buf, &version); @@ -992,8 +1020,10 @@ void fstDestroy(Fst *fst) { bool fstGet(Fst *fst, FstSlice *b, Output *out) { FstNode *root = fstGetRoot(fst); Output tOut = 0; - for (uint32_t i = 0; i < b->dLen; i++) { - uint8_t inp = b->data[i]; + int32_t len; + uint8_t *data = fstSliceData(b, &len); + for (uint32_t i = 0; i < len; i++) { + uint8_t inp = data[i]; Output res = 0; bool null = fstNodeFindInput(root, inp, &res); if (null) { return false; } @@ -1046,9 +1076,10 @@ Output fstEmptyFinalOutput(Fst *fst, bool *null) { bool fstVerify(Fst *fst) { uint32_t checkSum = fst->meta->checkSum; - FstSlice *data = fst->data; + int32_t len; + uint8_t *data = fstSliceData(fst->data, &len); TSCKSUM initSum = 0; - if (!taosCheckChecksumWhole(data->data, data->dLen)) { + if (!taosCheckChecksumWhole(data, len)) { return false; } return true; @@ -1058,9 +1089,14 @@ bool fstVerify(Fst *fst) { FstBoundWithData* fstBoundStateCreate(FstBound type, FstSlice *data) { FstBoundWithData *b = calloc(1, sizeof(FstBoundWithData)); if (b == NULL) { return NULL; } - + + if (data != NULL) { + b->data = fstSliceCopy(data, data->start, data->end); + } else { + b->data = fstSliceCreate(NULL, 0); + } b->type = type; - b->data = fstSliceCopy(data, data->start, data->end); + return b; } @@ -1145,8 +1181,10 @@ bool streamWithStateSeekMin(StreamWithState *sws, FstBoundWithData *min) { Output out = 0; void* autState = sws->aut->start(); - for (uint32_t i = 0; i < key->dLen; i++) { - uint8_t b = key->data[i]; + int32_t len; + uint8_t *data = fstSliceData(key, &len); + for (uint32_t i = 0; i < len; i++) { + uint8_t b = data[i]; uint64_t res = 0; bool null = fstNodeFindInput(node, b, &res); if (null == false) { @@ -1277,8 +1315,8 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); if (result == NULL) { return NULL; } - FstSlice slice = fstSliceCopy(data, 0, data->dLen - 1); - result->data = slice; + FstSlice s = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); + result->data = s; result->out = fOut; result->state = state; @@ -1293,5 +1331,42 @@ void streamStateDestroy(void *s) { //free(s->autoState); } +FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { + FstStreamBuilder *b = calloc(1, sizeof(FstStreamBuilder)); + if (NULL == b) { return NULL; } + + b->fst = fst; + b->aut = aut; + b->min = fstBoundStateCreate(Unbounded, NULL); + b->max = fstBoundStateCreate(Unbounded, NULL); + return b; +} +void fstStreamBuilderDestroy(FstStreamBuilder *b) { + free(b); +} +FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { + if (b == NULL) { return NULL; } + + if (type == GE) { + b->min->type = Included; + fstSliceDestroy(&(b->min->data)); + b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); + } else if (type == GT) { + b->min->type = Excluded; + fstSliceDestroy(&(b->min->data)); + b->min->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); + } else if (type == LE) { + b->max->type = Included; + fstSliceDestroy(&(b->max->data)); + b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); + } else if (type == LT) { + b->max->type = Excluded; + fstSliceDestroy(&(b->max->data)); + b->max->data = fstSliceDeepCopy(val, 0, FST_SLICE_LEN(val) - 1); + } + return b; +} + + diff --git a/source/libs/index/src/index_fst_automation.c b/source/libs/index/src/index_fst_automation.c index f2f48bbc8a..3d5efd30f3 100644 --- a/source/libs/index/src/index_fst_automation.c +++ b/source/libs/index/src/index_fst_automation.c @@ -12,3 +12,4 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index 952c710676..92949d1ff2 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -91,42 +91,88 @@ CompiledAddr unpackDelta(char *data, uint64_t len, uint64_t nodeAddr) { } // fst slice func -FstSlice fstSliceCreate(uint8_t *data, uint64_t dLen) { - FstSlice slice = {.data = data, .dLen = dLen, .start = 0, .end = dLen - 1}; - return slice; +// + +FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { + FstString *str = (FstString *)malloc(sizeof(FstString)); + str->ref = 1; + str->len = len; + str->data = malloc(len * sizeof(uint8_t)); + memcpy(str->data, data, len); + + FstSlice s = {.str = str, .start = 0, .end = len - 1}; + return s; } // just shallow copy -FstSlice fstSliceCopy(FstSlice *slice, int32_t start, int32_t end) { - FstSlice t; - if (start >= slice->dLen || end >= slice->dLen || start > end) { - t.data = NULL; - return t; - }; - - t.data = slice->data; - t.dLen = slice->dLen; - t.start = start; - t.end = end; +FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { + FstString *str = s->str; + str->ref++; + int32_t alen; + //uint8_t *buf = fstSliceData(s, &alen); + //start = buf + start - (buf - s->start); + //end = buf + end - (buf - s->start); + + FstSlice t = {.str = str, .start = start + s->start, .end = end + s->start}; return t; } -bool fstSliceEmpty(FstSlice *slice) { - return slice->data == NULL || slice->dLen <= 0; +FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { + + int32_t alen, tlen = end - start + 1; + uint8_t *data = fstSliceData(s, &alen); + assert(tlen <= alen); + + uint8_t *buf = malloc(sizeof(uint8_t) * tlen); + memcpy(buf, data, tlen); + + FstString *str = malloc(sizeof(FstString)); + str->data = buf; + str->len = tlen; + str->ref = 1; + + FstSlice ans; + ans.str = str; + ans.start = 0; + ans.end = tlen - 1; + return ans; +} +bool fstSliceEmpty(FstSlice *s) { + return s->str == NULL || s->start < 0 || s->end < 0; +} + +uint8_t *fstSliceData(FstSlice *s, int32_t *size) { + FstString *str = s->str; + if (size != NULL) { + *size = s->end - s->start + 1; + } + return str->data + s->start; +} +void fstSliceDestroy(FstSlice *s) { + FstString *str = s->str; + str->ref--; + if (str->ref <= 0) { + free(str->data); + free(str); + s->str = NULL; + } } int fstSliceCompare(FstSlice *a, FstSlice *b) { - int32_t aLen = (a->end - a->start + 1); - int32_t bLen = (b->end - b->start + 1); - int32_t mLen = (aLen < bLen ? aLen : bLen); - for (int i = 0; i < mLen; i++) { - uint8_t x = a->data[i + a->start]; - uint8_t y = b->data[i + b->start]; - if (x == y) { continue; } + int32_t alen, blen; + uint8_t *aBuf = fstSliceData(a, &alen); + uint8_t *bBuf = fstSliceData(b, &blen); + + + uint32_t i, j; + for (i = 0, j = 0; i < alen && j < blen; i++, j++) { + uint8_t x = aBuf[i]; + uint8_t y = bBuf[j]; + if (x == y) { continue;} else if (x < y) { return -1; } - else { return 1; } - } - if (aLen == bLen) { return 0; } - else if (aLen < bLen) { return -1; } - else { return 1; } + else { return 1; }; + } + if (i < alen) { return 1; } + else if (j < blen) { return -1; } + else { return 0; } } From f6908c8d1429ec741b9c014c288f6d413036a5dc Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Dec 2021 08:46:22 +0800 Subject: [PATCH 03/18] refactor builder struct --- source/libs/index/src/index_fst.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index f7ed124f43..e0ffbdbe9f 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1315,7 +1315,7 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta StreamWithStateResult *result = calloc(1, sizeof(StreamWithStateResult)); if (result == NULL) { return NULL; } - FstSlice s = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); + result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); result->data = s; result->out = fOut; result->state = state; @@ -1342,6 +1342,8 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { return b; } void fstStreamBuilderDestroy(FstStreamBuilder *b) { + fstSliceDestroy(&b->min); + fstSliceDestroy(&b->max); free(b); } FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { From 7ee1cf62ca436f6dac136177532b637c2afed09e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Dec 2021 12:00:20 +0800 Subject: [PATCH 04/18] refactor builder struct --- source/libs/index/src/index_fst.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index e0ffbdbe9f..dd43890209 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -129,13 +129,12 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) { FstSlice *s = &bs; - size_t lsz = (size_t)(s->end - s->start + 1); // data len size_t ssz = taosArrayGetSize(node->stack); // stack size - uint64_t count = 0; + int32_t lsz; // data len + uint8_t *data = fstSliceData(s, &lsz); for (size_t i = 0; i < ssz && i < lsz; i++) { FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); - uint8_t *data = fstSliceData(s, NULL); if (un->last->inp == data[i]) { count++; } else { @@ -662,6 +661,7 @@ static const char *fstNodeState(FstNode *node) { void fstNodeDestroy(FstNode *node) { + fstSliceDestroy(&node->data); free(node); } FstTransitions* fstNodeTransitions(FstNode *node) { @@ -825,6 +825,7 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { FstSlice sub = fstSliceCopy(s, prefixLen, s->end); fstUnFinishedNodesAddSuffix(b->unfinished, sub, out); + fstSliceDestroy(&sub); return; } @@ -1316,7 +1317,6 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta if (result == NULL) { return NULL; } result->data = fstSliceCopy(data, 0, FST_SLICE_LEN(data) - 1); - result->data = s; result->out = fOut; result->state = state; @@ -1342,8 +1342,8 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut) { return b; } void fstStreamBuilderDestroy(FstStreamBuilder *b) { - fstSliceDestroy(&b->min); - fstSliceDestroy(&b->max); + fstSliceDestroy(&b->min->data); + fstSliceDestroy(&b->max->data); free(b); } FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type) { From 716c0045f8aa32ddff8e8fd8cb7d6d8c9726010a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 1 Dec 2021 17:29:59 +0800 Subject: [PATCH 05/18] refactor builder struct --- source/libs/index/inc/index_fst_util.h | 4 ++-- source/libs/index/src/index_fst.c | 8 ++++---- source/libs/index/src/index_fst_util.c | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index a3cef493eb..d87e5bd57e 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -82,8 +82,8 @@ typedef struct FstSlice { FstSlice fstSliceCreate(uint8_t *data, uint64_t len); FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end); FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end); -bool fstSliceEmpty(FstSlice *s); -int fstSliceCompare(FstSlice *s1, FstSlice *s2); +bool fstSliceIsEmpty(FstSlice *s); +int fstSliceCompare(FstSlice *s1, FstSlice *s2); void fstSliceDestroy(FstSlice *s); uint8_t *fstSliceData(FstSlice *s, int32_t *sz); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index dd43890209..ca1c7910ed 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -92,7 +92,7 @@ void fstUnFinishedNodesTopLastFreeze(FstUnFinishedNodes *nodes, CompiledAddr add } void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output out) { FstSlice *s = &bs; - if (fstSliceEmpty(s)) { + if (fstSliceIsEmpty(s)) { return; } size_t sz = taosArrayGetSize(nodes->stack) - 1; @@ -801,7 +801,7 @@ bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in) { void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { FstSlice *s = &bs; - if (fstSliceEmpty(s)) { + if (fstSliceIsEmpty(s)) { b->len = 1; fstUnFinishedNodesSetRootOutput(b->unfinished, in); return; @@ -831,7 +831,7 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup) { FstSlice *input = &bs; - if (fstSliceEmpty(&b->last)) { + if (fstSliceIsEmpty(&b->last)) { // deep copy or not b->last = fstSliceCopy(&bs, input->start, input->end); } else { @@ -1115,7 +1115,7 @@ bool fstBoundWithDataIsEmpty(FstBoundWithData *bound) { if (bound->type == Unbounded) { return true; } else { - return fstSliceEmpty(&bound->data); + return fstSliceIsEmpty(&bound->data); } } diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index 92949d1ff2..1c7a1a9e9d 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -122,7 +122,7 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { assert(tlen <= alen); uint8_t *buf = malloc(sizeof(uint8_t) * tlen); - memcpy(buf, data, tlen); + memcpy(buf, data + start, tlen); FstString *str = malloc(sizeof(FstString)); str->data = buf; @@ -135,7 +135,7 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { ans.end = tlen - 1; return ans; } -bool fstSliceEmpty(FstSlice *s) { +bool fstSliceIsEmpty(FstSlice *s) { return s->str == NULL || s->start < 0 || s->end < 0; } From e3ccb28c097e4ba845c7293c134dd38a7adb8a57 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Dec 2021 10:32:58 +0800 Subject: [PATCH 06/18] update fst write --- source/libs/index/inc/index_fst.h | 8 +++----- source/libs/index/src/index_fst.c | 14 ++++++++++++-- source/libs/index/src/index_fst_util.c | 11 +++++------ 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index fe669c8567..cd5073187c 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -268,11 +268,8 @@ FstNode* fstGetNode(Fst *fst, CompiledAddr); FstNode* fstGetRoot(Fst *fst); FstType fstGetType(Fst *fst); CompiledAddr fstGetRootAddr(Fst *fst); - - - -Output fstEmptyFinalOutput(Fst *fst, bool *null); -bool fstVerify(Fst *fst); +Output fstEmptyFinalOutput(Fst *fst, bool *null); +bool fstVerify(Fst *fst); //refactor this function @@ -304,6 +301,7 @@ typedef struct StreamWithStateResult { } StreamWithStateResult; StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *state); +void swsResultDestroy(StreamWithStateResult *result); typedef void* (*StreamCallback)(void *); StreamWithState *streamWithStateCreate(Fst *fst, Automation *automation, FstBoundWithData *min, FstBoundWithData *max) ; diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index ca1c7910ed..68d2011b8f 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -1301,12 +1301,16 @@ StreamWithStateResult *streamWithStateNextWith(StreamWithState *sws, StreamCallb if (fstBoundWithDataExceededBy(sws->endAt, &slice)) { taosArrayDestroyEx(sws->stack, streamStateDestroy); sws->stack = (SArray *)taosArrayInit(256, sizeof(StreamState)); + fstSliceDestroy(&slice); return NULL; } if (FST_NODE_IS_FINAL(nextNode) && isMatch) { FstOutput fOutput = {.null = false, .out = out + FST_NODE_FINAL_OUTPUT(nextNode)}; - return swsResultCreate(&slice, fOutput , tState); + StreamWithStateResult *result = swsResultCreate(&slice, fOutput , tState); + fstSliceDestroy(&slice); + return result; } + fstSliceDestroy(&slice); } return NULL; @@ -1321,8 +1325,14 @@ StreamWithStateResult *swsResultCreate(FstSlice *data, FstOutput fOut, void *sta result->state = state; return result; - } +void swsResultDestroy(StreamWithStateResult *result) { + if (NULL == result) { return; } + + fstSliceDestroy(&result->data); + free(result); +} + void streamStateDestroy(void *s) { if (NULL == s) { return; } StreamState *ss = (StreamState *)s; diff --git a/source/libs/index/src/index_fst_util.c b/source/libs/index/src/index_fst_util.c index 1c7a1a9e9d..532b0b8ac3 100644 --- a/source/libs/index/src/index_fst_util.c +++ b/source/libs/index/src/index_fst_util.c @@ -107,7 +107,6 @@ FstSlice fstSliceCreate(uint8_t *data, uint64_t len) { FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { FstString *str = s->str; str->ref++; - int32_t alen; //uint8_t *buf = fstSliceData(s, &alen); //start = buf + start - (buf - s->start); //end = buf + end - (buf - s->start); @@ -117,9 +116,10 @@ FstSlice fstSliceCopy(FstSlice *s, int32_t start, int32_t end) { } FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { - int32_t alen, tlen = end - start + 1; - uint8_t *data = fstSliceData(s, &alen); - assert(tlen <= alen); + int32_t tlen = end - start + 1; + int32_t slen; + uint8_t *data = fstSliceData(s, &slen); + assert(tlen <= slen); uint8_t *buf = malloc(sizeof(uint8_t) * tlen); memcpy(buf, data + start, tlen); @@ -136,7 +136,7 @@ FstSlice fstSliceDeepCopy(FstSlice *s, int32_t start, int32_t end) { return ans; } bool fstSliceIsEmpty(FstSlice *s) { - return s->str == NULL || s->start < 0 || s->end < 0; + return s->str == NULL || s->str->len == 0 || s->start < 0 || s->end < 0; } uint8_t *fstSliceData(FstSlice *s, int32_t *size) { @@ -161,7 +161,6 @@ int fstSliceCompare(FstSlice *a, FstSlice *b) { uint8_t *aBuf = fstSliceData(a, &alen); uint8_t *bBuf = fstSliceData(b, &blen); - uint32_t i, j; for (i = 0, j = 0; i < alen && j < blen; i++, j++) { uint8_t x = aBuf[i]; From 4bab45c039be102094218eaccb5293d125ef06f7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 2 Dec 2021 15:48:30 +0800 Subject: [PATCH 07/18] refactor builder struct --- source/libs/index/inc/index_fst.h | 6 +- .../index/inc/index_fst_counting_writer.h | 30 +++++++- .../index/src/index_fst_counting_writer.c | 76 +++++++++++++++++-- 3 files changed, 104 insertions(+), 8 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index cd5073187c..96838d5843 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -86,14 +86,16 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, typedef struct FstBuilder { FstCountingWriter *wrt; // The FST raw data is written directly to `wtr`. FstUnFinishedNodes *unfinished; // The stack of unfinished nodes - FstRegistry* registry; // A map of finished nodes. - FstSlice last; // The last word added + FstRegistry* registry; // A map of finished nodes. + FstSlice last; // The last word added CompiledAddr lastAddr; // The address of the last compiled node uint64_t len; // num of keys added } FstBuilder; FstBuilder *fstBuilderCreate(void *w, FstType ty); + + void fstBuilderDestroy(FstBuilder *b); void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 4650804034..42d77cacb0 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -16,6 +16,34 @@ #ifndef __INDEX_FST_COUNTING_WRITER_H__ #define __INDEX_FST_COUNTING_WRITER_H__ +#include "tfile.h" + + +#define DefaultMem 1024*1024 + +static char tmpFile[] = "/tmp/index"; +typedef enum WriterType {TMemory, TFile} WriterType; + +typedef struct WriterCtx { + int (*write)(struct WriterCtx *ctx, uint8_t *buf, int len); + int (*read)(struct WriterCtx *ctx, uint8_t *buf, int len); + int (*flush)(struct WriterCtx *ctx); + WriterType type; + union { + int fd; + void *mem; + }; + int32_t offset; + int32_t limit; +} WriterCtx; + +static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len); +static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len); +static int writeCtxDoFlush(WriterCtx *ctx); + +WriterCtx* writerCtxCreate(WriterType type); +void writerCtxDestroy(WriterCtx *w); + typedef uint32_t CheckSummer; @@ -25,7 +53,7 @@ typedef struct FstCountingWriter { CheckSummer summer; } FstCountingWriter; -uint64_t fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen); +int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen); int fstCountingWriterFlush(FstCountingWriter *write); diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index a0a2c380f1..f04f6eddad 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -16,24 +16,88 @@ #include "index_fst_util.h" #include "index_fst_counting_writer.h" +static int writeCtxDoWrite(WriterCtx *ctx, uint8_t *buf, int len) { + if (ctx->offset + len > ctx->limit) { + return -1; + } + + if (ctx->type == TFile) { + assert(len != tfWrite(ctx->fd, buf, len)); + } else { + memcpy(ctx->mem + ctx->offset, buf, len); + } + ctx->offset += len; + return len; +} +static int writeCtxDoRead(WriterCtx *ctx, uint8_t *buf, int len) { + if (ctx->type == TFile) { + tfRead(ctx->fd, buf, len); + } else { + memcpy(buf, ctx->mem + ctx->offset, len); + } + ctx->offset += len; + + return 1; +} +static int writeCtxDoFlush(WriterCtx *ctx) { + if (ctx->type == TFile) { + //tfFlush(ctx->fd); + } else { + // do nothing + } + return 1; +} + +WriterCtx* writerCtxCreate(WriterType type) { + WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); + if (ctx == NULL) { return NULL; } + + ctx->type == type; + if (ctx->type == TFile) { + ctx->fd = tfOpenCreateWriteAppend(tmpFile); + } else if (ctx->type == TMemory) { + ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); + } + ctx->write = writeCtxDoWrite; + ctx->read = writeCtxDoRead; + ctx->flush = writeCtxDoFlush; + + ctx->offset = 0; + ctx->limit = DefaultMem; + + return ctx; +} +void writerCtxDestroy(WriterCtx *ctx) { + if (ctx->type == TMemory) { + free(ctx->mem); + } else { + tfClose(ctx->fd); + } + free(ctx); +} + + FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } - - cw->wrt = wrt; + + cw->wrt = (void *)(writerCtxCreate(TMemory)); return cw; } void fstCountingWriterDestroy(FstCountingWriter *cw) { // free wrt object: close fd or free mem + writerCtxDestroy((WriterCtx *)(cw->wrt)); free(cw); } -uint64_t fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen) { +int fstCountingWriterWrite(FstCountingWriter *write, uint8_t *buf, uint32_t bufLen) { if (write == NULL) { return 0; } // update checksum // write data to file/socket or mem - - write->count += bufLen; + WriterCtx *ctx = write->wrt; + + int nWrite = ctx->write(ctx, buf, bufLen); + write->count += nWrite; return bufLen; } @@ -41,6 +105,8 @@ uint32_t fstCountingWriterMaskedCheckSum(FstCountingWriter *write) { return 0; } int fstCountingWriterFlush(FstCountingWriter *write) { + WriterCtx *ctx = write->wrt; + ctx->flush(ctx); //write->wtr->flush return 1; } From 6d7388de9109bddff5b130fbde14b126d2652dac Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Thu, 2 Dec 2021 19:56:43 +0800 Subject: [PATCH 08/18] TD-10431 process show msg --- include/common/taosmsg.h | 5 +- source/dnode/mgmt/impl/src/dndTransport.c | 2 +- source/dnode/mnode/impl/inc/mndDef.h | 34 +-- source/dnode/mnode/impl/inc/mndInt.h | 14 + source/dnode/mnode/impl/inc/mndShow.h | 4 + source/dnode/mnode/impl/src/mndCluster.c | 69 +++++ source/dnode/mnode/impl/src/mndConnect.c | 155 +++++++++++ source/dnode/mnode/impl/src/mndDb.c | 20 +- source/dnode/mnode/impl/src/mndShow.c | 314 +++++++++++++++++++++- 9 files changed, 593 insertions(+), 24 deletions(-) create mode 100644 source/dnode/mnode/impl/src/mndConnect.c diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 9bae38e3a7..826934c4a8 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -153,7 +153,8 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DUMMY9, "dummy9" ) #define TSDB_IE_TYPE_DNODE_EXT 6 #define TSDB_IE_TYPE_DNODE_STATE 7 -enum _mgmt_table { +typedef enum _mgmt_table { + TSDB_MGMT_TABLE_START, TSDB_MGMT_TABLE_ACCT, TSDB_MGMT_TABLE_USER, TSDB_MGMT_TABLE_DB, @@ -175,7 +176,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_TP, TSDB_MGMT_TABLE_FUNCTION, TSDB_MGMT_TABLE_MAX, -}; +} EShowType; #define TSDB_ALTER_TABLE_ADD_TAG_COLUMN 1 #define TSDB_ALTER_TABLE_DROP_TAG_COLUMN 2 diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 44ca1d1407..f3e71564b2 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -157,7 +157,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.label = "DND-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = dndProcessResponse; - rpcInit.sessions = 8; + rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 9facb8829c..dc27656dbf 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -281,22 +281,24 @@ typedef struct SFuncObj { int16_t type; } SFuncObj; -typedef struct { - int8_t type; - int8_t maxReplica; - int16_t numOfColumns; - int32_t index; - int32_t rowSize; - int32_t numOfRows; - int32_t numOfReads; - uint16_t payloadLen; - void *pIter; - void *pVgIter; - void **ppShow; - char db[TSDB_FULL_DB_NAME_LEN]; - int16_t offset[TSDB_MAX_COLUMNS]; - int32_t bytes[TSDB_MAX_COLUMNS]; - char payload[]; +typedef struct SShowObj SShowObj; +typedef struct SShowObj { + int8_t type; + int8_t maxReplica; + int16_t numOfColumns; + int32_t id; + int32_t rowSize; + int32_t numOfRows; + int32_t numOfReads; + uint16_t payloadLen; + void *pIter; + void *pVgIter; + SMnode *pMnode; + SShowObj **ppShow; + char db[TSDB_FULL_DB_NAME_LEN]; + int16_t offset[TSDB_MAX_COLUMNS]; + int32_t bytes[TSDB_MAX_COLUMNS]; + char payload[]; } SShowObj; typedef struct SMnodeMsg { diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 8910ed4e63..e5643847a5 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -18,6 +18,7 @@ #include "mndDef.h" #include "sdb.h" +#include "tcache.h" #include "tqueue.h" #ifdef __cplusplus @@ -27,6 +28,9 @@ extern "C" { typedef int32_t (*MndMsgFp)(SMnode *pMnode, SMnodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); +typedef int32_t (*ShowMetaFp)(SMnode *pMnode, STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); +typedef int32_t (*ShowRetrieveFp)(SMnode *pMnode, SShowObj *pShow, char *data, int32_t rows, void *pConn); +typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct { const char *name; @@ -34,6 +38,14 @@ typedef struct { MndCleanupFp cleanupFp; } SMnodeStep; +typedef struct { + int32_t showId; + ShowMetaFp metaFps[TSDB_MGMT_TABLE_MAX]; + ShowRetrieveFp retrieveFps[TSDB_MGMT_TABLE_MAX]; + ShowFreeIterFp freeIterFps[TSDB_MGMT_TABLE_MAX]; + SCacheObj *cache; +} SShowMgmt; + typedef struct SMnode { int32_t dnodeId; int32_t clusterId; @@ -45,6 +57,7 @@ typedef struct SMnode { SSdb *pSdb; SDnode *pDnode; SArray *pSteps; + SShowMgmt showMgmt; MndMsgFp msgFp[TSDB_MSG_TYPE_MAX]; SendMsgToDnodeFp sendMsgToDnodeFp; SendMsgToMnodeFp sendMsgToMnodeFp; @@ -53,6 +66,7 @@ typedef struct SMnode { int32_t sver; int32_t statusInterval; int32_t mnodeEqualVnodeNum; + int32_t shellActivityTimer; char *timezone; char *locale; char *charset; diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index 06c18cb029..31c0b69c85 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -24,6 +24,10 @@ extern "C" { int32_t mndInitShow(SMnode *pMnode); void mndCleanupShow(SMnode *pMnode); +void mnodeAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); +void mnodeAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); +void mnodeAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); +void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 1780c88d6b..dbe5dd4f89 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "mndCluster.h" #include "mndTrans.h" +#include "mndShow.h" #define SDB_CLUSTER_VER 1 @@ -94,6 +95,71 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } + +// static int32_t mnodeGetClusterMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) { +// int32_t cols = 0; +// SSchema *pSchema = pMeta->schema; + +// pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE; +// pSchema[cols].type = TSDB_DATA_TYPE_BINARY; +// strcpy(pSchema[cols].name, "clusterId"); +// pSchema[cols].bytes = htons(pShow->bytes[cols]); +// cols++; + +// pShow->bytes[cols] = 8; +// pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; +// strcpy(pSchema[cols].name, "create_time"); +// pSchema[cols].bytes = htons(pShow->bytes[cols]); +// cols++; + +// pMeta->numOfColumns = htons(cols); +// strcpy(pMeta->tableFname, "show cluster"); +// pShow->numOfColumns = cols; + +// pShow->offset[0] = 0; +// for (int32_t i = 1; i < cols; ++i) { +// pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; +// } + +// pShow->numOfRows = 1; +// pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + +// return 0; +// } + +// static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows, void *pConn) { +// int32_t numOfRows = 0; +// int32_t cols = 0; +// char * pWrite; +// SClusterObj *pCluster = NULL; + +// while (numOfRows < rows) { +// pShow->pIter = mnodeGetNextCluster(pShow->pIter, &pCluster); +// if (pCluster == NULL) break; + +// cols = 0; + +// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; +// STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pCluster->uid, TSDB_CLUSTER_ID_LEN); +// cols++; + +// pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; +// *(int64_t *) pWrite = pCluster->createdTime; +// cols++; + +// mnodeDecClusterRef(pCluster); +// numOfRows++; +// } + +// mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); +// pShow->numOfReads += numOfRows; +// return numOfRows; +// } + +// static void mnodeCancelGetNextCluster(void *pIter) { +// sdbFreeIter(tsClusterSdb, pIter); +// } + int32_t mndInitCluster(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_CLUSTER, .keyType = SDB_KEY_INT32, @@ -104,6 +170,9 @@ int32_t mndInitCluster(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndClusterActionUpdate, .deleteFp = (SdbDeleteFp)mndClusterActionDelete}; + // mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); + // mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); + // mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster); return sdbSetTable(pMnode->pSdb, table); } diff --git a/source/dnode/mnode/impl/src/mndConnect.c b/source/dnode/mnode/impl/src/mndConnect.c new file mode 100644 index 0000000000..8e62eacfab --- /dev/null +++ b/source/dnode/mnode/impl/src/mndConnect.c @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndShow.h" + + + +// static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { +// SHeartBeatRsp *pRsp = (SHeartBeatRsp *)rpcMallocCont(sizeof(SHeartBeatRsp)); +// if (pRsp == NULL) { +// return TSDB_CODE_MND_OUT_OF_MEMORY; +// } + +// SHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; +// if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) { +// rpcFreeCont(pRsp); +// return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code +// } + +// SRpcConnInfo connInfo = {0}; +// rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo); + +// int32_t connId = htonl(pHBMsg->connId); +// SConnObj *pConn = mnodeAccquireConn(connId, connInfo.user, connInfo.clientIp, connInfo.clientPort); +// if (pConn == NULL) { +// pHBMsg->pid = htonl(pHBMsg->pid); +// pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort, pHBMsg->pid, pHBMsg->appName); +// } + +// if (pConn == NULL) { +// // do not close existing links, otherwise +// // mError("failed to create connId, close connect"); +// // pRsp->killConnection = 1; +// } else { +// pRsp->connId = htonl(pConn->connId); +// mnodeSaveQueryStreamList(pConn, pHBMsg); + +// if (pConn->killed != 0) { +// pRsp->killConnection = 1; +// } + +// if (pConn->streamId != 0) { +// pRsp->streamId = htonl(pConn->streamId); +// pConn->streamId = 0; +// } + +// if (pConn->queryId != 0) { +// pRsp->queryId = htonl(pConn->queryId); +// pConn->queryId = 0; +// } +// } + +// int32_t onlineDnodes = 0, totalDnodes = 0; +// mnodeGetOnlineAndTotalDnodesNum(&onlineDnodes, &totalDnodes); + +// pRsp->onlineDnodes = htonl(onlineDnodes); +// pRsp->totalDnodes = htonl(totalDnodes); +// mnodeGetMnodeEpSetForShell(&pRsp->epSet, false); + +// pMsg->rpcRsp.rsp = pRsp; +// pMsg->rpcRsp.len = sizeof(SHeartBeatRsp); + +// mnodeReleaseConn(pConn); +// return TSDB_CODE_SUCCESS; +// } + +// static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { +// SConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont; +// SConnectRsp *pConnectRsp = NULL; +// int32_t code = TSDB_CODE_SUCCESS; + +// SRpcConnInfo connInfo = {0}; +// if (rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo) != 0) { +// mError("thandle:%p is already released while process connect msg", pMsg->rpcMsg.handle); +// code = TSDB_CODE_MND_INVALID_CONNECTION; +// goto connect_over; +// } + +// code = taosCheckVersion(pConnectMsg->clientVersion, version, 3); +// if (code != TSDB_CODE_SUCCESS) { +// goto connect_over; +// } + +// SUserObj *pUser = pMsg->pUser; +// SAcctObj *pAcct = pUser->pAcct; + +// if (pConnectMsg->db[0]) { +// char dbName[TSDB_TABLE_FNAME_LEN * 3] = {0}; +// sprintf(dbName, "%x%s%s", pAcct->acctId, TS_PATH_DELIMITER, pConnectMsg->db); +// SDbObj *pDb = mnodeGetDb(dbName); +// if (pDb == NULL) { +// code = TSDB_CODE_MND_INVALID_DB; +// goto connect_over; +// } + +// if (pDb->status != TSDB_DB_STATUS_READY) { +// mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); +// code = TSDB_CODE_MND_DB_IN_DROPPING; +// mnodeDecDbRef(pDb); +// goto connect_over; +// } +// mnodeDecDbRef(pDb); +// } + +// pConnectRsp = rpcMallocCont(sizeof(SConnectRsp)); +// if (pConnectRsp == NULL) { +// code = TSDB_CODE_MND_OUT_OF_MEMORY; +// goto connect_over; +// } + +// pConnectMsg->pid = htonl(pConnectMsg->pid); +// SConnObj *pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort, pConnectMsg->pid, pConnectMsg->appName); +// if (pConn == NULL) { +// code = terrno; +// } else { +// pConnectRsp->connId = htonl(pConn->connId); +// mnodeReleaseConn(pConn); +// } + +// sprintf(pConnectRsp->acctId, "%x", pAcct->acctId); +// memcpy(pConnectRsp->serverVersion, version, TSDB_VERSION_LEN); +// pConnectRsp->writeAuth = pUser->writeAuth; +// pConnectRsp->superAuth = pUser->superAuth; + +// mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet, false); + +// dnodeGetClusterId(pConnectRsp->clusterId); + +// connect_over: +// if (code != TSDB_CODE_SUCCESS) { +// if (pConnectRsp) rpcFreeCont(pConnectRsp); +// mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); +// } else { +// mLInfo("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); +// pMsg->rpcRsp.rsp = pConnectRsp; +// pMsg->rpcRsp.len = sizeof(SConnectRsp); +// } + +// return code; +// } + + diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 3a5a221677..0ad144d2df 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -18,4 +18,22 @@ #include "mndInt.h" int32_t mndInitDb(SMnode *pMnode) { return 0; } -void mndCleanupDb(SMnode *pMnode) {} \ No newline at end of file +void mndCleanupDb(SMnode *pMnode) {} + + +// static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { +// SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont; + +// int32_t code = TSDB_CODE_SUCCESS; +// if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pUseDbMsg->db); +// if (pMsg->pDb == NULL) { +// return TSDB_CODE_MND_INVALID_DB; +// } + +// if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { +// mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); +// return TSDB_CODE_MND_DB_IN_DROPPING; +// } + +// return code; +// } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 9938e95a73..af6a203006 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -14,8 +14,314 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndShow.h" -int32_t mndInitShow(SMnode *pMnode) { return 0; } -void mndCleanupShow(SMnode *pMnode) {} \ No newline at end of file +static int32_t mndProcessShowMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg); +static int32_t mndProcessRetrieveMsg(SMnode *pMnode, SMnodeMsg *pMsg); +static bool mndCheckRetrieveFinished(SShowObj *pShow); +static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow); +static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); +static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow); +static void mndFreeShowObj(void *ppShow); +static char *mndShowStr(int32_t showType); + +int32_t mndInitShow(SMnode *pMnode) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + + pMgmt->cache = taosCacheInit(TSDB_CACHE_PTR_KEY, 5, true, mndFreeShowObj, "show"); + if (pMgmt->cache == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to alloc show cache since %s", terrstr()); + return -1; + } + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SHOW, mndProcessShowMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_SHOW_RETRIEVE, mndProcessRetrieveMsg); + return 0; +} + +void mndCleanupShow(SMnode *pMnode) {} + +static int32_t mndProcessShowMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + SShowMsg *pMsg = pMnodeMsg->rpcMsg.pCont; + int8_t type = pMsg->type; + uint16_t payloadLen = htonl(pMsg->payloadLen); + + if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) { + terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; + mError("failed to process show msg since %s", terrstr()); + return -1; + } + + ShowMetaFp metaFp = pMgmt->metaFps[type]; + if (metaFp == NULL) { + terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; + mError("failed to process show-meta msg:%s since no message handle", mndShowStr(type)); + return -1; + } + + int32_t size = sizeof(SShowObj) + payloadLen; + SShowObj *pShow = calloc(1, size); + if (pShow != NULL) { + pShow->pMnode = pMnode; + pShow->type = type; + pShow->payloadLen = payloadLen; + memcpy(pShow->db, pMsg->db, TSDB_FULL_DB_NAME_LEN); + memcpy(pShow->payload, pMsg->payload, payloadLen); + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); + return -1; + } + + if (mndPutShowObj(pMnode, pShow) == 0) { + mError("failed to process show-meta msg:%s since %s", mndShowStr(type), terrstr()); + free(pShow); + return -1; + } + + size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; + SShowRsp *pRsp = rpcMallocCont(size); + if (pRsp == NULL) { + mndReleaseShowObj(pShow, true); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("show:%d, failed to process show-meta msg:%s since malloc rsp error", pShow->id, mndShowStr(type)); + return -1; + } + + pRsp->qhandle = htobe64((uint64_t)pShow); + + int32_t code = (*metaFp)(pMnode, &pRsp->tableMeta, pShow, pMnodeMsg->rpcMsg.handle); + mDebug("show:%d, type:%s, get meta finished, numOfRows:%d cols:%d result:%s", pShow->id, mndShowStr(type), + pShow->numOfRows, pShow->numOfColumns, tstrerror(code)); + + if (code == TSDB_CODE_SUCCESS) { + pMnodeMsg->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; + pMnodeMsg->pCont = pRsp; + mndReleaseShowObj(pShow, false); + return TSDB_CODE_SUCCESS; + } else { + rpcFreeCont(pRsp); + mndReleaseShowObj(pShow, true); + return code; + } +} + +static int32_t mndProcessRetrieveMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + int32_t rowsToRead = 0; + int32_t size = 0; + int32_t rowsRead = 0; + + SRetrieveTableMsg *pRetrieve = pMnodeMsg->rpcMsg.pCont; + pRetrieve->qhandle = htobe64(pRetrieve->qhandle); + SShowObj *pShow = (SShowObj *)pRetrieve->qhandle; + + /* + * in case of server restart, apps may hold qhandle created by server before + * restart, which is actually invalid, therefore, signature check is required. + */ + if (mndAcquireShowObj(pMnode, pShow) != 0) { + terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; + mError("failed to process show-retrieve msg:%p since %s", pShow, terrstr()); + return -1; + } + + ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; + if (retrieveFp == NULL) { + mndReleaseShowObj(pShow, false); + terrno = TSDB_CODE_MSG_NOT_PROCESSED; + mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + return -1; + } + + mDebug("show:%d, type:%s, start retrieve data, numOfReads:%d numOfRows:%d", pShow->id, mndShowStr(pShow->type), + pShow->numOfReads, pShow->numOfRows); + + if (mndCheckRetrieveFinished(pShow)) { + mDebug("show:%d, read finished, numOfReads:%d numOfRows:%d", pShow->id, pShow->numOfReads, pShow->numOfRows); + pShow->numOfReads = pShow->numOfRows; + } + + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + rowsToRead = pShow->numOfRows - pShow->numOfReads; + } + + /* return no more than 100 tables in one round trip */ + if (rowsToRead > 100) rowsToRead = 100; + + /* + * the actual number of table may be larger than the value of pShow->numOfRows, if a query is + * issued during a continuous create table operation. Therefore, rowToRead may be less than 0. + */ + if (rowsToRead < 0) rowsToRead = 0; + size = pShow->rowSize * rowsToRead; + + size += 100; + SRetrieveTableRsp *pRsp = rpcMallocCont(size); + if (pRsp == NULL) { + mndReleaseShowObj(pShow, false); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("show:%d, failed to retrieve data since %s", pShow->id, terrstr()); + return -1; + } + + // if free flag is set, client wants to clean the resources + if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + rowsRead = (*retrieveFp)(pMnode, pShow, pRsp->data, rowsToRead, pMnodeMsg->rpcMsg.handle); + } + + mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); + + pRsp->numOfRows = htonl(rowsRead); + pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision + + pMnodeMsg->pCont = pRsp; + pMnodeMsg->contLen = size; + + if (rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { + pRsp->completed = 1; + mDebug("%p, retrieve completed", pShow); + mndReleaseShowObj(pShow, true); + } else { + mDebug("%p, retrieve not completed yet", pShow); + mndReleaseShowObj(pShow, false); + } + + return TSDB_CODE_SUCCESS; +} + +static char *mndShowStr(int32_t showType) { + switch (showType) { + case TSDB_MGMT_TABLE_ACCT: + return "show accounts"; + case TSDB_MGMT_TABLE_USER: + return "show users"; + case TSDB_MGMT_TABLE_DB: + return "show databases"; + case TSDB_MGMT_TABLE_TABLE: + return "show tables"; + case TSDB_MGMT_TABLE_DNODE: + return "show dnodes"; + case TSDB_MGMT_TABLE_MNODE: + return "show mnodes"; + case TSDB_MGMT_TABLE_VGROUP: + return "show vgroups"; + case TSDB_MGMT_TABLE_METRIC: + return "show stables"; + case TSDB_MGMT_TABLE_MODULE: + return "show modules"; + case TSDB_MGMT_TABLE_QUERIES: + return "show queries"; + case TSDB_MGMT_TABLE_STREAMS: + return "show streams"; + case TSDB_MGMT_TABLE_VARIABLES: + return "show configs"; + case TSDB_MGMT_TABLE_CONNS: + return "show connections"; + case TSDB_MGMT_TABLE_SCORES: + return "show scores"; + case TSDB_MGMT_TABLE_GRANTS: + return "show grants"; + case TSDB_MGMT_TABLE_VNODES: + return "show vnodes"; + case TSDB_MGMT_TABLE_CLUSTER: + return "show clusters"; + case TSDB_MGMT_TABLE_STREAMTABLES: + return "show streamtables"; + case TSDB_MGMT_TABLE_TP: + return "show topics"; + default: + return "undefined"; + } +} + +static bool mndCheckRetrieveFinished(SShowObj *pShow) { + if (pShow->pIter == NULL && pShow->numOfReads != 0) { + return true; + } + return false; +} + +static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow) { + TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pShow; + + SShowMgmt *pMgmt = &pMnode->showMgmt; + SShowObj **ppShow = taosCacheAcquireByKey(pMgmt->cache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE)); + if (ppShow) { + mTrace("show:%d, data:%p acquired from cache", pShow->id, ppShow); + return 0; + } + + return -1; +} + +static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { + SMnode *pMnode = pShow->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + SShowObj **ppShow = (SShowObj **)pShow->ppShow; + taosCacheRelease(pMgmt->cache, (void **)(&ppShow), forceRemove); + mDebug("show:%d, data:%p released from cache, force:%d", pShow->id, ppShow, forceRemove); +} + +static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + int32_t lifeSpan = pMnode->shellActivityTimer * 6 * 1000; + + TSDB_CACHE_PTR_TYPE val = (TSDB_CACHE_PTR_TYPE)pShow; + pShow->id = atomic_add_fetch_32(&pMgmt->showId, 1); + SShowObj **ppShow = + taosCachePut(pMgmt->cache, &val, sizeof(TSDB_CACHE_PTR_TYPE), &pShow, sizeof(TSDB_CACHE_PTR_TYPE), lifeSpan); + if (ppShow == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("show:%d, failed to put into cache", pShow->id); + return -1; + } + + mTrace("show:%d, data:%p put into cache", pShow->id, ppShow); + return 0; +} + +static void mndFreeShowObj(void *ppShow) { + SShowObj *pShow = *(SShowObj **)ppShow; + SMnode *pMnode = pShow->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + + ShowFreeIterFp freeFp = pMgmt->freeIterFps[pShow->type]; + if (freeFp != NULL) { + if (pShow->pVgIter != NULL) { + // only used in 'show vnodes "ep"' + (*freeFp)(pMnode, pShow->pVgIter); + } + if (pShow->pIter != NULL) { + (*freeFp)(pMnode, pShow->pIter); + } + } + + mDebug("show:%d, data:%p destroyed", pShow->id, ppShow); + tfree(pShow); +} + +void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { + if (rows < capacity) { + for (int32_t i = 0; i < numOfCols; ++i) { + memmove(data + pShow->offset[i] * rows, data + pShow->offset[i] * capacity, pShow->bytes[i] * rows); + } + } +} + +void mnodeAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + pMgmt->metaFps[showType] = fp; +} + +void mnodeAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + pMgmt->retrieveFps[showType] = fp; +} + +void mnodeAddShowFreeIterHandle(SMnode *pMnode, EShowType showType, ShowFreeIterFp fp) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + pMgmt->freeIterFps[showType] = fp; +} From 5f0fd023cb157e7b1cd65edafe106acc3ebfcca0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 3 Dec 2021 10:45:32 +0800 Subject: [PATCH 09/18] add test case --- source/libs/index/inc/index_fst.h | 9 ++ source/libs/index/inc/index_fst_automation.h | 8 +- source/libs/index/inc/index_fst_common.h | 9 ++ .../index/inc/index_fst_counting_writer.h | 8 ++ source/libs/index/inc/index_fst_node.h | 8 ++ source/libs/index/inc/index_fst_registry.h | 8 ++ source/libs/index/inc/index_fst_util.h | 7 ++ source/libs/index/src/index_fst.c | 8 +- .../index/src/index_fst_counting_writer.c | 2 +- source/libs/index/test/CMakeLists.txt | 1 - source/libs/index/test/indexTests.cpp | 119 ++++++++++-------- 11 files changed, 131 insertions(+), 56 deletions(-) diff --git a/source/libs/index/inc/index_fst.h b/source/libs/index/inc/index_fst.h index 96838d5843..eb288e0aa2 100644 --- a/source/libs/index/inc/index_fst.h +++ b/source/libs/index/inc/index_fst.h @@ -16,6 +16,9 @@ #ifndef __INDEX_FST_H__ #define __INDEX_FST_H__ +#ifdef __cplusplus +extern "C" { +#endif #include "tarray.h" #include "index_fst_util.h" @@ -98,6 +101,7 @@ FstBuilder *fstBuilderCreate(void *w, FstType ty); void fstBuilderDestroy(FstBuilder *b); void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in); +bool fstBuilderInsert(FstBuilder *b, FstSlice bs, Output in); OrderType fstBuilderCheckLastKey(FstBuilder *b, FstSlice bs, bool ckDup); void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate); CompiledAddr fstBuilderCompile(FstBuilder *b, FstBuilderNode *bn); @@ -324,4 +328,9 @@ FstStreamBuilder *fstStreamBuilderCreate(Fst *fst, Automation *aut); FstStreamBuilder *fstStreamBuilderRange(FstStreamBuilder *b, FstSlice *val, RangeType type); + +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_automation.h b/source/libs/index/inc/index_fst_automation.h index c9df9c219e..480d3110c4 100644 --- a/source/libs/index/inc/index_fst_automation.h +++ b/source/libs/index/inc/index_fst_automation.h @@ -15,6 +15,10 @@ #ifndef __INDEX_FST_AUTAOMATION_H__ #define __INDEX_FST_AUTAOMATION_H__ +#ifdef __cplusplus +extern "C" { +#endif + typedef struct AutomationCtx AutomationCtx; typedef struct StartWith { @@ -42,6 +46,8 @@ typedef struct Automation { void *data; } Automation; - +#ifdef __cplusplus +} +#endif #endif diff --git a/source/libs/index/inc/index_fst_common.h b/source/libs/index/inc/index_fst_common.h index b261f4090c..9c802faa33 100644 --- a/source/libs/index/inc/index_fst_common.h +++ b/source/libs/index/inc/index_fst_common.h @@ -1,7 +1,16 @@ #ifndef __INDEX_FST_COMM_H__ #define __INDEX_FST_COMM_H__ + extern const uint8_t COMMON_INPUTS[]; extern char const COMMON_INPUTS_INV[]; +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_counting_writer.h b/source/libs/index/inc/index_fst_counting_writer.h index 42d77cacb0..9280461780 100644 --- a/source/libs/index/inc/index_fst_counting_writer.h +++ b/source/libs/index/inc/index_fst_counting_writer.h @@ -16,6 +16,10 @@ #ifndef __INDEX_FST_COUNTING_WRITER_H__ #define __INDEX_FST_COUNTING_WRITER_H__ +#ifdef __cplusplus +extern "C" { +#endif + #include "tfile.h" @@ -72,6 +76,10 @@ uint8_t fstCountingWriterPackUint(FstCountingWriter *writer, uint64_t n); #define FST_WRITER_INTER_WRITER(writer) (writer->wtr) #define FST_WRITE_CHECK_SUMMER(writer) (writer->summer) +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index 0645aa1158..d1b45a16ab 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -16,6 +16,10 @@ #ifndef __INDEX_FST_NODE_H__ #define __INDEX_FST_NODE_H__ +#ifdef __cplusplus +extern "C" { +#endif + #include "index_fst_util.h" #include "index_fst_counting_writer.h" @@ -45,4 +49,8 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); void fstBuilderNodeDestroy(FstBuilderNode *node); +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_registry.h b/source/libs/index/inc/index_fst_registry.h index 2504d7ccff..1d89e57e52 100644 --- a/source/libs/index/inc/index_fst_registry.h +++ b/source/libs/index/inc/index_fst_registry.h @@ -15,6 +15,10 @@ #ifndef __FST_REGISTRY_H__ #define __FST_REGISTRY_H__ +#ifdef __cplusplus +extern "C" { +#endif + #include "index_fst_util.h" #include "tarray.h" #include "index_fst_node.h" @@ -59,4 +63,8 @@ void fstRegistryDestroy(FstRegistry *registry); FstRegistryEntry* fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNode); void fstRegistryEntryDestroy(FstRegistryEntry *entry); +#ifdef __cplusplus +} +#endif + #endif diff --git a/source/libs/index/inc/index_fst_util.h b/source/libs/index/inc/index_fst_util.h index d87e5bd57e..4af885816f 100644 --- a/source/libs/index/inc/index_fst_util.h +++ b/source/libs/index/inc/index_fst_util.h @@ -17,6 +17,10 @@ #ifndef __INDEX_FST_UTIL_H__ #define __INDEX_FST_UTIL_H__ +#ifdef __cplusplus +extern "C" { +#endif + #include "tarray.h" #include "index_fst_common.h" @@ -89,5 +93,8 @@ uint8_t *fstSliceData(FstSlice *s, int32_t *sz); #define FST_SLICE_LEN(s) (s->end - s->start + 1) +#ifdef __cplusplus +} +#endif #endif diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 68d2011b8f..9e5c1f5d78 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -52,7 +52,7 @@ void fstUnFinishedNodesPushEmpty(FstUnFinishedNodes *nodes, bool isFinal) { FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); node->isFinal = isFinal; node->finalOutput = 0; - node->trans = NULL; + node->trans = taosArrayInit(16, sizeof(FstTransition)); FstBuilderNodeUnfinished un = {.node = node, .last = NULL}; taosArrayPush(nodes->stack, &un); @@ -112,7 +112,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; - n->trans = NULL; + n->trans = taosArrayInit(16, sizeof(FstTransition)); //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); //trn->inp = s->data[i]; @@ -806,13 +806,13 @@ void fstBuilderInsertOutput(FstBuilder *b, FstSlice bs, Output in) { fstUnFinishedNodesSetRootOutput(b->unfinished, in); return; } - Output out; //if (in != 0) { //if let Some(in) = in // prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); //} else { // prefixLen = fstUnFinishedNodesFindCommPrefix(b->unfinished, bs); // out = 0; //} + Output out; uint64_t prefixLen = fstUnFinishedNodesFindCommPrefixAndSetOutput(b->unfinished, bs, in, &out); if (prefixLen == FST_SLICE_LEN(s)) { @@ -857,7 +857,7 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { } addr = fstBuilderCompile(b, n); assert(addr != NONE_ADDRESS); - fstBuilderNodeDestroy(n); + //fstBuilderNodeDestroy(n); } fstUnFinishedNodesTopLastFreeze(b->unfinished, addr); return; diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index f04f6eddad..1989a5a425 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -81,7 +81,7 @@ FstCountingWriter *fstCountingWriterCreate(void *wrt) { FstCountingWriter *cw = calloc(1, sizeof(FstCountingWriter)); if (cw == NULL) { return NULL; } - cw->wrt = (void *)(writerCtxCreate(TMemory)); + cw->wrt = (void *)(writerCtxCreate(TFile)); return cw; } void fstCountingWriterDestroy(FstCountingWriter *cw) { diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index f2a7442a5a..f84f874a23 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -1,7 +1,6 @@ add_executable(indexTest "") target_sources(indexTest PRIVATE - "../src/index.c" "indexTests.cpp" ) target_include_directories ( indexTest diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 857060ce5e..b94a241489 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -3,58 +3,79 @@ #include #include "index.h" #include "indexInt.h" +#include "index_fst.h" +#include "index_fst_util.h" +#include "index_fst_counting_writer.h" -TEST(IndexTest, index_create_test) { - SIndexOpts *opts = indexOptsCreate(); - SIndex *index = indexOpen(opts, "./test"); - if (index == NULL) { - std::cout << "index open failed" << std::endl; - } +//TEST(IndexTest, index_create_test) { +// SIndexOpts *opts = indexOptsCreate(); +// SIndex *index = indexOpen(opts, "./test"); +// if (index == NULL) { +// std::cout << "index open failed" << std::endl; +// } +// +// +// // write +// for (int i = 0; i < 100000; i++) { +// SIndexMultiTerm* terms = indexMultiTermCreate(); +// std::string val = "field"; +// +// indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size()); +// +// val.append(std::to_string(i)); +// indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size()); +// +// val.insert(0, std::to_string(i)); +// indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size()); +// +// val.append("const"); +// indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size()); +// +// +// indexPut(index, terms, i); +// indexMultiTermDestroy(terms); +// } +// +// +// // query +// SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); +// +// indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX); +// indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM); +// +// SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); +// indexSearch(index, multiQuery, result); +// +// std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl; +// for (int i = 0; i < taosArrayGetSize(result); i++) { +// int *v = (int *)taosArrayGet(result, i); +// std::cout << "value --->" << *v << std::endl; +// } +// // add more test case +// indexMultiTermQueryDestroy(multiQuery); +// +// indexOptsDestroy(opts); +// indexClose(index); +// // +//} - - // write - for (int i = 0; i < 100000; i++) { - SIndexMultiTerm* terms = indexMultiTermCreate(); - std::string val = "field"; - - indexMultiTermAdd(terms, "tag1", strlen("tag1"), val.c_str(), val.size()); - - val.append(std::to_string(i)); - indexMultiTermAdd(terms, "tag2", strlen("tag2"), val.c_str(), val.size()); - - val.insert(0, std::to_string(i)); - indexMultiTermAdd(terms, "tag3", strlen("tag3"), val.c_str(), val.size()); - - val.append("const"); - indexMultiTermAdd(terms, "tag4", strlen("tag4"), val.c_str(), val.size()); - - - indexPut(index, terms, i); - indexMultiTermDestroy(terms); - } - - - // query - SIndexMultiTermQuery *multiQuery = indexMultiTermQueryCreate(MUST); - - indexMultiTermQueryAdd(multiQuery, "tag1", strlen("tag1"), "field", strlen("field"), QUERY_PREFIX); - indexMultiTermQueryAdd(multiQuery, "tag3", strlen("tag3"), "0field0", strlen("0field0"), QUERY_TERM); - - SArray *result = (SArray *)taosArrayInit(10, sizeof(int)); - indexSearch(index, multiQuery, result); - - std::cout << "taos'size : " << taosArrayGetSize(result) << std::endl; - for (int i = 0; i < taosArrayGetSize(result); i++) { - int *v = (int *)taosArrayGet(result, i); - std::cout << "value --->" << *v << std::endl; - } - // add more test case - indexMultiTermQueryDestroy(multiQuery); - - indexOptsDestroy(opts); - indexClose(index); - // +int main(int argc, char** argv) { + std::string str("Hello world"); + FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); + Output val = 10; + FstBuilder *b = fstBuilderCreate(NULL, 1); + fstBuilderInsert(b, key, val); + fstBuilderFinish(b); + fstBuilderDestroy(b); + fstSliceDestroy(&key); + return 1; } + +//TEST(IndexFstBuilder, IndexFstInput) { +// +//} + + From 0f682ca20a2d162101f0bacb4f4e89632ae7d03a Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 3 Dec 2021 14:16:32 +0800 Subject: [PATCH 10/18] refactor builder struct --- source/libs/index/inc/index_fst_node.h | 1 + source/libs/index/src/index_fst.c | 1 - source/libs/index/src/index_fst_node.c | 32 +++++++++++++++++++--- source/libs/index/src/index_fst_registry.c | 8 +++--- source/libs/index/test/indexTests.cpp | 7 ++++- 5 files changed, 39 insertions(+), 10 deletions(-) diff --git a/source/libs/index/inc/index_fst_node.h b/source/libs/index/inc/index_fst_node.h index d1b45a16ab..87eb7cb746 100644 --- a/source/libs/index/inc/index_fst_node.h +++ b/source/libs/index/inc/index_fst_node.h @@ -46,6 +46,7 @@ FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src); void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src); //bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr); +bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2); void fstBuilderNodeDestroy(FstBuilderNode *node); diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index 9e5c1f5d78..b2d21cd0b0 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -167,7 +167,6 @@ uint64_t fstUnFinishedNodesFindCommPrefixAndSetOutput(FstUnFinishedNodes *node, } if (addPrefix != 0) { fstBuilderNodeUnfinishedAddOutputPrefix(un, addPrefix); - } } return i; diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index b33b8e4428..af69f98f32 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -18,7 +18,7 @@ FstBuilderNode *fstBuilderNodeDefault() { FstBuilderNode *bn = malloc(sizeof(FstBuilderNode)); bn->isFinal = false; bn->finalOutput = 0; - bn->trans = NULL; + bn->trans = taosArrayInit(16, sizeof(FstTransition)); return bn; } void fstBuilderNodeDestroy(FstBuilderNode *node) { @@ -27,6 +27,25 @@ void fstBuilderNodeDestroy(FstBuilderNode *node) { taosArrayDestroy(node->trans); free(node); } + +bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { + if (n1 == n2) { return true; } + + if (n1->isFinal != n2->isFinal || + n1->finalOutput != n2->finalOutput || + taosArrayGetSize(n1->trans) != taosArrayGetSize(n2->trans)) { + return false; + } + size_t sz = taosArrayGetSize(n1->trans); + for (size_t i = 0; i < sz; i++) { + FstTransition *t1 = taosArrayGet(n1->trans, i); + FstTransition *t2 = taosArrayGet(n2->trans, i); + if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { + return false; + } + } + return true; +} FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { FstBuilderNode *node = malloc(sizeof(FstBuilderNode)); if (node == NULL) { return NULL; } @@ -53,12 +72,17 @@ void fstBuilderNodeCloneFrom(FstBuilderNode *dst, FstBuilderNode *src) { dst->isFinal = src->isFinal; dst->finalOutput = src->finalOutput; - // avoid mem leak + //release free avoid mem leak taosArrayDestroy(dst->trans); - dst->trans = src->trans; - src->trans = NULL; + size_t sz = taosArrayGetSize(src->trans); + dst->trans = taosArrayInit(sz, sizeof(FstTransition)); + for (size_t i = 0; i < sz; i++) { + FstTransition *trn = taosArrayGet(src->trans, i); + taosArrayPush(dst->trans, trn); + } } + //bool fstBuilderNodeCompileTo(FstBuilderNode *b, FstCountingWriter *wrt, CompiledAddr lastAddr, CompiledAddr startAddr) { //size_t sz = taosArrayGetSize(b->trans); diff --git a/source/libs/index/src/index_fst_registry.c b/source/libs/index/src/index_fst_registry.c index b25964e0e7..8fb0dbfcaa 100644 --- a/source/libs/index/src/index_fst_registry.c +++ b/source/libs/index/src/index_fst_registry.c @@ -112,7 +112,7 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo if (end - start == 1) { FstRegistryCell *cell = taosArrayGet(registry->table, start); //cell->isNode && - if (cell->addr != NONE_ADDRESS && cell->node == bNode) { + if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { entry->state = FOUND; entry->addr = cell->addr ; return entry; @@ -123,13 +123,13 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo } } else if (end - start == 2) { FstRegistryCell *cell1 = taosArrayGet(registry->table, start); - if (cell1->addr != NONE_ADDRESS && cell1->node == bNode) { + if (cell1->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell1->node, bNode)) { entry->state = FOUND; entry->addr = cell1->addr; return entry; } FstRegistryCell *cell2 = taosArrayGet(registry->table, start + 1); - if (cell2->addr != NONE_ADDRESS && cell2->node == bNode) { + if (cell2->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell2->node, bNode)) { entry->state = FOUND; entry->addr = cell2->addr; // must swap here @@ -147,7 +147,7 @@ FstRegistryEntry *fstRegistryGetEntry(FstRegistry *registry, FstBuilderNode *bNo uint32_t i = start; for (; i < end; i++) { FstRegistryCell *cell = (FstRegistryCell *)taosArrayGet(registry->table, i); - if (cell->addr != NONE_ADDRESS && cell->node == bNode) { + if (cell->addr != NONE_ADDRESS && fstBuilderNodeEqual(cell->node, bNode)) { entry->state = FOUND; entry->addr = cell->addr; fstRegistryCellPromote(registry->table, i, start); diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index b94a241489..9f3d1b962e 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -63,11 +63,16 @@ //} int main(int argc, char** argv) { - std::string str("Hello world"); + std::string str("abc"); FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); Output val = 10; + + std::string str1("bcd"); + FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size()); + Output val2 = 10; FstBuilder *b = fstBuilderCreate(NULL, 1); fstBuilderInsert(b, key, val); + fstBuilderInsert(b, key1, val2); fstBuilderFinish(b); fstBuilderDestroy(b); fstSliceDestroy(&key); From 6de9692b7d759d85fadcd2001cbcae748129ee58 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Fri, 3 Dec 2021 14:28:54 +0800 Subject: [PATCH 11/18] refactor wal --- include/libs/wal/wal.h | 28 +++++--- include/util/tfile.h | 1 + source/libs/wal/inc/walInt.h | 4 +- source/libs/wal/src/walIndex.c | 125 ++++++++++++++++----------------- source/libs/wal/src/walMgmt.c | 66 +++++++---------- source/libs/wal/src/walWrite.c | 113 ++++++++++++++++++++++++++++- source/util/src/tfile.c | 5 ++ 7 files changed, 225 insertions(+), 117 deletions(-) diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index e77540bf90..b514648bbd 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -39,6 +39,14 @@ typedef enum { } EWalType; typedef struct { + //union { + //uint32_t info; + //struct { + //uint32_t sver:3; + //uint32_t msgtype: 5; + //uint32_t reserved : 24; + //}; + //}; int8_t sver; uint8_t msgType; int8_t reserved[2]; @@ -71,13 +79,17 @@ typedef struct { #define WAL_FILESET_MAX 128 #define WAL_IDX_ENTRY_SIZE (sizeof(int64_t)*2) -#define WAL_CUR_POS_READ_ONLY 1 -#define WAL_CUR_FILE_READ_ONLY 2 +#define WAL_CUR_POS_WRITABLE 1 +#define WAL_CUR_FILE_WRITABLE 2 +#define WAL_CUR_FAILED 4 typedef struct SWal { // cfg int32_t vgId; int32_t fsyncPeriod; // millisecond + int32_t fsyncSeq; + int32_t rollPeriod; // second + int64_t segSize; EWalType level; //reference int64_t refId; @@ -86,7 +98,7 @@ typedef struct SWal { int64_t curIdxTfd; //current version int64_t curVersion; - int64_t curOffset; + int64_t curLogOffset; //current file version int64_t curFileFirstVersion; int64_t curFileLastVersion; @@ -94,8 +106,10 @@ typedef struct SWal { int64_t firstVersion; int64_t snapshotVersion; int64_t lastVersion; - //fsync status - int32_t fsyncSeq; + int64_t lastFileName; + //roll status + int64_t lastRollSeq; + int64_t lastFileWriteSize; //ctl int32_t curStatus; pthread_mutex_t mutex; @@ -119,12 +133,10 @@ int32_t walAlter(SWal *, SWalCfg *pCfg); void walClose(SWal *); // write -//int64_t walWriteWithMsgType(SWal*, int8_t msgType, void* body, int32_t bodyLen); int64_t walWrite(SWal *, int64_t index, uint8_t msgType, void *body, int32_t bodyLen); -//int64_t walWriteBatch(SWal *, void **bodies, int32_t *bodyLen, int32_t batchSize); +void walFsync(SWal *, bool force); // apis for lifecycle management -void walFsync(SWal *, bool force); int32_t walCommit(SWal *, int64_t ver); // truncate after int32_t walRollback(SWal *, int64_t ver); diff --git a/include/util/tfile.h b/include/util/tfile.h index ff62c9e341..3d0e2177ac 100644 --- a/include/util/tfile.h +++ b/include/util/tfile.h @@ -28,6 +28,7 @@ void tfCleanup(); // the same syntax as UNIX standard open/close/read/write // but FD is int64_t and will never be reused +int64_t tfOpenRead(const char *pathname); int64_t tfOpenReadWrite(const char *pathname); int64_t tfOpenCreateWrite(const char *pathname); int64_t tfOpenCreateWriteAppend(const char *pathname); diff --git a/source/libs/wal/inc/walInt.h b/source/libs/wal/inc/walInt.h index 42ede49c6b..285d7e2576 100644 --- a/source/libs/wal/inc/walInt.h +++ b/source/libs/wal/inc/walInt.h @@ -17,14 +17,16 @@ #define _TD_WAL_INT_H_ #include "wal.h" +#include "compare.h" #ifdef __cplusplus extern "C" { #endif -int walRotate(SWal* pWal); int walGetFile(SWal* pWal, int32_t version); +int64_t walGetSeq(); + #ifdef __cplusplus } #endif diff --git a/source/libs/wal/src/walIndex.c b/source/libs/wal/src/walIndex.c index 2569af841f..1aa64b34b5 100644 --- a/source/libs/wal/src/walIndex.c +++ b/source/libs/wal/src/walIndex.c @@ -20,9 +20,38 @@ #include "tfile.h" #include "walInt.h" -int walSeekVerImpl(SWal *pWal, int64_t ver) { - //close old file +static int walSeekFilePos(SWal* pWal, int64_t ver) { int code = 0; + + int64_t idxTfd = pWal->curIdxTfd; + int64_t logTfd = pWal->curLogTfd; + + //seek position + int64_t offset = (ver - pWal->curFileFirstVersion) * WAL_IDX_ENTRY_SIZE; + code = tfLseek(idxTfd, offset, SEEK_SET); + if(code != 0) { + + } + int64_t readBuf[2]; + code = tfRead(idxTfd, readBuf, sizeof(readBuf)); + if(code != 0) { + + } + //TODO:deserialize + ASSERT(readBuf[0] == ver); + code = tfLseek(logTfd, readBuf[1], SEEK_CUR); + if (code != 0) { + + } + pWal->curLogOffset = readBuf[1]; + pWal->curVersion = ver; + return code; +} + +static int walChangeFile(SWal *pWal, int64_t ver) { + int code = 0; + int64_t idxTfd, logTfd; + char fnameStr[WAL_FILE_LEN]; code = tfClose(pWal->curLogTfd); if(code != 0) { //TODO @@ -32,29 +61,36 @@ int walSeekVerImpl(SWal *pWal, int64_t ver) { //TODO } //bsearch in fileSet - int fName = 0;//TODO - //open the right file - char fNameStr[WAL_FILE_LEN]; - sprintf(fNameStr, "%d."WAL_INDEX_SUFFIX, fName); - bool closed = 1; //TODO:read only - int64_t idxTfd = tfOpenReadWrite(fNameStr); - sprintf(fNameStr, "%d."WAL_LOG_SUFFIX, fName); - int64_t logTfd = tfOpenReadWrite(fNameStr); - //seek position - int64_t offset = (ver - fName) * WAL_IDX_ENTRY_SIZE; - tfLseek(idxTfd, offset, SEEK_SET); - //set cur version, cur file version and cur status - pWal->curFileFirstVersion = fName; - pWal->curFileLastVersion = 1;//TODO + int64_t* pRet = taosArraySearch(pWal->fileSet, &ver, compareInt64Val, TD_LE); + ASSERT(pRet != NULL); + int64_t fname = *pRet; + if(fname < pWal->lastFileName) { + pWal->curStatus &= ~WAL_CUR_FILE_WRITABLE; + pWal->curFileLastVersion = pRet[1]-1; + sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + idxTfd = tfOpenRead(fnameStr); + sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + logTfd = tfOpenRead(fnameStr); + } else { + pWal->curStatus |= WAL_CUR_FILE_WRITABLE; + pWal->curFileLastVersion = -1; + sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + idxTfd = tfOpenReadWrite(fnameStr); + sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + logTfd = tfOpenReadWrite(fnameStr); + } + + pWal->curFileFirstVersion = fname; pWal->curLogTfd = logTfd; pWal->curIdxTfd = idxTfd; - pWal->curVersion = ver; - pWal->curOffset = offset; - pWal->curStatus = 0;//TODO return code; } int walSeekVer(SWal *pWal, int64_t ver) { + if((!(pWal->curStatus & WAL_CUR_FAILED)) + && ver == pWal->curVersion) { + return 0; + } if(ver > pWal->lastVersion) { //TODO: some records are skipped return -1; @@ -64,54 +100,13 @@ int walSeekVer(SWal *pWal, int64_t ver) { return -1; } if(ver < pWal->snapshotVersion) { - //TODO: seek snapshotted log + //TODO: seek snapshotted log, invalid in some cases } - if(ver >= pWal->curFileFirstVersion - && ((pWal->curFileLastVersion == -1 && ver <= pWal->lastVersion) || (ver <= pWal->curFileLastVersion))) { - - } - if(ver < pWal->curFileFirstVersion || (pWal->curFileLastVersion != -1 && ver > pWal->curFileLastVersion)) { - int index = 0; - index = 1; - //back up to avoid inconsistency - int64_t curVersion = pWal->curVersion; - int64_t curOffset = pWal->curOffset; - int64_t curFileFirstVersion = pWal->curFileFirstVersion; - int64_t curFileLastVersion = pWal->curFileLastVersion; - if(walSeekVerImpl(pWal, ver) < 0) { - //TODO: errno - pWal->curVersion = curVersion; - pWal->curOffset = curOffset; - pWal->curFileFirstVersion = curFileFirstVersion; - pWal->curFileLastVersion = curFileLastVersion; - return -1; - } + if(ver < pWal->curFileFirstVersion || + (pWal->curFileLastVersion != -1 && ver > pWal->curFileLastVersion)) { + walChangeFile(pWal, ver); } + walSeekFilePos(pWal, ver); return 0; } - -int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { - int code = 0; - //get index file - if(!tfValid(pWal->curIdxTfd)) { - code = TAOS_SYSTEM_ERROR(errno); - wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); - } - if(pWal->curVersion != ver) { - if(walSeekVer(pWal, ver) != 0) { - //TODO: some records are skipped - return -1; - } - } - //check file checksum - //append index - return 0; -} - -int walRotateIndex(SWal *pWal) { - //check file checksum - //create new file - //switch file - return 0; -} diff --git a/source/libs/wal/src/walMgmt.c b/source/libs/wal/src/walMgmt.c index d60cdfe118..bc2e687069 100644 --- a/source/libs/wal/src/walMgmt.c +++ b/source/libs/wal/src/walMgmt.c @@ -26,57 +26,44 @@ int32_t walGetNextFile(SWal *pWal, int64_t *nextFileId); int32_t walGetOldFile(SWal *pWal, int64_t curFileId, int32_t minDiff, int64_t *oldFileId); int32_t walGetNewFile(SWal *pWal, int64_t *newFileId); -static pthread_mutex_t walInitLock = PTHREAD_MUTEX_INITIALIZER; -static int8_t walInited = 0; - typedef struct { int32_t refSetId; - int32_t seq; + uint32_t seq; int8_t stop; + int8_t inited; pthread_t thread; - pthread_mutex_t mutex; } SWalMgmt; -static SWalMgmt tsWal = {0}; +static SWalMgmt tsWal = {0, .seq = 1}; static int32_t walCreateThread(); static void walStopThread(); static int32_t walInitObj(SWal *pWal); static void walFreeObj(void *pWal); -int32_t walInit() { - //TODO: change to atomic - pthread_mutex_lock(&walInitLock); - if(walInited) { - pthread_mutex_unlock(&walInitLock); - return 0; - } else { - walInited = 1; - pthread_mutex_unlock(&walInitLock); - } +int64_t walGetSeq() { + return (int64_t)atomic_load_32(&tsWal.seq); +} + +int32_t walInit() { + int8_t old = atomic_val_compare_exchange_8(&tsWal.inited, 0, 1); + if(old == 1) return 0; - int32_t code = 0; tsWal.refSetId = taosOpenRef(TSDB_MIN_VNODES, walFreeObj); - code = pthread_mutex_init(&tsWal.mutex, NULL); - if (code != 0) { - wError("failed to init wal mutex since %s", tstrerror(code)); - return code; - } - - code = walCreateThread(); + int code = walCreateThread(); if (code != 0) { wError("failed to init wal module since %s", tstrerror(code)); + atomic_store_8(&tsWal.inited, 0); return code; } wInfo("wal module is initialized, rsetId:%d", tsWal.refSetId); - return code; + return 0; } void walCleanUp() { walStopThread(); taosCloseRef(tsWal.refSetId); - pthread_mutex_destroy(&tsWal.mutex); wInfo("wal module is cleaned up"); } @@ -92,7 +79,7 @@ static int walLoadFileset(SWal *pWal) { char *name = ent->d_name; name[WAL_NOSUFFIX_LEN] = 0; //validate file name by regex matching - if(1 /* regex match */) { + if(1 /* TODO:regex match */) { int64_t fnameInt64 = atoll(name); taosArrayPush(pWal->fileSet, &fnameInt64); } @@ -133,6 +120,7 @@ SWal *walOpen(const char *path, SWalCfg *pCfg) { walFreeObj(pWal); return NULL; } + walLoadFileset(pWal); wDebug("vgId:%d, wal:%p is opened, level:%d fsyncPeriod:%d", pWal->vgId, pWal, pWal->level, pWal->fsyncPeriod); @@ -164,6 +152,9 @@ void walClose(SWal *pWal) { pthread_mutex_lock(&pWal->mutex); tfClose(pWal->curLogTfd); + tfClose(pWal->curIdxTfd); + taosArrayDestroy(pWal->fileSet); + pWal->fileSet = NULL; pthread_mutex_unlock(&pWal->mutex); taosRemoveRef(tsWal.refSetId, pWal->refId); } @@ -188,6 +179,9 @@ static void walFreeObj(void *wal) { wDebug("vgId:%d, wal:%p is freed", pWal->vgId, pWal); tfClose(pWal->curLogTfd); + tfClose(pWal->curIdxTfd); + taosArrayDestroy(pWal->fileSet); + pWal->fileSet = NULL; pthread_mutex_destroy(&pWal->mutex); tfree(pWal); } @@ -197,7 +191,7 @@ static bool walNeedFsync(SWal *pWal) { return false; } - if (tsWal.seq % pWal->fsyncSeq == 0) { + if (atomic_load_32(&tsWal.seq) % pWal->fsyncSeq == 0) { return true; } @@ -206,16 +200,14 @@ static bool walNeedFsync(SWal *pWal) { static void walUpdateSeq() { taosMsleep(WAL_REFRESH_MS); - if (++tsWal.seq <= 0) { - tsWal.seq = 1; - } + atomic_add_fetch_32(&tsWal.seq, 1); } static void walFsyncAll() { SWal *pWal = taosIterateRef(tsWal.refSetId, 0); while (pWal) { if (walNeedFsync(pWal)) { - wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, tsWal.seq); + wTrace("vgId:%d, do fsync, level:%d seq:%d rseq:%d", pWal->vgId, pWal->level, pWal->fsyncSeq, atomic_load_32(&tsWal.seq)); int32_t code = tfFsync(pWal->curLogTfd); if (code != 0) { wError("vgId:%d, file:%"PRId64".log, failed to fsync since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(code)); @@ -226,16 +218,12 @@ static void walFsyncAll() { } static void *walThreadFunc(void *param) { - int stop = 0; setThreadName("wal"); while (1) { walUpdateSeq(); walFsyncAll(); - pthread_mutex_lock(&tsWal.mutex); - stop = tsWal.stop; - pthread_mutex_unlock(&tsWal.mutex); - if (stop) break; + if (atomic_load_8(&tsWal.stop)) break; } return NULL; @@ -258,9 +246,7 @@ static int32_t walCreateThread() { } static void walStopThread() { - pthread_mutex_lock(&tsWal.mutex); - tsWal.stop = 1; - pthread_mutex_unlock(&tsWal.mutex); + atomic_store_8(&tsWal.stop, 1); if (taosCheckPthreadValid(tsWal.thread)) { pthread_join(tsWal.thread, NULL); diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 7563ec02c7..69c83a9912 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -26,10 +26,24 @@ int32_t walCommit(SWal *pWal, int64_t ver) { } int32_t walRollback(SWal *pWal, int64_t ver) { + //TODO: ftruncate return 0; } int32_t walTakeSnapshot(SWal *pWal, int64_t ver) { + pWal->snapshotVersion = ver; + + //mark files safe to delete + int64_t* pRet = taosArraySearch(pWal->fileSet, &ver, compareInt64Val, TD_LE); + if(pRet != pWal->fileSet->pData) { + //delete files until less than retention size + + //find first file that exceeds retention time + + } + + //delete files living longer than retention limit + //remove file from fileset return 0; } @@ -124,13 +138,102 @@ void walRemoveAllOldFiles(void *handle) { } #endif +static int walRoll(SWal *pWal) { + int code = 0; + code = tfClose(pWal->curIdxTfd); + if(code != 0) { + return code; + } + code = tfClose(pWal->curLogTfd); + if(code != 0) { + return code; + } + int64_t idxTfd, logTfd; + //create new file + int64_t newFileFirstVersion = pWal->lastVersion + 1; + char fnameStr[WAL_FILE_LEN]; + sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, newFileFirstVersion); + idxTfd = tfOpenCreateWrite(fnameStr); + sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, newFileFirstVersion); + logTfd = tfOpenCreateWrite(fnameStr); + + taosArrayPush(pWal->fileSet, &newFileFirstVersion); + + //switch file + pWal->curIdxTfd = idxTfd; + pWal->curLogTfd = logTfd; + //change status + pWal->curFileLastVersion = -1; + pWal->curFileFirstVersion = newFileFirstVersion; + pWal->curVersion = newFileFirstVersion; + pWal->curLogOffset = 0; + pWal->curStatus = WAL_CUR_FILE_WRITABLE & WAL_CUR_POS_WRITABLE; + + pWal->lastFileName = newFileFirstVersion; + pWal->lastFileWriteSize = 0; + pWal->lastRollSeq = walGetSeq(); + return 0; +} + +int walChangeFileToLast(SWal *pWal) { + int64_t idxTfd, logTfd; + int64_t* pRet = taosArrayGetLast(pWal->fileSet); + ASSERT(pRet != NULL); + int64_t fname = *pRet; + + char fnameStr[WAL_FILE_LEN]; + sprintf(fnameStr, "%"PRId64"."WAL_INDEX_SUFFIX, fname); + idxTfd = tfOpenReadWrite(fnameStr); + sprintf(fnameStr, "%"PRId64"."WAL_LOG_SUFFIX, fname); + logTfd = tfOpenReadWrite(fnameStr); + //switch file + pWal->curIdxTfd = idxTfd; + pWal->curLogTfd = logTfd; + //change status + pWal->curFileLastVersion = -1; + pWal->curFileFirstVersion = fname; + pWal->curVersion = fname; + pWal->curLogOffset = 0; + pWal->curStatus = WAL_CUR_FILE_WRITABLE; + return 0; +} + +int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { + int code = 0; + //get index file + if(!tfValid(pWal->curIdxTfd)) { + code = TAOS_SYSTEM_ERROR(errno); + wError("vgId:%d, file:%"PRId64".idx, failed to open since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); + } + int64_t writeBuf[2] = { ver, offset }; + int size = tfWrite(pWal->curIdxTfd, writeBuf, sizeof(writeBuf)); + if(size != sizeof(writeBuf)) { + //TODO: + } + return 0; +} + int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t bodyLen) { if (pWal == NULL) return -1; // no wal - if (!tfValid(pWal->curLogTfd)) return 0; if (pWal->level == TAOS_WAL_NOLOG) return 0; - if (index > pWal->lastVersion + 1) return -1; + + if (index == pWal->lastVersion + 1) { + int64_t passed = walGetSeq() - pWal->lastRollSeq; + if(passed > pWal->rollPeriod) { + walRoll(pWal); + } else if(pWal->lastFileWriteSize > pWal->segSize) { + walRoll(pWal); + } else { + walChangeFileToLast(pWal); + } + } else { + //reject skip log or rewrite log + //must truncate explicitly first + return -1; + } + if (!tfValid(pWal->curLogTfd)) return 0; pWal->head.version = index; int32_t code = 0; @@ -155,8 +258,12 @@ int64_t walWrite(SWal *pWal, int64_t index, uint8_t msgType, void *body, int32_t code = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%"PRId64".log, failed to write since %s", pWal->vgId, pWal->curFileFirstVersion, strerror(errno)); } - //TODO:write idx + walWriteIndex(pWal, index, pWal->curLogOffset); + pWal->curLogOffset += sizeof(SWalHead) + bodyLen; + //set status + pWal->lastVersion = index; + pthread_mutex_unlock(&pWal->mutex); return code; diff --git a/source/util/src/tfile.c b/source/util/src/tfile.c index 5b61186b12..5d4789aae6 100644 --- a/source/util/src/tfile.c +++ b/source/util/src/tfile.c @@ -53,6 +53,11 @@ static int64_t tfOpenImp(int32_t fd) { return rid; } +int64_t tfOpenRead(const char *pathname, int32_t flags) { + int32_t fd = taosOpenFileRead(pathname); + return tfOpenImp(fd); +} + int64_t tfOpenReadWrite(const char *pathname, int32_t flags) { int32_t fd = taosOpenFileReadWrite(pathname); return tfOpenImp(fd); From 5857ab5b28a6fa6a21749b3bc2d2b54d81688962 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 3 Dec 2021 14:36:41 +0800 Subject: [PATCH 12/18] TD-10431 process mnode profile --- include/common/taosmsg.h | 51 +- include/util/tdef.h | 2 +- include/util/tutil.h | 4 +- source/dnode/mnode/impl/inc/mndDb.h | 2 + source/dnode/mnode/impl/inc/mndDef.h | 8 +- source/dnode/mnode/impl/inc/mndInt.h | 10 +- source/dnode/mnode/impl/inc/mndMnode.h | 1 + source/dnode/mnode/impl/inc/mndShow.h | 6 +- source/dnode/mnode/impl/src/mndCluster.c | 6 +- source/dnode/mnode/impl/src/mndConnect.c | 155 ----- source/dnode/mnode/impl/src/mndDb.c | 48 +- source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndProfile.c | 761 ++++++++++++++++++++++- source/dnode/mnode/impl/src/mndShow.c | 10 +- source/dnode/mnode/impl/src/mndUser.c | 21 +- source/util/src/tutil.c | 7 + 16 files changed, 862 insertions(+), 234 deletions(-) delete mode 100644 source/dnode/mnode/impl/src/mndConnect.c diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index 826934c4a8..e29accd249 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -353,11 +353,9 @@ typedef struct { } SUpdateTableTagValMsg; typedef struct { - char clientVersion[TSDB_VERSION_LEN]; - char msgVersion[TSDB_VERSION_LEN]; - char db[TSDB_TABLE_FNAME_LEN]; - char appName[TSDB_APPNAME_LEN]; int32_t pid; + char app[TSDB_APP_NAME_LEN]; + char db[TSDB_DB_NAME_LEN]; } SConnectMsg; typedef struct SEpSet { @@ -368,15 +366,14 @@ typedef struct SEpSet { } SEpSet; typedef struct { - char acctId[TSDB_ACCT_ID_LEN]; - char serverVersion[TSDB_VERSION_LEN]; - char clusterId[TSDB_CLUSTER_ID_LEN]; - int8_t writeAuth; - int8_t superAuth; - int8_t reserved1; - int8_t reserved2; - int32_t connId; - SEpSet epSet; + int32_t acctId; + int32_t clusterId; + int32_t connId; + int8_t superAuth; + int8_t readAuth; + int8_t writeAuth; + int8_t reserved[5]; + SEpSet epSet; } SConnectRsp; typedef struct { @@ -875,23 +872,23 @@ typedef struct { } SStreamDesc; typedef struct { - char clientVer[TSDB_VERSION_LEN]; - uint32_t connId; - int32_t pid; - int32_t numOfQueries; - int32_t numOfStreams; - char appName[TSDB_APPNAME_LEN]; - char pData[]; + int32_t connId; + int32_t pid; + int32_t numOfQueries; + int32_t numOfStreams; + char app[TSDB_APP_NAME_LEN]; + char pData[]; } SHeartBeatMsg; typedef struct { - uint32_t queryId; - uint32_t streamId; - uint32_t totalDnodes; - uint32_t onlineDnodes; - uint32_t connId; - int8_t killConnection; - SEpSet epSet; + int32_t connId; + int32_t queryId; + int32_t streamId; + int32_t totalDnodes; + int32_t onlineDnodes; + int8_t killConnection; + int8_t reserved[3]; + SEpSet epSet; } SHeartBeatRsp; typedef struct { diff --git a/include/util/tdef.h b/include/util/tdef.h index 76df8887a0..897f51f5c1 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -174,7 +174,7 @@ do { \ #define TSDB_MAX_SQL_SHOW_LEN 512 #define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb -#define TSDB_APPNAME_LEN TSDB_UNI_LEN +#define TSDB_APP_NAME_LEN TSDB_UNI_LEN /** * In some scenarios uint16_t (0~65535) is used to store the row len. diff --git a/include/util/tutil.h b/include/util/tutil.h index 8dbcb7e8d5..573dee9339 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -41,8 +41,10 @@ char * paGetToken(char *src, char **token, int32_t *tokenLen); int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]); int32_t taosHexStrToByteArray(char hexstr[], char bytes[]); -char * taosIpStr(uint32_t ipInt); +char *taosIpStr(uint32_t ipInt); uint32_t ip2uint(const char *const ip_addr); +void taosIp2String(uint32_t ip, char *str); +void taosIpPort2String(uint32_t ip, uint16_t port, char *str); static FORCE_INLINE void taosEncryptPass(uint8_t *inBuf, size_t inLen, char *target) { MD5_CTX context; diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index acccb62603..91f502be7d 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -24,6 +24,8 @@ extern "C" { int32_t mndInitDb(SMnode *pMnode); void mndCleanupDb(SMnode *pMnode); +SDbObj *mndAcquireDb(SMnode *pMnode, char *db); +void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index dc27656dbf..aaf86c15b6 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -185,9 +185,11 @@ typedef struct SUserObj { char acct[TSDB_USER_LEN]; int64_t createdTime; int64_t updateTime; - int8_t rootAuth; + int8_t superAuth; + int8_t readAuth; + int8_t writeAuth; + int32_t acctId; SHashObj *prohibitDbHash; - SAcctObj *pAcct; } SUserObj; typedef struct { @@ -303,6 +305,8 @@ typedef struct SShowObj { typedef struct SMnodeMsg { char user[TSDB_USER_LEN]; + char db[TSDB_FULL_DB_NAME_LEN]; + int32_t acctId; SMnode *pMnode; int16_t received; int16_t successed; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index e5643847a5..0f4dd0633b 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -28,8 +28,8 @@ extern "C" { typedef int32_t (*MndMsgFp)(SMnode *pMnode, SMnodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); -typedef int32_t (*ShowMetaFp)(SMnode *pMnode, STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); -typedef int32_t (*ShowRetrieveFp)(SMnode *pMnode, SShowObj *pShow, char *data, int32_t rows, void *pConn); +typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +typedef int32_t (*ShowRetrieveFp)(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); typedef void (*ShowFreeIterFp)(SMnode *pMnode, void *pIter); typedef struct { @@ -46,6 +46,11 @@ typedef struct { SCacheObj *cache; } SShowMgmt; +typedef struct { + int32_t connId; + SCacheObj *cache; +} SProfileMgmt; + typedef struct SMnode { int32_t dnodeId; int32_t clusterId; @@ -58,6 +63,7 @@ typedef struct SMnode { SDnode *pDnode; SArray *pSteps; SShowMgmt showMgmt; + SProfileMgmt profileMgmt; MndMsgFp msgFp[TSDB_MSG_TYPE_MAX]; SendMsgToDnodeFp sendMsgToDnodeFp; SendMsgToMnodeFp sendMsgToMnodeFp; diff --git a/source/dnode/mnode/impl/inc/mndMnode.h b/source/dnode/mnode/impl/inc/mndMnode.h index 53f7b733f2..64bbb13b60 100644 --- a/source/dnode/mnode/impl/inc/mndMnode.h +++ b/source/dnode/mnode/impl/inc/mndMnode.h @@ -25,6 +25,7 @@ extern "C" { int32_t mndInitMnode(SMnode *pMnode); void mndCleanupMnode(SMnode *pMnode); bool mndIsMnode(SMnode *pMnode, int32_t dnodeId); +void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndShow.h b/source/dnode/mnode/impl/inc/mndShow.h index 31c0b69c85..e7a3fcd45f 100644 --- a/source/dnode/mnode/impl/inc/mndShow.h +++ b/source/dnode/mnode/impl/inc/mndShow.h @@ -24,9 +24,9 @@ extern "C" { int32_t mndInitShow(SMnode *pMnode); void mndCleanupShow(SMnode *pMnode); -void mnodeAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); -void mnodeAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); -void mnodeAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); +void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp); +void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp); +void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType msgType, ShowFreeIterFp fp); void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index dbe5dd4f89..6013ac0c31 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -170,9 +170,9 @@ int32_t mndInitCluster(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndClusterActionUpdate, .deleteFp = (SdbDeleteFp)mndClusterActionDelete}; - // mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); - // mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); - // mnodeAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster); + // mndAddShowMetaHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeGetClusterMeta); + // mndAddShowRetrieveHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeRetrieveClusters); + // mndAddShowFreeIterHandle(TSDB_MGMT_TABLE_CLUSTER, mnodeCancelGetNextCluster); return sdbSetTable(pMnode->pSdb, table); } diff --git a/source/dnode/mnode/impl/src/mndConnect.c b/source/dnode/mnode/impl/src/mndConnect.c deleted file mode 100644 index 8e62eacfab..0000000000 --- a/source/dnode/mnode/impl/src/mndConnect.c +++ /dev/null @@ -1,155 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "mndShow.h" - - - -// static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { -// SHeartBeatRsp *pRsp = (SHeartBeatRsp *)rpcMallocCont(sizeof(SHeartBeatRsp)); -// if (pRsp == NULL) { -// return TSDB_CODE_MND_OUT_OF_MEMORY; -// } - -// SHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; -// if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) { -// rpcFreeCont(pRsp); -// return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code -// } - -// SRpcConnInfo connInfo = {0}; -// rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo); - -// int32_t connId = htonl(pHBMsg->connId); -// SConnObj *pConn = mnodeAccquireConn(connId, connInfo.user, connInfo.clientIp, connInfo.clientPort); -// if (pConn == NULL) { -// pHBMsg->pid = htonl(pHBMsg->pid); -// pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort, pHBMsg->pid, pHBMsg->appName); -// } - -// if (pConn == NULL) { -// // do not close existing links, otherwise -// // mError("failed to create connId, close connect"); -// // pRsp->killConnection = 1; -// } else { -// pRsp->connId = htonl(pConn->connId); -// mnodeSaveQueryStreamList(pConn, pHBMsg); - -// if (pConn->killed != 0) { -// pRsp->killConnection = 1; -// } - -// if (pConn->streamId != 0) { -// pRsp->streamId = htonl(pConn->streamId); -// pConn->streamId = 0; -// } - -// if (pConn->queryId != 0) { -// pRsp->queryId = htonl(pConn->queryId); -// pConn->queryId = 0; -// } -// } - -// int32_t onlineDnodes = 0, totalDnodes = 0; -// mnodeGetOnlineAndTotalDnodesNum(&onlineDnodes, &totalDnodes); - -// pRsp->onlineDnodes = htonl(onlineDnodes); -// pRsp->totalDnodes = htonl(totalDnodes); -// mnodeGetMnodeEpSetForShell(&pRsp->epSet, false); - -// pMsg->rpcRsp.rsp = pRsp; -// pMsg->rpcRsp.len = sizeof(SHeartBeatRsp); - -// mnodeReleaseConn(pConn); -// return TSDB_CODE_SUCCESS; -// } - -// static int32_t mnodeProcessConnectMsg(SMnodeMsg *pMsg) { -// SConnectMsg *pConnectMsg = pMsg->rpcMsg.pCont; -// SConnectRsp *pConnectRsp = NULL; -// int32_t code = TSDB_CODE_SUCCESS; - -// SRpcConnInfo connInfo = {0}; -// if (rpcGetConnInfo(pMsg->rpcMsg.handle, &connInfo) != 0) { -// mError("thandle:%p is already released while process connect msg", pMsg->rpcMsg.handle); -// code = TSDB_CODE_MND_INVALID_CONNECTION; -// goto connect_over; -// } - -// code = taosCheckVersion(pConnectMsg->clientVersion, version, 3); -// if (code != TSDB_CODE_SUCCESS) { -// goto connect_over; -// } - -// SUserObj *pUser = pMsg->pUser; -// SAcctObj *pAcct = pUser->pAcct; - -// if (pConnectMsg->db[0]) { -// char dbName[TSDB_TABLE_FNAME_LEN * 3] = {0}; -// sprintf(dbName, "%x%s%s", pAcct->acctId, TS_PATH_DELIMITER, pConnectMsg->db); -// SDbObj *pDb = mnodeGetDb(dbName); -// if (pDb == NULL) { -// code = TSDB_CODE_MND_INVALID_DB; -// goto connect_over; -// } - -// if (pDb->status != TSDB_DB_STATUS_READY) { -// mError("db:%s, status:%d, in dropping", pDb->name, pDb->status); -// code = TSDB_CODE_MND_DB_IN_DROPPING; -// mnodeDecDbRef(pDb); -// goto connect_over; -// } -// mnodeDecDbRef(pDb); -// } - -// pConnectRsp = rpcMallocCont(sizeof(SConnectRsp)); -// if (pConnectRsp == NULL) { -// code = TSDB_CODE_MND_OUT_OF_MEMORY; -// goto connect_over; -// } - -// pConnectMsg->pid = htonl(pConnectMsg->pid); -// SConnObj *pConn = mnodeCreateConn(connInfo.user, connInfo.clientIp, connInfo.clientPort, pConnectMsg->pid, pConnectMsg->appName); -// if (pConn == NULL) { -// code = terrno; -// } else { -// pConnectRsp->connId = htonl(pConn->connId); -// mnodeReleaseConn(pConn); -// } - -// sprintf(pConnectRsp->acctId, "%x", pAcct->acctId); -// memcpy(pConnectRsp->serverVersion, version, TSDB_VERSION_LEN); -// pConnectRsp->writeAuth = pUser->writeAuth; -// pConnectRsp->superAuth = pUser->superAuth; - -// mnodeGetMnodeEpSetForShell(&pConnectRsp->epSet, false); - -// dnodeGetClusterId(pConnectRsp->clusterId); - -// connect_over: -// if (code != TSDB_CODE_SUCCESS) { -// if (pConnectRsp) rpcFreeCont(pConnectRsp); -// mLError("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); -// } else { -// mLInfo("user:%s login from %s, result:%s", connInfo.user, taosIpStr(connInfo.clientIp), tstrerror(code)); -// pMsg->rpcRsp.rsp = pConnectRsp; -// pMsg->rpcRsp.len = sizeof(SConnectRsp); -// } - -// return code; -// } - - diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0ad144d2df..d990e926bb 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -14,26 +14,38 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndDb.h" -int32_t mndInitDb(SMnode *pMnode) { return 0; } -void mndCleanupDb(SMnode *pMnode) {} +static int32_t mnodeProcessUseMsg(SMnode *pMnode, SMnodeMsg *pMsg); +int32_t mndInitDb(SMnode *pMnode) { + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mnodeProcessUseMsg); + return 0; +} -// static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { -// SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont; +void mndCleanupDb(SMnode *pMnode) {} -// int32_t code = TSDB_CODE_SUCCESS; -// if (pMsg->pDb == NULL) pMsg->pDb = mnodeGetDb(pUseDbMsg->db); -// if (pMsg->pDb == NULL) { -// return TSDB_CODE_MND_INVALID_DB; -// } - -// if (pMsg->pDb->status != TSDB_DB_STATUS_READY) { -// mError("db:%s, status:%d, in dropping", pMsg->pDb->name, pMsg->pDb->status); -// return TSDB_CODE_MND_DB_IN_DROPPING; -// } +SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_DB, db); +} -// return code; -// } +void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pDb); +} + +static int32_t mnodeProcessUseMsg(SMnode *pMnode, SMnodeMsg *pMsg) { + SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont; + + strncpy(pMsg->db, pUseDbMsg->db, TSDB_FULL_DB_NAME_LEN); + + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pDb != NULL) { + mndReleaseDb(pMnode, pDb); + return 0; + } else { + mError("db:%s, failed to process use db msg since %s", pMsg->db, terrstr()); + return -1; + } +} diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index d2ace31a36..c1c17e4983 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -135,4 +135,6 @@ bool mndIsMnode(SMnode *pMnode, int32_t dnodeId) { sdbRelease(pSdb, pMnodeObj); return true; -} \ No newline at end of file +} + +void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) {} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 845e50210a..ef596f8ee1 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -14,8 +14,761 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "mndInt.h" +#include "mndProfile.h" +#include "mndDb.h" +#include "mndMnode.h" +#include "mndShow.h" +#include "mndUser.h" -int32_t mndInitProfile(SMnode *pMnode) { return 0; } -void mndCleanupProfile(SMnode *pMnode) {} \ No newline at end of file +#define QUERY_ID_SIZE 20 +#define QUERY_OBJ_ID_SIZE 18 +#define SUBQUERY_INFO_SIZE 6 +#define QUERY_STREAM_SAVE_SIZE 20 + +typedef struct { + char user[TSDB_USER_LEN]; + char app[TSDB_APP_NAME_LEN]; // app name that invokes taosc + int32_t pid; // pid of app that invokes taosc + int32_t connId; + int8_t killed; + int8_t align; + uint16_t port; + uint32_t ip; + int64_t stime; + int64_t lastAccess; + int32_t queryId; + int32_t streamId; + int32_t numOfQueries; + int32_t numOfStreams; + SStreamDesc *pStreams; + SQueryDesc *pQueries; +} SConnObj; + +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app); +static void mndFreeConn(SConnObj *pConn); +static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId, char *user, uint32_t ip, uint16_t port); +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); +static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); +static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); +static int32_t mndProcessHeartBeatMsg(SMnode *pMnode, SMnodeMsg *pMsg); +static int32_t mndProcessConnectMsg(SMnode *pMnode, SMnodeMsg *pMsg); +static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); +static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow); +static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter); +static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow); +static int32_t mndRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, SMnodeMsg *pMsg); +static void mndCancelGetNextStream(SMnode *pMnode, void *pIter); + +int32_t mndInitProfile(SMnode *pMnode) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + int32_t connCheckTime = pMnode->shellActivityTimer * 2; + pMgmt->cache = taosCacheInit(TSDB_DATA_TYPE_INT, connCheckTime, true, (__cache_free_fn_t)mndFreeConn, "conn"); + if (pMgmt->cache == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("failed to alloc profile cache since %s", terrstr()); + return -1; + } + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_HEARTBEAT, mndProcessHeartBeatMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessConnectMsg); + + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); + return 0; +} + +void mndCleanupProfile(SMnode *pMnode) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + if (pMgmt->cache != NULL) { + taosCacheCleanup(pMgmt->cache); + pMgmt->cache = NULL; + } +} + +static SConnObj *mndCreateConn(SMnode *pMnode, char *user, uint32_t ip, uint16_t port, int32_t pid, const char *app) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + int32_t connId = atomic_add_fetch_32(&pMgmt->connId, 1); + if (connId == 0) atomic_add_fetch_32(&pMgmt->connId, 1); + + SConnObj connObj = {.pid = pid, + .connId = connId, + .killed = 0, + .port = port, + .ip = ip, + .stime = taosGetTimestampMs(), + .lastAccess = 0, + .queryId = 0, + .streamId = 0, + .numOfQueries = 0, + .numOfStreams = 0, + .pStreams = NULL, + .pQueries = NULL}; + + connObj.lastAccess = connObj.stime; + tstrncpy(connObj.user, user, TSDB_USER_LEN); + tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); + + int32_t keepTime = pMnode->shellActivityTimer * 3; + SConnObj *pConn = taosCachePut(pMgmt->cache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), keepTime * 1000); + + mDebug("conn:%d, is created, user:%s", connId, user); + return pConn; +} + +static void mndFreeConn(SConnObj *pConn) { + tfree(pConn->pQueries); + tfree(pConn->pStreams); + mDebug("conn:%d, is destroyed", pConn->connId); +} + +static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId, char *newUser, uint32_t newIp, uint16_t newPort) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + if (pConn == NULL) { + mDebug("conn:%d, already destroyed, user:%s", connId, newUser); + return NULL; + } + + if (pConn->ip != newIp || pConn->port != newPort /* || strcmp(pConn->user, newUser) != 0 */) { + char oldIpStr[30]; + char newIpStr[30]; + taosIp2String(pConn->ip, oldIpStr); + taosIp2String(newIp, newIpStr); + mDebug("conn:%d, incoming conn user:%s ip:%s:%u, not match exist user:%s ip:%s:%u", connId, newUser, newIpStr, + newPort, pConn->user, oldIpStr, pConn->port); + + if (pMgmt->connId < connId) pMgmt->connId = connId + 1; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + return NULL; + } + + int32_t keepTime = pMnode->shellActivityTimer * 3; + pConn->lastAccess = keepTime * 1000 + (uint64_t)taosGetTimestampMs(); + return pConn; +} + +static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + if (pConn == NULL) return; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); +} + +static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + *pConn = NULL; + + pIter = taosHashIterate(pMgmt->cache->pHashTable, pIter); + if (pIter == NULL) return NULL; + + SCacheDataNode **pNode = pIter; + if (pNode == NULL || *pNode == NULL) { + taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); + return NULL; + } + + *pConn = (SConnObj *)((*pNode)->data); + return pIter; +} + +static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); +} + +static int32_t mndProcessConnectMsg(SMnode *pMnode, SMnodeMsg *pMsg) { + SConnectMsg *pReq = pMsg->rpcMsg.pCont; + pReq->pid = htonl(pReq->pid); + + SRpcConnInfo info = {0}; + if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { + mError("user:%s, failed to login while get connection info since %s", pMsg->user, terrstr()); + return -1; + } + + char ip[30]; + taosIp2String(info.clientIp, ip); + + if (pReq->db[0]) { + snprintf(pMsg->db, TSDB_FULL_DB_NAME_LEN, "%d%s%s", pMsg->acctId, TS_PATH_DELIMITER, pReq->db); + SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + mError("user:%s, failed to login from %s while use db:%s since %s", pMsg->user, ip, pReq->db, terrstr()); + return -1; + } + mndReleaseDb(pMnode, pDb); + } + + SConnObj *pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + if (pConn == NULL) { + mError("user:%s, failed to login from %s while create connection since %s", pMsg->user, ip, terrstr()); + return -1; + } + + SConnectRsp *pRsp = rpcMallocCont(sizeof(SConnectRsp)); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("user:%s, failed to login from %s while create rsp since %s", pMsg->user, ip, terrstr()); + return -1; + } + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser != NULL) { + pRsp->acctId = htonl(pUser->acctId); + pRsp->superAuth = pUser->superAuth; + pRsp->readAuth = pUser->readAuth; + pRsp->writeAuth = pUser->writeAuth; + mndReleaseUser(pMnode, pUser); + } + + pRsp->acctId = htonl(pUser->acctId); + pRsp->clusterId = htonl(pMnode->clusterId); + pRsp->connId = htonl(pConn->connId); + mndGetMnodeEpSet(pMnode, &pRsp->epSet); + + pMsg->contLen = sizeof(SConnectRsp); + pMsg->pCont = pRsp; + mDebug("user:%s, login from %s, conn:%d", info.user, ip, pConn->connId); + return 0; +} + +static int32_t mndProcessHeartBeatMsg(SMnode *pMnode, SMnodeMsg *pMsg) { + SHeartBeatMsg *pReq = pMsg->rpcMsg.pCont; + pReq->connId = htonl(pReq->connId); + pReq->pid = htonl(pReq->pid); + + SRpcConnInfo info = {0}; + if (rpcGetConnInfo(pMsg->rpcMsg.handle, &info) != 0) { + mError("user:%s, connId:%d failed to process hb since %s", pMsg->user, pReq->connId, terrstr()); + return -1; + } + + SConnObj *pConn = mndAcquireConn(pMnode, pReq->connId, info.user, info.clientIp, info.clientPort); + if (pConn == NULL) { + pConn = mndCreateConn(pMnode, info.user, info.clientIp, info.clientPort, pReq->pid, pReq->app); + if (pConn == NULL) { + mError("user:%s, conn:%d is freed and failed to create new conn since %s", pMsg->user, pReq->connId, terrstr()); + return -1; + } else { + mDebug("user:%s, conn:%d is freed and create a new conn:%d", pMsg->user, pReq->connId, pConn->connId); + } + } + + SHeartBeatRsp *pRsp = rpcMallocCont(sizeof(SHeartBeatRsp)); + if (pRsp == NULL) { + mndReleaseConn(pMnode, pConn); + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("user:%s, conn:%d failed to process hb while create rsp since %s", pMsg->user, pReq->connId, terrstr()); + return -1; + } + + pRsp->connId = htonl(pConn->connId); + pRsp->killConnection = pConn->killed; + mndGetMnodeEpSet(pMnode, &pRsp->epSet); + mndReleaseConn(pMnode, pConn); + + pMsg->contLen = sizeof(SConnectRsp); + pMsg->pCont = pRsp; + return 0; +} + +static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + return TSDB_CODE_MND_NO_RIGHTS; + } + mndReleaseUser(pMnode, pUser); + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "connId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "user"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + // app name + pShow->bytes[cols] = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "program"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + // app pid + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "pid"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ip:port"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "login_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "last_access"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = taosHashGetSize(pMgmt->cache->pHashTable); + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + int32_t numOfRows = 0; + SConnObj *pConnObj = NULL; + int32_t cols = 0; + char *pWrite; + char ipStr[TSDB_IPv4ADDR_LEN + 6]; + + while (numOfRows < rows) { + pShow->pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); + if (pConnObj == NULL) break; + + cols = 0; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pConnObj->connId; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + cols++; + + // app name + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->app, pShow->bytes[cols]); + cols++; + + // app pid + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = pConnObj->pid; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + taosIpPort2String(pConnObj->ip, pConnObj->port, ipStr); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = pConnObj->stime; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + if (pConnObj->lastAccess < pConnObj->stime) pConnObj->lastAccess = pConnObj->stime; + *(int64_t *)pWrite = pConnObj->lastAccess; + cols++; + + numOfRows++; + } + + pShow->numOfReads += numOfRows; + + return numOfRows; +} + +static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + return TSDB_CODE_MND_NO_RIGHTS; + } + mndReleaseUser(pMnode, pUser); + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = QUERY_ID_SIZE + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "query_id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "user"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ip:port"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 24; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "qid"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "created_time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; + strcpy(pSchema[cols].name, "time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sql_obj_id"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "pid"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ep"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 1; + pSchema[cols].type = TSDB_DATA_TYPE_BOOL; + strcpy(pSchema[cols].name, "stable_query"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "sub_queries"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sub_query_info"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sql"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = 1000000; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pMsg->pMnode; + int32_t numOfRows = 0; + SConnObj *pConnObj = NULL; + int32_t cols = 0; + char * pWrite; + void * pIter; + char str[TSDB_IPv4ADDR_LEN + 6] = {0}; + + while (numOfRows < rows) { + pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); + if (pConnObj == NULL) { + pShow->pIter = pIter; + break; + } + + if (numOfRows + pConnObj->numOfQueries >= rows) { + mndCancelGetNextConn(pMnode, pIter); + break; + } + + pShow->pIter = pIter; + for (int32_t i = 0; i < pConnObj->numOfQueries; ++i) { + SQueryDesc *pDesc = pConnObj->pQueries + i; + cols = 0; + + snprintf(str, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->queryId)); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + snprintf(str, tListLen(str), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + cols++; + + char handleBuf[24] = {0}; + snprintf(handleBuf, tListLen(handleBuf), "%" PRIu64, htobe64(pDesc->qId)); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, handleBuf, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->stime); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->useconds); + cols++; + + snprintf(str, tListLen(str), "0x%" PRIx64, htobe64(pDesc->sqlObjId)); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, str, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = htonl(pDesc->pid); + cols++; + + char epBuf[TSDB_EP_LEN + 1] = {0}; + snprintf(epBuf, tListLen(epBuf), "%s:%u", pDesc->fqdn, pConnObj->port); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, epBuf, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(bool *)pWrite = pDesc->stableQuery; + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = htonl(pDesc->numOfSub); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->subSqlInfo, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]); + cols++; + + numOfRows++; + } + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); +} + +static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + return TSDB_CODE_MND_NO_RIGHTS; + } + mndReleaseUser(pMnode, pUser); + + int32_t cols = 0; + SSchema *pSchema = pMeta->schema; + + pShow->bytes[cols] = QUERY_ID_SIZE + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "streamId"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "user"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "dest table"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "ip:port"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "created time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; + strcpy(pSchema[cols].name, "exec time"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 8; + pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; + strcpy(pSchema[cols].name, "time(us)"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sql"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pShow->bytes[cols] = 4; + pSchema[cols].type = TSDB_DATA_TYPE_INT; + strcpy(pSchema[cols].name, "cycles"); + pSchema[cols].bytes = htons(pShow->bytes[cols]); + cols++; + + pMeta->numOfColumns = htons(cols); + pShow->numOfColumns = cols; + + pShow->offset[0] = 0; + for (int32_t i = 1; i < cols; ++i) { + pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1]; + } + + pShow->numOfRows = 1000000; + pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1]; + + return 0; +} + +static int32_t mndRetrieveStreams(SShowObj *pShow, char *data, int32_t rows, SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + int32_t numOfRows = 0; + SConnObj *pConnObj = NULL; + int32_t cols = 0; + char *pWrite; + void *pIter; + char ipStr[TSDB_IPv4ADDR_LEN + 6]; + + while (numOfRows < rows) { + pIter = mndGetNextConn(pMnode, pShow->pIter, &pConnObj); + if (pConnObj == NULL) { + pShow->pIter = pIter; + break; + } + + if (numOfRows + pConnObj->numOfStreams >= rows) { + mndCancelGetNextConn(pMnode, pIter); + break; + } + + pShow->pIter = pIter; + for (int32_t i = 0; i < pConnObj->numOfStreams; ++i) { + SStreamDesc *pDesc = pConnObj->pStreams + i; + cols = 0; + + snprintf(ipStr, QUERY_ID_SIZE + 1, "%u:%u", pConnObj->connId, htonl(pDesc->streamId)); + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pConnObj->user, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->dstTable, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + snprintf(ipStr, sizeof(ipStr), "%s:%u", taosIpStr(pConnObj->ip), pConnObj->port); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, ipStr, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->ctime); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->stime); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int64_t *)pWrite = htobe64(pDesc->useconds); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDesc->sql, pShow->bytes[cols]); + cols++; + + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + *(int32_t *)pWrite = (int32_t)htobe64(pDesc->num); + cols++; + + numOfRows++; + } + } + + pShow->numOfReads += numOfRows; + return numOfRows; +} + +static void mndCancelGetNextStream(SMnode *pMnode, void *pIter) { + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); +} diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index af6a203006..790f5c1737 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -92,7 +92,7 @@ static int32_t mndProcessShowMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg) { pRsp->qhandle = htobe64((uint64_t)pShow); - int32_t code = (*metaFp)(pMnode, &pRsp->tableMeta, pShow, pMnodeMsg->rpcMsg.handle); + int32_t code = (*metaFp)(pMnodeMsg,pShow, &pRsp->tableMeta); mDebug("show:%d, type:%s, get meta finished, numOfRows:%d cols:%d result:%s", pShow->id, mndShowStr(type), pShow->numOfRows, pShow->numOfColumns, tstrerror(code)); @@ -169,7 +169,7 @@ static int32_t mndProcessRetrieveMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg) { // if free flag is set, client wants to clean the resources if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { - rowsRead = (*retrieveFp)(pMnode, pShow, pRsp->data, rowsToRead, pMnodeMsg->rpcMsg.handle); + rowsRead = (*retrieveFp)(pMnodeMsg, pShow, pRsp->data, rowsToRead); } mDebug("show:%d, stop retrieve data, rowsRead:%d rowsToRead:%d", pShow->id, rowsRead, rowsToRead); @@ -311,17 +311,17 @@ void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capa } } -void mnodeAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp) { +void mndAddShowMetaHandle(SMnode *pMnode, EShowType showType, ShowMetaFp fp) { SShowMgmt *pMgmt = &pMnode->showMgmt; pMgmt->metaFps[showType] = fp; } -void mnodeAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp) { +void mndAddShowRetrieveHandle(SMnode *pMnode, EShowType showType, ShowRetrieveFp fp) { SShowMgmt *pMgmt = &pMnode->showMgmt; pMgmt->retrieveFps[showType] = fp; } -void mnodeAddShowFreeIterHandle(SMnode *pMnode, EShowType showType, ShowFreeIterFp fp) { +void mndAddShowFreeIterHandle(SMnode *pMnode, EShowType showType, ShowFreeIterFp fp) { SShowMgmt *pMgmt = &pMnode->showMgmt; pMgmt->freeIterFps[showType] = fp; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index abbe41a60d..78869bd416 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -30,7 +30,7 @@ static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_BINARY(pRaw, dataPos, pUser->acct, TSDB_USER_LEN) SDB_SET_INT64(pRaw, dataPos, pUser->createdTime) SDB_SET_INT64(pRaw, dataPos, pUser->updateTime) - SDB_SET_INT8(pRaw, dataPos, pUser->rootAuth) + SDB_SET_INT8(pRaw, dataPos, pUser->superAuth) SDB_SET_DATALEN(pRaw, dataPos); return pRaw; @@ -56,7 +56,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_BINARY(pRaw, pRow, dataPos, pUser->acct, TSDB_USER_LEN) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->createdTime) SDB_GET_INT64(pRaw, pRow, dataPos, &pUser->updateTime) - SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->rootAuth) + SDB_GET_INT8(pRaw, pRow, dataPos, &pUser->superAuth) return pRow; } @@ -70,12 +70,14 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) { return -1; } - pUser->pAcct = sdbAcquire(pSdb, SDB_ACCT, pUser->acct); - if (pUser->pAcct == NULL) { + SAcctObj *pAcct = sdbAcquire(pSdb, SDB_ACCT, pUser->acct); + if (pAcct == NULL) { terrno = TSDB_CODE_MND_ACCT_NOT_EXIST; mError("user:%s, failed to perform insert action since %s", pUser->user, terrstr()); return -1; } + pUser->acctId = pAcct->acctId; + sdbRelease(pSdb, pAcct); return 0; } @@ -87,11 +89,6 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { pUser->prohibitDbHash = NULL; } - if (pUser->pAcct != NULL) { - sdbRelease(pSdb, pUser->pAcct); - pUser->pAcct = NULL; - } - return 0; } @@ -102,7 +99,7 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDs memcpy(pSrcUser->acct, pDstUser->acct, TSDB_USER_LEN); pSrcUser->createdTime = pDstUser->createdTime; pSrcUser->updateTime = pDstUser->updateTime; - pSrcUser->rootAuth = pDstUser->rootAuth; + pSrcUser->superAuth = pDstUser->superAuth; return 0; } @@ -115,7 +112,7 @@ static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char userObj.updateTime = userObj.createdTime; if (strcmp(user, TSDB_DEFAULT_USER) == 0) { - userObj.rootAuth = 1; + userObj.superAuth = 1; } SSdbRaw *pRaw = mndUserActionEncode(&userObj); @@ -145,7 +142,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.rootAuth = 0; + userObj.superAuth = 0; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, pMsg->rpcMsg.handle); if (pTrans == NULL) return -1; diff --git a/source/util/src/tutil.c b/source/util/src/tutil.c index ee524e4448..b9d2da6939 100644 --- a/source/util/src/tutil.c +++ b/source/util/src/tutil.c @@ -410,3 +410,10 @@ char *taosIpStr(uint32_t ipInt) { return ipStr; } +void taosIp2String(uint32_t ip, char *str) { + sprintf(str, "%u.%u.%u.%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24)); +} + +void taosIpPort2String(uint32_t ip, uint16_t port, char *str) { + sprintf(str, "%u.%u.%u.%u:%u", ip & 0xFF, (ip >> 8) & 0xFF, (ip >> 16) & 0xFF, (uint8_t)(ip >> 24), port); +} \ No newline at end of file From 8a850dd3f2fb650e7c5b367e5f2b4fe2d4f7e84b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 3 Dec 2021 15:01:26 +0800 Subject: [PATCH 13/18] TD-10431 process mnode profile --- include/common/taosmsg.h | 1 + source/dnode/mgmt/impl/src/dndTransport.c | 1 + source/dnode/mnode/impl/inc/mndInt.h | 2 +- source/dnode/mnode/impl/src/mndDb.c | 9 +- source/dnode/mnode/impl/src/mndDnode.c | 9 +- source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndProfile.c | 142 ++++++++++++++++++++-- source/dnode/mnode/impl/src/mndShow.c | 10 +- source/dnode/mnode/impl/src/mndUser.c | 3 +- source/dnode/mnode/impl/src/mnode.c | 2 +- 10 files changed, 157 insertions(+), 26 deletions(-) diff --git a/include/common/taosmsg.h b/include/common/taosmsg.h index e29accd249..e14788be47 100644 --- a/include/common/taosmsg.h +++ b/include/common/taosmsg.h @@ -81,6 +81,7 @@ TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_ALTER_STABLE, "alter-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_DROP_STABLE, "drop-stable" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_STABLE_VGROUP, "stable-vgroup" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_QUERY, "kill-query" ) +TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_STREAM, "kill-stream" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_KILL_CONN, "kill-conn" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_HEARTBEAT, "heartbeat" ) TAOS_DEFINE_MESSAGE_TYPE( TSDB_MSG_TYPE_SHOW, "show" ) diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index f3e71564b2..98a0b8e308 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -76,6 +76,7 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TSDB_MSG_TYPE_DROP_STABLE] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_STABLE_VGROUP] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_QUERY] = dndProcessMnodeWriteMsg; + pMgmt->msgFp[TSDB_MSG_TYPE_KILL_STREAM] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_KILL_CONN] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TSDB_MSG_TYPE_HEARTBEAT] = dndProcessMnodeReadMsg; pMgmt->msgFp[TSDB_MSG_TYPE_SHOW] = dndProcessMnodeReadMsg; diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 0f4dd0633b..dbca6ad057 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -25,7 +25,7 @@ extern "C" { #endif -typedef int32_t (*MndMsgFp)(SMnode *pMnode, SMnodeMsg *pMsg); +typedef int32_t (*MndMsgFp)(SMnodeMsg *pMsg); typedef int32_t (*MndInitFp)(SMnode *pMnode); typedef void (*MndCleanupFp)(SMnode *pMnode); typedef int32_t (*ShowMetaFp)(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index d990e926bb..f01a43dfef 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" -static int32_t mnodeProcessUseMsg(SMnode *pMnode, SMnodeMsg *pMsg); +static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg); int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_USE_DB, mnodeProcessUseMsg); @@ -35,10 +35,11 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb) { sdbRelease(pSdb, pDb); } -static int32_t mnodeProcessUseMsg(SMnode *pMnode, SMnodeMsg *pMsg) { - SUseDbMsg *pUseDbMsg = pMsg->rpcMsg.pCont; +static int32_t mnodeProcessUseMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SUseDbMsg *pUse = pMsg->rpcMsg.pCont; - strncpy(pMsg->db, pUseDbMsg->db, TSDB_FULL_DB_NAME_LEN); + strncpy(pMsg->db, pUse->db, TSDB_FULL_DB_NAME_LEN); SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db); if (pDb != NULL) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index a6cee3efc5..24e6113161 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -226,7 +226,8 @@ static void mndParseStatusMsg(SStatusMsg *pStatus) { pStatus->clusterCfg.checkTime = htobe64(pStatus->clusterCfg.checkTime); } -static int32_t mndProcessStatusMsg(SMnode *pMnode, SMnodeMsg *pMsg) { +static int32_t mndProcessStatusMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; SStatusMsg *pStatus = pMsg->rpcMsg.pCont; mndParseStatusMsg(pStatus); @@ -315,11 +316,11 @@ static int32_t mndProcessStatusMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessCreateDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropDnodeMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessConfigDnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessConfigDnodeMsg(SMnodeMsg *pMsg) { return 0; } int32_t mndInitDnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_DNODE, diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index c1c17e4983..e3464cd327 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -103,9 +103,9 @@ static int32_t mndCreateDefaultMnode(SMnode *pMnode) { return sdbWrite(pMnode->pSdb, pRaw); } -static int32_t mndProcessCreateMnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessCreateMnodeMsg(SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessDropMnodeMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessDropMnodeMsg(SMnodeMsg *pMsg) { return 0; } int32_t mndInitMnode(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_MNODE, diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index ef596f8ee1..d70d045a04 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -50,8 +50,11 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, int32_t connId, char *user, uint static void mndReleaseConn(SMnode *pMnode, SConnObj *pConn); static void *mndGetNextConn(SMnode *pMnode, void *pIter, SConnObj **pConn); static void mndCancelGetNextConn(SMnode *pMnode, void *pIter); -static int32_t mndProcessHeartBeatMsg(SMnode *pMnode, SMnodeMsg *pMsg); -static int32_t mndProcessConnectMsg(SMnode *pMnode, SMnodeMsg *pMsg); +static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg); +static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg); +static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg); +static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg); +static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg); static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta); static int32_t mndRetrieveConns(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj *pShow); @@ -74,10 +77,14 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_HEARTBEAT, mndProcessHeartBeatMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessConnectMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_QUERY, mndProcessKillQueryMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessKillStreamMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_CONN, mndProcessKillConnectionMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndRetrieveConns); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndCancelGetNextConn); + return 0; } @@ -183,7 +190,8 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { taosHashCancelIterate(pMgmt->cache->pHashTable, pIter); } -static int32_t mndProcessConnectMsg(SMnode *pMnode, SMnodeMsg *pMsg) { +static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; SConnectMsg *pReq = pMsg->rpcMsg.pCont; pReq->pid = htonl(pReq->pid); @@ -240,7 +248,8 @@ static int32_t mndProcessConnectMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } -static int32_t mndProcessHeartBeatMsg(SMnode *pMnode, SMnodeMsg *pMsg) { +static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; SHeartBeatMsg *pReq = pMsg->rpcMsg.pCont; pReq->connId = htonl(pReq->connId); pReq->pid = htonl(pReq->pid); @@ -280,6 +289,118 @@ static int32_t mndProcessHeartBeatMsg(SMnode *pMnode, SMnodeMsg *pMsg) { return 0; } +static int32_t mndProcessKillQueryMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + mInfo("kill query msg is received, queryId:%s", pKill->queryId); + + const char delim = ':'; + char *connIdStr = strtok(pKill->queryId, &delim); + char *queryIdStr = strtok(NULL, &delim); + + if (queryIdStr == NULL || connIdStr == NULL) { + mError("failed to kill query, queryId:%s", pKill->queryId); + terrno = TSDB_CODE_MND_INVALID_QUERY_ID; + return -1; + } + + int32_t queryId = (int32_t)strtol(queryIdStr, NULL, 10); + + int32_t connId = atoi(connIdStr); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + if (pConn == NULL) { + mError("connId:%s, failed to kill queryId:%d, conn not exist", connIdStr, queryId); + terrno = TSDB_CODE_MND_INVALID_CONN_ID; + return -1; + } else { + mInfo("connId:%s, queryId:%d is killed by user:%s", connIdStr, queryId, pMsg->user); + pConn->queryId = queryId; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + return 0; + } +} + +static int32_t mndProcessKillStreamMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + SKillQueryMsg *pKill = pMsg->rpcMsg.pCont; + mInfo("kill stream msg is received, streamId:%s", pKill->queryId); + + const char delim = ':'; + char *connIdStr = strtok(pKill->queryId, &delim); + char *streamIdStr = strtok(NULL, &delim); + + if (streamIdStr == NULL || connIdStr == NULL) { + mError("failed to kill stream, streamId:%s", pKill->queryId); + terrno = TSDB_CODE_MND_INVALID_STREAM_ID; + return -1; + } + + int32_t streamId = (int32_t)strtol(streamIdStr, NULL, 10); + int32_t connId = atoi(connIdStr); + + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + if (pConn == NULL) { + mError("connId:%s, failed to kill streamId:%d, conn not exist", connIdStr, streamId); + terrno = TSDB_CODE_MND_INVALID_CONN_ID; + return -1; + } else { + mInfo("connId:%s, streamId:%d is killed by user:%s", connIdStr, streamId, pMsg->user); + pConn->streamId = streamId; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + return TSDB_CODE_SUCCESS; + } +} + +static int32_t mndProcessKillConnectionMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; + SProfileMgmt *pMgmt = &pMnode->profileMgmt; + + SUserObj *pUser = mndAcquireUser(pMnode, pMsg->user); + if (pUser == NULL) return 0; + if (!pUser->superAuth) { + mndReleaseUser(pMnode, pUser); + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + mndReleaseUser(pMnode, pUser); + + SKillConnMsg *pKill = pMsg->rpcMsg.pCont; + int32_t connId = atoi(pKill->queryId); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + if (pConn == NULL) { + mError("connId:%s, failed to kill, conn not exist", pKill->queryId); + terrno = TSDB_CODE_MND_INVALID_CONN_ID; + return -1; + } else { + mInfo("connId:%s, is killed by user:%s", pKill->queryId, pMsg->user); + pConn->killed = 1; + taosCacheRelease(pMgmt->cache, (void **)&pConn, false); + return TSDB_CODE_SUCCESS; + } +} + static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg *pMeta) { SMnode *pMnode = pMsg->pMnode; SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -288,7 +409,8 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaMsg * if (pUser == NULL) return 0; if (!pUser->superAuth) { mndReleaseUser(pMnode, pUser); - return TSDB_CODE_MND_NO_RIGHTS; + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; } mndReleaseUser(pMnode, pUser); @@ -415,7 +537,8 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj * if (pUser == NULL) return 0; if (!pUser->superAuth) { mndReleaseUser(pMnode, pUser); - return TSDB_CODE_MND_NO_RIGHTS; + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; } mndReleaseUser(pMnode, pUser); @@ -519,8 +642,8 @@ static int32_t mndRetrieveQueries(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t numOfRows = 0; SConnObj *pConnObj = NULL; int32_t cols = 0; - char * pWrite; - void * pIter; + char *pWrite; + void *pIter; char str[TSDB_IPv4ADDR_LEN + 6] = {0}; while (numOfRows < rows) { @@ -621,7 +744,8 @@ static int32_t mndGetStreamMeta(SMnodeMsg *pMsg, STableMetaMsg *pMeta, SShowObj if (pUser == NULL) return 0; if (!pUser->superAuth) { mndReleaseUser(pMnode, pUser); - return TSDB_CODE_MND_NO_RIGHTS; + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; } mndReleaseUser(pMnode, pUser); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 790f5c1737..ff9f2a9805 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "mndShow.h" -static int32_t mndProcessShowMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg); -static int32_t mndProcessRetrieveMsg(SMnode *pMnode, SMnodeMsg *pMsg); +static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg); +static int32_t mndProcessRetrieveMsg( SMnodeMsg *pMsg); static bool mndCheckRetrieveFinished(SShowObj *pShow); static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow); static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove); @@ -42,7 +42,8 @@ int32_t mndInitShow(SMnode *pMnode) { void mndCleanupShow(SMnode *pMnode) {} -static int32_t mndProcessShowMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg) { +static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { + SMnode *pMnode = pMnodeMsg->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; SShowMsg *pMsg = pMnodeMsg->rpcMsg.pCont; int8_t type = pMsg->type; @@ -108,7 +109,8 @@ static int32_t mndProcessShowMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg) { } } -static int32_t mndProcessRetrieveMsg(SMnode *pMnode, SMnodeMsg *pMnodeMsg) { +static int32_t mndProcessRetrieveMsg(SMnodeMsg *pMnodeMsg) { + SMnode *pMnode = pMnodeMsg->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; int32_t rowsToRead = 0; int32_t size = 0; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 78869bd416..d3a8cf56be 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -180,7 +180,8 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, return 0; } -static int32_t mndProcessCreateUserMsg(SMnode *pMnode, SMnodeMsg *pMsg) { +static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { + SMnode *pMnode = pMsg->pMnode; SCreateUserMsg *pCreate = pMsg->rpcMsg.pCont; if (pCreate->user[0] == 0) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index e390deda6d..3a5b6c470b 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -386,7 +386,7 @@ static void mndProcessRpcMsg(SMnodeMsg *pMsg) { goto PROCESS_RPC_END; } - code = (*fp)(pMnode, pMsg); + code = (*fp)(pMsg); if (code != 0) { code = terrno; mError("msg:%p, app:%p failed to process since %s", pMsg, ahandle, terrstr()); From 3e34c4cd2645bd9ceaeadb4eeaff5fbc7ec70824 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 3 Dec 2021 17:22:44 +0800 Subject: [PATCH 14/18] TD-10431 process mnode profile --- source/dnode/mnode/impl/src/mndProfile.c | 53 +++++++++- source/dnode/mnode/impl/src/mndShow.c | 126 ++++++++++++----------- 2 files changed, 118 insertions(+), 61 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d70d045a04..eb9926cfb6 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -248,6 +248,41 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { return 0; } +static int32_t mnodeSaveQueryStreamList(SConnObj *pConn, SHeartBeatMsg *pMsg) { + pConn->numOfQueries = 0; + pConn->numOfStreams = 0; + int32_t numOfQueries = htonl(pMsg->numOfQueries); + int32_t numOfStreams = htonl(pMsg->numOfStreams); + + if (numOfQueries > 0) { + if (pConn->pQueries == NULL) { + pConn->pQueries = calloc(sizeof(SQueryDesc), QUERY_STREAM_SAVE_SIZE); + } + + pConn->numOfQueries = MIN(QUERY_STREAM_SAVE_SIZE, numOfQueries); + + int32_t saveSize = pConn->numOfQueries * sizeof(SQueryDesc); + if (saveSize > 0 && pConn->pQueries != NULL) { + memcpy(pConn->pQueries, pMsg->pData, saveSize); + } + } + + if (numOfStreams > 0) { + if (pConn->pStreams == NULL) { + pConn->pStreams = calloc(sizeof(SStreamDesc), QUERY_STREAM_SAVE_SIZE); + } + + pConn->numOfStreams = MIN(QUERY_STREAM_SAVE_SIZE, numOfStreams); + + int32_t saveSize = pConn->numOfStreams * sizeof(SStreamDesc); + if (saveSize > 0 && pConn->pStreams != NULL) { + memcpy(pConn->pStreams, pMsg->pData + numOfQueries * sizeof(SQueryDesc), saveSize); + } + } + + return TSDB_CODE_SUCCESS; +} + static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { SMnode *pMnode = pMsg->pMnode; SHeartBeatMsg *pReq = pMsg->rpcMsg.pCont; @@ -279,8 +314,24 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { return -1; } + mnodeSaveQueryStreamList(pConn, pReq); + if (pConn->killed != 0) { + pRsp->killConnection = 1; + } + + if (pConn->streamId != 0) { + pRsp->streamId = htonl(pConn->streamId); + pConn->streamId = 0; + } + + if (pConn->queryId != 0) { + pRsp->queryId = htonl(pConn->queryId); + pConn->queryId = 0; + } + pRsp->connId = htonl(pConn->connId); - pRsp->killConnection = pConn->killed; + pRsp->totalDnodes = htnol(1); + pRsp->onlineDnodes = htonl(1); mndGetMnodeEpSet(pMnode, &pRsp->epSet); mndReleaseConn(pMnode, pConn); diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index ff9f2a9805..e652f94957 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -40,7 +40,72 @@ int32_t mndInitShow(SMnode *pMnode) { return 0; } -void mndCleanupShow(SMnode *pMnode) {} +void mndCleanupShow(SMnode *pMnode) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + if (pMgmt->cache != NULL) { + taosCacheCleanup(pMgmt->cache); + pMgmt->cache = NULL; + } +} + +static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow) { + TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pShow; + + SShowMgmt *pMgmt = &pMnode->showMgmt; + SShowObj **ppShow = taosCacheAcquireByKey(pMgmt->cache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE)); + if (ppShow) { + mTrace("show:%d, data:%p acquired from cache", pShow->id, ppShow); + return 0; + } + + return -1; +} + +static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { + SMnode *pMnode = pShow->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + SShowObj **ppShow = (SShowObj **)pShow->ppShow; + taosCacheRelease(pMgmt->cache, (void **)(&ppShow), forceRemove); + mDebug("show:%d, data:%p released from cache, force:%d", pShow->id, ppShow, forceRemove); +} + +static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow) { + SShowMgmt *pMgmt = &pMnode->showMgmt; + int32_t lifeSpan = pMnode->shellActivityTimer * 6 * 1000; + + TSDB_CACHE_PTR_TYPE val = (TSDB_CACHE_PTR_TYPE)pShow; + pShow->id = atomic_add_fetch_32(&pMgmt->showId, 1); + SShowObj **ppShow = + taosCachePut(pMgmt->cache, &val, sizeof(TSDB_CACHE_PTR_TYPE), &pShow, sizeof(TSDB_CACHE_PTR_TYPE), lifeSpan); + if (ppShow == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + mError("show:%d, failed to put into cache", pShow->id); + return -1; + } + + mTrace("show:%d, data:%p put into cache", pShow->id, ppShow); + return 0; +} + +static void mndFreeShowObj(void *ppShow) { + SShowObj *pShow = *(SShowObj **)ppShow; + SMnode *pMnode = pShow->pMnode; + SShowMgmt *pMgmt = &pMnode->showMgmt; + + ShowFreeIterFp freeFp = pMgmt->freeIterFps[pShow->type]; + if (freeFp != NULL) { + if (pShow->pVgIter != NULL) { + // only used in 'show vnodes "ep"' + (*freeFp)(pMnode, pShow->pVgIter); + } + if (pShow->pIter != NULL) { + (*freeFp)(pMnode, pShow->pIter); + } + } + + mDebug("show:%d, data:%p destroyed", pShow->id, ppShow); + tfree(pShow); +} static int32_t mndProcessShowMsg(SMnodeMsg *pMnodeMsg) { SMnode *pMnode = pMnodeMsg->pMnode; @@ -246,65 +311,6 @@ static bool mndCheckRetrieveFinished(SShowObj *pShow) { return false; } -static int32_t mndAcquireShowObj(SMnode *pMnode, SShowObj *pShow) { - TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE)pShow; - - SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowObj **ppShow = taosCacheAcquireByKey(pMgmt->cache, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE)); - if (ppShow) { - mTrace("show:%d, data:%p acquired from cache", pShow->id, ppShow); - return 0; - } - - return -1; -} - -static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { - SMnode *pMnode = pShow->pMnode; - SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowObj **ppShow = (SShowObj **)pShow->ppShow; - taosCacheRelease(pMgmt->cache, (void **)(&ppShow), forceRemove); - mDebug("show:%d, data:%p released from cache, force:%d", pShow->id, ppShow, forceRemove); -} - -static int32_t mndPutShowObj(SMnode *pMnode, SShowObj *pShow) { - SShowMgmt *pMgmt = &pMnode->showMgmt; - int32_t lifeSpan = pMnode->shellActivityTimer * 6 * 1000; - - TSDB_CACHE_PTR_TYPE val = (TSDB_CACHE_PTR_TYPE)pShow; - pShow->id = atomic_add_fetch_32(&pMgmt->showId, 1); - SShowObj **ppShow = - taosCachePut(pMgmt->cache, &val, sizeof(TSDB_CACHE_PTR_TYPE), &pShow, sizeof(TSDB_CACHE_PTR_TYPE), lifeSpan); - if (ppShow == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:%d, failed to put into cache", pShow->id); - return -1; - } - - mTrace("show:%d, data:%p put into cache", pShow->id, ppShow); - return 0; -} - -static void mndFreeShowObj(void *ppShow) { - SShowObj *pShow = *(SShowObj **)ppShow; - SMnode *pMnode = pShow->pMnode; - SShowMgmt *pMgmt = &pMnode->showMgmt; - - ShowFreeIterFp freeFp = pMgmt->freeIterFps[pShow->type]; - if (freeFp != NULL) { - if (pShow->pVgIter != NULL) { - // only used in 'show vnodes "ep"' - (*freeFp)(pMnode, pShow->pVgIter); - } - if (pShow->pIter != NULL) { - (*freeFp)(pMnode, pShow->pIter); - } - } - - mDebug("show:%d, data:%p destroyed", pShow->id, ppShow); - tfree(pShow); -} - void mnodeVacuumResult(char *data, int32_t numOfCols, int32_t rows, int32_t capacity, SShowObj *pShow) { if (rows < capacity) { for (int32_t i = 0; i < numOfCols; ++i) { From 608b96d38474c99d8347a1537199dfaec208693c Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 3 Dec 2021 20:52:44 +0800 Subject: [PATCH 15/18] TD-10431 dnode test01 --- source/dnode/mgmt/impl/CMakeLists.txt | 4 + source/dnode/mgmt/impl/test/CMakeLists.txt | 1 + .../mgmt/impl/test/test01/CMakeLists.txt | 25 +++ source/dnode/mgmt/impl/test/test01/deploy.cpp | 0 .../dnode/mgmt/impl/test/test01/dndTest01.cpp | 156 ++++++++++++++++++ source/dnode/mnode/impl/src/mndAcct.c | 99 +++++++---- source/dnode/mnode/impl/src/mndUser.c | 132 +++++++++------ 7 files changed, 330 insertions(+), 87 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/test01/CMakeLists.txt create mode 100644 source/dnode/mgmt/impl/test/test01/deploy.cpp create mode 100644 source/dnode/mgmt/impl/test/test01/dndTest01.cpp diff --git a/source/dnode/mgmt/impl/CMakeLists.txt b/source/dnode/mgmt/impl/CMakeLists.txt index 3eadf24164..51131ede6a 100644 --- a/source/dnode/mgmt/impl/CMakeLists.txt +++ b/source/dnode/mgmt/impl/CMakeLists.txt @@ -14,3 +14,7 @@ target_include_directories( PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/mgmt" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/CMakeLists.txt b/source/dnode/mgmt/impl/test/CMakeLists.txt new file mode 100644 index 0000000000..95ffa548b2 --- /dev/null +++ b/source/dnode/mgmt/impl/test/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory(test01) \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/test01/CMakeLists.txt b/source/dnode/mgmt/impl/test/test01/CMakeLists.txt new file mode 100644 index 0000000000..313f21a2f6 --- /dev/null +++ b/source/dnode/mgmt/impl/test/test01/CMakeLists.txt @@ -0,0 +1,25 @@ +add_executable(dndTest01 "") + +target_sources(dndTest01 + PRIVATE + "dndTest01.cpp" +) + +target_link_libraries( + dndTest01 + PUBLIC dnode + PUBLIC gtest_main +) + +target_include_directories(dndTest01 + PUBLIC + "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" + "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" +) + +enable_testing() + +add_test( + NAME dndTest01 + COMMAND dndTest01 +) diff --git a/source/dnode/mgmt/impl/test/test01/deploy.cpp b/source/dnode/mgmt/impl/test/test01/deploy.cpp new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/dnode/mgmt/impl/test/test01/dndTest01.cpp b/source/dnode/mgmt/impl/test/test01/dndTest01.cpp new file mode 100644 index 0000000000..8cff0d32a1 --- /dev/null +++ b/source/dnode/mgmt/impl/test/test01/dndTest01.cpp @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include "tthread.h" + +#include "dnode.h" +#include "trpc.h" + +typedef struct { + SDnode* pDnode; + pthread_t* threadId; +} SServer; + +void* runServer(void* param) { + SServer* pServer = param; + while (1) { + taosMsleep(100); + pthread_testcancel(); + } +} + +void initOption(SDnodeOpt* pOption) { + pOption->sver = 1; + pOption->numOfCores = 1; + pOption->numOfSupportMnodes = 1; + pOption->numOfSupportVnodes = 1; + pOption->numOfSupportQnodes = 1; + pOption->statusInterval = 1; + pOption->mnodeEqualVnodeNum = 1; + pOption->numOfThreadsPerCore = 1; + pOption->ratioOfQueryCores = 1; + pOption->maxShellConns = 1000; + pOption->shellActivityTimer = 30; + pOption->serverPort = 9527; + strncpy(pOption->dataDir, "./test01"); + strncpy(pOption->localEp, "localhost:9527"); + strncpy(pOption->localFqdn, "localhost"); + tstrncpy(pOption->firstEp, "localhost:9527"); +} + +Server* createServer() { + SDnodeOpt option = {0}; + initOption(&option); + + SDnode* pDnode = dndInit(&option); + ASSERT(pDnode); + + Server* pServer = calloc(1, sizeof(SServer)); + ASSERT(pServer); + + pServer->pDnode = pDnode; + pServer->threadId = taosCreateThread(runServer, pServer); + ASSERT(pServer->threadId); + + return pServer; +} + +void dropServer(SServer* pServer) { + if (pServer->threadId != NULL) { + taosDestoryThread(pServer->threadId); + } +} + +typedef struct { + void* clientRpc; + SRpcMsg* pRsp; + tsem_t sem; +} SClient; + +static void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { + SClient* pClient = parent; + pClient->pRsp = pMsg; + tsem_post(pMgmt->clientRpc); +} + +SClient* createClient() { + SClient* pClient = calloc(1, sizeof(SClient)); + ASSERT(pClient); + + char secretEncrypt[32] = {0}; + char* pass = "taosdata"; + taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); + + SRpcInit rpcInit; + memset(&rpcInit, 0, sizeof(rpcInit)); + rpcInit.label = "DND-C"; + rpcInit.numOfThreads = 1; + rpcInit.cfp = processClientRsp; + rpcInit.sessions = 1024; + rpcInit.connType = TAOS_CONN_CLIENT; + rpcInit.idleTime = 30 * 1000; + rpcInit.user = "root"; + rpcInit.ckey = "key"; + rpcInit.parent = pDnode; + rpcInit.secret = (char*)secretEncrypt; + rpcInit.parent = pClient; + // rpcInit.spi = 1; + + pClient->clientRpc = rpcOpen(&rpcInit); + ASSERT(pClient->clientRpc); + + tsem_init(&pClient->sem, 0, 0); +} + +void dropClient(SClient* pClient) { + tsem_destroy(&pClient->sem); + rpcClose(pClient->clientRpc); +} + +void sendMsg(SClient* pClient, SRpcMsg* pMsg) { + SEpSet epSet = {0}; + epSet.inUse = 0; + epSet.numOfEps = 1; + epSet.port[0] = 9527; + strcpy(epSet.fqdn[0], "localhost"); + + rpcSendRequest(pMgmt->clientRpc, &epSet, pMsg, NULL); + tsem_wait(pMgmt->clientRpc); +} + +class DndTest01 : public ::testing::Test { + protected: + void SetUp() override { + pServer = createServer(); + pClient = createClient(); + } + void TearDown() override { + dropServer(pServer); + dropClient(pClient); + } + + SServer* pServer; + SClient* pClient; +}; + +TEST_F(DndTest01, connectMsg) { + SConnectMsg *pReq = rpcMallocCont() + + +} diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 23328506e6..e3d37cd9f9 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -14,10 +14,61 @@ */ #define _DEFAULT_SOURCE -#include "mndInt.h" +#include "mndAcct.h" +#include "mndShow.h" #define SDB_ACCT_VER 1 +static int32_t mnodeCreateDefaultAcct(SMnode *pMnode); +static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct); +static SSdbRow *mnodeAcctActionDecode(SSdbRaw *pRaw); +static int32_t mnodeAcctActionInsert(SSdb *pSdb, SAcctObj *pAcct); +static int32_t mnodeAcctActionDelete(SSdb *pSdb, SAcctObj *pAcct); +static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *pDstAcct); +static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg); +static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg); +static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg); + +int32_t mndInitAcct(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_ACCT, + .keyType = SDB_KEY_BINARY, + .deployFp = mnodeCreateDefaultAcct, + .encodeFp = (SdbEncodeFp)mnodeAcctActionEncode, + .decodeFp = (SdbDecodeFp)mnodeAcctActionDecode, + .insertFp = (SdbInsertFp)mnodeAcctActionInsert, + .updateFp = (SdbUpdateFp)mnodeAcctActionUpdate, + .deleteFp = (SdbDeleteFp)mnodeAcctActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_ACCT, mndProcessCreateAcctMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_ACCT, mndProcessAlterAcctMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_ACCT, mndProcessDropAcctMsg); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupAcct(SMnode *pMnode) {} + +static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { + SAcctObj acctObj = {0}; + tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); + acctObj.createdTime = taosGetTimestampMs(); + acctObj.updateTime = acctObj.createdTime; + acctObj.acctId = 1; + acctObj.cfg = (SAcctCfg){.maxUsers = 1024, + .maxDbs = 1024, + .maxTimeSeries = INT32_MAX, + .maxStreams = 8092, + .maxStorage = INT64_MAX, + .accessState = TSDB_VN_ALL_ACCCESS}; + + SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj); + if (pRaw == NULL) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mTrace("acct:%s, will be created while deploy sdb", acctObj.acct); + return sdbWrite(pMnode->pSdb, pRaw); +} + static SSdbRaw *mnodeAcctActionEncode(SAcctObj *pAcct) { SSdbRaw *pRaw = sdbAllocRaw(SDB_ACCT, SDB_ACCT_VER, sizeof(SAcctObj)); if (pRaw == NULL) return NULL; @@ -92,40 +143,20 @@ static int32_t mnodeAcctActionUpdate(SSdb *pSdb, SAcctObj *pSrcAcct, SAcctObj *p return 0; } -static int32_t mnodeCreateDefaultAcct(SMnode *pMnode) { - int32_t code = 0; - - SAcctObj acctObj = {0}; - tstrncpy(acctObj.acct, TSDB_DEFAULT_USER, TSDB_USER_LEN); - acctObj.createdTime = taosGetTimestampMs(); - acctObj.updateTime = acctObj.createdTime; - acctObj.acctId = 1; - acctObj.cfg = (SAcctCfg){.maxUsers = 1024, - .maxDbs = 1024, - .maxTimeSeries = INT32_MAX, - .maxStreams = 8092, - .maxStorage = INT64_MAX, - .accessState = TSDB_VN_ALL_ACCCESS}; - - SSdbRaw *pRaw = mnodeAcctActionEncode(&acctObj); - if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - - mTrace("acct:%s, will be created while deploy sdb", acctObj.acct); - return sdbWrite(pMnode->pSdb, pRaw); +static int32_t mndProcessCreateAcctMsg(SMnodeMsg *pMnodeMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process create acct msg since %s", terrstr()); + return -1; } -int32_t mndInitAcct(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_ACCT, - .keyType = SDB_KEY_BINARY, - .deployFp = mnodeCreateDefaultAcct, - .encodeFp = (SdbEncodeFp)mnodeAcctActionEncode, - .decodeFp = (SdbDecodeFp)mnodeAcctActionDecode, - .insertFp = (SdbInsertFp)mnodeAcctActionInsert, - .updateFp = (SdbUpdateFp)mnodeAcctActionUpdate, - .deleteFp = (SdbDeleteFp)mnodeAcctActionDelete}; - - return sdbSetTable(pMnode->pSdb, table); +static int32_t mndProcessAlterAcctMsg(SMnodeMsg *pMnodeMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process create acct msg since %s", terrstr()); + return -1; } -void mndCleanupAcct(SMnode *pMnode) {} +static int32_t mndProcessDropAcctMsg(SMnodeMsg *pMnodeMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process create acct msg since %s", terrstr()); + return -1; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index d3a8cf56be..020de13e36 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -14,12 +14,76 @@ */ #define _DEFAULT_SOURCE +#include "mndUser.h" +#include "mndShow.h" #include "mndSync.h" #include "mndTrans.h" #include "tkey.h" #define SDB_USER_VER 1 +static int32_t mndCreateDefaultUsers(SMnode *pMnode); +static SSdbRaw *mndUserActionEncode(SUserObj *pUser); +static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw); +static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser); +static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser); +static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDstUser); +static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg); +static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg); +static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg); +static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg); + +int32_t mndInitUser(SMnode *pMnode) { + SSdbTable table = {.sdbType = SDB_USER, + .keyType = SDB_KEY_BINARY, + .deployFp = (SdbDeployFp)mndCreateDefaultUsers, + .encodeFp = (SdbEncodeFp)mndUserActionEncode, + .decodeFp = (SdbDecodeFp)mndUserActionDecode, + .insertFp = (SdbInsertFp)mndUserActionInsert, + .updateFp = (SdbUpdateFp)mndUserActionUpdate, + .deleteFp = (SdbDeleteFp)mndUserActionDelete}; + + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_USER, mndProcessCreateUserMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_ALTER_USER, mndProcessAlterUserMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_DROP_USER, mndProcessDropUserMsg); + + return sdbSetTable(pMnode->pSdb, table); +} + +void mndCleanupUser(SMnode *pMnode) {} + +static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { + SUserObj userObj = {0}; + tstrncpy(userObj.user, user, TSDB_USER_LEN); + tstrncpy(userObj.acct, acct, TSDB_USER_LEN); + taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); + userObj.createdTime = taosGetTimestampMs(); + userObj.updateTime = userObj.createdTime; + + if (strcmp(user, TSDB_DEFAULT_USER) == 0) { + userObj.superAuth = 1; + } + + SSdbRaw *pRaw = mndUserActionEncode(&userObj); + if (pRaw == NULL) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + + mTrace("user:%s, will be created while deploy sdb", userObj.user); + return sdbWrite(pMnode->pSdb, pRaw); +} + +static int32_t mndCreateDefaultUsers(SMnode *pMnode) { + if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { + return -1; + } + + if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { + return -1; + } + + return 0; +} + static SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, SDB_USER_VER, sizeof(SUserObj)); if (pRaw == NULL) return NULL; @@ -103,36 +167,14 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDs return 0; } -static int32_t mndCreateDefaultUser(SMnode *pMnode, char *acct, char *user, char *pass) { - SUserObj userObj = {0}; - tstrncpy(userObj.user, user, TSDB_USER_LEN); - tstrncpy(userObj.acct, acct, TSDB_USER_LEN); - taosEncryptPass((uint8_t *)pass, strlen(pass), userObj.pass); - userObj.createdTime = taosGetTimestampMs(); - userObj.updateTime = userObj.createdTime; - - if (strcmp(user, TSDB_DEFAULT_USER) == 0) { - userObj.superAuth = 1; - } - - SSdbRaw *pRaw = mndUserActionEncode(&userObj); - if (pRaw == NULL) return -1; - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - - mTrace("user:%s, will be created while deploy sdb", userObj.user); - return sdbWrite(pMnode->pSdb, pRaw); +SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { + SSdb *pSdb = pMnode->pSdb; + return sdbAcquire(pSdb, SDB_USER, &userName); } -static int32_t mndCreateDefaultUsers(SMnode *pMnode) { - if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { - return -1; - } - - if (mndCreateDefaultUser(pMnode, TSDB_DEFAULT_USER, "_" TSDB_DEFAULT_USER, TSDB_DEFAULT_PASS) != 0) { - return -1; - } - - return 0; +void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pUser); } static int32_t mndCreateUser(SMnode *pMnode, char *acct, char *user, char *pass, SMnodeMsg *pMsg) { @@ -222,30 +264,14 @@ static int32_t mndProcessCreateUserMsg(SMnodeMsg *pMsg) { return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -int32_t mndInitUser(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_USER, - .keyType = SDB_KEY_BINARY, - .deployFp = (SdbDeployFp)mndCreateDefaultUsers, - .encodeFp = (SdbEncodeFp)mndUserActionEncode, - .decodeFp = (SdbDecodeFp)mndUserActionDecode, - .insertFp = (SdbInsertFp)mndUserActionInsert, - .updateFp = (SdbUpdateFp)mndUserActionUpdate, - .deleteFp = (SdbDeleteFp)mndUserActionDelete}; - - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CREATE_USER, mndProcessCreateUserMsg); - - return sdbSetTable(pMnode->pSdb, table); -} - -void mndCleanupUser(SMnode *pMnode) {} - -SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { - SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_USER, &userName); -} - -void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { - SSdb *pSdb = pMnode->pSdb; - sdbRelease(pSdb, pUser); +static int32_t mndProcessAlterUserMsg(SMnodeMsg *pMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process alter user msg since %s", terrstr()); + return -1; } +static int32_t mndProcessDropUserMsg(SMnodeMsg *pMsg) { + terrno = TSDB_CODE_MND_MSG_NOT_PROCESSED; + mError("failed to process drop user msg since %s", terrstr()); + return -1; +} \ No newline at end of file From aff44f1a47d7a3c166306ad488815823f8cd8aa5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Fri, 3 Dec 2021 21:55:22 +0800 Subject: [PATCH 16/18] TD-10431 dnode test01 --- .../mgmt/impl/test/test01/CMakeLists.txt | 5 +- .../test/test01/{dndTest01.cpp => test01.cpp} | 56 +++++++++++++------ .../impl/test/{test01 => util}/deploy.cpp | 0 3 files changed, 43 insertions(+), 18 deletions(-) rename source/dnode/mgmt/impl/test/test01/{dndTest01.cpp => test01.cpp} (71%) rename source/dnode/mgmt/impl/test/{test01 => util}/deploy.cpp (100%) diff --git a/source/dnode/mgmt/impl/test/test01/CMakeLists.txt b/source/dnode/mgmt/impl/test/test01/CMakeLists.txt index 313f21a2f6..0652ee8381 100644 --- a/source/dnode/mgmt/impl/test/test01/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/test01/CMakeLists.txt @@ -2,12 +2,15 @@ add_executable(dndTest01 "") target_sources(dndTest01 PRIVATE - "dndTest01.cpp" + "test01.cpp" + "../util/deploy.cpp" ) target_link_libraries( dndTest01 PUBLIC dnode + PUBLIC util + PUBLIC os PUBLIC gtest_main ) diff --git a/source/dnode/mgmt/impl/test/test01/dndTest01.cpp b/source/dnode/mgmt/impl/test/test01/test01.cpp similarity index 71% rename from source/dnode/mgmt/impl/test/test01/dndTest01.cpp rename to source/dnode/mgmt/impl/test/test01/test01.cpp index 8cff0d32a1..e1f3729633 100644 --- a/source/dnode/mgmt/impl/test/test01/dndTest01.cpp +++ b/source/dnode/mgmt/impl/test/test01/test01.cpp @@ -14,13 +14,16 @@ */ #include -#include -#include -#include -#include "tthread.h" +#include "os.h" #include "dnode.h" +#include "taosmsg.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" #include "trpc.h" +#include "tthread.h" +#include "ulog.h" typedef struct { SDnode* pDnode; @@ -28,7 +31,7 @@ typedef struct { } SServer; void* runServer(void* param) { - SServer* pServer = param; + SServer* pServer = (SServer*)param; while (1) { taosMsleep(100); pthread_testcancel(); @@ -49,19 +52,19 @@ void initOption(SDnodeOpt* pOption) { pOption->shellActivityTimer = 30; pOption->serverPort = 9527; strncpy(pOption->dataDir, "./test01"); - strncpy(pOption->localEp, "localhost:9527"); - strncpy(pOption->localFqdn, "localhost"); - tstrncpy(pOption->firstEp, "localhost:9527"); + strcpy(pOption->localEp, "localhost:9527"); + strcpy(pOption->localFqdn, "localhost"); + strcpy(pOption->firstEp, "localhost:9527"); } -Server* createServer() { +SServer* createServer() { SDnodeOpt option = {0}; initOption(&option); SDnode* pDnode = dndInit(&option); ASSERT(pDnode); - Server* pServer = calloc(1, sizeof(SServer)); + SServer* pServer = (SServer*)calloc(1, sizeof(SServer)); ASSERT(pServer); pServer->pDnode = pDnode; @@ -84,13 +87,13 @@ typedef struct { } SClient; static void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { - SClient* pClient = parent; + SClient* pClient = (SClient*)parent; pClient->pRsp = pMsg; - tsem_post(pMgmt->clientRpc); + tsem_post(&pClient->sem); } SClient* createClient() { - SClient* pClient = calloc(1, sizeof(SClient)); + SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); ASSERT(pClient); char secretEncrypt[32] = {0}; @@ -107,7 +110,7 @@ SClient* createClient() { rpcInit.idleTime = 30 * 1000; rpcInit.user = "root"; rpcInit.ckey = "key"; - rpcInit.parent = pDnode; + rpcInit.parent = pClient; rpcInit.secret = (char*)secretEncrypt; rpcInit.parent = pClient; // rpcInit.spi = 1; @@ -130,8 +133,8 @@ void sendMsg(SClient* pClient, SRpcMsg* pMsg) { epSet.port[0] = 9527; strcpy(epSet.fqdn[0], "localhost"); - rpcSendRequest(pMgmt->clientRpc, &epSet, pMsg, NULL); - tsem_wait(pMgmt->clientRpc); + rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL); + tsem_wait(&pClient->sem); } class DndTest01 : public ::testing::Test { @@ -150,7 +153,26 @@ class DndTest01 : public ::testing::Test { }; TEST_F(DndTest01, connectMsg) { - SConnectMsg *pReq = rpcMallocCont() + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + pReq->pid = 1234; + strcpy(pReq->app, "test01"); + strcpy(pReq->app, ""); + SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SConnectMsg), .msgType = TSDB_MSG_TYPE_AUTH}; + sendMsg(pClient, &rpcMsg); + + SConnectRsp* pRsp = (SConnectRsp*)pClient->pRsp; + EXPECT_NE(pRsp, NULL); + EXPECT_EQ(pRsp->acctId, 1); + EXPECT_GT(pRsp->clusterId, 0); + EXPECT_GT(pRsp->connId, 1); + EXPECT_EQ(pRsp->superAuth, 1); + EXPECT_EQ(pRsp->readAuth, 1); + EXPECT_EQ(pRsp->writeAuth, 1); + + EXPECT_EQ(pRsp->epSet.inUse, 0); + EXPECT_EQ(pRsp->epSet.numOfEps, 1); + EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } diff --git a/source/dnode/mgmt/impl/test/test01/deploy.cpp b/source/dnode/mgmt/impl/test/util/deploy.cpp similarity index 100% rename from source/dnode/mgmt/impl/test/test01/deploy.cpp rename to source/dnode/mgmt/impl/test/util/deploy.cpp From 7d3a09c5013025c45335fef1509479665942fb3b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 4 Dec 2021 10:23:39 +0800 Subject: [PATCH 17/18] TD-10431 dnode test01 --- include/dnode/mnode/mnode.h | 1 + source/dnode/mgmt/impl/src/dndMnode.c | 3 +- .../mgmt/impl/test/test01/CMakeLists.txt | 5 +- .../dnode/mgmt/impl/test/test01/dndTest01.cpp | 95 +++++++++++++++++++ source/dnode/mgmt/impl/test/util/deploy.cpp | 0 .../test01.cpp => util/dndTestDeploy.cpp} | 93 +++++------------- .../dnode/mgmt/impl/test/util/dndTestDeploy.h | 43 +++++++++ source/dnode/mnode/impl/inc/mndUser.h | 2 +- source/dnode/mnode/impl/src/mndProfile.c | 5 +- source/dnode/mnode/impl/src/mndUser.c | 4 +- source/dnode/mnode/impl/src/mnode.c | 1 + source/libs/transport/src/rpcMain.c | 4 - 12 files changed, 173 insertions(+), 83 deletions(-) create mode 100644 source/dnode/mgmt/impl/test/test01/dndTest01.cpp delete mode 100644 source/dnode/mgmt/impl/test/util/deploy.cpp rename source/dnode/mgmt/impl/test/{test01/test01.cpp => util/dndTestDeploy.cpp} (62%) create mode 100644 source/dnode/mgmt/impl/test/util/dndTestDeploy.h diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index c7415af0d5..0a897b93f8 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -57,6 +57,7 @@ typedef struct { int32_t sver; int32_t statusInterval; int32_t mnodeEqualVnodeNum; + int32_t shellActivityTimer; char *timezone; char *locale; char *charset; diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 4ec08d5fb3..4afce4655d 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -334,6 +334,7 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->sver = pDnode->opt.sver; pOption->statusInterval = pDnode->opt.statusInterval; pOption->mnodeEqualVnodeNum = pDnode->opt.mnodeEqualVnodeNum; + pOption->shellActivityTimer = pDnode->opt.shellActivityTimer; pOption->timezone = pDnode->opt.timezone; pOption->charset = pDnode->opt.charset; pOption->locale = pDnode->opt.locale; @@ -675,7 +676,7 @@ void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SMnodeMgmt *pMgmt = &pDnode->mmgmt; SMnode *pMnode = dndAcquireMnode(pDnode); - if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) { + if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) { SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno}; rpcSendResponse(&rsp); rpcFreeCont(pMsg->pCont); diff --git a/source/dnode/mgmt/impl/test/test01/CMakeLists.txt b/source/dnode/mgmt/impl/test/test01/CMakeLists.txt index 0652ee8381..c3037dfdff 100644 --- a/source/dnode/mgmt/impl/test/test01/CMakeLists.txt +++ b/source/dnode/mgmt/impl/test/test01/CMakeLists.txt @@ -2,8 +2,8 @@ add_executable(dndTest01 "") target_sources(dndTest01 PRIVATE - "test01.cpp" - "../util/deploy.cpp" + "dndTest01.cpp" + "../util/dndTestDeploy.cpp" ) target_link_libraries( @@ -18,6 +18,7 @@ target_include_directories(dndTest01 PUBLIC "${CMAKE_SOURCE_DIR}/include/server/dnode/mgmt" "${CMAKE_CURRENT_SOURCE_DIR}/../../inc" + "${CMAKE_CURRENT_SOURCE_DIR}/../util" ) enable_testing() diff --git a/source/dnode/mgmt/impl/test/test01/dndTest01.cpp b/source/dnode/mgmt/impl/test/test01/dndTest01.cpp new file mode 100644 index 0000000000..70f649cc07 --- /dev/null +++ b/source/dnode/mgmt/impl/test/test01/dndTest01.cpp @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "dndTestDeploy.h" + +class DndTest01 : public ::testing::Test { + protected: + void SetUp() override { + pServer = createServer("/tmp/dndTest01"); + pClient = createClient("root", "taosdata"); + } + void TearDown() override { + dropServer(pServer); + dropClient(pClient); + } + + SServer* pServer; + SClient* pClient; +}; + +TEST_F(DndTest01, connectMsg) { + SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); + pReq->pid = htonl(1234); + strcpy(pReq->app, "test01"); + strcpy(pReq->db, ""); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = pReq; + rpcMsg.contLen = sizeof(SConnectMsg); + rpcMsg.msgType = TSDB_MSG_TYPE_CONNECT; + + sendMsg(pClient, &rpcMsg); + + SConnectRsp* pRsp = (SConnectRsp*)pClient->pRsp; + ASSERT(pRsp); + pRsp->acctId = htonl(pRsp->acctId); + pRsp->clusterId = htonl(pRsp->clusterId); + pRsp->connId = htonl(pRsp->connId); + pRsp->epSet.port[0] = htonl(pRsp->epSet.port[0]); + + EXPECT_EQ(pRsp->acctId, 1); + EXPECT_GT(pRsp->clusterId, 0); + EXPECT_GT(pRsp->connId, 1); + EXPECT_EQ(pRsp->superAuth, 1); + EXPECT_EQ(pRsp->readAuth, 1); + EXPECT_EQ(pRsp->writeAuth, 1); + + EXPECT_EQ(pRsp->epSet.inUse, 0); + EXPECT_EQ(pRsp->epSet.numOfEps, 1); + EXPECT_EQ(pRsp->epSet.port[0], 9527); + EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); +} + +// TEST_F(DndTest01, heartbeatMsg) { +// SHeartBeatMsg* pReq = (SHeartBeatMsg*)rpcMallocCont(sizeof(SHeartBeatMsg)); +// pReq->connId = htonl(1); +// pReq->pid = htonl(1234); +// pReq->numOfQueries = htonl(0); +// pReq->numOfStreams = htonl(0); +// strcpy(pReq->app, "test01"); + +// SRpcMsg rpcMsg = {0}; +// rpcMsg.pCont = pReq; +// rpcMsg.contLen = sizeof(SHeartBeatMsg); +// rpcMsg.msgType = TSDB_MSG_TYPE_HEARTBEAT; + +// sendMsg(pClient, &rpcMsg); + +// SHeartBeatRsp* pRsp = (SHeartBeatRsp*)pClient->pRsp; +// ASSERT(pRsp); + // pRsp->epSet.port[0] = htonl(pRsp->epSet.port[0]); + +// EXPECT_EQ(htonl(pRsp->connId), 1); +// EXPECT_GT(htonl(pRsp->queryId), 0); +// EXPECT_GT(htonl(pRsp->streamId), 1); +// EXPECT_EQ(htonl(pRsp->totalDnodes), 1); +// EXPECT_EQ(htonl(pRsp->onlineDnodes), 1); +// EXPECT_EQ(pRsp->killConnection, 0); + // EXPECT_EQ(pRsp->epSet.inUse, 0); + // EXPECT_EQ(pRsp->epSet.numOfEps, 1); + // EXPECT_EQ(pRsp->epSet.port[0], 9527); + // EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); +// } diff --git a/source/dnode/mgmt/impl/test/util/deploy.cpp b/source/dnode/mgmt/impl/test/util/deploy.cpp deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/dnode/mgmt/impl/test/test01/test01.cpp b/source/dnode/mgmt/impl/test/util/dndTestDeploy.cpp similarity index 62% rename from source/dnode/mgmt/impl/test/test01/test01.cpp rename to source/dnode/mgmt/impl/test/util/dndTestDeploy.cpp index e1f3729633..4b4f0fa763 100644 --- a/source/dnode/mgmt/impl/test/test01/test01.cpp +++ b/source/dnode/mgmt/impl/test/util/dndTestDeploy.cpp @@ -13,22 +13,16 @@ * along with this program. If not, see . */ -#include -#include "os.h" +#include "dndTestDeploy.h" -#include "dnode.h" -#include "taosmsg.h" -#include "tconfig.h" -#include "tglobal.h" -#include "tnote.h" -#include "trpc.h" -#include "tthread.h" -#include "ulog.h" - -typedef struct { - SDnode* pDnode; - pthread_t* threadId; -} SServer; +void initLog(char *path) { + mDebugFlag = 207; + char temp[PATH_MAX]; + snprintf(temp, PATH_MAX, "%s/taosdlog", path); + if (taosInitLog(temp, tsNumOfLogLines, 1) != 0) { + printf("failed to init log file\n"); + } +} void* runServer(void* param) { SServer* pServer = (SServer*)param; @@ -38,7 +32,7 @@ void* runServer(void* param) { } } -void initOption(SDnodeOpt* pOption) { +void initOption(SDnodeOpt* pOption, char *path) { pOption->sver = 1; pOption->numOfCores = 1; pOption->numOfSupportMnodes = 1; @@ -51,15 +45,18 @@ void initOption(SDnodeOpt* pOption) { pOption->maxShellConns = 1000; pOption->shellActivityTimer = 30; pOption->serverPort = 9527; - strncpy(pOption->dataDir, "./test01"); + strcpy(pOption->dataDir, path); strcpy(pOption->localEp, "localhost:9527"); strcpy(pOption->localFqdn, "localhost"); strcpy(pOption->firstEp, "localhost:9527"); + + taosRemoveDir(path); + taosMkDir(path); } -SServer* createServer() { +SServer* createServer(char *path) { SDnodeOpt option = {0}; - initOption(&option); + initOption(&option, path); SDnode* pDnode = dndInit(&option); ASSERT(pDnode); @@ -80,24 +77,18 @@ void dropServer(SServer* pServer) { } } -typedef struct { - void* clientRpc; - SRpcMsg* pRsp; - tsem_t sem; -} SClient; - -static void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { +void processClientRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { SClient* pClient = (SClient*)parent; pClient->pRsp = pMsg; + taosMsleep(100000); tsem_post(&pClient->sem); } -SClient* createClient() { +SClient* createClient(char *user, char *pass) { SClient* pClient = (SClient*)calloc(1, sizeof(SClient)); ASSERT(pClient); - char secretEncrypt[32] = {0}; - char* pass = "taosdata"; + char secretEncrypt[32] = {0}; taosEncryptPass((uint8_t*)pass, strlen(pass), secretEncrypt); SRpcInit rpcInit; @@ -108,7 +99,7 @@ SClient* createClient() { rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; rpcInit.idleTime = 30 * 1000; - rpcInit.user = "root"; + rpcInit.user = user; rpcInit.ckey = "key"; rpcInit.parent = pClient; rpcInit.secret = (char*)secretEncrypt; @@ -119,6 +110,8 @@ SClient* createClient() { ASSERT(pClient->clientRpc); tsem_init(&pClient->sem, 0, 0); + + return pClient; } void dropClient(SClient* pClient) { @@ -136,43 +129,3 @@ void sendMsg(SClient* pClient, SRpcMsg* pMsg) { rpcSendRequest(pClient->clientRpc, &epSet, pMsg, NULL); tsem_wait(&pClient->sem); } - -class DndTest01 : public ::testing::Test { - protected: - void SetUp() override { - pServer = createServer(); - pClient = createClient(); - } - void TearDown() override { - dropServer(pServer); - dropClient(pClient); - } - - SServer* pServer; - SClient* pClient; -}; - -TEST_F(DndTest01, connectMsg) { - SConnectMsg* pReq = (SConnectMsg*)rpcMallocCont(sizeof(SConnectMsg)); - pReq->pid = 1234; - strcpy(pReq->app, "test01"); - strcpy(pReq->app, ""); - - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SConnectMsg), .msgType = TSDB_MSG_TYPE_AUTH}; - - sendMsg(pClient, &rpcMsg); - - SConnectRsp* pRsp = (SConnectRsp*)pClient->pRsp; - EXPECT_NE(pRsp, NULL); - EXPECT_EQ(pRsp->acctId, 1); - EXPECT_GT(pRsp->clusterId, 0); - EXPECT_GT(pRsp->connId, 1); - EXPECT_EQ(pRsp->superAuth, 1); - EXPECT_EQ(pRsp->readAuth, 1); - EXPECT_EQ(pRsp->writeAuth, 1); - - EXPECT_EQ(pRsp->epSet.inUse, 0); - EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9527); - EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); -} diff --git a/source/dnode/mgmt/impl/test/util/dndTestDeploy.h b/source/dnode/mgmt/impl/test/util/dndTestDeploy.h new file mode 100644 index 0000000000..6c47b03fe7 --- /dev/null +++ b/source/dnode/mgmt/impl/test/util/dndTestDeploy.h @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include "os.h" + +#include "dnode.h" +#include "taosmsg.h" +#include "tconfig.h" +#include "tglobal.h" +#include "tnote.h" +#include "trpc.h" +#include "tthread.h" +#include "ulog.h" + +typedef struct { + SDnode* pDnode; + pthread_t* threadId; +} SServer; + +typedef struct { + void* clientRpc; + SRpcMsg* pRsp; + tsem_t sem; +} SClient; + +SServer* createServer(char* path); +void dropServer(SServer* pServer); +SClient* createClient(char *user, char *pass); +void dropClient(SClient* pClient); +void sendMsg(SClient* pClient, SRpcMsg* pMsg); diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 4d31a87b19..e4ecdf3283 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -24,7 +24,7 @@ extern "C" { int32_t mndInitUser(SMnode *pMnode); void mndCleanupUser(SMnode *pMnode); -SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName); +SUserObj *mndAcquireUser(SMnode *pMnode, char *userName); void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index eb9926cfb6..2afb5f0665 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -78,7 +78,7 @@ int32_t mndInitProfile(SMnode *pMnode) { mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_HEARTBEAT, mndProcessHeartBeatMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessConnectMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_QUERY, mndProcessKillQueryMsg); - mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_CONNECT, mndProcessKillStreamMsg); + mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_STREAM, mndProcessKillStreamMsg); mndSetMsgHandle(pMnode, TSDB_MSG_TYPE_KILL_CONN, mndProcessKillConnectionMsg); mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_CONNS, mndGetConnsMeta); @@ -237,7 +237,6 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) { mndReleaseUser(pMnode, pUser); } - pRsp->acctId = htonl(pUser->acctId); pRsp->clusterId = htonl(pMnode->clusterId); pRsp->connId = htonl(pConn->connId); mndGetMnodeEpSet(pMnode, &pRsp->epSet); @@ -330,7 +329,7 @@ static int32_t mndProcessHeartBeatMsg(SMnodeMsg *pMsg) { } pRsp->connId = htonl(pConn->connId); - pRsp->totalDnodes = htnol(1); + pRsp->totalDnodes = htonl(1); pRsp->onlineDnodes = htonl(1); mndGetMnodeEpSet(pMnode, &pRsp->epSet); mndReleaseConn(pMnode, pConn); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 020de13e36..f59ef8f8e3 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -167,9 +167,9 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pSrcUser, SUserObj *pDs return 0; } -SUserObj *mndAcquireUser(SMnode *pMnode, const char *userName) { +SUserObj *mndAcquireUser(SMnode *pMnode, char *userName) { SSdb *pSdb = pMnode->pSdb; - return sdbAcquire(pSdb, SDB_USER, &userName); + return sdbAcquire(pSdb, SDB_USER, userName); } void mndReleaseUser(SMnode *pMnode, SUserObj *pUser) { diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 3a5b6c470b..b99de4a019 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -206,6 +206,7 @@ static int32_t mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->sver = pOption->sver; pMnode->statusInterval = pOption->statusInterval; pMnode->mnodeEqualVnodeNum = pOption->mnodeEqualVnodeNum; + pMnode->shellActivityTimer = pOption->shellActivityTimer; pMnode->timezone = strdup(pOption->timezone); pMnode->locale = strdup(pOption->locale); pMnode->charset = strdup(pOption->charset); diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index a2041c76fc..e392351366 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -507,7 +507,6 @@ void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { } int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { -#if 0 SRpcConn *pConn = (SRpcConn *)thandle; if (pConn->user[0] == 0) return -1; @@ -516,9 +515,6 @@ int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { // pInfo->serverIp = pConn->destIp; tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); -#else - strcpy(pInfo->user, "root"); -#endif return 0; } From d0844e5dda06e73e4b20571f341f656d68fab860 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 4 Dec 2021 21:40:09 +0800 Subject: [PATCH 18/18] refactor builder struct --- source/libs/index/src/index_fst.c | 31 ++++++++++--------- .../index/src/index_fst_counting_writer.c | 5 ++- source/libs/index/src/index_fst_node.c | 20 +++++++----- source/libs/index/test/indexTests.cpp | 10 +++--- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/source/libs/index/src/index_fst.c b/source/libs/index/src/index_fst.c index b2d21cd0b0..418d1f2bda 100644 --- a/source/libs/index/src/index_fst.c +++ b/source/libs/index/src/index_fst.c @@ -62,7 +62,7 @@ FstBuilderNode *fstUnFinishedNodesPopRoot(FstUnFinishedNodes *nodes) { assert(taosArrayGetSize(nodes->stack) == 1); FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); - assert(un->last == NULL); + //assert(un->last == NULL); return un->node; } @@ -70,6 +70,7 @@ FstBuilderNode *fstUnFinishedNodesPopFreeze(FstUnFinishedNodes *nodes, CompiledA FstBuilderNodeUnfinished *un = taosArrayPop(nodes->stack); fstBuilderNodeUnfinishedLastCompiled(un, addr); free(un->last); // TODO add func FstLastTransitionFree() + un->last = NULL; return un->node; } @@ -99,8 +100,6 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output FstBuilderNodeUnfinished *un = taosArrayGet(nodes->stack, sz); assert(un->last == NULL); - - //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); //trn->inp = s->data[s->start]; //trn->out = out; @@ -108,7 +107,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output uint8_t *data = fstSliceData(s, &len); un->last = fstLastTransitionCreate(data[0], out); - for (uint64_t i = 0; i < len; i++) { + for (uint64_t i = 1; i < len; i++) { FstBuilderNode *n = malloc(sizeof(FstBuilderNode)); n->isFinal = false; n->finalOutput = 0; @@ -117,7 +116,7 @@ void fstUnFinishedNodesAddSuffix(FstUnFinishedNodes *nodes, FstSlice bs, Output //FstLastTransition *trn = malloc(sizeof(FstLastTransition)); //trn->inp = s->data[i]; //trn->out = out; - FstLastTransition *trn = fstLastTransitionCreate(data[i], out); + FstLastTransition *trn = fstLastTransitionCreate(data[i], 0); FstBuilderNodeUnfinished un = {.node = n, .last = trn}; taosArrayPush(nodes->stack, &un); @@ -131,8 +130,8 @@ uint64_t fstUnFinishedNodesFindCommPrefix(FstUnFinishedNodes *node, FstSlice bs) size_t ssz = taosArrayGetSize(node->stack); // stack size uint64_t count = 0; - int32_t lsz; // data len - uint8_t *data = fstSliceData(s, &lsz); + int32_t lsz; // data len + uint8_t *data = fstSliceData(s, &lsz); for (size_t i = 0; i < ssz && i < lsz; i++) { FstBuilderNodeUnfinished *un = taosArrayGet(node->stack, i); if (un->last->inp == data[i]) { @@ -854,7 +853,7 @@ void fstBuilderCompileFrom(FstBuilder *b, uint64_t istate) { } else { n = fstUnFinishedNodesPopFreeze(b->unfinished, addr); } - addr = fstBuilderCompile(b, n); + addr = fstBuilderCompile(b, n); assert(addr != NONE_ADDRESS); //fstBuilderNodeDestroy(n); } @@ -890,22 +889,26 @@ void* fstBuilderInsertInner(FstBuilder *b) { FstBuilderNode *rootNode = fstUnFinishedNodesPopRoot(b->unfinished); CompiledAddr rootAddr = fstBuilderCompile(b, rootNode); - uint8_t buf64[8] = {0}; + char buf64[8] = {0}; - taosEncodeFixedU64((void **)&buf64, b->len); + void *pBuf64 = buf64; + taosEncodeFixedU64(&pBuf64, b->len); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - taosEncodeFixedU64((void **)&buf64, rootAddr); + pBuf64 = buf64; + taosEncodeFixedU64(&pBuf64, rootAddr); fstCountingWriterWrite(b->wrt, buf64, sizeof(buf64)); - uint8_t buf32[4] = {0}; + char buf32[4] = {0}; + void *pBuf32 = buf32; uint32_t sum = fstCountingWriterMaskedCheckSum(b->wrt); - taosEncodeFixedU32((void **)&buf32, sum); + taosEncodeFixedU32(&pBuf32, sum); fstCountingWriterWrite(b->wrt, buf32, sizeof(buf32)); fstCountingWriterFlush(b->wrt); + //fstCountingWriterDestroy(b->wrt); + //b->wrt = NULL; return b->wrt; - } void fstBuilderFinish(FstBuilder *b) { fstBuilderInsertInner(b); diff --git a/source/libs/index/src/index_fst_counting_writer.c b/source/libs/index/src/index_fst_counting_writer.c index 1989a5a425..6820292e65 100644 --- a/source/libs/index/src/index_fst_counting_writer.c +++ b/source/libs/index/src/index_fst_counting_writer.c @@ -52,9 +52,12 @@ WriterCtx* writerCtxCreate(WriterType type) { WriterCtx *ctx = calloc(1, sizeof(WriterCtx)); if (ctx == NULL) { return NULL; } - ctx->type == type; + ctx->type = type; if (ctx->type == TFile) { ctx->fd = tfOpenCreateWriteAppend(tmpFile); + if (ctx->fd < 0) { + + } } else if (ctx->type == TMemory) { ctx->mem = calloc(1, DefaultMem * sizeof(uint8_t)); } diff --git a/source/libs/index/src/index_fst_node.c b/source/libs/index/src/index_fst_node.c index af69f98f32..5abe8ad5a0 100644 --- a/source/libs/index/src/index_fst_node.c +++ b/source/libs/index/src/index_fst_node.c @@ -30,20 +30,26 @@ void fstBuilderNodeDestroy(FstBuilderNode *node) { bool fstBuilderNodeEqual(FstBuilderNode *n1, FstBuilderNode *n2) { if (n1 == n2) { return true; } + if (n1 == NULL || n2 == NULL ) { + return false; + } - if (n1->isFinal != n2->isFinal || - n1->finalOutput != n2->finalOutput || - taosArrayGetSize(n1->trans) != taosArrayGetSize(n2->trans)) { - return false; - } - size_t sz = taosArrayGetSize(n1->trans); - for (size_t i = 0; i < sz; i++) { + if (n1->isFinal != n2->isFinal || n1->finalOutput != n2->finalOutput) { + return false; + } + size_t s1 = n1->trans? taosArrayGetSize(n1->trans): 0; + size_t s2 = n2->trans? taosArrayGetSize(n2->trans): 0; + if (s1 != s2) { + return false; + } + for (size_t i = 0; i < s1; i++) { FstTransition *t1 = taosArrayGet(n1->trans, i); FstTransition *t2 = taosArrayGet(n2->trans, i); if (t1->inp != t2->inp || t1->out != t2->out || t1->addr != t2->addr) { return false; } } + return true; } FstBuilderNode *fstBuilderNodeClone(FstBuilderNode *src) { diff --git a/source/libs/index/test/indexTests.cpp b/source/libs/index/test/indexTests.cpp index 9f3d1b962e..14ff2caf12 100644 --- a/source/libs/index/test/indexTests.cpp +++ b/source/libs/index/test/indexTests.cpp @@ -65,14 +65,14 @@ int main(int argc, char** argv) { std::string str("abc"); FstSlice key = fstSliceCreate((uint8_t *)str.c_str(), str.size()); - Output val = 10; + Output val = 1; - std::string str1("bcd"); - FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size()); - Output val2 = 10; + //std::string str1("bcd"); + //FstSlice key1 = fstSliceCreate((uint8_t *)str1.c_str(), str1.size()); + //Output val2 = 10; FstBuilder *b = fstBuilderCreate(NULL, 1); fstBuilderInsert(b, key, val); - fstBuilderInsert(b, key1, val2); + //fstBuilderInsert(b, key1, val2); fstBuilderFinish(b); fstBuilderDestroy(b); fstSliceDestroy(&key);