diff --git a/.gitignore b/.gitignore index a69d2e44bd..a6e222d2e9 100644 --- a/.gitignore +++ b/.gitignore @@ -130,3 +130,4 @@ tools/COPYING tools/BUGS tools/taos-tools tools/taosws-rs +tags diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index a36a7513f3..0bfb057f20 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -29,6 +29,7 @@ extern "C" { typedef struct SBuffer SBuffer; typedef struct SSchema SSchema; +typedef struct SSchema2 SSchema2; typedef struct STColumn STColumn; typedef struct STSchema STSchema; typedef struct SValue SValue; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 6d782af909..77b9d2d681 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -144,6 +144,8 @@ typedef enum _mgmt_table { #define TSDB_ALTER_TABLE_UPDATE_TAG_BYTES 8 #define TSDB_ALTER_TABLE_UPDATE_OPTIONS 9 #define TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME 10 +#define TSDB_ALTER_TABLE_ADD_TAG_INDEX 11 +#define TSDB_ALTER_TABLE_DROP_TAG_INDEX 12 #define TSDB_FILL_NONE 0 #define TSDB_FILL_NULL 1 @@ -294,6 +296,15 @@ struct SSchema { char name[TSDB_COL_NAME_LEN]; }; +struct SSchema2 { + int8_t type; + int8_t flags; + col_id_t colId; + int32_t bytes; + char name[TSDB_COL_NAME_LEN]; + char alias[TSDB_COL_NAME_LEN]; +}; + typedef struct { char tbName[TSDB_TABLE_NAME_LEN]; char stbName[TSDB_TABLE_NAME_LEN]; @@ -347,8 +358,19 @@ void tFreeSSubmitRsp(SSubmitRsp* pRsp); #define COL_CLR_SET(FLG) ((FLG) &= (~(COL_SET_VAL | COL_SET_NULL))) #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON) +#define IS_IDX_ON(s) (((s)->flags & 0x02) == COL_IDX_ON) #define IS_SET_NULL(s) (((s)->flags & COL_SET_NULL) == COL_SET_NULL) +#define SSCHMEA_SET_IDX_ON(s) \ + do { \ + (s)->flags |= COL_IDX_ON; \ + } while (0) + +#define SSCHMEA_SET_IDX_OFF(s) \ + do { \ + (s)->flags &= (~COL_IDX_ON); \ + } while (0) + #define SSCHMEA_TYPE(s) ((s)->type) #define SSCHMEA_FLAGS(s) ((s)->flags) #define SSCHMEA_COLID(s) ((s)->colId) @@ -1240,6 +1262,17 @@ typedef struct { int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); int32_t tDeserializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); +typedef struct { + char colName[TSDB_COL_NAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + int64_t stbUid; + int64_t dbUid; + int64_t reserved[8]; +} SDropIndexReq; + +int32_t tSerializeSDropIdxReq(void* buf, int32_t bufLen, SDropIndexReq* pReq); +int32_t tDeserializeSDropIdxReq(void* buf, int32_t bufLen, SDropIndexReq* pReq); + typedef struct { int64_t dbUid; char db[TSDB_DB_FNAME_LEN]; @@ -2813,6 +2846,22 @@ typedef struct { int32_t tSerializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); int32_t tDeserializeSMDropSmaReq(void* buf, int32_t bufLen, SMDropSmaReq* pReq); +typedef struct { + char dbFName[TSDB_DB_FNAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; + char idxName[TSDB_COL_NAME_LEN]; + int8_t idxType; +} SCreateTagIndexReq; + +int32_t tSerializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexReq* pReq); +int32_t tDeserializeSCreateTagIdxReq(void* buf, int32_t bufLen, SCreateTagIndexReq* pReq); + +typedef SMDropSmaReq SDropTagIndexReq; + +int32_t tSerializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); +int32_t tDeserializeSDropTagIdxReq(void* buf, int32_t bufLen, SDropTagIndexReq* pReq); + typedef struct { int8_t version; // for compatibility(default 0) int8_t intervalUnit; // MACRO: TIME_UNIT_XXX diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 44ee26ef24..7aae38f7ba 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -220,6 +220,9 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "vnode-drop-ttl-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TRIM, "vnode-trim", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "vnode-commit", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DROP_INDEX, "vnode-drop-index", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_DISABLE_WRITE, "vnode-disable-write", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_MAX_MSG, "vnd-max", NULL, NULL) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 548f520db6..56c10f2463 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -257,6 +257,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_STB_OPTION TAOS_DEF_ERROR_CODE(0, 0x036E) #define TSDB_CODE_MND_INVALID_ROW_BYTES TAOS_DEF_ERROR_CODE(0, 0x036F) + // mnode-func #define TSDB_CODE_MND_INVALID_FUNC_NAME TAOS_DEF_ERROR_CODE(0, 0x0370) // #define TSDB_CODE_MND_INVALID_FUNC_LEN TAOS_DEF_ERROR_CODE(0, 0x0371) // 2.x @@ -269,6 +270,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_FUNC_COMMENT TAOS_DEF_ERROR_CODE(0, 0x0378) #define TSDB_CODE_MND_INVALID_FUNC_RETRIEVE TAOS_DEF_ERROR_CODE(0, 0x0379) + + // mnode-db #define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) #define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) // @@ -370,6 +373,11 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481) #define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482) +// mnode-tag-indxe + +#define TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0483) +#define TSDB_CODE_MND_TAG_INDEX_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0484) + // dnode // #define TSDB_CODE_DND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0400) // 2.x // #define TSDB_CODE_DND_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0401) // 2.x diff --git a/include/util/tdef.h b/include/util/tdef.h index 04f21570ff..85c89744ed 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -104,6 +104,7 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_INDEX_TYPE_SMA "SMA" #define TSDB_INDEX_TYPE_FULLTEXT "FULLTEXT" +#define TSDB_INDEX_TYPE_NORMAL "NORMAL" #define TSDB_INS_USER_STABLES_DBNAME_COLID 2 diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 60002425f7..b83e7ee976 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -1273,7 +1273,7 @@ static int32_t smlInsertData(SSmlHandle *info) { } static void smlPrintStatisticInfo(SSmlHandle *info) { - uError( + uDebug( "SML:0x%" PRIx64 " smlInsertLines result, code:%d,lineNum:%d,stable num:%d,ctable num:%d,create stable num:%d,alter stable tag num:%d,alter stable col num:%d \ parse cost:%" PRId64 ",schema cost:%" PRId64 ",bind cost:%" PRId64 ",rpc cost:%" PRId64 ",total cost:%" PRId64 diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 0e185fa1c0..6198f923f4 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -120,6 +120,8 @@ static const SSysDbTableSchema userIdxSchema[] = { {.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false}, + {.name = "column_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "index_type", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysDbTableSchema userStbsSchema[] = { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e5f56e1fa7..e180959d1e 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -918,6 +918,65 @@ int32_t tDeserializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) tDecoderClear(&decoder); return 0; } + +int32_t tSerializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->stbName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->colName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->idxName) < 0) return -1; + if (tEncodeI8(&encoder, pReq->idxType) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} +int32_t tDeserializeSCreateTagIdxReq(void *buf, int32_t bufLen, SCreateTagIndexReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->stbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->colName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->idxName) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->idxType) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} +int32_t tSerializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + tEndEncode(&encoder); + + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} +int32_t tDeserializeSDropTagIdxReq(void *buf, int32_t bufLen, SDropTagIndexReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + + return 0; +} int32_t tSerializeSMCreateFullTextReq(void *buf, int32_t bufLen, SMCreateFullTextReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3984,6 +4043,44 @@ int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq tDecoderClear(&decoder); return 0; } +int32_t tSerializeSDropIdxReq(void *buf, int32_t bufLen, SDropIndexReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->colName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->stb) < 0) return -1; + if (tEncodeI64(&encoder, pReq->stbUid) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dbUid) < 0) return -1; + for (int32_t i = 0; i < 8; ++i) { + if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; + // TODO + return 0; +} +int32_t tDeserializeSDropIdxReq(void *buf, int32_t bufLen, SDropIndexReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->colName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->stb) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->stbUid) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dbUid) < 0) return -1; + for (int32_t i = 0; i < 8; ++i) { + if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; + } + tEndDecode(&decoder); + + tDecoderClear(&decoder); + // TODO + return 0; +} int32_t tSerializeSCompactVnodeReq(void *buf, int32_t bufLen, SCompactVnodeReq *pReq) { SEncoder encoder = {0}; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 7d11bc7082..fe65c3dde9 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -157,6 +157,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_AUTH, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SHOW_VARIABLES, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_SERVER_VERSION, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_INDEX, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_MERGE_QUERY, mmPutMsgToQueryQueue, 1) == NULL) goto _OVER; @@ -182,6 +184,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_HASHRANGE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMPACT_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DISABLE_WRITE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT, mmPutMsgToSyncQueue, 1) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index 2d6a4954e9..5cf408a905 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -522,6 +522,8 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SCH_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_INDEX, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 6ff1dc576e..ac0c68f652 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -389,6 +389,18 @@ typedef struct { SSchemaWrapper schemaTag; // for dstVgroup } SSmaObj; +typedef struct { + char name[TSDB_TABLE_FNAME_LEN]; + char stb[TSDB_TABLE_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + char dstTbName[TSDB_TABLE_FNAME_LEN]; + char colName[TSDB_COL_NAME_LEN]; + int64_t createdTime; + int64_t uid; + int64_t stbUid; + int64_t dbUid; +} SIdxObj; + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char db[TSDB_DB_FNAME_LEN]; diff --git a/source/dnode/mnode/impl/inc/mndIndex.h b/source/dnode/mnode/impl/inc/mndIndex.h new file mode 100644 index 0000000000..2d5479bc9b --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndIndex.h @@ -0,0 +1,34 @@ +#ifndef _TD_MND_IDX_H_ +#define _TD_MND_IDX_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t mndInitIdx(SMnode *pMnode); +void mndCleanupIdx(SMnode *pMnode); +SIdxObj *mndAcquireIdx(SMnode *pMnode, char *Name); +void mndReleaseIdx(SMnode *pMnode, SIdxObj *pSma); +int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb); +int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); +int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *pIdx); +int32_t mndGetTableIdx(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist); + +int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq); + +int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); + +int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); +int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_IDX_H_*/ \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndIndexComm.h b/source/dnode/mnode/impl/inc/mndIndexComm.h new file mode 100644 index 0000000000..68112f0a2d --- /dev/null +++ b/source/dnode/mnode/impl/inc/mndIndexComm.h @@ -0,0 +1,43 @@ + +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_MND_IDX_COMM_H_ +#define _TD_MND_IDX_COMM_H_ + +#include "mndInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SSIdx { + int type; // sma or idx + void *pIdx; +} SSIdx; + +// retrieve sma index and tag index +typedef struct { + void *pSmaIter; + void *pIdxIter; +} SSmaAndTagIter; + +int32_t mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_MND_IDX_COMM_H_*/ \ No newline at end of file diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 6a40316923..ac0924aab9 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -42,6 +42,11 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t const char *mndGetStbStr(const char *src); +int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew); +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId); +void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen, void *alterOriData, + int32_t alterOriDataLen); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 1943ab0b0f..2f7fa26a87 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -17,6 +17,7 @@ #include "mndDb.h" #include "mndCluster.h" #include "mndDnode.h" +#include "mndIndex.h" #include "mndPrivilege.h" #include "mndShow.h" #include "mndSma.h" @@ -1053,6 +1054,7 @@ static int32_t mndDropDb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb) { /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndDropStreamByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndDropSmasByDb(pMnode, pTrans, pDb) != 0) goto _OVER; + if (mndDropIdxsByDb(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndUserRemoveDb(pMnode, pTrans, pDb->name) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndIndex.c b/source/dnode/mnode/impl/src/mndIndex.c new file mode 100644 index 0000000000..016f01b032 --- /dev/null +++ b/source/dnode/mnode/impl/src/mndIndex.c @@ -0,0 +1,877 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define _DEFAULT_SOURCE +#include "mndIndex.h" +#include "mndDb.h" +#include "mndDnode.h" +#include "mndIndexComm.h" +#include "mndInfoSchema.h" +#include "mndMnode.h" +#include "mndPrivilege.h" +#include "mndScheduler.h" +#include "mndShow.h" +#include "mndStb.h" +#include "mndStream.h" +#include "mndTrans.h" +#include "mndUser.h" +#include "mndVgroup.h" +#include "parser.h" +#include "tname.h" + +#define TSDB_IDX_VER_NUMBER 1 +#define TSDB_IDX_RESERVE_SIZE 64 + +static SSdbRaw *mndIdxActionEncode(SIdxObj *pSma); +static SSdbRow *mndIdxActionDecode(SSdbRaw *pRaw); +static int32_t mndIdxActionInsert(SSdb *pSdb, SIdxObj *pIdx); +static int32_t mndIdxActionDelete(SSdb *pSdb, SIdxObj *pIdx); +static int32_t mndIdxActionUpdate(SSdb *pSdb, SIdxObj *pOld, SIdxObj *pNew); +static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq); +// static int32_t mndProcessDropIdxReq(SRpcMsg *pReq); +static int32_t mndProcessGetIdxReq(SRpcMsg *pReq); +static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq); +// static int32_t mndRetrieveIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +// static void mndCancelGetNextIdx(SMnode *pMnode, void *pIter); +static void mndDestroyIdxObj(SIdxObj *pIdxObj); + +static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb); + +int32_t mndInitIdx(SMnode *pMnode) { + SSdbTable table = { + .sdbType = SDB_IDX, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndIdxActionEncode, + .decodeFp = (SdbDecodeFp)mndIdxActionDecode, + .insertFp = (SdbInsertFp)mndIdxActionInsert, + .updateFp = (SdbUpdateFp)mndIdxActionUpdate, + .deleteFp = (SdbDeleteFp)mndIdxActionDelete, + }; + + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIdxReq); + // mndSetMsgHandle(pMnode, TDMT_MND_DROP_INDEX, mndProcessDropIdxReq); + + mndSetMsgHandle(pMnode, TDMT_VND_CREATE_INDEX_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_VND_DROP_INDEX_RSP, mndTransProcessRsp); + + // mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessCreateIdxReq); + // mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessDropIdxReq); + // mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndTransProcessRsp); + // mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndTransProcessRsp); + // mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIdxReq); + // mndSetMsgHandle(pMnode, TDMT_MND_GET_TABLE_INDEX, mndProcessGetTbIdxReq); + + // type same with sma + // mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveIdx); + // mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextIdx); + return sdbSetTable(pMnode->pSdb, table); +} + +static int32_t mndFindSuperTableTagId(const SStbObj *pStb, const char *tagName) { + for (int32_t tag = 0; tag < pStb->numOfTags; tag++) { + if (strcasecmp(pStb->pTags[tag].name, tagName) == 0) { + return tag; + } + } + + return -1; +} + +int mndSetCreateIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (!mndVgroupInDb(pVgroup, pDb->uid)) { + sdbRelease(pSdb, pVgroup); + continue; + } + + void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen, NULL, 0); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_INDEX; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} +static void *mndBuildDropIdxReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStbObj, SIdxObj *pIdx, int32_t *contLen) { + int32_t len = 0; + int32_t ret = 0; + + SDropIndexReq req = {0}; + memcpy(req.colName, pIdx->colName, sizeof(pIdx->colName)); + memcpy(req.stb, pIdx->stb, sizeof(pIdx->stb)); + req.dbUid = pIdx->dbUid; + req.stbUid = pIdx->stbUid; + + mInfo("idx: %s start to build drop index req", pIdx->name); + + len = tSerializeSDropIdxReq(NULL, 0, &req); + if (ret < 0) { + goto _err; + } + + len += sizeof(SMsgHead); + SMsgHead *pHead = taosMemoryCalloc(1, len); + if (pHead == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + + pHead->contLen = htonl(len); + pHead->vgId = htonl(pVgroup->vgId); + + void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead)); + tSerializeSDropIdxReq(pBuf, len - sizeof(SMsgHead), &req); + *contLen = len; + return pHead; +_err: + + return NULL; +} +int mndSetDropIdxRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (!mndVgroupInDb(pVgroup, pDb->uid)) { + sdbRelease(pSdb, pVgroup); + continue; + } + + int32_t len; + void *pReq = mndBuildDropIdxReq(pMnode, pVgroup, pStb, pIdx, &len); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = len; + action.msgType = TDMT_VND_DROP_INDEX; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + +void mndCleanupIdx(SMnode *pMnode) { + // do nothing + return; +} + +static SSdbRaw *mndIdxActionEncode(SIdxObj *pIdx) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + + // int32_t size = + // sizeof(SSmaObj) + pSma->exprLen + pSma->tagsFilterLen + pSma->sqlLen + pSma->astLen + TSDB_IDX_RESERVE_SIZE; + int32_t size = sizeof(SIdxObj) + TSDB_IDX_RESERVE_SIZE; + + SSdbRaw *pRaw = sdbAllocRaw(SDB_IDX, TSDB_IDX_VER_NUMBER, size); + if (pRaw == NULL) goto _OVER; + + int32_t dataPos = 0; + SDB_SET_BINARY(pRaw, dataPos, pIdx->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pIdx->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pIdx->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pIdx->dstTbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pIdx->colName, TSDB_COL_NAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pIdx->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pIdx->uid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pIdx->stbUid, _OVER) + SDB_SET_INT64(pRaw, dataPos, pIdx->dbUid, _OVER) + + SDB_SET_RESERVE(pRaw, dataPos, TSDB_IDX_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) + + terrno = 0; + +_OVER: + if (terrno != 0) { + mError("idx:%s, failed to encode to raw:%p since %s", pIdx->name, pRaw, terrstr()); + sdbFreeRaw(pRaw); + return NULL; + } + + mTrace("idx:%s, encode to raw:%p, row:%p", pIdx->name, pRaw, pIdx); + return pRaw; +} + +static SSdbRow *mndIdxActionDecode(SSdbRaw *pRaw) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + SSdbRow *pRow = NULL; + SIdxObj *pIdx = NULL; + + int8_t sver = 0; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; + + if (sver != TSDB_IDX_VER_NUMBER) { + terrno = TSDB_CODE_SDB_INVALID_DATA_VER; + goto _OVER; + } + + pRow = sdbAllocRow(sizeof(SIdxObj)); + if (pRow == NULL) goto _OVER; + + pIdx = sdbGetRowObj(pRow); + if (pIdx == NULL) goto _OVER; + + int32_t dataPos = 0; + + SDB_GET_BINARY(pRaw, dataPos, pIdx->name, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pIdx->stb, TSDB_TABLE_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pIdx->db, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pIdx->dstTbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pIdx->colName, TSDB_COL_NAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pIdx->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pIdx->uid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pIdx->stbUid, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pIdx->dbUid, _OVER) + + SDB_GET_RESERVE(pRaw, dataPos, TSDB_IDX_RESERVE_SIZE, _OVER) + + terrno = 0; + +_OVER: + if (terrno != 0) { + taosMemoryFree(pRow); + return NULL; + } + + mTrace("idx:%s, decode from raw:%p, row:%p", pIdx->name, pRaw, pIdx); + return pRow; +} + +static int32_t mndIdxActionInsert(SSdb *pSdb, SIdxObj *pIdx) { + mTrace("idx:%s, perform insert action, row:%p", pIdx->name, pIdx); + return 0; +} + +static int32_t mndIdxActionDelete(SSdb *pSdb, SIdxObj *pIdx) { + mTrace("idx:%s, perform delete action, row:%p", pIdx->name, pIdx); + return 0; +} + +static int32_t mndIdxActionUpdate(SSdb *pSdb, SIdxObj *pOld, SIdxObj *pNew) { + // lock no not + if (strncmp(pOld->colName, pNew->colName, TSDB_COL_NAME_LEN) != 0) { + memcpy(pOld->colName, pNew->colName, sizeof(pNew->colName)); + } + mTrace("idx:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + return 0; +} + +SIdxObj *mndAcquireIdx(SMnode *pMnode, char *idxName) { + SSdb *pSdb = pMnode->pSdb; + SIdxObj *pIdx = sdbAcquire(pSdb, SDB_IDX, idxName); + if (pIdx == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST; + } + return pIdx; +} + +void mndReleaseIdx(SMnode *pMnode, SIdxObj *pIdx) { + SSdb *pSdb = pMnode->pSdb; + sdbRelease(pSdb, pIdx); +} + +SDbObj *mndAcquireDbByIdx(SMnode *pMnode, const char *idxName) { + SName name = {0}; + tNameFromString(&name, idxName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + char db[TSDB_TABLE_FNAME_LEN] = {0}; + tNameGetFullDbName(&name, db); + + return mndAcquireDb(pMnode, db); +} + +int32_t mndSetCreateIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) return -1; + + return 0; +} + +int32_t mndSetCreateIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +int32_t mndSetAlterIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + sdbFreeRaw(pRedoRaw); + return -1; + } + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +int32_t mndSetAlterIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) { + sdbFreeRaw(pCommitRaw); + return -1; + } + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} + +static int32_t mndSetCreateIdxVgroupRedoLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_CREATING) != 0) return -1; + return 0; +} + +static int32_t mndSetCreateIdxVgroupCommitLogs(SMnode *pMnode, STrans *pTrans, SVgObj *pVgroup) { + SSdbRaw *pVgRaw = mndVgroupActionEncode(pVgroup); + if (pVgRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pVgRaw) != 0) return -1; + if (sdbSetRawStatus(pVgRaw, SDB_STATUS_READY) != 0) return -1; + return 0; +} + +// static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pStb) { +// SStbObj stbObj = {0}; +// taosRLockLatch(&pStb->lock); +// memcpy(&stbObj, pStb, sizeof(SStbObj)); +// taosRUnLockLatch(&pStb->lock); +// stbObj.numOfColumns = 0; +// stbObj.pColumns = NULL; +// stbObj.numOfTags = 0; +// stbObj.pTags = NULL; +// stbObj.numOfFuncs = 0; +// stbObj.pFuncs = NULL; +// stbObj.updateTime = taosGetTimestampMs(); +// stbObj.lock = 0; +// stbObj.tagVer++; + +// SSdbRaw *pCommitRaw = mndStbActionEncode(&stbObj); +// if (pCommitRaw == NULL) return -1; +// if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; +// if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; +// +// return 0; +//} + +static void mndDestroyIdxObj(SIdxObj *pIdxObj) { + if (pIdxObj) { + // do nothing + } +} + +static int32_t mndProcessCreateIdxReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SStbObj *pStb = NULL; + SIdxObj *pIdx = NULL; + + SDbObj *pDb = NULL; + SCreateTagIndexReq createReq = {0}; + + if (tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mInfo("idx:%s start to create", createReq.idxName); + // if (mndCheckCreateIdxReq(&createReq) != 0) { + // goto _OVER; + // } + + pDb = mndAcquireDbByStb(pMnode, createReq.stbName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, createReq.stbName); + if (pStb == NULL) { + mError("idx:%s, failed to create since stb:%s not exist", createReq.idxName, createReq.stbName); + goto _OVER; + } + SSIdx idx = {0}; + if (mndAcquireGlobalIdx(pMnode, createReq.idxName, SDB_IDX, &idx) == 0) { + pIdx = idx.pIdx; + } else { + goto _OVER; + } + if (pIdx != NULL) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + goto _OVER; + } + + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + goto _OVER; + } + + code = mndAddIndex(pMnode, pReq, &createReq, pDb, pStb); + if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) { + return terrno; + } else { + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + } + +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("stb:%s, failed to create since %s", createReq.idxName, terrstr()); + } + + mndReleaseStb(pMnode, pStb); + mndReleaseIdx(pMnode, pIdx); + mndReleaseDb(pMnode, pDb); + + return code; +} + +int32_t mndSetDropIdxRedoLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pRedoRaw = mndIdxActionEncode(pIdx); + if (pRedoRaw == NULL) return -1; + if (mndTransAppendRedolog(pTrans, pRedoRaw) != 0) return -1; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPING) != 0) return -1; + + return 0; +} + +int32_t mndSetDropIdxCommitLogs(SMnode *pMnode, STrans *pTrans, SIdxObj *pIdx) { + SSdbRaw *pCommitRaw = mndIdxActionEncode(pIdx); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + + return 0; +} + +static int32_t mndProcessGetTbIdxReq(SRpcMsg *pReq) { + // + return 0; +} + +int32_t mndRetrieveTagIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + SIdxObj *pIdx = NULL; + int32_t cols = 0; + + SDbObj *pDb = NULL; + if (strlen(pShow->db) > 0) { + pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + } + SSmaAndTagIter *pIter = pShow->pIter; + int invalid = -1; + while (numOfRows < rows) { + pIter->pIdxIter = sdbFetch(pSdb, SDB_IDX, pIter->pIdxIter, (void **)&pIdx); + if (pIter->pIdxIter == NULL) break; + + if (NULL != pDb && pIdx->dbUid != pDb->uid) { + sdbRelease(pSdb, pIdx); + continue; + } + + cols = 0; + + SName idxName = {0}; + tNameFromString(&idxName, pIdx->name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + char n1[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + STR_TO_VARSTR(n1, (char *)tNameGetTableName(&idxName)); + + char n2[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n2, (char *)mndGetDbStr(pIdx->db)); + + SName stbName = {0}; + tNameFromString(&stbName, pIdx->stb, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + char n3[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(n3, (char *)tNameGetTableName(&stbName)); + + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)n1, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)n2, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)n3, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + colDataAppend(pColInfo, numOfRows, (const char *)&invalid, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pIdx->createdTime, false); + + char col[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(col, (char *)pIdx->colName); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)col, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + char tag[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tag, (char *)"tag_index"); + colDataAppend(pColInfo, numOfRows, (const char *)tag, false); + + numOfRows++; + sdbRelease(pSdb, pIdx); + } + + mndReleaseDb(pMnode, pDb); + pShow->numOfRows += numOfRows; + return numOfRows; +} + +// static void mndCancelGetNextIdx(SMnode *pMnode, void *pIter) { +// SSdb *pSdb = pMnode->pSdb; +// +// sdbCancelFetch(pSdb, pIter); +//} +static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) { + // impl + return TSDB_CODE_SUCCESS; +} + +static int32_t mndSetUpdateIdxStbCommitLogs(SMnode *pMnode, STrans *pTrans, SStbObj *pOld, SStbObj *pNew, char *tagName, + int on) { + taosRLockLatch(&pOld->lock); + memcpy(pNew, pOld, sizeof(SStbObj)); + taosRUnLockLatch(&pOld->lock); + + pNew->pTags = NULL; + pNew->updateTime = taosGetTimestampMs(); + pNew->lock = 0; + + int32_t tag = mndFindSuperTableTagId(pOld, tagName); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + SSchema *pTag = pNew->pTags + tag; + + if (on == 1) { + if (IS_IDX_ON(pTag)) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } else { + SSCHMEA_SET_IDX_ON(pTag); + } + } else { + if (!IS_IDX_ON(pTag)) { + terrno = TSDB_CODE_MND_SMA_NOT_EXIST; + } else { + SSCHMEA_SET_IDX_OFF(pTag); + pTag->flags = 0; + } + } + pNew->tagVer++; + + SSdbRaw *pCommitRaw = mndStbActionEncode(pNew); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) return -1; + + return 0; +} +int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, SIdxObj *pIdx) { + // impl later + int32_t code = 0; + SStbObj newStb = {0}; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb-index"); + if (pTrans == NULL) goto _OVER; + + // mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); + mndTransSetDbName(pTrans, pDb->name, pStb->name); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + + mndTransSetSerial(pTrans); + + if (mndSetCreateIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + if (mndSetCreateIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + + if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newStb, pIdx->colName, 1) != 0) goto _OVER; + if (mndSetCreateIdxRedoActions(pMnode, pTrans, pDb, &newStb, pIdx) != 0) goto _OVER; + + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + // mndDestoryIdxObj(pIdx); + if (newStb.pTags != NULL) { + taosMemoryFree(newStb.pTags); + taosMemoryFree(newStb.pColumns); + } + mndTransDrop(pTrans); + return code; +} + +static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *req, SDbObj *pDb, SStbObj *pStb) { + SIdxObj idxObj = {0}; + memcpy(idxObj.name, req->idxName, TSDB_TABLE_FNAME_LEN); + memcpy(idxObj.stb, pStb->name, TSDB_TABLE_FNAME_LEN); + memcpy(idxObj.db, pDb->name, TSDB_DB_FNAME_LEN); + memcpy(idxObj.colName, req->colName, TSDB_COL_NAME_LEN); + + idxObj.createdTime = taosGetTimestampMs(); + idxObj.uid = mndGenerateUid(req->idxName, strlen(req->idxName)); + idxObj.stbUid = pStb->uid; + idxObj.dbUid = pStb->dbUid; + + int32_t code = -1; + // SField *pField0 = NULL; + + // SStbObj stbObj = {0}; + // SStbObj *pNew = &stbObj; + + // taosRLockLatch(&pOld->lock); + // memcpy(&stbObj, pOld, sizeof(SStbObj)); + // taosRUnLockLatch(&pOld->lock); + + // stbObj.pColumns = NULL; + // stbObj.pTags = NULL; + // stbObj.updateTime = taosGetTimestampMs(); + // stbObj.lock = 0; + + int32_t tag = mndFindSuperTableTagId(pStb, req->colName); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } else if (tag == 0) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } + + col_id_t colId = pStb->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pStb->name, pStb->uid, colId) != 0) { + return -1; + } + + SSchema *pTag = pStb->pTags + tag; + if (IS_IDX_ON(pTag)) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } + code = mndAddIndexImpl(pMnode, pReq, pDb, pStb, &idxObj); + + return code; +} + +static int32_t mndDropIdx(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SIdxObj *pIdx) { + int32_t code = -1; + SStbObj *pStb = NULL; + STrans *pTrans = NULL; + + SStbObj newObj = {0}; + + pStb = mndAcquireStb(pMnode, pIdx->stb); + if (pStb == NULL) goto _OVER; + + pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB, pReq, "drop-index"); + if (pTrans == NULL) goto _OVER; + + mInfo("trans:%d, used to drop idx:%s", pTrans->id, pIdx->name); + mndTransSetDbName(pTrans, pDb->name, NULL); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + + mndTransSetSerial(pTrans); + if (mndSetDropIdxRedoLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) goto _OVER; + + if (mndSetUpdateIdxStbCommitLogs(pMnode, pTrans, pStb, &newObj, pIdx->colName, 0) != 0) goto _OVER; + if (mndSetDropIdxRedoActions(pMnode, pTrans, pDb, &newObj, pIdx) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + code = 0; + +_OVER: + taosMemoryFree(newObj.pTags); + taosMemoryFree(newObj.pColumns); + + mndTransDrop(pTrans); + mndReleaseStb(pMnode, pStb); + return code; +} +int32_t mndProcessDropTagIdxReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDbObj *pDb = NULL; + SIdxObj *pIdx = NULL; + + SDropTagIndexReq req = {0}; + if (tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + mInfo("idx:%s, start to drop", req.name); + SSIdx idx = {0}; + if (mndAcquireGlobalIdx(pMnode, req.name, SDB_IDX, &idx) == 0) { + pIdx = idx.pIdx; + } else { + goto _OVER; + } + if (pIdx == NULL) { + if (req.igNotExists) { + mInfo("idx:%s, not exist, ignore not exist is set", req.name); + code = 0; + goto _OVER; + } else { + terrno = TSDB_CODE_MND_TAG_INDEX_NOT_EXIST; + goto _OVER; + } + } + + pDb = mndAcquireDbByIdx(pMnode, req.name); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + goto _OVER; + } + + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + goto _OVER; + } + + code = mndDropIdx(pMnode, pReq, pDb, pIdx); + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("idx:%s, failed to drop since %s", req.name, terrstr()); + } + mndReleaseIdx(pMnode, pIdx); + mndReleaseDb(pMnode, pDb); + return code; +} +static int32_t mndProcessGetIdxReq(SRpcMsg *pReq) { + // do nothing + return 0; +} + +int32_t mndDropIdxsByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SIdxObj *pIdx = NULL; + pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx); + if (pIter == NULL) break; + + if (pIdx->stbUid == pStb->uid) { + if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { + sdbRelease(pSdb, pIdx); + sdbCancelFetch(pSdb, pIdx); + return -1; + } + } + + sdbRelease(pSdb, pIdx); + } + + return 0; +} + +int32_t mndGetIdxsByTagName(SMnode *pMnode, SStbObj *pStb, char *tagName, SIdxObj *idx) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SIdxObj *pIdx = NULL; + pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx); + if (pIter == NULL) break; + + if (pIdx->stbUid == pStb->uid && strcasecmp(pIdx->colName, tagName) == 0) { + memcpy((char *)idx, (char *)pIdx, sizeof(SIdxObj)); + sdbRelease(pSdb, pIdx); + return 0; + } + + sdbRelease(pSdb, pIdx); + } + + return -1; +} +int32_t mndDropIdxsByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + + while (1) { + SIdxObj *pIdx = NULL; + pIter = sdbFetch(pSdb, SDB_IDX, pIter, (void **)&pIdx); + if (pIter == NULL) break; + + if (pIdx->dbUid == pDb->uid) { + if (mndSetDropIdxCommitLogs(pMnode, pTrans, pIdx) != 0) { + sdbRelease(pSdb, pIdx); + sdbCancelFetch(pSdb, pIdx); + return -1; + } + } + + sdbRelease(pSdb, pIdx); + } + + return 0; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndIndexCom.c b/source/dnode/mnode/impl/src/mndIndexCom.c new file mode 100644 index 0000000000..2858d7548e --- /dev/null +++ b/source/dnode/mnode/impl/src/mndIndexCom.c @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "mndIndex.h" +#include "mndIndexComm.h" +#include "mndSma.h" + +static void *mndGetIdx(SMnode *pMnode, char *name, int type) { + SSdb *pSdb = pMnode->pSdb; + void *pIdx = sdbAcquire(pSdb, type, name); + if (pIdx == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { + terrno = 0; + } + return pIdx; +} + +int mndAcquireGlobalIdx(SMnode *pMnode, char *name, int type, SSIdx *idx) { + SSmaObj *pSma = mndGetIdx(pMnode, name, SDB_SMA); + SIdxObj *pIdx = mndGetIdx(pMnode, name, SDB_IDX); + + terrno = 0; + + if (pSma == NULL && pIdx == NULL) return 0; + + if (pSma != NULL) { + if (type == SDB_SMA) { + idx->type = SDB_SMA; + idx->pIdx = pSma; + } else { + mndReleaseSma(pMnode, pSma); + terrno = TSDB_CODE_MND_SMA_ALREADY_EXIST; + return -1; + } + } else { + if (type == SDB_IDX) { + idx->type = SDB_IDX; + idx->pIdx = pIdx; + } else { + mndReleaseIdx(pMnode, pIdx); + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } + } + return 0; +} diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index c3f37d86e3..75411f6089 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -21,6 +21,7 @@ #include "mndDnode.h" #include "mndFunc.h" #include "mndGrant.h" +#include "mndIndex.h" #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPerfSchema.h" @@ -425,6 +426,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-vgroup", mndInitVgroup, mndCleanupVgroup) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stb", mndInitStb, mndCleanupStb) != 0) return -1; if (mndAllocStep(pMnode, "mnode-sma", mndInitSma, mndCleanupSma) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-idx", mndInitIdx, mndCleanupIdx) != 0) return -1; if (mndAllocStep(pMnode, "mnode-infos", mndInitInfos, mndCleanupInfos) != 0) return -1; if (mndAllocStep(pMnode, "mnode-perfs", mndInitPerfs, mndCleanupPerfs) != 0) return -1; if (mndAllocStep(pMnode, "mnode-db", mndInitDb, mndCleanupDb) != 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 90baf57c52..c10851cfad 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -17,6 +17,8 @@ #include "mndSma.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndIndex.h" +#include "mndIndexComm.h" #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPrivilege.h" @@ -43,9 +45,13 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq); static int32_t mndProcessGetSmaReq(SRpcMsg *pReq); static int32_t mndProcessGetTbSmaReq(SRpcMsg *pReq); static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); -static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); static void mndDestroySmaObj(SSmaObj *pSmaObj); +// sma and tag index comm func +static int32_t mndProcessDropIdxReq(SRpcMsg *pReq); +static int32_t mndRetrieveIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); +static void mndCancelRetrieveIdx(SMnode *pMnode, void *pIter); + int32_t mndInitSma(SMnode *pMnode) { SSdbTable table = { .sdbType = SDB_SMA, @@ -58,14 +64,14 @@ int32_t mndInitSma(SMnode *pMnode) { }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_SMA, mndProcessCreateSmaReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessDropSmaReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessDropIdxReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_TABLE_INDEX, mndProcessGetTbSmaReq); - mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); - mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveIdx); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelRetrieveIdx); return sdbSetTable(pMnode->pSdb, table); } @@ -712,8 +718,13 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { terrno = TSDB_CODE_MND_STREAM_ALREADY_EXIST; goto _OVER; } + SSIdx idx = {0}; + if (mndAcquireGlobalIdx(pMnode, createReq.name, SDB_SMA, &idx) == 0) { + pSma = idx.pIdx; + } else { + goto _OVER; + } - pSma = mndAcquireSma(pMnode, createReq.name); if (pSma != NULL) { if (createReq.igExists) { mInfo("sma:%s, already exist in sma:%s, ignore exist is set", createReq.name, pSma->name); @@ -959,7 +970,12 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { mInfo("sma:%s, start to drop", dropReq.name); - pSma = mndAcquireSma(pMnode, dropReq.name); + SSIdx idx = {0}; + if (mndAcquireGlobalIdx(pMnode, dropReq.name, SDB_SMA, &idx) == 0) { + pSma = idx.pIdx; + } else { + goto _OVER; + } if (pSma == NULL) { if (dropReq.igNotExists) { mInfo("sma:%s, not exist, ignore not exist is set", dropReq.name); @@ -998,7 +1014,14 @@ static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp int32_t code = -1; SSmaObj *pSma = NULL; - pSma = mndAcquireSma(pMnode, indexReq->indexFName); + SSIdx idx = {0}; + if (0 == mndAcquireGlobalIdx(pMnode, indexReq->indexFName, SDB_SMA, &idx)) { + pSma = idx.pIdx; + } else { + *exist = false; + return 0; + } + if (pSma == NULL) { *exist = false; return 0; @@ -1207,10 +1230,10 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pDb = mndAcquireDb(pMnode, pShow->db); if (pDb == NULL) return 0; } - + SSmaAndTagIter *pIter = pShow->pIter; while (numOfRows < rows) { - pShow->pIter = sdbFetch(pSdb, SDB_SMA, pShow->pIter, (void **)&pSma); - if (pShow->pIter == NULL) break; + pIter->pSmaIter = sdbFetch(pSdb, SDB_SMA, pIter->pSmaIter, (void **)&pSma); + if (pIter->pSmaIter == NULL) break; if (NULL != pDb && pSma->dbUid != pDb->uid) { sdbRelease(pSdb, pSma); @@ -1247,6 +1270,18 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)&pSma->createdTime, false); + char col[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(col, (char *)""); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)col, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + + char tag[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tag, (char *)"sma_index"); + colDataAppend(pColInfo, numOfRows, (const char *)tag, false); + numOfRows++; sdbRelease(pSdb, pSma); } @@ -1256,7 +1291,30 @@ static int32_t mndRetrieveSma(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc return numOfRows; } -static void mndCancelGetNextSma(SMnode *pMnode, void *pIter) { - SSdb *pSdb = pMnode->pSdb; - sdbCancelFetch(pSdb, pIter); +// sma and tag index comm func +static int32_t mndProcessDropIdxReq(SRpcMsg *pReq) { + int ret = mndProcessDropSmaReq(pReq); + if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST) { + terrno = 0; + ret = mndProcessDropTagIdxReq(pReq); + } + return ret; +} + +static int32_t mndRetrieveIdx(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { + if (pShow->pIter == NULL) { + pShow->pIter = taosMemoryCalloc(1, sizeof(SSmaAndTagIter)); + } + int32_t read = mndRetrieveSma(pReq, pShow, pBlock, rows); + if (read < rows) read += mndRetrieveTagIdx(pReq, pShow, pBlock, rows - read); + return read; +} +static void mndCancelRetrieveIdx(SMnode *pMnode, void *pIter) { + SSmaAndTagIter *p = pIter; + if (p != NULL) { + SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, p->pSmaIter); + sdbCancelFetch(pSdb, p->pIdxIter); + } + taosMemoryFree(p); } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 6dedeaf8f3..499136084e 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -17,6 +17,7 @@ #include "mndStb.h" #include "mndDb.h" #include "mndDnode.h" +#include "mndIndex.h" #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPerfSchema.h" @@ -49,7 +50,11 @@ static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); static int32_t mndProcessTableCfgReq(SRpcMsg *pReq); static int32_t mndAlterStbImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void *alterOriData, int32_t alterOriDataLen); -static int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbname, int64_t suid, col_id_t colId); +static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, + void *alterOriData, int32_t alterOriDataLen, const SMAlterStbReq *pAlter); + +static int32_t mndProcessCreateIndexReq(SRpcMsg *pReq); +static int32_t mndProcessDropIndexReq(SRpcMsg *pReq); int32_t mndInitStb(SMnode *pMnode) { SSdbTable table = { @@ -72,7 +77,12 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq); -// mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); + // mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq); + + // mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIndexReq); + // mndSetMsgHandle(pMnode, TDMT_MND_DROP_INDEX, mndProcessDropIndexReq); + // mndSetMsgHandle(pMnode, TDMT_VND_CREATE_INDEX_RSP, mndTransProcessRsp); + // mndSetMsgHandle(pMnode, TDMT_VND_DROP_INDEX_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STB, mndCancelGetNextStb); @@ -426,8 +436,8 @@ static FORCE_INLINE int32_t schemaExColIdCompare(const void *colId, const void * return 0; } -static void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen, - void *alterOriData, int32_t alterOriDataLen) { +void *mndBuildVCreateStbReq(SMnode *pMnode, SVgObj *pVgroup, SStbObj *pStb, int32_t *pContLen, void *alterOriData, + int32_t alterOriDataLen) { SEncoder encoder = {0}; int32_t contLen; SName name = {0}; @@ -802,6 +812,9 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat SSchema *pSchema = &pDst->pTags[i]; pSchema->type = pField->type; pSchema->bytes = pField->bytes; + if (i == 0) { + SSCHMEA_SET_IDX_ON(pSchema); + } memcpy(pSchema->name, pField->name, TSDB_COL_NAME_LEN); pSchema->colId = pDst->nextColId; pDst->nextColId++; @@ -1086,7 +1099,7 @@ static int32_t mndCheckAlterStbReq(SMAlterStbReq *pAlter) { return 0; } -static int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { +int32_t mndAllocStbSchemas(const SStbObj *pOld, SStbObj *pNew) { pNew->pTags = taosMemoryCalloc(pNew->numOfTags, sizeof(SSchema)); pNew->pColumns = taosMemoryCalloc(pNew->numOfColumns, sizeof(SSchema)); if (pNew->pTags == NULL || pNew->pColumns == NULL) { @@ -1348,6 +1361,10 @@ static int32_t mndDropSuperTableTag(SMnode *pMnode, const SStbObj *pOld, SStbObj pNew->numOfTags--; pNew->tagVer++; + + // if (mndDropIndexByTag(pMnode, pOld, tagName) != 0) { + // return -1; + // } mInfo("stb:%s, start to drop tag %s", pNew->name, tagName); return 0; } @@ -1609,6 +1626,43 @@ static int32_t mndSetAlterStbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj return 0; } +static int32_t mndSetAlterStbRedoActions2(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb, + void *alterOriData, int32_t alterOriDataLen) { + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + int32_t contLen; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + if (!mndVgroupInDb(pVgroup, pDb->uid)) { + sdbRelease(pSdb, pVgroup); + continue; + } + + void *pReq = mndBuildVCreateStbReq(pMnode, pVgroup, pStb, &contLen, alterOriData, alterOriDataLen); + if (pReq == NULL) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + STransAction action = {0}; + action.epSet = mndGetVgroupEpset(pMnode, pVgroup); + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_VND_CREATE_INDEX; + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pVgroup); + return -1; + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { taosRLockLatch(&pStb->lock); @@ -1647,6 +1701,7 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa SSchema *pSrcSchema = &pStb->pTags[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; + pSchema->flags = pSrcSchema->flags; pSchema->colId = pSrcSchema->colId; pSchema->bytes = pSrcSchema->bytes; } @@ -1888,6 +1943,74 @@ _OVER: return code; } +static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, + void *alterOriData, int32_t alterOriDataLen, const SMAlterStbReq *pAlter) { + int32_t code = -1; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "alter-stb"); + if (pTrans == NULL) goto _OVER; + + mInfo("trans:%d, used to alter stb:%s", pTrans->id, pStb->name); + mndTransSetDbName(pTrans, pDb->name, pStb->name); + + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + + if (needRsp) { + void *pCont = NULL; + int32_t contLen = 0; + if (mndBuildSMAlterStbRsp(pDb, pStb, &pCont, &contLen) != 0) goto _OVER; + mndTransSetRpcRsp(pTrans, pCont, contLen); + } + + if (pAlter->alterType == TSDB_ALTER_TABLE_DROP_TAG) { + SIdxObj idxObj = {0}; + SField *pField0 = taosArrayGet(pAlter->pFields, 0); + bool exist = false; + if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) { + exist = true; + } + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + + if (exist == true) { + if (mndSetDropIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + if (mndSetDropIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + } + + if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + } else if (pAlter->alterType == TSDB_ALTER_TABLE_UPDATE_TAG_NAME) { + SIdxObj idxObj = {0}; + SField *pField0 = taosArrayGet(pAlter->pFields, 0); + SField *pField1 = taosArrayGet(pAlter->pFields, 1); + const char *oTagName = pField0->name; + const char *nTagName = pField1->name; + bool exist = false; + + if (mndGetIdxsByTagName(pMnode, pStb, pField0->name, &idxObj) == 0) { + exist = true; + } + + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + + if (exist == true) { + memcpy(idxObj.colName, nTagName, strlen(nTagName)); + idxObj.colName[strlen(nTagName)] = 0; + if (mndSetAlterIdxRedoLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + if (mndSetAlterIdxCommitLogs(pMnode, pTrans, &idxObj) != 0) goto _OVER; + } + + if (mndSetAlterStbRedoActions(pMnode, pTrans, pDb, pStb, alterOriData, alterOriDataLen) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + } + code = 0; + +_OVER: + mndTransDrop(pTrans); + return code; +} + static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *pAlter, SDbObj *pDb, SStbObj *pOld) { bool needRsp = true; int32_t code = -1; @@ -1901,7 +2024,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p stbObj.pTags = NULL; stbObj.updateTime = taosGetTimestampMs(); stbObj.lock = 0; - + bool updateTagIndex = false; switch (pAlter->alterType) { case TSDB_ALTER_TABLE_ADD_TAG: code = mndAddSuperTableTag(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); @@ -1909,9 +2032,11 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p case TSDB_ALTER_TABLE_DROP_TAG: pField0 = taosArrayGet(pAlter->pFields, 0); code = mndDropSuperTableTag(pMnode, pOld, &stbObj, pField0->name); + updateTagIndex = true; break; case TSDB_ALTER_TABLE_UPDATE_TAG_NAME: code = mndAlterStbTagName(pMnode, pOld, &stbObj, pAlter->pFields); + updateTagIndex = true; break; case TSDB_ALTER_TABLE_UPDATE_TAG_BYTES: pField0 = taosArrayGet(pAlter->pFields, 0); @@ -1939,7 +2064,11 @@ static int32_t mndAlterStb(SMnode *pMnode, SRpcMsg *pReq, const SMAlterStbReq *p } if (code != 0) goto _OVER; - code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp, pReq->pCont, pReq->contLen); + if (updateTagIndex == false) { + code = mndAlterStbImp(pMnode, pReq, pDb, &stbObj, needRsp, pReq->pCont, pReq->contLen); + } else { + code = mndAlterStbAndUpdateTagIdxImp(pMnode, pReq, pDb, &stbObj, needRsp, pReq->pCont, pReq->contLen, pAlter); + } _OVER: taosMemoryFreeClear(stbObj.pTags); @@ -2072,9 +2201,9 @@ static int32_t mndDropStb(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *p if (mndSetDropStbRedoLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbCommitLogs(pMnode, pTrans, pStb) != 0) goto _OVER; if (mndSetDropStbRedoActions(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndDropIdxsByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndDropSmasByStb(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - code = 0; _OVER: @@ -2182,9 +2311,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, return 0; } -static int32_t mndProcessDropTtltbReq(SRpcMsg *pRsp) { - return 0; -} +static int32_t mndProcessDropTtltbReq(SRpcMsg *pRsp) { return 0; } static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; @@ -2499,30 +2626,30 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t } } -//static int32_t mndProcessRetrieveStbReq(SRpcMsg *pReq) { -// SMnode *pMnode = pReq->info.node; -// SShowMgmt *pMgmt = &pMnode->showMgmt; -// SShowObj *pShow = NULL; -// int32_t rowsToRead = SHOW_STEP_SIZE; -// int32_t rowsRead = 0; +// static int32_t mndProcessRetrieveStbReq(SRpcMsg *pReq) { +// SMnode *pMnode = pReq->info.node; +// SShowMgmt *pMgmt = &pMnode->showMgmt; +// SShowObj *pShow = NULL; +// int32_t rowsToRead = SHOW_STEP_SIZE; +// int32_t rowsRead = 0; // -// SRetrieveTableReq retrieveReq = {0}; -// if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { -// terrno = TSDB_CODE_INVALID_MSG; -// return -1; -// } +// SRetrieveTableReq retrieveReq = {0}; +// if (tDeserializeSRetrieveTableReq(pReq->pCont, pReq->contLen, &retrieveReq) != 0) { +// terrno = TSDB_CODE_INVALID_MSG; +// return -1; +// } // -// SMnode *pMnode = pReq->info.node; -// SSdb *pSdb = pMnode->pSdb; -// int32_t numOfRows = 0; -// SDbObj *pDb = NULL; -// ESdbStatus objStatus = 0; +// SMnode *pMnode = pReq->info.node; +// SSdb *pSdb = pMnode->pSdb; +// int32_t numOfRows = 0; +// SDbObj *pDb = NULL; +// ESdbStatus objStatus = 0; // -// SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); -// if (pUser == NULL) return 0; -// bool sysinfo = pUser->sysInfo; +// SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); +// if (pUser == NULL) return 0; +// bool sysinfo = pUser->sysInfo; // -// // Append the information_schema database into the result. +// // Append the information_schema database into the result. //// if (!pShow->sysDbRsp) { //// SDbObj infoschemaDb = {0}; //// setInformationSchemaDbCfg(pMnode, &infoschemaDb); @@ -2775,7 +2902,6 @@ void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t // return TSDB_CODE_SUCCESS; //} - static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -2886,8 +3012,8 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc return numOfRows; } -static int32_t buildDbColsInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, - const char* dbName, const char* tbName) { +static int32_t buildDbColsInfoBlock(const SSDataBlock *p, const SSysTableMeta *pSysDbTableMeta, size_t size, + const char *dbName, const char *tbName) { char tName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char dName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; @@ -2897,19 +3023,19 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock* p, const SSysTableMeta* p STR_TO_VARSTR(typeName, "SYSTEM_TABLE"); for (int32_t i = 0; i < size; ++i) { - const SSysTableMeta* pm = &pSysDbTableMeta[i]; -// if (pm->sysInfo) { -// continue; -// } - if(tbName[0] && strncmp(tbName, pm->name, TSDB_TABLE_NAME_LEN) != 0){ + const SSysTableMeta *pm = &pSysDbTableMeta[i]; + // if (pm->sysInfo) { + // continue; + // } + if (tbName[0] && strncmp(tbName, pm->name, TSDB_TABLE_NAME_LEN) != 0) { continue; } STR_TO_VARSTR(tName, pm->name); - for(int32_t j = 0; j < pm->colNum; j++){ + for (int32_t j = 0; j < pm->colNum; j++) { // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + SColumnInfoData *pColInfoData = taosArrayGet(p->pDataBlock, 0); colDataAppend(pColInfoData, numOfRows, tName, false); // database name @@ -2931,18 +3057,17 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock* p, const SSysTableMeta* p char colTypeStr[VARSTR_HEADER_SIZE + 32]; int colTypeLen = sprintf(varDataVal(colTypeStr), "%s", tDataTypes[colType].name); if (colType == TSDB_DATA_TYPE_VARCHAR) { - colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", - (int32_t)(pm->schema[j].bytes - VARSTR_HEADER_SIZE)); + colTypeLen += + sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", (int32_t)(pm->schema[j].bytes - VARSTR_HEADER_SIZE)); } else if (colType == TSDB_DATA_TYPE_NCHAR) { - colTypeLen += sprintf( - varDataVal(colTypeStr) + colTypeLen, "(%d)", - (int32_t)((pm->schema[j].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)((pm->schema[j].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - colDataAppend(pColInfoData, numOfRows, (char*)colTypeStr, false); + colDataAppend(pColInfoData, numOfRows, (char *)colTypeStr, false); pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (const char*)&pm->schema[j].bytes, false); + colDataAppend(pColInfoData, numOfRows, (const char *)&pm->schema[j].bytes, false); for (int32_t k = 6; k <= 8; ++k) { pColInfoData = taosArrayGet(p->pDataBlock, k); colDataAppendNULL(pColInfoData, numOfRows); @@ -2955,11 +3080,12 @@ static int32_t buildDbColsInfoBlock(const SSDataBlock* p, const SSysTableMeta* p return numOfRows; } -static int32_t buildSysDbColsInfo(SSDataBlock* p, char* db, char* tb) { +static int32_t buildSysDbColsInfo(SSDataBlock *p, char *db, char *tb) { size_t size = 0; - const SSysTableMeta* pSysDbTableMeta = NULL; + const SSysTableMeta *pSysDbTableMeta = NULL; - if(db[0] && strncmp(db, TSDB_INFORMATION_SCHEMA_DB, TSDB_DB_FNAME_LEN) != 0 && strncmp(db, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_DB_FNAME_LEN) != 0){ + if (db[0] && strncmp(db, TSDB_INFORMATION_SCHEMA_DB, TSDB_DB_FNAME_LEN) != 0 && + strncmp(db, TSDB_PERFORMANCE_SCHEMA_DB, TSDB_DB_FNAME_LEN) != 0) { return p->info.rows; } @@ -2985,7 +3111,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pDb == NULL) return terrno; } - char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + char typeName[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; STR_TO_VARSTR(typeName, "SUPER_TABLE"); while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_STB, pShow->pIter, (void **)&pStb); @@ -2999,7 +3125,7 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB SName name = {0}; char stbName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; mndExtractTbNameFromStbFullName(pStb->name, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN); - if(pShow->filterTb[0] && strncmp(pShow->filterTb, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN) != 0){ + if (pShow->filterTb[0] && strncmp(pShow->filterTb, &stbName[VARSTR_HEADER_SIZE], TSDB_TABLE_NAME_LEN) != 0) { sdbRelease(pSdb, pStb); continue; } @@ -3012,8 +3138,8 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB tNameGetDbName(&name, varDataVal(db)); varDataSetLen(db, strlen(varDataVal(db))); - for(int i = 0; i < pStb->numOfColumns; i++){ - int32_t cols = 0; + for (int i = 0; i < pStb->numOfColumns; i++) { + int32_t cols = 0; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)stbName, false); @@ -3038,16 +3164,15 @@ static int32_t mndRetrieveStbCol(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", (int32_t)(pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE)); } else if (colType == TSDB_DATA_TYPE_NCHAR) { - colTypeLen += sprintf( - varDataVal(colTypeStr) + colTypeLen, "(%d)", - (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + colTypeLen += sprintf(varDataVal(colTypeStr) + colTypeLen, "(%d)", + (int32_t)((pStb->pColumns[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } varDataSetLen(colTypeStr, colTypeLen); - colDataAppend(pColInfo, numOfRows, (char*)colTypeStr, false); + colDataAppend(pColInfo, numOfRows, (char *)colTypeStr, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char*)&pStb->pColumns[i].bytes, false); - while(cols < pShow->numOfColumns) { + colDataAppend(pColInfo, numOfRows, (const char *)&pStb->pColumns[i].bytes, false); + while (cols < pShow->numOfColumns) { pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppendNULL(pColInfo, numOfRows); } @@ -3082,3 +3207,136 @@ const char *mndGetStbStr(const char *src) { if (posStb == NULL) return posDb; return posStb; } + +static int32_t mndCheckIndexReq(SCreateTagIndexReq *pReq) { + // impl + return TSDB_CODE_SUCCESS; +} + +/*int32_t mndAddIndexImpl(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SStbObj *pStb, bool needRsp, void *sql, + int32_t len) { + // impl later + int32_t code = 0; + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_DB_INSIDE, pReq, "create-stb-index"); + if (pTrans == NULL) goto _OVER; + + mInfo("trans:%d, used to add index to stb:%s", pTrans->id, pStb->name); + mndTransSetDbName(pTrans, pDb->name, pStb->name); + if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + + if (mndSetAlterStbRedoLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbCommitLogs(pMnode, pTrans, pDb, pStb) != 0) goto _OVER; + if (mndSetAlterStbRedoActions2(pMnode, pTrans, pDb, pStb, sql, len) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; + + return code; + +_OVER: + mndTransDrop(pTrans); + return code; +} +static int32_t mndAddIndex(SMnode *pMnode, SRpcMsg *pReq, SCreateTagIndexReq *tagIdxReq, SDbObj *pDb, SStbObj *pOld) { + bool needRsp = true; + int32_t code = -1; + SField *pField0 = NULL; + + SStbObj stbObj = {0}; + SStbObj *pNew = &stbObj; + + taosRLockLatch(&pOld->lock); + memcpy(&stbObj, pOld, sizeof(SStbObj)); + taosRUnLockLatch(&pOld->lock); + + stbObj.pColumns = NULL; + stbObj.pTags = NULL; + stbObj.updateTime = taosGetTimestampMs(); + stbObj.lock = 0; + + int32_t tag = mndFindSuperTableTagIndex(pOld, tagIdxReq->colName); + if (tag < 0) { + terrno = TSDB_CODE_MND_TAG_NOT_EXIST; + return -1; + } + col_id_t colId = pOld->pTags[tag].colId; + if (mndCheckColAndTagModifiable(pMnode, pOld->name, pOld->uid, colId) != 0) { + return -1; + } + if (mndAllocStbSchemas(pOld, pNew) != 0) { + return -1; + } + + SSchema *pTag = pNew->pTags + tag; + if (IS_IDX_ON(pTag)) { + terrno = TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST; + return -1; + } else { + pTag->flags |= COL_IDX_ON; + } + pNew->tagVer++; + + code = mndAddIndexImpl(pMnode, pReq, pDb, pNew, needRsp, pReq->pCont, pReq->contLen); + + return code; +} +static int32_t mndProcessCreateIndexReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SCreateTagIndexReq tagIdxReq = {0}; + + if (tDeserializeSCreateTagIdxReq(pReq->pCont, pReq->contLen, &tagIdxReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mInfo("stb:%s, start to alter", tagIdxReq.stbName); + + if (mndCheckIndexReq(&tagIdxReq) != TSDB_CODE_SUCCESS) { + goto _OVER; + } + + pDb = mndAcquireDbByStb(pMnode, tagIdxReq.dbFName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_INVALID_DB; + goto _OVER; + } + + pStb = mndAcquireStb(pMnode, tagIdxReq.stbName); + if (pStb == NULL) { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + goto _OVER; + } + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + goto _OVER; + } + + code = mndAddIndex(pMnode, pReq, &tagIdxReq, pDb, pStb); + if (terrno == TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST || terrno == TSDB_CODE_MND_TAG_NOT_EXIST) { + return terrno; + } else { + if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; + } +_OVER: + if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) { + mError("stb:%s, failed to create index since %s", tagIdxReq.stbName, terrstr()); + } + mndReleaseStb(pMnode, pStb); + mndReleaseDb(pMnode, pDb); + return code; +} +static int32_t mndProcessDropIndexReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; + SDropTagIndexReq dropReq = {0}; + if (tDeserializeSDropTagIdxReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + // + return TSDB_CODE_SUCCESS; +_OVER: + return code; +}*/ diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6bb286cd6f..5776d8a559 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1008,6 +1008,12 @@ static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans) { if (0 == mndBuildSMCreateStbRsp(pMnode, pTrans->dbname, pTrans->stbname, &pCont, &contLen) != 0) { mndTransSetRpcRsp(pTrans, pCont, contLen); } + } else if (pTrans->originRpcType == TDMT_MND_CREATE_INDEX) { + void *pCont = NULL; + int32_t contLen = 0; + if (0 == mndBuildSMCreateStbRsp(pMnode, pTrans->dbname, pTrans->stbname, &pCont, &contLen) != 0) { + mndTransSetRpcRsp(pTrans, pCont, contLen); + } } if (pTrans->rpcRspLen != 0) { diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 5a44e4279f..8d2cec478c 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -147,7 +147,8 @@ typedef enum { SDB_STB = 18, SDB_DB = 19, SDB_FUNC = 20, - SDB_MAX = 21 + SDB_IDX = 21, + SDB_MAX = 22 } ESdbType; typedef struct SSdbRaw { diff --git a/source/dnode/mnode/sdb/src/sdbHash.c b/source/dnode/mnode/sdb/src/sdbHash.c index 505dee3d87..cc3dba2e07 100644 --- a/source/dnode/mnode/sdb/src/sdbHash.c +++ b/source/dnode/mnode/sdb/src/sdbHash.c @@ -60,6 +60,8 @@ const char *sdbTableName(ESdbType type) { return "db"; case SDB_FUNC: return "func"; + case SDB_IDX: + return "idx"; default: return "undefine"; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index d383d26565..acfaccafe2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -59,7 +59,7 @@ void vnodePostClose(SVnode *pVnode); void vnodeSyncCheckTimeout(SVnode *pVnode); void vnodeClose(SVnode *pVnode); int32_t vnodeSyncCommit(SVnode *pVnode); -int32_t vnodeBegin(SVnode* pVnode); +int32_t vnodeBegin(SVnode *pVnode); int32_t vnodeStart(SVnode *pVnode); void vnodeStop(SVnode *pVnode); @@ -137,6 +137,7 @@ typedef struct SMetaFltParam { int16_t type; void *val; bool reverse; + bool equal; int (*filterFunc)(void *a, void *b, int16_t type); } SMetaFltParam; @@ -270,8 +271,8 @@ int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, // int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock2(STqReader *pReader); bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData** pSubmitTbDataRet); -int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData** pSubmitTbDataRet); +int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); +int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); // int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReader *pReader); // int32_t tqRetrieveTaosxBlock(STqReader *pReader, SArray *blocks, SArray *schemas); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index f04e06f270..eda47bf5de 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -142,6 +142,9 @@ int32_t metaGetTbTSchemaEx(SMeta* pMeta, tb_uid_t suid, tb_uid_t uid, in int metaGetTableEntryByName(SMetaReader* pReader, const char* name); int metaAlterCache(SMeta* pMeta, int32_t nPage); +int metaAddIndexToSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); +int metaDropIndexFromSTable(SMeta* pMeta, int64_t version, SDropIndexReq* pReq); + int64_t metaGetTimeSeriesNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid, int lock); void metaCloseCtbCursor(SMCtbCursor* pCtbCur, int lock); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index eb7c71222c..91518f7a0b 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -1108,26 +1108,30 @@ int32_t metaFilterCreateTime(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) } int32_t valid = 0; - while (1) { + int32_t count = 0; + + static const int8_t TRY_ERROR_LIMIT = 1; + do { void *entryKey = NULL; int32_t nEntryKey = -1; valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, NULL, NULL); if (valid < 0) break; SCtimeIdxKey *p = entryKey; + if (count > TRY_ERROR_LIMIT) break; int32_t cmp = (*param->filterFunc)((void *)&p->ctime, (void *)&pCtimeKey->ctime, param->type); - if (cmp == 0) taosArrayPush(pUids, &p->uid); - - if (param->reverse == false) { - if (cmp == -1) break; - } else if (param->reverse) { - if (cmp == 1) break; + if (cmp == 0) + taosArrayPush(pUids, &p->uid); + else { + if (param->equal == true) { + if (count > TRY_ERROR_LIMIT) break; + count++; + } } - valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) break; - } + } while (1); END: if (pCursor->pMeta) metaULock(pCursor->pMeta); @@ -1162,29 +1166,34 @@ int32_t metaFilterTableName(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { if (tdbTbcMoveTo(pCursor->pCur, pName, strlen(pName) + 1, &cmp) < 0) { goto END; } - bool first = true; int32_t valid = 0; - while (1) { + int32_t count = 0; + + int32_t TRY_ERROR_LIMIT = 1; + do { void *pEntryKey = NULL, *pEntryVal = NULL; int32_t nEntryKey = -1, nEntryVal = 0; valid = tdbTbcGet(pCursor->pCur, (const void **)pEntryKey, &nEntryKey, (const void **)&pEntryVal, &nEntryVal); if (valid < 0) break; + if (count > TRY_ERROR_LIMIT) break; + char *pTableKey = (char *)pEntryKey; cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type); if (cmp == 0) { tb_uid_t tuid = *(tb_uid_t *)pEntryVal; taosArrayPush(pUids, &tuid); - } else if (cmp == 1) { - // next } else { - break; + if (param->equal == true) { + if (count > TRY_ERROR_LIMIT) break; + count++; + } } valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) { break; } - } + } while (1); END: if (pCursor->pMeta) metaULock(pCursor->pMeta); @@ -1240,7 +1249,7 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { pCursor->type = param->type; metaRLock(pMeta); - ret = tdbTbcOpen(pMeta->pCtimeIdx, &pCursor->pCur, NULL); + ret = tdbTbcOpen(pMeta->pTagIdx, &pCursor->pCur, NULL); if (ret < 0) { goto END; } @@ -1284,32 +1293,40 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { goto END; } - bool first = true; + int count = 0; int32_t valid = 0; - while (1) { + bool found = false; + + static const int8_t TRY_ERROR_LIMIT = 1; + + /// src: [[suid, cid1, type1]....[suid, cid2, type2]....[suid, cid3, type3]...] + /// target: [suid, cid2, type2] + do { void *entryKey = NULL, *entryVal = NULL; int32_t nEntryKey, nEntryVal; valid = tdbTbcGet(pCursor->pCur, (const void **)&entryKey, &nEntryKey, (const void **)&entryVal, &nEntryVal); if (valid < 0) { - tdbFree(entryVal); break; } + if (count > TRY_ERROR_LIMIT) { + break; + } + STagIdxKey *p = entryKey; if (p == NULL) break; - if (p->type != pCursor->type) { - if (first) { - valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); - if (valid < 0) break; - continue; - } else { + + if (p->type != pCursor->type || p->suid != pCursor->suid || p->cid != pCursor->cid) { + if (found == true) break; + count++; + valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); + if (valid < 0) { break; + } else { + continue; } } - if (p->suid != pKey->suid) { - break; - } - first = false; + int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); if (cmp == 0) { // match @@ -1320,17 +1337,18 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes); } taosArrayPush(pUids, &tuid); - } else if (cmp == 1) { - // not match but should continue to iter + found = true; } else { - // not match and no more result - break; + if (param->equal == true) { + if (count > TRY_ERROR_LIMIT) break; + count++; + } } valid = param->reverse ? tdbTbcMoveToPrev(pCursor->pCur) : tdbTbcMoveToNext(pCursor->pCur); if (valid < 0) { break; } - } + } while (1); END: if (pCursor->pMeta) metaULock(pCursor->pMeta); @@ -1373,19 +1391,19 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) { isLock = true; } -// if (taosHashGet(tags, &p->uid, sizeof(tb_uid_t)) == NULL) { - void *val = NULL; - int32_t len = 0; - if (metaGetTableTagByUid(pMeta, suid, p->uid, &val, &len, false) == 0) { - p->pTagVal = taosMemoryMalloc(len); - memcpy(p->pTagVal, val, len); - tdbFree(val); - } else { - metaError("vgId:%d, failed to table tags, suid: %" PRId64 ", uid: %" PRId64 "", TD_VID(pMeta->pVnode), suid, - p->uid); - } + // if (taosHashGet(tags, &p->uid, sizeof(tb_uid_t)) == NULL) { + void *val = NULL; + int32_t len = 0; + if (metaGetTableTagByUid(pMeta, suid, p->uid, &val, &len, false) == 0) { + p->pTagVal = taosMemoryMalloc(len); + memcpy(p->pTagVal, val, len); + tdbFree(val); + } else { + metaError("vgId:%d, failed to table tags, suid: %" PRId64 ", uid: %" PRId64 "", TD_VID(pMeta->pVnode), suid, + p->uid); } -// } + } + // } if (isLock) metaULock(pMeta); return 0; } @@ -1399,7 +1417,8 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) { SHashObj *pSepecifiedUidMap = NULL; size_t numOfElems = taosArrayGetSize(pUidTagInfo); if (numOfElems > 0) { - pSepecifiedUidMap = taosHashInit(numOfElems / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + pSepecifiedUidMap = + taosHashInit(numOfElems / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < numOfElems; i++) { STUidTagInfo *pTagInfo = taosArrayGet(pUidTagInfo, i); taosHashPut(pSepecifiedUidMap, &pTagInfo->uid, sizeof(uint64_t), &i, sizeof(int32_t)); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 335cfc32d5..94aa464354 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -394,6 +394,301 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { tdbTbcClose(pUidIdxc); return 0; } +int metaAddIndexToSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + + STbDbKey tbDbKey = {0}; + + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; + void *pData = NULL; + int nData = 0; + int64_t oversion; + SDecoder dc = {0}; + int32_t ret; + int32_t c = -2; + tb_uid_t suid = pReq->suid; + + // get super table + if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData) != 0) { + ret = -1; + goto _err; + } + + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + + tDecoderInit(&dc, pData, nData); + ret = metaDecodeEntry(&dc, &oStbEntry); + if (ret < 0) { + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.pSchema == NULL || oStbEntry.stbEntry.schemaTag.pSchema == NULL) { + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.version == pReq->schemaTag.version) { + goto _err; + } + + if (oStbEntry.stbEntry.schemaTag.nCols != pReq->schemaTag.nCols) { + goto _err; + } + + int diffIdx = -1; + for (int i = 0; i < pReq->schemaTag.nCols; i++) { + SSchema *pNew = pReq->schemaTag.pSchema + i; + SSchema *pOld = oStbEntry.stbEntry.schemaTag.pSchema + i; + if (pNew->type != pOld->type || pNew->colId != pOld->colId || pNew->bytes != pOld->bytes || + strncmp(pOld->name, pNew->name, sizeof(pNew->name))) { + goto _err; + } + if (IS_IDX_ON(pNew) && !IS_IDX_ON(pOld)) { + if (diffIdx != -1) goto _err; + diffIdx = i; + } + } + + if (diffIdx == -1 && diffIdx == 0) { + goto _err; + } + + // Get target schema info + SSchemaWrapper *pTagSchema = &pReq->schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + SSchema *pCol = pTagSchema->pSchema + diffIdx; + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey = NULL, *pVal = NULL; + int nKey = 0, nVal = 0; + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (rc < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCtbIdxc); + pCtbIdxc = NULL; + break; + } + if (((SCtbIdxKey *)pKey)->suid != suid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + SCtbIdxKey *table = (SCtbIdxKey *)pKey; + STagVal tagVal = {.cid = pCol->colId}; + tTagGet((const STag *)pVal, &tagVal); + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); + tdbFree(pKey); + tdbFree(pVal); + if (rc < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); + goto _err; + } + + metaWLock(pMeta); + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + metaULock(pMeta); + metaDestroyTagIdxKey(pTagIdxKey); + } + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = pReq->suid; + nStbEntry.name = pReq->name; + nStbEntry.stbEntry.schemaRow = pReq->schemaRow; + nStbEntry.stbEntry.schemaTag = pReq->schemaTag; + + metaWLock(pMeta); + // update table.db + metaSaveToTbDb(pMeta, &nStbEntry); + // update uid index + metaUpdateUidIdx(pMeta, &nStbEntry); + metaULock(pMeta); + + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + tdbTbcClose(pCtbIdxc); + return TSDB_CODE_SUCCESS; +_err: + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + return TSDB_CODE_VND_COL_ALREADY_EXISTS; +} +int metaDropIndexFromSTable(SMeta *pMeta, int64_t version, SDropIndexReq *pReq) { + SMetaEntry oStbEntry = {0}; + SMetaEntry nStbEntry = {0}; + + STbDbKey tbDbKey = {0}; + TBC *pUidIdxc = NULL; + TBC *pTbDbc = NULL; + int ret = 0; + int c = -2; + void *pData = NULL; + int nData = 0; + int64_t oversion; + SDecoder dc = {0}; + + tb_uid_t suid = pReq->stbUid; + + if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pData, &nData) != 0) { + ret = -1; + goto _err; + } + + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pData)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pData, &nData); + tDecoderInit(&dc, pData, nData); + ret = metaDecodeEntry(&dc, &oStbEntry); + if (ret < 0) { + goto _err; + } + + SSchema *pCol = NULL; + int32_t colId = -1; + for (int i = 0; i < oStbEntry.stbEntry.schemaTag.nCols; i++) { + SSchema *schema = oStbEntry.stbEntry.schemaTag.pSchema + i; + if (0 == strncmp(schema->name, pReq->colName, sizeof(pReq->colName))) { + if (i != 0 || IS_IDX_ON(schema)) { + pCol = schema; + } + break; + } + } + + if (pCol == NULL) { + goto _err; + } + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey = NULL, *pVal = NULL; + int nKey = 0, nVal = 0; + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (rc < 0) { + tdbFree(pKey); + tdbFree(pVal); + tdbTbcClose(pCtbIdxc); + pCtbIdxc = NULL; + break; + } + if (((SCtbIdxKey *)pKey)->suid != suid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + SCtbIdxKey *table = (SCtbIdxKey *)pKey; + STagVal tagVal = {.cid = pCol->colId}; + tTagGet((const STag *)pVal, &tagVal); + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + rc = metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, table->uid, &pTagIdxKey, &nTagIdxKey); + tdbFree(pKey); + tdbFree(pVal); + if (rc < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + tdbTbcClose(pCtbIdxc); + goto _err; + } + + metaWLock(pMeta); + tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + metaULock(pMeta); + metaDestroyTagIdxKey(pTagIdxKey); + } + + // clear idx flag + SSCHMEA_SET_IDX_OFF(pCol); + + nStbEntry.version = version; + nStbEntry.type = TSDB_SUPER_TABLE; + nStbEntry.uid = oStbEntry.uid; + nStbEntry.name = oStbEntry.name; + + SSchemaWrapper *row = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaRow); + SSchemaWrapper *tag = tCloneSSchemaWrapper(&oStbEntry.stbEntry.schemaTag); + + nStbEntry.stbEntry.schemaRow = *row; + nStbEntry.stbEntry.schemaTag = *tag; + nStbEntry.stbEntry.rsmaParam = oStbEntry.stbEntry.rsmaParam; + + metaWLock(pMeta); + // update table.db + metaSaveToTbDb(pMeta, &nStbEntry); + // update uid index + metaUpdateUidIdx(pMeta, &nStbEntry); + metaULock(pMeta); + + tDeleteSSchemaWrapper(tag); + tDeleteSSchemaWrapper(row); + + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + tdbTbcClose(pCtbIdxc); + return TSDB_CODE_SUCCESS; +_err: + if (oStbEntry.pBuf) taosMemoryFree(oStbEntry.pBuf); + tDecoderClear(&dc); + tdbFree(pData); + + return -1; +} int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMetaRsp **pMetaRsp) { SMetaEntry me = {0}; @@ -649,31 +944,38 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tDecoderInit(&tdc, tData, tLen); metaDecodeEntry(&tdc, &stbEntry); - const SSchema *pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; - if (pTagColumn->type == TSDB_DATA_TYPE_JSON) { + + SSchema *pTagColumn = NULL; + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; metaDelJsonVarFromIdx(pMeta, &e, pTagColumn); } else { - STagIdxKey *pTagIdxKey = NULL; - int32_t nTagIdxKey; + for (int i = 0; i < pTagSchema->nCols; i++) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[i]; + if (!IS_IDX_ON(pTagColumn)) continue; + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; - const void *pTagData = NULL; - int32_t nTagData = 0; + const void *pTagData = NULL; + int32_t nTagData = 0; - STagVal tagVal = {.cid = pTagColumn->colId}; - tTagGet((const STag *)e.ctbEntry.pTags, &tagVal); - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; + STagVal tagVal = {.cid = pTagColumn->colId}; + tTagGet((const STag *)e.ctbEntry.pTags, &tagVal); + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + + if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, + &pTagIdxKey, &nTagIdxKey) == 0) { + tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); + } + metaDestroyTagIdxKey(pTagIdxKey); } - - if (metaCreateTagIdxKey(e.ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, uid, - &pTagIdxKey, &nTagIdxKey) == 0) { - tdbTbDelete(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, pMeta->txn); - } - metaDestroyTagIdxKey(pTagIdxKey); } tDecoderClear(&tdc); } @@ -931,12 +1233,11 @@ static int metaAlterTableColumn(SMeta *pMeta, int64_t version, SVAlterTbReq *pAl entry.version = version; - metaDeleteNcolIdx(pMeta, &oldEntry); - metaUpdateNcolIdx(pMeta, &entry); - // do actual write metaWLock(pMeta); + metaDeleteNcolIdx(pMeta, &oldEntry); + metaUpdateNcolIdx(pMeta, &entry); // save to table db metaSaveToTbDb(pMeta, &entry); @@ -1117,9 +1418,7 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA // save to uid.idx metaUpdateUidIdx(pMeta, &ctbEntry); - if (iCol == 0) { - metaUpdateTagIdx(pMeta, &ctbEntry); - } + metaUpdateTagIdx(pMeta, &ctbEntry); if (NULL == ctbEntry.ctbEntry.pTags) { metaError("meta/table: null tags, update tag val failed."); @@ -1254,6 +1553,243 @@ static int metaUpdateTableOptions(SMeta *pMeta, int64_t version, SVAlterTbReq *p return 0; } +static int metaAddTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + SMetaEntry stbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t uid, suid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + SDecoder dc = {0}; + + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + return -1; + } + uid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { + ret = -1; + goto _err; + } + suid = ((SUidIdxVal *)pVal)[0].suid; + + STbDbKey tbDbKey = {0}; + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + tDecoderInit(&dc, pVal, nVal); + ret = metaDecodeEntry(&dc, &stbEntry); + if (ret < 0) { + goto _err; + } + + // Get target schema info + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + SSchema *pCol = NULL; + int32_t iCol = 0; + for (;;) { + pCol = NULL; + if (iCol >= pTagSchema->nCols) break; + pCol = &pTagSchema->pSchema[iCol]; + if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; + iCol++; + } + + if (iCol == 0) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + if (pCol == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + + /* + * iterator all pTdDbc by uid and version + */ + TBC *pCtbIdxc = NULL; + tdbTbcOpen(pMeta->pCtbIdx, &pCtbIdxc, NULL); + int rc = tdbTbcMoveTo(pCtbIdxc, &(SCtbIdxKey){.suid = suid, .uid = INT64_MIN}, sizeof(SCtbIdxKey), &c); + if (rc < 0) { + tdbTbcClose(pCtbIdxc); + goto _err; + } + for (;;) { + void *pKey, *pVal; + int nKey, nVal; + rc = tdbTbcNext(pCtbIdxc, &pKey, &nKey, &pVal, &nVal); + if (rc < 0) break; + if (((SCtbIdxKey *)pKey)->suid != uid) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + STagIdxKey *pTagIdxKey = NULL; + int32_t nTagIdxKey; + + const void *pTagData = NULL; + int32_t nTagData = 0; + + STagVal tagVal = {.cid = pCol->colId}; + tTagGet((const STag *)pVal, &tagVal); + if (IS_VAR_DATA_TYPE(pCol->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pCol->type].bytes; + } + if (metaCreateTagIdxKey(suid, pCol->colId, pTagData, nTagData, pCol->type, uid, &pTagIdxKey, &nTagIdxKey) < 0) { + metaDestroyTagIdxKey(pTagIdxKey); + goto _err; + } + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + metaDestroyTagIdxKey(pTagIdxKey); + } + return 0; + +_err: + // tDecoderClear(&dc1); + // tDecoderClear(&dc2); + // if (ctbEntry.pBuf) taosMemoryFree(ctbEntry.pBuf); + // if (stbEntry.pBuf) tdbFree(stbEntry.pBuf); + // tdbTbcClose(pTbDbc); + // tdbTbcClose(pUidIdxc); + return -1; +} + +typedef struct SMetaPair { + void *key; + int nkey; +} SMetaPair; + +static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterTbReq) { + SMetaEntry stbEntry = {0}; + void *pVal = NULL; + int nVal = 0; + int ret; + int c; + tb_uid_t suid; + int64_t oversion; + const void *pData = NULL; + int nData = 0; + SDecoder dc = {0}; + + if (pAlterTbReq->tagName == NULL) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + // search name index + ret = tdbTbGet(pMeta->pNameIdx, pAlterTbReq->tbName, strlen(pAlterTbReq->tbName) + 1, &pVal, &nVal); + if (ret < 0) { + terrno = TSDB_CODE_TDB_TABLE_NOT_EXIST; + return -1; + } + suid = *(tb_uid_t *)pVal; + tdbFree(pVal); + pVal = NULL; + + if (tdbTbGet(pMeta->pUidIdx, &suid, sizeof(tb_uid_t), &pVal, &nVal) == -1) { + ret = -1; + goto _err; + } + + STbDbKey tbDbKey = {0}; + tbDbKey.uid = suid; + tbDbKey.version = ((SUidIdxVal *)pVal)[0].version; + tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &pVal, &nVal); + + tDecoderInit(&dc, pVal, nVal); + ret = metaDecodeEntry(&dc, &stbEntry); + if (ret < 0) { + goto _err; + } + + // Get targe schema info + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + SSchema *pCol = NULL; + int32_t iCol = 0; + for (;;) { + pCol = NULL; + if (iCol >= pTagSchema->nCols) break; + pCol = &pTagSchema->pSchema[iCol]; + if (strcmp(pCol->name, pAlterTbReq->tagName) == 0) break; + iCol++; + } + if (iCol == 0) { + // cannot drop 1th tag index + terrno = -1; + goto _err; + } + if (pCol == NULL) { + terrno = TSDB_CODE_VND_COL_NOT_EXISTS; + goto _err; + } + + if (IS_IDX_ON(pCol)) { + terrno = TSDB_CODE_VND_COL_ALREADY_EXISTS; + goto _err; + } + + SArray *tagIdxList = taosArrayInit(512, sizeof(SMetaPair)); + + TBC *pTagIdxc = NULL; + tdbTbcOpen(pMeta->pTagIdx, &pTagIdxc, NULL); + int rc = + tdbTbcMoveTo(pTagIdxc, &(STagIdxKey){.suid = suid, .cid = INT32_MIN, .type = pCol->type}, sizeof(STagIdxKey), &c); + for (;;) { + void *pKey, *pVal; + int nKey, nVal; + rc = tdbTbcNext(pTagIdxc, &pKey, &nKey, &pVal, &nVal); + STagIdxKey *pIdxKey = (STagIdxKey *)pKey; + if (pIdxKey->suid != suid || pIdxKey->cid != pCol->colId) { + tdbFree(pKey); + tdbFree(pVal); + continue; + } + + SMetaPair pair = {.key = pKey, nKey = nKey}; + taosArrayPush(tagIdxList, &pair); + } + tdbTbcClose(pTagIdxc); + + metaWLock(pMeta); + for (int i = 0; i < taosArrayGetSize(tagIdxList); i++) { + SMetaPair *pair = taosArrayGet(tagIdxList, i); + tdbTbDelete(pMeta->pTagIdx, pair->key, pair->nkey, pMeta->txn); + } + metaULock(pMeta); + + taosArrayDestroy(tagIdxList); + + // set pCol->flags; INDEX_ON + return 0; +_err: + return -1; +} + int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMetaRsp *pMetaRsp) { switch (pReq->action) { case TSDB_ALTER_TABLE_ADD_COLUMN: @@ -1265,6 +1801,10 @@ int metaAlterTable(SMeta *pMeta, int64_t version, SVAlterTbReq *pReq, STableMeta return metaUpdateTableTagVal(pMeta, version, pReq); case TSDB_ALTER_TABLE_UPDATE_OPTIONS: return metaUpdateTableOptions(pMeta, version, pReq); + case TSDB_ALTER_TABLE_ADD_TAG_INDEX: + return metaAddTagIndex(pMeta, version, pReq); + case TSDB_ALTER_TABLE_DROP_TAG_INDEX: + return metaDropTagIndex(pMeta, version, pReq); default: terrno = TSDB_CODE_VND_INVALID_TABLE_ACTION; return -1; @@ -1429,36 +1969,43 @@ static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry) { goto end; } - pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag; + if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) { + pTagColumn = &stbEntry.stbEntry.schemaTag.pSchema[0]; + STagVal tagVal = {.cid = pTagColumn->colId}; - STagVal tagVal = {.cid = pTagColumn->colId}; - if (pTagColumn->type != TSDB_DATA_TYPE_JSON) { - tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); - if (IS_VAR_DATA_TYPE(pTagColumn->type)) { - pTagData = tagVal.pData; - nTagData = (int32_t)tagVal.nData; - } else { - pTagData = &(tagVal.i64); - nTagData = tDataTypes[pTagColumn->type].bytes; - } - } else { - // pTagData = pCtbEntry->ctbEntry.pTags; - // nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; pTagData = pCtbEntry->ctbEntry.pTags; nTagData = ((const STag *)pCtbEntry->ctbEntry.pTags)->len; ret = metaSaveJsonVarToIdx(pMeta, pCtbEntry, pTagColumn); goto end; - } - if (pTagData != NULL) { - if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, - pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { - ret = -1; - goto end; + } else { + for (int i = 0; i < pTagSchema->nCols; i++) { + pTagColumn = &pTagSchema->pSchema[i]; + if (i != 0 && !IS_IDX_ON(pTagColumn)) continue; + + STagVal tagVal = {.cid = pTagColumn->colId}; + tTagGet((const STag *)pCtbEntry->ctbEntry.pTags, &tagVal); + if (IS_VAR_DATA_TYPE(pTagColumn->type)) { + pTagData = tagVal.pData; + nTagData = (int32_t)tagVal.nData; + } else { + pTagData = &(tagVal.i64); + nTagData = tDataTypes[pTagColumn->type].bytes; + } + + if (pTagData != NULL) { + if (metaCreateTagIdxKey(pCtbEntry->ctbEntry.suid, pTagColumn->colId, pTagData, nTagData, pTagColumn->type, + pCtbEntry->uid, &pTagIdxKey, &nTagIdxKey) < 0) { + ret = -1; + goto end; + } + tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); + } + metaDestroyTagIdxKey(pTagIdxKey); } - tdbTbUpsert(pMeta->pTagIdx, pTagIdxKey, nTagIdxKey, NULL, 0, pMeta->txn); } end: - metaDestroyTagIdxKey(pTagIdxKey); + // metaDestroyTagIdxKey(pTagIdxKey); tDecoderClear(&dc); tdbFree(pData); return ret; diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 8d4e70cff9..dac712b4e9 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -96,6 +96,7 @@ int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg, bool direct) { metaRsp.numOfColumns = schema.nCols; metaRsp.precision = pVnode->config.tsdbCfg.precision; metaRsp.sversion = schema.version; + metaRsp.tversion = schemaTag.version; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (metaRsp.numOfColumns + metaRsp.numOfTags)); memcpy(metaRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); @@ -264,26 +265,25 @@ _exit: return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void vnodeFreeSBatchRspMsg(void* p) { +static FORCE_INLINE void vnodeFreeSBatchRspMsg(void *p) { if (NULL == p) { return; } - SBatchRspMsg* pRsp = (SBatchRspMsg*)p; + SBatchRspMsg *pRsp = (SBatchRspMsg *)p; rpcFreeCont(pRsp->msg); } - int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { - int32_t code = 0; - int32_t rspSize = 0; - SBatchReq batchReq = {0}; - SBatchMsg *req = NULL; + int32_t code = 0; + int32_t rspSize = 0; + SBatchReq batchReq = {0}; + SBatchMsg *req = NULL; SBatchRspMsg rsp = {0}; - SBatchRsp batchRsp = {0}; - SRpcMsg reqMsg = *pMsg; - SRpcMsg rspMsg = {0}; - void *pRsp = NULL; + SBatchRsp batchRsp = {0}; + SRpcMsg reqMsg = *pMsg; + SRpcMsg rspMsg = {0}; + void *pRsp = NULL; if (tDeserializeSBatchReq(pMsg->pCont, pMsg->contLen, &batchReq)) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -291,7 +291,7 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) { goto _exit; } - int32_t msgNum = taosArrayGetSize(batchReq.pMsgs); + int32_t msgNum = taosArrayGetSize(batchReq.pMsgs); if (msgNum >= MAX_META_MSG_IN_BATCH) { code = TSDB_CODE_INVALID_MSG; qError("too many msgs %d in vnode batch meta req", msgNum); @@ -405,7 +405,8 @@ void vnodeResetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsert, pLoad->numOfInsertReqs, 64, "nInsert"); VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nInsertSuccess, pLoad->numOfInsertSuccessReqs, 64, "nInsertSuccess"); VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsert, pLoad->numOfBatchInsertReqs, 64, "nBatchInsert"); - VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64, "nBatchInsertSuccess"); + VNODE_GET_LOAD_RESET_VALS(pVnode->statis.nBatchInsertSuccess, pLoad->numOfBatchInsertSuccessReqs, 64, + "nBatchInsertSuccess"); } void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 89e6f4ca7a..9a44fefef3 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -33,6 +33,8 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p static int32_t vnodeProcessTrimReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessBatchDeleteReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp); static int32_t vnodePreprocessCreateTableReq(SVnode *pVnode, SDecoder *pCoder, int64_t ctime, int64_t *pUid) { @@ -419,6 +421,12 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp case TDMT_VND_COMMIT: needCommit = true; break; + case TDMT_VND_CREATE_INDEX: + vnodeProcessCreateIndexReq(pVnode, version, pReq, len, pRsp); + break; + case TDMT_VND_DROP_INDEX: + vnodeProcessDropIndexReq(pVnode, version, pReq, len, pRsp); + break; case TDMT_VND_COMPACT: vnodeProcessCompactVnodeReq(pVnode, version, pReq, len, pRsp); goto _exit; @@ -1567,6 +1575,49 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq _err: return code; } +static int32_t vnodeProcessCreateIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SVCreateStbReq req = {0}; + SDecoder dc = {0}; + + pRsp->msgType = TDMT_VND_CREATE_INDEX_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + tDecoderInit(&dc, pReq, len); + // decode req + if (tDecodeSVCreateStbReq(&dc, &req) < 0) { + terrno = TSDB_CODE_INVALID_MSG; + tDecoderClear(&dc); + return -1; + } + if (metaAddIndexToSTable(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + goto _err; + } + tDecoderClear(&dc); + return 0; +_err: + tDecoderClear(&dc); + return -1; +} +static int32_t vnodeProcessDropIndexReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { + SDropIndexReq req = {0}; + pRsp->msgType = TDMT_VND_CREATE_INDEX_RSP; + pRsp->code = TSDB_CODE_SUCCESS; + pRsp->pCont = NULL; + pRsp->contLen = 0; + + if (tDeserializeSDropIdxReq(pReq, len, &req)) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + if (metaDropIndexFromSTable(pVnode->pMeta, version, &req) < 0) { + pRsp->code = terrno; + return -1; + } + return TSDB_CODE_SUCCESS; +} static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { SCompactVnodeReq req = {0}; @@ -1581,4 +1632,4 @@ static int32_t vnodeProcessCompactVnodeReq(SVnode *pVnode, int64_t version, void vnodeBegin(pVnode); return 0; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e003f7eb0c..d9b0210ed5 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -43,12 +43,20 @@ typedef struct tagFilterAssist { SArray* cInfoList; } tagFilterAssist; -static int32_t removeInvalidUid(SArray* uids, SHashObj* tags); -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* pTagCond); -static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond); -static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, - SNode* pTagIndexCond, STableListInfo* pListInfo, const char* idstr); -static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle); +typedef enum { + FILTER_NO_LOGIC = 1, + FILTER_AND, + FILTER_OTHER, +} FilterCondType; + +static FilterCondType checkTagCond(SNode* cond); +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); +static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* list, SNode* pTagCond); + +static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, + SNode* pTagIndexCond, STableListInfo* pListInfo, const char* idstr); +static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, + void* metaHandle); static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } @@ -89,14 +97,12 @@ size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { return rowSize; } -static void freeEx(void* p) { - taosMemoryFree(*(void**)p); -} +static void freeEx(void* p) { taosMemoryFree(*(void**)p); } void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { taosMemoryFreeClear(pGroupResInfo->pBuf); if (pGroupResInfo->freeItem) { -// taosArrayDestroy(pGroupResInfo->pRows); + // taosArrayDestroy(pGroupResInfo->pRows); taosArrayDestroyEx(pGroupResInfo->pRows, freeEx); pGroupResInfo->freeItem = false; pGroupResInfo->pRows = NULL; @@ -143,7 +149,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in // todo move away and record this during create window while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { - /*void* key = */tSimpleHashGetKey(pData, &keyLen); + /*void* key = */ tSimpleHashGetKey(pData, &keyLen); bufLen += keyLen + sizeof(SResultRowPosition); } @@ -153,7 +159,7 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SSHashObj* pHashmap, in while ((pData = tSimpleHashIterate(pHashmap, pData, &iter)) != NULL) { void* key = tSimpleHashGetKey(pData, &keyLen); - SResKeyPos* p = (SResKeyPos*) (pGroupResInfo->pBuf + offset); + SResKeyPos* p = (SResKeyPos*)(pGroupResInfo->pBuf + offset); p->groupId = *(uint64_t*)key; p->pos = *(SResultRowPosition*)pData; @@ -421,7 +427,7 @@ static void releaseColInfoData(void* pCol) { } void freeItem(void* p) { - STUidTagInfo *pInfo = p; + STUidTagInfo* pInfo = p; if (pInfo->pTagVal != NULL) { taosMemoryFree(pInfo->pTagVal); } @@ -462,7 +468,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis SArray* pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo)); for (int32_t i = 0; i < rows; ++i) { STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); - STUidTagInfo info = {.uid = pkeyInfo->uid}; + STUidTagInfo info = {.uid = pkeyInfo->uid}; taosArrayPush(pUidTagList, &info); } @@ -660,22 +666,32 @@ static int tableUidCompare(const void* a, const void* b) { } static int32_t filterTableInfoCompare(const void* a, const void* b) { - STUidTagInfo* p1 = (STUidTagInfo*) a; - STUidTagInfo* p2 = (STUidTagInfo*) b; + STUidTagInfo* p1 = (STUidTagInfo*)a; + STUidTagInfo* p2 = (STUidTagInfo*)b; if (p1->uid == p2->uid) { return 0; } - return p1->uid < p2->uid? -1:1; + return p1->uid < p2->uid ? -1 : 1; } -static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* cond) { +static FilterCondType checkTagCond(SNode* cond) { + if (nodeType(cond) == QUERY_NODE_OPERATOR) { + return FILTER_NO_LOGIC; + } + if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { + return FILTER_AND; + } + return FILTER_OTHER; +} + +static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond) { int32_t ret = -1; int32_t ntype = nodeType(cond); if (ntype == QUERY_NODE_OPERATOR) { - ret = optimizeTbnameInCondImpl(metaHandle, pRes, cond); + ret = optimizeTbnameInCondImpl(metaHandle, list, cond); } if (ntype != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { @@ -694,50 +710,23 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes SListCell* cell = pList->pHead; for (int i = 0; i < len; i++) { if (cell == NULL) break; - if (optimizeTbnameInCondImpl(metaHandle, pRes, cell->pNode) == 0) { + if (optimizeTbnameInCondImpl(metaHandle, list, cell->pNode) == 0) { hasTbnameCond = true; break; } cell = cell->pNext; } - taosArraySort(pRes, filterTableInfoCompare); - taosArrayRemoveDuplicate(pRes, filterTableInfoCompare, NULL); + taosArraySort(list, filterTableInfoCompare); + taosArrayRemoveDuplicate(list, filterTableInfoCompare, NULL); if (hasTbnameCond) { - ret = metaGetTableTagsByUids(metaHandle, suid, pRes); -// removeInvalidUid(pRes, tags); + ret = metaGetTableTagsByUids(metaHandle, suid, list); } return ret; } -#if 0 -/* - * handle invalid uid - */ -static int32_t removeInvalidUid(SArray* uids, SHashObj* tags) { - int32_t size = taosArrayGetSize(uids); - if (size <= 0) { - return 0; - } - - SArray* validUid = taosArrayInit(size, sizeof(STUidTagInfo)); - - for (int32_t i = 0; i < size; i++) { - STUidTagInfo* p = taosArrayGet(uids, i); - if (taosHashGet(tags, &p->uid, sizeof(int64_t)) != NULL) { - taosArrayPush(validUid, p); - } - } - - taosArraySwap(uids, validUid); - taosArrayDestroy(validUid); - return 0; -} - -#endif - // only return uid that does not contained in pExistedUidList static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { @@ -763,7 +752,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis int32_t numOfTables = taosArrayGetSize(pTbList); SHashObj* uHash = NULL; - size_t numOfExisted = taosArrayGetSize(pExistedUidList); // len > 0 means there already have uids + size_t numOfExisted = taosArrayGetSize(pExistedUidList); // len > 0 means there already have uids if (numOfExisted > 0) { uHash = taosHashInit(numOfExisted / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for (int i = 0; i < numOfExisted; i++) { @@ -789,7 +778,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis return -1; } } else { -// qWarn("failed to get tableIds from by table name: %s, reason: %s", name, tstrerror(terrno)); + // qWarn("failed to get tableIds from by table name: %s, reason: %s", name, tstrerror(terrno)); terrno = 0; } } @@ -818,7 +807,8 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) { taosMemoryFree(payload); } -static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, void* metaHandle) { +static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList, + void* metaHandle) { SSDataBlock* pResBlock = createDataBlock(); if (pResBlock == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -851,7 +841,7 @@ static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTa char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; if (p1->name != NULL) { STR_TO_VARSTR(str, p1->name); - } else { // name is not retrieved during filter + } else { // name is not retrieved during filter metaGetTableNameByUid(metaHandle, p1->uid, str); } @@ -901,7 +891,7 @@ static void doSetQualifiedUid(SArray* pUidList, const SArray* pUidTagList, bool* taosArrayClear(pUidList); int32_t numOfTables = taosArrayGetSize(pUidTagList); - for(int32_t i = 0; i < numOfTables; ++i) { + for (int32_t i = 0; i < numOfTables; ++i) { uint64_t uid = ((STUidTagInfo*)taosArrayGet(pUidTagList, i))->uid; qDebug("tagfilter get uid:%" PRId64 ", res:%d", uid, pResultList[i]); @@ -917,14 +907,15 @@ static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) { return; } - for(int32_t i = 0; i < numOfExisted; ++i) { - uint64_t* uid = taosArrayGet(pUidList, i); + for (int32_t i = 0; i < numOfExisted; ++i) { + uint64_t* uid = taosArrayGet(pUidList, i); STUidTagInfo info = {.uid = *uid}; taosArrayPush(pUidTagList, &info); } } -static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle) { +static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle, + SIdxFltStatus status) { if (pTagCond == NULL) { return TSDB_CODE_SUCCESS; } @@ -957,13 +948,15 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN SArray* pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo)); copyExistedUids(pUidTagList, pUidList); + FilterCondType condType = checkTagCond(pTagCond); + int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->suid, pUidTagList, pTagCond); if (filter == 0) { // tbname in filter is activated, do nothing and return taosArrayClear(pUidList); int32_t numOfRows = taosArrayGetSize(pUidTagList); taosArrayEnsureCap(pUidList, numOfRows); - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { STUidTagInfo* pInfo = taosArrayGet(pUidTagList, i); taosArrayPush(pUidList, &pInfo->uid); } @@ -971,8 +964,11 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN terrno = 0; goto end; } else { - // here we retrieve all tags from the vnode table-meta store - code = metaGetTableTags(metaHandle, pListInfo->suid, pUidTagList); + if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { + code = metaGetTableTagsByUids(metaHandle, pListInfo->suid, pUidList); + } else { + code = metaGetTableTags(metaHandle, pListInfo->suid, pUidList); + } if (code != TSDB_CODE_SUCCESS) { qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->suid); terrno = code; @@ -1009,9 +1005,9 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN goto end; } - doSetQualifiedUid(pUidList, pUidTagList, (bool*) output.columnData->pData); + doSetQualifiedUid(pUidList, pUidTagList, (bool*)output.columnData->pData); - end: +end: taosHashCleanup(ctx.colHash); taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); @@ -1031,12 +1027,13 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, pListInfo->suid = pScanNode->suid; SArray* pUidList = taosArrayInit(8, sizeof(uint64_t)); + SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { if (metaIsTableExist(metaHandle, pScanNode->uid)) { taosArrayPush(pUidList, &pScanNode->uid); } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1048,7 +1045,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, genTagFilterDigest(pTagCond, &context); bool acquired = false; - metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, &acquired); + metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, + &acquired); if (acquired) { qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pUidList)); goto _end; @@ -1061,20 +1059,22 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } else { // failed to find the result in the cache, let try to calculate the results if (pTagIndexCond) { - void* pIndex = tsdbGetIvtIdx(metaHandle); + void* pIndex = tsdbGetIvtIdx(metaHandle); SIndexMetaArg metaArg = { .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = pIndex, .suid = pScanNode->uid}; SIdxFltStatus status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, pUidList, &status); if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake -// qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); + // qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); code = TDB_CODE_SUCCESS; + } else { + qInfo("succ to get filter result, table num: %d", (int)taosArrayGetSize(pUidList)); } } } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle); + code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { goto _end; } @@ -1106,7 +1106,7 @@ _end: return TSDB_CODE_OUT_OF_MEMORY; } - qTrace("tagfilter get uid:%" PRIu64", %s", info.uid, idstr); + qTrace("tagfilter get uid:%" PRIu64 ", %s", info.uid, idstr); } taosArrayDestroy(pUidList); @@ -1837,7 +1837,7 @@ int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalGroupIndex, STableKeyInfo** pKeyInfo, int32_t* size) { int32_t totalGroups = tableListGetOutputGroups(pTableList); - int32_t numOfTables = tableListGetSize(pTableList); + int32_t numOfTables = tableListGetSize(pTableList); if (ordinalGroupIndex < 0 || ordinalGroupIndex >= totalGroups) { return TSDB_CODE_INVALID_PARA; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 1f0b7bb5a4..0af9cee2d6 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -2331,13 +2331,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->base.dataReader = NULL; pTaskInfo->streamInfo.lastStatus.uid = -1; - // code = tsdbReaderOpen(pHandle->vnode, &pTSInfo->base.cond, pList, num, pTSInfo->pResBlock, - // &pTSInfo->base.dataReader, NULL); - // if (code != 0) { - // terrno = code; - // destroyTableScanOperatorInfo(pTableScanOp); - // goto _error; - // } } if (pHandle->initTqReader) { diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index b55f3dbd7c..1b2f135064 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -134,7 +134,7 @@ static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName); static void destroySysScanOperator(void* param); static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code); static SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo); -static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse); +static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse, bool* equal); static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, SMetaReader* smrChildTable, const char* dbname, const char* tableName, @@ -164,7 +164,8 @@ int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); @@ -182,9 +183,9 @@ int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool reverse = false; + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; int ret = func(&vgId, &pVal->datum.i, TSDB_DATA_TYPE_BIGINT); @@ -198,9 +199,9 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool reverse = false, equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; SMetaFltParam param = {.suid = 0, @@ -208,6 +209,7 @@ int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { .type = TSDB_DATA_TYPE_VARCHAR, .val = pVal->datum.p, .reverse = reverse, + .equal = equal, .filterFunc = func}; return -1; } @@ -217,9 +219,9 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool reverse = false, equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; SMetaFltParam param = {.suid = 0, @@ -227,6 +229,7 @@ int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { .type = TSDB_DATA_TYPE_BIGINT, .val = &pVal->datum.i, .reverse = reverse, + .equal = equal, .filterFunc = func}; int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); @@ -239,8 +242,8 @@ int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -251,8 +254,8 @@ int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -263,8 +266,8 @@ int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -275,8 +278,8 @@ int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -287,8 +290,8 @@ int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { SOperatorNode* pOper = (SOperatorNode*)pNode; SValueNode* pVal = (SValueNode*)pOper->pRight; bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + bool equal = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse, &equal); if (func == NULL) return -1; return -1; } @@ -359,10 +362,13 @@ void extractTbnameSlotId(SSysTableScanInfo* pInfo, const SScanPhy static void sysTableScanFillTbName(SOperatorInfo* pOperator, const SSysTableScanInfo* pInfo, const char* name, SSDataBlock* pBlock); -__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { +__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse, bool* equal) { if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { *reverse = true; } + if (ctype == OP_TYPE_EQUAL) { + *equal = true; + } if (ctype == OP_TYPE_LOWER_THAN) return optSysFilterFuncImpl__LowerThan; else if (ctype == OP_TYPE_LOWER_EQUAL) diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 0bb454571a..ffca6ad97f 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -22,8 +22,8 @@ #define MAX_INDEX_KEY_LEN 256 // test only, change later #define MEM_TERM_LIMIT 10 * 10000 -#define MEM_THRESHOLD 8 * 512 * 1024 // 8M -#define MEM_SIGNAL_QUIT MEM_THRESHOLD * 20 +#define MEM_THRESHOLD 128 * 1024 * 1024 // 8M +#define MEM_SIGNAL_QUIT MEM_THRESHOLD * 5 #define MEM_ESTIMATE_RADIO 1.5 static void idxMemRef(MemTable* tbl); diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 5c14424b7e..0d13db2f35 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -383,12 +383,13 @@ static FORCE_INLINE int sifEqual(void *a, void *b, int16_t dtype) { //__compar_fn_t func = idxGetCompar(dtype); return (int)tDoCompare(func, QUERY_TERM, a, b); } -static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reverse) { +static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reverse, bool *equal) { if (type == QUERY_LESS_EQUAL || type == QUERY_LESS_THAN) { *reverse = true; } else { *reverse = false; } + if (type == QUERY_LESS_EQUAL) return sifLessEqual; else if (type == QUERY_LESS_THAN) @@ -398,6 +399,7 @@ static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reve else if (type == QUERY_GREATER_THAN) return sifGreaterThan; else if (type == QUERY_TERM) { + *equal = true; return sifEqual; } return NULL; @@ -474,14 +476,15 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP ret = indexJsonSearch(arg->ivtIdx, mtm, output->result); indexMultiTermQueryDestroy(mtm); } else { - bool reverse; - FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse); + bool reverse = false, equal = false; + FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse, &equal); SMetaFltParam param = {.suid = arg->suid, .cid = left->colId, .type = left->colValType, .val = right->condValue, .reverse = reverse, + .equal = equal, .filterFunc = filterFunc}; char buf[128] = {0}; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 564f2fe8d4..f3bd91f786 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -787,7 +787,7 @@ static void setColumnInfoBySchema(const SRealTableNode* pTable, const SSchema* p pCol->tableType = pTable->pMeta->tableType; pCol->colId = pColSchema->colId; pCol->colType = (tagFlag >= 0 ? COLUMN_TYPE_TAG : COLUMN_TYPE_COLUMN); - pCol->hasIndex = (0 == tagFlag); + pCol->hasIndex = ((0 == tagFlag) || (pColSchema != NULL && IS_IDX_ON(pColSchema))); pCol->node.resType.type = pColSchema->type; pCol->node.resType.bytes = pColSchema->bytes; if (TSDB_DATA_TYPE_TIMESTAMP == pCol->node.resType.type) { @@ -5391,6 +5391,28 @@ static int32_t buildCreateFullTextReq(STranslateContext* pCxt, SCreateIndexStmt* return TSDB_CODE_SUCCESS; } +static int32_t buildCreateTagIndexReq(STranslateContext* pCxt, SCreateIndexStmt* pStmt, SCreateTagIndexReq* pReq) { + SName name; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->indexDbName, pStmt->indexName, &name), pReq->idxName); + memset(&name, 0, sizeof(SName)); + + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name), pReq->stbName); + memset(&name, 0, sizeof(SName)); + + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->dbName, strlen(pStmt->dbName)); + tNameGetFullDbName(&name, pReq->dbFName); + + SNode* pNode = NULL; + ASSERT(LIST_LENGTH(pStmt->pCols) == 1); + FOREACH(pNode, pStmt->pCols) { + SColumnNode* p = (SColumnNode*)pNode; + memcpy(pReq->colName, p->colName, sizeof(p->colName)); + } + + // impl later + return TSDB_CODE_SUCCESS; +} + static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { SMCreateFullTextReq createFTReq = {0}; int32_t code = buildCreateFullTextReq(pCxt, pStmt, &createFTReq); @@ -5401,9 +5423,20 @@ static int32_t translateCreateFullTextIndex(STranslateContext* pCxt, SCreateInde return code; } +static int32_t translateCreateNormalIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { + SCreateTagIndexReq createTagIdxReq = {0}; + int32_t code = buildCreateTagIndexReq(pCxt, pStmt, &createTagIdxReq); + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_CREATE_INDEX, (FSerializeFunc)tSerializeSCreateTagIdxReq, &createTagIdxReq); + } + return code; +} + static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { if (INDEX_TYPE_FULLTEXT == pStmt->indexType) { return translateCreateFullTextIndex(pCxt, pStmt); + } else if (INDEX_TYPE_NORMAL == pStmt->indexType) { + return translateCreateNormalIndex(pCxt, pStmt); } return translateCreateSmaIndex(pCxt, pStmt); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 593ca767b8..d03cfff022 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -209,6 +209,10 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize" TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_COMMENT, "Invalid func comment") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_RETRIEVE, "Invalid func retrieve msg") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_ALREADY_EXIST, "Tag index already exists") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_TAG_INDEX_NOT_EXIST, "Tag index not exists") + + // mnode-db TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists") diff --git a/tests/script/tsim/tagindex/add_index.sim b/tests/script/tsim/tagindex/add_index.sim new file mode 100644 index 0000000000..4e883ef150 --- /dev/null +++ b/tests/script/tsim/tagindex/add_index.sim @@ -0,0 +1,224 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step0 +$dbPrefix = ta_3_db +$tbPrefix = ta_3_tb +$mtPrefix = ta_3_mt +$tbNum = 100 +$rowNum = 20 +$totalNum = 200 + +print =============== create database +sql create database $dbPrefix +sql use $dbPrefix + +print =============== create super table and register tag index +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mtPrefix tags( $i , $i , $i , $i , $i ); + $i = $i + 1 +endw + +sql show tables +if $rows != $tbNum then + return -1 +endi + + + +print =============== insert data into each table +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql insert into $tb values(now, $i ); + $i = $i + 1 +endw + + +sql create index ti2 on $mtPrefix (t2) + +print ==== test name conflict +# +sql_error create index ti3 on $mtPrefix(t2) + +sql_error create index ti2 on $mtPrefix(t2) +sql_error create index ti2 on $mtPrefix(t1) +sql_error create index ti2 on $mtPrefix(t3) +sql_error create index ti2 on $mtPrefix(txx) + + +print ===== test operator equal + +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2= $i ; + if $rows != 1 then + return -1 + endi + $i = $i + 1 +endw + + + +print ===== test operator great equal +# great equal than +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 >= $i ; + $tmp = $tbNum - $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator great +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 > $i ; + $tmp = $tbNum - $i + $tmp = $tmp - 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator lower +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 <= $i ; + $tmp = $i + 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator lower than +# lower equal than +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 < $i ; + $tmp = $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== add table after index created +$interval = $tbNum + $tbNum +$i = $interval +$limit = $interval + $tbNum +while $i < $limit + $tb = $tbPrefix . $i + sql insert into $tb using $mtPrefix tags( $i , $i , $i , $i , $i ) values( now, $i ) + $i = $i + 1 +endw + + + +print ===== add table after index created (opeator great equal than) +# great equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 >= $i ; + $tmp = $limit - $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print ===== add table after index created (opeator great than) +# great than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 > $i ; + $tmp = $limit - $i + $tmp = $tmp - 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== drop table after index created + +# drop table +$dropTbNum = $tbNum / 2 +$i = 0 + +while $i < $tbNum + $tb = $tbPrefix . $i + sql drop table $tb + $i = $i + 1 +endw + +print ===== drop table after index created(opeator lower than) + +# lower equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 < $i ; + $tmp = $i - $interval + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print ===== drop table after index created(opeator lower equal than) + +# lower equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 <= $i ; + + $tmp = $i - $interval + $tmp = $tmp + 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print === show index + +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi + + +print === drop index ti2 +sql drop index ti2 + +print === drop not exist index +sql_error drop index t2 +sql_error drop index t3 + + + +sql_error create index ti0 on $mtPrefix (t1) +sql_error create index t2i on ta_3_tb17 (t2) + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/tagindex/drop_index.sim b/tests/script/tsim/tagindex/drop_index.sim new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/script/tsim/tagindex/perf.sim b/tests/script/tsim/tagindex/perf.sim new file mode 100644 index 0000000000..88d7579ecc --- /dev/null +++ b/tests/script/tsim/tagindex/perf.sim @@ -0,0 +1,225 @@ + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step0 +$dbPrefix = ta_3_db +$tbPrefix = ta_3_tb +$mtPrefix = ta_3_mt +$tbNum = 100000 +$rowNum = 20 +$totalNum = 200 + +print =============== create database +sql create database $dbPrefix +sql use $dbPrefix + +print =============== create super table and register tag index +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mtPrefix tags( $i , $i , $i , $i , $i ); + $i = $i + 1 +endw + +sql show tables +if $rows != $tbNum then + return -1 +endi + + + +print =============== insert data into each table +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql insert into $tb values(now, $i ); + $i = $i + 1 +endw + + +sql create index ti2 on $mtPrefix (t2) + +print ==== test name conflict +# +sql_error create index ti3 on $mtPrefix(t2) + +sql_error create index ti2 on $mtPrefix(t2) +sql_error create index ti2 on $mtPrefix(t1) +sql_error create index ti2 on $mtPrefix(t3) +sql_error create index ti2 on $mtPrefix(txx) + + +print ===== test operator equal + +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2= $i ; + if $rows != 1 then + return -1 + endi + $i = $i + 1 +endw + + + +print ===== test operator great equal +# great equal than +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 >= $i ; + $tmp = $tbNum - $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator great +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 > $i ; + $tmp = $tbNum - $i + $tmp = $tmp - 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator lower +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 <= $i ; + $tmp = $i + 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== test operator lower than +# lower equal than +$i = 0 +while $i < $tbNum + sql select * from $mtPrefix where t2 < $i ; + $tmp = $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== add table after index created +$interval = $tbNum + $tbNum +$i = $interval +$limit = $interval + $tbNum +while $i < $limit + $tb = $tbPrefix . $i + sql insert into $tb using $mtPrefix tags( $i , $i , $i , $i , $i ) values( now, $i ) + $i = $i + 1 +endw + + + +print ===== add table after index created (opeator great equal than) +# great equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 >= $i ; + $tmp = $limit - $i + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print ===== add table after index created (opeator great than) +# great than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 > $i ; + $tmp = $limit - $i + $tmp = $tmp - 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + +print ===== drop table after index created + +# drop table +$dropTbNum = $tbNum / 2 +$i = 0 + +while $i < $tbNum + $tb = $tbPrefix . $i + sql drop table $tb + $i = $i + 1 +endw + +print ===== drop table after index created(opeator lower than) + +# lower equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 < $i ; + $tmp = $i - $interval + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print ===== drop table after index created(opeator lower equal than) + +# lower equal than +$i = $interval +while $i < $limit + sql select * from $mtPrefix where t2 <= $i ; + + $tmp = $i - $interval + $tmp = $tmp + 1 + if $rows != $tmp then + return -1 + endi + $i = $i + 1 +endw + + +print === show index + +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi + + +print === drop index ti2 +sql drop index ti2 + +print === drop not exist index +sql_error drop index t2 +sql_error drop index t3 + + + +sql_error create index ti0 on $mtPrefix (t1) +sql_error create index t2i on ta_3_tb17 (t2) + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/tagindex/sma_and_tag_index.sim b/tests/script/tsim/tagindex/sma_and_tag_index.sim new file mode 100644 index 0000000000..b15d22d439 --- /dev/null +++ b/tests/script/tsim/tagindex/sma_and_tag_index.sim @@ -0,0 +1,162 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step0 +$dbPrefix = ta_3_db +$dbPrefix1 = ta_3_db1 +$tbPrefix = ta_3_tb +$mtPrefix = ta_3_mt +$tbNum = 1 +$rowNum = 20 +$totalNum = 200 + +print =============== create database +sql create database $dbPrefix +sql use $dbPrefix + +print =============== create super table and register tag index +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) + +sql show stables +if $rows != 1 then + return -1 +endi + +print =============== create child table + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mtPrefix tags( $i , $i , $i , $i , $i ); + $i = $i + 1 +endw + +sql show tables +if $rows != $tbNum then + return -1 +endi + + + +print =============== insert data into each table +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql insert into $tb values(now, $i ); + $i = $i + 1 +endw + + +print ==== create sma and tag index, global name conflict +sql create sma index t2i on $mtPrefix function(max(c1)) interval(6m,10s) sliding(6m); +sql_error create index t2i on $mtPrefix (t2) +sql drop index t2i + + +print ==== create tagindex and sma index, global name conflict +sql create index t2i on $mtPrefix (t2) +sql_error create sma index t2i on $mtPrefix function(max(c1)) interval(6m,10s) sliding(6m); + +sql drop index t2i + + +print ===== iter sma and tag index + +sql create index tagt2i on $mtPrefix (t2) +sql create sma index smat2i on $mtPrefix function(max(c1)) interval(6m,10s) sliding(6m); + + +sql select * from information_schema.ins_indexes +if $rows != 2 then + return -1 +endi + +sql drop index smat2i + +$i = 0 +$smaPre = sma3 +while $i < 5 + $sma = $smaPre . $i + $i = $i + 1 + sql create sma index $sma on $mtPrefix function(max(c1)) interval(6m,10s) sliding(6m); +endw + +sql select * from information_schema.ins_indexes +if $rows != 6 then + return -1 +endi + + + +sql drop stable $mtPrefix +sql select * from information_schema.ins_indexes +if $rows != 0 then + return -1 +endi + +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) +sql create index tagt2i on $mtPrefix (t2) +sql drop database $dbPrefix + +sql select * from information_schema.ins_indexes +if $rows != 0 then + return -1 +endi + + +print ===== drop tag and del tag index + +sql create database $dbPrefix +sql use $dbPrefix + +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) +sql create index tagt2i on $mtPrefix (t2) +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi +sql alter table $mtPrefix drop tag t2 + +sql select * from information_schema.ins_indexes +if $rows != 0 then + return -1 +endi + + +print ==== rename tag name, and update index colName +sql create index tagt3i on $mtPrefix (t3) +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi + +sql alter table $mtPrefix rename tag t3 txxx +sql select * from information_schema.ins_indexes +if $rows != 1 then + return -1 +endi + +if $data05 != txxx then + return -1 +endi + + + +print ====== diff db has same index name + +sql create database $dbPrefix1 +sql use $dbPrefix1 + +sql create table if not exists $mtPrefix (ts timestamp, c1 int) tags (t1 int, t2 int, t3 int, t4 int, t5 int) +sql create index tagt3i on $mtPrefix (t3) +sql select * from information_schema.ins_indexes + +if $rows != 2 then + return -1 +endi + + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index c682d79c42..2325b01c35 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -22,7 +22,7 @@ class TDTestCase: tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") tdSql.query("select count(*) from information_schema.ins_columns") - tdSql.checkData(0, 0, 267) + tdSql.checkData(0, 0, 269) tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") tdSql.checkRows(14)