diff --git a/Jenkinsfile2 b/Jenkinsfile2 index c9c9c3a7ca..49e13b7831 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -8,135 +8,202 @@ def skipbuild=0 def win_stop=0 def abortPreviousBuilds() { - def currentJobName = env.JOB_NAME - def currentBuildNumber = env.BUILD_NUMBER.toInteger() - def jobs = Jenkins.instance.getItemByFullName(currentJobName) - def builds = jobs.getBuilds() + def currentJobName = env.JOB_NAME + def currentBuildNumber = env.BUILD_NUMBER.toInteger() + def jobs = Jenkins.instance.getItemByFullName(currentJobName) + def builds = jobs.getBuilds() - for (build in builds) { - if (!build.isBuilding()) { - continue; + for (build in builds) { + if (!build.isBuilding()) { + continue; + } + + if (currentBuildNumber == build.getNumber().toInteger()) { + continue; + } + + build.doKill() //doTerm(),doKill(),doTerm() } - - if (currentBuildNumber == build.getNumber().toInteger()) { - continue; - } - - build.doKill() //doTerm(),doKill(),doTerm() - } } // abort previous build abortPreviousBuilds() def abort_previous(){ - def buildNumber = env.BUILD_NUMBER as int - if (buildNumber > 1) milestone(buildNumber - 1) - milestone(buildNumber) + def buildNumber = env.BUILD_NUMBER as int + if (buildNumber > 1) milestone(buildNumber - 1) + milestone(buildNumber) } def pre_test(){ - sh'hostname' + sh 'hostname' sh ''' - date - sudo rmtaos || echo "taosd has not installed" + date + sudo rmtaos || echo "taosd has not installed" ''' sh ''' - killall -9 taosd ||echo "no taosd running" - killall -9 gdb || echo "no gdb running" - killall -9 python3.8 || echo "no python program running" - cd ${WKC} + killall -9 taosd ||echo "no taosd running" + killall -9 gdb || echo "no gdb running" + killall -9 python3.8 || echo "no python program running" + cd ${WKC} ''' script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WKC} - git checkout master - ''' + if (env.CHANGE_TARGET == 'master') { + sh ''' + cd ${WKC} + git checkout master + ''' + } else if(env.CHANGE_TARGET == '2.0') { + sh ''' + cd ${WKC} + git checkout 2.0 + ''' + } else if(env.CHANGE_TARGET == '3.0') { + sh ''' + cd ${WKC} + git checkout 3.0 + [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. + ''' + } else { + sh ''' + cd ${WKC} + git checkout develop + ''' } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WKC} - git checkout 2.0 - ''' - } - else if(env.CHANGE_TARGET == '3.0'){ - sh ''' - cd ${WKC} - git checkout 3.0 - [ -d contrib/bdb ] && cd contrib/bdb && git clean -fxd && cd ../.. - ''' - } - else{ - sh ''' - cd ${WKC} - git checkout develop - ''' - } } - sh''' - cd ${WKC} - git pull >/dev/null - git fetch origin +refs/pull/${CHANGE_ID}/merge - git checkout -qf FETCH_HEAD - git submodule update --init --recursive + sh ''' + cd ${WKC} + git pull >/dev/null + git fetch origin +refs/pull/${CHANGE_ID}/merge + git checkout -qf FETCH_HEAD + git submodule update --init --recursive ''' - sh''' - cd ${WKC} - export TZ=Asia/Harbin - date - rm -rf debug - mkdir debug - cd debug - cmake .. > /dev/null - make -j4> /dev/null + sh ''' + cd ${WKC} + export TZ=Asia/Harbin + date + rm -rf debug + mkdir debug + cd debug + cmake .. > /dev/null + make -j4> /dev/null ''' - sh''' - cd ${WKPY} - git reset --hard - git pull - pip3 install . + sh ''' + cd ${WKPY} + git reset --hard + git pull + pip3 install . + ''' + return 1 +} +def pre_test_win(){ + bat ''' + hostname + date /t + time /t + taskkill /f /t /im python.exe + taskkill /f /t /im bash.exe + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + rd /s /Q C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine\\debug + exit 0 + ''' + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git reset --hard + git fetch || git fetch + git checkout -f + ''' + script { + if (env.CHANGE_TARGET == 'master') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout master + ''' + } else if(env.CHANGE_TARGET == '2.0') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout 2.0 + ''' + } else if(env.CHANGE_TARGET == '3.0') { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout 3.0 + ''' + } else { + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git checkout develop + ''' + } + } + bat ''' + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + git branch + git pull || git pull + git fetch origin +refs/pull/%CHANGE_ID%/merge + git checkout -qf FETCH_HEAD + ''' +} +def pre_test_build_win() { + bat ''' + echo "building ..." + time /t + cd C:\\workspace\\%EXECUTOR_NUMBER%\\TDengine + mkdir debug + cd debug + call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" x64 + set CL=/MP8 + cmake .. -G "NMake Makefiles JOM" + jom -j 4 || exit 8 + time /t ''' return 1 } pipeline { - agent none - options { skipDefaultCheckout() } - environment{ - WK = '/var/lib/jenkins/workspace/TDinternal' - WKC= '/var/lib/jenkins/workspace/TDengine' - WKPY= '/var/lib/jenkins/workspace/taos-connector-python' - } - stages { - stage('pre_build'){ - agent{label " slave3_0 || slave15 || slave16 || slave17 "} - options { skipDefaultCheckout() } - when { - changeRequest() - } - steps { - script{ - abort_previous() - abortPreviousBuilds() + agent none + options { skipDefaultCheckout() } + environment{ + WK = '/var/lib/jenkins/workspace/TDinternal' + WKC= '/var/lib/jenkins/workspace/TDengine' + WKPY= '/var/lib/jenkins/workspace/taos-connector-python' + } + stages { + stage('run test') { + parallel { + stage('windows test') { + agent {label " windows11 "} + steps { + pre_test_win() + pre_test_build_win() + } + } + stage('linux test') { + agent{label " slave3_0 || slave15 || slave16 || slave17 "} + options { skipDefaultCheckout() } + when { + changeRequest() + } + steps { + timeout(time: 45, unit: 'MINUTES'){ + pre_test() + sh ''' + cd ${WKC}/debug + ctest -VV + ''' + sh ''' + export LD_LIBRARY_PATH=${WKC}/debug/build/lib + cd ${WKC}/tests/system-test + ./fulltest.sh + ''' + sh ''' + cd ${WKC}/tests + ./test-all.sh b1fq + ''' + } + } + } } - timeout(time: 45, unit: 'MINUTES'){ - pre_test() - sh''' - cd ${WKC}/debug - ctest -VV - ''' - sh''' - export LD_LIBRARY_PATH=${WKC}/debug/build/lib - cd ${WKC}/tests/system-test - ./fulltest.sh - ''' - sh''' - cd ${WKC}/tests - ./test-all.sh b1fq - ''' - } - } - } - } - post { + } + } + post { success { emailext ( subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]' SUCCESS", diff --git a/cmake/cmake.define b/cmake/cmake.define index 1655154506..d1d9266bca 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -2,6 +2,8 @@ cmake_minimum_required(VERSION 3.16) set(CMAKE_VERBOSE_MAKEFILE OFF) +SET(BUILD_SHARED_LIBS "OFF") + #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin) diff --git a/example/src/tstream.c b/example/src/tstream.c index 77a002499e..5bd833213d 100644 --- a/example/src/tstream.c +++ b/example/src/tstream.c @@ -81,9 +81,10 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = taos_query( - pConn, - "create stream stream1 trigger window_close as select min(k), max(k), sum(k) as sum_of_k from tu1 interval(10m)"); + pRes = + taos_query(pConn, + "create stream stream1 trigger window_close as select _wstartts, min(k), max(k), sum(k) as sum_of_k " + "from tu1 interval(10m)"); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 3a98102661..19b108dcbf 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -54,6 +54,11 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ } while (0) +#define colDataSetNotNull_f(bm_, r_) \ + do { \ + BMCharPos(bm_, r_) &= ~(1u << (7u - BitPos(r_))); \ + } while (0) + #define colDataIsNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] == -1) #define colDataSetNull_var(pColumnInfoData, row) (pColumnInfoData->varmeta.offset[row] = -1) @@ -63,16 +68,15 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); #define colDataGetNumData(p1_, r_) ((p1_)->pData + ((r_) * (p1_)->info.bytes)) // SColumnInfoData, rowNumber -#define colDataGetData(p1_, r_) \ - ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) \ - : colDataGetNumData(p1_, r_)) +#define colDataGetData(p1_, r_) \ + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_)) static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { - if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON){ - if(colDataIsNull_var(pColumnInfoData, row)){ + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) { + if (colDataIsNull_var(pColumnInfoData, row)) { return true; } - char *data = colDataGetVarData(pColumnInfoData, row); + char* data = colDataGetVarData(pColumnInfoData, row); return (*data == TSDB_DATA_TYPE_NULL); } @@ -80,7 +84,7 @@ static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, return false; } - if (pColumnInfoData->info.type== TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) { + if (pColumnInfoData->info.type == TSDB_DATA_TYPE_VARCHAR || pColumnInfoData->info.type == TSDB_DATA_TYPE_NCHAR) { return colDataIsNull_var(pColumnInfoData, row); } else { if (pColumnInfoData->nullbitmap == NULL) { @@ -132,7 +136,7 @@ static FORCE_INLINE void colDataAppendNULL(SColumnInfoData* pColumnInfoData, uin static FORCE_INLINE void colDataAppendNNULL(SColumnInfoData* pColumnInfoData, uint32_t start, size_t nRows) { if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { for (int32_t i = start; i < start + nRows; ++i) { - colDataSetNull_var(pColumnInfoData,i); // it is a null value of VAR type. + colDataSetNull_var(pColumnInfoData, i); // it is a null value of VAR type. } } else { for (int32_t i = start; i < start + nRows; ++i) { @@ -228,6 +232,8 @@ void blockDebugShowData(const SArray* dataBlocks); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t uid, tb_uid_t suid); +SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema); + static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { return blockDataGetSerialMetaSize(pBlock) + blockDataGetSize(pBlock); } @@ -241,10 +247,10 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32 static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { - int32_t* actualLen = (int32_t*) data; + int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); - uint64_t* groupId = (uint64_t*) data; + uint64_t* groupId = (uint64_t*)data; data += sizeof(uint64_t); int32_t* colSizes = (int32_t*)data; diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 36286d5944..11d0f34c45 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -61,11 +61,11 @@ extern "C" { // ----------------- TSDB COLUMN DEFINITION #pragma pack(push, 1) typedef struct { - col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) - int32_t type : 8; // column type - int32_t bytes : 24; // column bytes (0~16M) - int32_t flags : 8; // flags: 0 no index, 1 SCHEMA_SMA_ON, 2 SCHEMA_IDX_ON - int32_t offset : 24; // point offset in STpRow after the header part. + col_id_t colId; // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1)) + int8_t type; // column type + int8_t flags; // flags: 0 no index, 1 SCHEMA_SMA_ON, 2 SCHEMA_IDX_ON + int32_t bytes; // column bytes (0~16M) + int32_t offset; // point offset in STpRow after the header part. } STColumn; #pragma pack(pop) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index a8053d8854..3889784f2f 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -73,7 +73,8 @@ typedef uint16_t tmsg_t; enum { CONN_TYPE__QUERY = 1, CONN_TYPE__TMQ, CONN_TYPE__MAX }; enum { - HEARTBEAT_KEY_DBINFO = 1, + HEARTBEAT_KEY_USER_AUTHINFO = 1, + HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_MQ_TMP, }; @@ -245,6 +246,8 @@ int32_t tInitSubmitMsgIter(const SSubmitReq* pMsg, SSubmitMsgIter* pIter); int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); int32_t tInitSubmitBlkIter(SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); +// for debug +int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq* pReq, STSchema* pSchema); typedef struct { int32_t index; // index of failed block in submit blocks @@ -424,7 +427,9 @@ int32_t tDeserializeSGetUserAuthReq(void* buf, int32_t bufLen, SGetUserAuthReq* typedef struct { char user[TSDB_USER_LEN]; + int32_t version; int8_t superAuth; + SHashObj* createdDbs; SHashObj* readDbs; SHashObj* writeDbs; } SGetUserAuthRsp; @@ -667,10 +672,20 @@ typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; + int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SGetUserAuthRsp +} SUserAuthBatchRsp; + +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); + + typedef struct { char db[TSDB_DB_FNAME_LEN]; } SCompactDbReq; @@ -2130,7 +2145,7 @@ static FORCE_INLINE int32_t tDecodeSSchemaWrapper(SCoder* pDecoder, SSchemaWrapp if (tDecodeI32v(pDecoder, &pSW->nCols) < 0) return -1; if (tDecodeI32v(pDecoder, &pSW->sver) < 0) return -1; - pSW->pSchema = (SSchema*)tCoderMalloc(pDecoder, sizeof(SSchema) * pSW->nCols); + pSW->pSchema = (SSchema*)taosMemoryCalloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) return -1; for (int32_t i = 0; i < pSW->nCols; i++) { if (tDecodeSSchema(pDecoder, &pSW->pSchema[i]) < 0) return -1; @@ -2576,6 +2591,28 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); } +#define TD_AUTO_CREATE_TABLE 0x1 +typedef struct { + int64_t suid; + int64_t uid; + int32_t sver; + uint64_t nData; + const void* pData; + SVCreateTbReq cTbReq; +} SVSubmitBlk; + +typedef struct { + int32_t flags; + int32_t nBlocks; + union { + SArray* pArray; + SVSubmitBlk* pBlocks; + }; +} SVSubmitReq; + +int32_t tEncodeSVSubmitReq(SCoder* pCoder, const SVSubmitReq* pReq); +int32_t tDecodeSVSubmitReq(SCoder* pCoder, SVSubmitReq* pReq); + #pragma pack(pop) #ifdef __cplusplus diff --git a/include/common/trow.h b/include/common/trow.h index 02f84b372e..f87ea1f009 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -141,7 +141,7 @@ typedef struct { /// row total length uint32_t len; /// row version - uint64_t ver; + // uint64_t ver; /// the inline data, maybe a tuple or a k-v tuple char data[]; } STSRow; @@ -176,7 +176,7 @@ typedef struct { #define TD_ROW_DATA(r) ((r)->data) #define TD_ROW_LEN(r) ((r)->len) #define TD_ROW_KEY(r) ((r)->ts) -#define TD_ROW_VER(r) ((r)->ver) +// #define TD_ROW_VER(r) ((r)->ver) #define TD_ROW_KEY_ADDR(r) (r) // N.B. If without STSchema, getExtendedRowSize() is used to get the rowMaxBytes and @@ -622,7 +622,6 @@ static FORCE_INLINE int32_t tdSRowSetTpInfo(SRowBuilder *pBuilder, int32_t nCols return TSDB_CODE_SUCCESS; } - /** * @brief To judge row type: STpRow/SKvRow * @@ -758,7 +757,6 @@ static int32_t tdSRowGetBuf(SRowBuilder *pBuilder, void *pBuf) { return TSDB_CODE_SUCCESS; } - /** * @brief 由调用方管理存储空间的分配及释放,一次输入多个参数 * @@ -1250,16 +1248,16 @@ static FORCE_INLINE int32_t tdGetColDataOfRow(SCellVal *pVal, SDataCol *pCol, in } /** - * @brief - * - * @param pRow - * @param colId - * @param colType - * @param flen - * @param offset + * @brief + * + * @param pRow + * @param colId + * @param colType + * @param flen + * @param offset * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE + * @param pVal + * @return FORCE_INLINE */ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t colType, int32_t flen, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { @@ -1273,14 +1271,14 @@ static FORCE_INLINE bool tdSTpRowGetVal(STSRow *pRow, col_id_t colId, col_type_t } /** - * @brief - * - * @param pRow - * @param colId - * @param offset + * @brief + * + * @param pRow + * @param colId + * @param offset * @param colIdx start from 0 - * @param pVal - * @return FORCE_INLINE + * @param pVal + * @return FORCE_INLINE */ static FORCE_INLINE bool tdSKvRowGetVal(STSRow *pRow, col_id_t colId, uint32_t offset, col_id_t colIdx, SCellVal *pVal) { @@ -1397,14 +1395,14 @@ static void tdSCellValPrint(SCellVal *pVal, int8_t colType) { } } -static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char* tag) { +static void tdSRowPrint(STSRow *row, STSchema *pSchema, const char *tag) { STSRowIter iter = {0}; tdSTSRowIterInit(&iter, pSchema); tdSTSRowIterReset(&iter, row); printf("%s >>>", tag); for (int i = 0; i < pSchema->numOfCols; ++i) { STColumn *stCol = pSchema->columns + i; - SCellVal sVal = { 255, NULL}; + SCellVal sVal = {255, NULL}; if (!tdSTSRowIterNext(&iter, stCol->colId, stCol->type, &sVal)) { break; } diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 30d1bd0a51..04a24c4f32 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -40,6 +40,11 @@ enum { CTG_DBG_STB_RENT_NUM, }; +typedef enum { + AUTH_TYPE_READ = 1, + AUTH_TYPE_WRITE, + AUTH_TYPE_OTHER, +} AUTH_TYPE; typedef struct SCatalogReq { SArray *pTableName; // element is SNAME @@ -57,6 +62,7 @@ typedef struct SMetaData { typedef struct SCatalogCfg { uint32_t maxTblCacheNum; uint32_t maxDBCacheNum; + uint32_t maxUserCacheNum; uint32_t dbRentSec; uint32_t stbRentSec; } SCatalogCfg; @@ -77,6 +83,11 @@ typedef struct SDbVgVersion { int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT } SDbVgVersion; +typedef struct SUserAuthVersion { + char user[TSDB_USER_LEN]; + int32_t version; +} SUserAuthVersion; + typedef SDbCfgRsp SDbCfgInfo; typedef SUserIndexRsp SIndexInfo; @@ -219,12 +230,18 @@ int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableMetaVersion **stable int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion **dbs, uint32_t *num); +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num); + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg); int32_t catalogGetIndexInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* indexName, SIndexInfo* pInfo); int32_t catalogGetUdfInfo(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* funcName, SFuncInfo** pInfo); +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass); + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); + /** * Destroy catalog and relase all resources diff --git a/include/libs/function/tudf.h b/include/libs/function/tudf.h index e49f5cac45..0000972f5e 100644 --- a/include/libs/function/tudf.h +++ b/include/libs/function/tudf.h @@ -22,6 +22,7 @@ #include "tmsg.h" #include "tcommon.h" #include "function.h" +#include "tdatablock.h" #ifdef __cplusplus extern "C" { @@ -54,14 +55,14 @@ int32_t setupUdf(char udfName[], UdfcFuncHandle *handle); typedef struct SUdfColumnMeta { int16_t type; - int32_t bytes; // <0 var length, others fixed length bytes + int32_t bytes; uint8_t precision; uint8_t scale; } SUdfColumnMeta; typedef struct SUdfColumnData { int32_t numOfRows; - bool varLengthColumn; + int32_t rowsAlloc; union { struct { int32_t nullBitmapLen; @@ -72,9 +73,10 @@ typedef struct SUdfColumnData { struct { int32_t varOffsetsLen; - char *varOffsets; + int32_t *varOffsets; int32_t payloadLen; char *payload; + int32_t payloadAllocLen; } varLenCol; }; } SUdfColumnData; @@ -131,10 +133,114 @@ typedef int32_t (*TUdfSetupFunc)(); typedef int32_t (*TUdfTeardownFunc)(); //TODO: add API to check function arguments type, number etc. -//TODO: another way to manage memory is provide api for UDF to add data to SUdfColumnData and UDF framework will allocate memory. -// then UDF framework will free the memory -//typedef int32_t addFixedLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t colBytes, char* data); -//typedef int32_t addVariableLengthColumnData(SColumnData *columnData, int rowIndex, bool isNull, int32_t dataLen, char * data); + +#define UDF_MEMORY_EXP_GROWTH 1.5 + +static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn* pColumn, int32_t newCapacity) { + SUdfColumnMeta *meta = &pColumn->colMeta; + SUdfColumnData *data = &pColumn->colData; + + if (newCapacity== 0 || newCapacity <= data->rowsAlloc) { + return TSDB_CODE_SUCCESS; + } + + int allocCapacity = TMAX(data->rowsAlloc, 8); + while (allocCapacity < newCapacity) { + allocCapacity *= UDF_MEMORY_EXP_GROWTH; + } + + if (IS_VAR_DATA_TYPE(meta->type)) { + char* tmp = taosMemoryRealloc(data->varLenCol.varOffsets, sizeof(int32_t) * allocCapacity); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + data->varLenCol.varOffsets = (int32_t*)tmp; + data->varLenCol.varOffsetsLen = sizeof(int32_t) * allocCapacity; + // for payload, add data in udfColDataAppend + } else { + char* tmp = taosMemoryRealloc(data->fixLenCol.nullBitmap, BitmapLen(allocCapacity)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + data->fixLenCol.nullBitmap = tmp; + data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); + if (meta->type == TSDB_DATA_TYPE_NULL) { + return TSDB_CODE_SUCCESS; + } + + tmp = taosMemoryRealloc(data->fixLenCol.data, allocCapacity* meta->bytes); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + data->fixLenCol.data = tmp; + data->fixLenCol.dataLen = allocCapacity* meta->bytes; + } + + data->rowsAlloc = allocCapacity; + + return TSDB_CODE_SUCCESS; +} + +static FORCE_INLINE int32_t udfColSetRow(SUdfColumn* pColumn, uint32_t currentRow, const char* pData, bool isNull) { + SUdfColumnMeta *meta = &pColumn->colMeta; + SUdfColumnData *data = &pColumn->colData; + udfColEnsureCapacity(pColumn, currentRow+1); + bool isVarCol = IS_VAR_DATA_TYPE(meta->type); + if (isNull) { + if (isVarCol) { + data->varLenCol.varOffsets[currentRow] = -1; + } else { + colDataSetNull_f(data->fixLenCol.nullBitmap, currentRow); + } + } else { + if (!isVarCol) { + colDataSetNotNull_f(data->fixLenCol.nullBitmap, currentRow); + memcpy(data->fixLenCol.data + meta->bytes * currentRow, pData, meta->bytes); + } else { + int32_t dataLen = varDataTLen(pData); + if (meta->type == TSDB_DATA_TYPE_JSON) { + if (*pData == TSDB_DATA_TYPE_NULL) { + dataLen = 0; + } else if (*pData == TSDB_DATA_TYPE_NCHAR) { + dataLen = varDataTLen(pData + CHAR_BYTES); + } else if (*pData == TSDB_DATA_TYPE_BIGINT || *pData == TSDB_DATA_TYPE_DOUBLE) { + dataLen = LONG_BYTES; + } else if (*pData == TSDB_DATA_TYPE_BOOL) { + dataLen = CHAR_BYTES; + } + dataLen += CHAR_BYTES; + } + + if (data->varLenCol.payloadAllocLen < data->varLenCol.payloadLen + dataLen) { + uint32_t newSize = data->varLenCol.payloadAllocLen; + if (newSize <= 1) { + newSize = 8; + } + + while (newSize < data->varLenCol.payloadLen + dataLen) { + newSize = newSize * UDF_MEMORY_EXP_GROWTH; + } + + char *buf = taosMemoryRealloc(data->varLenCol.payload, newSize); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + data->varLenCol.payload = buf; + data->varLenCol.payloadAllocLen = newSize; + } + + uint32_t len = data->varLenCol.payloadLen; + data->varLenCol.varOffsets[currentRow] = len; + + memcpy(data->varLenCol.payload + len, pData, dataLen); + data->varLenCol.payloadLen += dataLen; + } + } + data->numOfRows = TMAX(currentRow + 1, data->numOfRows); + return 0; +} typedef int32_t (*TUdfFreeUdfColumnFunc)(SUdfColumn* column); typedef int32_t (*TUdfScalarProcFunc)(SUdfDataBlock* block, SUdfColumn *resultCol); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2e5574511b..42180833df 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -70,8 +70,10 @@ typedef struct { } STaskDispatcherShuffle; typedef struct { - int8_t reserved; + int8_t reserved; + SSchemaWrapper* pSchemaWrapper; // not applicable to encoder and decoder + STSchema* pTSchema; SHashObj* pHash; // groupId to tbuid } STaskSinkTb; diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 0e7d486eab..a7d1522d12 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -68,6 +68,7 @@ typedef int (*RpcAfp)(void *parent, char *tableId, char *spi, char *encrypt, cha typedef bool (*RpcRfp)(int32_t code); typedef struct SRpcInit { + char localFqdn[TSDB_FQDN_LEN]; uint16_t localPort; // local port char * label; // for debug purpose int numOfThreads; // number of threads to handle connections diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 62c3771669..213a6930ee 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -161,6 +161,7 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); +bool taosValidIpAndPort(uint32_t ip, uint16_t port); TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port); int32_t taosKeepTcpAlive(TdSocketPtr pSocket); TdSocketPtr taosAcceptTcpConnectSocket(TdSocketServerPtr pServerSocket, struct sockaddr *destAddr, int *addrLen); diff --git a/include/os/osString.h b/include/os/osString.h index a4100652c3..026cb33ad9 100644 --- a/include/os/osString.h +++ b/include/os/osString.h @@ -59,6 +59,8 @@ bool taosMbsToUcs4(const char *mbs, size_t mbs_len, TdUcs4 *ucs4, int32_t ucs int32_t tasoUcs4Compare(TdUcs4 *f1_ucs4, TdUcs4 *f2_ucs4, int32_t bytes); TdUcs4* tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4); bool taosValidateEncodec(const char *encodec); +int32_t taosHexEncode(const char *src, char *dst, int32_t len); +int32_t taosHexDecode(const char *src, char *dst, int32_t len); int32_t taosWcharWidth(TdWchar wchar); int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index b11a49fa1a..fc39e80c1e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -28,6 +28,27 @@ static int32_t hbMqHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq static int32_t hbMqHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { return 0; } +static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { + int32_t code = 0; + + SUserAuthBatchRsp batchRsp = {0}; + if (tDeserializeSUserAuthBatchRsp(value, valueLen, &batchRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfBatchs = taosArrayGetSize(batchRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserAuthRsp *rsp = taosArrayGet(batchRsp.pArray, i); + tscDebug("hb user auth rsp, user:%s, version:%d", rsp->user, rsp->version); + + catalogUpdateUserAuthInfo(pCatalog, rsp); + } + + tFreeSUserAuthBatchRsp(&batchRsp); + return TSDB_CODE_SUCCESS; +} + static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { int32_t code = 0; @@ -148,6 +169,24 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { for (int32_t i = 0; i < kvNum; ++i) { SKv *kv = taosArrayGet(pRsp->info, i); switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb user auth info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + int64_t *clusterId = (int64_t *)info->param; + struct SCatalog *pCatalog = NULL; + + int32_t code = catalogGetHandle(*clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + break; + } + + hbProcessUserAuthInfoRsp(kv->value, kv->valueLen, pCatalog); + break; + } case HEARTBEAT_KEY_DBINFO: { if (kv->valueLen <= 0 || NULL == kv->value) { tscError("invalid hb db info, len:%d, value:%p", kv->valueLen, kv->value); @@ -327,6 +366,39 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } +int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { + SUserAuthVersion *users = NULL; + uint32_t userNum = 0; + int32_t code = 0; + + code = catalogGetExpiredUsers(pCatalog, &users, &userNum); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + + if (userNum <= 0) { + return TSDB_CODE_SUCCESS; + } + + for (int32_t i = 0; i < userNum; ++i) { + SUserAuthVersion *user = &users[i]; + user->version = htonl(user->version); + } + + SKv kv = { + .key = HEARTBEAT_KEY_USER_AUTHINFO, + .valueLen = sizeof(SUserAuthVersion) * userNum, + .value = users, + }; + + tscDebug("hb got %d expired users, valueLen:%d", userNum, kv.valueLen); + + taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); + + return TSDB_CODE_SUCCESS; +} + + int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SDbVgVersion *dbs = NULL; uint32_t dbNum = 0; @@ -407,6 +479,11 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); + code = hbGetExpiredUserInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + code = hbGetExpiredDBInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 2874220121..d7ffa282df 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -587,15 +587,34 @@ TEST(testCase, projection_query_tables) { } taos_free_result(pRes); + pRes = taos_query(pConn, "create stable st2 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + pRes = taos_query(pConn, "create table tu using st1 tags(1)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } taos_free_result(pRes); - for(int32_t i = 0; i < 10000; ++i) { - char sql[512] = {0}; - sprintf(sql, "insert into tu values(now+%da, %d)", i, i); + pRes = taos_query(pConn, "create table tu2 using st2 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + for(int32_t i = 0; i < 10000000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); TAOS_RES* p = taos_query(pConn, sql); if (taos_errno(p) != 0) { printf("failed to insert data, reason:%s\n", taos_errstr(p)); @@ -604,24 +623,44 @@ TEST(testCase, projection_query_tables) { taos_free_result(p); } - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); + printf("start to insert next table\n"); + + for(int32_t i = 0; i < 10000000; i += 20) { + char sql[1024] = {0}; + sprintf(sql, + "insert into tu2 values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" + "(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)", + i, i, i + 1, i + 1, i + 2, i + 2, i + 3, i + 3, i + 4, i + 4, i + 5, i + 5, i + 6, i + 6, i + 7, i + 7, + i + 8, i + 8, i + 9, i + 9, i + 10, i + 10, i + 11, i + 11, i + 12, i + 12, i + 13, i + 13, i + 14, i + 14, + i + 15, i + 15, i + 16, i + 16, i + 17, i + 17, i + 18, i + 18, i + 19, i + 19); + TAOS_RES* p = taos_query(pConn, sql); + if (taos_errno(p) != 0) { + printf("failed to insert data, reason:%s\n", taos_errstr(p)); + } + + taos_free_result(p); } - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); +// pRes = taos_query(pConn, "select * from tu"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } - taos_free_result(pRes); +// taos_free_result(pRes); taos_close(pConn); } @@ -659,7 +698,7 @@ TEST(testCase, agg_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use db"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); if (taos_errno(pRes) != 0) { printf("failed to use db, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -695,5 +734,6 @@ TEST(testCase, agg_query_tables) { taos_free_result(pRes); taos_close(pConn); } +#endif #pragma GCC diagnostic pop diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 4946c9690b..a4b36dcb67 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1594,3 +1594,68 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks return TSDB_CODE_SUCCESS; } + +SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema) { + SSubmitReq* ret = NULL; + + // cal size + int32_t cap = sizeof(SSubmitReq); + int32_t sz = taosArrayGetSize(pBlocks); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + int32_t rows = pDataBlock->info.rows; + // TODO min + int32_t rowSize = pDataBlock->info.rowSize; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + cap += sizeof(SSubmitBlk) + rows * maxLen; + } + + // assign data + ret = taosMemoryCalloc(1, cap); + ret->version = htonl(1); + ret->length = htonl(cap - sizeof(SSubmitReq)); + ret->numOfBlocks = htonl(sz); + + void* submitBlk = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + + SSubmitBlk* blkHead = submitBlk; + blkHead->numOfRows = htons(pDataBlock->info.rows); + blkHead->schemaLen = 0; + blkHead->sversion = htonl(pTSchema->version); + // TODO + blkHead->suid = 0; + blkHead->uid = htobe64(pDataBlock->info.uid); + + int32_t rows = pDataBlock->info.rows; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + /*blkHead->dataLen = htonl(rows * maxLen);*/ + blkHead->dataLen = 0; + + void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); + STSRow* rowData = blockData; + + for (int32_t j = 0; j < pDataBlock->info.rows; j++) { + SRowBuilder rb = {0}; + tdSRowInit(&rb, pTSchema->version); + tdSRowSetTpInfo(&rb, pTSchema->numOfCols, pTSchema->flen); + tdSRowResetBuf(&rb, rowData); + + for (int32_t k = 0; k < pTSchema->numOfCols; k++) { + const STColumn* pColumn = &pTSchema->columns[k]; + SColumnInfoData* pColData = taosArrayGet(pDataBlock->pDataBlock, k); + void* data = colDataGetData(pColData, j); + tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NORM, data, true, pColumn->offset, k); + } + int32_t rowLen = TD_ROW_LEN(rowData); + rowData = POINTER_SHIFT(rowData, rowLen); + blkHead->dataLen += rowLen; + } + int32_t len = blkHead->dataLen; + blkHead->dataLen = htonl(len); + blkHead = POINTER_SHIFT(blkHead, len); + } + + return ret; +} diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index df598046e1..e659b9ec04 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "tdataformat.h" #include "tcoding.h" +#include "tdatablock.h" #include "tlog.h" static void dataColSetNEleNull(SDataCol *pCol, int nEle); @@ -128,6 +129,50 @@ void *tdDecodeSchema(void *buf, STSchema **pRSchema) { return buf; } +#if 0 +int32_t tEncodeSTColumn(SCoder *pEncoder, const STColumn *pCol) { + if (tEncodeI16(pEncoder, pCol->colId) < 0) return -1; + if (tEncodeI8(pEncoder, pCol->type) < 0) return -1; + if (tEncodeI8(pEncoder, pCol->sma) < 0) return -1; + if (tEncodeI32(pEncoder, pCol->bytes) < 0) return -1; + if (tEncodeI32(pEncoder, pCol->offset) < 0) return -1; + return pEncoder->pos; +} + +int32_t tDecodeSTColumn(SCoder *pDecoder, STColumn *pCol) { + if (tDecodeI16(pDecoder, &pCol->colId) < 0) return -1; + if (tDecodeI8(pDecoder, &pCol->type) < 0) return -1; + if (tDecodeI8(pDecoder, &pCol->sma) < 0) return -1; + if (tDecodeI32(pDecoder, &pCol->bytes) < 0) return -1; + if (tDecodeI32(pDecoder, &pCol->offset) < 0) return -1; + return 0; +} + +int32_t tEncodeSchema(SCoder *pEncoder, const STSchema *pSchema) { + if (tEncodeI32(pEncoder, pSchema->numOfCols) < 0) return -1; + if (tEncodeI16(pEncoder, pSchema->version) < 0) return -1; + if (tEncodeU16(pEncoder, pSchema->flen) < 0) return -1; + if (tEncodeI32(pEncoder, pSchema->vlen) < 0) return -1; + if (tEncodeI32(pEncoder, pSchema->tlen) < 0) return -1; + + for (int32_t i = 0; i < schemaNCols(pSchema); i++) { + const STColumn *pCol = schemaColAt(pSchema, i); + if (tEncodeSTColumn(pEncoder, pCol) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSchema(SCoder *pDecoder, STSchema *pSchema) { + if (tDecodeI32(pDecoder, &pSchema->numOfCols) < 0) return -1; + if (tDecodeI16(pDecoder, &pSchema->version) < 0) return -1; + if (tDecodeU16(pDecoder, &pSchema->flen) < 0) return -1; + if (tDecodeI32(pDecoder, &pSchema->vlen) < 0) return -1; + if (tDecodeI32(pDecoder, &pSchema->tlen) < 0) return -1; + + return 0; +} +#endif + int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, schema_ver_t version) { if (pBuilder == NULL) return -1; @@ -908,4 +953,4 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch taosArrayDestroy(stashRow); return buffer; } -#endif \ No newline at end of file +#endif diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 6235dcf895..6278b52a04 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -103,6 +103,25 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { } } +int32_t tPrintFixedSchemaSubmitReq(const SSubmitReq *pReq, STSchema *pTschema) { + SSubmitMsgIter msgIter = {0}; + if (tInitSubmitMsgIter(pReq, &msgIter) < 0) return -1; + while (true) { + SSubmitBlk *pBlock = NULL; + if (tGetSubmitMsgNext(&msgIter, &pBlock) < 0) return -1; + if (pBlock == NULL) break; + SSubmitBlkIter blkIter = {0}; + tInitSubmitBlkIter(&msgIter, pBlock, &blkIter); + STSRowIter rowIter = {0}; + tdSTSRowIterInit(&rowIter, pTschema); + STSRow *row; + while ((row = tGetSubmitBlkNext(&blkIter)) != NULL) { + tdSRowPrint(row, pTschema, "stream"); + } + } + return 0; +} + int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) { if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; @@ -1145,31 +1164,47 @@ int32_t tDeserializeSGetUserAuthReq(void *buf, int32_t bufLen, SGetUserAuthReq * return 0; } -int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - SCoder encoder = {0}; - tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - - if (tStartEncode(&encoder) < 0) return -1; - if (tEncodeCStr(&encoder, pRsp->user) < 0) return -1; - if (tEncodeI8(&encoder, pRsp->superAuth) < 0) return -1; +int32_t tSerializeSGetUserAuthRspImpl(SCoder *pEncoder, SGetUserAuthRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->user) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->superAuth) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs); int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); - if (tEncodeI32(&encoder, numOfReadDbs) < 0) return -1; - if (tEncodeI32(&encoder, numOfWriteDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfCreatedDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfReadDbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfWriteDbs) < 0) return -1; - char *db = taosHashIterate(pRsp->readDbs, NULL); + char *db = taosHashIterate(pRsp->createdDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; + db = taosHashIterate(pRsp->createdDbs, db); + } + + db = taosHashIterate(pRsp->readDbs, NULL); + while (db != NULL) { + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->readDbs, db); } db = taosHashIterate(pRsp->writeDbs, NULL); while (db != NULL) { - if (tEncodeCStr(&encoder, db) < 0) return -1; + if (tEncodeCStr(pEncoder, db) < 0) return -1; db = taosHashIterate(pRsp->writeDbs, db); } + return 0; +} + +int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + if (tSerializeSGetUserAuthRspImpl(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1177,39 +1212,58 @@ int32_t tSerializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pR return tlen; } -int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { - pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); - pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false); +int32_t tDeserializeSGetUserAuthRspImpl(SCoder *pDecoder, SGetUserAuthRsp *pRsp) { + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { return -1; } - SCoder decoder = {0}; - tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - - if (tStartDecode(&decoder) < 0) return -1; - if (tDecodeCStrTo(&decoder, pRsp->user) < 0) return -1; - if (tDecodeI8(&decoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->user) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->superAuth) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->version) < 0) return -1; + int32_t numOfCreatedDbs = 0; int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; - if (tDecodeI32(&decoder, &numOfReadDbs) < 0) return -1; - if (tDecodeI32(&decoder, &numOfWriteDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfCreatedDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfReadDbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfWriteDbs) < 0) return -1; + + for (int32_t i = 0; i < numOfCreatedDbs; ++i) { + char db[TSDB_DB_FNAME_LEN] = {0}; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; + int32_t len = strlen(db) + 1; + taosHashPut(pRsp->createdDbs, db, len, db, len); + } for (int32_t i = 0; i < numOfReadDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->readDbs, db, len, db, len); } for (int32_t i = 0; i < numOfWriteDbs; ++i) { char db[TSDB_DB_FNAME_LEN] = {0}; - if (tDecodeCStrTo(&decoder, db) < 0) return -1; + if (tDecodeCStrTo(pDecoder, db) < 0) return -1; int32_t len = strlen(db) + 1; taosHashPut(pRsp->writeDbs, db, len, db, len); } + return 0; +} + + +int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDeserializeSGetUserAuthRspImpl(&decoder, pRsp) < 0) return -1; + tEndDecode(&decoder); tCoderClear(&decoder); @@ -1217,6 +1271,7 @@ int32_t tDeserializeSGetUserAuthRsp(void *buf, int32_t bufLen, SGetUserAuthRsp * } void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { + taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->writeDbs); } @@ -2036,6 +2091,62 @@ void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } +int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSGetUserAuthRspImpl(&encoder, pUserAuthRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp){ + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserAuthRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp rsp = {0}; + if (tDeserializeSGetUserAuthRspImpl(&decoder, &rsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &rsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp){ + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserAuthRsp *pUserAuthRsp = taosArrayGet(pRsp->pArray, i); + tFreeSGetUserAuthRsp(pUserAuthRsp); + } + + taosArrayDestroy(pRsp->pArray); +} + + int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SCoder encoder = {0}; tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); @@ -3841,3 +3952,65 @@ int32_t tDecodeSVDropStbReq(SCoder *pCoder, SVDropStbReq *pReq) { tEndDecode(pCoder); return 0; } + +static int32_t tEncodeSVSubmitBlk(SCoder *pCoder, const SVSubmitBlk *pBlock, int32_t flags) { + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI64(pCoder, pBlock->suid) < 0) return -1; + if (tEncodeI64(pCoder, pBlock->uid) < 0) return -1; + if (tEncodeI32v(pCoder, pBlock->sver) < 0) return -1; + if (tEncodeBinary(pCoder, pBlock->pData, pBlock->nData) < 0) return -1; + + if (flags & TD_AUTO_CREATE_TABLE) { + if (tEncodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +static int32_t tDecodeSVSubmitBlk(SCoder *pCoder, SVSubmitBlk *pBlock, int32_t flags) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI64(pCoder, &pBlock->suid) < 0) return -1; + if (tDecodeI64(pCoder, &pBlock->uid) < 0) return -1; + if (tDecodeI32v(pCoder, &pBlock->sver) < 0) return -1; + if (tDecodeBinary(pCoder, &pBlock->pData, &pBlock->nData) < 0) return -1; + + if (flags & TD_AUTO_CREATE_TABLE) { + if (tDecodeSVCreateTbReq(pCoder, &pBlock->cTbReq) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} + +int32_t tEncodeSVSubmitReq(SCoder *pCoder, const SVSubmitReq *pReq) { + int32_t nBlocks = taosArrayGetSize(pReq->pArray); + + if (tStartEncode(pCoder) < 0) return -1; + + if (tEncodeI32v(pCoder, pReq->flags) < 0) return -1; + if (tEncodeI32v(pCoder, nBlocks) < 0) return -1; + for (int32_t iBlock = 0; iBlock < nBlocks; iBlock++) { + if (tEncodeSVSubmitBlk(pCoder, (SVSubmitBlk *)taosArrayGet(pReq->pArray, iBlock), pReq->flags) < 0) return -1; + } + + tEndEncode(pCoder); + return 0; +} + +int32_t tDecodeSVSubmitReq(SCoder *pCoder, SVSubmitReq *pReq) { + if (tStartDecode(pCoder) < 0) return -1; + + if (tDecodeI32v(pCoder, &pReq->flags) < 0) return -1; + if (tDecodeI32v(pCoder, &pReq->nBlocks) < 0) return -1; + pReq->pBlocks = tCoderMalloc(pCoder, sizeof(SVSubmitBlk) * pReq->nBlocks); + if (pReq->pBlocks == NULL) return -1; + for (int32_t iBlock = 0; iBlock < pReq->nBlocks; iBlock++) { + if (tDecodeSVSubmitBlk(pCoder, pReq->pBlocks + iBlock, pReq->flags) < 0) return -1; + } + + tEndDecode(pCoder); + return 0; +} \ No newline at end of file diff --git a/source/dnode/mgmt/implement/src/dmTransport.c b/source/dnode/mgmt/implement/src/dmTransport.c index 446894556e..114d7b6dfc 100644 --- a/source/dnode/mgmt/implement/src/dmTransport.c +++ b/source/dnode/mgmt/implement/src/dmTransport.c @@ -16,8 +16,8 @@ #define _DEFAULT_SOURCE #include "dmImp.h" -#define INTERNAL_USER "_dnd" -#define INTERNAL_CKEY "_key" +#define INTERNAL_USER "_dnd" +#define INTERNAL_CKEY "_key" #define INTERNAL_SECRET "_pwd" static void dmGetMnodeEpSet(SDnode *pDnode, SEpSet *pEpSet) { @@ -130,10 +130,10 @@ _OVER: } static void dmProcessMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; tmsg_t msgType = pMsg->msgType; bool isReq = msgType & 1u; - SMsgHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; + SMsgHandle * pHandle = &pTrans->msgHandles[TMSG_INDEX(msgType)]; SMgmtWrapper *pWrapper = pHandle->pNdWrapper; if (msgType == TDMT_DND_SERVER_STATUS) { @@ -517,7 +517,7 @@ static inline int32_t dmRetrieveUserAuthInfo(SDnode *pDnode, char *user, char *s SAuthReq authReq = {0}; tstrncpy(authReq.user, user, TSDB_USER_LEN); int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); - void *pReq = rpcMallocCont(contLen); + void * pReq = rpcMallocCont(contLen); tSerializeSAuthReq(pReq, contLen, &authReq); SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; @@ -547,6 +547,8 @@ static int32_t dmInitServer(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; + + strncpy(rpcInit.localFqdn, pDnode->data.localFqdn, strlen(pDnode->data.localFqdn)); rpcInit.localPort = pDnode->data.serverPort; rpcInit.label = "DND"; rpcInit.numOfThreads = tsNumOfRpcThreads; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index d516b0bf26..89638524ae 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -255,8 +255,10 @@ typedef struct { int64_t updateTime; int8_t superUser; int32_t acctId; + int32_t authVersion; SHashObj* readDbs; SHashObj* writeDbs; + SRWLatch lock; } SUserObj; typedef struct { @@ -419,6 +421,7 @@ typedef struct { typedef struct { char key[TSDB_PARTITION_KEY_LEN]; + int64_t dbUid; int64_t offset; } SMqOffsetObj; diff --git a/source/dnode/mnode/impl/inc/mndOffset.h b/source/dnode/mnode/impl/inc/mndOffset.h index 556bfe6a53..b468496165 100644 --- a/source/dnode/mnode/impl/inc/mndOffset.h +++ b/source/dnode/mnode/impl/inc/mndOffset.h @@ -37,6 +37,8 @@ static FORCE_INLINE int32_t mndMakePartitionKey(char *key, const char *cgroup, c return snprintf(key, TSDB_PARTITION_KEY_LEN, "%d:%s:%s", vgId, cgroup, topicName); } +int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndSma.h b/source/dnode/mnode/impl/inc/mndSma.h index 91c6e24e28..4a80f619d3 100644 --- a/source/dnode/mnode/impl/inc/mndSma.h +++ b/source/dnode/mnode/impl/inc/mndSma.h @@ -26,7 +26,6 @@ int32_t mndInitSma(SMnode *pMnode); void mndCleanupSma(SMnode *pMnode); SSmaObj *mndAcquireSma(SMnode *pMnode, char *smaName); void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma); -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSubscribe.h b/source/dnode/mnode/impl/inc/mndSubscribe.h index 7915006f27..5ca672e8dd 100644 --- a/source/dnode/mnode/impl/inc/mndSubscribe.h +++ b/source/dnode/mnode/impl/inc/mndSubscribe.h @@ -31,6 +31,8 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub); int32_t mndMakeSubscribeKey(char *key, const char *cgroup, const char *topicName); +int32_t mndDropSubByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index b3eb7f2f95..2140d0fa67 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -29,6 +29,7 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index f42829eddf..8aecc22454 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -33,6 +33,7 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5861f8d245..af7efb5543 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -17,9 +17,12 @@ #include "mndDb.h" #include "mndAuth.h" #include "mndDnode.h" +#include "mndOffset.h" #include "mndShow.h" #include "mndSma.h" #include "mndStb.h" +#include "mndSubscribe.h" +#include "mndTopic.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -41,16 +44,17 @@ static int32_t mndProcessCompactDbReq(SNodeMsg *pReq); static int32_t mndRetrieveDbs(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rowsCapacity); static void mndCancelGetNextDb(SMnode *pMnode, void *pIter); static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq); -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq); int32_t mndInitDb(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_DB, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndDbActionEncode, - .decodeFp = (SdbDecodeFp)mndDbActionDecode, - .insertFp = (SdbInsertFp)mndDbActionInsert, - .updateFp = (SdbUpdateFp)mndDbActionUpdate, - .deleteFp = (SdbDeleteFp)mndDbActionDelete}; + SSdbTable table = { + .sdbType = SDB_DB, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndDbActionEncode, + .decodeFp = (SdbDecodeFp)mndDbActionDecode, + .insertFp = (SdbInsertFp)mndDbActionInsert, + .updateFp = (SdbUpdateFp)mndDbActionUpdate, + .deleteFp = (SdbDeleteFp)mndDbActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_DB, mndProcessCreateDbReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_DB, mndProcessAlterDbReq); @@ -58,7 +62,6 @@ int32_t mndInitDb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_USE_DB, mndProcessUseDbReq); mndSetMsgHandle(pMnode, TDMT_MND_COMPACT_DB, mndProcessCompactDbReq); mndSetMsgHandle(pMnode, TDMT_MND_GET_DB_CFG, mndProcessGetDbCfgReq); - mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetIndexReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_DB, mndRetrieveDbs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_DB, mndCancelGetNextDb); @@ -191,6 +194,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pDb->lock); terrno = 0; @@ -219,17 +223,29 @@ static int32_t mndDbActionDelete(SSdb *pSdb, SDbObj *pDb) { static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { mTrace("db:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); taosWLockLatch(&pOld->lock); - SArray *pOldRetensions = pOld->cfg.pRetensions; pOld->updateTime = pNew->updateTime; pOld->cfgVersion = pNew->cfgVersion; pOld->vgVersion = pNew->vgVersion; - memcpy(&pOld->cfg, &pNew->cfg, sizeof(SDbCfg)); - pNew->cfg.pRetensions = pOldRetensions; + pOld->cfg.buffer = pNew->cfg.buffer; + pOld->cfg.pages = pNew->cfg.pages; + pOld->cfg.pageSize = pNew->cfg.pageSize; + pOld->cfg.daysPerFile = pNew->cfg.daysPerFile; + pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0; + pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1; + pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2; + pOld->cfg.fsyncPeriod = pNew->cfg.fsyncPeriod; + pOld->cfg.walLevel = pNew->cfg.walLevel; + pOld->cfg.strict = pNew->cfg.strict; + pOld->cfg.cacheLastRow = pNew->cfg.cacheLastRow; + pOld->cfg.replications = pNew->cfg.replications; taosWUnLockLatch(&pOld->lock); return 0; } -static int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { return sdbGetTableVer(pMnode->pSdb, SDB_VGROUP); } +static inline int32_t mndGetGlobalVgroupVersion(SMnode *pMnode) { + SSdb *pSdb = pMnode->pSdb; + return sdbGetTableVer(pSdb, SDB_VGROUP); +} SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SSdb *pSdb = pMnode->pSdb; @@ -635,69 +651,7 @@ static int32_t mndSetAlterDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *p return 0; } -void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { - SAlterVnodeReq alterReq = {0}; - alterReq.vgVersion = pVgroup->version; - alterReq.buffer = pDb->cfg.buffer; - alterReq.pages = pDb->cfg.pages; - alterReq.pageSize = pDb->cfg.pageSize; - alterReq.daysPerFile = pDb->cfg.daysPerFile; - alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; - alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; - alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; - alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; - alterReq.walLevel = pDb->cfg.walLevel; - alterReq.strict = pDb->cfg.strict; - alterReq.cacheLastRow = pDb->cfg.cacheLastRow; - alterReq.replica = pVgroup->replica; - alterReq.selfIndex = -1; - - for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &alterReq.replicas[v]; - SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; - SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); - if (pVgidDnode == NULL) { - return NULL; - } - - pReplica->id = pVgidDnode->id; - pReplica->port = pVgidDnode->port; - memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); - mndReleaseDnode(pMnode, pVgidDnode); - - if (pDnode->id == pVgid->dnodeId) { - alterReq.selfIndex = v; - } - } - - if (alterReq.selfIndex == -1) { - terrno = TSDB_CODE_MND_APP_ERROR; - return NULL; - } - - int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); - if (contLen < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - contLen += +sizeof(SMsgHead); - - void *pReq = taosMemoryMalloc(contLen); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - SMsgHead *pHead = pReq; - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(pVgroup->vgId); - - tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); - *pContLen = contLen; - return pReq; -} - -static int32_t mndBuilAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { +static int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup) { for (int32_t vn = 0; vn < pVgroup->replica; ++vn) { STransAction action = {0}; SVnodeGid *pVgid = pVgroup->vnodeGid + vn; @@ -733,7 +687,7 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * if (pIter == NULL) break; if (pVgroup->dbUid == pNew->uid) { - if (mndBuilAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { + if (mndBuildAlterVgroupAction(pMnode, pTrans, pNew, pVgroup) != 0) { sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pVgroup); return -1; @@ -749,19 +703,19 @@ static int32_t mndSetAlterDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj * static int32_t mndAlterDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pOld, SDbObj *pNew) { int32_t code = -1; STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_ALTER_DB, &pReq->rpcMsg); - if (pTrans == NULL) goto UPDATE_DB_OVER; + if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to alter db:%s", pTrans->id, pOld->name); mndTransSetDbInfo(pTrans, pOld); - if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto UPDATE_DB_OVER; - if (mndTransPrepare(pMnode, pTrans) != 0) goto UPDATE_DB_OVER; + if (mndSetAlterDbRedoLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbCommitLogs(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndSetAlterDbRedoActions(pMnode, pTrans, pOld, pNew) != 0) goto _OVER; + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; -UPDATE_DB_OVER: +_OVER: mndTransDrop(pTrans); return code; } @@ -775,7 +729,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto ALTER_DB_OVER; + goto _OVER; } mDebug("db:%s, start to alter", alterReq.db); @@ -783,24 +737,26 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto ALTER_DB_OVER; + goto _OVER; } pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto ALTER_DB_OVER; + goto _OVER; } if (mndCheckAlterDropCompactDbAuth(pUser, pDb) != 0) { - goto ALTER_DB_OVER; + goto _OVER; } SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); + dbObj.cfg.numOfRetensions = 0; + dbObj.cfg.pRetensions = NULL; code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) { - goto ALTER_DB_OVER; + goto _OVER; } dbObj.cfgVersion++; @@ -808,7 +764,7 @@ static int32_t mndProcessAlterDbReq(SNodeMsg *pReq) { code = mndAlterDb(pMnode, pReq, pDb, &dbObj); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; -ALTER_DB_OVER: +_OVER: if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } @@ -828,13 +784,13 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (tDeserializeSDbCfgReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto GET_DB_CFG_OVER; + goto _OVER; } pDb = mndAcquireDb(pMnode, cfgReq.db); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - goto GET_DB_CFG_OVER; + goto _OVER; } cfgRsp.numOfVgroups = pDb->cfg.numOfVgroups; @@ -863,7 +819,7 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto GET_DB_CFG_OVER; + goto _OVER; } tSerializeSDbCfgRsp(pRsp, contLen, &cfgRsp); @@ -873,9 +829,9 @@ static int32_t mndProcessGetDbCfgReq(SNodeMsg *pReq) { code = 0; -GET_DB_CFG_OVER: +_OVER: - if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0) { mError("db:%s, failed to get cfg since %s", cfgReq.db, terrstr()); } @@ -1027,6 +983,9 @@ static int32_t mndDropDb(SMnode *pMnode, SNodeMsg *pReq, SDbObj *pDb) { if (mndSetDropDbRedoLogs(pMnode, pTrans, pDb) != 0) goto _OVER; if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto _OVER; + /*if (mndDropOffsetByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropSubByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ + /*if (mndDropTopicByDB(pMnode, pTrans, pDb) != 0) goto _OVER;*/ if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto _OVER; int32_t rspLen = 0; @@ -1091,7 +1050,8 @@ _OVER: return code; } -void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { +static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) { + int32_t numOfTables = 0; int32_t vindex = 0; SSdb *pSdb = pMnode->pSdb; @@ -1102,8 +1062,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - *num += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; - + numOfTables += pVgroup->numOfTables / TSDB_TABLE_NUM_UNIT; vindex++; } @@ -1111,6 +1070,7 @@ void mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode, int32_t *num) { } sdbCancelFetch(pSdb, pIter); + return numOfTables; } static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { @@ -1164,8 +1124,7 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs return -1; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pReq == NULL || pReq->vgVersion < pDb->vgVersion || pReq->dbId != pDb->uid || numOfTable != pReq->numOfTable) { mndBuildDBVgroupInfo(pDb, pMnode, pRsp->pVgroupInfos); @@ -1189,7 +1148,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (tDeserializeSUseDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &usedbReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; - goto USE_DB_OVER; + goto _OVER; } char *p = strchr(usedbReq.db, '.'); @@ -1200,12 +1159,11 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { usedbRsp.pVgroupInfos = taosArrayInit(10, sizeof(SVgroupInfo)); if (usedbRsp.pVgroupInfos == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto USE_DB_OVER; + goto _OVER; } mndBuildDBVgroupInfo(NULL, pMnode, usedbRsp.pVgroupInfos); usedbRsp.vgVersion = vgVersion++; - } else { usedbRsp.vgVersion = usedbReq.vgVersion; } @@ -1226,15 +1184,15 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { } else { pUser = mndAcquireUser(pMnode, pReq->user); if (pUser == NULL) { - goto USE_DB_OVER; + goto _OVER; } if (mndCheckUseDbAuth(pUser, pDb) != 0) { - goto USE_DB_OVER; + goto _OVER; } if (mndExtractDbInfo(pMnode, pDb, &usedbRsp, &usedbReq) < 0) { - goto USE_DB_OVER; + goto _OVER; } code = 0; @@ -1246,7 +1204,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; code = -1; - goto USE_DB_OVER; + goto _OVER; } tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); @@ -1254,7 +1212,7 @@ static int32_t mndProcessUseDbReq(SNodeMsg *pReq) { pReq->pRsp = pRsp; pReq->rspLen = contLen; -USE_DB_OVER: +_OVER: if (code != 0) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } @@ -1292,8 +1250,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, continue; } - int32_t numOfTable = 0; - mndGetDBTableNum(pDb, pMnode, &numOfTable); + int32_t numOfTable = mndGetDBTableNum(pDb, pMnode); if (pDbVgVersion->vgVersion >= pDb->vgVersion && numOfTable == pDbVgVersion->numOfTable) { mDebug("db:%s, version & numOfTable not changed", pDbVgVersion->dbFName); @@ -1387,7 +1344,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in bool sysDb) { int32_t cols = 0; - int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; + int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; char *buf = taosMemoryMalloc(bytes); const char *name = mndGetDbStr(pDb->name); if (name != NULL) { @@ -1508,9 +1465,6 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in pColInfo = taosArrayGet(pBlock->pDataBlock, cols); colDataAppend(pColInfo, rows, (const char *)b, false); } - - // pWrite = getDataPosition(data, pShow, cols, rows, rowCapacity); - // *(int8_t *)pWrite = pDb->cfg.update; } static void setInformationSchemaDbCfg(SDbObj *pDbObj) { @@ -1538,7 +1492,6 @@ static void setPerfSchemaDbCfg(SDbObj *pDbObj) { static bool mndGetTablesOfDbFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { SVgObj *pVgroup = pObj; int32_t *numOfTables = p1; - *numOfTables += pVgroup->numOfTables; return true; } @@ -1588,49 +1541,3 @@ static void mndCancelGetNextDb(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } - -static int32_t mndProcessGetIndexReq(SNodeMsg *pReq) { - SUserIndexReq indexReq = {0}; - SMnode *pMnode = pReq->pNode; - int32_t code = -1; - SUserIndexRsp rsp = {0}; - bool exist = false; - - if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - goto _OVER; - } - - code = mndProcessGetSmaReq(pMnode, &indexReq, &rsp, &exist); - if (code) { - goto _OVER; - } - - if (!exist) { - // TODO GET INDEX FROM FULLTEXT - code = -1; - terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; - } else { - int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - code = -1; - goto _OVER; - } - - tSerializeSUserIndexRsp(pRsp, contLen, &rsp); - - pReq->pRsp = pRsp; - pReq->rspLen = contLen; - - code = 0; - } - -_OVER: - if (code != 0) { - mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); - } - - return code; -} diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index b9331e6e03..3ac2951b6f 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -314,7 +314,7 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { + if (createReq.bufSize < 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index f075348704..e0ba4076f5 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -231,3 +231,36 @@ static void mndCancelGetNextOffset(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +static int32_t mndSetDropOffsetCommitLogs(SMnode *pMnode, STrans *pTrans, SMqOffsetObj *pOffset) { + SSdbRaw *pCommitRaw = mndOffsetActionEncode(pOffset); + if (pCommitRaw == NULL) return -1; + if (mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) return -1; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_DROPPED) != 0) return -1; + return 0; +} + +int32_t mndDropOffsetByDB(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { + int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; + + void *pIter = NULL; + SMqOffsetObj *pOffset = NULL; + while (1) { + pIter = sdbFetch(pSdb, SDB_SUBSCRIBE, pIter, (void **)&pOffset); + if (pIter == NULL) break; + + if (pOffset->dbUid != pDb->uid) { + sdbRelease(pSdb, pOffset); + continue; + } + + if (mndSetDropOffsetCommitLogs(pMnode, pTrans, pOffset) < 0) { + goto END; + } + } + + code = 0; +END: + return code; +} diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 59d07fd4aa..2de337537f 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -403,6 +403,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SKv *kv = pIter; switch (kv->key) { + case HEARTBEAT_KEY_USER_AUTHINFO: { + void * rspMsg = NULL; + int32_t rspLen = 0; + mndValidateUserAuthInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserAuthVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_USER_AUTHINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } case HEARTBEAT_KEY_DBINFO: { void * rspMsg = NULL; int32_t rspLen = 0; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index b760dd6aa8..9274d35b8b 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -204,6 +204,8 @@ int32_t mndAddShuffledSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* p pTask->smaSink.smaId = pStream->smaId; } else { pTask->sinkType = TASK_SINK__TABLE; + pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); + ASSERT(pTask->tbSink.pSchemaWrapper); } // dispatch @@ -242,6 +244,7 @@ int32_t mndAddFixedSinkToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStr pTask->smaSink.smaId = pStream->smaId; } else { pTask->sinkType = TASK_SINK__TABLE; + pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); } // // dispatch @@ -316,6 +319,7 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pTask->smaSink.smaId = pStream->smaId; } else { pTask->sinkType = TASK_SINK__TABLE; + pTask->tbSink.pSchemaWrapper = tCloneSSchemaWrapper(&pStream->outputSchema); } #endif } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 0e3e8b68cc..8619df978b 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -40,6 +40,7 @@ static int32_t mndProcessMCreateSmaReq(SNodeMsg *pReq); static int32_t mndProcessMDropSmaReq(SNodeMsg *pReq); static int32_t mndProcessVCreateSmaRsp(SNodeMsg *pRsp); static int32_t mndProcessVDropSmaRsp(SNodeMsg *pRsp); +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq); static int32_t mndRetrieveSma(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextSma(SMnode *pMnode, void *pIter); @@ -56,6 +57,7 @@ int32_t mndInitSma(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_DROP_SMA, mndProcessMDropSmaReq); mndSetMsgHandle(pMnode, TDMT_VND_CREATE_SMA_RSP, mndProcessVCreateSmaRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_SMA_RSP, mndProcessVDropSmaRsp); + mndSetMsgHandle(pMnode, TDMT_MND_GET_INDEX, mndProcessGetSmaReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndRetrieveSma); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_INDEX, mndCancelGetNextSma); @@ -686,7 +688,7 @@ _OVER: return code; } -int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { +static int32_t mndGetSma(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexRsp *rsp, bool *exist) { int32_t code = -1; SSmaObj *pSma = NULL; @@ -715,6 +717,51 @@ int32_t mndProcessGetSmaReq(SMnode *pMnode, SUserIndexReq *indexReq, SUserIndexR } mndReleaseSma(pMnode, pSma); + return code; +} + +static int32_t mndProcessGetSmaReq(SNodeMsg *pReq) { + SUserIndexReq indexReq = {0}; + SMnode *pMnode = pReq->pNode; + int32_t code = -1; + SUserIndexRsp rsp = {0}; + bool exist = false; + + if (tDeserializeSUserIndexReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &indexReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + code = mndGetSma(pMnode, &indexReq, &rsp, &exist); + if (code) { + goto _OVER; + } + + if (!exist) { + // TODO GET INDEX FROM FULLTEXT + code = -1; + terrno = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; + } else { + int32_t contLen = tSerializeSUserIndexRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + code = -1; + goto _OVER; + } + + tSerializeSUserIndexRsp(pRsp, contLen, &rsp); + + pReq->pRsp = pRsp; + pReq->rspLen = contLen; + + code = 0; + } + +_OVER: + if (code != 0) { + mError("failed to get index %s since %s", indexReq.indexFName, terrstr()); + } return code; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index f3438bd0dc..43e2735984 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1050,7 +1050,7 @@ static int32_t mndTransExecuteActions(SMnode *pMnode, STrans *pTrans, SArray *pA static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->redoActions); - if (code != 0) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("failed to execute redoActions since %s", terrstr()); } return code; @@ -1058,7 +1058,7 @@ static int32_t mndTransExecuteRedoActions(SMnode *pMnode, STrans *pTrans) { static int32_t mndTransExecuteUndoActions(SMnode *pMnode, STrans *pTrans) { int32_t code = mndTransExecuteActions(pMnode, pTrans, pTrans->undoActions); - if (code != 0) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("failed to execute undoActions since %s", terrstr()); } return code; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5e15bdeb43..8b5fa06faf 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -186,6 +186,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { } SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pUser->lock); terrno = 0; @@ -228,11 +229,12 @@ static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { mTrace("user:%s, perform update action, old row:%p new row:%p", pOld->user, pOld, pNew); - memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); + taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; - + memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->writeDbs, pNew->writeDbs); + taosWUnLockLatch(&pOld->lock); return 0; } @@ -426,8 +428,12 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { } memcpy(&newUser, pUser, sizeof(SUserObj)); + + taosRLockLatch(&pUser->lock); newUser.readDbs = mndDupDbHash(pUser->readDbs); newUser.writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + if (newUser.readDbs == NULL || newUser.writeDbs == NULL) { goto _OVER; } @@ -451,13 +457,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_DB) { if (taosHashRemove(newUser.readDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_READ_DB) { taosHashClear(newUser.readDbs); + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_DB) { if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; @@ -467,13 +476,16 @@ static int32_t mndProcessAlterUserReq(SNodeMsg *pReq) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_DB) { if (taosHashRemove(newUser.writeDbs, alterReq.dbname, len) != 0) { terrno = TSDB_CODE_MND_DB_NOT_EXIST; goto _OVER; } + newUser.authVersion++; } else if (alterReq.alterType == TSDB_ALTER_USER_CLEAR_WRITE_DB) { taosHashClear(newUser.writeDbs); + newUser.authVersion++; } else { terrno = TSDB_CODE_MND_INVALID_ALTER_OPER; goto _OVER; @@ -576,6 +588,38 @@ _OVER: return code; } +static int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { + memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); + pRsp->superAuth = pUser->superUser; + pRsp->version = pUser->authVersion; + taosRLockLatch(&pUser->lock); + pRsp->readDbs = mndDupDbHash(pUser->readDbs); + pRsp->writeDbs = mndDupDbHash(pUser->writeDbs); + taosRUnLockLatch(&pUser->lock); + pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (NULL == pRsp->createdDbs) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SDbObj *pDb = NULL; + pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); + if (pIter == NULL) break; + + if (strcmp(pDb->createUser, pUser->user) == 0) { + int32_t len = strlen(pDb->name) + 1; + taosHashPut(pRsp->createdDbs, pDb->name, len, pDb->name, len); + } + + sdbRelease(pSdb, pDb); + } + + return 0; +} + static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { SMnode *pMnode = pReq->pNode; int32_t code = -1; @@ -596,25 +640,9 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { goto _OVER; } - memcpy(authRsp.user, pUser->user, TSDB_USER_LEN); - authRsp.superAuth = pUser->superUser; - authRsp.readDbs = mndDupDbHash(pUser->readDbs); - authRsp.writeDbs = mndDupDbHash(pUser->writeDbs); - - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; - while (1) { - SDbObj *pDb = NULL; - pIter = sdbFetch(pSdb, SDB_DB, pIter, (void **)&pDb); - if (pIter == NULL) break; - - if (strcmp(pDb->createUser, pUser->user) == 0) { - int32_t len = strlen(pDb->name) + 1; - taosHashPut(authRsp.readDbs, pDb->name, len, pDb->name, len); - taosHashPut(authRsp.writeDbs, pDb->name, len, pDb->name, len); - } - - sdbRelease(pSdb, pDb); + code = mndSetUserAuthRsp(pMnode, pUser, &authRsp); + if (code) { + goto _OVER; } int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &authRsp); @@ -631,6 +659,7 @@ static int32_t mndProcessGetUserAuthReq(SNodeMsg *pReq) { code = 0; _OVER: + mndReleaseUser(pMnode, pUser); tFreeSGetUserAuthRsp(&authRsp); @@ -681,3 +710,72 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { SSdb *pSdb = pMnode->pSdb; sdbCancelFetch(pSdb, pIter); } + +int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen) { + SUserAuthBatchRsp batchRsp = {0}; + batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserAuthRsp)); + if (batchRsp.pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + int32_t code = 0; + for (int32_t i = 0; i < numOfUses; ++i) { + SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + if (pUser == NULL) { + mError("user:%s, failed to auth user since %s", pUsers[i].user, terrstr()); + continue; + } + + if (pUser->authVersion <= pUsers[i].version) { + mndReleaseUser(pMnode, pUser); + continue; + } + + SGetUserAuthRsp rsp = {0}; + code = mndSetUserAuthRsp(pMnode, pUser, &rsp); + if (code) { + mndReleaseUser(pMnode, pUser); + tFreeSGetUserAuthRsp(&rsp); + goto _OVER; + } + + + taosArrayPush(batchRsp.pArray, &rsp); + mndReleaseUser(pMnode, pUser); + } + + if (taosArrayGetSize(batchRsp.pArray) <= 0) { + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + } + + int32_t rspLen = tSerializeSUserAuthBatchRsp(NULL, 0, &batchRsp); + void *pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeSUserAuthBatchRsp(&batchRsp); + return -1; + } + tSerializeSUserAuthBatchRsp(pRsp, rspLen, &batchRsp); + + *ppRsp = pRsp; + *pRspLen = rspLen; + + tFreeSUserAuthBatchRsp(&batchRsp); + return 0; + +_OVER: + + *ppRsp = NULL; + *pRspLen = 0; + + tFreeSUserAuthBatchRsp(&batchRsp); + return code; +} + + + diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 31e4a8ea7d..d1e4be1161 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -21,8 +21,8 @@ #include "mndShow.h" #include "mndTrans.h" -#define TSDB_VGROUP_VER_NUMBER 1 -#define TSDB_VGROUP_RESERVE_SIZE 64 +#define VGROUP_VER_NUMBER 1 +#define VGROUP_RESERVE_SIZE 64 static SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw); static int32_t mndVgroupActionInsert(SSdb *pSdb, SVgObj *pVgroup); @@ -34,19 +34,21 @@ static int32_t mndProcessAlterVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessDropVnodeRsp(SNodeMsg *pRsp); static int32_t mndProcessCompactVnodeRsp(SNodeMsg *pRsp); -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVgroup(SMnode *pMnode, void *pIter); -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows); +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextVnode(SMnode *pMnode, void *pIter); int32_t mndInitVgroup(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_VGROUP, - .keyType = SDB_KEY_INT32, - .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, - .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, - .insertFp = (SdbInsertFp)mndVgroupActionInsert, - .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, - .deleteFp = (SdbDeleteFp)mndVgroupActionDelete}; + SSdbTable table = { + .sdbType = SDB_VGROUP, + .keyType = SDB_KEY_INT32, + .encodeFp = (SdbEncodeFp)mndVgroupActionEncode, + .decodeFp = (SdbDecodeFp)mndVgroupActionDecode, + .insertFp = (SdbInsertFp)mndVgroupActionInsert, + .updateFp = (SdbUpdateFp)mndVgroupActionUpdate, + .deleteFp = (SdbDeleteFp)mndVgroupActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_DND_CREATE_VNODE_RSP, mndProcessCreateVnodeRsp); mndSetMsgHandle(pMnode, TDMT_VND_ALTER_VNODE_RSP, mndProcessAlterVnodeRsp); @@ -66,29 +68,29 @@ void mndCleanupVgroup(SMnode *pMnode) {} SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) { terrno = TSDB_CODE_OUT_OF_MEMORY; - SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, TSDB_VGROUP_VER_NUMBER, sizeof(SVgObj) + TSDB_VGROUP_RESERVE_SIZE); - if (pRaw == NULL) goto VG_ENCODE_OVER; + SSdbRaw *pRaw = sdbAllocRaw(SDB_VGROUP, VGROUP_VER_NUMBER, sizeof(SVgObj) + VGROUP_RESERVE_SIZE); + if (pRaw == NULL) goto _OVER; int32_t dataPos = 0; - SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->version, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, VG_ENCODE_OVER) - SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, VG_ENCODE_OVER) - SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_ENCODE_OVER) - SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, VG_ENCODE_OVER) - SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->vgId, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->createdTime, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->updateTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->version, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin, _OVER) + SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd, _OVER) + SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid, _OVER) + SDB_SET_INT8(pRaw, dataPos, pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, VG_ENCODE_OVER) + SDB_SET_INT32(pRaw, dataPos, pVgid->dnodeId, _OVER) } - SDB_SET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_ENCODE_OVER) - SDB_SET_DATALEN(pRaw, dataPos, VG_ENCODE_OVER) + SDB_SET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) + SDB_SET_DATALEN(pRaw, dataPos, _OVER) terrno = 0; -VG_ENCODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to encode to raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); sdbFreeRaw(pRaw); @@ -103,41 +105,41 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; int8_t sver = 0; - if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto VG_DECODE_OVER; + if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != TSDB_VGROUP_VER_NUMBER) { + if (sver != VGROUP_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; - goto VG_DECODE_OVER; + goto _OVER; } SSdbRow *pRow = sdbAllocRow(sizeof(SVgObj)); - if (pRow == NULL) goto VG_DECODE_OVER; + if (pRow == NULL) goto _OVER; SVgObj *pVgroup = sdbGetRowObj(pRow); - if (pVgroup == NULL) goto VG_DECODE_OVER; + if (pVgroup == NULL) goto _OVER; int32_t dataPos = 0; - SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, VG_DECODE_OVER) - SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, VG_DECODE_OVER) - SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, VG_DECODE_OVER) - SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, VG_DECODE_OVER) - SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->vgId, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->createdTime, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->updateTime, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->version, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashBegin, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgroup->hashEnd, _OVER) + SDB_GET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pVgroup->dbUid, _OVER) + SDB_GET_INT8(pRaw, dataPos, &pVgroup->replica, _OVER) for (int8_t i = 0; i < pVgroup->replica; ++i) { SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; - SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, VG_DECODE_OVER) + SDB_GET_INT32(pRaw, dataPos, &pVgid->dnodeId, _OVER) if (pVgroup->replica == 1) { pVgid->role = TAOS_SYNC_STATE_LEADER; } } - SDB_GET_RESERVE(pRaw, dataPos, TSDB_VGROUP_RESERVE_SIZE, VG_DECODE_OVER) + SDB_GET_RESERVE(pRaw, dataPos, VGROUP_RESERVE_SIZE, _OVER) terrno = 0; -VG_DECODE_OVER: +_OVER: if (terrno != 0) { mError("vgId:%d, failed to decode from raw:%p since %s", pVgroup->vgId, pRaw, terrstr()); taosMemoryFreeClear(pRow); @@ -254,6 +256,68 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg return pReq; } +void *mndBuildAlterVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { + SAlterVnodeReq alterReq = {0}; + alterReq.vgVersion = pVgroup->version; + alterReq.buffer = pDb->cfg.buffer; + alterReq.pages = pDb->cfg.pages; + alterReq.pageSize = pDb->cfg.pageSize; + alterReq.daysPerFile = pDb->cfg.daysPerFile; + alterReq.daysToKeep0 = pDb->cfg.daysToKeep0; + alterReq.daysToKeep1 = pDb->cfg.daysToKeep1; + alterReq.daysToKeep2 = pDb->cfg.daysToKeep2; + alterReq.fsyncPeriod = pDb->cfg.fsyncPeriod; + alterReq.walLevel = pDb->cfg.walLevel; + alterReq.strict = pDb->cfg.strict; + alterReq.cacheLastRow = pDb->cfg.cacheLastRow; + alterReq.replica = pVgroup->replica; + alterReq.selfIndex = -1; + + for (int32_t v = 0; v < pVgroup->replica; ++v) { + SReplica *pReplica = &alterReq.replicas[v]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; + SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + if (pVgidDnode == NULL) { + return NULL; + } + + pReplica->id = pVgidDnode->id; + pReplica->port = pVgidDnode->port; + memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); + mndReleaseDnode(pMnode, pVgidDnode); + + if (pDnode->id == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + + if (alterReq.selfIndex == -1) { + terrno = TSDB_CODE_MND_APP_ERROR; + return NULL; + } + + int32_t contLen = tSerializeSAlterVnodeReq(NULL, 0, &alterReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + contLen += +sizeof(SMsgHead); + + void *pReq = taosMemoryMalloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + SMsgHead *pHead = pReq; + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + tSerializeSAlterVnodeReq((char *)pReq + sizeof(SMsgHead), contLen, &alterReq); + *pContLen = contLen; + return pReq; +} + void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { SDropVnodeReq dropReq = {0}; dropReq.dnodeId = pDnode->id; @@ -372,12 +436,12 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { pVgroups = taosMemoryCalloc(pDb->cfg.numOfVgroups, sizeof(SVgObj)); if (pVgroups == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - goto ALLOC_VGROUP_OVER; + goto _OVER; } pArray = mndBuildDnodesArray(pMnode); if (pArray == NULL) { - goto ALLOC_VGROUP_OVER; + goto _OVER; } mDebug("db:%s, total %d dnodes used to create %d vgroups (%d vnodes)", pDb->name, (int32_t)taosArrayGetSize(pArray), @@ -410,7 +474,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { if (mndGetAvailableDnode(pMnode, pVgroup, pArray) != 0) { terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; - goto ALLOC_VGROUP_OVER; + goto _OVER; } allocedVgroups++; @@ -421,7 +485,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) { mDebug("db:%s, %d vgroups is alloced, replica:%d", pDb->name, pDb->cfg.numOfVgroups, pDb->cfg.replications); -ALLOC_VGROUP_OVER: +_OVER: if (code != 0) taosMemoryFree(pVgroups); taosArrayDestroy(pArray); return code; @@ -492,7 +556,7 @@ static int32_t mndGetVgroupMaxReplica(SMnode *pMnode, char *dbName, int8_t *pRep return 0; } -static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; @@ -533,14 +597,13 @@ static int32_t mndRetrieveVgroups(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* // default 3 replica for (int32_t i = 0; i < 3; ++i) { - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); if (i < pVgroup->replica) { colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vnodeGid[i].dnodeId, false); char buf1[20] = {0}; const char *role = syncStr(pVgroup->vnodeGid[i].role); - STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); + STR_WITH_MAXSIZE_TO_VARSTR(buf1, role, pShow->pMeta->pSchemas[cols].bytes); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf1, false); @@ -597,13 +660,12 @@ int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId) { return numOfVnodes; } -static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock* pBlock, int32_t rows) { +static int32_t mndRetrieveVnodes(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->pNode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SVgObj *pVgroup = NULL; int32_t cols = 0; -// int32_t dnodeId = pShow->replica; while (numOfRows < rows) { pShow->pIter = sdbFetch(pSdb, SDB_VGROUP, pShow->pIter, (void **)&pVgroup); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 0473fa375e..c8f832160b 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -24,6 +24,7 @@ class MndTestFunc : public ::testing::Test { void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size); void SetComment(SCreateFuncReq* pReq, const char* pComment); + void SetBufSize(SCreateFuncReq* pReq, int32_t size); }; Testbase MndTestFunc::test; @@ -40,6 +41,10 @@ void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { strcpy(pReq->pComment, pComment); } +void MndTestFunc::SetBufSize(SCreateFuncReq* pReq, int32_t size) { + pReq->bufSize = size; +} + TEST_F(MndTestFunc, 01_Show_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 0); @@ -96,6 +101,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { strcpy(createReq.name, "f1"); SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); + SetBufSize(&createReq, -1); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); void* pReq = rpcMallocCont(contLen); diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 85d0f76268..58e00ee34a 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -30,6 +30,7 @@ target_sources( "src/tsdb/tsdbFS.c" "src/tsdb/tsdbOpen.c" "src/tsdb/tsdbMemTable.c" + "src/tsdb/tsdbMemTable2.c" "src/tsdb/tsdbRead.c" "src/tsdb/tsdbReadImpl.c" "src/tsdb/tsdbSma.c" diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 078a493773..2716692bd7 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -40,7 +40,6 @@ typedef struct STable STable; int tsdbMemTableCreate(STsdb *pTsdb, STsdbMemTable **ppMemTable); void tsdbMemTableDestroy(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows); int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 5538a63b4b..3724a98a5e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -100,6 +100,7 @@ int32_t tsdbUpdateSmaWindow(STsdb* pTsdb, SSubmitReq* pMsg, int64_t version int32_t tsdbCreateTSma(STsdb* pTsdb, char* pMsg); int32_t tsdbInsertTSmaData(STsdb* pTsdb, int64_t indexUid, const char* msg); int tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp); +int tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, int32_t* pAffectedRows); tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, uint64_t taskId); tsdbReaderT tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableGroupInfo* groupList, uint64_t qId, @@ -125,7 +126,6 @@ int32_t tsdbUpdateTbUidList(STsdb* pTsdb, STbUidStore* pUidStore); void tsdbUidStoreDestory(STbUidStore* pStore); void* tsdbUidStoreFree(STbUidStore* pStore); int32_t tsdbTriggerRSma(STsdb* pTsdb, void* pMsg, int32_t inputType); -int32_t tsdbProcessSubmitReq(STsdb* pTsdb, int64_t version, void* pReq); typedef struct { int8_t streamType; // sma or other diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 411d5c451c..261b4aa8ae 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -926,6 +926,12 @@ int32_t tqProcessTaskDeploy(STQ* pTq, char* msg, int32_t msgLen) { pTask->ahandle = pTq->pVnode; if (pTask->sinkType == TASK_SINK__SMA) { pTask->smaSink.smaHandle = smaHandleRes; + } else if (pTask->sinkType == TASK_SINK__TABLE) { + ASSERT(pTask->tbSink.pSchemaWrapper); + ASSERT(pTask->tbSink.pSchemaWrapper->pSchema); + pTask->tbSink.pTSchema = + tdGetSTSChemaFromSSChema(&pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols); + ASSERT(pTask->tbSink.pTSchema); } taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), pTask, sizeof(SStreamTask)); diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index fc23534247..97b796c6a5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -69,10 +69,6 @@ static void tsdbDestroyCommitIters(SCommitH *pCommith); static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid); static void tsdbResetCommitFile(SCommitH *pCommith); static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid); -// static int tsdbCommitMeta(STsdbRepo *pRepo); -// static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); -// static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); -// static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); static int tsdbCommitToTable(SCommitH *pCommith, int tid); static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable); static int tsdbComparKeyBlock(const void *arg1, const void *arg2); diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index ff4d99f510..b52a34cfb2 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -191,9 +191,6 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey } int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, int32_t *pAffectedRows) { - // STsdbMeta *pMeta = pRepo->tsdbMeta; - // int32_t points = 0; - // STable *pTable = NULL; SSubmitBlkIter blkIter = {0}; STsdbMemTable *pMemTable = pTsdb->mem; void *tptr; @@ -221,8 +218,9 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo } // copy data to buffer pool - pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, pMsgIter->dataLen + sizeof(*pBlock)); - memcpy(pBlkCopy, pBlock, pMsgIter->dataLen + sizeof(*pBlock)); + int32_t tlen = pMsgIter->dataLen + pMsgIter->schemaLen + sizeof(*pBlock); + pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->mem->pPool, tlen); + memcpy(pBlkCopy, pBlock, tlen); tInitSubmitBlkIter(pMsgIter, pBlkCopy, &blkIter); if (blkIter.row == NULL) return 0; @@ -241,7 +239,7 @@ int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlo if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin; if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax; - (*pAffectedRows) += pMsgIter->numOfRows; + (*pAffectedRows) = pMsgIter->numOfRows; return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable2.c b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c new file mode 100644 index 0000000000..ae0d4630d1 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable2.c @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct SMemTable SMemTable; +typedef struct SMemData SMemData; +typedef struct SMemSkipList SMemSkipList; +typedef struct SMemSkipListCfg SMemSkipListCfg; + +struct SMemTable { + STsdb *pTsdb; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + int32_t nHash; + int32_t nBucket; + SMemData **pBuckets; +}; + +struct SMemSkipListCfg { + int8_t maxLevel; + int32_t nKey; + int32_t nData; +}; + +struct SMemSkipList { + int8_t level; + uint32_t seed; +}; + +struct SMemData { + SMemData *pHashNext; + tb_uid_t suid; + tb_uid_t uid; + TSKEY minKey; + TSKEY maxKey; + int64_t minVer; + int64_t maxVer; + int64_t nRows; + SMemSkipList sl; +}; + +// SMemTable +int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable **ppMemTb) { + SMemTable *pMemTb = NULL; + + pMemTb = taosMemoryCalloc(1, sizeof(*pMemTb)); + if (pMemTb == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + pMemTb->pTsdb = pTsdb; + pMemTb->minKey = TSKEY_MAX; + pMemTb->maxKey = TSKEY_MIN; + pMemTb->minVer = -1; + pMemTb->maxVer = -1; + pMemTb->nRows = 0; + pMemTb->nHash = 0; + pMemTb->nBucket = 1024; + pMemTb->pBuckets = taosMemoryCalloc(pMemTb->nBucket, sizeof(*pMemTb->pBuckets)); + if (pMemTb->pBuckets == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + *ppMemTb = pMemTb; + return 0; +} + +int32_t tsdbMemTableDestroy2(STsdb *pTsdb, SMemTable *pMT) { + // TODO + return 0; +} + +// SMemData + +// SMemSkipList \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 362e31ff61..480b131914 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -1962,6 +1962,21 @@ int32_t tsdbUpdateTbUidList(STsdb *pTsdb, STbUidStore *pStore) { return TSDB_CODE_SUCCESS; } +static int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { + if (!pReq) { + terrno = TSDB_CODE_INVALID_PTR; + return TSDB_CODE_FAILED; + } + + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + + if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { + return TSDB_CODE_FAILED; + } + + return TSDB_CODE_SUCCESS; +} + static int32_t tsdbFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { ASSERT(pMsg != NULL); SSubmitMsgIter msgIter = {0}; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 878dd5fca4..06da3b0e1d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -450,7 +450,7 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in SCoder coder = {0}; int ret; - pRsp->msgType = TDMT_VND_CREATE_STB_RSP; + pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; pRsp->pCont = NULL; pRsp->contLen = 0; pRsp->code = TSDB_CODE_SUCCESS; @@ -473,9 +473,13 @@ static int vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq, in /* code */ ret = metaDropTable(pVnode->pMeta, version, pDropTbReq); if (ret < 0) { - dropTbRsp.code = TSDB_CODE_SUCCESS; + if (pDropTbReq->igNotExists && terrno == TSDB_CODE_VND_TABLE_NOT_EXIST) { + dropTbRsp.code = TSDB_CODE_SUCCESS; + } else { + dropTbRsp.code = terrno; + } } else { - dropTbRsp.code = terrno; + dropTbRsp.code = TSDB_CODE_SUCCESS; } taosArrayPush(rsp.pArray, &dropTbRsp); @@ -488,20 +492,55 @@ _exit: } static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - SSubmitRsp rsp = {0}; + SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; + SSubmitMsgIter msgIter = {0}; + SSubmitBlk *pBlock; + SSubmitRsp rsp = {0}; + SVCreateTbReq createTbReq = {0}; + SCoder coder = {0}; + int32_t nRows; pRsp->code = 0; // handle the request - if (tsdbInsertData(pVnode->pTsdb, version, pSubmitReq, &rsp) < 0) { - pRsp->code = terrno; - return -1; + if (tInitSubmitMsgIter(pSubmitReq, &msgIter) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + goto _exit; } - // pRsp->msgType = TDMT_VND_SUBMIT_RSP; - // vnodeProcessSubmitReq(pVnode, ptr, pRsp); + for (;;) { + tGetSubmitMsgNext(&msgIter, &pBlock); + if (pBlock == NULL) break; + // create table for auto create table mode + if (msgIter.schemaLen > 0) { + tCoderInit(&coder, TD_LITTLE_ENDIAN, pBlock->data, msgIter.schemaLen, TD_DECODER); + if (tDecodeSVCreateTbReq(&coder, &createTbReq) < 0) { + pRsp->code = TSDB_CODE_INVALID_MSG; + tCoderClear(&coder); + goto _exit; + } + + if (metaCreateTable(pVnode->pMeta, version, &createTbReq) < 0) { + if (terrno != TSDB_CODE_TDB_TABLE_ALREADY_EXIST) { + pRsp->code = terrno; + tCoderClear(&coder); + goto _exit; + } + } + + tCoderClear(&coder); + } + + if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &nRows) < 0) { + pRsp->code = terrno; + goto _exit; + } + + rsp.numOfRows += nRows; + } + +_exit: // encode the response (TODO) pRsp->pCont = rpcMallocCont(sizeof(SSubmitRsp)); memcpy(pRsp->pCont, &rsp, sizeof(rsp)); @@ -511,18 +550,3 @@ static int vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq, in return 0; } - -int32_t tsdbProcessSubmitReq(STsdb *pTsdb, int64_t version, void *pReq) { - if (!pReq) { - terrno = TSDB_CODE_INVALID_PTR; - return TSDB_CODE_FAILED; - } - - SSubmitReq *pSubmitReq = (SSubmitReq *)pReq; - - if (tsdbInsertData(pTsdb, version, pSubmitReq, NULL) < 0) { - return TSDB_CODE_FAILED; - } - - return TSDB_CODE_SUCCESS; -} diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 918892b786..3e8528e3d9 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -54,6 +54,7 @@ enum { CTG_ACT_REMOVE_DB, CTG_ACT_REMOVE_STB, CTG_ACT_REMOVE_TBL, + CTG_ACT_UPDATE_USER, CTG_ACT_MAX }; @@ -95,8 +96,18 @@ typedef struct SCtgRentMgmt { SCtgRentSlot *slots; } SCtgRentMgmt; +typedef struct SCtgUserAuth { + int32_t version; + SRWLatch lock; + bool superUser; + SHashObj *createdDbs; + SHashObj *readDbs; + SHashObj *writeDbs; +} SCtgUserAuth; + typedef struct SCatalog { uint64_t clusterId; + SHashObj *userCache; //key:user, value:SCtgUserAuth SHashObj *dbCache; //key:dbname, value:SCtgDBCache SCtgRentMgmt dbRent; SCtgRentMgmt stbRent; @@ -124,6 +135,8 @@ typedef struct SCtgCacheStat { uint64_t vgMissNum; uint64_t tblHitNum; uint64_t tblMissNum; + uint64_t userHitNum; + uint64_t userMissNum; } SCtgCacheStat; typedef struct SCatalogStat { @@ -169,6 +182,11 @@ typedef struct SCtgRemoveTblMsg { uint64_t dbId; } SCtgRemoveTblMsg; +typedef struct SCtgUpdateUserMsg { + SCatalog* pCtg; + SGetUserAuthRsp userAuth; +} SCtgUpdateUserMsg; + typedef struct SCtgMetaAction { int32_t act; @@ -234,6 +252,8 @@ typedef struct SCtgAction { #define CTG_FLAG_SYS_DB 0x8 #define CTG_FLAG_FORCE_UPDATE 0x10 +#define CTG_FLAG_SET(_flag, _v) ((_flag) |= (_v)) + #define CTG_FLAG_IS_STB(_flag) ((_flag) & CTG_FLAG_STB) #define CTG_FLAG_IS_NOT_STB(_flag) ((_flag) & CTG_FLAG_NOT_STB) #define CTG_FLAG_IS_UNKNOWN_STB(_flag) ((_flag) & CTG_FLAG_UNKNOWN_STB) diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6f1f34a57b..f485f85809 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -24,6 +24,7 @@ int32_t ctgActUpdateTbl(SCtgMetaAction *action); int32_t ctgActRemoveDB(SCtgMetaAction *action); int32_t ctgActRemoveStb(SCtgMetaAction *action); int32_t ctgActRemoveTbl(SCtgMetaAction *action); +int32_t ctgActUpdateUser(SCtgMetaAction *action); extern SCtgDebug gCTGDebug; SCatalogMgmt gCtgMgmt = {0}; @@ -51,6 +52,11 @@ SCtgAction gCtgAction[CTG_ACT_MAX] = {{ CTG_ACT_REMOVE_TBL, "remove tbMeta", ctgActRemoveTbl + }, + { + CTG_ACT_UPDATE_USER, + "update user", + ctgActUpdateUser } }; @@ -357,6 +363,31 @@ _return: CTG_RET(code); } +int32_t ctgPushUpdateUserMsgInQueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncReq) { + int32_t code = 0; + SCtgMetaAction action= {.act = CTG_ACT_UPDATE_USER, .syncReq = syncReq}; + SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); + if (NULL == msg) { + ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); + CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); + } + + msg->pCtg = pCtg; + msg->userAuth = *pAuth; + + action.data = msg; + + CTG_ERR_JRET(ctgPushAction(pCtg, &action)); + + return TSDB_CODE_SUCCESS; + +_return: + + tFreeSGetUserAuthRsp(pAuth); + taosMemoryFreeClear(msg); + + CTG_RET(code); +} int32_t ctgAcquireVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgLock); @@ -687,6 +718,43 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEp return TSDB_CODE_SUCCESS; } +int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char *user, SGetUserAuthRsp *authRsp) { + char *msg = NULL; + int32_t msgLen = 0; + + ctgDebug("try to get user auth from mnode, user:%s", user); + + int32_t code = queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)]((void *)user, &msg, 0, &msgLen); + if (code) { + ctgError("Build get user auth msg failed, code:%x, db:%s", code, user); + CTG_ERR_RET(code); + } + + SRpcMsg rpcMsg = { + .msgType = TDMT_MND_GET_USER_AUTH, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + + rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp); + if (TSDB_CODE_SUCCESS != rpcRsp.code) { + ctgError("error rsp for get user auth, error:%s, user:%s", tstrerror(rpcRsp.code), user); + CTG_ERR_RET(rpcRsp.code); + } + + code = queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)](authRsp, rpcRsp.pCont, rpcRsp.contLen); + if (code) { + ctgError("Process get user auth rsp failed, code:%x, user:%s", code, user); + CTG_ERR_RET(code); + } + + ctgDebug("Got user auth from mnode, user:%s", user); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgIsTableMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { if (NULL == pCtg->dbCache) { @@ -859,6 +927,55 @@ int32_t ctgGetTableTypeFromCache(SCatalog* pCtg, const char* dbFName, const char return TSDB_CODE_SUCCESS; } +int32_t ctgChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { + if (NULL == pCtg->userCache) { + ctgDebug("empty user auth cache, user:%s", user); + goto _return; + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + if (NULL == pUser) { + ctgDebug("user not in cache, user:%s", user); + goto _return; + } + + *inCache = true; + + ctgDebug("Got user from cache, user:%s", user); + CTG_CACHE_STAT_ADD(userHitNum, 1); + + if (pUser->superUser) { + *pass = true; + return TSDB_CODE_SUCCESS; + } + + CTG_LOCK(CTG_READ, &pUser->lock); + if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + CTG_UNLOCK(CTG_READ, &pUser->lock); + return TSDB_CODE_SUCCESS; + } + + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + + CTG_UNLOCK(CTG_READ, &pUser->lock); + + return TSDB_CODE_SUCCESS; + +_return: + + *inCache = false; + CTG_CACHE_STAT_ADD(userMissNum, 1); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetTableMetaFromMnodeImpl(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, char *dbFName, char* tbName, STableMetaOutput* output) { SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; @@ -1952,6 +2069,45 @@ _return: CTG_RET(code); } +int32_t ctgChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + bool inCache = false; + int32_t code = 0; + + *pass = false; + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, user, dbFName, type, &inCache, pass)); + + if (inCache) { + return TSDB_CODE_SUCCESS; + } + + SGetUserAuthRsp authRsp = {0}; + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pRpc, pMgmtEps, user, &authRsp)); + + if (authRsp.superAuth) { + *pass = true; + goto _return; + } + + if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { + *pass = true; + goto _return; + } + + if (authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { + *pass = true; + } + + if (authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { + *pass = true; + } + +_return: + + ctgPushUpdateUserMsgInQueue(pCtg, &authRsp, false); + + return TSDB_CODE_SUCCESS; +} int32_t ctgActUpdateVg(SCtgMetaAction *action) { @@ -2121,6 +2277,67 @@ _return: CTG_RET(code); } +int32_t ctgActUpdateUser(SCtgMetaAction *action) { + int32_t code = 0; + SCtgUpdateUserMsg *msg = action->data; + SCatalog* pCtg = msg->pCtg; + + if (NULL == pCtg->userCache) { + pCtg->userCache = taosHashInit(gCtgMgmt.cfg.maxUserCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); + if (NULL == pCtg->userCache) { + ctgError("taosHashInit %d user cache failed", gCtgMgmt.cfg.maxUserCacheNum); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); + if (NULL == pUser) { + SCtgUserAuth userAuth = {0}; + + userAuth.version = msg->userAuth.version; + userAuth.superUser = msg->userAuth.superAuth; + userAuth.createdDbs = msg->userAuth.createdDbs; + userAuth.readDbs = msg->userAuth.readDbs; + userAuth.writeDbs = msg->userAuth.writeDbs; + + if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { + ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); + CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + } + + return TSDB_CODE_SUCCESS; + } + + pUser->version = msg->userAuth.version; + + CTG_LOCK(CTG_WRITE, &pUser->lock); + + taosHashCleanup(pUser->createdDbs); + pUser->createdDbs = msg->userAuth.createdDbs; + msg->userAuth.createdDbs = NULL; + + taosHashCleanup(pUser->readDbs); + pUser->readDbs = msg->userAuth.readDbs; + msg->userAuth.readDbs = NULL; + + taosHashCleanup(pUser->writeDbs); + pUser->writeDbs = msg->userAuth.writeDbs; + msg->userAuth.writeDbs = NULL; + + CTG_UNLOCK(CTG_WRITE, &pUser->lock); + +_return: + + + taosHashCleanup(msg->userAuth.createdDbs); + taosHashCleanup(msg->userAuth.readDbs); + taosHashCleanup(msg->userAuth.writeDbs); + + taosMemoryFreeClear(msg); + + CTG_RET(code); +} + void* ctgUpdateThreadFunc(void* param) { setThreadName("catalog"); @@ -2836,6 +3053,35 @@ int32_t catalogGetExpiredDBs(SCatalog* pCtg, SDbVgVersion **dbs, uint32_t *num) CTG_API_LEAVE(ctgMetaRentGet(&pCtg->dbRent, (void **)dbs, num, sizeof(SDbVgVersion))); } +int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion **users, uint32_t *num) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == users || NULL == num) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + *num = taosHashGetSize(pCtg->userCache); + if (*num > 0) { + *users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion)); + if (NULL == *users) { + ctgError("calloc %d userAuthVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); + } + } + + uint32_t i = 0; + SCtgUserAuth *pAuth = taosHashIterate(pCtg->userCache, NULL); + while (pAuth != NULL) { + void *key = taosHashGetKey(pAuth, NULL); + strncpy((*users)[i].user, key, sizeof((*users)[i].user)); + (*users)[i].version = pAuth->version; + pAuth = taosHashIterate(pCtg->userCache, pAuth); + } + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + + int32_t catalogGetDBCfg(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* dbFName, SDbCfgInfo* pDbCfg) { CTG_API_ENTER(); @@ -2880,6 +3126,31 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogChkAuth(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, const char* user, const char* dbFName, AUTH_TYPE type, bool *pass) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pRpc || NULL == pMgmtEps || NULL == user || NULL == dbFName || NULL == pass) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(ctgChkAuth(pCtg, pRpc, pMgmtEps, user, dbFName, type, pass)); + +_return: + + CTG_API_LEAVE(code); +} + +int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pAuth) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_API_LEAVE(ctgPushUpdateUserMsgInQueue(pCtg, pAuth, false)); +} + void catalogDestroy(void) { qInfo("start to destroy catalog"); diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index 15813b3cb0..c31cabda19 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -48,8 +48,7 @@ target_include_directories( PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( - udf1 PUBLIC os -) + udf1 PUBLIC os) add_library(udf2 MODULE test/udf2.c) target_include_directories( diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 8aedd59c33..6f82542aee 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -19,9 +19,6 @@ extern "C" { #endif -//TODO replaces them with fnDebug -//#define debugPrint(...) taosPrintLog("Function", DEBUG_INFO, 135, __VA_ARGS__) -#define debugPrint(...) {fprintf(stderr, __VA_ARGS__);fprintf(stderr, "\n");} enum { UDF_TASK_SETUP = 0, UDF_TASK_CALL = 1, @@ -107,7 +104,7 @@ void* decodeUdfRequest(const void *buf, SUdfRequest* request); int32_t encodeUdfResponse(void **buf, const SUdfResponse *response); void* decodeUdfResponse(const void* buf, SUdfResponse *response); -void freeUdfColumnData(SUdfColumnData *data); +void freeUdfColumnData(SUdfColumnData *data, SUdfColumnMeta *meta); void freeUdfColumn(SUdfColumn* col); void freeUdfDataDataBlock(SUdfDataBlock *block); diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 9aac2bfe0c..5b1573c88f 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -481,8 +481,8 @@ void* decodeUdfResponse(const void* buf, SUdfResponse* rsp) { return (void*)buf; } -void freeUdfColumnData(SUdfColumnData *data) { - if (data->varLengthColumn) { +void freeUdfColumnData(SUdfColumnData *data, SUdfColumnMeta *meta) { + if (IS_VAR_DATA_TYPE(meta->type)) { taosMemoryFree(data->varLenCol.varOffsets); data->varLenCol.varOffsets = NULL; taosMemoryFree(data->varLenCol.payload); @@ -496,7 +496,7 @@ void freeUdfColumnData(SUdfColumnData *data) { } void freeUdfColumn(SUdfColumn* col) { - freeUdfColumnData(&col->colData); + freeUdfColumnData(&col->colData, &col->colMeta); } void freeUdfDataDataBlock(SUdfDataBlock *block) { @@ -528,8 +528,7 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo udfCol->colMeta.scale = col->info.scale; udfCol->colMeta.precision = col->info.precision; udfCol->colData.numOfRows = udfBlock->numOfRows; - udfCol->colData.varLengthColumn = IS_VAR_DATA_TYPE(udfCol->colMeta.type); - if (udfCol->colData.varLengthColumn) { + if (IS_VAR_DATA_TYPE(udfCol->colMeta.type)) { udfCol->colData.varLenCol.varOffsetsLen = sizeof(int32_t) * udfBlock->numOfRows; udfCol->colData.varLenCol.varOffsets = taosMemoryMalloc(udfCol->colData.varLenCol.varOffsetsLen); memcpy(udfCol->colData.varLenCol.varOffsets, col->varmeta.offset, udfCol->colData.varLenCol.varOffsetsLen); @@ -555,7 +554,7 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { block->info.numOfCols = 1; block->info.rows = udfCol->colData.numOfRows; - block->info.hasVarCol = udfCol->colData.varLengthColumn; + block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type); block->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); taosArraySetSize(block->pDataBlock, 1); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 06fa49e1c2..ba9fca2969 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -75,8 +75,8 @@ typedef struct SUdf { char path[PATH_MAX]; uv_lib_t lib; + TUdfScalarProcFunc scalarProcFunc; - TUdfFreeUdfColumnFunc freeUdfColumn; TUdfAggStartFunc aggStartFunc; TUdfAggProcessFunc aggProcFunc; @@ -106,11 +106,6 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(processFuncName, udfName); uv_dlsym(&udf->lib, processFuncName, (void **)(&udf->scalarProcFunc)); - char freeFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; - char *freeSuffix = "_free"; - strncpy(freeFuncName, processFuncName, strlen(processFuncName)); - strncat(freeFuncName, freeSuffix, strlen(freeSuffix)); - uv_dlsym(&udf->lib, freeFuncName, (void **)(&udf->freeUdfColumn)); } else if (udf->funcType == TSDB_FUNC_TYPE_AGGREGATE) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(processFuncName, udfName); @@ -215,7 +210,7 @@ void udfdProcessRequest(uv_work_t *req) { udf->scalarProcFunc(&input, &output); convertUdfColumnToDataBlock(&output, &response.callRsp.resultData); - udf->freeUdfColumn(&output); + freeUdfColumn(&output); break; } case TSDB_UDF_CALL_AGG_INIT: { diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index b2367313ae..4384d326cb 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -18,52 +18,20 @@ int32_t udf1_destroy() { } int32_t udf1(SUdfDataBlock* block, SUdfColumn *resultCol) { - SUdfColumnData *resultData = &resultCol->colData; - resultData->numOfRows = block->numOfRows; - SUdfColumnData *srcData = &block->udfCols[0]->colData; - resultData->varLengthColumn = srcData->varLengthColumn; - - if (resultData->varLengthColumn) { - resultData->varLenCol.varOffsetsLen = srcData->varLenCol.varOffsetsLen; - resultData->varLenCol.varOffsets = malloc(resultData->varLenCol.varOffsetsLen); - memcpy(resultData->varLenCol.varOffsets, srcData->varLenCol.varOffsets, srcData->varLenCol.varOffsetsLen); - - resultData->varLenCol.payloadLen = srcData->varLenCol.payloadLen; - resultData->varLenCol.payload = malloc(resultData->varLenCol.payloadLen); - memcpy(resultData->varLenCol.payload, srcData->varLenCol.payload, srcData->varLenCol.payloadLen); - } else { - resultData->fixLenCol.nullBitmapLen = srcData->fixLenCol.nullBitmapLen; - resultData->fixLenCol.nullBitmap = malloc(resultData->fixLenCol.nullBitmapLen); - memcpy(resultData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmap, srcData->fixLenCol.nullBitmapLen); - - resultData->fixLenCol.dataLen = srcData->fixLenCol.dataLen; - resultData->fixLenCol.data = malloc(resultData->fixLenCol.dataLen); - memcpy(resultData->fixLenCol.data, srcData->fixLenCol.data, srcData->fixLenCol.dataLen); - for (int32_t i = 0; i < resultData->numOfRows; ++i) { - *(resultData->fixLenCol.data + i * sizeof(int32_t)) = 88; - } - } - SUdfColumnMeta *meta = &resultCol->colMeta; meta->bytes = 4; meta->type = TSDB_DATA_TYPE_INT; meta->scale = 0; meta->precision = 0; - return 0; -} -int32_t udf1_free(SUdfColumn *col) { - SUdfColumnData *data = &col->colData; - if (data->varLengthColumn) { - free(data->varLenCol.varOffsets); - data->varLenCol.varOffsets = NULL; - free(data->varLenCol.payload); - data->varLenCol.payload = NULL; - } else { - free(data->fixLenCol.nullBitmap); - data->fixLenCol.nullBitmap = NULL; - free(data->fixLenCol.data); - data->fixLenCol.data = NULL; + SUdfColumnData *resultData = &resultCol->colData; + resultData->numOfRows = block->numOfRows; + SUdfColumnData *srcData = &block->udfCols[0]->colData; + + for (int32_t i = 0; i < resultData->numOfRows; ++i) { + int32_t luckyNum = 88; + udfColSetRow(resultCol, i, (char*)&luckyNum, false); } + return 0; } \ No newline at end of file diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 6c85d9dff5..5dcacc4354 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -1971,7 +1971,18 @@ static int32_t datumToJson(const void* pObj, SJson* pJson) { case TSDB_DATA_TYPE_DOUBLE: code = tjsonAddDoubleToObject(pJson, jkValueDatum, pNode->datum.d); break; - case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_NCHAR: { + //cJSON only support utf-8 encoding. Convert memory content to hex string. + char *buf = taosMemoryCalloc(varDataLen(pNode->datum.p) * 2 + 1, sizeof(char)); + code = taosHexEncode(varDataVal(pNode->datum.p), buf, varDataLen(pNode->datum.p)); + if(code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + return TSDB_CODE_TSC_INVALID_VALUE; + } + code = tjsonAddStringToObject(pJson, jkValueDatum, buf); + taosMemoryFree(buf); + break; + } case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: code = tjsonAddStringToObject(pJson, jkValueDatum, varDataVal(pNode->datum.p)); @@ -2074,7 +2085,26 @@ static int32_t jsonToDatum(const SJson* pJson, void* pObj) { break; } varDataSetLen(pNode->datum.p, pNode->node.resType.bytes); - code = tjsonGetStringValue(pJson, jkValueDatum, varDataVal(pNode->datum.p)); + if (TSDB_DATA_TYPE_NCHAR == pNode->node.resType.type) { + char *buf = taosMemoryCalloc(1, pNode->node.resType.bytes * 2 + VARSTR_HEADER_SIZE + 1); + if (NULL == buf) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + code = tjsonGetStringValue(pJson, jkValueDatum, buf); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + break; + } + code = taosHexDecode(buf, varDataVal(pNode->datum.p), pNode->node.resType.bytes - VARSTR_HEADER_SIZE); + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(buf); + break; + } + taosMemoryFree(buf); + } else { + code = tjsonGetStringValue(pJson, jkValueDatum, varDataVal(pNode->datum.p)); + } break; } case TSDB_DATA_TYPE_JSON: diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index c021f65090..82c5d17f10 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -363,10 +363,8 @@ SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType d CHECK_OUT_OF_MEM(func); strcpy(func->functionName, "cast"); func->node.resType = dt; - if (TSDB_DATA_TYPE_BINARY == dt.type) { - func->node.resType.bytes += 2; - } else if (TSDB_DATA_TYPE_NCHAR == dt.type) { - func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE + 2; + if (TSDB_DATA_TYPE_NCHAR == dt.type) { + func->node.resType.bytes = func->node.resType.bytes * TSDB_NCHAR_SIZE; } nodesListMakeAppend(&func->pParameterList, pExpr); return (SNode*)func; diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 152aa6a01e..e82873b923 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -22,7 +22,6 @@ #include "ttime.h" #include "ttypes.h" -// clang-format off #define NEXT_TOKEN(pSql, sToken) \ do { \ int32_t index = 0; \ @@ -248,12 +247,11 @@ static int32_t getTableMetaImpl(SInsertParseContext* pCxt, SToken* pTname, bool } else { CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); + SVgroupInfo vg; + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); } - SVgroupInfo vg; - CHECK_CODE( - catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); - CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); - return TSDB_CODE_SUCCESS; } @@ -828,12 +826,21 @@ static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { return TSDB_CODE_SUCCESS; } -static int32_t storeTableMeta(SHashObj* pHash, const char* pName, int32_t len, STableMeta* pMeta) { +static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName* pTableName, const char* pName, + int32_t len, STableMeta* pMeta) { + SVgroupInfo vg; + SParseContext* pBasicCtx = pCxt->pComCxt; + CHECK_CODE( + catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, pTableName, &vg)); + CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg))); + + pMeta->uid = tGenIdPI64(); + pMeta->vgId = vg.vgId; + STableMeta* pBackup = NULL; if (TSDB_CODE_SUCCESS != cloneTableMeta(pMeta, &pBackup)) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } - pBackup->uid = tGenIdPI64(); return taosHashPut(pHash, pName, len, &pBackup, POINTER_BYTES); } @@ -856,7 +863,7 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken) if (TSDB_SUPER_TABLE != pCxt->pTableMeta->tableType) { return buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); } - CHECK_CODE(storeTableMeta(pCxt->pSubTableHashObj, tbFName, len, pCxt->pTableMeta)); + CHECK_CODE(storeTableMeta(pCxt, pCxt->pSubTableHashObj, &name, tbFName, len, pCxt->pTableMeta)); SSchema* pTagsSchema = getTableTagSchema(pCxt->pTableMeta); setBoundColumnInfo(&pCxt->tags, pTagsSchema, getNumOfTags(pCxt->pTableMeta)); @@ -1262,9 +1269,10 @@ int32_t qBuildStmtOutput(SQuery* pQuery, SHashObj* pVgHash, SHashObj* pBlockHash return TSDB_CODE_SUCCESS; } -int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *pName, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen){ - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; +int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, SName* pName, TAOS_MULTI_BIND* bind, + char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SMsgBuf pBuf = {.buf = msgBuf, .len = msgBufLen}; SParsedDataColInfo* tags = (SParsedDataColInfo*)boundTags; if (NULL == tags) { return TSDB_CODE_QRY_APP_ERROR; @@ -1312,11 +1320,10 @@ int32_t qBindStmtTagsValue(void *pBlock, void *boundTags, int64_t suid, SName *p return TSDB_CODE_SUCCESS; } - -int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen) { - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1391,10 +1398,11 @@ int32_t qBindStmtColsValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, in return TSDB_CODE_SUCCESS; } -int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBuf, int32_t msgBufLen, int32_t colIdx, int32_t rowNum) { - STableDataBlocks *pDataBlock = (STableDataBlocks *)pBlock; - SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); - int32_t extendedRowSize = getExtendedRowSize(pDataBlock); +int32_t qBindStmtSingleColValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, int32_t msgBufLen, int32_t colIdx, + int32_t rowNum) { + STableDataBlocks* pDataBlock = (STableDataBlocks*)pBlock; + SSchema* pSchema = getTableColumnSchema(pDataBlock->pTableMeta); + int32_t extendedRowSize = getExtendedRowSize(pDataBlock); SParsedDataColInfo* spd = &pDataBlock->boundColumnInfo; SRowBuilder* pBuilder = &pDataBlock->rowBuilder; SMemParam param = {.rb = pBuilder}; @@ -1459,7 +1467,7 @@ int32_t qBindStmtSingleColValue(void *pBlock, TAOS_MULTI_BIND *bind, char *msgBu } #ifdef TD_DEBUG_PRINT_ROW - if(rowEnd) { + if (rowEnd) { STSchema* pSTSchema = tdGetSTSChemaFromSSChema(&pSchema, spd->numOfCols); tdSRowPrint(row, pSTSchema, __func__); taosMemoryFree(pSTSchema); diff --git a/source/libs/parser/src/parInsertData.c b/source/libs/parser/src/parInsertData.c index ae05d2293e..4a96301d95 100644 --- a/source/libs/parser/src/parInsertData.c +++ b/source/libs/parser/src/parInsertData.c @@ -498,14 +498,9 @@ int32_t mergeTableDataBlocks(SHashObj* pHashObj, uint8_t payloadType, SArray** p ASSERT(blkKeyInfo.pKeyTuple != NULL && pBlocks->numOfRows > 0); } - int32_t len = pBlocks->numOfRows * - (isRawPayload ? (pOneTableBlock->rowSize + expandSize) : getExtendedRowSize(pOneTableBlock)) + - sizeof(STColumn) * getNumOfColumns(pOneTableBlock->pTableMeta); - // erase the empty space reserved for binary data int32_t finalLen = trimDataBlock(dataBuf->pData + dataBuf->size, pOneTableBlock, blkKeyInfo.pKeyTuple, isRawPayload); - assert(finalLen <= len); dataBuf->size += (finalLen + sizeof(SSubmitBlk)); assert(dataBuf->size <= dataBuf->nAllocSize); diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 286e7d3d44..822c214fe5 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -181,6 +181,25 @@ int32_t queryBuildRetrieveFuncMsg(void *input, char **msg, int32_t msgSize, int3 return TSDB_CODE_SUCCESS; } +int32_t queryBuildGetUserAuthMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SGetUserAuthReq req = {0}; + strncpy(req.user, input, sizeof(req.user)); + + int32_t bufLen = tSerializeSGetUserAuthReq(NULL, 0, &req); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSGetUserAuthReq(pBuf, bufLen, &req); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + + int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; SUseDbRsp usedbRsp = {0}; @@ -419,6 +438,20 @@ int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessGetUserAuthRsp(void *output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + if (tDeserializeSGetUserAuthRsp(msg, msgSize, (SGetUserAuthRsp *)output) != 0) { + qError("tDeserializeSGetUserAuthRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + return TSDB_CODE_SUCCESS; +} + + void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; @@ -427,6 +460,8 @@ void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryBuildGetUserAuthMsg; + queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; @@ -435,6 +470,7 @@ void initQueryModuleMsgHandle() { queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryProcessGetUserAuthRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4df6148a6e..77caf67a2b 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -709,6 +709,10 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int16_t outputType = GET_PARAM_TYPE(&pOutput[0]); int64_t outputLen = GET_PARAM_BYTES(&pOutput[0]); + if (IS_VAR_DATA_TYPE(outputType)) { + outputLen += VARSTR_HEADER_SIZE; + } + char *outputBuf = taosMemoryCalloc(outputLen * pInput[0].numOfRows, 1); char *output = outputBuf; @@ -790,29 +794,30 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_NCHAR: { int32_t outputCharLen = (outputLen - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE; + int32_t len; if (inputType == TSDB_DATA_TYPE_BOOL) { char tmp[8] = {0}; - int32_t len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" ); + len = sprintf(tmp, "%.*s", outputCharLen, *(int8_t *)input ? "true" : "false" ); bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { return TSDB_CODE_FAILED; } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_BINARY) { - int32_t len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; + len = outputCharLen > varDataLen(input) ? varDataLen(input) : outputCharLen; bool ret = taosMbsToUcs4(input + VARSTR_HEADER_SIZE, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { return TSDB_CODE_FAILED; } varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = TMIN(outputLen, varDataLen(input) + VARSTR_HEADER_SIZE); - memcpy(output, input, len); - varDataSetLen(output, len - VARSTR_HEADER_SIZE); + len = TMIN(outputLen - VARSTR_HEADER_SIZE, varDataLen(input)); + memcpy(output, input, len + VARSTR_HEADER_SIZE); + varDataSetLen(output, len); } else { char tmp[400] = {0}; NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); - int32_t len = (int32_t)strlen(tmp); + len = (int32_t)strlen(tmp); len = outputCharLen > len ? len : outputCharLen; bool ret = taosMbsToUcs4(tmp, len, (TdUcs4 *)varDataVal(output), outputLen - VARSTR_HEADER_SIZE, &len); if (!ret) { @@ -820,6 +825,10 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } varDataSetLen(output, len); } + //for constant conversion, need to set proper length of pOutput description + if (len < outputLen - VARSTR_HEADER_SIZE) { + pOutput->columnData->info.bytes = len; + } break; } default: { diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index cf3cd162aa..b8198364c2 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -152,8 +152,10 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in // sink if (pTask->sinkType == TASK_SINK__TABLE) { - // - blockDebugShowData(pRes); + /*blockDebugShowData(pRes);*/ + ASSERT(pTask->tbSink.pTSchema); + SSubmitReq* pReq = tdBlockToSubmit(pRes, pTask->tbSink.pTSchema); + tPrintFixedSchemaSubmitReq(pReq, pTask->tbSink.pTSchema); } else if (pTask->sinkType == TASK_SINK__SMA) { pTask->smaSink.smaHandle(pTask->ahandle, pTask->smaSink.smaId, pRes); // @@ -274,7 +276,8 @@ int32_t tEncodeSStreamTask(SCoder* pEncoder, const SStreamTask* pTask) { } if (pTask->sinkType == TASK_SINK__TABLE) { - if (tEncodeI8(pEncoder, pTask->tbSink.reserved) < 0) return -1; + /*if (tEncodeI8(pEncoder, pTask->tbSink.reserved) < 0) return -1;*/ + if (tEncodeSSchemaWrapper(pEncoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; } else if (pTask->sinkType == TASK_SINK__SMA) { if (tEncodeI64(pEncoder, pTask->smaSink.smaId) < 0) return -1; } else if (pTask->sinkType == TASK_SINK__FETCH) { @@ -318,7 +321,10 @@ int32_t tDecodeSStreamTask(SCoder* pDecoder, SStreamTask* pTask) { } if (pTask->sinkType == TASK_SINK__TABLE) { - if (tDecodeI8(pDecoder, &pTask->tbSink.reserved) < 0) return -1; + /*if (tDecodeI8(pDecoder, &pTask->tbSink.reserved) < 0) return -1;*/ + pTask->tbSink.pSchemaWrapper = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pTask->tbSink.pSchemaWrapper == NULL) return -1; + if (tDecodeSSchemaWrapper(pDecoder, pTask->tbSink.pSchemaWrapper) < 0) return -1; } else if (pTask->sinkType == TASK_SINK__SMA) { if (tDecodeI64(pDecoder, &pTask->smaSink.smaId) < 0) return -1; } else if (pTask->sinkType == TASK_SINK__FETCH) { diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index f776fb3764..846cf6f967 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -46,9 +46,21 @@ void* rpcOpen(const SRpcInit* pInit) { pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } + uint32_t ip = 0; + if (pInit->connType == TAOS_CONN_SERVER) { + ip = taosGetIpv4FromFqdn(pInit->localFqdn); + if (ip == 0xFFFFFFFF) { + tError("invalid fqdn: %s", pInit->localFqdn); + terrno = TSDB_CODE_RPC_FQDN_ERROR; + taosMemoryFree(pRpc); + return NULL; + } + } + pRpc->connType = pInit->connType; pRpc->idleTime = pInit->idleTime; - pRpc->tcphandle = (*taosInitHandle[pRpc->connType])(0, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + pRpc->tcphandle = + (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); if (pRpc->tcphandle == NULL) { taosMemoryFree(pRpc); return NULL; diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 1bd7d0857e..98e9e67ede 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -435,7 +435,7 @@ int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_ if (minNode) { SDelayTask* minTask = container_of(minNode, SDelayTask, node); if (minTask->execTime < task->execTime) { - timeoutMs = minTask->execTime <= now ? 0 : now - minTask->execTime; + timeoutMs = minTask->execTime <= now ? 0 : minTask->execTime - now; } } diff --git a/source/libs/transport/src/transSrv.c b/source/libs/transport/src/transSrv.c index 7378ca3241..ad3f520210 100644 --- a/source/libs/transport/src/transSrv.c +++ b/source/libs/transport/src/transSrv.c @@ -817,7 +817,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->pipe[i] = (uv_pipe_t*)taosMemoryCalloc(2, sizeof(uv_pipe_t)); - uv_os_sock_t fds[2]; if (uv_socketpair(SOCK_STREAM, 0, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { goto End; @@ -841,6 +840,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, goto End; } } + if (false == taosValidIpAndPort(srv->ip, srv->port)) { + tError("failed to bind, reason: %s", terrstr()); + goto End; + } if (false == addHandleToAcceptloop(srv)) { goto End; } diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 6d90773d1b..19e4defafc 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -44,7 +44,7 @@ int wordexp(char *words, wordexp_t *pwordexp, int flags) { return -1; } - printf("parse relative path:%s to abs path:%s\n", words, pwordexp->wordPos); + // printf("parse relative path:%s to abs path:%s\n", words, pwordexp->wordPos); return 0; } diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 6aa8520082..8cac660039 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -638,6 +638,48 @@ int32_t taosKeepTcpAlive(TdSocketPtr pSocket) { return 0; } +bool taosValidIpAndPort(uint32_t ip, uint16_t port) { + struct sockaddr_in serverAdd; + SocketFd fd; + int32_t reuse; + + // printf("open tcp server socket:0x%x:%hu", ip, port); + + bzero((char *)&serverAdd, sizeof(serverAdd)); + serverAdd.sin_family = AF_INET; + serverAdd.sin_addr.s_addr = ip; + serverAdd.sin_port = (uint16_t)htons(port); + + if ((fd = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) <= 2) { + // printf("failed to open TCP socket: %d (%s)", errno, strerror(errno)); + taosCloseSocketNoCheck1(fd); + return false; + } + + TdSocketPtr pSocket = (TdSocketPtr)taosMemoryMalloc(sizeof(TdSocket)); + if (pSocket == NULL) { + taosCloseSocketNoCheck1(fd); + return false; + } + pSocket->refId = 0; + pSocket->fd = fd; + + /* set REUSEADDR option, so the portnumber can be re-used */ + reuse = 1; + if (taosSetSockOpt(pSocket, SOL_SOCKET, SO_REUSEADDR, (void *)&reuse, sizeof(reuse)) < 0) { + // printf("setsockopt SO_REUSEADDR failed: %d (%s)", errno, strerror(errno)); + taosCloseSocket(&pSocket); + return NULL; + } + /* bind socket to server address */ + if (bind(pSocket->fd, (struct sockaddr *)&serverAdd, sizeof(serverAdd)) < 0) { + // printf("bind tcp server socket failed, 0x%x:%hu(%s)", ip, port, strerror(errno)); + taosCloseSocket(&pSocket); + return false; + } + taosCloseSocket(&pSocket); + return true; +} TdSocketServerPtr taosOpenTcpServerSocket(uint32_t ip, uint16_t port) { struct sockaddr_in serverAdd; SocketFd fd; diff --git a/source/os/src/osString.c b/source/os/src/osString.c index ed596a051d..375c5001f4 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -195,6 +195,36 @@ int32_t taosUcs4len(TdUcs4 *ucs4) { return n; } +//dst buffer size should be at least 2*len + 1 +int32_t taosHexEncode(const char *src, char *dst, int32_t len) { + if (!dst) { + return -1; + } + + for (int32_t i = 0; i < len; ++i) { + sprintf(dst + i * 2, "%02x", src[i] & 0xff); + } + + return 0; +} + +int32_t taosHexDecode(const char *src, char *dst, int32_t len) { + if (!dst) { + return -1; + } + + uint16_t hn, ln, out; + for (int i = 0, j = 0; i < len * 2; i += 2, ++j ) { + hn = src[i] > '9' ? src[i] - 'A' + 10 : src[i] - '0'; + ln = src[i + 1] > '9' ? src[i + 1] - 'A' + 10 : src[i + 1] - '0'; + + out = (hn << 4) | ln; + memcpy(dst + j, &out, 1); + } + + return 0; +} + int32_t taosWcharWidth(TdWchar wchar) { return wcwidth(wchar); } int32_t taosWcharsWidth(TdWchar *pWchar, int32_t size) { return wcswidth(pWchar, size); } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index d1c5e83975..b95e822df3 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -39,6 +39,7 @@ ./test.sh -f tsim/query/explain.sim ./test.sh -f tsim/query/session.sim ./test.sh -f tsim/query/scalarNull.sim +./test.sh -f tsim/query/udf.sim # ---- qnode ./test.sh -f tsim/qnode/basic1.sim diff --git a/tests/script/sh/copy_udf.sh b/tests/script/sh/copy_udf.sh new file mode 100755 index 0000000000..e1d9ff53d2 --- /dev/null +++ b/tests/script/sh/copy_udf.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set +e +#set -x + +echo "Executing copy_udf.sh" + +SCRIPT_DIR=`dirname $0` +cd $SCRIPT_DIR/../ + +IN_TDINTERNAL="community" +if [[ "$SCRIPT_DIR" == *"$IN_TDINTERNAL"* ]]; then + cd ../../.. +else + cd ../../ +fi + +TAOS_DIR=`pwd` +UDF1_DIR=`find $TAOS_DIR -name "libudf1.so"|grep lib|head -n1` +UDF2_DIR=`find $TAOS_DIR -name "libudf2.so"|grep lib|head -n1` + +echo $UDF1_DIR +echo $UDF2_DIR + +UDF_TMP=/tmp/udf +mkdir $UDF_TMP +rm $UDF_TMP/libudf1.so +rm $UDF_TMP/libudf2.so + +echo "Copy udf shared library files to $UDF_TMP" + +cp $UDF1_DIR $UDF_TMP +cp $UDF2_DIR $UDF_TMP diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index 417e53daff..40882306c8 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -66,7 +66,7 @@ print ============= create database # | REPLICA value [1 | 3] # | WAL value [1 | 2] -sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 1 WAL 2 VGROUPS 6 SINGLE_STABLE 1 +sql create database db CACHELAST 3 COMP 0 DAYS 345600 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1440000 PRECISION 'ns' REPLICA 3 WAL 2 VGROUPS 6 SINGLE_STABLE 1 sql show databases print rows: $rows print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 @@ -86,7 +86,7 @@ endi if $data3_db != 0 then # ntables return -1 endi -if $data4_db != 1 then # replica +if $data4_db != 3 then # replica return -1 endi if $data5_db != nostrict then # strict diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim new file mode 100644 index 0000000000..8acd07cfe4 --- /dev/null +++ b/tests/script/tsim/query/udf.sim @@ -0,0 +1,48 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c wallevel -v 2 +system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1 + +print ========= start dnode1 as LEADER +system sh/exec.sh -n dnode1 -s start +sleep 2000 +sql connect + +print ======== step1 udf +system sh/copy_udf.sh +sql create database udf vgroups 3; +sql use udf; +sql show databases; + +sql create table t (ts timestamp, f int); +sql insert into t values(now, 1)(now+1s, 2); + +sql create function udf1 as '/tmp/udf/libudf1.so' outputtype int bufSize 8; +sql create aggregate function udf2 as '/tmp/udf/libudf2.so' outputtype double bufSize 8; +sql show functions; +if $rows != 2 then + return -1 +endi +sql select udf1(f) from t; +if $rows != 2 then + return -1 +endi +if $data00 != 88 then + return -1 +endi +if $data10 != 88 then + return -1 +endi + +sql select udf2(f) from t; +if $rows != 1 then + return -1 +endi +if $data00 != 2.236067977 then + return -1 +endi + +#sql drop function udf1; +#sql drop function udf2; +system sh/exec.sh -n dnode1 -s stop -x SIGKILL diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py new file mode 100644 index 0000000000..a7d17bc41e --- /dev/null +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -0,0 +1,291 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import threading +import multiprocessing as mp +from numpy.lib.function_base import insert +import taos +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import datetime as dt +import time +# constant define +WAITS = 5 # wait seconds + +class TDTestCase: + # + # --------------- main frame ------------------- + # + + def caseDescription(self): + ''' + limit and offset keyword function test cases; + case1: limit offset base function test + case2: offset return valid + ''' + return + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + # init + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + # tdSql.prepare() + # self.create_tables(); + self.ts = 1500000000000 + + + # run case + def run(self): + + # test base case + self.test_case1() + tdLog.debug(" LIMIT test_case1 ............ [OK]") + + # test advance case + # self.test_case2() + # tdLog.debug(" LIMIT test_case2 ............ [OK]") + + # stop + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + # --------------- case ------------------- + + # create tables + def create_tables(self,dbname,stbname,count): + tdSql.execute("use %s" %dbname) + tdSql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + pre_create = "create table" + sql = pre_create + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + for i in range(count): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%3000 == 0: + tdSql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + def newcur(self,host,cfg): + user = "root" + password = "taosdata" + port =6030 + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) + cur=con.cursor() + print(cur) + return cur + + def new_create_tables(self,dbname,vgroups,stbname,tcountStart,tcountStop): + host = "chenhaoran02" + buildPath = self.getBuildPath() + config = buildPath+ "../sim/dnode1/cfg/" + + tsql=self.newcur(host,config) + tsql.execute("create database %s vgroups %d"%(dbname,vgroups)) + tsql.execute("use %s" %dbname) + tsql.execute("create stable %s(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)"%stbname) + + pre_create = "create table" + sql = pre_create + tcountStop=int(tcountStop) + tcountStart=int(tcountStart) + count=tcountStop-tcountStart + + tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # print(time.time()) + exeStartTime=time.time() + # print(type(tcountStop),type(tcountStart)) + for i in range(tcountStart,tcountStop): + sql += " %s_%d using %s tags(%d)"%(stbname,i,stbname,i+1) + if i >0 and i%20000 == 0: + # print(sql) + tsql.execute(sql) + sql = pre_create + # print(time.time()) + # end sql + if sql != pre_create: + # print(sql) + tsql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedCreate=count/spendTime + # tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + return + + + + # insert data + def insert_data(self, dbname, stbname, ts_start, tcountStart,tcountStop,rowCount): + tdSql.execute("use %s" %dbname) + pre_insert = "insert into " + sql = pre_insert + tcount=tcountStop-tcountStart + allRows=tcount*rowCount + tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbname, allRows)) + exeStartTime=time.time() + for i in range(tcountStart,tcountStop): + sql += " %s_%d values "%(stbname,i) + for j in range(rowCount): + sql += "(%d, %d, 'taos_%d') "%(ts_start + j*1000, j, j) + if j >0 and j%5000 == 0: + # print(sql) + tdSql.execute(sql) + sql = "insert into %s_%d values " %(stbname,i) + # end sql + if sql != pre_insert: + # print(sql) + tdSql.execute(sql) + exeEndTime=time.time() + spendTime=exeEndTime-exeStartTime + speedInsert=allRows/spendTime + # tdLog.debug("spent %.2fs to INSERT %d rows , insert rate is %.2f rows/s... [OK]"% (spendTime,allRows,speedInsert)) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + + # test case1 base + def test_case1(self): + tdLog.debug("-----create database and tables test------- ") + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + # tdSql.execute("create database db11 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + # tdSql.execute("create stable stb1(ts timestamp, c1 int, c2 binary(10)) tags(t1 int)") + + # tdSql.execute("create database db12 vgroups 1") + # # self.create_tables("db1", "stb1", 30*10000) + # tdSql.execute("use db1") + + # t1 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(1,)) + # t2 = threading.Thread(target=self.new_create_tables("db1", "stb1", 15*10000), args=(2,)) + # t1 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", 0,count/2,)) + # t2 = mp.Process(target=self.new_create_tables, args=("db1", "stb1", count/2,count,)) + + count=50000 + vgroups=1 + threads = [] + threadNumbers=2 + for i in range(threadNumbers): + threads.append(mp.Process(target=self.new_create_tables, args=("db1%d"%i, vgroups, "stb1", 0,count,))) + start_time = time.time() + for tr in threads: + tr.start() + for tr in threads: + tr.join() + end_time = time.time() + spendTime=end_time-start_time + speedCreate=count/spendTime + tdLog.debug("spent %.2fs to create 1 stable and %d table, create speed is %.2f table/s... [OK]"% (spendTime,count,speedCreate)) + # self.new_create_tables("db1", "stb1", 15*10000) + # self.new_create_tables("db1", "stb1", 15*10000) + + # tdSql.execute("create database db4 vgroups 4") + # self.create_tables("db4", "stb4", 30*10000) + + # tdSql.execute("create database db6 vgroups 6") + # self.create_tables("db6", "stb6", 30*10000) + + # tdSql.execute("create database db8 vgroups 8") + # self.create_tables("db8", "stb8", 30*10000) + + # tdSql.execute("create database db12 vgroups 12") + # self.create_tables("db12", "stb12", 30*10000) + + # tdSql.execute("create database db16 vgroups 16") + # self.create_tables("db16", "stb16", 30*10000) + return + + # test case2 base:insert data + def test_case2(self): + + tdLog.debug("-----insert data test------- ") + # drop database + tdSql.execute("drop database if exists db1") + tdSql.execute("drop database if exists db4") + tdSql.execute("drop database if exists db6") + tdSql.execute("drop database if exists db8") + tdSql.execute("drop database if exists db12") + tdSql.execute("drop database if exists db16") + + #create database and tables; + + tdSql.execute("create database db1 vgroups 1") + self.create_tables("db1", "stb1", 1*100) + self.insert_data("db1", "stb1", self.ts, 1*50,1*10000) + + + tdSql.execute("create database db4 vgroups 4") + self.create_tables("db4", "stb4", 1*100) + self.insert_data("db4", "stb4", self.ts, 1*100,1*10000) + + tdSql.execute("create database db6 vgroups 6") + self.create_tables("db6", "stb6", 1*100) + self.insert_data("db6", "stb6", self.ts, 1*100,1*10000) + + tdSql.execute("create database db8 vgroups 8") + self.create_tables("db8", "stb8", 1*100) + self.insert_data("db8", "stb8", self.ts, 1*100,1*10000) + + tdSql.execute("create database db12 vgroups 12") + self.create_tables("db12", "stb12", 1*100) + self.insert_data("db12", "stb12", self.ts, 1*100,1*10000) + + tdSql.execute("create database db16 vgroups 16") + self.create_tables("db16", "stb16", 1*100) + self.insert_data("db16", "stb16", self.ts, 1*100,1*10000) + + return + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/2-query/log.py b/tests/system-test/2-query/log.py index e6762b2d61..6e4c292183 100644 --- a/tests/system-test/2-query/log.py +++ b/tests/system-test/2-query/log.py @@ -191,13 +191,13 @@ class TDTestCase: def support_types(self): type_error_sql_lists = [ "select log(ts ,2 ) from t1" , - "select log(c7,2 ) from t1", - "select log(c8,2 ) from t1", - "select log(c9,2 ) from t1", - "select log(ts,2 ) from ct1" , - "select log(c7,2 ) from ct1", - "select log(c8,2 ) from ct1", - "select log(c9,2 ) from ct1", + "select log(c7,c2 ) from t1", + "select log(c8,c1 ) from t1", + "select log(c9,c2 ) from t1", + "select log(ts,c7 ) from ct1" , + "select log(c7,c9 ) from ct1", + "select log(c8,c2 ) from ct1", + "select log(c9,c1 ) from ct1", "select log(ts,2 ) from ct3" , "select log(c7,2 ) from ct3", "select log(c8,2 ) from ct3", diff --git a/tests/system-test/2-query/pow.py b/tests/system-test/2-query/pow.py new file mode 100644 index 0000000000..8b0137b411 --- /dev/null +++ b/tests/system-test/2-query/pow.py @@ -0,0 +1,652 @@ +import taos +import sys +import datetime +import inspect +import math +from util.log import * +from util.sql import * +from util.cases import * + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + def init(self, conn, powSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + + def prepare_datas(self): + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + def check_result_auto_pow2(self ,origin_query , pow_query): + + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else: + elem = math.pow(elem,2) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def check_result_auto_pow1(self ,origin_query , pow_query): + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + else : + elem = pow(elem ,1) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def check_result_auto_pow__10(self ,origin_query , pow_query): + pow_result = tdSql.getResult(pow_query) + origin_result = tdSql.getResult(origin_query) + + auto_result =[] + + for row in origin_result: + row_check = [] + for elem in row: + if elem == None: + elem = None + elif elem == 0: + elem = None + else: + elem = pow(elem ,-10) + row_check.append(elem) + auto_result.append(row_check) + + check_status = True + for row_index , row in enumerate(pow_result): + for col_index , elem in enumerate(row): + if auto_result[row_index][col_index] == None and not (auto_result[row_index][col_index] == None and elem == None): + check_status = False + elif auto_result[row_index][col_index] != None and (auto_result[row_index][col_index] - elem > 0.00000001): + check_status = False + else: + pass + if not check_status: + tdLog.notice("pow function value has not as expected , sql is \"%s\" "%pow_query ) + sys.exit(1) + else: + tdLog.info("pow value check pass , it work as expected ,sql is \"%s\" "%pow_query ) + + def test_errors(self): + error_sql_lists = [ + "select pow from t1", + # "select pow(-+--+c1 ,2) from t1", + # "select +-pow(c1,2) from t1", + # "select ++-pow(c1,2) from t1", + # "select ++--pow(c1,2) from t1", + # "select - -pow(c1,2)*0 from t1", + # "select pow(tbname+1,2) from t1 ", + "select pow(123--123,2)==1 from t1", + "select pow(c1,2) as 'd1' from t1", + "select pow(c1 ,c2 ,2) from t1", + "select pow(c1 ,NULL ,2) from t1", + "select pow(, 2) from t1;", + "select pow(pow(c1, 2) ab from t1)", + "select pow(c1 ,2 ) as int from t1", + "select pow from stb1", + # "select pow(-+--+c1) from stb1", + # "select +-pow(c1) from stb1", + # "select ++-pow(c1) from stb1", + # "select ++--pow(c1) from stb1", + # "select - -pow(c1)*0 from stb1", + # "select pow(tbname+1) from stb1 ", + "select pow(123--123 ,2)==1 from stb1", + "select pow(c1 ,2) as 'd1' from stb1", + "select pow(c1 ,c2 ,2 ) from stb1", + "select pow(c1 ,NULL,2) from stb1", + "select pow(,) from stb1;", + "select pow(pow(c1 , 2) ab from stb1)", + "select pow(c1 , 2) as int from stb1" + ] + for error_sql in error_sql_lists: + tdSql.error(error_sql) + + def support_types(self): + type_error_sql_lists = [ + "select pow(ts ,2 ) from t1" , + "select pow(c7,c1 ) from t1", + "select pow(c8,c2) from t1", + "select pow(c9,c3 ) from t1", + "select pow(ts,c4 ) from ct1" , + "select pow(c7,c5 ) from ct1", + "select pow(c8,c6 ) from ct1", + "select pow(c9,c8 ) from ct1", + "select pow(ts,2 ) from ct3" , + "select pow(c7,2 ) from ct3", + "select pow(c8,2 ) from ct3", + "select pow(c9,2 ) from ct3", + "select pow(ts,2 ) from ct4" , + "select pow(c7,2 ) from ct4", + "select pow(c8,2 ) from ct4", + "select pow(c9,2 ) from ct4", + "select pow(ts,2 ) from stb1" , + "select pow(c7,2 ) from stb1", + "select pow(c8,2 ) from stb1", + "select pow(c9,2 ) from stb1" , + + "select pow(ts,2 ) from stbbb1" , + "select pow(c7,2 ) from stbbb1", + + "select pow(ts,2 ) from tbname", + "select pow(c9,2 ) from tbname" + + ] + + for type_sql in type_error_sql_lists: + tdSql.error(type_sql) + + + type_sql_lists = [ + "select pow(c1,2 ) from t1", + "select pow(c2,2 ) from t1", + "select pow(c3,2 ) from t1", + "select pow(c4,2 ) from t1", + "select pow(c5,2 ) from t1", + "select pow(c6,2 ) from t1", + + "select pow(c1,2 ) from ct1", + "select pow(c2,2 ) from ct1", + "select pow(c3,2 ) from ct1", + "select pow(c4,2 ) from ct1", + "select pow(c5,2 ) from ct1", + "select pow(c6,2 ) from ct1", + + "select pow(c1,2 ) from ct3", + "select pow(c2,2 ) from ct3", + "select pow(c3,2 ) from ct3", + "select pow(c4,2 ) from ct3", + "select pow(c5,2 ) from ct3", + "select pow(c6,2 ) from ct3", + + "select pow(c1,2 ) from stb1", + "select pow(c2,2 ) from stb1", + "select pow(c3,2 ) from stb1", + "select pow(c4,2 ) from stb1", + "select pow(c5,2 ) from stb1", + "select pow(c6,2 ) from stb1", + + "select pow(c6,2) as alisb from stb1", + "select pow(c6,2) alisb from stb1", + ] + + for type_sql in type_sql_lists: + tdSql.query(type_sql) + + def basic_pow_function(self): + + # basic query + tdSql.query("select c1 from ct3") + tdSql.checkRows(0) + tdSql.query("select c1 from t1") + tdSql.checkRows(12) + tdSql.query("select c1 from stb1") + tdSql.checkRows(25) + + # used for empty table , ct3 is empty + tdSql.query("select pow(c1 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c2 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c3 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c4 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c5 ,2) from ct3") + tdSql.checkRows(0) + tdSql.query("select pow(c6 ,2) from ct3") + tdSql.checkRows(0) + + + # # used for regular table + tdSql.query("select pow(c1 ,2) from t1") + tdSql.checkData(0, 0, None) + tdSql.checkData(1 , 0, 1.000000000) + tdSql.checkData(3 , 0, 9.000000000) + tdSql.checkData(5 , 0, None) + + tdSql.query("select c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 4, 1.11000) + tdSql.checkData(3, 3, 33) + tdSql.checkData(5, 4, None) + + tdSql.query("select ts,c1, c2, c3 , c4, c5 from t1") + tdSql.checkData(1, 5, 1.11000) + tdSql.checkData(3, 4, 33) + tdSql.checkData(5, 5, None) + + self.check_result_auto_pow2( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,2), pow(c2 ,2) ,pow(c3, 2), pow(c4 ,2), pow(c5 ,2) from t1") + self.check_result_auto_pow1( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,1), pow(c2 ,1) ,pow(c3, 1), pow(c4 ,1), pow(c5 ,1) from t1") + self.check_result_auto_pow__10( "select c1, c2, c3 , c4, c5 from t1", "select pow(c1 ,-10), pow(c2 ,-10) ,pow(c3, -10), pow(c4 ,-10), pow(c5 ,-10) from t1") + + # used for sub table + tdSql.query("select c1 ,pow(c1 ,2) from ct1") + tdSql.checkData(0, 1, 64.000000000) + tdSql.checkData(1 , 1, 49.000000000) + tdSql.checkData(3 , 1, 25.000000000) + tdSql.checkData(4 , 1, 0) + + # # test bug fix for pow(c1,c2) + + tdSql.query("select c1, c5 ,pow(c1,c5) from ct4") + tdSql.checkData(0 , 2, None) + tdSql.checkData(1 , 2, 104577724.506799981) + tdSql.checkData(2 , 2, 3684781.623933245) + tdSql.checkData(3 , 2, 152225.429759376) + tdSql.checkData(4 , 2, 7573.273783071) + + + self.check_result_auto_pow2( "select c1, c2, c3 , c4, c5 from ct1", "select pow(c1,2), pow(c2,2) ,pow(c3,2), pow(c4,2), pow(c5,2) from ct1") + self.check_result_auto_pow__10( "select c1, c2, c3 , c4, c5 from ct1", "select pow(c1,-10), pow(c2,-10) ,pow(c3,-10), pow(c4,-10), pow(c5,-10) from ct1") + + # nest query for pow functions + tdSql.query("select c1 , pow(c1,2) ,pow(pow(c1,2),2) , pow(pow(pow(c1,2),2),2) from ct1;") + tdSql.checkData(0 , 0 , 8) + tdSql.checkData(0 , 1 , 64.000000000) + tdSql.checkData(0 , 2 , 4096.000000000) + tdSql.checkData(0 , 3 , 16777216.000000000) + + tdSql.checkData(1 , 0 , 7) + tdSql.checkData(1 , 1 , 49.000000000) + tdSql.checkData(1 , 2 , 2401.000000000) + tdSql.checkData(1 , 3 , 5764801.000000000) + + tdSql.checkData(4 , 0 , 0) + tdSql.checkData(4 , 1 , 0.000000000) + tdSql.checkData(4 , 2 , 0.000000000) + tdSql.checkData(4 , 3 , 0.000000000) + + # # used for stable table + + tdSql.query("select pow(c1, 2) from stb1") + tdSql.checkRows(25) + + + # used for not exists table + tdSql.error("select pow(c1, 2) from stbbb1") + tdSql.error("select pow(c1, 2) from tbname") + tdSql.error("select pow(c1, 2) from ct5") + + # mix with common col + tdSql.query("select c1, pow(c1 ,2) from ct1") + tdSql.checkData(0 , 0 ,8) + tdSql.checkData(0 , 1 ,64.000000000) + tdSql.checkData(4 , 0 ,0) + tdSql.checkData(4 , 1 ,0.000000000) + tdSql.query("select c1, pow(c1,2) from ct4") + tdSql.checkData(0 , 0 , None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(4 , 0 ,5) + tdSql.checkData(4 , 1 ,25.000000000) + tdSql.checkData(5 , 0 ,None) + tdSql.checkData(5 , 1 ,None) + + # mix with common functions + tdSql.query("select c1, pow(c1 ,2),pow(c1,2), log(pow(c1,2) ,2) from ct4 ") + tdSql.checkData(0 , 0 ,None) + tdSql.checkData(0 , 1 ,None) + tdSql.checkData(0 , 2 ,None) + tdSql.checkData(0 , 3 ,None) + + tdSql.checkData(3 , 0 , 6) + tdSql.checkData(3 , 1 ,36.000000000) + tdSql.checkData(3 , 2 ,36.000000000) + tdSql.checkData(3 , 3 ,5.169925001) + + tdSql.query("select c1, pow(c1,1),c5, floor(c5 ) from stb1 ") + + # # mix with agg functions , not support + tdSql.error("select c1, pow(c1 ,2),c5, count(c5) from stb1 ") + tdSql.error("select c1, pow(c1 ,2),c5, count(c5) from ct1 ") + tdSql.error("select pow(c1 ,2), count(c5) from stb1 ") + tdSql.error("select pow(c1 ,2), count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from ct1 ") + tdSql.error("select c1, count(c5) from stb1 ") + + # agg functions mix with agg functions + + tdSql.query("select max(c5), count(c5) from stb1") + tdSql.query("select max(c5), count(c5) from ct1") + + + # bug fix for count + tdSql.query("select count(c1) from ct4 ") + tdSql.checkData(0,0,9) + tdSql.query("select count(*) from ct4 ") + tdSql.checkData(0,0,12) + tdSql.query("select count(c1) from stb1 ") + tdSql.checkData(0,0,22) + tdSql.query("select count(*) from stb1 ") + tdSql.checkData(0,0,25) + + # # bug fix for compute + tdSql.query("select c1, pow(c1 ,2) -0 ,pow(c1-4 ,2)-0 from ct4 ") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 64.000000000) + tdSql.checkData(1, 2, 16.000000000) + + tdSql.query(" select c1, pow(c1 ,2) -0 ,pow(c1-0.1 ,2)-0.1 from ct4") + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 0, 8) + tdSql.checkData(1, 1, 64.000000000) + tdSql.checkData(1, 2, 62.310000000) + + tdSql.query("select c1, pow(c1, -10), c2, pow(c2, -10), c3, pow(c3, -10) from ct1") + + def test_big_number(self): + + tdSql.query("select c1, pow(c1, 100000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + + tdSql.query("select c1, pow(c1, 10000000000000) from ct1") # bigint to double data overflow + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000.0) from ct1") # 10000000000000000000000000.0 is a double value + tdSql.checkData(0, 1, None) + tdSql.checkData(1, 1, None) + tdSql.checkData(4, 1, 0.000000000) + + tdSql.query("select c1, pow(c1, 10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000) from ct1") # bigint to double data overflow + + def pow_base_test(self): + + # base is an regular number ,int or double + tdSql.query("select c1, pow(c1, 2) from ct1") + tdSql.checkData(0, 1,64.000000000) + tdSql.query("select c1, pow(c1, 2.0) from ct1") + tdSql.checkData(0, 1, 64.000000000) + + tdSql.query("select c1, pow(1, 2.0) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + + # # bug for compute in functions + # tdSql.query("select c1, abs(1/0) from ct1") + # tdSql.checkData(0, 0, 8) + # tdSql.checkData(0, 1, 1) + + tdSql.query("select c1, pow(1, 2.0) from ct1") + tdSql.checkData(0, 1, 1.000000000) + tdSql.checkRows(13) + + # two cols start pow(x,y) + tdSql.query("select c1,c2, pow(c1,c2) from ct1") + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, None) + tdSql.checkData(4, 2, 1.000000000) + + tdSql.query("select c1,c2, pow(c2,c1) from ct1") + tdSql.checkData(0, 2, 3897131646727578700481513520437089271808.000000000) + tdSql.checkData(1, 2, 17217033054561120738612297152331776.000000000) + tdSql.checkData(4, 2, 1.000000000) + + tdSql.query("select c1, pow(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 256.000000000) + tdSql.checkData(1, 1, 128.000000000) + tdSql.checkData(4, 1, 1.000000000) + + tdSql.query("select c1, pow(2.0 , c1) from ct1") + tdSql.checkData(0, 1, 256.000000000) + tdSql.checkData(1, 1, 128.000000000) + tdSql.checkData(4, 1, 1.000000000) + + def abs_func_filter(self): + tdSql.execute("use db") + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.checkRows(3) + tdSql.checkData(0,0,8) + tdSql.checkData(0,1,8.000000000) + tdSql.checkData(0,2,8.000000000) + tdSql.checkData(0,3,7.900000000) + tdSql.checkData(0,4,64.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,25.000000000) + + tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkRows(1) + tdSql.checkData(0,0,5) + tdSql.checkData(0,1,5.000000000) + tdSql.checkData(0,2,5.000000000) + tdSql.checkData(0,3,4.900000000) + tdSql.checkData(0,4,25.000000000) + + tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(pow(c1,2)-0.5) from ct4 where c10 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + def insert_data1(self, tbname, ts_start, count): + pre_insert = "insert into %s values"%tbname + sql = pre_insert + tdLog.debug("doing insert table %s rows=%d ..."%(tbname, count)) + for i in range(count): + sql += " (%d,%d,%d)"%(ts_start + i*1000, i , i+1) + if i >0 and i%30000 == 0: + tdSql.execute(sql) + sql = pre_insert + # end sql + if sql != pre_insert: + tdSql.execute(sql) + + tdLog.debug("INSERT TABLE DATA ............ [OK]") + return + + # test case1 base + # def test_case1(self): + # # + # # limit base function + # # + # # base no where + # sql = "select * from t1 limit 10" + # tdSql.waitedQuery(sql, 10, WAITS) + + +# +# add case with filename +# +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 0b7a71bbe4..1dfb160987 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -22,4 +22,5 @@ python3 ./test.py -f 2-query/abs.py python3 ./test.py -f 2-query/ceil.py python3 ./test.py -f 2-query/floor.py python3 ./test.py -f 2-query/round.py -python3 ./test.py -f 2-query/log.py \ No newline at end of file +python3 ./test.py -f 2-query/log.py +python3 ./test.py -f 2-query/pow.py diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index befcfc0731..08fe2cbc32 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -141,6 +141,9 @@ int32_t shellRunCommand(char *command) { *p++ = '\\'; } break; + default: + *p++ = '\\'; + break; } *p++ = c; esc = false; diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index c8ec31c48b..345b85d896 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -21,7 +21,7 @@ static void shellWorkAsClient() { SRpcInit rpcInit = {0}; SEpSet epSet = {.inUse = 0, .numOfEps = 1}; SRpcMsg rpcRsp = {0}; - void *clientRpc = NULL; + void * clientRpc = NULL; char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)("_pwd"), strlen("_pwd"), pass); @@ -116,6 +116,7 @@ static void shellWorkAsServer() { } SRpcInit rpcInit = {0}; + memcpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn)); rpcInit.localPort = pArgs->port; rpcInit.label = "CHK"; rpcInit.numOfThreads = tsNumOfRpcThreads; @@ -126,7 +127,7 @@ static void shellWorkAsServer() { void *serverRpc = rpcOpen(&rpcInit); if (serverRpc == NULL) { - printf("failed to init net test server since %s", terrstr()); + printf("failed to init net test server since %s\n", terrstr()); } else { printf("network test server is initialized, port:%u\n", pArgs->port); taosSetSignal(SIGTERM, shellNettestHandler);