diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 6145d8c00c..f0daf4b82a 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -532,7 +532,12 @@ TIMEDIFF(expr1, expr2 [, time_unit]) #### TIMETRUNCATE ```sql -TIMETRUNCATE(expr, time_unit) +TIMETRUNCATE(expr, time_unit [, ignore_timezone]) + +ignore_timezone: { + 0 + | 1 +} ``` **Description**: Truncate the input timestamp with unit specified by `time_unit` @@ -548,7 +553,10 @@ TIMETRUNCATE(expr, time_unit) 1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks) - The precision of the returned timestamp is same as the precision set for the current data base in use - If the input data is not formatted as a timestamp, the returned value is null. - +- If `1d` is used as `time_unit` to truncate the timestamp, `ignore_timezone` option can be set to indicate if the returned result is affected by client timezone or not. + For example, if client timezone is set to UTC+0800, TIMETRUNCATE('2020-01-01 23:00:00', 1d, 0) will return '2020-01-01 08:00:00'. + Otherwise, TIMETRUNCATE('2020-01-01 23:00:00', 1d, 1) will return '2020-01-01 00:00:00'. + If `ignore_timezone` option is omitted, the default value is set to 1. #### TIMEZONE diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index 54031db618..ba8dbb85d4 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -59,10 +59,10 @@ The different database framework specifications for various programming language | -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- | | **Connection Management** | Support | Support | Support | Support | Support | Support | | **Regular Query** | Support | Support | Support | Support | Support | Support | -| **Parameter Binding** | Not supported | Not supported | Not supported | Support | Not supported | Support | -| **Subscription (TMQ) ** | Not supported | Not supported | Not supported | Not supported | Not supported | Support | +| **Parameter Binding** | Not supported | Not supported | support | Support | Not supported | Support | +| **Subscription (TMQ) ** | Not supported | Not supported | support | Not supported | Not supported | Support | | **Schemaless** | Not supported | Not supported | Not supported | Not supported | Not supported | Not supported | -| **Bulk Pulling (based on WebSocket) ** | Support | Support | Not Supported | support | Not Supported | Supported | +| **Bulk Pulling (based on WebSocket) ** | Support | Support | Support | support | Support | Support | | **DataFrame** | Not supported | Support | Not supported | Not supported | Not supported | Not supported | :::warning diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index eecf564b90..a28a7cd66c 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -59,10 +59,10 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- | | **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 暂不支持 | 暂不支持 | 暂不支持 | 支持 | 暂不支持 | 支持 | -| **数据订阅（TMQ）** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 支持 | +| **参数绑定** | 暂不支持 | 暂不支持 | 支持 | 支持 | 暂不支持 | 支持 | +| **数据订阅（TMQ）** | 暂不支持 | 暂不支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | | **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | -| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | 支持 | 暂不支持 | 支持 | +| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | :::warning diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index a8a1edc9a6..afe90b8a93 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -533,7 +533,12 @@ TIMEDIFF(expr1, expr2 [, time_unit]) #### TIMETRUNCATE ```sql -TIMETRUNCATE(expr, time_unit) +TIMETRUNCATE(expr, time_unit [, ignore_timezone]) + +ignore_timezone: { + 0 + | 1 +} ``` **功能说明**：将时间戳按照指定时间单位 time_unit 进行截断。 @@ -549,6 +554,11 @@ TIMETRUNCATE(expr, time_unit) 1b(纳秒), 1u(微秒)，1a(毫秒)，1s(秒)，1m(分)，1h(小时)，1d(天), 1w(周)。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 - 输入包含不符合时间日期格式的字符串则返回 NULL。 +- 当使用 1d 作为时间单位对时间戳进行截断时， 可通过设置 ignore_timezone 参数指定返回结果的显示是否忽略客户端时区的影响。 + 例如客户端所配置时区为 UTC+0800, 则 TIMETRUNCATE('2020-01-01 23:00:00', 1d, 0) 返回结果为 '2020-01-01 08:00:00'。 + 而使用 TIMETRUNCATE('2020-01-01 23:00:00', 1d, 1) 设置忽略时区时，返回结果为 '2020-01-01 00:00:00' + ignore_timezone 如果忽略的话，则默认值为 1 。 + #### TIMEZONE @@ -1085,7 +1095,7 @@ ignore_negative: { ```sql DIFF(expr [, ignore_negative]) - + ignore_negative: { 0 | 1 diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index 5f57e203b9..1b1339340b 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -228,7 +228,7 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR newSize = 8; } - while (newSize < data->varLenCol.payloadLen + dataLen) { + while (newSize < (uint32_t)(data->varLenCol.payloadLen + dataLen)) { newSize = newSize * UDF_MEMORY_EXP_GROWTH; } @@ -248,7 +248,7 @@ static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentR data->varLenCol.payloadLen += dataLen; } } - data->numOfRows = (currentRow + 1 > data->numOfRows) ? (currentRow + 1) : data->numOfRows; + data->numOfRows = ((int32_t)(currentRow + 1) > data->numOfRows) ? (int32_t)(currentRow + 1) : data->numOfRows; return 0; } diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index b5a6ebadee..34fc480432 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -136,7 +136,7 @@ int32_t check_for_params(jobject jobj, jlong conn, jlong res) { } if ((TAOS_RES *)res == NULL) { - jniError("jobj:%p, conn:%p, res is null", jobj, (TAOS *)conn); + jniError("jobj:%p, conn:%p, param res is null", jobj, (TAOS *)conn); return JNI_RESULT_SET_NULL; } @@ -393,9 +393,8 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_freeResultSetImp( return code; } - taos_free_result((void *)res); jniDebug("jobj:%p, conn:%p, free resultset:%p", jobj, (TAOS *)con, (void *)res); - + taos_free_result((void *)res); return JNI_SUCCESS; } @@ -489,7 +488,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchRowImp(JNIEn numOfFields); return JNI_FETCH_END; } else { - jniDebug("jobj:%p, conn:%p, interrupted query", jobj, tscon); + jniDebug("jobj:%p, conn:%p, interrupted query. fetch row error code: %d, msg:%s", jobj, tscon, code, taos_errstr(result)); return JNI_RESULT_SET_NULL; } } @@ -584,7 +583,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_fetchBlockImp(JNI jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res); return JNI_FETCH_END; } else { - jniError("jobj:%p, conn:%p, query interrupted", jobj, tscon); + jniError("jobj:%p, conn:%p, query interrupted. fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres)); return JNI_RESULT_SET_NULL; } } diff --git a/source/client/src/clientTmqConnector.c b/source/client/src/clientTmqConnector.c index 42988b16fe..ccfc4980bc 100644 --- a/source/client/src/clientTmqConnector.c +++ b/source/client/src/clientTmqConnector.c @@ -303,7 +303,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_tmq_TMQConnector_fetchRawBlockImp( jniDebug("jobj:%p, conn:%p, resultset:%p, no data to retrieve", jobj, tscon, (void *)res); return JNI_FETCH_END; } else { - jniError("jobj:%p, conn:%p, query interrupted", jobj, tscon); + jniError("jobj:%p, conn:%p, query interrupted, tmq fetch block error code:%d, msg:%s", jobj, tscon, error_code, taos_errstr(tres)); return JNI_RESULT_SET_NULL; } } diff --git a/source/dnode/vnode/src/meta/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c index b2cd85e9fa..5eb27679bb 100644 --- a/source/dnode/vnode/src/meta/metaCommit.c +++ b/source/dnode/vnode/src/meta/metaCommit.c @@ -43,7 +43,13 @@ int metaBegin(SMeta *pMeta, int8_t fromSys) { TXN *metaGetTxn(SMeta *pMeta) { return pMeta->txn; } int metaCommit(SMeta *pMeta, TXN *txn) { return tdbCommit(pMeta->pEnv, txn); } int metaFinishCommit(SMeta *pMeta, TXN *txn) { return tdbPostCommit(pMeta->pEnv, txn); } -int metaPrepareAsyncCommit(SMeta *pMeta) { return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); } +int metaPrepareAsyncCommit(SMeta *pMeta) { + // return tdbPrepareAsyncCommit(pMeta->pEnv, pMeta->txn); + int code = 0; + code = tdbCommit(pMeta->pEnv, pMeta->txn); + + return code; +} // abort the meta txn int metaAbort(SMeta *pMeta) { return tdbAbort(pMeta->pEnv, pMeta->txn); } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index f9a598fec6..cd56468371 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -193,7 +193,6 @@ static void vnodePrepareCommit(SVnode *pVnode) { metaPrepareAsyncCommit(pVnode->pMeta); smaPrepareAsyncCommit(pVnode->pSma); - vnodeBufPoolUnRef(pVnode->inUse); pVnode->inUse = NULL; } @@ -283,11 +282,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { syncBeginSnapshot(pVnode->sync, pVnode->state.applied); // commit each sub-system - if (metaCommit(pVnode->pMeta, pInfo->txn) < 0) { - code = TSDB_CODE_FAILED; - TSDB_CHECK_CODE(code, lino, _exit); - } - code = tsdbCommit(pVnode->pTsdb, pInfo); TSDB_CHECK_CODE(code, lino, _exit); diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a2a827e7e5..07e480ee1d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1924,7 +1924,8 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int } static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (2 != LIST_LENGTH(pFunc->pParameterList)) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (2 != numOfParams && 3 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1935,9 +1936,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - int32_t ret = validateTimeUnitParam(dbPrec, (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1)); if (ret == TIME_UNIT_TOO_SMALL) { return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, @@ -1948,11 +1947,30 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ "TIMETRUNCATE function time unit parameter should be one of the following: [1b, 1u, 1a, 1s, 1m, 1h, 1d, 1w]"); } + if (3 == numOfParams) { + uint8_t para3Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 2))->resType.type; + if (!IS_INTEGER_TYPE(para3Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 2); + if (pValue->datum.i != 0 && pValue->datum.i != 1) { + return invaildFuncParaValueErrMsg(pErrBuf, len, pFunc->functionName); + } + } + + // add database precision as param + int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); if (code != TSDB_CODE_SUCCESS) { return code; } + // add client timezone as param + code = addTimezoneParam(pFunc->pParameterList); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d261d572f0..1de8a35308 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1174,12 +1174,35 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu } /** Time functions **/ +static int64_t offsetFromTz(char *timezone, int64_t factor) { + char *minStr = &timezone[3]; + int64_t minutes = taosStr2Int64(minStr, NULL, 10); + memset(minStr, 0, strlen(minStr)); + int64_t hours = taosStr2Int64(timezone, NULL, 10); + int64_t seconds = hours * 3600 + minutes * 60; + + return seconds * factor; + +} + int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(&pInput[0]); int64_t timeUnit, timePrec, timeVal = 0; + bool ignoreTz = true; + char timezone[20] = {0}; + GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + + int32_t timePrecIdx = 2, timeZoneIdx = 3; + if (inputNum == 5) { + timePrecIdx += 1; + timeZoneIdx += 1; + GET_TYPED_DATA(ignoreTz, bool, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + } + + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[timePrecIdx]), pInput[timePrecIdx].columnData->pData); + memcpy(timezone, varDataVal(pInput[timeZoneIdx].columnData->pData), varDataLen(pInput[timeZoneIdx].columnData->pData)); int64_t factor = TSDB_TICK_PER_SECOND(timePrec); int64_t unit = timeUnit * 1000 / factor; @@ -1294,13 +1317,29 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } case 86400000: { /* 1d */ if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS) { - timeVal = timeVal / 1000 / 86400 * 86400 * 1000; + if (ignoreTz) { + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000)) % (86400L * 1000); + } else { + timeVal = timeVal / 1000 / 86400 * 86400 * 1000; + } } else if (tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS) { - timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000; + if (ignoreTz) { + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000)) % (86400L * 1000000); + } else { + timeVal = timeVal / 1000000 / 86400 * 86400 * 1000000; + } } else if (tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { - timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000; + if (ignoreTz) { + timeVal = timeVal - (timeVal + offsetFromTz(timezone, 1000000000)) % (86400L * 1000000000); + } else { + timeVal = timeVal / 1000000000 / 86400 * 86400 * 1000000000; + } } else if (tsDigits <= TSDB_TIME_PRECISION_SEC_DIGITS) { - timeVal = timeVal * factor / factor / 86400 * 86400 * factor; + if (ignoreTz) { + timeVal = (timeVal - (timeVal + offsetFromTz(timezone, 1)) % (86400L)) * factor; + } else { + timeVal = timeVal * factor / factor / 86400 * 86400 * factor; + } } else { colDataAppendNULL(pOutput->columnData, i); continue; diff --git a/source/libs/sync/inc/syncEnv.h b/source/libs/sync/inc/syncEnv.h index 265da9703d..4dc5f58cfe 100644 --- a/source/libs/sync/inc/syncEnv.h +++ b/source/libs/sync/inc/syncEnv.h @@ -22,14 +22,12 @@ extern "C" { #include "syncInt.h" -#define TIMER_MAX_MS 0x7FFFFFFF -#define ENV_TICK_TIMER_MS 1000 -#define PING_TIMER_MS 5000 -#define ELECT_TIMER_MS_MIN 2500 -#define ELECT_TIMER_MS_MAX (ELECT_TIMER_MS_MIN * 2) -#define ELECT_TIMER_MS_RANGE (ELECT_TIMER_MS_MAX - ELECT_TIMER_MS_MIN) -#define HEARTBEAT_TIMER_MS 1000 -#define HEARTBEAT_TICK_NUM 20 +#define TIMER_MAX_MS 0x7FFFFFFF +#define ENV_TICK_TIMER_MS 1000 +#define PING_TIMER_MS 5000 +#define ELECT_TIMER_MS_MIN 2500 +#define HEARTBEAT_TIMER_MS 1000 +#define HEARTBEAT_TICK_NUM 20 typedef struct SSyncEnv { uint8_t isStart; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index a00b639a9c..b89383bef3 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1025,8 +1025,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } // timer ms init pSyncNode->pingBaseLine = PING_TIMER_MS; - pSyncNode->electBaseLine = ELECT_TIMER_MS_MIN; - pSyncNode->hbBaseLine = HEARTBEAT_TIMER_MS; + pSyncNode->electBaseLine = tsElectInterval; + pSyncNode->hbBaseLine = tsHeartbeatInterval; // init ping timer pSyncNode->pPingTimer = NULL; @@ -2384,7 +2384,7 @@ bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { continue; } - if (tsNow - recvTime > SYNC_HEART_TIMEOUT_MS) { + if (tsNow - recvTime > tsHeartbeatTimeout) { toCount++; } } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a316d78cdb..b8ecbe7515 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -548,6 +548,7 @@ _START_RECEIVER: sNTrace(pSyncNode, "snapshot receiver pre waitting for true time, now:%" PRId64 ", stime:%" PRId64, timeNow, pMsg->startTime); taosMsleep(10); + timeNow = taosGetTimestampMs(); } if (snapshotReceiverIsStart(pReceiver)) { @@ -883,9 +884,9 @@ int32_t syncNodeOnSnapshotReply(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { // receive ack is finish, close sender if (pMsg->ack == SYNC_SNAPSHOT_SEQ_END) { snapshotSenderStop(pSender, true); - SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); + SSyncLogReplMgr *pMgr = syncNodeGetLogReplMgr(pSyncNode, &pMsg->srcId); if (pMgr) { - syncLogReplMgrReset(pMgr); + syncLogReplMgrReset(pMgr); } return 0; } diff --git a/source/libs/tdb/src/db/tdbPCache.c b/source/libs/tdb/src/db/tdbPCache.c index 425e494ab6..b67fe562eb 100644 --- a/source/libs/tdb/src/db/tdbPCache.c +++ b/source/libs/tdb/src/db/tdbPCache.c @@ -184,9 +184,9 @@ SPage *tdbPCacheFetch(SPCache *pCache, const SPgid *pPgid, TXN *pTxn) { // TDB_PAGE_PGNO(pPage), pPage, nRef); if (pPage) { - tdbDebug("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); + tdbTrace("pcache/fetch page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); } else { - tdbDebug("pcache/fetch page %p", pPage); + tdbTrace("pcache/fetch page %p", pPage); } return pPage; @@ -202,7 +202,7 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { tdbPCacheLock(pCache); nRef = tdbUnrefPage(pPage); - tdbDebug("pcache/release page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); + tdbTrace("pcache/release page %p/%d/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id, nRef); if (nRef == 0) { // test the nRef again to make sure // it is safe th handle the page @@ -221,8 +221,6 @@ void tdbPCacheRelease(SPCache *pCache, SPage *pPage, TXN *pTxn) { // } } tdbPCacheUnlock(pCache); - // printf("thread %" PRId64 " relas page %d pgno %d pPage %p nRef %d\n", taosGetSelfPthreadId(), pPage->id, - // TDB_PAGE_PGNO(pPage), pPage, nRef); } int tdbPCacheGetPageSize(SPCache *pCache) { return pCache->szPage; } @@ -335,8 +333,7 @@ static void tdbPCachePinPage(SPCache *pCache, SPage *pPage) { pCache->nRecyclable--; - // printf("pin page %d pgno %d pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); - tdbDebug("pcache/pin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); + tdbTrace("pcache/pin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); } } @@ -349,7 +346,7 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { ASSERT(pPage->pLruNext == NULL); - tdbDebug("pCache:%p unpin page %p/%d/%d, nPages:%d", pCache, pPage, TDB_PAGE_PGNO(pPage), pPage->id, pCache->nPages); + tdbTrace("pCache:%p unpin page %p/%d/%d, nPages:%d", pCache, pPage, TDB_PAGE_PGNO(pPage), pPage->id, pCache->nPages); if (pPage->id < pCache->nPages) { pPage->pLruPrev = &(pCache->lru); pPage->pLruNext = pCache->lru.pLruNext; @@ -359,9 +356,9 @@ static void tdbPCacheUnpinPage(SPCache *pCache, SPage *pPage) { pCache->nRecyclable++; // printf("unpin page %d pgno %d pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); - tdbDebug("pcache/unpin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); + tdbTrace("pcache/unpin page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); } else { - tdbDebug("pcache destroy page: %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); + tdbTrace("pcache destroy page: %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); tdbPCacheRemovePageFromHash(pCache, pPage); tdbPageDestroy(pPage, tdbDefaultFree, NULL); @@ -381,7 +378,7 @@ static void tdbPCacheRemovePageFromHash(SPCache *pCache, SPage *pPage) { // printf("rmv page %d to hash, pgno %d, pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); } - tdbDebug("pcache/remove page %p/%d/%d from hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h); + tdbTrace("pcache/remove page %p/%d/%d from hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h); } static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { @@ -392,8 +389,7 @@ static void tdbPCacheAddPageToHash(SPCache *pCache, SPage *pPage) { pCache->nPage++; - // printf("add page %d to hash, pgno %d, pPage %p\n", pPage->id, TDB_PAGE_PGNO(pPage), pPage); - tdbDebug("pcache/add page %p/%d/%d to hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h); + tdbTrace("pcache/add page %p/%d/%d to hash %" PRIu32, pPage, TDB_PAGE_PGNO(pPage), pPage->id, h); } static int tdbPCacheOpenImpl(SPCache *pCache) { diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index 0653b6fc36..ac2725d8ec 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -81,7 +81,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) ASSERT(xFree); for (int iOvfl = 0; iOvfl < pPage->nOverflow; iOvfl++) { - tdbDebug("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); + tdbTrace("tdbPage/destroy/free ovfl cell: %p/%p", pPage->apOvfl[iOvfl], pPage); tdbOsFree(pPage->apOvfl[iOvfl]); } @@ -92,7 +92,7 @@ int tdbPageDestroy(SPage *pPage, void (*xFree)(void *arg, void *ptr), void *arg) } void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { - tdbDebug("page/zero: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); + tdbTrace("page/zero: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); pPage->pPageHdr = pPage->pData + szAmHdr; TDB_PAGE_NCELLS_SET(pPage, 0); TDB_PAGE_CCELLS_SET(pPage, pPage->pageSize - sizeof(SPageFtr)); @@ -109,7 +109,7 @@ void tdbPageZero(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell } void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell *, int, TXN *, SBTree *pBt)) { - tdbDebug("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); + tdbTrace("page/init: %p %" PRIu8 " %p", pPage, szAmHdr, xCellSize); pPage->pPageHdr = pPage->pData + szAmHdr; pPage->pCellIdx = pPage->pPageHdr + TDB_PAGE_HDR_SIZE(pPage); pPage->pFreeStart = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * TDB_PAGE_NCELLS(pPage); @@ -154,7 +154,7 @@ int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pCell, szCell); - tdbDebug("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage); + tdbTrace("tdbPage/insert/new ovfl cell: %p/%p", pNewCell, pPage); pPage->apOvfl[iOvfl] = pNewCell; pPage->aiOvfl[iOvfl] = idx; @@ -204,7 +204,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { if (pPage->aiOvfl[iOvfl] == idx) { // remove the over flow cell tdbOsFree(pPage->apOvfl[iOvfl]); - tdbDebug("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]); + tdbTrace("tdbPage/drop/free ovfl cell: %p", pPage->apOvfl[iOvfl]); for (; (++iOvfl) < pPage->nOverflow;) { pPage->aiOvfl[iOvfl - 1] = pPage->aiOvfl[iOvfl] - 1; pPage->apOvfl[iOvfl - 1] = pPage->apOvfl[iOvfl]; @@ -257,7 +257,7 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); pNewCell = (SCell *)tdbOsMalloc(szCell); memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); - tdbDebug("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); + tdbTrace("tdbPage/copy/new ovfl cell: %p/%p/%p", pNewCell, pToPage, pFromPage); } pToPage->apOvfl[iOvfl] = pNewCell; diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index 7f0fdf9efc..f638ec25a3 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -239,7 +239,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // ref page one more time so the page will not be release tdbRefPage(pPage); - tdbDebug("pager/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); + tdbTrace("pager/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id); // Set page as dirty pPage->isDirty = 1; diff --git a/tests/pytest/crash_gen/crash_gen_main.py b/tests/pytest/crash_gen/crash_gen_main.py index 4140728dcd..3c39b05e69 100755 --- a/tests/pytest/crash_gen/crash_gen_main.py +++ b/tests/pytest/crash_gen/crash_gen_main.py @@ -2022,11 +2022,10 @@ class TdSuperTable: conf.set("group.id", "tg2") conf.set("td.connect.user", "root") conf.set("td.connect.pass", "taosdata") - conf.set("enable.auto. - ", "true") - def tmq_commit_cb_print(tmq, resp, offset, param=None): - print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") - conf.set_auto_commit_cb(tmq_commit_cb_print, None) +# conf.set("enable.auto.commit", "true") +# def tmq_commit_cb_print(tmq, resp, offset, param=None): +# print(f"commit: {resp}, tmq: {tmq}, offset: {offset}, param: {param}") +# conf.set_auto_commit_cb(tmq_commit_cb_print, None) consumer = conf.new_consumer() topic_list = TaosTmqList() for topic in current_topic_list: diff --git a/tests/system-test/2-query/timetruncate.py b/tests/system-test/2-query/timetruncate.py index 32d6ef98e9..a59180c2b1 100644 --- a/tests/system-test/2-query/timetruncate.py +++ b/tests/system-test/2-query/timetruncate.py @@ -30,7 +30,7 @@ class TDTestCase: self.stbname = f'{self.dbname}.stb' self.ctbname = f'{self.dbname}.ctb' - def check_ms_timestamp(self,unit,date_time): + def check_ms_timestamp(self,unit,date_time, ignore_tz): if unit.lower() == '1a': for i in range(len(self.ts_str)): ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0])) @@ -50,13 +50,17 @@ class TDTestCase: elif unit.lower() == '1d': for i in range(len(self.ts_str)): ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0])) - tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24)*24*60*60*1000) + if ignore_tz == 0: + tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24)*24*60*60*1000) + else: + # assuming the client timezone is UTC+0800 + tdSql.checkEqual(ts_result,int(date_time[i] - (date_time[i] + 8 * 3600 * 1000) % (86400 * 1000))) elif unit.lower() == '1w': for i in range(len(self.ts_str)): ts_result = self.get_time.get_ms_timestamp(str(tdSql.queryResult[i][0])) tdSql.checkEqual(ts_result,int(date_time[i]/1000/60/60/24/7)*7*24*60*60*1000) - def check_us_timestamp(self,unit,date_time): + def check_us_timestamp(self,unit,date_time, ignore_tz): if unit.lower() == '1u': for i in range(len(self.ts_str)): ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0])) @@ -80,13 +84,17 @@ class TDTestCase: elif unit.lower() == '1d': for i in range(len(self.ts_str)): ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0])) - tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24)*24*60*60*1000*1000 ) + if ignore_tz == 0: + tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24)*24*60*60*1000*1000 ) + else: + # assuming the client timezone is UTC+0800 + tdSql.checkEqual(ts_result,int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000) % (86400 * 1000000))) elif unit.lower() == '1w': for i in range(len(self.ts_str)): ts_result = self.get_time.get_us_timestamp(str(tdSql.queryResult[i][0])) tdSql.checkEqual(ts_result,int(date_time[i]/1000/1000/60/60/24/7)*7*24*60*60*1000*1000) - def check_ns_timestamp(self,unit,date_time): + def check_ns_timestamp(self,unit,date_time, ignore_tz): if unit.lower() == '1b': for i in range(len(self.ts_str)): if self.rest_tag != 'rest': @@ -114,21 +122,26 @@ class TDTestCase: elif unit.lower() == '1d': for i in range(len(self.ts_str)): if self.rest_tag != 'rest': - tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24)*24*60*60*1000*1000*1000 ) + if ignore_tz == 0: + tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24)*24*60*60*1000*1000*1000 ) + else: + # assuming the client timezone is UTC+0800 + tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i] - (date_time[i] + 8 * 3600 * 1000000) % (86400 * 1000000))) elif unit.lower() == '1w': for i in range(len(self.ts_str)): if self.rest_tag != 'rest': tdSql.checkEqual(tdSql.queryResult[i][0],int(date_time[i]*1000/1000/1000/1000/1000/60/60/24/7)*7*24*60*60*1000*1000*1000) - def check_tb_type(self,unit,tb_type): + def check_tb_type(self,unit,tb_type,ignore_tz): if tb_type.lower() == 'ntb': - tdSql.query(f'select timetruncate(ts,{unit}) from {self.ntbname}') + tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.ntbname}') elif tb_type.lower() == 'ctb': - tdSql.query(f'select timetruncate(ts,{unit}) from {self.ctbname}') + tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.ctbname}') elif tb_type.lower() == 'stb': - tdSql.query(f'select timetruncate(ts,{unit}) from {self.stbname}') + tdSql.query(f'select timetruncate(ts,{unit},{ignore_tz}) from {self.stbname}') def data_check(self,date_time,precision,tb_type): + tz_options = [0, 1] for unit in self.time_unit: if (unit.lower() == '1u' and precision.lower() == 'ms') or (unit.lower() == '1b' and precision.lower() == 'us') or (unit.lower() == '1b' and precision.lower() == 'ms'): if tb_type.lower() == 'ntb': @@ -138,17 +151,20 @@ class TDTestCase: elif tb_type.lower() == 'stb': tdSql.error(f'select timetruncate(ts,{unit}) from {self.stbname}') elif precision.lower() == 'ms': - self.check_tb_type(unit,tb_type) - tdSql.checkRows(len(self.ts_str)) - self.check_ms_timestamp(unit,date_time) + for ignore_tz in tz_options: + self.check_tb_type(unit,tb_type,ignore_tz) + tdSql.checkRows(len(self.ts_str)) + self.check_ms_timestamp(unit,date_time,ignore_tz) elif precision.lower() == 'us': - self.check_tb_type(unit,tb_type) - tdSql.checkRows(len(self.ts_str)) - self.check_us_timestamp(unit,date_time) + for ignore_tz in tz_options: + self.check_tb_type(unit,tb_type,ignore_tz) + tdSql.checkRows(len(self.ts_str)) + self.check_us_timestamp(unit,date_time,ignore_tz) elif precision.lower() == 'ns': - self.check_tb_type(unit,tb_type) - tdSql.checkRows(len(self.ts_str)) - self.check_ns_timestamp(unit,date_time) + for ignore_tz in tz_options: + self.check_tb_type(unit,tb_type, ignore_tz) + tdSql.checkRows(len(self.ts_str)) + self.check_ns_timestamp(unit,date_time,ignore_tz) for unit in self.error_unit: if tb_type.lower() == 'ntb': tdSql.error(f'select timetruncate(ts,{unit}) from {self.ntbname}') @@ -181,8 +197,8 @@ class TDTestCase: date_time = self.get_time.time_transform(self.ts_str,precision) self.data_check(date_time,precision,'ctb') self.data_check(date_time,precision,'stb') - - + + def run(self): self.function_check_ntb() self.function_check_stb() diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 5be46fe54a..986806fdd8 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -113,6 +113,13 @@ int32_t shellRunSingleCommand(char *command) { } void shellRecordCommandToHistory(char *command) { + if (strncasecmp(command, "create user ", 12) == 0 || strncasecmp(command, "alter user ", 11) == 0) { + if (taosStrCaseStr(command, " pass ")) { + // have password command forbid record to history because security + return; + } + } + SShellHistory *pHistory = &shell.history; if (pHistory->hstart == pHistory->hend || pHistory->hist[(pHistory->hend + SHELL_MAX_HISTORY_SIZE - 1) % SHELL_MAX_HISTORY_SIZE] == NULL || @@ -135,7 +142,7 @@ int32_t shellRunCommand(char *command, bool recordHistory) { } // add help or help; - if(strcmp(command, "help") == 0 || strcmp(command, "help;") == 0) { + if(strncasecmp(command, "help;", 5) == 0) { showHelp(); return 0; }