This commit is contained in:
chenhaoran 2023-10-08 17:45:14 +08:00
commit d8f927b3f2
30 changed files with 409 additions and 144 deletions

View File

@ -1,7 +1,7 @@
# curl
ExternalProject_Add(curl
URL https://curl.se/download/curl-8.2.1.tar.gz
URL_HASH MD5 b25588a43556068be05e1624e0e74d41
URL_HASH MD5=b25588a43556068be05e1624e0e74d41
DOWNLOAD_NO_PROGRESS 1
DOWNLOAD_DIR "${TD_CONTRIB_DIR}/deps-download"
#GIT_REPOSITORY https://github.com/curl/curl.git

View File

@ -56,7 +56,7 @@ database_option: {
- WAL_FSYNC_PERIOD: specifies the interval (in milliseconds) at which data is written from the WAL to disk. This parameter takes effect only when the WAL parameter is set to 2. The default value is 3000. Enter a value between 0 and 180000. The value 0 indicates that incoming data is immediately written to disk.
- MAXROWS: specifies the maximum number of rows recorded in a block. The default value is 4096.
- MINROWS: specifies the minimum number of rows recorded in a block. The default value is 100.
- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. TDengine Enterprise supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; TDengine OSS does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP).
- KEEP: specifies the time for which data is retained. Enter a value between 1 and 365000. The default value is 3650. The value of the KEEP parameter must be greater than or equal to three times of the value of the DURATION parameter. TDengine automatically deletes data that is older than the value of the KEEP parameter. You can use m (minutes), h (hours), and d (days) as the unit, for example KEEP 100h or KEEP 10d. If you do not include a unit, d is used by default. TDengine Enterprise supports [Tiered Storage](https://docs.tdengine.com/tdinternal/arch/#tiered-storage) function, thus multiple KEEP values (comma separated and up to 3 values supported, and meet keep 0 <= keep 1 <= keep 2, e.g. KEEP 100h,100d,3650d) are supported; TDengine OSS does not support Tiered Storage function (although multiple keep values are configured, they do not take effect, only the maximum keep value is used as KEEP).
- PAGES: specifies the number of pages in the metadata storage engine cache on each vnode. Enter a value greater than or equal to 64. The default value is 256. The space occupied by metadata storage on each vnode is equal to the product of the values of the PAGESIZE and PAGES parameters. The space occupied by default is 1 MB.
- PAGESIZE: specifies the size (in KB) of each page in the metadata storage engine cache on each vnode. The default value is 4. Enter a value between 1 and 16384.
- PRECISION: specifies the precision at which a database records timestamps. Enter ms for milliseconds, us for microseconds, or ns for nanoseconds. The default value is ms.

View File

@ -56,7 +56,7 @@ database_option: {
- WAL_FSYNC_PERIOD当 WAL 参数设置为 2 时,落盘的周期。默认为 3000单位毫秒。最小为 0表示每次写入立即落盘最大为 180000即三分钟。
- MAXROWS文件块中记录的最大条数默认为 4096 条。
- MINROWS文件块中记录的最小条数默认为 100 条。
- KEEP表示数据文件保存的天数缺省值为 3650取值范围 [1, 365000],且必须大于或等于 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式,如 KEEP 100h、KEEP 10d 等,支持 m分钟、h小时和 d三个单位。也可以不写单位如 KEEP 50此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间(多个以英文逗号分隔,最多 3 个,满足 keep 0 <= keep 1 <= keep 2如 KEEP 100h,100d,3650d; 社区版不支持多级存储功能(即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间)。
- KEEP表示数据文件保存的天数缺省值为 3650取值范围 [1, 365000],且必须大于或等于3倍的 DURATION 参数值。数据库会自动删除保存时间超过 KEEP 值的数据。KEEP 可以使用加单位的表示形式,如 KEEP 100h、KEEP 10d 等,支持 m分钟、h小时和 d三个单位。也可以不写单位如 KEEP 50此时默认单位为天。企业版支持[多级存储](https://docs.taosdata.com/tdinternal/arch/#%E5%A4%9A%E7%BA%A7%E5%AD%98%E5%82%A8)功能, 因此, 可以设置多个保存时间(多个以英文逗号分隔,最多 3 个,满足 keep 0 <= keep 1 <= keep 2如 KEEP 100h,100d,3650d; 社区版不支持多级存储功能(即使配置了多个保存时间, 也不会生效, KEEP 会取最大的保存时间)。
- PAGES一个 VNODE 中元数据存储引擎的缓存页个数,默认为 256最小 64。一个 VNODE 元数据存储占用 PAGESIZE \* PAGES默认情况下为 1MB 内存。
- PAGESIZE一个 VNODE 中元数据存储引擎的页大小,单位为 KB默认为 4 KB。范围为 1 到 16384即 1 KB 到 16 MB。
- PRECISION数据库的时间戳精度。ms 表示毫秒us 表示微秒ns 表示纳秒,默认 ms 毫秒。

View File

@ -51,6 +51,9 @@ typedef enum {
} EGrantType;
int32_t grantCheck(EGrantType grant);
#ifdef TD_ENTERPRISE
int32_t grantAlterActiveCode(const char* old, const char* new, char* out, int8_t type);
#endif
#ifndef GRANTS_CFG
#ifdef TD_ENTERPRISE

View File

@ -206,6 +206,11 @@ typedef struct {
bool comp;
} SMonCfg;
typedef struct {
int8_t state;
tsem_t sem;
} SDmNotifyHandle;
int32_t monInit(const SMonCfg *pCfg);
void monCleanup();
void monRecordLog(int64_t ts, ELogLevel level, const char *content);

View File

@ -541,6 +541,13 @@ int32_t* taosGetErrno();
#define TSDB_CODE_GRANT_CPU_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080B)
#define TSDB_CODE_GRANT_STABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080C)
#define TSDB_CODE_GRANT_TABLE_LIMITED TAOS_DEF_ERROR_CODE(0, 0x080D)
#define TSDB_CODE_GRANT_PAR_IVLD_ACTIVE TAOS_DEF_ERROR_CODE(0, 0x080E)
#define TSDB_CODE_GRANT_PAR_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x080F)
#define TSDB_CODE_GRANT_PAR_DEC_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0810)
#define TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0811)
#define TSDB_CODE_GRANT_GEN_IVLD_KEY TAOS_DEF_ERROR_CODE(0, 0x0812)
#define TSDB_CODE_GRANT_GEN_APP_LIMIT TAOS_DEF_ERROR_CODE(0, 0x0813)
#define TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN TAOS_DEF_ERROR_CODE(0, 0x0814)
// sync
// #define TSDB_CODE_SYN_INVALID_CONFIG TAOS_DEF_ERROR_CODE(0, 0x0900) // 2.x

View File

@ -53,21 +53,26 @@ static void *dmStatusThreadFp(void *param) {
return NULL;
}
tsem_t dmNotifySem;
static void *dmNotifyThreadFp(void *param) {
SDmNotifyHandle dmNotifyHdl = {.state = 0};
static void *dmNotifyThreadFp(void *param) {
SDnodeMgmt *pMgmt = param;
int64_t lastTime = taosGetTimestampMs();
setThreadName("dnode-notify");
if (tsem_init(&dmNotifySem, 0, 0) != 0) {
if (tsem_init(&dmNotifyHdl.sem, 0, 0) != 0) {
return NULL;
}
bool wait = true;
while (1) {
if (pMgmt->pData->dropped || pMgmt->pData->stopped) break;
tsem_wait(&dmNotifySem);
if (wait) tsem_wait(&dmNotifyHdl.sem);
atomic_store_8(&dmNotifyHdl.state, 1);
dmSendNotifyReq(pMgmt);
if (1 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 0)) {
wait = true;
continue;
}
wait = false;
}
return NULL;
@ -189,11 +194,11 @@ int32_t dmStartNotifyThread(SDnodeMgmt *pMgmt) {
void dmStopNotifyThread(SDnodeMgmt *pMgmt) {
if (taosCheckPthreadValid(pMgmt->notifyThread)) {
tsem_post(&dmNotifySem);
tsem_post(&dmNotifyHdl.sem);
taosThreadJoin(pMgmt->notifyThread, NULL);
taosThreadClear(&pMgmt->notifyThread);
}
tsem_destroy(&dmNotifySem);
tsem_destroy(&dmNotifyHdl.sem);
}
int32_t dmStartMonitorThread(SDnodeMgmt *pMgmt) {

View File

@ -47,7 +47,7 @@ void mndUserFreeObj(SUserObj *pUser);
int64_t mndGetIpWhiteVer(SMnode *pMnode);
void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock);
void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock);
int32_t mndRefreshUserIpWhiteList(SMnode *pMnode);

View File

@ -294,6 +294,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.daysToKeep0 = pNew->cfg.daysToKeep0;
pOld->cfg.daysToKeep1 = pNew->cfg.daysToKeep1;
pOld->cfg.daysToKeep2 = pNew->cfg.daysToKeep2;
pOld->cfg.keepTimeOffset = pNew->cfg.keepTimeOffset;
pOld->cfg.walFsyncPeriod = pNew->cfg.walFsyncPeriod;
pOld->cfg.walLevel = pNew->cfg.walLevel;
pOld->cfg.walRetentionPeriod = pNew->cfg.walRetentionPeriod;

View File

@ -147,7 +147,7 @@ static int32_t mndCreateDefaultDnode(SMnode *pMnode) {
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;
mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
_OVER:
mndTransDrop(pTrans);
@ -752,7 +752,7 @@ static int32_t mndCreateDnode(SMnode *pMnode, SRpcMsg *pReq, SCreateDnodeReq *pC
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
code = 0;
mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, dnodeObj.fqdn, IP_WHITE_ADD, 1);
_OVER:
mndTransDrop(pTrans);
sdbFreeRaw(pRaw);
@ -763,7 +763,9 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
SSdbRaw *pRaw = NULL;
STrans *pTrans = NULL;
SDnodeObj *pDnode = NULL;
SArray *failRecord = NULL;
bool cfgAll = pCfgReq->dnodeId == -1;
int32_t cfgAllErr = 0;
int32_t iter = 0;
SSdb *pSdb = pMnode->pSdb;
@ -777,28 +779,64 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
goto _OVER;
}
SDnodeObj tmpDnode = *pDnode;
if (action == DND_ACTIVE_CODE) {
#ifndef TD_CHECK_ACTIVE
strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN);
#else
if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
if (TSDB_CODE_DUP_KEY != terrno) {
mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
if (cfgAll) { // alter all dnodes:
if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t));
if (failRecord) taosArrayPush(failRecord, &pDnode->id);
if (0 == cfgAllErr) cfgAllErr = terrno; // output 1st terrno.
}
} else {
terrno = 0; // no action for dup active code
}
if (cfgAll) continue;
goto _OVER;
}
#endif
} else if (action == DND_CONN_ACTIVE_CODE) {
#ifndef TD_CHECK_ACTIVE
strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN);
#else
if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
if (TSDB_CODE_DUP_KEY != terrno) {
mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
if (cfgAll) {
if (!failRecord) failRecord = taosArrayInit(1, sizeof(int32_t));
if (failRecord) taosArrayPush(failRecord, &pDnode->id);
if (0 == cfgAllErr) cfgAllErr = terrno;
}
} else {
terrno = 0;
}
if (cfgAll) continue;
goto _OVER;
}
#endif
} else {
terrno = TSDB_CODE_INVALID_CFG;
goto _OVER;
}
if (!pTrans) {
pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "config-dnode");
if (!pTrans) goto _OVER;
if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER;
}
SDnodeObj tmpDnode = *pDnode;
if (action == DND_ACTIVE_CODE) {
strncpy(tmpDnode.active, pCfgReq->value, TSDB_ACTIVE_KEY_LEN);
} else if (action == DND_CONN_ACTIVE_CODE) {
strncpy(tmpDnode.connActive, pCfgReq->value, TSDB_CONN_ACTIVE_KEY_LEN);
} else {
terrno = TSDB_CODE_INVALID_CFG;
goto _OVER;
}
pRaw = mndDnodeActionEncode(&tmpDnode);
if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) goto _OVER;
(void)sdbSetRawStatus(pRaw, SDB_STATUS_READY);
pRaw = NULL;
mInfo("dnode:%d, config dnode, cfg:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle,
mInfo("dnode:%d, config dnode:%d, app:%p config:%s value:%s", pDnode->id, pCfgReq->dnodeId, pReq->info.ahandle,
pCfgReq->config, pCfgReq->value);
if (cfgAll) {
@ -816,12 +854,19 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
_OVER:
if (cfgAll) {
sdbRelease(pSdb, pDnode);
if (cfgAllErr != 0) terrno = cfgAllErr;
int32_t nFail = taosArrayGetSize(failRecord);
if (nFail > 0) {
mError("config dnode, cfg:%d, app:%p config:%s value:%s. total:%d, fail:%d", pCfgReq->dnodeId, pReq->info.ahandle,
pCfgReq->config, pCfgReq->value, iter, nFail);
}
} else {
mndReleaseDnode(pMnode, pDnode);
}
sdbCancelFetch(pSdb, pIter);
mndTransDrop(pTrans);
sdbFreeRaw(pRaw);
taosArrayDestroy(failRecord);
return terrno;
}
@ -1045,7 +1090,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
mndUpdateIpWhite(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1);
mndUpdateIpWhiteForAllUser(pMnode, TSDB_DEFAULT_USER, pDnode->fqdn, IP_WHITE_DROP, 1);
code = 0;
_OVER:
@ -1191,7 +1236,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
strcpy(dcfgReq.config, "monitor");
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag);
} else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) {
} else if (strncasecmp(cfgReq.config, "ttlpushinterval", 14) == 0) {
int32_t optLen = strlen("ttlpushinterval");
int32_t flag = -1;
int32_t code = mndMCfgGetValInt32(&cfgReq, optLen, &flag);
@ -1262,7 +1307,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) {
snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%s", cfgReq.value);
if (mndConfigDnode(pMnode, pReq, &cfgReq, opt) != 0) {
mError("dnode:%d, failed to config activeCode since %s", cfgReq.dnodeId, terrstr());
mError("dnode:%d, failed to config activeCode since %s. conf:%s, val:%s", cfgReq.dnodeId, terrstr(),
cfgReq.config, cfgReq.value);
return -1;
}
return 0;

View File

@ -131,6 +131,7 @@ void grantAdd(EGrantType grant, uint64_t value) {}
void grantRestore(EGrantType grant, uint64_t value) {}
int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
int32_t grantAlterActiveCode(const char *old, const char *new, char *out, int8_t type) { return TSDB_CODE_SUCCESS; }
#endif

View File

@ -543,6 +543,8 @@ STrans *mndAcquireTrans(SMnode *pMnode, int32_t transId) {
STrans *pTrans = sdbAcquire(pMnode->pSdb, SDB_TRANS, &transId);
if (pTrans == NULL) {
terrno = TSDB_CODE_MND_TRANS_NOT_EXIST;
} else {
taosThreadMutexInit(&pTrans->mutex, NULL);
}
return pTrans;
}

View File

@ -58,7 +58,7 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBloc
static void mndCancelGetNextPrivileges(SMnode *pMnode, void *pIter);
SHashObj *mndFetchAllIpWhite(SMnode *pMnode);
static int32_t mndProcesSRetrieveIpWhiteReq(SRpcMsg *pReq);
bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type);
bool mndUpdateIpWhiteImpl(SHashObj *pIpWhiteTab, char *user, char *fqdn, int8_t type);
void ipWhiteMgtUpdateAll(SMnode *pMnode);
typedef struct {
@ -103,7 +103,8 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) {
for (int i = 0; i < taosArrayGetSize(fqdns); i++) {
char *fqdn = taosArrayGetP(fqdns, i);
mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD);
update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, TSDB_DEFAULT_USER, fqdn, IP_WHITE_ADD);
update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, IP_WHITE_ADD);
}
for (int i = 0; i < taosArrayGetSize(fqdns); i++) {
@ -115,8 +116,7 @@ int32_t ipWhiteMgtUpdate(SMnode *pMnode, char *user, SIpWhiteList *pNew) {
// for (int i = 0; i < taosArrayGetSize(pUserNames); i++) {
// taosMemoryFree(taosArrayGetP(pUserNames, i));
// }
//taosArrayDestroy(pUserNames);
// taosArrayDestroy(pUserNames);
if (update) ipWhiteMgt.ver++;
@ -302,7 +302,7 @@ int32_t mndRefreshUserIpWhiteList(SMnode *pMnode) {
return 0;
}
void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) {
void mndUpdateIpWhiteForAllUser(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_t lock) {
if (lock) {
taosThreadRwlockWrlock(&ipWhiteMgt.rw);
if (ipWhiteMgt.ver == 0) {
@ -313,6 +313,20 @@ void mndUpdateIpWhite(SMnode *pMnode, char *user, char *fqdn, int8_t type, int8_
}
bool update = mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, user, fqdn, type);
void *pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, NULL);
while (pIter) {
size_t klen = 0;
char *key = taosHashGetKey(pIter, &klen);
char *keyDup = taosMemoryCalloc(1, klen + 1);
memcpy(keyDup, key, klen);
update |= mndUpdateIpWhiteImpl(ipWhiteMgt.pIpWhiteTab, keyDup, fqdn, type);
taosMemoryFree(keyDup);
pIter = taosHashIterate(ipWhiteMgt.pIpWhiteTab, pIter);
}
if (update) ipWhiteMgt.ver++;
if (lock) taosThreadRwlockUnlock(&ipWhiteMgt.rw);

View File

@ -132,9 +132,10 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name);
int32_t metaGetCachedTbGroup(void *pVnode, tb_uid_t suid, const uint8_t *pKey, int32_t keyLen, SArray **pList);
int32_t metaPutTbGroupToCache(void *pVnode, uint64_t suid, const void *pKey, int32_t keyLen, void *pPayload,
int32_t payloadLen);
bool metaTbInFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
int32_t metaPutTbToFilterCache(void *pVnode, tb_uid_t suid, int8_t type);
int32_t metaSizeOfTbFilterCache(void *pVnode, int8_t type);
bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type);
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type);
int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type);
int32_t metaInitTbFilterCache(SMeta *pMeta);
int32_t metaGetStbStats(void *pVnode, int64_t uid, int64_t *numOfTables, int32_t *numOfCols);

View File

@ -14,6 +14,13 @@
*/
#include "meta.h"
#ifdef TD_ENTERPRISE
extern const char* tkLogStb[];
extern const char* tkAuditStb[];
extern const int tkLogStbNum;
extern const int tkAuditStbNum;
#endif
#define TAG_FILTER_RES_KEY_LEN 32
#define META_CACHE_BASE_BUCKET 1024
#define META_CACHE_STATS_BUCKET 16
@ -69,6 +76,7 @@ struct SMetaCache {
struct STbFilterCache {
SHashObj* pStb;
SHashObj* pStbName;
} STbFilterCache;
};
@ -178,6 +186,13 @@ int32_t metaCacheOpen(SMeta* pMeta) {
goto _err2;
}
pCache->STbFilterCache.pStbName =
taosHashInit(0, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK);
if (pCache->STbFilterCache.pStbName == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err2;
}
pMeta->pCache = pCache;
return code;
@ -204,6 +219,7 @@ void metaCacheClose(SMeta* pMeta) {
taosHashCleanup(pMeta->pCache->STbGroupResCache.pTableEntry);
taosHashCleanup(pMeta->pCache->STbFilterCache.pStb);
taosHashCleanup(pMeta->pCache->STbFilterCache.pStbName);
taosMemoryFree(pMeta->pCache);
pMeta->pCache = NULL;
@ -893,30 +909,59 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
return TSDB_CODE_SUCCESS;
}
bool metaTbInFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
bool metaTbInFilterCache(SMeta *pMeta, const void* key, int8_t type) {
if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t))) {
return true;
}
if (type == 0 && taosHashGet(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid))) {
if (type == 1 && taosHashGet(pMeta->pCache->STbFilterCache.pStbName, key, strlen(key))) {
return true;
}
return false;
}
int32_t metaPutTbToFilterCache(void* pVnode, tb_uid_t suid, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
int32_t metaPutTbToFilterCache(SMeta *pMeta, const void* key, int8_t type) {
if (type == 0) {
return taosHashPut(pMeta->pCache->STbFilterCache.pStb, &suid, sizeof(suid), NULL, 0);
return taosHashPut(pMeta->pCache->STbFilterCache.pStb, key, sizeof(tb_uid_t), NULL, 0);
}
if (type == 1) {
return taosHashPut(pMeta->pCache->STbFilterCache.pStbName, key, strlen(key), NULL, 0);
}
return 0;
}
int32_t metaSizeOfTbFilterCache(void* pVnode, int8_t type) {
SMeta* pMeta = ((SVnode*)pVnode)->pMeta;
int32_t metaSizeOfTbFilterCache(SMeta *pMeta, int8_t type) {
if (type == 0) {
return taosHashGetSize(pMeta->pCache->STbFilterCache.pStb);
}
return 0;
}
}
int32_t metaInitTbFilterCache(SMeta* pMeta) {
#ifdef TD_ENTERPRISE
int32_t tbNum = 0;
const char** pTbArr = NULL;
const char* dbName = NULL;
if (!(dbName = strchr(pMeta->pVnode->config.dbname, '.'))) return 0;
if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
tbNum = tkLogStbNum;
pTbArr = (const char**)&tkLogStb;
} else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) {
tbNum = tkAuditStbNum;
pTbArr = (const char**)&tkAuditStb;
}
if (tbNum && pTbArr) {
for (int32_t i = 0; i < tbNum; ++i) {
if (metaPutTbToFilterCache(pMeta, pTbArr[i], 1) != 0) {
return terrno ? terrno : -1;
}
}
}
#else
#endif
return 0;
}

View File

@ -176,6 +176,10 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta, int8_t rollback) {
goto _err;
}
if (metaInitTbFilterCache(pMeta) != 0) {
goto _err;
}
metaDebug("vgId:%d, meta is opened", TD_VID(pVnode));
*ppMeta = pMeta;

View File

@ -15,7 +15,7 @@
#include "meta.h"
extern tsem_t dmNotifySem;
extern SDmNotifyHandle dmNotifyHdl;
static int metaSaveJsonVarToIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
static int metaDelJsonVarFromIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry, const SSchema *pSchema);
@ -28,7 +28,7 @@ static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME);
static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry);
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid);
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t *pSysTbl);
static void metaDestroyTagIdxKey(STagIdxKey *pTagIdxKey);
// opt ins_tables query
static int metaUpdateBtimeIdx(SMeta *pMeta, const SMetaEntry *pME);
@ -198,7 +198,11 @@ static inline void metaTimeSeriesNotifyCheck(SMeta *pMeta) {
#if defined(TD_ENTERPRISE) && !defined(_TD_DARWIN_64)
int64_t nTimeSeries = metaGetTimeSeriesNum(pMeta, 0);
int64_t deltaTS = nTimeSeries - pMeta->pVnode->config.vndStats.numOfReportedTimeSeries;
if (deltaTS > tsTimeSeriesThreshold) tsem_post(&dmNotifySem);
if (deltaTS > tsTimeSeriesThreshold) {
if (0 == atomic_val_compare_exchange_8(&dmNotifyHdl.state, 1, 2)) {
tsem_post(&dmNotifyHdl.sem);
}
}
#endif
}
@ -303,7 +307,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb
for (int32_t iChild = 0; iChild < taosArrayGetSize(tbUidList); iChild++) {
tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUidList, iChild);
metaDropTableByUid(pMeta, uid, NULL, NULL);
metaDropTableByUid(pMeta, uid, NULL, NULL, NULL);
}
// drop super table
@ -392,6 +396,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
nStbEntry.stbEntry.schemaTag = pReq->schemaTag;
int32_t deltaCol = pReq->schemaRow.nCols - oStbEntry.stbEntry.schemaRow.nCols;
bool updStat = deltaCol != 0 && !metaTbInFilterCache(pMeta, pReq->name, 1);
metaWLock(pMeta);
// compare two entry
@ -407,15 +412,16 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) {
// metaStatsCacheDrop(pMeta, nStbEntry.uid);
if (deltaCol != 0) {
if (updStat) {
metaUpdateStbStats(pMeta, pReq->suid, 0, deltaCol);
}
metaULock(pMeta);
if (deltaCol != 0) {
if (updStat) {
int64_t ctbNum;
metaGetStbStats(pMeta->pVnode, pReq->suid, &ctbNum, NULL);
pMeta->pVnode->config.vndStats.numOfTimeSeries += (ctbNum * deltaCol);
metaTimeSeriesNotifyCheck(pMeta);
}
_exit:
@ -760,6 +766,8 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
}
metaReaderClear(&mr);
bool sysTbl = (pReq->type == TSDB_CHILD_TABLE) && metaTbInFilterCache(pMeta, pReq->ctb.stbName, 1);
// build SMetaEntry
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
me.version = ver;
@ -796,9 +804,12 @@ int metaCreateTable(SMeta *pMeta, int64_t ver, SVCreateTbReq *pReq, STableMetaRs
#endif
++pStats->numOfCTables;
int32_t nCols = 0;
metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols);
pStats->numOfTimeSeries += nCols - 1;
if (!sysTbl) {
int32_t nCols = 0;
metaGetStbStats(pMeta->pVnode, me.ctbEntry.suid, 0, &nCols);
pStats->numOfTimeSeries += nCols - 1;
}
metaWLock(pMeta);
metaUpdateStbStats(pMeta, me.ctbEntry.suid, 1, 0);
@ -852,6 +863,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
int rc = 0;
tb_uid_t uid = 0;
tb_uid_t suid = 0;
int8_t sysTbl = 0;
int type;
rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData);
@ -862,12 +874,12 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
uid = *(tb_uid_t *)pData;
metaWLock(pMeta);
rc = metaDropTableByUid(pMeta, uid, &type, &suid);
rc = metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl);
metaULock(pMeta);
if (rc < 0) goto _exit;
if (type == TSDB_CHILD_TABLE) {
if (!sysTbl && type == TSDB_CHILD_TABLE) {
int32_t nCols = 0;
SVnodeStats *pStats = &pMeta->pVnode->config.vndStats;
if (metaGetStbStats(pMeta->pVnode, suid, NULL, &nCols) == 0) {
@ -898,9 +910,10 @@ void metaDropTables(SMeta *pMeta, SArray *tbUids) {
for (int i = 0; i < taosArrayGetSize(tbUids); ++i) {
tb_uid_t uid = *(tb_uid_t *)taosArrayGet(tbUids, i);
tb_uid_t suid = 0;
int8_t sysTbl = 0;
int type;
metaDropTableByUid(pMeta, uid, &type, &suid);
if (type == TSDB_CHILD_TABLE && suid != 0 && suidHash) {
metaDropTableByUid(pMeta, uid, &type, &suid, &sysTbl);
if (!sysTbl && type == TSDB_CHILD_TABLE && suid != 0 && suidHash) {
int64_t *pVal = tSimpleHashGet(suidHash, &suid, sizeof(tb_uid_t));
if (pVal) {
nCtbDropped = *pVal + 1;
@ -1059,7 +1072,7 @@ static int metaDeleteTtl(SMeta *pMeta, const SMetaEntry *pME) {
return ttlMgrDeleteTtl(pMeta->pTtlMgr, &ctx);
}
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid) {
static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *pSuid, int8_t* pSysTbl) {
void *pData = NULL;
int nData = 0;
int rc = 0;
@ -1088,7 +1101,6 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
void *tData = NULL;
int tLen = 0;
if (tdbTbGet(pMeta->pUidIdx, &e.ctbEntry.suid, sizeof(tb_uid_t), &tData, &tLen) == 0) {
STbDbKey tbDbKey = {.uid = e.ctbEntry.suid, .version = ((SUidIdxVal *)tData)[0].version};
if (tdbTbGet(pMeta->pTbDb, &tbDbKey, sizeof(tbDbKey), &tData, &tLen) == 0) {
@ -1098,6 +1110,8 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type, tb_uid_t *p
tDecoderInit(&tdc, tData, tLen);
metaDecodeEntry(&tdc, &stbEntry);
if (pSysTbl) *pSysTbl = metaTbInFilterCache(pMeta, stbEntry.name, 1) ? 1 : 0;
SSchema *pTagColumn = NULL;
SSchemaWrapper *pTagSchema = &stbEntry.stbEntry.schemaTag;
if (pTagSchema->nCols == 1 && pTagSchema->pSchema[0].type == TSDB_DATA_TYPE_JSON) {

View File

@ -566,42 +566,55 @@ int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) {
}
#ifdef TD_ENTERPRISE
#define TK_LOG_STB_NUM 19
static const char *tkLogStb[TK_LOG_STB_NUM] = {"cluster_info",
"data_dir",
"dnodes_info",
"d_info",
"grants_info",
"keeper_monitor",
"logs",
"log_dir",
"log_summary",
"m_info",
"taosadapter_restful_http_request_fail",
"taosadapter_restful_http_request_in_flight",
"taosadapter_restful_http_request_summary_milliseconds",
"taosadapter_restful_http_request_total",
"taosadapter_system_cpu_percent",
"taosadapter_system_mem_percent",
"temp_dir",
"vgroups_info",
"vnodes_role"};
const char *tkLogStb[] = {"cluster_info",
"data_dir",
"dnodes_info",
"d_info",
"grants_info",
"keeper_monitor",
"logs",
"log_dir",
"log_summary",
"m_info",
"taosadapter_restful_http_request_fail",
"taosadapter_restful_http_request_in_flight",
"taosadapter_restful_http_request_summary_milliseconds",
"taosadapter_restful_http_request_total",
"taosadapter_system_cpu_percent",
"taosadapter_system_mem_percent",
"temp_dir",
"vgroups_info",
"vnodes_role"};
const char *tkAuditStb[] = {"operations"};
const int tkLogStbNum = ARRAY_SIZE(tkLogStb);
const int tkAuditStbNum = ARRAY_SIZE(tkAuditStb);
// exclude stbs of taoskeeper log
static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) {
char *dbName = strchr(pVnode->config.dbname, '.');
if (!dbName || 0 != strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
return 0;
int32_t tbSize = 0;
int32_t tbNum = 0;
const char **pTbArr = NULL;
const char *dbName = NULL;
if (!(dbName = strchr(pVnode->config.dbname, '.'))) return 0;
if (0 == strncmp(++dbName, "log", TSDB_DB_NAME_LEN)) {
tbNum = tkLogStbNum;
pTbArr = (const char **)&tkLogStb;
} else if (0 == strncmp(dbName, "audit", TSDB_DB_NAME_LEN)) {
tbNum = tkAuditStbNum;
pTbArr = (const char **)&tkAuditStb;
}
int32_t tbSize = metaSizeOfTbFilterCache(pVnode, 0);
if (tbSize < TK_LOG_STB_NUM) {
for (int32_t i = 0; i < TK_LOG_STB_NUM; ++i) {
tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, tkLogStb[i]);
if (suid != 0) {
metaPutTbToFilterCache(pVnode, suid, 0);
if (tbNum && pTbArr) {
tbSize = metaSizeOfTbFilterCache(pVnode->pMeta, 0);
if (tbSize < tbNum) {
for (int32_t i = 0; i < tbNum; ++i) {
tb_uid_t suid = metaGetTableEntryUidByName(pVnode->pMeta, pTbArr[i]);
if (suid != 0) {
metaPutTbToFilterCache(pVnode->pMeta, &suid, 0);
}
}
tbSize = metaSizeOfTbFilterCache(pVnode->pMeta, 0);
}
tbSize = metaSizeOfTbFilterCache(pVnode, 0);
}
return tbSize;
@ -611,7 +624,7 @@ static int32_t vnodeGetTimeSeriesBlackList(SVnode *pVnode) {
static bool vnodeTimeSeriesFilter(void *arg1, void *arg2) {
SVnode *pVnode = (SVnode *)arg1;
if (metaTbInFilterCache(pVnode, *(tb_uid_t *)(arg2), 0)) {
if (metaTbInFilterCache(pVnode->pMeta, arg2, 0)) {
return true;
}
return false;
@ -626,9 +639,9 @@ int32_t vnodeGetTimeSeriesNum(SVnode *pVnode, int64_t *num) {
}
int32_t tbFilterSize = 0;
#ifdef TD_ENTERPRISE
#ifdef TD_ENTERPRISE
tbFilterSize = vnodeGetTimeSeriesBlackList(pVnode);
#endif
#endif
if ((!tbFilterSize && vnodeGetStbIdList(pVnode, 0, suidList) < 0) ||
(tbFilterSize && vnodeGetStbIdListByFilter(pVnode, 0, suidList, vnodeTimeSeriesFilter, pVnode) < 0)) {

View File

@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "parInt.h"
#include "parTranslater.h"
#include "parInt.h"
#include "catalog.h"
#include "cmdnodes.h"
@ -1209,37 +1209,37 @@ static EDealRes translateNormalValue(STranslateContext* pCxt, SValueNode* pVal,
break;
}
case TSDB_DATA_TYPE_VARBINARY: {
if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY){
if (pVal->node.resType.type != TSDB_DATA_TYPE_BINARY) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, pVal->literal);
}
void* data = NULL;
void* data = NULL;
uint32_t size = 0;
bool isHexChar = isHex(pVal->literal, strlen(pVal->literal));
if(isHexChar){
if(!isValidateHex(pVal->literal, strlen(pVal->literal))){
bool isHexChar = isHex(pVal->literal, strlen(pVal->literal));
if (isHexChar) {
if (!isValidateHex(pVal->literal, strlen(pVal->literal))) {
return TSDB_CODE_PAR_INVALID_VARBINARY;
}
if(taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0){
if (taosHex2Ascii(pVal->literal, strlen(pVal->literal), &data, &size) < 0) {
return TSDB_CODE_OUT_OF_MEMORY;
}
}else{
} else {
size = pVal->node.resType.bytes;
data = pVal->literal;
}
if (size + VARSTR_HEADER_SIZE > targetDt.bytes) {
if(isHexChar) taosMemoryFree(data);
if (isHexChar) taosMemoryFree(data);
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_VALUE_TOO_LONG, pVal->literal);
}
pVal->datum.p = taosMemoryCalloc(1, size + VARSTR_HEADER_SIZE);
if (NULL == pVal->datum.p) {
if(isHexChar) taosMemoryFree(data);
if (isHexChar) taosMemoryFree(data);
return generateDealNodeErrMsg(pCxt, TSDB_CODE_OUT_OF_MEMORY);
}
varDataSetLen(pVal->datum.p, size);
memcpy(varDataVal(pVal->datum.p), data, size);
if(isHexChar) taosMemoryFree(data);
if (isHexChar) taosMemoryFree(data);
break;
}
case TSDB_DATA_TYPE_VARCHAR:
@ -1789,7 +1789,6 @@ static int32_t translateBlockDistFunc(STranslateContext* pCtx, SFunctionNode* pF
return TSDB_CODE_SUCCESS;
}
static bool isStarParam(SNode* pNode) { return nodesIsStar(pNode) || nodesIsTableStar(pNode); }
static int32_t translateMultiResFunc(STranslateContext* pCxt, SFunctionNode* pFunc) {
@ -2811,7 +2810,8 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
pJoinTable->table.precision = calcJoinTablePrecision(pJoinTable);
pJoinTable->table.singleTable = joinTableIsSingleTable(pJoinTable);
code = translateExpr(pCxt, &pJoinTable->pOnCond);
pJoinTable->hasSubQuery = (nodeType(pJoinTable->pLeft) != QUERY_NODE_REAL_TABLE) || (nodeType(pJoinTable->pRight) != QUERY_NODE_REAL_TABLE);
pJoinTable->hasSubQuery = (nodeType(pJoinTable->pLeft) != QUERY_NODE_REAL_TABLE) ||
(nodeType(pJoinTable->pRight) != QUERY_NODE_REAL_TABLE);
if (nodeType(pJoinTable->pLeft) == QUERY_NODE_JOIN_TABLE) {
((SJoinTableNode*)pJoinTable->pLeft)->isLowLevelJoin = true;
}
@ -2827,7 +2827,7 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) {
return code;
}
static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList** pCols) {
static int32_t createAllColumns(STranslateContext* pCxt, bool igTags, SNodeList** pCols) {
*pCols = nodesMakeList();
if (NULL == *pCols) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_OUT_OF_MEMORY);
@ -2870,7 +2870,7 @@ static SNode* createMultiResFunc(SFunctionNode* pSrcFunc, SExprNode* pExpr) {
} else {
len = snprintf(buf, sizeof(buf) - 1, "%s(%s.%s)", pSrcFunc->functionName, pCol->tableAlias, pCol->colName);
taosCreateMD5Hash(buf, len);
strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1);
strncpy(pFunc->node.aliasName, buf, TSDB_COL_NAME_LEN - 1);
len = snprintf(buf, sizeof(buf) - 1, "%s(%s)", pSrcFunc->functionName, pCol->colName);
// note: userAlias could be truncated here
strncpy(pFunc->node.userAlias, buf, TSDB_COL_NAME_LEN - 1);
@ -2992,11 +2992,8 @@ static int32_t createTags(STranslateContext* pCxt, SNodeList** pOutput) {
return TSDB_CODE_SUCCESS;
}
#ifndef TD_ENTERPRISE
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
return TSDB_CODE_SUCCESS;
}
int32_t biRewriteSelectStar(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; }
#endif
static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) {
@ -3712,7 +3709,8 @@ static int32_t removeConstantValueFromList(SNodeList** pList) {
SNode* pNode = NULL;
WHERE_EACH(pNode, *pList) {
if (nodeType(pNode) == QUERY_NODE_VALUE ||
(nodeType(pNode) == QUERY_NODE_FUNCTION && fmIsConstantResFunc((SFunctionNode*)pNode) && fmIsScalarFunc(((SFunctionNode*)pNode)->funcId))) {
(nodeType(pNode) == QUERY_NODE_FUNCTION && fmIsConstantResFunc((SFunctionNode*)pNode) &&
fmIsScalarFunc(((SFunctionNode*)pNode)->funcId))) {
ERASE_NODE(*pList);
continue;
}
@ -4557,9 +4555,9 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
daysPerFile = (-1 == daysPerFile ? dbCfg.daysPerFile : daysPerFile);
daysToKeep0 = (-1 == daysToKeep0 ? dbCfg.daysToKeep0 : daysToKeep0);
}
if (daysPerFile > daysToKeep0) {
if (daysPerFile > daysToKeep0 / 3) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid duration value, should be keep2 >= keep1 >= keep0 >= duration");
"Invalid duration value, should be keep2 >= keep1 >= keep0 >= 3 * duration");
}
return TSDB_CODE_SUCCESS;
}
@ -8018,9 +8016,29 @@ static int32_t insertCondIntoSelectStmt(SSelectStmt* pSelect, SNode* pCond) {
if (pSelect->pWhere == NULL) {
pSelect->pWhere = pCond;
} else {
SNode* pWhere = NULL;
createLogicCondNode(pSelect->pWhere, pCond, &pWhere, LOGIC_COND_TYPE_AND);
pSelect->pWhere = pWhere;
SNodeList* pLogicCondListWhere = NULL;
SNodeList* pLogicCondList2 = NULL;
if (nodeType(pSelect->pWhere) == QUERY_NODE_LOGIC_CONDITION &&
((SLogicConditionNode*)pSelect->pWhere)->condType == LOGIC_COND_TYPE_AND) {
pLogicCondListWhere = ((SLogicConditionNode*)pSelect->pWhere)->pParameterList;
} else {
nodesListMakeAppend(&pLogicCondListWhere, pSelect->pWhere);
}
if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION &&
((SLogicConditionNode*)pCond)->condType == LOGIC_COND_TYPE_AND) {
pLogicCondList2 = ((SLogicConditionNode*)pCond)->pParameterList;
} else {
nodesListMakeAppend(&pLogicCondList2, pCond);
}
nodesListAppendList(pLogicCondListWhere, pLogicCondList2);
SLogicConditionNode* pWhere = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
pWhere->condType = LOGIC_COND_TYPE_AND;
pWhere->pParameterList = pLogicCondListWhere;
pSelect->pWhere = (SNode*)pWhere;
}
return TSDB_CODE_SUCCESS;
}
@ -8094,7 +8112,6 @@ static int32_t addShowKindCond(const SShowStmt* pShow, SSelectStmt* pSelect) {
return TSDB_CODE_SUCCESS;
}
static int32_t createShowCondition(const SShowStmt* pShow, SSelectStmt* pSelect) {
SNode* pDbCond = NULL;
SNode* pTbCond = NULL;

View File

@ -430,6 +430,13 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_QUERYTIME_LIMITED, "Query time limited by
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_CPU_LIMITED, "CPU cores limited by license")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_STABLE_LIMITED, "STable creation limited by license")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TABLE_LIMITED, "Table creation limited by license")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_ACTIVE, "Invalid active code")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_IVLD_KEY, "Invalid key to parse active code")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KEY, "Invalid key to decode active code")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_PAR_DEC_IVLD_KLEN, "Invalid klen to decode active code")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_IVLD_KEY, "Invalid key to gen active code")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_APP_LIMIT, "Limited app num to gen active code")
TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_GEN_ENC_IVLD_KLEN, "Invalid klen to encode active code")
// sync
TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout")

View File

@ -815,6 +815,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-20582.py
,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/insertMix.py -N 3
,,n,system-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/stt.py -N 3
,,n,system-test,python3 ./test.py -f eco-system/meta/database/keep_time_offset.py
#tsim test
,,y,script,./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim

View File

@ -17,7 +17,7 @@ $db = $dbPrefix . $i
$mt = $mtPrefix . $i
sql drop database if exists $db
sql create database $db duration 10 keep 20,20,20
sql create database $db duration 3 keep 20,20,20
sql use $db
sql_error alter database $db keep "20"
@ -27,8 +27,8 @@ sql_error alter database $db keep 20.0
sql_error alter database $db keep 20.0,20.0,20.0
sql_error alter database $db keep 0,0,0
sql_error alter database $db keep -1,-1,-1
sql_error alter database $db keep 9,20
sql_error alter database $db keep 9,9,9
sql_error alter database $db keep 8,20
sql_error alter database $db keep 8,9,9
sql_error alter database $db keep 20,20,19
sql_error alter database $db keep 20,19,20
sql_error alter database $db keep 20,19,19

View File

@ -17,7 +17,7 @@ $db = $dbPrefix . $i
$mt = $mtPrefix . $i
sql drop database if exists $db
sql create database $db duration 10 keep 20
sql create database $db duration 3 keep 20
sql use $db
sql select * from information_schema.ins_databases
if $rows != 3 then
@ -36,7 +36,7 @@ sql_error alter database $db keep 0,0,0
sql_error alter database $db keep 3
sql_error alter database $db keep -1,-1,-1
sql alter database $db keep 20,20
sql_error alter database $db keep 9,9,9
sql_error alter database $db keep 8,9,9
sql_error alter database $db keep 20,20,19
sql_error alter database $db keep 20,19,20
sql_error alter database $db keep 20,19,19

View File

@ -150,26 +150,26 @@ sql_error create database $db keep 12,11,12
sql_error create database $db keep 8
sql_error create database $db keep 12,11
sql_error create database $db keep 365001,365001,365001
sql create database dbk0 keep 19
sql create database dbk0 keep 39
sql select * from information_schema.ins_databases
if $rows != 3 then
return -1
endi
if $data27 != 27360m,27360m,27360m then
if $data27 != 56160m,56160m,56160m then
return -1
endi
sql drop database dbk0
sql create database dbka keep 19,20
sql create database dbka keep 39,40
sql select * from information_schema.ins_databases
if $rows != 3 then
return -1
endi
if $data27 != 27360m,28800m,28800m then
if $data27 != 56160m,57600m,57600m then
return -1
endi
sql drop database dbka
sql create database dbk1 keep 11,11,11
sql create database dbk1 duration 3 keep 11,11,11
sql select * from information_schema.ins_databases
if $rows != 3 then
return -1
@ -178,7 +178,7 @@ if $data27 != 15840m,15840m,15840m then
return -1
endi
sql drop database dbk1
sql create database dbk2 keep 11,12,13
sql create database dbk2 duration 3 keep 11,12,13
sql select * from information_schema.ins_databases
if $rows != 3 then
return -1
@ -187,7 +187,7 @@ if $data27 != 15840m,17280m,18720m then
return -1
endi
sql drop database dbk2
sql create database dbk3 keep 11,11,13
sql create database dbk3 duration 3 keep 11,11,13
sql select * from information_schema.ins_databases
if $rows != 3 then
return -1
@ -196,7 +196,7 @@ if $data27 != 15840m,15840m,18720m then
return -1
endi
sql drop database dbk3
sql create database dbk4 keep 11,13,13
sql create database dbk4 duration 3 keep 11,13,13
sql select * from information_schema.ins_databases
if $rows != 3 then
return -1

View File

@ -94,4 +94,14 @@ print $rows
if $rows != 5 then
return -1
endi
sql show child db2.tables like '%'
print $rows
if $rows != 5 then
return -1
endi
sql show normal db2.tables like '%'
print $rows
if $rows != 0 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT

View File

@ -242,11 +242,11 @@ class TDTestCase:
self.str510 = self.str255 + self.str255
tdSql.error('alter dnode 1 "activeCode" "a"')
tdSql.error('alter dnode 1 "activeCode" "' + self.str107 + '"')
tdSql.execute('alter all dnodes "activeCode" "' + self.str108 + '"')
# tdSql.error('alter all dnodes "activeCode" "' + self.str108 + '"')
tdSql.error('alter dnode 1 "activeCode" "' + self.str109 + '"')
tdSql.error('alter all dnodes "activeCode" "' + self.str510 + '"')
tdSql.query(f'select * from information_schema.ins_dnodes')
tdSql.checkEqual(tdSql.queryResult[0][8],self.str108)
tdSql.checkEqual(tdSql.queryResult[0][8],"")
tdSql.execute('alter dnode 1 "activeCode" ""')
tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes')
tdSql.checkEqual(tdSql.queryResult[0][0],"")
@ -257,14 +257,14 @@ class TDTestCase:
tdSql.error('alter all dnodes "cActiveCode" "' + self.str255 + '"')
tdSql.error('alter all dnodes "cActiveCode" "' + self.str256 + '"')
tdSql.error('alter all dnodes "cActiveCode" "' + self.str257 + '"')
tdSql.execute('alter all dnodes "cActiveCode" "' + self.str254 + '"')
# tdSql.error('alter all dnodes "cActiveCode" "' + self.str254 + '"')
tdSql.error('alter dnode 1 "cActiveCode" "' + self.str510 + '"')
tdSql.query(f'select active_code,c_active_code from information_schema.ins_dnodes')
tdSql.checkEqual(tdSql.queryResult[0][0],"")
tdSql.checkEqual(tdSql.queryResult[0][1],self.str254)
tdSql.execute('alter dnode 1 "cActiveCode" "' + self.str109 + '"')
tdSql.checkEqual(tdSql.queryResult[0][1],"")
# tdSql.error('alter dnode 1 "cActiveCode" "' + self.str109 + '"')
tdSql.query(f'show dnodes')
tdSql.checkEqual(tdSql.queryResult[0][9],self.str109)
tdSql.checkEqual(tdSql.queryResult[0][9],"")
tdSql.execute('alter all dnodes "cActiveCode" ""')
tdSql.query(f'select c_active_code from information_schema.ins_dnodes')
tdSql.checkEqual(tdSql.queryResult[0][0],'')

View File

@ -100,7 +100,7 @@ class TDTestCase:
tdDnodes.deploy(1,cfg)
tdDnodes.start(1)
tdSql.execute("create database test duration 1")
tdSql.execute("create database test duration 8h")
tdSql.execute("use test")
tdSql.execute("create table stb(ts timestamp, c int) tags(t int)")
@ -117,7 +117,7 @@ class TDTestCase:
tdLog.info("================= step3")
tdSql.execute('drop database test')
for i in range(10):
tdSql.execute("create database test%d duration 1" %(i))
tdSql.execute("create database test%d duration 8h" %(i))
tdSql.execute("use test%d" %(i))
tdSql.execute("create table tb (ts timestamp,i int)")
for j in range(10):
@ -216,7 +216,7 @@ class TDTestCase:
tdDnodes.deploy(1,cfg)
tdDnodes.start(1)
tdSql.execute('create database dbtest')
tdSql.execute('create database dbtest duration 3')
tdSql.execute('use dbtest')
tdSql.execute('create table stb (ts timestamp,c0 int) tags(t0 int)')
tdSql.execute('create table tb1 using stb tags(1)')

View File

@ -15,7 +15,7 @@ class TDTestCase:
self.stbname = "stb"
self.ctbname = "ctb"
self.keep_value = "2d,2d,2d"
self.duration_value = "1d"
self.duration_value = "16h"
self.offset_time = 5
self.sleep_time = self.offset_time*2

View File

@ -0,0 +1,66 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import re
from util.log import *
from util.cases import *
from util.sql import *
from util.common import *
from util.sqlset import *
class TDTestCase:
def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar)
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor())
self.setsql = TDSetSql()
def create_db(self):
hours = 8
# create
keep_str = f"KEEP_TIME_OFFSET {hours}"
tdSql.execute(f"create database db {keep_str}")
# check result
tdSql.query("select `keep_time_offset` from information_schema.ins_databases where name='db'")
tdSql.checkData(0, 0, hours)
# alter
hours = 4
keep_str = f"KEEP_TIME_OFFSET {hours}"
tdSql.execute(f"alter database db {keep_str}")
# check result
tdSql.query("select `keep_time_offset` from information_schema.ins_databases where name='db'")
tdSql.checkData(0, 0, hours)
def check_old_syntax(self):
# old syntax would not support again
tdSql.error("alter dnode 1 'keeptimeoffset 10';")
def run(self):
# check new syntax right
self.create_db()
# check old syntax error
self.check_old_syntax()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

View File

@ -125,7 +125,10 @@ void build_code(HuffmanTree *huffmanTree, node n, int len, unsigned long out1, u
huffmanTree->code[n->c] = (unsigned long*)malloc(2*sizeof(unsigned long));
if(len<=64)
{
(huffmanTree->code[n->c])[0] = out1 << (64 - len);
if(len == 0)
(huffmanTree->code[n->c])[0] = 0;
else
(huffmanTree->code[n->c])[0] = out1 << (64 - len);
(huffmanTree->code[n->c])[1] = out2;
}
else