xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/3_liaohj

This commit is contained in:
Haojun Liao 2023-10-30 11:57:54 +08:00
parent 980f6fc46e 53e9c483c9
commit b0700d6095
26 changed files with 4679 additions and 4218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
deps/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/arm/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/darwin/x64/dm_static/libdmodule.a vendored
View File

Binary file not shown.

BIN
deps/win/x64/dm_static/dmodule.lib vendored
View File

Binary file not shown.

BIN
deps/x86/dm_static/libdmodule.a vendored
View File

Binary file not shown.

3
include/client/taos.h
View File

@ -338,8 +338,11 @@ typedef struct tmq_raw_data {
DLL_EXPORT int32_t tmq_get_raw(TAOS_RES *res, tmq_raw_data *raw);
DLL_EXPORT int32_t tmq_write_raw(TAOS *taos, tmq_raw_data raw);
DLL_EXPORT int taos_write_raw_block(TAOS *taos, int numOfRows, char *pData, const char *tbname);
DLL_EXPORT int taos_write_raw_block_with_reqid(TAOS *taos, int numOfRows, char *pData, const char *tbname, int64_t reqid);
DLL_EXPORT int taos_write_raw_block_with_fields(TAOS *taos, int rows, char *pData, const char *tbname,
TAOS_FIELD *fields, int numFields);
DLL_EXPORT int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
TAOS_FIELD *fields, int numFields, int64_t reqid);
DLL_EXPORT void tmq_free_raw(tmq_raw_data raw);
// Returning null means error. Returned result need to be freed by tmq_free_json_meta

4
include/common/tgrant.h
View File

@ -51,11 +51,7 @@ typedef enum {
} EGrantType;
int32_t grantCheck(EGrantType grant);
#ifndef TD_GRANT_OPTIMIZE
int32_t grantAlterActiveCode(const char* old, const char* newer, char* out, int8_t type);
#else
int32_t grantAlterActiveCode(int32_t did, const char* old, const char* newer, char* out, int8_t type);
#endif
#ifndef GRANTS_CFG
#ifdef TD_ENTERPRISE

3
include/common/tmsg.h
View File

@ -1568,6 +1568,9 @@ typedef struct {
typedef struct {
int32_t id;
int8_t isMnode;
#ifdef TD_GRANT_HB_OPTIMIZE
int8_t offlineReason;
#endif
SEp ep;
char active[TSDB_ACTIVE_KEY_LEN];
char connActive[TSDB_CONN_ACTIVE_KEY_LEN];

229
include/common/ttokendef.h
View File

@ -246,120 +246,121 @@
#define TK_INSERT 227
#define TK_NULL 228
#define TK_NK_QUESTION 229
#define TK_NK_ARROW 230
#define TK_ROWTS 231
#define TK_QSTART 232
#define TK_QEND 233
#define TK_QDURATION 234
#define TK_WSTART 235
#define TK_WEND 236
#define TK_WDURATION 237
#define TK_IROWTS 238
#define TK_ISFILLED 239
#define TK_CAST 240
#define TK_NOW 241
#define TK_TODAY 242
#define TK_TIMEZONE 243
#define TK_CLIENT_VERSION 244
#define TK_SERVER_VERSION 245
#define TK_SERVER_STATUS 246
#define TK_CURRENT_USER 247
#define TK_CASE 248
#define TK_WHEN 249
#define TK_THEN 250
#define TK_ELSE 251
#define TK_BETWEEN 252
#define TK_IS 253
#define TK_NK_LT 254
#define TK_NK_GT 255
#define TK_NK_LE 256
#define TK_NK_GE 257
#define TK_NK_NE 258
#define TK_MATCH 259
#define TK_NMATCH 260
#define TK_CONTAINS 261
#define TK_IN 262
#define TK_JOIN 263
#define TK_INNER 264
#define TK_SELECT 265
#define TK_NK_HINT 266
#define TK_DISTINCT 267
#define TK_WHERE 268
#define TK_PARTITION 269
#define TK_BY 270
#define TK_SESSION 271
#define TK_STATE_WINDOW 272
#define TK_EVENT_WINDOW 273
#define TK_SLIDING 274
#define TK_FILL 275
#define TK_VALUE 276
#define TK_VALUE_F 277
#define TK_NONE 278
#define TK_PREV 279
#define TK_NULL_F 280
#define TK_LINEAR 281
#define TK_NEXT 282
#define TK_HAVING 283
#define TK_RANGE 284
#define TK_EVERY 285
#define TK_ORDER 286
#define TK_SLIMIT 287
#define TK_SOFFSET 288
#define TK_LIMIT 289
#define TK_OFFSET 290
#define TK_ASC 291
#define TK_NULLS 292
#define TK_ABORT 293
#define TK_AFTER 294
#define TK_ATTACH 295
#define TK_BEFORE 296
#define TK_BEGIN 297
#define TK_BITAND 298
#define TK_BITNOT 299
#define TK_BITOR 300
#define TK_BLOCKS 301
#define TK_CHANGE 302
#define TK_COMMA 303
#define TK_CONCAT 304
#define TK_CONFLICT 305
#define TK_COPY 306
#define TK_DEFERRED 307
#define TK_DELIMITERS 308
#define TK_DETACH 309
#define TK_DIVIDE 310
#define TK_DOT 311
#define TK_EACH 312
#define TK_FAIL 313
#define TK_FILE 314
#define TK_FOR 315
#define TK_GLOB 316
#define TK_ID 317
#define TK_IMMEDIATE 318
#define TK_IMPORT 319
#define TK_INITIALLY 320
#define TK_INSTEAD 321
#define TK_ISNULL 322
#define TK_KEY 323
#define TK_MODULES 324
#define TK_NK_BITNOT 325
#define TK_NK_SEMI 326
#define TK_NOTNULL 327
#define TK_OF 328
#define TK_PLUS 329
#define TK_PRIVILEGE 330
#define TK_RAISE 331
#define TK_RESTRICT 332
#define TK_ROW 333
#define TK_SEMI 334
#define TK_STAR 335
#define TK_STATEMENT 336
#define TK_STRICT 337
#define TK_STRING 338
#define TK_TIMES 339
#define TK_VALUES 340
#define TK_VARIABLE 341
#define TK_VIEW 342
#define TK_WAL 343
#define TK_NK_ALIAS 230
#define TK_NK_ARROW 231
#define TK_ROWTS 232
#define TK_QSTART 233
#define TK_QEND 234
#define TK_QDURATION 235
#define TK_WSTART 236
#define TK_WEND 237
#define TK_WDURATION 238
#define TK_IROWTS 239
#define TK_ISFILLED 240
#define TK_CAST 241
#define TK_NOW 242
#define TK_TODAY 243
#define TK_TIMEZONE 244
#define TK_CLIENT_VERSION 245
#define TK_SERVER_VERSION 246
#define TK_SERVER_STATUS 247
#define TK_CURRENT_USER 248
#define TK_CASE 249
#define TK_WHEN 250
#define TK_THEN 251
#define TK_ELSE 252
#define TK_BETWEEN 253
#define TK_IS 254
#define TK_NK_LT 255
#define TK_NK_GT 256
#define TK_NK_LE 257
#define TK_NK_GE 258
#define TK_NK_NE 259
#define TK_MATCH 260
#define TK_NMATCH 261
#define TK_CONTAINS 262
#define TK_IN 263
#define TK_JOIN 264
#define TK_INNER 265
#define TK_SELECT 266
#define TK_NK_HINT 267
#define TK_DISTINCT 268
#define TK_WHERE 269
#define TK_PARTITION 270
#define TK_BY 271
#define TK_SESSION 272
#define TK_STATE_WINDOW 273
#define TK_EVENT_WINDOW 274
#define TK_SLIDING 275
#define TK_FILL 276
#define TK_VALUE 277
#define TK_VALUE_F 278
#define TK_NONE 279
#define TK_PREV 280
#define TK_NULL_F 281
#define TK_LINEAR 282
#define TK_NEXT 283
#define TK_HAVING 284
#define TK_RANGE 285
#define TK_EVERY 286
#define TK_ORDER 287
#define TK_SLIMIT 288
#define TK_SOFFSET 289
#define TK_LIMIT 290
#define TK_OFFSET 291
#define TK_ASC 292
#define TK_NULLS 293
#define TK_ABORT 294
#define TK_AFTER 295
#define TK_ATTACH 296
#define TK_BEFORE 297
#define TK_BEGIN 298
#define TK_BITAND 299
#define TK_BITNOT 300
#define TK_BITOR 301
#define TK_BLOCKS 302
#define TK_CHANGE 303
#define TK_COMMA 304
#define TK_CONCAT 305
#define TK_CONFLICT 306
#define TK_COPY 307
#define TK_DEFERRED 308
#define TK_DELIMITERS 309
#define TK_DETACH 310
#define TK_DIVIDE 311
#define TK_DOT 312
#define TK_EACH 313
#define TK_FAIL 314
#define TK_FILE 315
#define TK_FOR 316
#define TK_GLOB 317
#define TK_ID 318
#define TK_IMMEDIATE 319
#define TK_IMPORT 320
#define TK_INITIALLY 321
#define TK_INSTEAD 322
#define TK_ISNULL 323
#define TK_KEY 324
#define TK_MODULES 325
#define TK_NK_BITNOT 326
#define TK_NK_SEMI 327
#define TK_NOTNULL 328
#define TK_OF 329
#define TK_PLUS 330
#define TK_PRIVILEGE 331
#define TK_RAISE 332
#define TK_RESTRICT 333
#define TK_ROW 334
#define TK_SEMI 335
#define TK_STAR 336
#define TK_STATEMENT 337
#define TK_STRICT 338
#define TK_STRING 339
#define TK_TIMES 340
#define TK_VALUES 341
#define TK_VARIABLE 342
#define TK_VIEW 343
#define TK_WAL 344

13
source/client/src/clientRawBlockWrite.c
View File

@ -1338,6 +1338,11 @@ end:
int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const char* tbname, TAOS_FIELD* fields,
int numFields) {
return taos_write_raw_block_with_fields_with_reqid(taos, rows, pData, tbname, fields, numFields, 0);
}
int taos_write_raw_block_with_fields_with_reqid(TAOS *taos, int rows, char *pData, const char *tbname,
TAOS_FIELD *fields, int numFields, int64_t reqid){
if (!taos || !pData || !tbname) {
return TSDB_CODE_INVALID_PARA;
}
@ -1347,7 +1352,7 @@ int taos_write_raw_block_with_fields(TAOS* taos, int rows, char* pData, const ch
SHashObj* pVgHash = NULL;
uDebug("taos_write_raw_block_with_fields called");
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid);
if (!pRequest) {
uError("WriteRaw:createRequest error request is null");
code = terrno;
@ -1427,6 +1432,10 @@ end:
}
int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname) {
return taos_write_raw_block_with_reqid(taos, rows, pData, tbname, 0);
}
int taos_write_raw_block_with_reqid(TAOS* taos, int rows, char* pData, const char* tbname, int64_t reqid) {
if (!taos || !pData || !tbname) {
return TSDB_CODE_INVALID_PARA;
}
@ -1436,7 +1445,7 @@ int taos_write_raw_block(TAOS* taos, int rows, char* pData, const char* tbname)
SHashObj* pVgHash = NULL;
uDebug("taos_write_raw_block called");
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, 0);
SRequestObj* pRequest = (SRequestObj*)createRequest(*(int64_t*)taos, TSDB_SQL_INSERT, reqid);
if (!pRequest) {
uError("WriteRaw:createRequest error request is null");
code = terrno;

2
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -163,7 +163,7 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;

8
source/dnode/mgmt/node_mgmt/inc/dmMgmt.h
View File

@ -97,7 +97,11 @@ int32_t dmMarkWrapper(SMgmtWrapper *pWrapper);
void dmReleaseWrapper(SMgmtWrapper *pWrapper);
int32_t dmInitVars(SDnode *pDnode);
void dmClearVars(SDnode *pDnode);
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
int32_t dmInitModule(SDnode *pDnode, SMgmtWrapper *wrappers);
#else
int32_t dmInitModule(SDnode *pDnode);
#endif
bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper);
SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper);
void dmSetStatus(SDnode *pDnode, EDndRunStatus stype);
@ -119,7 +123,11 @@ int32_t dmInitStatusClient(SDnode *pDnode);
void dmCleanupClient(SDnode *pDnode);
void dmCleanupStatusClient(SDnode *pDnode);
SMsgCb dmGetMsgcb(SDnode *pDnode);
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers);
#else
int32_t dmInitMsgHandle(SDnode *pDnode);
#endif
int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg);
// dmMonitor.c

77
source/dnode/mgmt/node_mgmt/src/dmMgmt.c
View File

@ -66,9 +66,15 @@ int32_t dmInitDnode(SDnode *pDnode) {
goto _OVER;
}
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
if (dmInitModule(pDnode, pDnode->wrappers) != 0) {
goto _OVER;
}
#else
if (dmInitModule(pDnode) != 0) {
goto _OVER;
}
#endif
indexInit(tsNumOfCommitThreads);
streamMetaInit();
@ -107,6 +113,77 @@ void dmCleanupDnode(SDnode *pDnode) {
dDebug("dnode is closed, ptr:%p", pDnode);
}
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
int32_t dmInitVars(SDnode *pDnode) {
SDnodeData *pData = &pDnode->data;
pData->dnodeId = 0;
pData->clusterId = 0;
pData->dnodeVer = 0;
pData->engineVer = 0;
pData->updateTime = 0;
pData->rebootTime = taosGetTimestampMs();
pData->dropped = 0;
pData->stopped = 0;
pData->dnodeHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK);
if (pData->dnodeHash == NULL) {
dError("failed to init dnode hash");
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (dmReadEps(pData) != 0) {
dError("failed to read file since %s", terrstr());
return -1;
}
if (pData->dropped) {
dError("dnode will not start since its already dropped");
return -1;
}
taosThreadRwlockInit(&pData->lock, NULL);
taosThreadMutexInit(&pDnode->mutex, NULL);
return 0;
}
void dmClearVars(SDnode *pDnode) {
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype];
taosMemoryFreeClear(pWrapper->path);
taosThreadRwlockDestroy(&pWrapper->lock);
}
if (pDnode->lockfile != NULL) {
taosUnLockFile(pDnode->lockfile);
taosCloseFile(&pDnode->lockfile);
pDnode->lockfile = NULL;
}
SDnodeData *pData = &pDnode->data;
taosThreadRwlockWrlock(&pData->lock);
if (pData->oldDnodeEps != NULL) {
if (dmWriteEps(pData) == 0) {
dmRemoveDnodePairs(pData);
}
taosArrayDestroy(pData->oldDnodeEps);
pData->oldDnodeEps = NULL;
}
if (pData->dnodeEps != NULL) {
taosArrayDestroy(pData->dnodeEps);
pData->dnodeEps = NULL;
}
if (pData->dnodeHash != NULL) {
taosHashCleanup(pData->dnodeHash);
pData->dnodeHash = NULL;
}
taosThreadRwlockUnlock(&pData->lock);
taosThreadRwlockDestroy(&pData->lock);
taosThreadMutexDestroy(&pDnode->mutex);
memset(&pDnode->mutex, 0, sizeof(pDnode->mutex));
}
#endif
void dmSetStatus(SDnode *pDnode, EDndRunStatus status) {
if (pDnode->status != status) {
dDebug("dnode status set from %s to %s", dmStatStr(pDnode->status), dmStatStr(status));

28
source/dnode/mgmt/node_mgmt/src/dmTransport.c
View File

@ -251,6 +251,33 @@ _OVER:
dmReleaseWrapper(pWrapper);
}
#if defined(TD_MODULE_OPTIMIZE) || !defined(TD_ENTERPRISE)
int32_t dmInitMsgHandle(SDnode *pDnode, SMgmtWrapper *wrappers) {
SDnodeTrans *pTrans = &pDnode->trans;
for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) {
SMgmtWrapper *pWrapper = wrappers + ntype;
SArray *pArray = (*pWrapper->func.getHandlesFp)();
if (pArray == NULL) return -1;
for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) {
SMgmtHandle *pMgmt = taosArrayGet(pArray, i);
SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)];
if (pMgmt->needCheckVgId) {
pHandle->needCheckVgId = pMgmt->needCheckVgId;
}
if (!pMgmt->needCheckVgId) {
pHandle->defaultNtype = ntype;
}
pWrapper->msgFps[TMSG_INDEX(pMgmt->msgType)] = pMgmt->msgFp;
}
taosArrayDestroy(pArray);
}
return 0;
}
#else
int32_t dmInitMsgHandle(SDnode *pDnode) {
SDnodeTrans *pTrans = &pDnode->trans;
@ -276,6 +303,7 @@ int32_t dmInitMsgHandle(SDnode *pDnode) {
return 0;
}
#endif
static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) {
SDnode *pDnode = dmInstance();

11
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -397,6 +397,9 @@ void mndGetDnodeData(SMnode *pMnode, SArray *pDnodeInfo) {
SDnodeInfo dInfo;
dInfo.id = pDnode->id;
dInfo.ep.port = pDnode->port;
#ifdef TD_GRANT_HB_OPTIMIZE
dInfo.offlineReason = pDnode->offlineReason;
#endif
tstrncpy(dInfo.ep.fqdn, pDnode->fqdn, TSDB_FQDN_LEN);
tstrncpy(dInfo.active, pDnode->active, TSDB_ACTIVE_KEY_LEN);
tstrncpy(dInfo.connActive, pDnode->connActive, TSDB_CONN_ACTIVE_KEY_LEN);
@ -781,11 +784,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
SDnodeObj tmpDnode = *pDnode;
if (action == DND_ACTIVE_CODE) {
#ifndef TD_GRANT_OPTIMIZE
if (grantAlterActiveCode(pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
#else
if (grantAlterActiveCode(pDnode->id, pDnode->active, pCfgReq->value, tmpDnode.active, 0) != 0) {
#endif
if (TSDB_CODE_DUP_KEY != terrno) {
mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());
@ -801,11 +800,7 @@ static int32_t mndConfigDnode(SMnode *pMnode, SRpcMsg *pReq, SMCfgDnodeReq *pCfg
goto _OVER;
}
} else if (action == DND_CONN_ACTIVE_CODE) {
#ifndef TD_GRANT_OPTIMIZE
if (grantAlterActiveCode(pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
#else
if (grantAlterActiveCode(pDnode->id, pDnode->connActive, pCfgReq->value, tmpDnode.connActive, 1) != 0) {
#endif
if (TSDB_CODE_DUP_KEY != terrno) {
mError("dnode:%d, config dnode:%d, app:%p config:%s value:%s failed since %s", pDnode->id, pCfgReq->dnodeId,
pReq->info.ahandle, pCfgReq->config, pCfgReq->value, terrstr());

4
source/dnode/mnode/impl/src/mndGrant.c
View File

@ -131,13 +131,9 @@ void grantAdd(EGrantType grant, uint64_t value) {}
void grantRestore(EGrantType grant, uint64_t value) {}
int32_t dmProcessGrantReq(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
int32_t dmProcessGrantNotify(void *pInfo, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; }
#ifndef TD_GRANT_OPTIMIZE
int32_t grantAlterActiveCode(const char *old, const char *new, char *out, int8_t type) { return TSDB_CODE_SUCCESS; }
#else
int32_t grantAlterActiveCode(int32_t did, const char *old, const char *new, char *out, int8_t type) {
return TSDB_CODE_SUCCESS;
}
#endif
#endif

6
source/dnode/mnode/impl/src/mndSync.c
View File

@ -95,7 +95,11 @@ static int32_t mndTransValidatePrepareAction(SMnode *pMnode, STrans *pTrans, STr
}
_OUT:
taosMemoryFreeClear(pRow);
if (pRow) {
SdbDeleteFp deleteFp = pSdb->deleteFps[pRaw->type];
if (deleteFp) (*deleteFp)(pSdb, pRow->pObj, false);
taosMemoryFreeClear(pRow);
}
return code;
}

3
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -1521,7 +1521,8 @@ ETsdbFsState tsdbSnapGetFsState(SVnode* pVnode) {
return pVnode->pTsdb->pFS->fsstate;
}
for (int32_t lvl = 0; lvl < TSDB_RETENTION_MAX; ++lvl) {
if (SMA_RSMA_GET_TSDB(pVnode, lvl)->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
STsdb* pTsdb = SMA_RSMA_GET_TSDB(pVnode, lvl);
if (pTsdb && pTsdb->pFS->fsstate != TSDB_FS_STATE_NORMAL) {
return TSDB_FS_STATE_INCOMPLETE;
}
}

3
source/libs/parser/inc/sql.y
View File

@ -787,6 +787,7 @@ table_alias(A) ::= NK_ID(B).
%type column_alias { SToken }
%destructor column_alias { }
column_alias(A) ::= NK_ID(B). { A = B; }
column_alias(A) ::= NK_ALIAS(B). { A = B; }
%type user_name { SToken }
%destructor user_name { }
@ -873,6 +874,8 @@ expression_list(A) ::= expression_list(B) NK_COMMA expr_or_subquery(C).
column_reference(A) ::= column_name(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
column_reference(A) ::= table_name(B) NK_DOT column_name(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }
column_reference(A) ::= NK_ALIAS(B). { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
column_reference(A) ::= table_name(B) NK_DOT NK_ALIAS(C). { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }
pseudo_column(A) ::= ROWTS(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
pseudo_column(A) ::= TBNAME(B). { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }

31
source/libs/parser/src/parTokenizer.c
View File

@ -627,9 +627,21 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
case 't':
case 'F':
case 'f': {
for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t)z[i]]; i++) {
bool hasNonAsciiChars = false;
for (i = 1;; i++) {
if ((z[i] & 0x80) != 0) {
// utf-8 characters
// currently, we support using utf-8 characters only in alias
hasNonAsciiChars = true;
} else if (isIdChar[(uint8_t)z[i]]) {
} else {
break;
}
}
if (hasNonAsciiChars) {
*tokenId = TK_NK_ALIAS; // must be alias
return i;
}
if ((i == 4 && strncasecmp(z, "true", 4) == 0) || (i == 5 && strncasecmp(z, "false", 5) == 0)) {
*tokenId = TK_NK_BOOL;
return i;
@ -638,10 +650,21 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
return i;
}
default: {
if (((*z & 0x80) != 0) || !isIdChar[(uint8_t)*z]) {
if ((*z & 0x80) == 0 && !isIdChar[(uint8_t)*z]) {
break;
}
for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t)z[i]]; i++) {
bool hasNonAsciiChars = false;
for (i = 1; ; i++) {
if ((z[i] & 0x80) != 0) {
hasNonAsciiChars = true;
} else if (isIdChar[(uint8_t)z[i]]){
} else {
break;
}
}
if (hasNonAsciiChars) {
*tokenId = TK_NK_ALIAS;
return i;
}
*tokenId = tKeywordCode(z, i);
return i;

97
source/libs/parser/src/parTranslater.c
View File

@ -4576,7 +4576,7 @@ static int32_t checkDbEnumOption(STranslateContext* pCxt, const char* pName, int
return TSDB_CODE_SUCCESS;
}
static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions) {
static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRetentions, int8_t precision) {
if (NULL == pRetentions) {
return TSDB_CODE_SUCCESS;
}
@ -4599,11 +4599,55 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
SValueNode* pFreq = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 0);
SValueNode* pKeep = (SValueNode*)nodesListGetNode(((SNodeListNode*)pRetention)->pNodeList, 1);
if (pFreq->datum.i <= 0 || 'n' == pFreq->unit || 'y' == pFreq->unit || pFreq->datum.i >= pKeep->datum.i ||
(NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) ||
(NULL != pPrevKeep && pPrevKeep->datum.i > pKeep->datum.i)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION, "Invalid option retentions");
ASSERTS(pFreq->isDuration && pKeep->isDuration, "Retentions freq/keep should have unit");
// check unit
if (pFreq->isDuration && TIME_UNIT_SECOND != pFreq->unit && TIME_UNIT_MINUTE != pFreq->unit &&
TIME_UNIT_HOUR != pFreq->unit && TIME_UNIT_DAY != pFreq->unit && TIME_UNIT_WEEK != pFreq->unit) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(freq): %s, only s, m, h, d, w allowed", pFreq->literal);
}
if (pKeep->isDuration && TIME_UNIT_MINUTE != pKeep->unit && TIME_UNIT_HOUR != pKeep->unit &&
TIME_UNIT_DAY != pKeep->unit) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(keep): %s, only m, h, d allowed", pKeep->literal);
}
// check value range
if (pFreq->datum.i <= 0) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(freq): %s should larger than 0", pFreq->literal);
}
int64_t keepMinute = pKeep->datum.i / getUnitPerMinute(pKeep->node.resType.precision);
int64_t tsdbMaxKeep = TSDB_TIME_PRECISION_NANO == precision ? TSDB_MAX_KEEP_NS : TSDB_MAX_KEEP;
if (keepMinute < TSDB_MIN_KEEP || keepMinute > tsdbMaxKeep) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(keep): %" PRId64 "m, valid range: [%" PRIi64
"m, %" PRId64 "m]",
keepMinute, TSDB_MIN_KEEP, tsdbMaxKeep);
}
// check relationships
if (pFreq->datum.i >= pKeep->datum.i) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(freq/keep): %s should larger than %s", pKeep->literal,
pFreq->literal);
}
if (NULL != pPrevFreq && pPrevFreq->datum.i >= pFreq->datum.i) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(freq): %s should larger than %s", pFreq->literal,
pPrevFreq->literal);
}
if (NULL != pPrevKeep && pPrevKeep->datum.i > pKeep->datum.i) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DB_OPTION,
"Invalid option retentions(keep): %s should not larger than %s",
pPrevKeep->literal, pKeep->literal);
}
pPrevFreq = pFreq;
pPrevKeep = pKeep;
}
@ -4723,7 +4767,7 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName
TSDB_DB_SINGLE_STABLE_OFF);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbRetentionsOption(pCxt, pOptions->pRetentions);
code = checkDbRetentionsOption(pCxt, pOptions->pRetentions, pOptions->precision);
}
if (TSDB_CODE_SUCCESS == code) {
code = checkDbEnumOption(pCxt, "schemaless", pOptions->schemaless, TSDB_DB_SCHEMALESS_ON, TSDB_DB_SCHEMALESS_OFF);
@ -5021,7 +5065,7 @@ static int32_t checkTableSmaOption(STranslateContext* pCxt, SCreateTableStmt* pS
}
static bool validRollupFunc(const char* pFunc) {
static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"};
static const char* rollupFuncs[] = {"avg", "sum", "min", "max", "last", "first"};
static const int32_t numOfRollupFuncs = (sizeof(rollupFuncs) / sizeof(char*));
for (int i = 0; i < numOfRollupFuncs; ++i) {
if (0 == strcmp(rollupFuncs[i], pFunc)) {
@ -5031,6 +5075,17 @@ static bool validRollupFunc(const char* pFunc) {
return false;
}
static bool aggrRollupFunc(const char* pFunc) {
static const char* aggrRollupFuncs[] = {"avg", "sum"};
static const int32_t numOfAggrRollupFuncs = (sizeof(aggrRollupFuncs) / sizeof(char*));
for (int i = 0; i < numOfAggrRollupFuncs; ++i) {
if (0 == strcmp(aggrRollupFuncs[i], pFunc)) {
return true;
}
}
return false;
}
static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs, bool createStable,
SDbCfgInfo* pDbCfg) {
if (NULL == pFuncs) {
@ -5104,7 +5159,8 @@ static int32_t checkTableTagsSchema(STranslateContext* pCxt, SHashObj* pHash, SN
return code;
}
static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, int32_t ntags, SNodeList* pCols) {
static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, int32_t ntags, SNodeList* pCols,
SNodeList* pRollupFuncs) {
int32_t ncols = LIST_LENGTH(pCols);
if (ncols < TSDB_MIN_COLUMNS) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
@ -5114,13 +5170,19 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in
int32_t code = TSDB_CODE_SUCCESS;
bool first = true;
int32_t colIndex = 0;
int32_t rowSize = 0;
SNode* pNode = NULL;
char* pFunc = NULL;
bool isAggrRollup = false;
if (pRollupFuncs) {
pFunc = ((SFunctionNode*)nodesListGetNode(pRollupFuncs, 0))->functionName;
isAggrRollup = aggrRollupFunc(pFunc);
}
FOREACH(pNode, pCols) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if (first) {
first = false;
if (0 == colIndex) {
if (TSDB_DATA_TYPE_TIMESTAMP != pCol->dataType.type) {
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FIRST_COLUMN);
}
@ -5140,6 +5202,15 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in
code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_VAR_COLUMN_LEN);
}
}
if (TSDB_CODE_SUCCESS == code && isAggrRollup && 0 != colIndex) {
if (pCol->dataType.type != TSDB_DATA_TYPE_FLOAT && pCol->dataType.type != TSDB_DATA_TYPE_DOUBLE) {
code =
generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN,
"Invalid column type: %s, only float/double allowed for %s", pCol->colName, pFunc);
}
}
if (TSDB_CODE_SUCCESS == code) {
code = taosHashPut(pHash, pCol->colName, len, &pCol, POINTER_BYTES);
}
@ -5148,6 +5219,8 @@ static int32_t checkTableColsSchema(STranslateContext* pCxt, SHashObj* pHash, in
} else {
break;
}
// next column
++colIndex;
}
if (TSDB_CODE_SUCCESS == code && rowSize > TSDB_MAX_BYTES_PER_ROW) {
@ -5166,7 +5239,7 @@ static int32_t checkTableSchema(STranslateContext* pCxt, SCreateTableStmt* pStmt
int32_t code = checkTableTagsSchema(pCxt, pHash, pStmt->pTags);
if (TSDB_CODE_SUCCESS == code) {
code = checkTableColsSchema(pCxt, pHash, LIST_LENGTH(pStmt->pTags), pStmt->pCols);
code = checkTableColsSchema(pCxt, pHash, LIST_LENGTH(pStmt->pTags), pStmt->pCols, pStmt->pOptions->pRollupFuncs);
}
taosHashCleanup(pHash);

8052
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

4
tests/parallel_test/container_build.sh
View File

@ -68,9 +68,9 @@ docker run \
-v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
-v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
-v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1"
# -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
if [[ -d ${WORKDIR}/debugNoSan ]] ;then
echo "delete ${WORKDIR}/debugNoSan"
@ -97,10 +97,10 @@ docker run \
-v ${REP_REAL_PATH}/community/contrib/cpp-stub/:${REP_DIR}/community/contrib/cpp-stub \
-v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \
-v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_SANITIZER=1 -DTOOLS_SANITIZE=true -DTOOLS_BUILD_TYPE=Debug -DBUILD_TAOSX=true -DJEMALLOC_ENABLED=0;make -j 10|| exit 1 "
# -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
mv ${REP_REAL_PATH}/debug ${WORKDIR}/debugSan
ret=$?

73
tests/system-test/1-insert/create_retentions.py
View File

@ -52,13 +52,41 @@ class TDTestCase:
@property
def create_databases_sql_err(self):
return [
"create database db1 retentions 0s:1d",
"create database db3 retentions 1s:0d",
"create database db1 retentions 1s:1y",
# check grammar
"create database db1 retentions",
"create database db1 retentions 1s:1d,2s:2d,3s:3d,4s:4d",
# check unit
"create database db1 retentions 1b:1d",
"create database db1 retentions 1u:1d",
"create database db1 retentions 1a:1d",
"create database db1 retentions 1n:1d",
"create database db1 retentions 1y:1d",
"create database db1 retentions 1s:86400s",
"create database db1 retentions 1s:86400000a",
"create database db1 retentions 1s:86400000000u",
"create database db1 retentions 1s:86400000000000b",
"create database db1 retentions 1s:1w",
"create database db1 retentions 1s:1n",
"create database db2 retentions 1w:1d ;",
"create database db5 retentions 1s:1d,3s:3d,2s:2d",
"create database db1 retentions 1s:1n,2s:2d,3s:3d,4s:4d",
"create database db1 retentions 1s:1y",
# check value range
"create database db1 retentions -1s:1d",
"create database db1 retentions 0s:1d",
"create database db3 retentions 1s:-1d",
"create database db3 retentions 1s:0d",
"create database db3 retentions 1s:1439m",
"create database db3 retentions 1s:365001d",
"create database db3 retentions 1s:8760001h",
"create database db3 retentions 1s:525600001m",
"create database db3 retentions 1s:106581d precision 'ns'",
"create database db3 retentions 1s:2557921h precision 'ns'",
"create database db3 retentions 1s:153475201m precision 'ns'",
# check relationships
"create database db5 retentions 1441m:1440m,2d:3d",
"create database db5 retentions 2m:1d,1s:2d",
"create database db5 retentions 1s:2880m,2s:2879m",
"create database db5 retentions 1s:1d,2s:2d,2s:3d",
"create database db5 retentions 1s:1d,3s:2d,2s:3d",
"create database db1 retentions 1s:1d,2s:3d,3s:2d",
]
@property
@ -92,6 +120,16 @@ class TDTestCase:
f"create stable {dbname}.stb24 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " ,
f"create stable {dbname}.stb25 ({PRIMARY_COL} timestamp, {INT_COL} int) " ,
f"create stable {dbname}.stb26 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " ,
# only float/double allowd for avg/sum
f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)",
f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BINT_COL} bigint) tags (tag1 int) rollup(avg)",
f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BOOL_COL} bool) tags (tag1 int) rollup(avg)",
f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BINARY_COL} binary(10)) tags (tag1 int) rollup(avg)",
f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum)",
f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BINT_COL} bigint) tags (tag1 int) rollup(sum)",
f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BOOL_COL} bool) tags (tag1 int) rollup(sum)",
f"create stable {dbname}.stb11 ({PRIMARY_COL} timestamp, {BINARY_COL} binary(10)) tags (tag1 int) rollup(sum)",
# watermark, max_delay: [0, 900000], [ms, s, m, ?]
f"create stable stb17 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) max_delay 1u",
@ -108,10 +146,10 @@ class TDTestCase:
@property
def create_stable_sql_current(self):
return [
f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)",
f"create stable stb1 ({PRIMARY_COL} timestamp, {FLOAT_COL} float) tags (tag1 int) rollup(avg)",
f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s max_delay 1m",
f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s max_delay 1m",
f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m",
f"create stable stb4 ({PRIMARY_COL} timestamp, {DOUBLE_COL} double) tags (tag1 int) rollup(sum) watermark 5s max_delay 1m",
f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s max_delay 1m",
f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m",
f"create stable stb7 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s max_delay 1m sma({INT_COL})",
@ -154,6 +192,12 @@ class TDTestCase:
{INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned, {BINARY_COL} binary(16)
) tags ({INT_TAG} int) rollup({rsma_type}) watermark 5s,5s max_delay 5s,5s
'''
elif rsma_type.lower().strip() in ("sum", "avg"):
create_stb_sql = f'''create table {dbname}.{stb}(
ts timestamp, {DOUBLE_COL} double, {DOUBLE_COL}_1 double, {DOUBLE_COL}_2 double, {DOUBLE_COL}_3 double,
{FLOAT_COL} float, {DOUBLE_COL}_4 double, {FLOAT_COL}_1 float, {FLOAT_COL}_2 float, {FLOAT_COL}_3 float,
{DOUBLE_COL}_5 double) tags ({INT_TAG} int) rollup({rsma_type}) watermark 5s,5s max_delay 5s,5s
'''
else:
create_stb_sql = f'''create table {dbname}.{stb}(
ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint,
@ -200,11 +244,16 @@ class TDTestCase:
{data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]},
{data.utint_data[i]}, {data.usint_data[i]}, {data.uint_data[i]}, {data.ubint_data[i]}, '{data.vchar_data[i]}'
'''
else:
elif rsma_type.lower().strip() in ("sum", "avg"):
row_data = f'''
{data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]},
{data.utint_data[i]}, {data.usint_data[i]}, {data.uint_data[i]}, {data.ubint_data[i]}
'''
else:
row_data = f'''
{data.double_data[i]}, {data.double_data[i]}, {data.double_data[i]}, {data.double_data[i]}, {data.float_data[i]}, {data.double_data[i]},
{data.float_data[i]}, {data.float_data[i]}, {data.float_data[i]}, {data.double_data[i]}
'''
else:
row_data = f'''
{data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]},
@ -245,17 +294,17 @@ class TDTestCase:
tdSql.query(f"select count(*) from {DB3}.{STBNAME} where ts > now()-5m")
tdSql.checkData(0, 0, self.rows * db3_ctb_num)
tdSql.checkRows(1)
tdSql.query(f"select {INT_COL} from {DB3}.{CTBNAME} where ts > now()-4d")
tdSql.query(f"select {FLOAT_COL} from {DB3}.{CTBNAME} where ts > now()-4d")
# not stable
#tdSql.checkData(0, 0, self.rows-1)
tdSql.query(f"select {INT_COL} from {DB3}.{CTBNAME} where ts > now()-6d")
tdSql.query(f"select {DOUBLE_COL} from {DB3}.{CTBNAME} where ts > now()-6d")
# not stable
# tdSql.checkData(0, 0, self.rows-1)
# from ...pytest.util.sql import tdSql
tdLog.printNoPrefix("==========step2.1.1 : alter stb schemaL drop column")
tdSql.query(f"select {BINT_COL} from {DB3}.{STBNAME}")
tdSql.query(f"select {FLOAT_COL} from {DB3}.{STBNAME}")
#tdSql.execute(f"alter stable {DB3}.stb1 drop column {BINT_COL}")
# not support alter stable schema anymore
tdSql.error(f"alter stable {DB3}.stb1 drop column {BINT_COL}")

29
tests/system-test/2-query/select_null.py
View File

@ -432,10 +432,39 @@ class TDTestCase:
self.ts_3036("%s" %self.db)
self.test_select_as_chinese_characters();
endTime = time.time()
print("total time %ds" % (endTime - startTime))
def test_select_as_chinese_characters(self):
tdSql.execute("use sel_null")
tdSql.query("select ts as 时间戳, c0 as c第一列, t0 标签1 from sel_null.stb0_0 limit 10", queryTimes=1)
tdSql.checkRows(10)
tdSql.query("select 时间戳 from (select ts as 时间戳, c0 as c第一列, t0 标签1 from sel_null.stb0_0) where 时间戳 > '2023-1-1' and c第一列 != 0 and 标签1 == 0 limit 10", queryTimes=1)
tdSql.checkRows(10)
tdSql.query("select count(*) as 计数 from sel_null.stb0_0 partition by c0 as 分组列", queryTimes=1)
tdSql.error("create database 数据库")
tdSql.error("create table sel_null.中文库 (ts timestamp, c2 int)")
tdSql.error("create table sel_null.table1(ts timestamp, 列2 int)")
tdSql.execute("create stable sel_null.stable1(ts timestamp, `值` int) tags(`标签1` int, `标签2` int)")
tdSql.execute('insert into sel_null.ct1 using sel_null.stable1 tags(1, 1) values(now, 1)', queryTimes=1)
tdSql.execute('insert into sel_null.ct1 using sel_null.stable1 tags(2, 2) values(now, 2)', queryTimes=1)
tdSql.execute('insert into sel_null.ct1 using sel_null.stable1 tags(2, 2) values(now, 3)', queryTimes=1)
tdSql.query('select 值 , 标签1 from sel_null.stable1', queryTimes=1)
tdSql.query('select case 值 when 标签1 then 标签1 else 标签2 end from sel_null.stable1', queryTimes=1)
tdSql.query('select count(*) from sel_null.stable1 group by 值 having sum(标签1) > 0', queryTimes=1)
tdSql.query('show table tags `标签1` 标签n from sel_null.stable1', queryTimes=1)
tdSql.query('create sma index a on sel_null.stable1 FUNCTION (sum(值)) interval(1s)', queryTimes=1)
tdSql.query('select count(值) from sel_null.stable1', queryTimes=1)
tdSql.query('select stable1.值 from sel_null.stable1', queryTimes=1)
tdSql.query('select stable1.值 from sel_null.stable1 order by 值', queryTimes=1)
tdSql.execute('create stable sel_null.join_stable(`时间戳` timestamp, c1 int) tags(`标签1` int)', queryTimes=1)
tdSql.query('select a.值 from sel_null.stable1 a join sel_null.join_stable b on a.ts = 时间戳;', queryTimes=1)
tdSql.query('select a.值 from sel_null.stable1 a join sel_null.join_stable b on a.ts = b.时间戳;', queryTimes=1)
tdSql.execute('create user user1 pass "asd"', queryTimes=1)
tdSql.execute('grant write on sel_null.stable1 with 标签1 = 1 to user1',queryTimes=1)
tdSql.execute('select count(*) from sel_null.stable1 state_window(值)', queryTimes=1)
def stop(self):
tdSql.close()

217
tools/shell/src/shellEngine.c
View File

@ -22,6 +22,23 @@
#include "shellAuto.h"
#include "shellInt.h"
typedef struct {
const char *sql;
bool vertical;
tsem_t sem;
int64_t numOfRows; // the num of this batch
int64_t numOfAllRows;
int32_t numFields;
TAOS_FIELD *fields;
int32_t precision;
int32_t maxColNameLen; // for vertical print
int32_t width[TSDB_MAX_COLUMNS]; // for horizontal print
uint64_t resShowMaxNum;
} tsDumpInfo;
static bool shellIsEmptyCommand(const char *cmd);
static int32_t shellRunSingleCommand(char *command);
static void shellRecordCommandToHistory(char *command);
@ -31,8 +48,8 @@ static char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision);
static int64_t shellDumpResultToFile(const char *fname, TAOS_RES *tres);
static void shellPrintNChar(const char *str, int32_t length, int32_t width);
static void shellPrintGeometry(const unsigned char *str, int32_t length, int32_t width);
static int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql);
static int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql);
static void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo* dump_info);
static void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo* dump_info);
static int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql);
static void shellReadHistory();
static void shellWriteHistory();
@ -702,49 +719,67 @@ bool shellIsShowQuery(const char *sql) {
return false;
}
int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
void init_dump_info(tsDumpInfo *dump_info, TAOS_RES *tres, const char *sql, bool vertical) {
dump_info->sql = sql;
dump_info->vertical = vertical;
tsem_init(&dump_info->sem, 0, 0);
dump_info->numOfAllRows = 0;
dump_info->numFields = taos_num_fields(tres);
dump_info->fields = taos_fetch_fields(tres);
dump_info->precision = taos_result_precision(tres);
dump_info->resShowMaxNum = UINT64_MAX;
if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(dump_info->sql)) {
dump_info->resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
}
int32_t num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int32_t precision = taos_result_precision(tres);
int32_t maxColNameLen = 0;
for (int32_t col = 0; col < num_fields; col++) {
int32_t len = (int32_t)strlen(fields[col].name);
if (len > maxColNameLen) {
maxColNameLen = len;
if (vertical) {
dump_info->maxColNameLen = 0;
for (int32_t col = 0; col < dump_info->numFields; col++) {
int32_t len = (int32_t)strlen(dump_info->fields[col].name);
if (len > dump_info->maxColNameLen) {
dump_info->maxColNameLen = len;
}
}
} else {
for (int32_t col = 0; col < dump_info->numFields; col++) {
dump_info->width[col] = shellCalcColWidth(dump_info->fields + col, dump_info->precision);
}
}
}
uint64_t resShowMaxNum = UINT64_MAX;
if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) {
resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
void shellVerticalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
printf("\033[31mtaos_fetch_row failed.\033[0m\n");
return;
}
int64_t numOfRows = 0;
int32_t showMore = 1;
do {
if (numOfRows < resShowMaxNum) {
printf("*************************** %"PRId64".row ***************************\r\n", numOfRows + 1);
int64_t numOfPintRows = dump_info->numOfAllRows;
int numOfPrintRowsThisOne = 0;
int32_t *length = taos_fetch_lengths(tres);
while (row != NULL) {
printf("*************************** %" PRId64 ".row ***************************\r\n", numOfPintRows + 1);
for (int32_t i = 0; i < num_fields; i++) {
TAOS_FIELD *field = fields + i;
int32_t *length = taos_fetch_lengths(tres);
int32_t padding = (int32_t)(maxColNameLen - strlen(field->name));
printf("%*.s%s: ", padding, " ", field->name);
for (int32_t i = 0; i < dump_info->numFields; i++) {
TAOS_FIELD *field = dump_info->fields + i;
shellPrintField((const char *)row[i], field, 0, length[i], precision);
putchar('\r');
putchar('\n');
}
} else if (showMore) {
int32_t padding = (int32_t)(dump_info->maxColNameLen - strlen(field->name));
printf("%*.s%s: ", padding, " ", field->name);
shellPrintField((const char *)row[i], field, 0, length[i], dump_info->precision);
putchar('\r');
putchar('\n');
}
numOfPintRows++;
numOfPrintRowsThisOne++;
if (numOfPintRows == dump_info->resShowMaxNum) {
printf("\r\n");
printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM);
printf(" You can use the `LIMIT` clause to get fewer result to show.\r\n");
@ -752,14 +787,16 @@ int64_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) {
printf("\r\n");
printf(" You can use Ctrl+C to stop the underway fetching.\r\n");
printf("\r\n");
showMore = 0;
return;
}
numOfRows++;
row = taos_fetch_row(tres);
} while (row != NULL);
if (numOfPrintRowsThisOne == dump_info->numOfRows) {
return;
}
return numOfRows;
row = taos_fetch_row(tres);
}
return;
}
int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision) {
@ -856,47 +893,38 @@ void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields) {
putchar('\n');
}
int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) {
void shellHorizontalPrintResult(TAOS_RES *tres, tsDumpInfo *dump_info) {
TAOS_ROW row = taos_fetch_row(tres);
if (row == NULL) {
return 0;
printf("\033[31mtaos_fetch_row failed.\033[0m\n");
return;
}
int32_t num_fields = taos_num_fields(tres);
TAOS_FIELD *fields = taos_fetch_fields(tres);
int32_t precision = taos_result_precision(tres);
int32_t width[TSDB_MAX_COLUMNS];
for (int32_t col = 0; col < num_fields; col++) {
width[col] = shellCalcColWidth(fields + col, precision);
int64_t numOfPintRows = dump_info->numOfAllRows;
int numOfPrintRowsThisOne = 0;
if (numOfPintRows == 0) {
shellPrintHeader(dump_info->fields, dump_info->width, dump_info->numFields);
}
shellPrintHeader(fields, width, num_fields);
uint64_t resShowMaxNum = UINT64_MAX;
if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsShowWhole(sql)) {
resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM;
}
int64_t numOfRows = 0;
int32_t showMore = 1;
do {
while (row != NULL) {
int32_t *length = taos_fetch_lengths(tres);
if (numOfRows < resShowMaxNum) {
for (int32_t i = 0; i < num_fields; i++) {
putchar(' ');
shellPrintField((const char *)row[i], fields + i, width[i], length[i], precision);
putchar(' ');
putchar('|');
}
putchar('\r');
putchar('\n');
} else if (showMore) {
for (int32_t i = 0; i < dump_info->numFields; i++) {
putchar(' ');
shellPrintField((const char *)row[i], dump_info->fields + i, dump_info->width[i], length[i],
dump_info->precision);
putchar(' ');
putchar('|');
}
putchar('\r');
putchar('\n');
numOfPintRows++;
numOfPrintRowsThisOne++;
if (numOfPintRows == dump_info->resShowMaxNum) {
printf("\r\n");
printf(" Notice: The result shows only the first %d rows.\r\n", SHELL_DEFAULT_RES_SHOW_NUM);
if (shellIsShowQuery(sql)) {
if (shellIsShowQuery(dump_info->sql)) {
printf(" You can use '>>' to redirect the whole set of the result to a specified file.\r\n");
} else {
printf(" You can use the `LIMIT` clause to get fewer result to show.\r\n");
@ -905,28 +933,53 @@ int64_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) {
printf("\r\n");
printf(" You can use Ctrl+C to stop the underway fetching.\r\n");
printf("\r\n");
showMore = 0;
return;
}
numOfRows++;
row = taos_fetch_row(tres);
} while (row != NULL);
if (numOfPrintRowsThisOne == dump_info->numOfRows) {
return;
}
return numOfRows;
row = taos_fetch_row(tres);
}
return;
}
void shellDumpResultCallback(void *param, TAOS_RES *tres, int num_of_rows) {
tsDumpInfo *dump_info = (tsDumpInfo *)param;
if (num_of_rows > 0) {
dump_info->numOfRows = num_of_rows;
if (dump_info->numOfAllRows < dump_info->resShowMaxNum) {
if (dump_info->vertical) {
shellVerticalPrintResult(tres, dump_info);
} else {
shellHorizontalPrintResult(tres, dump_info);
}
}
dump_info->numOfAllRows += num_of_rows;
taos_fetch_rows_a(tres, shellDumpResultCallback, param);
} else {
if (num_of_rows < 0) {
printf("\033[31masync retrieve failed, code: %d\033[0m\n", num_of_rows);
}
tsem_post(&dump_info->sem);
}
}
int64_t shellDumpResult(TAOS_RES *tres, char *fname, int32_t *error_no, bool vertical, const char *sql) {
int64_t numOfRows = 0;
int64_t num_of_rows = 0;
if (fname != NULL) {
numOfRows = shellDumpResultToFile(fname, tres);
} else if (vertical) {
numOfRows = shellVerticalPrintResult(tres, sql);
num_of_rows = shellDumpResultToFile(fname, tres);
} else {
numOfRows = shellHorizontalPrintResult(tres, sql);
tsDumpInfo dump_info;
init_dump_info(&dump_info, tres, sql, vertical);
taos_fetch_rows_a(tres, shellDumpResultCallback, &dump_info);
tsem_wait(&dump_info.sem);
num_of_rows = dump_info.numOfAllRows;
}
*error_no = taos_errno(tres);
return numOfRows;
return num_of_rows;
}
void shellReadHistory() {