xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #7512 from taosdata/feature/szhou/schemaless 

[schemaless][RunCITest]
This commit is contained in:
dapan1121 2021-08-27 10:29:29 +08:00 committed by GitHub
parent d8bcb00701 a2a845faeb
commit f9e22c1632
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1223 additions and 434 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/client/inc/tsclient.h
View File

@ -492,6 +492,7 @@ bool tscHasReachLimitation(SQueryInfo *pQueryInfo, SSqlRes *pRes);
void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32_t numOfCols);
char *tscGetErrorMsgPayload(SSqlCmd *pCmd);
int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql);
int32_t tscInvalidOperationMsg(char *msg, const char *additionalInfo, const char *sql);
int32_t tscSQLSyntaxErrMsg(char* msg, const char* additionalInfo, const char* sql);

9
src/client/src/tscAsync.c
View File

@ -363,15 +363,6 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
}
if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_STMT_INSERT)) { // stmt insert
STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
code = tscGetTableMeta(pSql, pTableMetaInfo);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {
taosReleaseRef(tscObjRef, pSql->self);
return;
} else {
assert(code == TSDB_CODE_SUCCESS);
}
(*pSql->fp)(pSql->param, pSql, code);
} else if (TSDB_QUERY_HAS_TYPE(pCmd->insertParam.insertType, TSDB_QUERY_TYPE_FILE_INSERT)) { // file insert
tscImportDataFromFile(pSql);

554
src/client/src/tscParseLineProtocol.c
View File

@ -32,9 +32,6 @@ typedef struct {
uint8_t type;
int16_t length;
char* value;
//===================================
uint32_t fieldSchemaIdx;
} TAOS_SML_KV;
typedef struct {
@ -47,9 +44,6 @@ typedef struct {
// first kv must be timestamp
TAOS_SML_KV* fields;
int32_t fieldNum;
//================================
uint32_t schemaIdx;
} TAOS_SML_DATA_POINT;
typedef enum {
@ -62,10 +56,23 @@ typedef enum {
typedef struct {
uint64_t id;
SHashObj* smlDataToSchema;
} SSmlLinesInfo;
//=================================================================================================
static uint64_t linesSmlHandleId = 0;
uint64_t genLinesSmlId() {
uint64_t id;
do {
id = atomic_add_fetch_64(&linesSmlHandleId, 1);
} while (id == 0);
return id;
}
int compareSmlColKv(const void* p1, const void* p2) {
TAOS_SML_KV* kv1 = (TAOS_SML_KV*)p1;
TAOS_SML_KV* kv2 = (TAOS_SML_KV*)p2;
@ -168,11 +175,46 @@ static int32_t buildSmlKvSchema(TAOS_SML_KV* smlKv, SHashObj* hash, SArray* arra
taosHashPut(hash, field.name, tagKeyLen, &fieldIdx, sizeof(fieldIdx));
}
smlKv->fieldSchemaIdx = (uint32_t)fieldIdx;
uintptr_t valPointer = (uintptr_t)smlKv;
taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &fieldIdx, sizeof(fieldIdx));
return 0;
}
static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen,
SSmlLinesInfo* info) {
tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id);
qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv);
SStringBuilder sb; memset(&sb, 0, sizeof(sb));
char sTableName[TSDB_TABLE_NAME_LEN] = {0};
strtolower(sTableName, point->stableName);
taosStringBuilderAppendString(&sb, sTableName);
for (int j = 0; j < point->tagNum; ++j) {
taosStringBuilderAppendChar(&sb, ',');
TAOS_SML_KV* tagKv = point->tags + j;
char tagName[TSDB_COL_NAME_LEN] = {0};
strtolower(tagName, tagKv->key);
taosStringBuilderAppendString(&sb, tagName);
taosStringBuilderAppendChar(&sb, '=');
taosStringBuilderAppend(&sb, tagKv->value, tagKv->length);
}
size_t len = 0;
char* keyJoined = taosStringBuilderGetResult(&sb, &len);
MD5_CTX context;
MD5Init(&context);
MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len);
MD5Final(&context);
*tableNameLen = snprintf(tableName, *tableNameLen,
"t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0],
context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6],
context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11],
context.digest[12], context.digest[13], context.digest[14], context.digest[15]);
taosStringBuilderDestroy(&sb);
tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName);
return 0;
}
static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint, SArray* stableSchemas, SSmlLinesInfo* info) {
int32_t code = 0;
SHashObj* sname2shema = taosHashInit(32,
@ -203,6 +245,15 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint,
for (int j = 0; j < point->tagNum; ++j) {
TAOS_SML_KV* tagKv = point->tags + j;
if (!point->childTableName) {
char childTableName[TSDB_TABLE_NAME_LEN];
int32_t tableNameLen = TSDB_TABLE_NAME_LEN;
getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info);
point->childTableName = calloc(1, tableNameLen+1);
strncpy(point->childTableName, childTableName, tableNameLen);
point->childTableName[tableNameLen] = '\0';
}
code = buildSmlKvSchema(tagKv, pStableSchema->tagHash, pStableSchema->tags, info);
if (code != 0) {
tscError("SML:0x%"PRIx64" build data point schema failed. point no.: %d, tag key: %s", info->id, i, tagKv->key);
@ -219,7 +270,8 @@ static int32_t buildDataPointSchemas(TAOS_SML_DATA_POINT* points, int numPoint,
}
}
point->schemaIdx = (uint32_t)stableIdx;
uintptr_t valPointer = (uintptr_t)point;
taosHashPut(info->smlDataToSchema, &valPointer, sizeof(uintptr_t), &stableIdx, sizeof(stableIdx));
}
size_t numStables = taosArrayGetSize(stableSchemas);
@ -319,7 +371,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf
buildColumnDescription(action->alterSTable.field, result+n, capacity-n, &outBytes);
TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery
code = taos_errno(res);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%"PRIx64" apply schema action. error: %s", info->id, taos_errstr(res));
}
taos_free_result(res);
if (code == TSDB_CODE_MND_FIELD_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) {
TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE");
code = taos_errno(res2);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
}
taos_free_result(res2);
}
break;
}
case SCHEMA_ACTION_ADD_TAG: {
@ -328,7 +392,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf
result+n, capacity-n, &outBytes);
TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery
code = taos_errno(res);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res));
}
taos_free_result(res);
if (code == TSDB_CODE_MND_TAG_ALREAY_EXIST || code == TSDB_CODE_TSC_DUP_COL_NAMES) {
TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE");
code = taos_errno(res2);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
}
taos_free_result(res2);
}
break;
}
case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: {
@ -337,7 +413,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf
capacity-n, &outBytes);
TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery
code = taos_errno(res);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res));
}
taos_free_result(res);
if (code == TSDB_CODE_MND_INVALID_COLUMN_LENGTH || code == TSDB_CODE_TSC_INVALID_COLUMN_LENGTH) {
TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE");
code = taos_errno(res2);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
}
taos_free_result(res2);
}
break;
}
case SCHEMA_ACTION_CHANGE_TAG_SIZE: {
@ -346,7 +434,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf
capacity-n, &outBytes);
TAOS_RES* res = taos_query(taos, result); //TODO async doAsyncQuery
code = taos_errno(res);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res));
}
taos_free_result(res);
if (code == TSDB_CODE_MND_INVALID_TAG_LENGTH || code == TSDB_CODE_TSC_INVALID_TAG_LENGTH) {
TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE");
code = taos_errno(res2);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
}
taos_free_result(res2);
}
break;
}
case SCHEMA_ACTION_CREATE_STABLE: {
@ -375,7 +475,19 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf
outBytes = snprintf(pos, freeBytes, ")");
TAOS_RES* res = taos_query(taos, result);
code = taos_errno(res);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%"PRIx64" apply schema action. error : %s", info->id, taos_errstr(res));
}
taos_free_result(res);
if (code == TSDB_CODE_MND_TABLE_ALREADY_EXIST) {
TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE");
code = taos_errno(res2);
if (code != TSDB_CODE_SUCCESS) {
tscError("SML:0x%" PRIx64 " apply schema action. reset query cache. error: %s", info->id, taos_errstr(res2));
}
taos_free_result(res2);
}
break;
}
@ -385,7 +497,7 @@ static int32_t applySchemaAction(TAOS* taos, SSchemaAction* action, SSmlLinesInf
free(result);
if (code != 0) {
tscError("SML:0x%"PRIx64 "apply schema action failure. %s", info->id, tstrerror(code));
tscError("SML:0x%"PRIx64 " apply schema action failure. %s", info->id, tstrerror(code));
}
return code;
}
@ -398,70 +510,12 @@ static int32_t destroySmlSTableSchema(SSmlSTableSchema* schema) {
return 0;
}
int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) {
int32_t code = 0;
STscObj *pObj = (STscObj *)taos;
if (pObj == NULL || pObj->signature != pObj) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return TSDB_CODE_TSC_DISCONNECTED;
}
tscDebug("SML:0x%"PRIx64" load table schema. super table name: %s", info->id, tableName);
char tableNameLowerCase[TSDB_TABLE_NAME_LEN];
strtolower(tableNameLowerCase, tableName);
char sql[256];
snprintf(sql, 256, "describe %s", tableNameLowerCase);
TAOS_RES* res = taos_query(taos, sql);
code = taos_errno(res);
if (code != 0) {
tscError("SML:0x%"PRIx64" describe table failure. %s", info->id, taos_errstr(res));
taos_free_result(res);
return code;
}
taos_free_result(res);
SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
if (pSql == NULL){
tscError("failed to allocate memory, reason:%s", strerror(errno));
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return code;
}
pSql->pTscObj = taos;
pSql->signature = pSql;
pSql->fp = NULL;
SStrToken tableToken = {.z=tableNameLowerCase, .n=(uint32_t)strlen(tableNameLowerCase), .type=TK_ID};
tGetToken(tableNameLowerCase, &tableToken.type);
// Check if the table name available or not
if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
sprintf(pSql->cmd.payload, "table name is invalid");
tscFreeSqlObj(pSql);
return code;
}
SName sname = {0};
if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) {
tscFreeSqlObj(pSql);
return code;
}
char fullTableName[TSDB_TABLE_FNAME_LEN] = {0};
memset(fullTableName, 0, tListLen(fullTableName));
tNameExtractFullName(&sname, fullTableName);
tscFreeSqlObj(pSql);
static int32_t fillDbSchema(STableMeta* tableMeta, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) {
schema->tags = taosArrayInit(8, sizeof(SSchema));
schema->fields = taosArrayInit(64, sizeof(SSchema));
schema->tagHash = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false);
schema->fieldHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, false);
size_t size = 0;
STableMeta* tableMeta = NULL;
taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void **)&tableMeta, &size);
tstrncpy(schema->sTableName, tableName, strlen(tableName)+1);
schema->precision = tableMeta->tableInfo.precision;
for (int i=0; i<tableMeta->tableInfo.numOfColumns; ++i) {
@ -484,9 +538,93 @@ int32_t loadTableMeta(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSm
size_t tagIndex = taosArrayGetSize(schema->tags) - 1;
taosHashPut(schema->tagHash, field.name, strlen(field.name), &tagIndex, sizeof(tagIndex));
}
tscDebug("SML:0x%"PRIx64 " load table meta succeed. table name: %s, columns number: %d, tag number: %d, precision: %d",
tscDebug("SML:0x%"PRIx64 " load table schema succeed. table name: %s, columns number: %d, tag number: %d, precision: %d",
info->id, tableName, tableMeta->tableInfo.numOfColumns, tableMeta->tableInfo.numOfTags, schema->precision);
free(tableMeta); tableMeta = NULL;
return TSDB_CODE_SUCCESS;
}
static int32_t retrieveTableMeta(TAOS* taos, char* tableName, STableMeta** pTableMeta, SSmlLinesInfo* info) {
int32_t code = 0;
int32_t retries = 0;
STableMeta* tableMeta = NULL;
while (retries++ < TSDB_MAX_REPLICA && tableMeta == NULL) {
STscObj* pObj = (STscObj*)taos;
if (pObj == NULL || pObj->signature != pObj) {
terrno = TSDB_CODE_TSC_DISCONNECTED;
return TSDB_CODE_TSC_DISCONNECTED;
}
tscDebug("SML:0x%" PRIx64 " retrieve table meta. super table name: %s", info->id, tableName);
char tableNameLowerCase[TSDB_TABLE_NAME_LEN];
strtolower(tableNameLowerCase, tableName);
char sql[256];
snprintf(sql, 256, "describe %s", tableNameLowerCase);
TAOS_RES* res = taos_query(taos, sql);
code = taos_errno(res);
if (code != 0) {
tscError("SML:0x%" PRIx64 " describe table failure. %s", info->id, taos_errstr(res));
taos_free_result(res);
return code;
}
taos_free_result(res);
SSqlObj* pSql = calloc(1, sizeof(SSqlObj));
if (pSql == NULL) {
tscError("SML:0x%" PRIx64 " failed to allocate memory, reason:%s", info->id, strerror(errno));
code = TSDB_CODE_TSC_OUT_OF_MEMORY;
return code;
}
pSql->pTscObj = taos;
pSql->signature = pSql;
pSql->fp = NULL;
registerSqlObj(pSql);
SStrToken tableToken = {.z = tableNameLowerCase, .n = (uint32_t)strlen(tableNameLowerCase), .type = TK_ID};
tGetToken(tableNameLowerCase, &tableToken.type);
// Check if the table name available or not
if (tscValidateName(&tableToken) != TSDB_CODE_SUCCESS) {
code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH;
sprintf(pSql->cmd.payload, "table name is invalid");
tscFreeRegisteredSqlObj(pSql);
return code;
}
SName sname = {0};
if ((code = tscSetTableFullName(&sname, &tableToken, pSql)) != TSDB_CODE_SUCCESS) {
tscFreeRegisteredSqlObj(pSql);
return code;
}
char fullTableName[TSDB_TABLE_FNAME_LEN] = {0};
memset(fullTableName, 0, tListLen(fullTableName));
tNameExtractFullName(&sname, fullTableName);
tscFreeRegisteredSqlObj(pSql);
size_t size = 0;
taosHashGetCloneExt(tscTableMetaMap, fullTableName, strlen(fullTableName), NULL, (void**)&tableMeta, &size);
}
if (tableMeta != NULL) {
*pTableMeta = tableMeta;
return TSDB_CODE_SUCCESS;
} else {
tscError("SML:0x%" PRIx64 " failed to retrieve table meta. super table name: %s", info->id, tableName);
return TSDB_CODE_TSC_NO_META_CACHED;
}
}
static int32_t loadTableSchemaFromDB(TAOS* taos, char* tableName, SSmlSTableSchema* schema, SSmlLinesInfo* info) {
int32_t code = 0;
STableMeta* tableMeta = NULL;
code = retrieveTableMeta(taos, tableName, &tableMeta, info);
if (code == TSDB_CODE_SUCCESS) {
assert(tableMeta != NULL);
fillDbSchema(tableMeta, tableName, schema, info);
free(tableMeta);
tableMeta = NULL;
}
return code;
}
@ -498,7 +636,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo*
SSmlSTableSchema dbSchema;
memset(&dbSchema, 0, sizeof(SSmlSTableSchema));
code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info);
code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info);
if (code == TSDB_CODE_MND_INVALID_TABLE_NAME) {
SSchemaAction schemaAction = {0};
schemaAction.action = SCHEMA_ACTION_CREATE_STABLE;
@ -507,7 +645,7 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo*
schemaAction.createSTable.tags = pointSchema->tags;
schemaAction.createSTable.fields = pointSchema->fields;
applySchemaAction(taos, &schemaAction, info);
code = loadTableMeta(taos, pointSchema->sTableName, &dbSchema, info);
code = loadTableSchemaFromDB(taos, pointSchema->sTableName, &dbSchema, info);
if (code != 0) {
tscError("SML:0x%"PRIx64" reconcile point schema failed. can not create %s", info->id, pointSchema->sTableName);
return code;
@ -567,74 +705,6 @@ static int32_t modifyDBSchemas(TAOS* taos, SArray* stableSchemas, SSmlLinesInfo*
return 0;
}
static int32_t getSmlMd5ChildTableName(TAOS_SML_DATA_POINT* point, char* tableName, int* tableNameLen,
SSmlLinesInfo* info) {
tscDebug("SML:0x%"PRIx64" taos_sml_insert get child table name through md5", info->id);
qsort(point->tags, point->tagNum, sizeof(TAOS_SML_KV), compareSmlColKv);
SStringBuilder sb; memset(&sb, 0, sizeof(sb));
char sTableName[TSDB_TABLE_NAME_LEN] = {0};
strtolower(sTableName, point->stableName);
taosStringBuilderAppendString(&sb, sTableName);
for (int j = 0; j < point->tagNum; ++j) {
taosStringBuilderAppendChar(&sb, ',');
TAOS_SML_KV* tagKv = point->tags + j;
char tagName[TSDB_COL_NAME_LEN] = {0};
strtolower(tagName, tagKv->key);
taosStringBuilderAppendString(&sb, tagName);
taosStringBuilderAppendChar(&sb, '=');
taosStringBuilderAppend(&sb, tagKv->value, tagKv->length);
}
size_t len = 0;
char* keyJoined = taosStringBuilderGetResult(&sb, &len);
MD5_CTX context;
MD5Init(&context);
MD5Update(&context, (uint8_t *)keyJoined, (uint32_t)len);
MD5Final(&context);
*tableNameLen = snprintf(tableName, *tableNameLen,
"t_%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x%02x", context.digest[0],
context.digest[1], context.digest[2], context.digest[3], context.digest[4], context.digest[5], context.digest[6],
context.digest[7], context.digest[8], context.digest[9], context.digest[10], context.digest[11],
context.digest[12], context.digest[13], context.digest[14], context.digest[15]);
taosStringBuilderDestroy(&sb);
tscDebug("SML:0x%"PRIx64" child table name: %s", info->id, tableName);
return 0;
}
static int32_t changeChildTableTagValue(TAOS* taos, const char* cTableName, const char* tagName, TAOS_BIND* bind, SSmlLinesInfo* info) {
char sql[512];
sprintf(sql, "alter table %s set tag %s=?", cTableName, tagName);
int32_t code;
TAOS_STMT* stmt = taos_stmt_init(taos);
code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql));
if (code != 0) {
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
return code;
}
code = taos_stmt_bind_param(stmt, bind);
if (code != 0) {
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
return code;
}
code = taos_stmt_execute(stmt);
if (code != 0) {
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
return code;
}
code = taos_stmt_close(stmt);
if (code != 0) {
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
return code;
}
return code;
}
static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, const char* sTableName,
SArray* tagsSchema, SArray* tagsBind, SSmlLinesInfo* info) {
size_t numTags = taosArrayGetSize(tagsSchema);
@ -673,28 +743,28 @@ static int32_t creatChildTableIfNotExists(TAOS* taos, const char* cTableName, co
free(sql);
if (code != 0) {
tfree(stmt);
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_prepare returns %d:%s", info->id, code, tstrerror(code));
taos_stmt_close(stmt);
return code;
}
code = taos_stmt_bind_param(stmt, TARRAY_GET_START(tagsBind));
if (code != 0) {
tfree(stmt);
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_bind_param returns %d:%s", info->id, code, tstrerror(code));
taos_stmt_close(stmt);
return code;
}
code = taos_stmt_execute(stmt);
if (code != 0) {
tfree(stmt);
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_execute returns %d:%s", info->id, code, tstrerror(code));
taos_stmt_close(stmt);
return code;
}
code = taos_stmt_close(stmt);
if (code != 0) {
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_close return %d:%s", info->id, code, tstrerror(code));
return code;
}
return code;
@ -726,27 +796,29 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols
tscDebug("SML:0x%"PRIx64" insert rows into child table %s. num of rows: %zu", info->id, cTableName, taosArrayGetSize(rowsBind));
int32_t code = 0;
int32_t try = 0;
TAOS_STMT* stmt = taos_stmt_init(taos);
if (stmt == NULL) {
tfree(sql);
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
code = taos_stmt_prepare(stmt, sql, (unsigned long)strlen(sql));
tfree(sql);
if (code != 0) {
tfree(stmt);
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_prepare return %d:%s", info->id, code, tstrerror(code));
taos_stmt_close(stmt);
return code;
}
bool tryAgain = false;
int32_t try = 0;
do {
code = taos_stmt_set_tbname(stmt, cTableName);
if (code != 0) {
tfree(stmt);
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_set_tbname return %d:%s", info->id, code, tstrerror(code));
taos_stmt_close(stmt);
return code;
}
@ -755,31 +827,52 @@ static int32_t insertChildTableBatch(TAOS* taos, char* cTableName, SArray* cols
TAOS_BIND* colsBinds = taosArrayGetP(rowsBind, i);
code = taos_stmt_bind_param(stmt, colsBinds);
if (code != 0) {
tfree(stmt);
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_bind_param return %d:%s", info->id, code, tstrerror(code));
taos_stmt_close(stmt);
return code;
}
code = taos_stmt_add_batch(stmt);
if (code != 0) {
tfree(stmt);
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_add_batch return %d:%s", info->id, code, tstrerror(code));
taos_stmt_close(stmt);
return code;
}
}
code = taos_stmt_execute(stmt);
if (code != 0) {
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
tscError("SML:0x%"PRIx64" taos_stmt_execute return %d:%s, try:%d", info->id, code, tstrerror(code), try);
}
} while (code == TSDB_CODE_TDB_TABLE_RECONFIGURE && try++ < TSDB_MAX_REPLICA);
if (code != 0) {
tscError("SML:0x%"PRIx64" %s", info->id, taos_stmt_errstr(stmt));
taos_stmt_close(stmt);
} else {
taos_stmt_close(stmt);
}
tryAgain = false;
if ((code == TSDB_CODE_TDB_INVALID_TABLE_ID
|| code == TSDB_CODE_VND_INVALID_VGROUP_ID
|| code == TSDB_CODE_TDB_TABLE_RECONFIGURE
|| code == TSDB_CODE_APP_NOT_READY
|| code == TSDB_CODE_RPC_NETWORK_UNAVAIL) && try++ < TSDB_MAX_REPLICA) {
tryAgain = true;
}
if (code == TSDB_CODE_TDB_INVALID_TABLE_ID || code == TSDB_CODE_VND_INVALID_VGROUP_ID) {
TAOS_RES* res2 = taos_query(taos, "RESET QUERY CACHE");
int32_t code2 = taos_errno(res2);
if (code2 != TSDB_CODE_SUCCESS) {
tscError("SML:0x%" PRIx64 " insert child table. reset query cache. error: %s", info->id, taos_errstr(res2));
}
taos_free_result(res2);
if (tryAgain) {
taosMsleep(50 * (2 << try));
}
}
if (code == TSDB_CODE_APP_NOT_READY || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) {
if (tryAgain) {
taosMsleep( 50 * (2 << try));
}
}
} while (tryAgain);
taos_stmt_close(stmt);
return code;
}
@ -787,16 +880,10 @@ static int32_t arrangePointsByChildTableName(TAOS_SML_DATA_POINT* points, int nu
SHashObj* cname2points, SArray* stableSchemas, SSmlLinesInfo* info) {
for (int32_t i = 0; i < numPoints; ++i) {
TAOS_SML_DATA_POINT * point = points + i;
if (!point->childTableName) {
char childTableName[TSDB_TABLE_NAME_LEN];
int32_t tableNameLen = TSDB_TABLE_NAME_LEN;
getSmlMd5ChildTableName(point, childTableName, &tableNameLen, info);
point->childTableName = calloc(1, tableNameLen+1);
strncpy(point->childTableName, childTableName, tableNameLen);
point->childTableName[tableNameLen] = '\0';
}
SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, point->schemaIdx);
uintptr_t valPointer = (uintptr_t)point;
size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t));
assert(pSchemaIndex != NULL);
SSmlSTableSchema* stableSchema = taosArrayGet(stableSchemas, *pSchemaIndex);
for (int j = 0; j < point->tagNum; ++j) {
TAOS_SML_KV* kv = point->tags + j;
@ -840,16 +927,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam
TAOS_SML_DATA_POINT * pDataPoint = taosArrayGetP(cTablePoints, i);
for (int j = 0; j < pDataPoint->tagNum; ++j) {
TAOS_SML_KV* kv = pDataPoint->tags + j;
tagKVs[kv->fieldSchemaIdx] = kv;
}
}
int32_t notNullTagsIndices[TSDB_MAX_TAGS] = {0};
int32_t numNotNullTags = 0;
for (int32_t i = 0; i < numTags; ++i) {
if (tagKVs[i] != NULL) {
notNullTagsIndices[numNotNullTags] = i;
++numNotNullTags;
uintptr_t valPointer = (uintptr_t)kv;
size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t));
assert(pFieldSchemaIdx != NULL);
tagKVs[*pFieldSchemaIdx] = kv;
}
}
@ -863,7 +944,10 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam
for (int j = 0; j < numTags; ++j) {
if (tagKVs[j] == NULL) continue;
TAOS_SML_KV* kv = tagKVs[j];
TAOS_BIND* bind = taosArrayGet(tagBinds, kv->fieldSchemaIdx);
uintptr_t valPointer = (uintptr_t)kv;
size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t));
assert(pFieldSchemaIdx != NULL);
TAOS_BIND* bind = taosArrayGet(tagBinds, *pFieldSchemaIdx);
bind->buffer_type = kv->type;
bind->length = malloc(sizeof(uintptr_t*));
*bind->length = kv->length;
@ -871,65 +955,8 @@ static int32_t applyChildTableTags(TAOS* taos, char* cTableName, char* sTableNam
bind->is_null = NULL;
}
// select tag1,tag2,... from stable where tbname in (ctable)
char* sql = malloc(tsMaxSQLStringLen+1);
int freeBytes = tsMaxSQLStringLen + 1;
snprintf(sql, freeBytes, "select tbname, ");
for (int i = 0; i < numNotNullTags ; ++i) {
snprintf(sql + strlen(sql), freeBytes-strlen(sql), "%s,", tagKVs[notNullTagsIndices[i]]->key);
}
snprintf(sql + strlen(sql) - 1, freeBytes - strlen(sql) + 1,
" from %s where tbname in (\'%s\')", sTableName, cTableName);
sql[strlen(sql)] = '\0';
int32_t code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info);
TAOS_RES* result = taos_query(taos, sql);
free(sql);
int32_t code = taos_errno(result);
if (code != 0) {
tscError("SML:0x%"PRIx64" get child table %s tags failed. error string %s", info->id, cTableName, taos_errstr(result));
goto cleanup;
}
// check tag value and set tag values if different
TAOS_ROW row = taos_fetch_row(result);
if (row != NULL) {
int numFields = taos_field_count(result);
TAOS_FIELD* fields = taos_fetch_fields(result);
int* lengths = taos_fetch_lengths(result);
for (int i = 1; i < numFields; ++i) {
uint8_t dbType = fields[i].type;
int32_t length = lengths[i];
char* val = row[i];
TAOS_SML_KV* tagKV = tagKVs[notNullTagsIndices[i-1]];
if (tagKV->type != dbType) {
tscError("SML:0x%"PRIx64" child table %s tag %s type mismatch. point type : %d, db type : %d",
info->id, cTableName, tagKV->key, tagKV->type, dbType);
return TSDB_CODE_TSC_INVALID_VALUE;
}
assert(tagKV->value);
if (val == NULL || length != tagKV->length || memcmp(tagKV->value, val, length) != 0) {
TAOS_BIND* bind = taosArrayGet(tagBinds, tagKV->fieldSchemaIdx);
code = changeChildTableTagValue(taos, cTableName, tagKV->key, bind, info);
if (code != 0) {
tscError("SML:0x%"PRIx64" change child table tag failed. table name %s, tag %s", info->id, cTableName, tagKV->key);
goto cleanup;
}
}
}
tscDebug("SML:0x%"PRIx64" successfully applied point tags. child table: %s", info->id, cTableName);
} else {
code = creatChildTableIfNotExists(taos, cTableName, sTableName, sTableSchema->tags, tagBinds, info);
if (code != 0) {
goto cleanup;
}
}
cleanup:
taos_free_result(result);
for (int i = 0; i < taosArrayGetSize(tagBinds); ++i) {
TAOS_BIND* bind = taosArrayGet(tagBinds, i);
free(bind->length);
@ -963,7 +990,10 @@ static int32_t applyChildTableFields(TAOS* taos, SSmlSTableSchema* sTableSchema,
}
for (int j = 0; j < point->fieldNum; ++j) {
TAOS_SML_KV* kv = point->fields + j;
TAOS_BIND* bind = colBinds + kv->fieldSchemaIdx;
uintptr_t valPointer = (uintptr_t)kv;
size_t* pFieldSchemaIdx = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t));
assert(pFieldSchemaIdx != NULL);
TAOS_BIND* bind = colBinds + *pFieldSchemaIdx;
bind->buffer_type = kv->type;
bind->length = malloc(sizeof(uintptr_t*));
*bind->length = kv->length;
@ -1000,9 +1030,11 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t
while (pCTablePoints) {
SArray* cTablePoints = *pCTablePoints;
TAOS_SML_DATA_POINT* point = taosArrayGetP(cTablePoints, 0);
SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, point->schemaIdx);
uintptr_t valPointer = (uintptr_t)point;
size_t* pSchemaIndex = taosHashGet(info->smlDataToSchema, &valPointer, sizeof(uintptr_t));
assert(pSchemaIndex != NULL);
SSmlSTableSchema* sTableSchema = taosArrayGet(stableSchemas, *pSchemaIndex);
tscDebug("SML:0x%"PRIx64" apply child table tags. child table: %s", info->id, point->childTableName);
code = applyChildTableTags(taos, point->childTableName, point->stableName, sTableSchema, cTablePoints, info);
@ -1014,7 +1046,7 @@ static int32_t applyDataPoints(TAOS* taos, TAOS_SML_DATA_POINT* points, int32_t
tscDebug("SML:0x%"PRIx64" apply child table points. child table: %s", info->id, point->childTableName);
code = applyChildTableFields(taos, sTableSchema, point->childTableName, cTablePoints, info);
if (code != 0) {
tscError("Apply child table fields failed. child table %s, error %s", point->childTableName, tstrerror(code));
tscError("SML:0x%"PRIx64" Apply child table fields failed. child table %s, error %s", info->id, point->childTableName, tstrerror(code));
goto cleanup;
}
@ -1034,10 +1066,11 @@ cleanup:
return code;
}
int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) {
int tscSmlInsert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint, SSmlLinesInfo* info) {
tscDebug("SML:0x%"PRIx64" taos_sml_insert. number of points: %d", info->id, numPoint);
int32_t code = TSDB_CODE_SUCCESS;
info->smlDataToSchema = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, false);
tscDebug("SML:0x%"PRIx64" build data point schemas", info->id);
SArray* stableSchemas = taosArrayInit(32, sizeof(SSmlSTableSchema)); // SArray<STableColumnsSchema>
@ -1067,6 +1100,15 @@ clean_up:
taosArrayDestroy(schema->tags);
}
taosArrayDestroy(stableSchemas);
taosHashCleanup(info->smlDataToSchema);
return code;
}
int taos_sml_insert(TAOS* taos, TAOS_SML_DATA_POINT* points, int numPoint) {
SSmlLinesInfo* info = calloc(1, sizeof(SSmlLinesInfo));
info->id = genLinesSmlId();
int code = tscSmlInsert(taos, points, numPoint, info);
free(info);
return code;
}
@ -2076,18 +2118,6 @@ int32_t tscParseLine(const char* sql, TAOS_SML_DATA_POINT* smlData, SSmlLinesInf
//=========================================================================
static uint64_t linesSmlHandleId = 0;
uint64_t genLinesSmlId() {
uint64_t id;
do {
id = atomic_add_fetch_64(&linesSmlHandleId, 1);
} while (id == 0);
return id;
}
void destroySmlDataPoint(TAOS_SML_DATA_POINT* point) {
for (int i=0; i<point->tagNum; ++i) {
free((point->tags+i)->key);
@ -2157,7 +2187,7 @@ int taos_insert_lines(TAOS* taos, char* lines[], int numLines) {
}
TAOS_SML_DATA_POINT* points = TARRAY_GET_START(lpPoints);
code = taos_sml_insert(taos, points, (int)numPoints, info);
code = tscSmlInsert(taos, points, (int)numPoints, info);
if (code != 0) {
tscError("SML:0x%"PRIx64" taos_sml_insert error: %s", info->id, tstrerror((code)));
}

4
src/client/src/tscPrepare.c
View File

@ -1540,6 +1540,8 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
pRes->qId = 0;
pRes->numOfRows = 1;
registerSqlObj(pSql);
strtolower(pSql->sqlstr, sql);
tscDebugL("0x%"PRIx64" SQL: %s", pSql->self, pSql->sqlstr);
@ -1549,8 +1551,6 @@ int taos_stmt_prepare(TAOS_STMT* stmt, const char* sql, unsigned long length) {
pSql->cmd.insertParam.numOfParams = 0;
pSql->cmd.batchSize = 0;
registerSqlObj(pSql);
int32_t ret = stmtParseInsertTbTags(pSql, pStmt);
if (ret != TSDB_CODE_SUCCESS) {
STMT_RET(ret);

67
src/client/src/tscSQLParser.c
View File

@ -117,7 +117,7 @@ static int32_t validateColumnName(char* name);
static int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType);
static int32_t setCompactVnodeInfo(SSqlObj* pSql, struct SSqlInfo* pInfo);
static bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField);
static int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField);
static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo);
static bool hasNormalColumnFilter(SQueryInfo* pQueryInfo);
@ -1538,9 +1538,7 @@ static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pC
/*
* tags name /column name is truncated in sql.y
*/
bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
//const char* msg1 = "timestamp not allowed in tags";
const char* msg2 = "duplicated column names";
int32_t validateOneTag(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
const char* msg3 = "tag length too long";
const char* msg4 = "invalid tag name";
const char* msg5 = "invalid binary/nchar tag length";
@ -1555,8 +1553,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
// no more max columns
if (numOfTags + numOfCols >= TSDB_MAX_COLUMNS) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7);
}
// no more than 6 tags
@ -1564,8 +1561,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
char msg[128] = {0};
sprintf(msg, "tags no more than %d", TSDB_MAX_TAGS);
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg);
}
// no timestamp allowable
@ -1575,8 +1571,7 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
//}
if ((pTagField->type < TSDB_DATA_TYPE_BOOL) || (pTagField->type > TSDB_DATA_TYPE_UBIGINT)) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
SSchema* pTagSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);
@ -1588,20 +1583,17 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
// length less than TSDB_MAX_TASG_LEN
if (nLen + pTagField->bytes > TSDB_MAX_TAGS_LEN) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
// tags name can not be a keyword
if (validateColumnName(pTagField->name) != TSDB_CODE_SUCCESS) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
// binary(val), val can not be equalled to or less than 0
if ((pTagField->type == TSDB_DATA_TYPE_BINARY || pTagField->type == TSDB_DATA_TYPE_NCHAR) && pTagField->bytes <= 0) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
// field name must be unique
@ -1609,17 +1601,15 @@ bool validateOneTags(SSqlCmd* pCmd, TAOS_FIELD* pTagField) {
for (int32_t i = 0; i < numOfTags + numOfCols; ++i) {
if (strncasecmp(pTagField->name, pSchema[i].name, sizeof(pTagField->name) - 1) == 0) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
return false;
return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pTagField->name, NULL);
}
}
return true;
return TSDB_CODE_SUCCESS;
}
bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
int32_t validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
const char* msg1 = "too many columns";
const char* msg2 = "duplicated column names";
const char* msg3 = "column length too long";
const char* msg4 = "invalid data type";
const char* msg5 = "invalid column name";
@ -1634,18 +1624,15 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
// no more max columns
if (numOfCols >= TSDB_MAX_COLUMNS || numOfTags + numOfCols >= TSDB_MAX_COLUMNS) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
if (pColField->type < TSDB_DATA_TYPE_BOOL || pColField->type > TSDB_DATA_TYPE_UBIGINT) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
if (validateColumnName(pColField->name) != TSDB_CODE_SUCCESS) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5);
}
SSchema* pSchema = tscGetTableSchema(pTableMeta);
@ -1656,25 +1643,22 @@ bool validateOneColumn(SSqlCmd* pCmd, TAOS_FIELD* pColField) {
}
if (pColField->bytes <= 0) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
// length less than TSDB_MAX_BYTES_PER_ROW
if (nLen + pColField->bytes > TSDB_MAX_BYTES_PER_ROW) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
return false;
return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
// field name must be unique
for (int32_t i = 0; i < numOfTags + numOfCols; ++i) {
if (strncasecmp(pColField->name, pSchema[i].name, sizeof(pColField->name) - 1) == 0) {
invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2);
return false;
return tscErrorMsgWithCode(TSDB_CODE_TSC_DUP_COL_NAMES, tscGetErrorMsgPayload(pCmd), pColField->name, NULL);
}
}
return true;
return TSDB_CODE_SUCCESS;
}
/* is contained in pFieldList or not */
@ -6068,7 +6052,6 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
const char* msg19 = "invalid new tag name";
const char* msg20 = "table is not super table";
const char* msg21 = "only binary/nchar column length could be modified";
const char* msg22 = "new column length should be bigger than old one";
const char* msg23 = "only column length coulbe be modified";
const char* msg24 = "invalid binary/nchar column length";
@ -6120,8 +6103,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
TAOS_FIELD* p = taosArrayGet(pFieldList, 0);
if (!validateOneTags(pCmd, p)) {
return TSDB_CODE_TSC_INVALID_OPERATION;
int32_t ret = validateOneTag(pCmd, p);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
@ -6298,8 +6282,9 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
TAOS_FIELD* p = taosArrayGet(pFieldList, 0);
if (!validateOneColumn(pCmd, p)) {
return TSDB_CODE_TSC_INVALID_OPERATION;
int32_t ret = validateOneColumn(pCmd, p);
if (ret != TSDB_CODE_SUCCESS) {
return ret;
}
tscFieldInfoAppend(&pQueryInfo->fieldsInfo, p);
@ -6362,7 +6347,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
if (pItem->bytes <= pColSchema->bytes) {
return invalidOperationMsg(pMsg, msg22);
return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, pMsg, pItem->name, NULL);
}
SSchema* pSchema = (SSchema*) pTableMetaInfo->pTableMeta->schema;
@ -6413,7 +6398,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
}
if (pItem->bytes <= pColSchema->bytes) {
return invalidOperationMsg(pMsg, msg22);
return tscErrorMsgWithCode(TSDB_CODE_TSC_INVALID_TAG_LENGTH, pMsg, pItem->name, NULL);
}
SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);

25
src/client/src/tscUtil.c
View File

@ -4168,6 +4168,31 @@ int32_t tscInvalidOperationMsg(char* msg, const char* additionalInfo, const char
return TSDB_CODE_TSC_INVALID_OPERATION;
}
int32_t tscErrorMsgWithCode(int32_t code, char* dstBuffer, const char* errMsg, const char* sql) {
const char* msgFormat1 = "%s:%s";
const char* msgFormat2 = "%s:\'%s\' (%s)";
const char* msgFormat3 = "%s:\'%s\'";
const int32_t BACKWARD_CHAR_STEP = 0;
if (sql == NULL) {
assert(errMsg != NULL);
sprintf(dstBuffer, msgFormat1, tstrerror(code), errMsg);
return code;
}
char buf[64] = {0}; // only extract part of sql string
strncpy(buf, (sql - BACKWARD_CHAR_STEP), tListLen(buf) - 1);
if (errMsg != NULL) {
sprintf(dstBuffer, msgFormat2, tstrerror(code), buf, errMsg);
} else {
sprintf(dstBuffer, msgFormat3, tstrerror(code), buf); // no additional information for invalid sql error
}
return code;
}
bool tscHasReachLimitation(SQueryInfo* pQueryInfo, SSqlRes* pRes) {
assert(pQueryInfo != NULL && pQueryInfo->clauseLimit != 0);
return (pQueryInfo->clauseLimit > 0 && pRes->numOfClauseTotal >= pQueryInfo->clauseLimit);

6
src/inc/taoserror.h
View File

@ -103,6 +103,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSC_FILE_EMPTY TAOS_DEF_ERROR_CODE(0, 0x021A) //"File is empty")
#define TSDB_CODE_TSC_LINE_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x021B) //"Syntax error in Line")
#define TSDB_CODE_TSC_NO_META_CACHED TAOS_DEF_ERROR_CODE(0, 0x021C) //"No table meta cached")
#define TSDB_CODE_TSC_DUP_COL_NAMES TAOS_DEF_ERROR_CODE(0, 0x021D) //"duplicated column names")
#define TSDB_CODE_TSC_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021E) //"Invalid tag length")
#define TSDB_CODE_TSC_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x021F) //"Invalid column length")
// mnode
#define TSDB_CODE_MND_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0300) //"Message not processed")
@ -185,6 +188,9 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_INVALID_FUNC TAOS_DEF_ERROR_CODE(0, 0x0374) //"Invalid func")
#define TSDB_CODE_MND_INVALID_FUNC_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x0375) //"Invalid func bufSize")
#define TSDB_CODE_MND_INVALID_TAG_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0376) //"invalid tag length")
#define TSDB_CODE_MND_INVALID_COLUMN_LENGTH TAOS_DEF_ERROR_CODE(0, 0x0377) //"invalid column length")
#define TSDB_CODE_MND_DB_NOT_SELECTED TAOS_DEF_ERROR_CODE(0, 0x0380) //"Database not specified or available")
#define TSDB_CODE_MND_DB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0381) //"Database already exists")
#define TSDB_CODE_MND_INVALID_DB_OPTION TAOS_DEF_ERROR_CODE(0, 0x0382) //"Invalid database options")

17
src/mnode/src/mnodeTable.c
View File

@ -1246,13 +1246,13 @@ static int32_t mnodeAddSuperTableTag(SMnodeMsg *pMsg, SSchema schema[], int32_t
if (mnodeFindSuperTableColumnIndex(pStable, schema[i].name) > 0) {
mError("msg:%p, app:%p stable:%s, add tag, column:%s already exist", pMsg, pMsg->rpcMsg.ahandle,
pStable->info.tableId, schema[i].name);
return TSDB_CODE_MND_TAG_ALREAY_EXIST;
return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
}
if (mnodeFindSuperTableTagIndex(pStable, schema[i].name) > 0) {
mError("msg:%p, app:%p stable:%s, add tag, tag:%s already exist", pMsg, pMsg->rpcMsg.ahandle,
pStable->info.tableId, schema[i].name);
return TSDB_CODE_MND_FIELD_ALREAY_EXIST;
return TSDB_CODE_MND_TAG_ALREAY_EXIST;
}
}
@ -1518,6 +1518,13 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg) {
// update
SSchema *schema = (SSchema *) (pStable->schema + col);
ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR);
if (pAlter->schema[0].bytes <= schema->bytes) {
mError("msg:%p, app:%p stable:%s, modify column len. column:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle,
pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes);
return TSDB_CODE_MND_INVALID_COLUMN_LENGTH;
}
schema->bytes = pAlter->schema[0].bytes;
pStable->sversion++;
mInfo("msg:%p, app:%p stable %s, start to modify column %s len to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,
@ -1548,6 +1555,12 @@ static int32_t mnodeChangeSuperTableTag(SMnodeMsg *pMsg) {
// update
SSchema *schema = (SSchema *) (pStable->schema + col + pStable->numOfColumns);
ASSERT(schema->type == TSDB_DATA_TYPE_BINARY || schema->type == TSDB_DATA_TYPE_NCHAR);
if (pAlter->schema[0].bytes <= schema->bytes) {
mError("msg:%p, app:%p stable:%s, modify tag len. tag:%s, len from %d to %d", pMsg, pMsg->rpcMsg.ahandle,
pStable->info.tableId, name, schema->bytes, pAlter->schema[0].bytes);
return TSDB_CODE_MND_INVALID_TAG_LENGTH;
}
schema->bytes = pAlter->schema[0].bytes;
pStable->tversion++;
mInfo("msg:%p, app:%p stable %s, start to modify tag len %s to %d", pMsg, pMsg->rpcMsg.ahandle, pStable->info.tableId,

6
src/util/src/terror.c
View File

@ -112,6 +112,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too lon
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DUP_COL_NAMES, "duplicated column names")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TAG_LENGTH, "Invalid tag length")
TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_COLUMN_LENGTH, "Invalid column length")
// mnode
TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, "Message not processed")
@ -194,6 +197,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_FUNC_ALREADY_EXIST, "Func already exists")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC, "Invalid func")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_FUNC_BUFSIZE, "Invalid func bufSize")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_TAG_LENGTH, "invalid tag length")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COLUMN_LENGTH, "invalid column length")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_NOT_SELECTED, "Database not specified or available")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_DB_ALREADY_EXIST, "Database already exists")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_DB_OPTION, "Invalid database options")

118
tests/examples/c/schemaless.c
View File

@ -61,7 +61,7 @@ int main(int argc, char* argv[]) {
time_t ct = time(0);
int64_t ts = ct * 1000;
char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms";
char* lineFormat = "sta%d,t0=true,t1=127i8,t2=32767i16,t3=%di32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=254u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" %lldms";
char** lines = calloc(numSuperTables * numChildTables * numRowsPerChildTable, sizeof(char*));
int l = 0;
@ -75,7 +75,7 @@ int main(int argc, char* argv[]) {
}
}
}
shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable);
//shuffle(lines, numSuperTables * numChildTables * numRowsPerChildTable);
printf("%s\n", "begin taos_insert_lines");
int64_t begin = getTimeInUs();
@ -83,119 +83,5 @@ int main(int argc, char* argv[]) {
int64_t end = getTimeInUs();
printf("code: %d, %s. time used: %"PRId64"\n", code, tstrerror(code), end-begin);
char* lines_000_0[] = {
"sta1,id=sta1_1,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us"
};
code = taos_insert_lines(taos, lines_000_0 , sizeof(lines_000_0)/sizeof(char*));
if (0 == code) {
printf("taos_insert_lines() lines_000_0 should return error\n");
return -1;
}
char* lines_000_1[] = {
"sta2,id=\"sta2_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=255u8,t6=32770u16,t7=2147483699u32,t8=9223372036854775899u64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639001"
};
code = taos_insert_lines(taos, lines_000_1 , sizeof(lines_000_1)/sizeof(char*));
if (0 == code) {
printf("taos_insert_lines() lines_000_1 should return error\n");
return -1;
}
char* lines_000_2[] = {
"sta3,id=\"sta3_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=255u8,c6=32770u16,c7=2147483699u32,c8=9223372036854775899u64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0"
};
code = taos_insert_lines(taos, lines_000_2 , sizeof(lines_000_2)/sizeof(char*));
if (0 != code) {
printf("taos_insert_lines() lines_000_2 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
char* lines_001_0[] = {
"sta4,t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000us",
};
code = taos_insert_lines(taos, lines_001_0 , sizeof(lines_001_0)/sizeof(char*));
if (0 != code) {
printf("taos_insert_lines() lines_001_0 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
char* lines_001_1[] = {
"sta5,id=\"sta5_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639001"
};
code = taos_insert_lines(taos, lines_001_1 , sizeof(lines_001_1)/sizeof(char*));
if (0 != code) {
printf("taos_insert_lines() lines_001_1 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
char* lines_001_2[] = {
"sta6,id=\"sta6_1\",t0=true,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t9=11.12345f32,t10=22.123456789f64,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c0=true,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c9=11.12345f32,c10=22.123456789f64,c11=\"binaryValue\",c12=L\"ncharValue\" 0"
};
code = taos_insert_lines(taos, lines_001_2 , sizeof(lines_001_2)/sizeof(char*));
if (0 != code) {
printf("taos_insert_lines() lines_001_2 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
char* lines_002[] = {
"stb,id=\"stb_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639000000ns",
"stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833639019us",
"stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006833640ms",
"stc,id=\"stc_1\",t20=t,t21=T,t22=true,t23=True,t24=TRUE,t25=f,t26=F,t27=false,t28=False,t29=FALSE,t10=33.12345,t11=\"binaryTagValue\",t12=L\"ncharTagValue\" c20=t,c21=T,c22=true,c23=True,c24=TRUE,c25=f,c26=F,c27=false,c28=False,c29=FALSE,c10=33.12345,c11=\"binaryValue\",c12=L\"ncharValue\" 1626006834s"
};
code = taos_insert_lines(taos, lines_002 , sizeof(lines_002)/sizeof(char*));
if (0 != code) {
printf("taos_insert_lines() lines_002 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
//Duplicate key check;
char* lines_003_1[] = {
"std,id=\"std_3_1\",t1=4i64,Id=\"std\",t2=true c1=true 1626006834s"
};
code = taos_insert_lines(taos, lines_003_1 , sizeof(lines_003_1)/sizeof(char*));
if (0 == code) {
printf("taos_insert_lines() lines_003_1 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
char* lines_003_2[] = {
"std,id=\"std_3_2\",tag1=4i64,Tag2=true,tAg3=2,TaG2=\"dup!\" c1=true 1626006834s"
};
code = taos_insert_lines(taos, lines_003_2 , sizeof(lines_003_2)/sizeof(char*));
if (0 == code) {
printf("taos_insert_lines() lines_003_2 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
char* lines_003_3[] = {
"std,id=\"std_3_3\",tag1=4i64 field1=true,Field2=2,FIElD1=\"dup!\",fIeLd4=true 1626006834s"
};
code = taos_insert_lines(taos, lines_003_3 , sizeof(lines_003_3)/sizeof(char*));
if (0 == code) {
printf("taos_insert_lines() lines_003_3 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
char* lines_003_4[] = {
"std,id=\"std_3_4\",tag1=4i64,dupkey=4i16,tag2=T field1=true,dUpkEy=1e3f32,field2=\"1234\" 1626006834s"
};
code = taos_insert_lines(taos, lines_003_4 , sizeof(lines_003_4)/sizeof(char*));
if (0 == code) {
printf("taos_insert_lines() lines_003_4 return code:%d (%s)\n", code, (char*)tstrerror(code));
return -1;
}
return 0;
}

545
tests/pytest/query/queryDiffColsOr.py Normal file
View File

@ -0,0 +1,545 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
from copy import deepcopy
from util.log import tdLog
from util.cases import tdCases
from util.sql import tdSql
from util.common import tdCom
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
def insertData(self, tb_name):
insert_sql_list = [f'insert into {tb_name} values ("2021-01-01 12:00:00", 1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)',
f'insert into {tb_name} values ("2021-01-05 12:00:00", 2, 2, 1, 3, 1.1, 1.1, "binary", "nchar", true, 2)',
f'insert into {tb_name} values ("2021-01-07 12:00:00", 1, 3, 1, 2, 1.1, 1.1, "binary", "nchar", true, 3)',
f'insert into {tb_name} values ("2021-01-09 12:00:00", 1, 2, 4, 3, 1.1, 1.1, "binary", "nchar", true, 4)',
f'insert into {tb_name} values ("2021-01-11 12:00:00", 1, 2, 5, 5, 1.1, 1.1, "binary", "nchar", true, 5)',
f'insert into {tb_name} values ("2021-01-13 12:00:00", 1, 2, 1, 3, 6.6, 1.1, "binary", "nchar", true, 6)',
f'insert into {tb_name} values ("2021-01-15 12:00:00", 1, 2, 1, 3, 1.1, 7.7, "binary", "nchar", true, 7)',
f'insert into {tb_name} values ("2021-01-17 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary8", "nchar", true, 8)',
f'insert into {tb_name} values ("2021-01-19 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar9", true, 9)',
f'insert into {tb_name} values ("2021-01-21 12:00:00", 1, 2, 1, 3, 1.1, 1.1, "binary", "nchar", false, 10)',
f'insert into {tb_name} values ("2021-01-23 12:00:00", 1, 3, 1, 3, 1.1, 1.1, Null, Null, false, 11)'
]
for sql in insert_sql_list:
tdSql.execute(sql)
def initTb(self):
tdCom.cleanTb()
tb_name = tdCom.getLongName(8, "letters")
tdSql.execute(
f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int)")
self.insertData(tb_name)
return tb_name
def initStb(self):
tdCom.cleanTb()
tb_name = tdCom.getLongName(8, "letters")
tdSql.execute(
f"CREATE TABLE {tb_name} (ts timestamp, c1 tinyint, c2 smallint, c3 int, c4 bigint, c5 float, c6 double, c7 binary(100), c8 nchar(200), c9 bool, c10 int) tags (t1 tinyint, t2 smallint, t3 int, t4 bigint, t5 float, t6 double, t7 binary(100), t8 nchar(200), t9 bool, t10 int)")
tdSql.execute(
f'CREATE TABLE {tb_name}_sub using {tb_name} tags (1, 1, 1, 3, 1.1, 1.1, "binary", "nchar", true, 1)')
self.insertData(f'{tb_name}_sub')
return tb_name
def queryLastC10(self, query_sql, multi=False):
if multi:
res = tdSql.query(query_sql.replace('c10', 'last(*)'), True)
else:
res = tdSql.query(query_sql.replace('*', 'last(*)'), True)
return int(res[0][-1])
def queryFullColType(self, tb_name):
## ts
query_sql = f'select * from {tb_name} where ts > "2021-01-11 12:00:00" or ts < "2021-01-13 12:00:00"'
tdSql.query(query_sql)
tdSql.checkRows(11)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## != or
query_sql = f'select * from {tb_name} where c1 != 1 or c2 = 3'
tdSql.query(query_sql)
tdSql.checkRows(3)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## <> or
query_sql = f'select * from {tb_name} where c1 <> 1 or c3 = 3'
tdSql.query(query_sql)
tdSql.checkRows(1)
tdSql.checkEqual(self.queryLastC10(query_sql), 2)
## >= or
query_sql = f'select * from {tb_name} where c1 >= 2 or c3 = 4'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 4)
## <= or
query_sql = f'select * from {tb_name} where c1 <= 1 or c3 = 4'
tdSql.query(query_sql)
tdSql.checkRows(10)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## <> or is Null
query_sql = f'select * from {tb_name} where c1 <> 1 or c7 is Null'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## > or is not Null
query_sql = f'select * from {tb_name} where c2 > 2 or c8 is not Null'
tdSql.query(query_sql)
tdSql.checkRows(11)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## > or < or >= or <= or != or <> or = Null
query_sql = f'select * from {tb_name} where c1 > 1 or c2 < 2 or c3 >= 4 or c4 <= 2 or c5 != 1.1 or c6 <> 1.1 or c7 is Null'
tdSql.query(query_sql)
tdSql.checkRows(8)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## tiny small int big or
query_sql = f'select * from {tb_name} where c1 = 2 or c2 = 3 or c3 = 4 or c4 = 5'
tdSql.query(query_sql)
tdSql.checkRows(5)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## float double binary nchar bool or
query_sql = f'select * from {tb_name} where c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false'
tdSql.query(query_sql)
tdSql.checkRows(6)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## all types or
query_sql = f'select * from {tb_name} where c1=2 or c2=3 or c3=4 or c4=5 or c5=6.6 or c6=7.7 or c7="binary8" or c8="nchar9" or c9=false'
tdSql.query(query_sql)
tdSql.checkRows(10)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
def queryMultiExpression(self, tb_name):
## condition_A and condition_B or condition_C (> < >=)
query_sql = f'select * from {tb_name} where c1 > 2 and c2 < 4 or c3 >= 4'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 5)
## (condition_A and condition_B) or condition_C (<= != <>)
query_sql = f'select * from {tb_name} where (c1 <= 1 and c2 != 2) or c4 <> 3'
tdSql.query(query_sql)
tdSql.checkRows(4)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## condition_A and (condition_B or condition_C) (Null not Null)
query_sql = f'select * from {tb_name} where c1 is not Null and (c6 = 7.7 or c8 is Null)'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## condition_A or condition_B and condition_C (> < >=)
query_sql = f'select * from {tb_name} where c1 > 2 or c2 < 4 and c3 >= 4'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 5)
## (condition_A or condition_B) and condition_C (<= != <>)
query_sql = f'select * from {tb_name} where (c1 <= 1 or c2 != 2) and c4 <> 3'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 5)
## condition_A or (condition_B and condition_C) (Null not Null)
query_sql = f'select * from {tb_name} where c6 >= 7.7 or (c1 is not Null and c3 =5)'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 7)
## condition_A or (condition_B and condition_C) or condition_D (> != < Null)
query_sql = f'select * from {tb_name} where c1 != 1 or (c2 >2 and c3 < 1) or c7 is Null'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## condition_A and (condition_B or condition_C) and condition_D (>= = <= not Null)
query_sql = f'select * from {tb_name} where c4 >= 4 and (c1 = 2 or c5 <= 1.1) and c7 is not Null'
tdSql.query(query_sql)
tdSql.checkRows(1)
tdSql.checkEqual(self.queryLastC10(query_sql), 5)
## (condition_A and condition_B) or (condition_C or condition_D) (Null >= > =)
query_sql = f'select * from {tb_name} where (c8 is Null and c1 >= 1) or (c3 > 3 or c4 =2)'
tdSql.query(query_sql)
tdSql.checkRows(4)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## (condition_A or condition_B) or condition_C or (condition_D and condition_E) (>= <= = not Null <>)
query_sql = f'select * from {tb_name} where (c1 >= 2 or c2 <= 1) or c3 = 4 or (c7 is not Null and c6 <> 1.1)'
tdSql.query(query_sql)
tdSql.checkRows(4)
tdSql.checkEqual(self.queryLastC10(query_sql), 7)
## condition_A or (condition_B and condition_C) or (condition_D and condition_E) and condition_F
query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 1 and c3 <4) or (c3 >= 4 or c7 is not Null) and c9 <> true'
tdSql.query(query_sql)
tdSql.checkRows(3)
tdSql.checkEqual(self.queryLastC10(query_sql), 10)
## (condition_A or (condition_B and condition_C) or (condition_D and condition_E)) and condition_F
query_sql = f'select * from {tb_name} where (c1 != 1 or (c2 <= 2 and c3 >= 4) or (c3 >= 4 or c7 is not Null)) and c9 != false'
tdSql.query(query_sql)
tdSql.checkRows(9)
tdSql.checkEqual(self.queryLastC10(query_sql), 9)
## (condition_A or condition_B) or (condition_C or condition_D) and (condition_E or condition_F or condition_G)
query_sql = f'select * from {tb_name} where c1 != 1 or (c2 <= 3 and c3 > 4) and c3 <= 5 and (c7 is not Null and c9 != false)'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 5)
def queryMultiIn(self, tb_name):
## in and in
query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar")'
tdSql.query(query_sql)
tdSql.checkRows(8)
tdSql.checkEqual(self.queryLastC10(query_sql), 10)
## in or in
query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (1, 4)'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 2)
## in and in or condition_A
query_sql = f'select * from {tb_name} where c7 in ("binary") and c8 in ("nchar") or c10 != 10'
tdSql.query(query_sql)
tdSql.checkRows(11)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## in or in and condition_A
query_sql = f'select * from {tb_name} where c7 in ("binary") or c8 in ("nchar") and c10 != 10'
tdSql.query(query_sql)
tdSql.checkRows(10)
tdSql.checkEqual(self.queryLastC10(query_sql), 10)
## in or in or condition_A
query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 != true'
tdSql.query(query_sql)
tdSql.checkRows(4)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## in or in or in or in
query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) or c9 in (false) or c10 in (5, 6 ,22)'
tdSql.query(query_sql)
tdSql.checkRows(6)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## in or in and in or in
query_sql = f'select * from {tb_name} where c1 in (2, 4) or c2 in (3, 4) and c9 in (false) or c10 in (5, 6 ,22)'
tdSql.query(query_sql)
tdSql.checkRows(4)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## condition_A or in or condition_B and in
query_sql = f'select * from {tb_name} where c1 = 2 or c2 in (2, 4) and c9 = false or c10 in (6 ,22)'
tdSql.query(query_sql)
tdSql.checkRows(3)
tdSql.checkEqual(self.queryLastC10(query_sql), 10)
## in and condition_A or in and in and condition_B
query_sql = f'select * from {tb_name} where c1 in (2, 3) and c2 <> 3 or c10 <= 4 and c10 in (4 ,22) and c9 != false'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 4)
## (in and condition_A or in) and in and condition_B
query_sql = f'select * from {tb_name} where (c1 in (2, 3) and c2 <> 3 or c10 <= 4) and c10 in (4 ,22) and c9 != false'
tdSql.query(query_sql)
tdSql.checkRows(1)
tdSql.checkEqual(self.queryLastC10(query_sql), 4)
def queryMultiLike(self, tb_name):
## like and like
query_sql = f'select * from {tb_name} where c7 like "bi%" and c8 like ("ncha_")'
tdSql.query(query_sql)
tdSql.checkRows(9)
tdSql.checkEqual(self.queryLastC10(query_sql), 10)
## like or like
query_sql = f'select * from {tb_name} where c7 like "binar12345" or c8 like "nchar_"'
tdSql.query(query_sql)
tdSql.checkRows(1)
tdSql.checkEqual(self.queryLastC10(query_sql), 9)
## like and like or condition_A
query_sql = f'select * from {tb_name} where c7 like "binary_" and c8 like "ncha_" or c1 != 1'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 8)
## like or like and condition_A
query_sql = f'select * from {tb_name} where c7 like ("binar_") or c8 like ("nchar_") and c10 != 8'
tdSql.query(query_sql)
tdSql.checkRows(9)
tdSql.checkEqual(self.queryLastC10(query_sql), 10)
## like or like or condition_A
query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6'
tdSql.query(query_sql)
tdSql.checkRows(3)
tdSql.checkEqual(self.queryLastC10(query_sql), 9)
## like or like or like or like
query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("nchar_") or c10 = 6 or c7 is Null'
tdSql.query(query_sql)
tdSql.checkRows(4)
tdSql.checkEqual(self.queryLastC10(query_sql), 11)
## like or like and like or like
query_sql = f'select * from {tb_name} where c7 like ("binary_") or c8 like ("ncha_") and c10 = 6 or c10 = 9'
tdSql.query(query_sql)
tdSql.checkRows(3)
tdSql.checkEqual(self.queryLastC10(query_sql), 9)
## condition_A or like or condition_B and like
query_sql = f'select * from {tb_name} where c1 = 2 or c7 like "binary_" or c10 = 3 and c8 like "ncha%"'
tdSql.query(query_sql)
tdSql.checkRows(3)
tdSql.checkEqual(self.queryLastC10(query_sql), 8)
## like and condition_A or like and like and condition_B
query_sql = f'select * from {tb_name} where c7 like "bin%" and c2 = 3 or c10 <= 4 and c7 like "binar_" and c8 like "ncha_"'
tdSql.query(query_sql)
tdSql.checkRows(4)
tdSql.checkEqual(self.queryLastC10(query_sql), 4)
## (like and condition_A or like) and like and condition_B
query_sql = f'select * from {tb_name} where (c7 like "bin%" and c2 = 3 or c8 like "nchar_") and c7 like "binar_" and c9 != false'
tdSql.query(query_sql)
tdSql.checkRows(2)
tdSql.checkEqual(self.queryLastC10(query_sql), 9)
def queryPreCal(self, tb_name):
## avg sum condition_A or condition_B
query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 5 or c8 is Null'
res = tdSql.query(query_sql, True)[0]
tdSql.checkEqual(int(res[0]), 3)
tdSql.checkEqual(int(res[1]), 6)
## avg sum condition_A or condition_B or condition_C
query_sql = f'select avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false '
res = tdSql.query(query_sql, True)[0]
tdSql.checkEqual(int(res[0]), 2)
tdSql.checkEqual(int(res[1]), 6)
## count avg sum condition_A or condition_B or condition_C interval
query_sql = f'select count(*), avg(c3), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c9 = false interval(16d)'
res = tdSql.query(query_sql, True)
tdSql.checkRows(2)
tdSql.checkEqual(int(res[0][1]), 1)
tdSql.checkEqual(int(res[0][2]), 4)
tdSql.checkEqual(int(res[0][3]), 4)
tdSql.checkEqual(int(res[1][1]), 2)
tdSql.checkEqual(int(res[1][2]), 1)
tdSql.checkEqual(int(res[1][3]), 2)
## count avg sum condition_A or condition_B or in and like or condition_C interval
query_sql = f'select count(*), sum(c3) from {tb_name} where c10 = 4 or c8 is Null or c2 in (1, 2) and c7 like "binary_" or c1 <> 1 interval(16d)'
res = tdSql.query(query_sql, True)
tdSql.checkRows(2)
tdSql.checkEqual(int(res[0][1]), 2)
tdSql.checkEqual(int(res[0][2]), 5)
tdSql.checkEqual(int(res[1][1]), 2)
tdSql.checkEqual(int(res[1][2]), 2)
def queryMultiTb(self, tb_name):
## select from (condition_A or condition_B)
query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3)'
res = tdSql.query(query_sql, True)
tdSql.checkRows(3)
tdSql.checkEqual(int(res[2][0]), 11)
## select from (condition_A or condition_B) where condition_A or condition_B
query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 >=3) where c1 =2 or c4 = 2'
res = tdSql.query(query_sql, True)
tdSql.checkRows(2)
tdSql.checkEqual(int(res[1][0]), 3)
## select from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in
query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true)'
res = tdSql.query(query_sql, True)
tdSql.checkRows(7)
tdSql.checkEqual(int(res[6][0]), 10)
## select count avg sum from (condition_A or condition_B and like and in) where condition_A or condition_B or like and in interval
query_sql = f'select count(*), avg(c6), sum(c3) from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) where c1 != 2 or c3 = 1 or c8 like "ncha_" and c9 in (true) interval(8d)'
res = tdSql.query(query_sql, True)
tdSql.checkRows(3)
tdSql.checkEqual(int(res[0][1]), 3)
tdSql.checkEqual(int(res[0][2]), 1)
tdSql.checkEqual(int(res[0][3]), 10)
tdSql.checkEqual(int(res[1][1]), 3)
tdSql.checkEqual(int(res[1][2]), 3)
tdSql.checkEqual(int(res[1][3]), 3)
tdSql.checkEqual(int(res[2][1]), 1)
tdSql.checkEqual(int(res[2][2]), 1)
tdSql.checkEqual(int(res[2][3]), 1)
## cname
query_sql = f'select c10 from (select * from {tb_name} where c1 >1 or c2 = 2 and c7 like "binar_" and c4 in (3, 5)) a where a.c1 != 2 or a.c3 = 1 or a.c8 like "ncha_" and a.c9 in (true)'
res = tdSql.query(query_sql, True)
tdSql.checkRows(7)
tdSql.checkEqual(int(res[6][0]), 10)
## multi cname
query_sql = f'select b.c10 from (select * from {tb_name} where c9 = true or c2 = 2) a, (select * from {tb_name} where c7 like "binar_" or c4 in (3, 5)) b where a.ts = b.ts'
res = tdSql.query(query_sql, True)
tdSql.checkRows(10)
tdSql.checkEqual(int(res[9][0]), 10)
def checkTbColTypeOperator(self):
'''
Ordinary table full column type and operator
'''
tb_name = self.initTb()
self.queryFullColType(tb_name)
def checkStbColTypeOperator(self):
'''
Super table full column type and operator
'''
tb_name = self.initStb()
self.queryFullColType(tb_name)
def checkTbMultiExpression(self):
'''
Ordinary table multiExpression
'''
tb_name = self.initTb()
self.queryMultiExpression(tb_name)
def checkStbMultiExpression(self):
'''
Super table multiExpression
'''
tb_name = self.initStb()
self.queryMultiExpression(tb_name)
def checkTbMultiIn(self):
'''
Ordinary table multiIn
'''
tb_name = self.initTb()
self.queryMultiIn(tb_name)
def checkStbMultiIn(self):
'''
Super table multiIn
'''
tb_name = self.initStb()
self.queryMultiIn(tb_name)
def checkTbMultiLike(self):
'''
Ordinary table multiLike
'''
tb_name = self.initTb()
self.queryMultiLike(tb_name)
def checkStbMultiLike(self):
'''
Super table multiLike
'''
tb_name = self.initStb()
self.queryMultiLike(tb_name)
def checkTbPreCal(self):
'''
Ordinary table precal
'''
tb_name = self.initTb()
self.queryPreCal(tb_name)
def checkStbPreCal(self):
'''
Super table precal
'''
tb_name = self.initStb()
self.queryPreCal(tb_name)
def checkMultiTb(self):
'''
test "or" in multi ordinary table
'''
tb_name = self.initTb()
self.queryMultiTb(tb_name)
def checkMultiStb(self):
'''
test "or" in multi super table
'''
tb_name = self.initStb()
self.queryMultiTb(tb_name)
# tb_name1 = tdCom.getLongName(8, "letters")
# tb_name2 = tdCom.getLongName(8, "letters")
# tb_name3 = tdCom.getLongName(8, "letters")
# tdSql.execute(
# f"CREATE TABLE {tb_name1} (ts timestamp, c1 tinyint, c2 smallint, c3 int)")
# tdSql.execute(
# f"CREATE TABLE {tb_name2} (ts timestamp, c1 tinyint, c2 smallint, c3 int)")
# tdSql.execute(
# f"CREATE TABLE {tb_name3} (ts timestamp, c1 tinyint, c2 smallint, c3 int)")
# insert_sql_list = [f'insert into {tb_name1} values ("2021-01-01 12:00:00", 1, 5, 1)',
# f'insert into {tb_name1} values ("2021-01-03 12:00:00", 2, 4, 1)',
# f'insert into {tb_name1} values ("2021-01-05 12:00:00", 3, 2, 1)',
# f'insert into {tb_name2} values ("2021-01-01 12:00:00", 4, 2, 1)',
# f'insert into {tb_name2} values ("2021-01-02 12:00:00", 5, 1, 1)',
# f'insert into {tb_name2} values ("2021-01-04 12:00:00", 1, 2, 1)',
# f'insert into {tb_name3} values ("2021-01-02 12:00:00", 4, 2, 1)',
# f'insert into {tb_name3} values ("2021-01-06 12:00:00", 5, 1, 1)',
# f'insert into {tb_name3} values ("2021-01-07 12:00:00", 1, 2, 1)',
# ]
# for sql in insert_sql_list:
# tdSql.execute(sql)
# tdSql.query(
# f'select * from {tb_name1} t1, {tb_name2}, {tb_name3} t3 t2 where (t1.ts=t2.ts or t2.ts=t3.ts)')
# tdSql.checkRows(4)
def run(self):
tdSql.prepare()
self.checkTbColTypeOperator()
self.checkStbColTypeOperator()
self.checkTbMultiExpression()
self.checkStbMultiExpression()
self.checkTbMultiIn()
self.checkStbMultiIn()
self.checkTbMultiLike()
self.checkStbMultiLike()
self.checkTbPreCal()
self.checkStbPreCal()
self.checkMultiTb()
self.checkMultiStb()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

301
tests/pytest/tools/schemalessInsertPerformance.py Normal file
View File

@ -0,0 +1,301 @@
###################################################################
# Copyright (c) 2021 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import traceback
import random
import string
from taos.error import LinesError
import datetime
import time
from copy import deepcopy
import numpy as np
from util.log import *
from util.cases import *
from util.sql import *
from util.common import tdCom
import threading
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug("start to execute %s" % __file__)
tdSql.init(conn.cursor(), logSql)
self._conn = conn
def genRandomTs(self):
year = random.randint(2000, 2021)
month = random.randint(10, 12)
day = random.randint(10, 29)
hour = random.randint(10, 24)
minute = random.randint(10, 59)
second = random.randint(10, 59)
m_second = random.randint(101, 199)
date_time = f'{year}-{month}-{day} {hour}:{minute}:{second}'
print(date_time)
timeArray = time.strptime(date_time, "%Y-%m-%d %H:%M:%S")
ts = int(time.mktime(timeArray))
print("------", ts)
# timestamp = time.mktime(datetime.datetime.strptime(date_time, "%Y-%m-%d %H:%M:%S.%f").timetuple())
return f'{ts}s'
def genMultiColStr(self, int_count=4, double_count=0, binary_count=0):
"""
genType must be tag/col
"""
col_str = ""
if double_count == 0 and binary_count == 0:
for i in range(0, int_count):
if i < (int_count-1):
col_str += f'c{i}={random.randint(0, 255)}i32,'
else:
col_str += f'c{i}={random.randint(0, 255)}i32 '
elif double_count > 0 and binary_count == 0:
for i in range(0, int_count):
col_str += f'c{i}={random.randint(0, 255)}i32,'
for i in range(0, double_count):
if i < (double_count-1):
col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,'
else:
col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64 '
elif double_count == 0 and binary_count > 0:
for i in range(0, int_count):
col_str += f'c{i}={random.randint(0, 255)}i32,'
for i in range(0, binary_count):
if i < (binary_count-1):
col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\",'
else:
col_str += f'c{i+int_count}=\"{tdCom.getLongName(5, "letters")}\" '
elif double_count > 0 and binary_count > 0:
for i in range(0, int_count):
col_str += f'c{i}={random.randint(0, 255)}i32,'
for i in range(0, double_count):
col_str += f'c{i+int_count}={random.randint(1, 255)}.{i}f64,'
for i in range(0, binary_count):
if i < (binary_count-1):
col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\",'
else:
col_str += f'c{i+int_count+double_count}=\"{tdCom.getLongName(5, "letters")}\" '
return col_str
def genLongSql(self, int_count=4, double_count=0, binary_count=0, init=False):
if init:
tag_str = f'id="init",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"'
else:
tag_str = f'id="sub_{tdCom.getLongName(5, "letters")}_{tdCom.getLongName(5, "letters")}",t0={random.randint(0, 65535)}i32,t1=\"{tdCom.getLongName(10, "letters")}\"'
col_str = self.genMultiColStr(int_count, double_count, binary_count)
long_sql = 'stb' + ',' + tag_str + ' ' + col_str + '0'
return long_sql
def getPerfSql(self, count=4, init=False):
if count == 4:
input_sql = self.genLongSql(init=init)
elif count == 1000:
input_sql = self.genLongSql(400, 400, 200, init=init)
elif count == 4000:
input_sql = self.genLongSql(1900, 1900, 200, init=init)
return input_sql
def tableGenerator(self, count=4, table_count=1000):
for i in range(table_count):
yield self.getPerfSql(count)
def genTableList(self, count=4, table_count=10000):
table_list = list()
for i in range(1, table_count+1):
table_list.append(self.getPerfSql(count))
return table_list
def splitTableList(self, count=4, thread_count=10, table_count=1000):
per_list_len = int(table_count/thread_count)
table_list = self.genTableList(count=count)
# ts = int(time.time())
list_of_group = zip(*(iter(table_list),) *per_list_len)
end_list = [list(i) for i in list_of_group] # i is a tuple
count = len(table_list) % per_list_len
end_list.append(table_list[-count:]) if count !=0 else end_list
return table_list, end_list
def rowsGenerator(self, end_list):
ts = int(time.time())
input_sql_list = list()
for elm_list in end_list:
for elm in elm_list:
for i in range(1, 10000):
ts -= 1
elm_new = self.replaceLastStr(elm, str(ts)) + 's'
input_sql_list.append(elm_new)
yield input_sql_list
# def insertRows(self, count=4, thread_count=10):
# table_list = self.splitTableList(count=count, thread_count=thread_count)[0]
# for
def replaceLastStr(self, str, new):
list_ori = list(str)
list_ori[-1] = new
return ''.join(list_ori)
def genDataList(self, table_list, row_count=10):
data_list = list()
ts = int(time.time())
for table_str in table_list:
for i in range(1, row_count+1):
ts -= 1
table_str_new = self.replaceLastStr(table_str, f'{str(ts)}s')
data_list.append(table_str_new)
print(data_list)
return data_list
def insertRows(self, count=4, table_count=1000):
table_generator = self.tableGenerator(count=count, table_count=table_count)
for table_name in table_generator:
pass
def perfTableInsert(self):
table_generator = self.tableGenerator()
for input_sql in table_generator:
self._conn.insert_lines([input_sql])
# for i in range(10):
# self._conn.insert_lines([input_sql])
def perfDataInsert(self, count=4):
table_generator = self.tableGenerator(count=count)
ts = int(time.time())
for input_sql in table_generator:
print("input_sql-----------", input_sql)
self._conn.insert_lines([input_sql])
for i in range(100000):
ts -= 1
input_sql_new = self.replaceLastStr(input_sql, str(ts)) + 's'
print("input_sql_new---------", input_sql_new)
self._conn.insert_lines([input_sql_new])
def batchInsertTable(self, batch_list):
for insert_list in batch_list:
print(threading.current_thread().name, "length=", len(insert_list))
print(threading.current_thread().name, 'firstline', insert_list[0])
print(threading.current_thread().name, 'lastline:', insert_list[-1])
self._conn.insert_lines(insert_list)
print(threading.current_thread().name, 'end')
def genTableThread(self, thread_count=10):
threads = list()
for i in range(thread_count):
t = threading.Thread(target=self.perfTableInsert)
threads.append(t)
return threads
def genMultiThread(self, count, thread_count=10):
threads = list()
for i in range(thread_count):
t = threading.Thread(target=self.perfDataInsert,args=(count,))
threads.append(t)
return threads
def multiThreadRun(self, threads):
for t in threads:
t.start()
for t in threads:
t.join()
def createStb(self, count=4):
input_sql = self.getPerfSql(count=count, init=True)
self._conn.insert_lines([input_sql])
def threadInsertTable(self, end_list, thread_count=10):
threads = list()
for i in range(thread_count):
t = threading.Thread(target=self.batchInsertTable, args=(end_list,))
threads.append(t)
return threads
def finalRun(self):
self.createStb()
table_list, end_list = self.splitTableList()
batchInsertTableThread = self.threadInsertTable(end_list=end_list)
self.multiThreadRun(batchInsertTableThread)
# print(end_list)
# def createTb(self, count=4):
# input_sql = self.getPerfSql(count=count)
# for i in range(10000):
# self._conn.insert_lines([input_sql])
# def createTb1(self, count=4):
# start_time = time.time()
# self.multiThreadRun(self.genMultiThread(input_sql))
# end_time = time.time()
# return end_time - start_time
# def calInsertTableTime(self):
# start_time = time.time()
# self.createStb()
# self.multiThreadRun(self.genMultiThread())
# end_time = time.time()
# return end_time - start_time
def calRunTime(self, count=4):
start_time = time.time()
self.createStb()
self.multiThreadRun(self.genMultiThread(count=count))
end_time = time.time()
return end_time - start_time
def calRunTime1(self, count=4):
start_time = time.time()
self.createStb()
self.multiThreadRun(self.perfTableInsert())
# self.perfTableInsert()
# def schemalessInsertPerfTest(self, count=4):
# input_sql = self.getPerfSql(count)
# self.calRunTime(input_sql)
# def test(self):
# sql1 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=37i32,c1=217i32,c2=3i32,c3=88i32 1626006833640ms'
# sql2 = 'stb,id="init",t0=14865i32,t1="tvnqbjuqck" c0=38i32,c1=217i32,c2=3i32,c3=88i32 1626006833641ms'
# self._conn.insert_lines([sql1])
# self._conn.insert_lines([sql2])
def run(self):
print("running {}".format(__file__))
tdSql.prepare()
self.finalRun()
# print(self.calRunTime1(count=4))
# print(self.calRunTime(count=4))
# print(self.genRandomTs())
# self.calInsertTableTime()
# self.test()
# table_list = self.splitTableList()[0]
# data_list = self.genDataList(table_list)
# print(len(data_list))
# end_list = [['stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0','stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0'], ['stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0','stb,id="sub_vzvfx_dbuxp",t0=9961i32,t1="zjjfayhfep" c0=83i32,c1=169i32,c2=177i32,c3=4i32 0']]
# rowsGenerator = self.rowsGenerator(end_list)
# for i in rowsGenerator:
# print(i)
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

4
tests/pytest/util/common.py
View File

@ -1,4 +1,4 @@
###################################################################
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
@ -50,4 +50,4 @@ class TDCom:
def close(self):
self.cursor.close()
tdCom = TDCom()
tdCom = TDCom()