xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #23532 from taosdata/szhou/continue-coing 

enhance: insert into super table performance improvement
This commit is contained in:
dapan1121 2023-11-06 08:43:33 +08:00 committed by GitHub
parent 0af6f2fb0e e1ea15c8f2
commit faf7001828
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 54 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
source/libs/parser/src/parInsertSql.c
View File

@ -1594,11 +1594,9 @@ static int32_t parseTbnameToken(SInsertParseContext* pCxt, SStbRowsDataContext*
if (pToken->n > 0) {
if (pToken->n <= TSDB_TABLE_NAME_LEN - 1) {
memcpy(ctbName, pToken->z, pToken->n);
ctbName[pToken->n] = '\0';
memcpy(pStbRowsCxt->ctbName.tname, pToken->z, pToken->n);
pStbRowsCxt->ctbName.tname[pToken->n] = '\0';
*pFoundCtbName = true;
tNameSetDbName(&pStbRowsCxt->ctbName, pStbRowsCxt->stbName.acctId, pStbRowsCxt->stbName.dbname, strlen(pStbRowsCxt->stbName.dbname));
tNameAddTbName(&pStbRowsCxt->ctbName, ctbName, pToken->n);
} else {
return buildInvalidOperationMsg(&pCxt->msg, "tbname is too long");
}
@ -1644,10 +1642,13 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
SArray* pTagNames = pStbRowsCxt->aTagNames;
SArray* pTagVals = pStbRowsCxt->aTagVals;
bool canParseTagsAfter = !pStbRowsCxt->pTagCond && !pStbRowsCxt->hasTimestampTag;
int32_t numOfCols = getNumOfColumns(pStbRowsCxt->pStbMeta);
int32_t tbnameIdx = getTbnameSchemaIndex(pStbRowsCxt->pStbMeta);
for (int i = 0; i < pCols->numOfBound && (code) == TSDB_CODE_SUCCESS; ++i) {
const char* pTmpSql = *ppSql;
bool ignoreComma = false;
NEXT_TOKEN_WITH_PREV_EXT(*ppSql, *pToken, &ignoreComma);
if (ignoreComma) {
code = buildSyntaxErrMsg(&pCxt->msg, "invalid data or symbol", pTmpSql);
break;
@ -1657,12 +1658,11 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
code = generateSyntaxErrMsg(&pCxt->msg, TSDB_CODE_PAR_INVALID_COLUMNS_NUM);
break;
}
if (pCols->pColIndex[i] < getNumOfColumns(pStbRowsCxt->pStbMeta)) {
if (pCols->pColIndex[i] < numOfCols) {
const SSchema* pSchema = &pSchemas[pCols->pColIndex[i]];
SColVal* pVal = taosArrayGet(pStbRowsCxt->aColVals, pCols->pColIndex[i]);
code = parseValueToken(pCxt, ppSql, pToken, (SSchema*)pSchema, getTableInfo(pStbRowsCxt->pStbMeta).precision, pVal);
} else if (pCols->pColIndex[i] < getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) {
} else if (pCols->pColIndex[i] < tbnameIdx) {
const SSchema* pTagSchema = &pSchemas[pCols->pColIndex[i]];
if (canParseTagsAfter) {
tagTokens[(*pNumOfTagTokens)] = *pToken;
@ -1675,7 +1675,7 @@ static int32_t doGetStbRowValues(SInsertParseContext* pCxt, SVnodeModifyOpStmt*
}
}
}
else if (pCols->pColIndex[i] == getTbnameSchemaIndex(pStbRowsCxt->pStbMeta)) {
else if (pCols->pColIndex[i] == tbnameIdx) {
char ctbName[TSDB_TABLE_NAME_LEN];
code = parseTbnameToken(pCxt, pStbRowsCxt, pToken, ctbName, bFoundTbName);
}
@ -1771,12 +1771,6 @@ static int32_t processCtbAutoCreationAndCtbMeta(SInsertParseContext* pCxt, SVnod
return code;
}
static void resetStbRowsDataContextPreStbRow(SStbRowsDataContext* pStbRowsCxt) {
pStbRowsCxt->pCtbMeta->tableType = TSDB_CHILD_TABLE;
pStbRowsCxt->pCtbMeta->suid = pStbRowsCxt->pStbMeta->uid;
insInitColValues(pStbRowsCxt->pStbMeta, pStbRowsCxt->aColVals);
}
static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) {
if (pStbRowsCxt == NULL) return;
@ -1791,19 +1785,15 @@ static void clearStbRowsDataContext(SStbRowsDataContext* pStbRowsCxt) {
taosArrayClear(pStbRowsCxt->aTagVals);
clearColValArray(pStbRowsCxt->aColVals);
taosArrayClear(pStbRowsCxt->aColVals);
tTagFree(pStbRowsCxt->pTag);
pStbRowsCxt->pTag = NULL;
pStbRowsCxt->pCtbMeta->uid = 0;
pStbRowsCxt->pCtbMeta->vgId = 0;
tdDestroySVCreateTbReq(pStbRowsCxt->pCreateCtbReq);
taosMemoryFreeClear(pStbRowsCxt->pCreateCtbReq);
}
static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt, const char** ppSql,
SStbRowsDataContext* pStbRowsCxt, bool* pGotRow, SToken* pToken) {
resetStbRowsDataContextPreStbRow(pStbRowsCxt);
bool bFirstTable = false;
int32_t code = getStbRowValues(pCxt, pStmt, ppSql, pStbRowsCxt, pGotRow, pToken, &bFirstTable);
if (code != TSDB_CODE_SUCCESS || !*pGotRow) {
@ -1825,12 +1815,13 @@ static int32_t parseOneStbRow(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pSt
insCheckTableDataOrder(pTableDataCxt, TD_ROW_KEY(*pRow));
}
}
if (code == TSDB_CODE_SUCCESS) {
*pGotRow = true;
}
clearStbRowsDataContext(pStbRowsCxt);
return TSDB_CODE_SUCCESS;
}
@ -2100,6 +2091,10 @@ static int32_t constructStbRowsDataContext(SVnodeModifyOpStmt* pStmt, SStbRowsDa
collectUseTable(&pStbRowsCxt->stbName, pStmt->pTableNameHashObj);
collectUseDatabase(&pStbRowsCxt->stbName, pStmt->pDbFNameHashObj);
pStbRowsCxt->ctbName.type = TSDB_TABLE_NAME_T;
pStbRowsCxt->ctbName.acctId = pStbRowsCxt->stbName.acctId;
memcpy(pStbRowsCxt->ctbName.dbname, pStbRowsCxt->stbName.dbname, sizeof(pStbRowsCxt->stbName.dbname));
pStbRowsCxt->pTagCond = pStmt->pTagCond;
pStbRowsCxt->pStbMeta = pStmt->pTableMeta;

62
tests/script/api/insert_stb.c
View File

@ -21,9 +21,16 @@
#include <stdlib.h>
#include <string.h>
#include <sys/time.h>
#include <stdint.h>
#include "taos.h" // TAOS header file
static int64_t currTimeInUs() {
struct timeval start_time;
gettimeofday(&start_time, NULL);
return (start_time.tv_sec) * 1000000 + (start_time.tv_usec);
}
static void executeSql(TAOS *taos, char *command) {
int i;
TAOS_RES *pSql = NULL;
@ -52,7 +59,7 @@ static void executeSql(TAOS *taos, char *command) {
taos_free_result(pSql);
}
void testInsert(TAOS *taos, char *qstr) {
void testInsert(TAOS *taos, char *qstr, double* pElapsedTime) {
executeSql(taos, "drop database if exists demo2");
executeSql(taos, "create database demo2");
executeSql(taos, "use demo2");
@ -60,8 +67,7 @@ void testInsert(TAOS *taos, char *qstr) {
executeSql(taos, "create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10)) tags(t1 int, t2 float, t3 binary(10))");
printf("success to create table\n");
struct timeval start_time;
gettimeofday(&start_time, NULL);
int64_t ts1 = currTimeInUs();
for (int tblIdx = 0; tblIdx < 10; ++tblIdx) {
int len = 0;
@ -84,15 +90,16 @@ void testInsert(TAOS *taos, char *qstr) {
taos_free_result(result1);
}
}
struct timeval end_time;
gettimeofday(&end_time, NULL);
double elapsed_time = (double)(end_time.tv_sec - start_time.tv_sec) +
(double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
printf("elapsed time: %.3f\n", elapsed_time);
int64_t ts2 = currTimeInUs();
double elapsedTime = (double)(ts2-ts1) / 1000000.0;
*pElapsedTime = elapsedTime;
printf("elapsed time: %.3f\n", elapsedTime);
executeSql(taos, "drop database if exists demo2");
}
void testInsertStb(TAOS *taos, char *qstr) {
void testInsertStb(TAOS *taos, char *qstr, double *pElapsedTime) {
executeSql(taos, "drop database if exists demo");
executeSql(taos, "create database demo");
executeSql(taos, "use demo");
@ -100,8 +107,7 @@ void testInsertStb(TAOS *taos, char *qstr) {
executeSql(taos, "create table st (ts timestamp, ti tinyint, si smallint, i int, bi bigint, f float, d double, b binary(10)) tags(t1 int, t2 float, t3 binary(10))");
printf("success to create table\n");
struct timeval start_time;
gettimeofday(&start_time, NULL);
int64_t ts1 = currTimeInUs();
for (int tblIdx = 0; tblIdx < 10; ++tblIdx) {
int len = 0;
@ -125,12 +131,11 @@ void testInsertStb(TAOS *taos, char *qstr) {
taos_free_result(result1);
}
}
struct timeval end_time;
gettimeofday(&end_time, NULL);
double elapsed_time = (double)(end_time.tv_sec - start_time.tv_sec) +
(double)(end_time.tv_usec - start_time.tv_usec) / 1000000.0;
printf("elapsed time: %.3f\n", elapsed_time);
int64_t ts2 = currTimeInUs();
double elapsedTime = (double)(ts2 - ts1) / 1000000.0;
*pElapsedTime = elapsedTime;
printf("elapsed time: %.3f\n", elapsedTime);
executeSql(taos, "drop database if exists demo");
}
@ -149,13 +154,26 @@ int main(int argc, char *argv[]) {
exit(1);
}
char* qstr = malloc(1024*1024);
printf("test insert into tb using stb\n\n");
for (int i =0; i < 5; ++i) {
testInsert(taos, qstr);
{
printf("test insert into tb using stb\n\n");
double sum = 0;
for (int i =0; i < 5; ++i) {
double elapsed = 0;
testInsert(taos, qstr, &elapsed);
sum += elapsed;
}
printf("average insert tb using stb time : %.3f\n", sum/5);
}
printf("test insert into stb tbname\n\n");
for (int i =0; i < 5; ++i) {
testInsertStb(taos, qstr);
{
printf("test insert into stb\n\n");
double sum = 0;
for (int i =0; i < 5; ++i) {
double elapsed = 0;
testInsertStb(taos, qstr, &elapsed);
sum += elapsed;
}
printf("average insert into stb time : %.3f\n", sum/5);
}
free(qstr);
taos_close(taos);