xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge pull request #7460 from taosdata/enhance/TD-6184-D 

[TD-6184]<enhance>: optimize insert from imported file
This commit is contained in:
Hongze Cheng 2021-08-19 15:04:57 +08:00 committed by GitHub
parent 49925cccf0 021fc6af21
commit 619d9712c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 8 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/client/inc/tsclient.h
View File

@ -123,17 +123,15 @@ typedef struct {
int32_t kvLen; // len of SKVRow int32_t kvLen; // len of SKVRow
} SMemRowInfo; } SMemRowInfo;
typedef struct { typedef struct {
uint8_t memRowType; uint8_t memRowType; // default is 0, that is SDataRow
uint8_t compareStat; // 0 unknown, 1 need compare, 2 no need uint8_t compareStat; // 0 no need, 1 need compare
TDRowTLenT dataRowInitLen;
TDRowTLenT kvRowInitLen; TDRowTLenT kvRowInitLen;
SMemRowInfo *rowInfo; SMemRowInfo *rowInfo;
} SMemRowBuilder; } SMemRowBuilder;
typedef enum { typedef enum {
ROW_COMPARE_UNKNOWN = 0, ROW_COMPARE_NO_NEED = 0,
ROW_COMPARE_NEED = 1, ROW_COMPARE_NEED = 1,
ROW_COMPARE_NO_NEED = 2,
} ERowCompareStat; } ERowCompareStat;
int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec); int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec);

20
src/client/src/tscParseInsert.c
View File

@ -51,20 +51,18 @@ int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint3
} }
} }
// default compareStat is ROW_COMPARE_NO_NEED
if (nBoundCols == 0) { // file input if (nBoundCols == 0) { // file input
pBuilder->memRowType = SMEM_ROW_DATA; pBuilder->memRowType = SMEM_ROW_DATA;
pBuilder->compareStat = ROW_COMPARE_NO_NEED;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else { } else {
float boundRatio = ((float)nBoundCols / (float)nCols); float boundRatio = ((float)nBoundCols / (float)nCols);
if (boundRatio < KVRatioKV) { if (boundRatio < KVRatioKV) {
pBuilder->memRowType = SMEM_ROW_KV; pBuilder->memRowType = SMEM_ROW_KV;
pBuilder->compareStat = ROW_COMPARE_NO_NEED;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} else if (boundRatio > KVRatioData) { } else if (boundRatio > KVRatioData) {
pBuilder->memRowType = SMEM_ROW_DATA; pBuilder->memRowType = SMEM_ROW_DATA;
pBuilder->compareStat = ROW_COMPARE_NO_NEED;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
pBuilder->compareStat = ROW_COMPARE_NEED; pBuilder->compareStat = ROW_COMPARE_NEED;
@ -76,7 +74,6 @@ int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint3
} }
} }
pBuilder->dataRowInitLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx); pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx);
if (nRows > 0) { if (nRows > 0) {
@ -86,7 +83,7 @@ int initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint3
} }
for (int i = 0; i < nRows; ++i) { for (int i = 0; i < nRows; ++i) {
(pBuilder->rowInfo + i)->dataLen = pBuilder->dataRowInitLen; (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
(pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen; (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen;
} }
} }
@ -460,7 +457,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
STableMeta * pTableMeta = pDataBlocks->pTableMeta; STableMeta * pTableMeta = pDataBlocks->pTableMeta;
SSchema * schema = tscGetTableSchema(pTableMeta); SSchema * schema = tscGetTableSchema(pTableMeta);
SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder; SMemRowBuilder * pBuilder = &pDataBlocks->rowBuilder;
int32_t dataLen = pBuilder->dataRowInitLen; int32_t dataLen = spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE;
int32_t kvLen = pBuilder->kvRowInitLen; int32_t kvLen = pBuilder->kvRowInitLen;
bool isParseBindParam = false; bool isParseBindParam = false;
@ -809,13 +806,12 @@ int tscSortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlk
// allocate memory // allocate memory
size_t nAlloc = nRows * sizeof(SBlockKeyTuple); size_t nAlloc = nRows * sizeof(SBlockKeyTuple);
if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) { if (pBlkKeyInfo->pKeyTuple == NULL || pBlkKeyInfo->maxBytesAlloc < nAlloc) {
size_t nRealAlloc = nAlloc + 10 * sizeof(SBlockKeyTuple); char *tmp = trealloc(pBlkKeyInfo->pKeyTuple, nAlloc);
char * tmp = trealloc(pBlkKeyInfo->pKeyTuple, nRealAlloc);
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp; pBlkKeyInfo->pKeyTuple = (SBlockKeyTuple *)tmp;
pBlkKeyInfo->maxBytesAlloc = (int32_t)nRealAlloc; pBlkKeyInfo->maxBytesAlloc = (int32_t)nAlloc;
} }
memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc); memset(pBlkKeyInfo->pKeyTuple, 0, nAlloc);
@ -1726,12 +1722,6 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow
goto _error; goto _error;
} }
if (TSDB_CODE_SUCCESS !=
(ret = initMemRowBuilder(&pTableDataBlock->rowBuilder, 0, tinfo.numOfColumns, pTableDataBlock->numOfParams,
pTableDataBlock->boundColumnInfo.allNullLen))) {
goto _error;
}
while ((readLen = tgetline(&line, &n, fp)) != -1) { while ((readLen = tgetline(&line, &n, fp)) != -1) {
if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) { if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {
line[--readLen] = 0; line[--readLen] = 0;