From 2d2316eeec7d105fa38769bbf90116346729a319 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 12 Mar 2021 15:32:58 +0800 Subject: [PATCH 1/5] [td-3277]: fix the taosd crash caused by fetching the too long binary data in tags. --- src/query/src/qExecutor.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f8119b0d4a..57efcf4b34 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2913,7 +2913,9 @@ static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, return; } - tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), type); + int32_t maxLen = bytes - VARSTR_HEADER_SIZE; + int32_t len = (varDataLen(val) > maxLen)? maxLen:varDataLen(val); + tVariantCreateFromBinary(tag, varDataVal(val), len, type); } else { if (isNull(val, type)) { tag->nType = TSDB_DATA_TYPE_NULL; @@ -7070,8 +7072,15 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type return; } - if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - memcpy(output, val, varDataTLen(val)); + if (IS_VAR_DATA_TYPE(type)) { + // Binary data overflows for sort of unknown reasons. Let trim the overflow data + if (varDataTLen(val) > bytes) { + int32_t len = bytes; + memcpy(varDataVal(output), varDataVal(val), len); + varDataSetLen(output, len); + } else { + varDataCopy(output, val); + } } else { memcpy(output, val, bytes); } From 1e6cc776f83a6726188a3dcdf57b8799029f7cbf Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 12 Mar 2021 17:24:18 +0800 Subject: [PATCH 2/5] add tag value check to create table --- src/inc/taoserror.h | 1 + src/tsdb/src/tsdbMeta.c | 29 +++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 3eb197868b..97749f5b78 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -235,6 +235,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO TAOS_DEF_ERROR_CODE(0, 0x0612) //"Invalid information to create table") #define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) //"No available disk") #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) //"TSDB messed message") +#define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) //"TSDB invalid tag value") // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 5e2e0fce1d..cdde2bc91f 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -20,7 +20,7 @@ static int tsdbCompareSchemaVersion(const void *key1, const void *key2); static char * getTagIndexKey(const void *pData); static STable *tsdbNewTable(); -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper); +static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable); static void tsdbFreeTable(STable *pTable); static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx, bool lock); static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock); @@ -43,6 +43,7 @@ static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STabl static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); +static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema); // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { @@ -87,7 +88,7 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { super = tsdbGetTableByUid(pMeta, pCfg->superUid); if (super == NULL) { // super table not exists, try to create it newSuper = true; - super = tsdbCreateTableFromCfg(pCfg, true); + super = tsdbCreateTableFromCfg(pCfg, true, NULL); if (super == NULL) goto _err; } else { if (TABLE_TYPE(super) != TSDB_SUPER_TABLE || TABLE_UID(super) != pCfg->superUid) { @@ -108,7 +109,7 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { } } - table = tsdbCreateTableFromCfg(pCfg, false); + table = tsdbCreateTableFromCfg(pCfg, false, super); if (table == NULL) goto _err; // Register to meta @@ -674,7 +675,7 @@ static STable *tsdbNewTable() { return pTable; } -static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { +static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper, STable *pSTable) { STable *pTable = NULL; size_t tsize = 0; @@ -726,6 +727,9 @@ static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper) { if (pCfg->type == TSDB_CHILD_TABLE) { TABLE_SUID(pTable) = pCfg->superUid; + if (tsdbCheckTableTagVal(pCfg->tagValues, pSTable->tagSchema) < 0) { + goto _err; + } pTable->tagVal = tdKVRowDup(pCfg->tagValues); if (pTable->tagVal == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -1302,3 +1306,20 @@ static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid) { return 0; } + +static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema) { + for (size_t i = 0; i < kvRowNCols(pKVRow); i++) { + SColIdx * pColIdx = kvRowColIdxAt(pKVRow, i); + STColumn *pCol = tdGetColOfID(pSchema, pColIdx->colId); + + if ((pCol == NULL) || (!IS_VAR_DATA_TYPE(pCol->type))) continue; + + void *pValue = tdGetKVRowValOfCol(pKVRow, pCol->colId); + if (varDataTLen(pValue) > pCol->bytes) { + terrno = TSDB_CODE_TDB_IVLD_TAG_VAL; + return -1; + } + } + + return 0; +} \ No newline at end of file From b434b54d7713500bdf6a29ba5edd563a2db7b407 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Mon, 15 Mar 2021 16:54:10 +0800 Subject: [PATCH 3/5] [TD-3307]: fix too long string append with length >= 64 --- src/plugins/http/src/httpParser.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 4ce54a8ee6..669b352ab6 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -110,7 +110,7 @@ static void httpCleanupString(HttpString *str) { static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { if (str->size == 0) { str->pos = 0; - str->size = 64; + str->size = len; str->str = malloc(str->size); } else if (str->pos + len + 1 >= str->size) { str->size += len; @@ -715,10 +715,12 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, if (parser->method) { ok = httpOnRequestLine(parser, parser->method, parser->target, parser->version); + /* if (parser->target) { free(parser->target); parser->target = NULL; } + */ } httpClearString(&parser->str); From 18c0457af25ac59ff5d70caa9fadaafc37f71275 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 16 Mar 2021 10:19:00 +0800 Subject: [PATCH 4/5] [td-225]fix null tag caused crash. --- src/query/src/qExecutor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1b3d8ec294..c8295f27cf 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2720,7 +2720,7 @@ static void doSetTagValueInParam(void* pTable, int32_t tagColId, tVariant *tag, val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); } - if (isNull(val, type)) { + if (val == NULL || isNull(val, type)) { tag->nType = TSDB_DATA_TYPE_NULL; return; } @@ -6524,7 +6524,7 @@ static void doSetTagValueToResultBuf(char* output, const char* val, int16_t type if (IS_VAR_DATA_TYPE(type)) { // Binary data overflows for sort of unknown reasons. Let trim the overflow data if (varDataTLen(val) > bytes) { - int32_t len = bytes; + int32_t len = bytes - VARSTR_HEADER_SIZE; // remain available space memcpy(varDataVal(output), varDataVal(val), len); varDataSetLen(output, len); } else { From d99b2bed1b170132d528c77484639b9ceea044fb Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 16 Mar 2021 14:24:31 +0800 Subject: [PATCH 5/5] http/string: append need one more byte for null --- src/plugins/http/src/httpParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/http/src/httpParser.c b/src/plugins/http/src/httpParser.c index 669b352ab6..18cea56cfe 100644 --- a/src/plugins/http/src/httpParser.c +++ b/src/plugins/http/src/httpParser.c @@ -110,7 +110,7 @@ static void httpCleanupString(HttpString *str) { static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) { if (str->size == 0) { str->pos = 0; - str->size = len; + str->size = len + 1; str->str = malloc(str->size); } else if (str->pos + len + 1 >= str->size) { str->size += len;