From 2742159fde19020be9dec4d28b94aecaa4ffbdf7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 29 Apr 2021 23:16:33 +0800 Subject: [PATCH 01/19] [td-3967]1). fix escape char error in sql. 2) refactor. --- src/client/inc/tschemautil.h | 2 +- src/client/src/tscParseInsert.c | 33 ++++++++++++------- src/client/src/tscPrepare.c | 2 +- src/client/src/tscSQLParser.c | 14 ++++---- src/client/src/tscSql.c | 2 +- src/client/src/tscUtil.c | 8 ++--- src/client/tests/timeParseTest.cpp | 2 +- src/common/inc/tname.h | 2 +- src/common/inc/tvariant.h | 2 +- src/common/src/tname.c | 2 +- src/common/src/tvariant.c | 14 ++++---- src/os/tests/test.cpp | 2 +- src/query/inc/qSqlparser.h | 2 +- src/query/src/qSqlParser.c | 6 ++-- src/query/src/sql.c | 6 ++-- src/query/tests/tsBufTest.cpp | 4 +-- src/query/tests/unitTest.cpp | 14 ++++---- src/util/inc/{tstoken.h => ttoken.h} | 29 ++++++++-------- .../qTokenizer.c => util/src/ttokenizer.c} | 32 +++++++++++++----- 19 files changed, 102 insertions(+), 76 deletions(-) rename src/util/inc/{tstoken.h => ttoken.h} (93%) rename src/{query/src/qTokenizer.c => util/src/ttokenizer.c} (92%) diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index a9dcd230a6..0026a27e19 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -21,8 +21,8 @@ extern "C" { #endif #include "taosmsg.h" -#include "tstoken.h" #include "tsclient.h" +#include "ttoken.h" /** * get the number of tags of this table diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 79ceb81cbc..a9d8afd09e 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -29,8 +29,7 @@ #include "taosdef.h" #include "tscLog.h" -#include "tscSubquery.h" -#include "tstoken.h" +#include "ttoken.h" #include "tdataformat.h" @@ -463,23 +462,33 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 // Remove quotation marks if (TK_STRING == sToken.type) { // delete escape character: \\, \', \" - char delim = sToken.z[0]; +// char delim = sToken.z[0]; int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < sToken.n - 1; ++k) { - if (sToken.z[k] == delim || sToken.z[k] == '\\') { - if (sToken.z[k + 1] == delim) { - cnt++; - tmpTokenBuf[j] = sToken.z[k + 1]; - j++; - k++; - continue; - } + if (sToken.z[k] == '\\') { + cnt++; + + tmpTokenBuf[j] = GET_ESCAPE_CHAR(sToken.z[k+1]); + j++; + k++; + continue; } +// if (sToken.z[k] == delim || sToken.z[k] == '\\') { +// if (sToken.z[k + 1] == delim) { +// cnt++; +// tmpTokenBuf[j] = sToken.z[k + 1]; +// j++;s +// k++; +// continue; +// } +// } + tmpTokenBuf[j] = sToken.z[k]; j++; } + tmpTokenBuf[j] = 0; sToken.z = tmpTokenBuf; sToken.n -= 2 + cnt; @@ -1005,7 +1014,7 @@ int validateTableName(char *tblName, int len, SStrToken* psTblToken) { psTblToken->n = len; psTblToken->type = TK_ID; - tSQLGetToken(psTblToken->z, &psTblToken->type); + tGetToken(psTblToken->z, &psTblToken->type); return tscValidateName(psTblToken); } diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index 4efaf7c2b5..73e39b471b 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -151,7 +151,7 @@ static int normalStmtPrepare(STscStmt* stmt) { while (sql[i] != 0) { SStrToken token = {0}; - token.n = tSQLGetToken(sql + i, &token.type); + token.n = tGetToken(sql + i, &token.type); if (token.type == TK_QUESTION) { sql[i] = 0; diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4cf1423c43..e33b8ef33a 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -21,19 +21,19 @@ #endif // __APPLE__ #include "os.h" -#include "ttype.h" -#include "texpr.h" #include "taos.h" #include "taosmsg.h" #include "tcompare.h" +#include "texpr.h" #include "tname.h" #include "tscLog.h" #include "tscUtil.h" #include "tschemautil.h" #include "tsclient.h" -#include "tstoken.h" #include "tstrbuild.h" +#include "ttoken.h" #include "ttokendef.h" +#include "ttype.h" #include "qUtil.h" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" @@ -4629,7 +4629,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t } } else { SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID}; - int32_t len = tSQLGetToken(pRight->value.pz, &token.type); + int32_t len = tGetToken(pRight->value.pz, &token.type); if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) { return TSDB_CODE_TSC_INVALID_SQL; @@ -5498,13 +5498,13 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) { } int32_t validateColumnName(char* name) { - bool ret = isKeyWord(name, (int32_t)strlen(name)); + bool ret = taosIsKeyWordToken(name, (int32_t)strlen(name)); if (ret) { return TSDB_CODE_TSC_INVALID_SQL; } SStrToken token = {.z = name}; - token.n = tSQLGetToken(name, &token.type); + token.n = tGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { return TSDB_CODE_TSC_INVALID_SQL; @@ -5515,7 +5515,7 @@ int32_t validateColumnName(char* name) { strntolower(token.z, token.z, token.n); token.n = (uint32_t)strtrim(token.z); - int32_t k = tSQLGetToken(token.z, &token.type); + int32_t k = tGetToken(token.z, &token.type); if (k != token.n) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index 36d5cec06f..02cd2bd5ef 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -962,7 +962,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t len = (int32_t)strtrim(tblName); SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; - tSQLGetToken(tblName, &sToken.type); + tGetToken(tblName, &sToken.type); // Check if the table name available or not if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index e8d7d5d03c..a9826f2fdf 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1519,7 +1519,7 @@ void tscColumnListDestroy(SArray* pColumnList) { static int32_t validateQuoteToken(SStrToken* pToken) { tscDequoteAndTrimToken(pToken); - int32_t k = tSQLGetToken(pToken->z, &pToken->type); + int32_t k = tGetToken(pToken->z, &pToken->type); if (pToken->type == TK_STRING) { return tscValidateName(pToken); @@ -1587,7 +1587,7 @@ int32_t tscValidateName(SStrToken* pToken) { tscStrToLower(pToken->z, pToken->n); //pToken->n = (uint32_t)strtrim(pToken->z); - int len = tSQLGetToken(pToken->z, &pToken->type); + int len = tGetToken(pToken->z, &pToken->type); // single token, validate it if (len == pToken->n) { @@ -1613,7 +1613,7 @@ int32_t tscValidateName(SStrToken* pToken) { pToken->n = (uint32_t)strtrim(pToken->z); } - pToken->n = tSQLGetToken(pToken->z, &pToken->type); + pToken->n = tGetToken(pToken->z, &pToken->type); if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1630,7 +1630,7 @@ int32_t tscValidateName(SStrToken* pToken) { pToken->z = sep + 1; pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1); - int32_t len = tSQLGetToken(pToken->z, &pToken->type); + int32_t len = tGetToken(pToken->z, &pToken->type); if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) { return TSDB_CODE_TSC_INVALID_SQL; } diff --git a/src/client/tests/timeParseTest.cpp b/src/client/tests/timeParseTest.cpp index d7325430cd..ba06a6b9aa 100644 --- a/src/client/tests/timeParseTest.cpp +++ b/src/client/tests/timeParseTest.cpp @@ -4,7 +4,7 @@ #include #include "taos.h" -#include "tstoken.h" +#include "ttoken.h" #include "tutil.h" int main(int argc, char** argv) { diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 465b298973..cacd6d2ae7 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -18,7 +18,7 @@ #include "os.h" #include "taosmsg.h" -#include "tstoken.h" +#include "ttoken.h" #include "tvariant.h" typedef struct SDataStatis { diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index f8f715c6ca..21b7fd8223 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -16,8 +16,8 @@ #ifndef TDENGINE_TVARIANT_H #define TDENGINE_TVARIANT_H -#include "tstoken.h" #include "tarray.h" +#include "ttoken.h" #ifdef __cplusplus extern "C" { diff --git a/src/common/src/tname.c b/src/common/src/tname.c index 787aa1e95b..65725455e8 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -2,7 +2,7 @@ #include "tutil.h" #include "tname.h" -#include "tstoken.h" +#include "ttoken.h" #include "tvariant.h" #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index c872d8731b..9988450c30 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -14,14 +14,14 @@ */ #include "os.h" -#include "tvariant.h" #include "hash.h" #include "taos.h" #include "taosdef.h" -#include "tstoken.h" +#include "ttoken.h" #include "ttokendef.h" -#include "tutil.h" #include "ttype.h" +#include "tutil.h" +#include "tvariant.h" void tVariantCreate(tVariant *pVar, SStrToken *token) { int32_t ret = 0; @@ -49,7 +49,7 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) { ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true); if (ret != 0) { SStrToken t = {0}; - tSQLGetToken(token->z, &t.type); + tGetToken(token->z, &t.type); if (t.type == TK_MINUS) { // it is a signed number which is greater than INT64_MAX or less than INT64_MIN pVar->nType = -1; // -1 means error type return; @@ -460,7 +460,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result *result = (int64_t) pVariant->dKey; } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen}; - /*int32_t n = */tSQLGetToken(pVariant->pz, &token.type); + /*int32_t n = */tGetToken(pVariant->pz, &token.type); if (token.type == TK_NULL) { if (releaseVariantPtr) { @@ -495,10 +495,10 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result wchar_t *endPtr = NULL; SStrToken token = {0}; - token.n = tSQLGetToken(pVariant->pz, &token.type); + token.n = tGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { - token.n = tSQLGetToken(pVariant->pz + token.n, &token.type); + token.n = tGetToken(pVariant->pz + token.n, &token.type); } if (token.type == TK_FLOAT) { diff --git a/src/os/tests/test.cpp b/src/os/tests/test.cpp index 600e5d71a7..bdd3114363 100644 --- a/src/os/tests/test.cpp +++ b/src/os/tests/test.cpp @@ -4,7 +4,7 @@ #include #include "taos.h" -#include "tstoken.h" +#include "ttoken.h" #include "tutil.h" int main(int argc, char** argv) { diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index 0a0587f701..70ba48e551 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -22,8 +22,8 @@ extern "C" { #include "taos.h" #include "taosmsg.h" -#include "tstoken.h" #include "tstrbuild.h" +#include "ttoken.h" #include "tvariant.h" #define ParseTOKENTYPE SStrToken diff --git a/src/query/src/qSqlParser.c b/src/query/src/qSqlParser.c index 6b38536b15..56d395e498 100644 --- a/src/query/src/qSqlParser.c +++ b/src/query/src/qSqlParser.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#include "os.h" #include "qSqlparser.h" +#include "os.h" #include "taosdef.h" #include "taosmsg.h" #include "tcmdtype.h" -#include "tstoken.h" #include "tstrbuild.h" +#include "ttoken.h" #include "ttokendef.h" #include "tutil.h" @@ -38,7 +38,7 @@ SSqlInfo qSqlParse(const char *pStr) { goto abort_parse; } - t0.n = tSQLGetToken((char *)&pStr[i], &t0.type); + t0.n = tGetToken((char *)&pStr[i], &t0.type); t0.z = (char *)(pStr + i); i += t0.n; diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 96d33a8ed6..8f36638dc9 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -26,14 +26,14 @@ #include /************ Begin %include sections from the grammar ************************/ +#include +#include #include #include #include -#include -#include #include "qSqlparser.h" #include "tcmdtype.h" -#include "tstoken.h" +#include "ttoken.h" #include "ttokendef.h" #include "tutil.h" #include "tvariant.h" diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 8ca636b834..dd7f03a494 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -3,10 +3,10 @@ #include #include +#include "qTsbuf.h" #include "taos.h" #include "tsdb.h" -#include "qTsbuf.h" -#include "tstoken.h" +#include "ttoken.h" #include "tutil.h" namespace { diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index 3406d83090..d2b058cf7c 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -21,7 +21,7 @@ int32_t testValidateName(char* name) { token.n = strlen(name); token.type = 0; - tSQLGetToken(name, &token.type); + tGetToken(name, &token.type); return tscValidateName(&token); } } @@ -691,32 +691,32 @@ TEST(testCase, tGetToken_Test) { char* s = ".123 "; uint32_t type = 0; - int32_t len = tSQLGetToken(s, &type); + int32_t len = tGetToken(s, &type); EXPECT_EQ(type, TK_FLOAT); EXPECT_EQ(len, strlen(s) - 1); char s1[] = "1.123e10 "; - len = tSQLGetToken(s1, &type); + len = tGetToken(s1, &type); EXPECT_EQ(type, TK_FLOAT); EXPECT_EQ(len, strlen(s1) - 1); char s4[] = "0xff "; - len = tSQLGetToken(s4, &type); + len = tGetToken(s4, &type); EXPECT_EQ(type, TK_HEX); EXPECT_EQ(len, strlen(s4) - 1); // invalid data type char s2[] = "e10 "; - len = tSQLGetToken(s2, &type); + len = tGetToken(s2, &type); EXPECT_FALSE(type == TK_FLOAT); char s3[] = "1.1.1.1"; - len = tSQLGetToken(s3, &type); + len = tGetToken(s3, &type); EXPECT_EQ(type, TK_IPTOKEN); EXPECT_EQ(len, strlen(s3)); char s5[] = "0x "; - len = tSQLGetToken(s5, &type); + len = tGetToken(s5, &type); EXPECT_FALSE(type == TK_HEX); } diff --git a/src/util/inc/tstoken.h b/src/util/inc/ttoken.h similarity index 93% rename from src/util/inc/tstoken.h rename to src/util/inc/ttoken.h index ab1ef7b279..c1e2170ac3 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/ttoken.h @@ -37,13 +37,25 @@ typedef struct SStrToken { char *z; } SStrToken; +extern const char escapeChar[]; + +/** + * check if it is a number or not + * @param pToken + * @return + */ +#define isNumber(tk) \ +((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN) + +#define GET_ESCAPE_CHAR(c) (escapeChar[(uint8_t)(c)]) + /** * tokenizer for sql string * @param z * @param tokenType * @return */ -uint32_t tSQLGetToken(char *z, uint32_t *tokenType); +uint32_t tGetToken(char *z, uint32_t *tokenType); /** * enhanced tokenizer for sql string. @@ -61,21 +73,12 @@ SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr); * @param len * @return */ -bool isKeyWord(const char *z, int32_t len); - -/** - * check if it is a number or not - * @param pToken - * @return - */ -#define isNumber(tk) \ -((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN) - +bool taosIsKeyWordToken(const char *z, int32_t len); /** * check if it is a token or not - * @param pToken - * @return token type, if it is not a number, TK_ILLEGAL will return + * @param pToken + * @return token type, if it is not a number, TK_ILLEGAL will return */ static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) { const char* z = pToken->z; diff --git a/src/query/src/qTokenizer.c b/src/util/src/ttokenizer.c similarity index 92% rename from src/query/src/qTokenizer.c rename to src/util/src/ttokenizer.c index 7869e27707..794420d55b 100644 --- a/src/query/src/qTokenizer.c +++ b/src/util/src/ttokenizer.c @@ -18,7 +18,7 @@ #include "hash.h" #include "hashfunc.h" #include "taosdef.h" -#include "tstoken.h" +#include "ttoken.h" #include "ttokendef.h" #include "tutil.h" @@ -232,6 +232,18 @@ static const char isIdChar[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ }; +const char escapeChar[] = { + /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 0x */ + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, /* 1x */ + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, /* 2x */ + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, /* 3x */ + 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,/* 4x */ + 0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,/* 5x */ + 0x60, 0x07, 0x08, 0x63, 0x64, 0x65, 0x0C, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x0A, 0x6F,/* 6x */ + 0x70, 0x71, 0x0D, 0x73, 0x09, 0x75, 0x0B, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,/* 7x */ +}; + static void* keywordHashTable = NULL; static void doInitKeywordsTable(void) { @@ -247,7 +259,7 @@ static void doInitKeywordsTable(void) { static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT; -int tSQLKeywordCode(const char* z, int n) { +static int32_t tKeywordCode(const char* z, int n) { pthread_once(&keywordsHashTableInit, doInitKeywordsTable); char key[512] = {0}; @@ -271,7 +283,7 @@ int tSQLKeywordCode(const char* z, int n) { * Return the length of the token that begins at z[0]. * Store the token type in *type before returning. */ -uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { +uint32_t tGetToken(char* z, uint32_t* tokenId) { uint32_t i; switch (*z) { case ' ': @@ -408,7 +420,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { continue; } - if (z[i] == delim ) { + if (z[i] == delim) { if (z[i + 1] == delim) { i++; } else { @@ -551,7 +563,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { } for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t) z[i]]; i++) { } - *tokenId = tSQLKeywordCode(z, i); + *tokenId = tKeywordCode(z, i); return i; } } @@ -584,7 +596,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { t = str[++(*i)]; } - t0.n = tSQLGetToken(&str[*i], &t0.type); + t0.n = tGetToken(&str[*i], &t0.type); break; // not support user specfied ignored symbol list @@ -613,7 +625,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { // support parse the 'db.tbl' format, notes: There should be no space on either side of the dot! if ('.' == str[*i + t0.n]) { - len = tSQLGetToken(&str[*i + t0.n + 1], &type); + len = tGetToken(&str[*i + t0.n + 1], &type); // only id and string are valid if ((TK_STRING != t0.type) && (TK_ID != t0.type)) { @@ -628,7 +640,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { } else { // support parse the -/+number format if ((isPrevOptr) && (t0.type == TK_MINUS || t0.type == TK_PLUS)) { - len = tSQLGetToken(&str[*i + t0.n], &type); + len = tGetToken(&str[*i + t0.n], &type); if (type == TK_INTEGER || type == TK_FLOAT) { t0.type = type; t0.n += len; @@ -642,7 +654,9 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) { return t0; } -bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); } +bool taosIsKeyWordToken(const char* z, int32_t len) { + return (tKeywordCode((char*)z, len) != TK_ID); +} void taosCleanupKeywordsTable() { void* m = keywordHashTable; From c64fd41be7f4dba7f929565acdf033a89bd07042 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Apr 2021 10:46:13 +0800 Subject: [PATCH 02/19] [td-3967]refactor. --- src/client/src/tscParseInsert.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index a9d8afd09e..35dc7bca55 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -475,16 +475,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 continue; } -// if (sToken.z[k] == delim || sToken.z[k] == '\\') { -// if (sToken.z[k + 1] == delim) { -// cnt++; -// tmpTokenBuf[j] = sToken.z[k + 1]; -// j++;s -// k++; -// continue; -// } -// } - tmpTokenBuf[j] = sToken.z[k]; j++; } From bdc887741ece56dc7e1afa9789338d55b652c35c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Apr 2021 10:47:42 +0800 Subject: [PATCH 03/19] [td-3967] --- src/client/src/tscParseInsert.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 35dc7bca55..298f1245e7 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -462,7 +462,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 // Remove quotation marks if (TK_STRING == sToken.type) { // delete escape character: \\, \', \" -// char delim = sToken.z[0]; int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < sToken.n - 1; ++k) { @@ -704,6 +703,8 @@ static int32_t doParseInsertStatement(SSqlCmd* pCmd, char **str, STableDataBlock } code = TSDB_CODE_TSC_INVALID_SQL; + + // todo the size should be limited to the current sql length char *tmpTokenBuf = calloc(1, 16*1024); // used for deleting Escape character: \\, \', \" if (NULL == tmpTokenBuf) { return TSDB_CODE_TSC_OUT_OF_MEMORY; From d78ac9848c3efd37107b886bd1f6e1780b805d86 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Apr 2021 17:12:20 +0800 Subject: [PATCH 04/19] [td-3967]1) refactor. --- src/client/src/tscParseInsert.c | 12 +++++++++--- .../script/general/parser/binary_escapeCharacter.sim | 9 +++++++++ tests/script/general/parser/testSuite.sim | 1 + 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 298f1245e7..9130118f80 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -462,13 +462,19 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 // Remove quotation marks if (TK_STRING == sToken.type) { // delete escape character: \\, \', \" + char delim = sToken.z[0]; + int32_t cnt = 0; int32_t j = 0; for (uint32_t k = 1; k < sToken.n - 1; ++k) { - if (sToken.z[k] == '\\') { - cnt++; + if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { + if (sToken.z[k] == '\\') { + tmpTokenBuf[j] = GET_ESCAPE_CHAR(sToken.z[k+1]); + } else { + tmpTokenBuf[j] = sToken.z[k + 1]; + } - tmpTokenBuf[j] = GET_ESCAPE_CHAR(sToken.z[k+1]); + cnt++; j++; k++; continue; diff --git a/tests/script/general/parser/binary_escapeCharacter.sim b/tests/script/general/parser/binary_escapeCharacter.sim index f0589d154f..1c6bc5b37f 100644 --- a/tests/script/general/parser/binary_escapeCharacter.sim +++ b/tests/script/general/parser/binary_escapeCharacter.sim @@ -93,5 +93,14 @@ if $data41 != @udp005@ then print "[ERROR] expect: udp005, act:$data41" endi +print ---------------------> TD-3967 +sql insert into tb values(now, '\\abc\\\\'); +sql insert into tb values(now, '\\abc\\\\'); +sql insert into tb values(now, '\\\\'); + +print ------------->sim bug +# sql_error insert into tb values(now, '\\\'); +# sql_error insert into tb values(now, '\\'); +# sql_error insert into tb values(now, '\\n'); system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index f05474d158..b605a04df7 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -55,4 +55,5 @@ run general/parser/sliding.sim run general/parser/function.sim run general/parser/stableOp.sim run general/parser/slimit_alter_tags.sim +run general/parser/binary_escapeCharacter.sim From b0ab4573bba52b91df50edde52819ecb7e6c1468 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Apr 2021 17:14:57 +0800 Subject: [PATCH 05/19] [td-3967]update the sim. --- tests/script/general/parser/binary_escapeCharacter.sim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/script/general/parser/binary_escapeCharacter.sim b/tests/script/general/parser/binary_escapeCharacter.sim index 1c6bc5b37f..b5bb10284b 100644 --- a/tests/script/general/parser/binary_escapeCharacter.sim +++ b/tests/script/general/parser/binary_escapeCharacter.sim @@ -100,7 +100,8 @@ sql insert into tb values(now, '\\\\'); print ------------->sim bug # sql_error insert into tb values(now, '\\\'); -# sql_error insert into tb values(now, '\\'); -# sql_error insert into tb values(now, '\\n'); +sql_error insert into tb values(now, '\'); +#sql_error insert into tb values(now, '\\\n'); +sql insert into tb values(now, '\n'); system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From b6e04604cb268e731e24f0141fac529752c80ab9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 7 May 2021 17:29:00 +0800 Subject: [PATCH 06/19] [td-3967] --- src/client/src/tscParseInsert.c | 4 ---- src/util/src/ttokenizer.c | 2 +- tests/pytest/insert/nchar-unicode.py | 5 ++++- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index 3882be56dc..923d95c888 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -468,11 +468,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1 int32_t j = 0; for (uint32_t k = 1; k < sToken.n - 1; ++k) { if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) { - if (sToken.z[k] == '\\') { - tmpTokenBuf[j] = GET_ESCAPE_CHAR(sToken.z[k+1]); - } else { tmpTokenBuf[j] = sToken.z[k + 1]; - } cnt++; j++; diff --git a/src/util/src/ttokenizer.c b/src/util/src/ttokenizer.c index 794420d55b..54da75cae0 100644 --- a/src/util/src/ttokenizer.c +++ b/src/util/src/ttokenizer.c @@ -415,7 +415,7 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) { int delim = z[0]; bool strEnd = false; for (i = 1; z[i]; i++) { - if (z[i] == '\\') { + if (z[i] == '\\') { // ignore the escaped character that follows this backslash i++; continue; } diff --git a/tests/pytest/insert/nchar-unicode.py b/tests/pytest/insert/nchar-unicode.py index c417a6bca2..4afcf5b760 100644 --- a/tests/pytest/insert/nchar-unicode.py +++ b/tests/pytest/insert/nchar-unicode.py @@ -57,12 +57,15 @@ class TDTestCase: # https://www.ltg.ed.ac.uk/~richard/unicode-sample.html # Basic Latin - data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~' + data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \\ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~' tdLog.info("insert Basic Latin %d length data: %s" % (len(data), data)) tdSql.execute("insert into tb values (now, '%s')" % data) tdSql.query("select * from tb") tdSql.checkRows(3) + + data = data.replace('\\\\', '\\') tdSql.checkData(2, 1, data) + # tdSql.execute("insert into tb values(now, 'abc')") # Latin-1 Supplement data = ' ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ­ ® ¯ ° ± ² ³ ´ µ ¶ · ¸ ¹ º » ¼ ½ ¾ ¿ À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ' From a0020b15ef900c68f23d7b15d9476c9b495e15d6 Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Sat, 8 May 2021 16:25:03 +0800 Subject: [PATCH 07/19] add assert info --- src/client/src/tscServer.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index d9405f300a..2035d6261f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -1973,6 +1973,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { return TSDB_CODE_TSC_INVALID_VALUE; } + assert(pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE || pTableMeta->tableType == TSDB_NORMAL_TABLE || pTableMeta->tableType == TSDB_STREAM_TABLE); + if (pTableMeta->tableType == TSDB_CHILD_TABLE) { // check if super table hashmap or not int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); From 75172ef7bbe21d1a5634d4d562e16801e6b92538 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 8 May 2021 20:44:29 +0800 Subject: [PATCH 08/19] [TD-4096] --- src/client/src/tscSQLParser.c | 45 +++++++++++++---------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4cf1423c43..4251695d7b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -64,7 +64,7 @@ static char* getAccountId(SSqlObj* pSql); static bool has(SArray* pFieldList, int32_t startIdx, const char* name); static char* cloneCurrentDBName(SSqlObj* pSql); -static bool hasSpecifyDB(SStrToken* pTableName); +static int32_t getDelimiterIndex(SStrToken* pTableName); static bool validateTableColumnInfo(SArray* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(SArray* pTagsList, SArray* pFieldList, SSqlCmd* pCmd); @@ -426,17 +426,11 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_DESCRIBE_TABLE: { const char* msg1 = "invalid table name"; - const char* msg2 = "table name too long"; SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - - if (!tscValidateTableNameLength(pToken->n)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - // additional msg has been attached already code = tscSetTableFullName(pTableMetaInfo, pToken, pSql); if (code != TSDB_CODE_SUCCESS) { @@ -447,17 +441,12 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_SHOW_CREATE_TABLE: { const char* msg1 = "invalid table name"; - const char* msg2 = "table name is too long"; SStrToken* pToken = taosArrayGet(pInfo->pMiscInfo->a, 0); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } - if (!tscValidateTableNameLength(pToken->n)) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - code = tscSetTableFullName(pTableMetaInfo, pToken, pSql); if (code != TSDB_CODE_SUCCESS) { return code; @@ -987,11 +976,13 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam const char* msg1 = "name too long"; const char* msg2 = "acctId too long"; const char* msg3 = "no acctId"; + const char* msg4 = "db name too long"; + const char* msg5 = "table name too long"; SSqlCmd* pCmd = &pSql->cmd; int32_t code = TSDB_CODE_SUCCESS; - - if (hasSpecifyDB(pTableName)) { // db has been specified in sql string so we ignore current db path + int32_t idx = getDelimiterIndex(pTableName); + if (idx != -1) { // db has been specified in sql string so we ignore current db path char* acctId = getAccountId(pSql); if (acctId == NULL || strlen(acctId) <= 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1001,6 +992,13 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam if (code != 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + if (idx >= TSDB_DB_NAME_LEN) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + if (pTableName->n - 1 - idx >= TSDB_TABLE_NAME_LEN) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); + } + char name[TSDB_TABLE_FNAME_LEN] = {0}; strncpy(name, pTableName->z, pTableName->n); @@ -1345,14 +1343,13 @@ static char* cloneCurrentDBName(SSqlObj* pSql) { } /* length limitation, strstr cannot be applied */ -static bool hasSpecifyDB(SStrToken* pTableName) { - for (uint32_t i = 0; i < pTableName->n; ++i) { - if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { - return true; +static int32_t getDelimiterIndex(SStrToken* pTableName) { + for (uint32_t i = 0; i < pTableName->n; ++i) { + if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { + return i; } } - - return false; + return -1; } int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { @@ -7483,11 +7480,3 @@ bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) { return false; } - - - - - - - - From 1729bc75ab163c2e7a1192333b45338ea8f7dbbd Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sun, 9 May 2021 00:34:48 +0800 Subject: [PATCH 09/19] [TD-4119] --- src/mnode/inc/mnodeDef.h | 2 +- src/mnode/src/mnodeShow.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/mnode/inc/mnodeDef.h b/src/mnode/inc/mnodeDef.h index ed1de1b87a..e052f34a33 100644 --- a/src/mnode/inc/mnodeDef.h +++ b/src/mnode/inc/mnodeDef.h @@ -249,7 +249,7 @@ typedef struct SAcctObj { } SAcctObj; typedef struct { - char db[TSDB_DB_NAME_LEN]; + char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN]; int8_t type; int16_t numOfColumns; int32_t index; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 03772f2724..c0fa6368f3 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -129,7 +129,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) { SShowObj *pShow = calloc(1, showObjSize); pShow->type = pShowMsg->type; pShow->payloadLen = htons(pShowMsg->payloadLen); - tstrncpy(pShow->db, pShowMsg->db, TSDB_DB_NAME_LEN); + tstrncpy(pShow->db, pShowMsg->db, TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN); memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen); pShow = mnodePutShowObj(pShow); From 7570894fbb68b43cfb583ae5c015894441fb3e35 Mon Sep 17 00:00:00 2001 From: wu champion Date: Mon, 10 May 2021 15:30:00 +0800 Subject: [PATCH 10/19] [TD-3960] add case for TD-3960 --- tests/pytest/fulltest.sh | 1 + .../functions/showOfflineThresholdIs864000.py | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/pytest/functions/showOfflineThresholdIs864000.py diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index b5aae6fcef..a748c9dd2d 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -305,6 +305,7 @@ python3 ./test.py -f functions/function_top.py -r 1 python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa_test2.py python3 ./test.py -f functions/function_stddev_td2555.py +python3 ./test.py -f functions/showOfflineThresholdIs864000.py python3 ./test.py -f insert/metadataUpdate.py python3 ./test.py -f query/last_cache.py python3 ./test.py -f query/last_row_cache.py diff --git a/tests/pytest/functions/showOfflineThresholdIs864000.py b/tests/pytest/functions/showOfflineThresholdIs864000.py new file mode 100644 index 0000000000..6cce869bf2 --- /dev/null +++ b/tests/pytest/functions/showOfflineThresholdIs864000.py @@ -0,0 +1,36 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to execute {__file__}") + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.query("show variables") + tdSql.checkData(51, 1, 864000) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file From 79b422390f9174ea23c56584e0dbefc2c79345ee Mon Sep 17 00:00:00 2001 From: dapan1121 <89396746@qq.com> Date: Mon, 10 May 2021 16:17:46 +0800 Subject: [PATCH 11/19] fix bug --- src/client/src/tscServer.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 2035d6261f..33325c7fd7 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2542,6 +2542,7 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } pTableMetaInfo->pTableMeta = (STableMeta *)tmp; + memset(pTableMetaInfo->pTableMeta, 0, size); pTableMetaInfo->tableMetaSize = size; } else { //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); From f2fa15156543728d68e3502970bcf4e2ee55381a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 May 2021 17:01:46 +0800 Subject: [PATCH 12/19] [td-3967]update the test case. --- .../taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java index 9014e82a9e..a3c46dc232 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java @@ -12,7 +12,7 @@ public class InsertSpecialCharacterJniTest { private static String tbname1 = "test"; private static String tbname2 = "weather"; private static String special_character_str_1 = "$asd$$fsfsf$"; - private static String special_character_str_2 = "\\asdfsfsf\\\\"; + private static String special_character_str_2 = "\\\\asdfsfsf\\\\"; private static String special_character_str_3 = "\\\\asdfsfsf\\"; private static String special_character_str_4 = "?asd??fsf?sf?"; private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; @@ -70,7 +70,7 @@ public class InsertSpecialCharacterJniTest { String f1 = new String(rs.getBytes(2)); //TODO: bug to be fixed // Assert.assertEquals(special_character_str_2, f1); - Assert.assertEquals(special_character_str_2.substring(0, special_character_str_1.length() - 2), f1); + Assert.assertEquals(special_character_str_2.substring(1, special_character_str_1.length() - 1), f1); String f2 = rs.getString(3); Assert.assertNull(f2); } From a9c5957165255f58b8fa56a274249de482033b5c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 00:08:30 +0800 Subject: [PATCH 13/19] [TD-4095]: taosdump long tablename. (#6076) Co-authored-by: Shuduo Sang --- src/kit/taosdump/taosdump.c | 115 +++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 46 deletions(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index e706ddefd6..f80ac069a0 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -110,14 +110,14 @@ typedef struct { } SColDes; typedef struct { - char name[TSDB_COL_NAME_LEN + 1]; + char name[TSDB_TABLE_NAME_LEN]; SColDes cols[]; } STableDef; extern char version[]; typedef struct { - char name[TSDB_DB_NAME_LEN + 1]; + char name[TSDB_DB_NAME_LEN]; char create_time[32]; int32_t ntables; int32_t vgroups; @@ -142,8 +142,8 @@ typedef struct { } SDbInfo; typedef struct { - char name[TSDB_TABLE_NAME_LEN + 1]; - char metric[TSDB_TABLE_NAME_LEN + 1]; + char name[TSDB_TABLE_NAME_LEN]; + char metric[TSDB_TABLE_NAME_LEN]; } STableRecord; typedef struct { @@ -155,7 +155,7 @@ typedef struct { pthread_t threadID; int32_t threadIndex; int32_t totalThreads; - char dbName[TSDB_TABLE_NAME_LEN + 1]; + char dbName[TSDB_DB_NAME_LEN]; void *taosCon; int64_t rowsOfDumpOut; int64_t tablesOfDumpOut; @@ -214,13 +214,13 @@ static struct argp_option options[] = { {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, // dump unit options {"all-databases", 'A', 0, 0, "Dump all databases.", 2}, - {"databases", 'B', 0, 0, "Dump assigned databases", 2}, + {"databases", 'D', 0, 0, "Dump assigned databases", 2}, // dump format options {"schemaonly", 's', 0, 0, "Only dump schema.", 3}, - {"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, + {"without-property", 'N', 0, 0, "Dump schema without properties.", 3}, {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, - {"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, + {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, @@ -341,15 +341,15 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'A': arguments->all_databases = true; break; - case 'B': + case 'D': arguments->databases = true; break; // dump format option case 's': arguments->schemaonly = true; break; - case 'M': - arguments->with_property = true; + case 'N': + arguments->with_property = false; break; case 'S': // parse time here. @@ -358,7 +358,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) { case 'E': arguments->end_time = atol(arg); break; - case 'N': + case 'B': arguments->data_batch = atoi(arg); if (arguments->data_batch >= INT16_MAX) { arguments->data_batch = INT16_MAX - 1; @@ -402,17 +402,17 @@ static resultStatistics g_resultStatistics = {0}; static FILE *g_fpOfResult = NULL; static int g_numOfCores = 1; -int taosDumpOut(struct arguments *arguments); -int taosDumpIn(struct arguments *arguments); -void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); -int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); -int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); -void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); -void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); -int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); -int taosCheckParam(struct arguments *arguments); -void taosFreeDbInfos(); +static int taosDumpOut(struct arguments *arguments); +static int taosDumpIn(struct arguments *arguments); +static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); +static int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); +static int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); +static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); +static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); +static int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); +static int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); +static int taosCheckParam(struct arguments *arguments); +static void taosFreeDbInfos(); static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); struct arguments g_args = { @@ -436,8 +436,8 @@ struct arguments g_args = { false, false, // dump format option - false, - false, + false, // schemeonly + true, // with_property 0, INT64_MAX, 1, @@ -959,7 +959,8 @@ int taosDumpOut(struct arguments *arguments) { goto _exit_failure; } - strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); + strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], + fields[TSDB_SHOW_DB_NAME_INDEX].bytes); if (arguments->with_property) { dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); @@ -967,7 +968,8 @@ int taosDumpOut(struct arguments *arguments) { dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); - strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); + strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], + fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); //dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); //dbInfos[count]->daysToKeep1; //dbInfos[count]->daysToKeep2; @@ -980,7 +982,8 @@ int taosDumpOut(struct arguments *arguments) { dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); - strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); + strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], + fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); //dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); } @@ -1095,7 +1098,9 @@ _exit_failure: return -1; } -int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { +int taosGetTableDes( + char* dbName, char *table, + STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { TAOS_ROW row = NULL; TAOS_RES* res = NULL; int count = 0; @@ -1113,7 +1118,7 @@ int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCo TAOS_FIELD *fields = taos_fetch_fields(res); - tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN); + tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN); while ((row = taos_fetch_row(res)) != NULL) { strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], @@ -1232,7 +1237,9 @@ int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCo return count; } -int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName) { +int32_t taosDumpTable( + char *table, char *metric, struct arguments *arguments, + FILE *fp, TAOS* taosCon, char* dbName) { int count = 0; STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); @@ -1346,14 +1353,17 @@ void* taosDumpOutWorkThreadFp(void *arg) ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); if (readLen <= 0) break; - int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &g_args, fp, pThread->taosCon, pThread->dbName); + int ret = taosDumpTable( + tableRecord.name, tableRecord.metric, &g_args, + fp, pThread->taosCon, pThread->dbName); if (ret >= 0) { // TODO: sum table count and table rows by self pThread->tablesOfDumpOut++; pThread->rowsOfDumpOut += ret; if (pThread->rowsOfDumpOut >= lastRowsPrint) { - printf(" %"PRId64 " rows already be dumpout from database %s\n", pThread->rowsOfDumpOut, pThread->dbName); + printf(" %"PRId64 " rows already be dumpout from database %s\n", + pThread->rowsOfDumpOut, pThread->dbName); lastRowsPrint += 5000000; } @@ -1364,9 +1374,12 @@ void* taosDumpOutWorkThreadFp(void *arg) memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); if (g_args.outpath[0] != 0) { - sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex); + sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", + g_args.outpath, pThread->dbName, + pThread->threadIndex, fileNameIndex); } else { - sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex); + sprintf(tmpBuf, "%s.tables.%d-%d.sql", + pThread->dbName, pThread->threadIndex, fileNameIndex); } fileNameIndex++; @@ -1391,14 +1404,15 @@ void* taosDumpOutWorkThreadFp(void *arg) static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName) { pthread_attr_t thattr; - SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); + SThreadParaObj *threadObj = + (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); for (int t = 0; t < numOfThread; ++t) { SThreadParaObj *pThread = threadObj + t; pThread->rowsOfDumpOut = 0; pThread->tablesOfDumpOut = 0; pThread->threadIndex = t; pThread->totalThreads = numOfThread; - tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN); + tstrncpy(pThread->dbName, dbName, TSDB_DB_NAME_LEN); pThread->taosCon = taosCon; pthread_attr_init(&thattr); @@ -1487,7 +1501,8 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp) while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); - strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); taosWrite(fd, &tableRecord, sizeof(STableRecord)); } @@ -1557,8 +1572,10 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao int32_t numOfTable = 0; while ((row = taos_fetch_row(res)) != NULL) { memset(&tableRecord, 0, sizeof(STableRecord)); - tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); - tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); + tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], + fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); + tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], + fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); taosWrite(fd, &tableRecord, sizeof(STableRecord)); @@ -1643,15 +1660,18 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, cha char* pstr = sqlstr; - pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s", dbName, tableDes->name); + pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s", + dbName, tableDes->name); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; if (counter == 0) { - pstr += sprintf(pstr, " (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); + pstr += sprintf(pstr, " (%s %s", + tableDes->cols[counter].field, tableDes->cols[counter].type); } else { - pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); + pstr += sprintf(pstr, ", %s %s", + tableDes->cols[counter].field, tableDes->cols[counter].type); } if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || @@ -1664,9 +1684,11 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, cha for (; counter < numOfCols; counter++) { if (counter == count_temp) { - pstr += sprintf(pstr, ") TAGS (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); + pstr += sprintf(pstr, ") TAGS (%s %s", + tableDes->cols[counter].field, tableDes->cols[counter].type); } else { - pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); + pstr += sprintf(pstr, ", %s %s", + tableDes->cols[counter].field, tableDes->cols[counter].type); } if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || @@ -1693,7 +1715,8 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols char *pstr = NULL; pstr = tmpBuf; - pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", dbName, tableDes->name, dbName, metric); + pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", + dbName, tableDes->name, dbName, metric); for (; counter < numOfCols; counter++) { if (tableDes->cols[counter].note[0] != '\0') break; From 12228d0d9636608e17ded5257361adb75f139432 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 09:54:03 +0800 Subject: [PATCH 14/19] [TD-4073]: taosdemo rest query output to result file. (#6072) * [TD-4073]: taosdemo rest query output to result file. * [TD-4073]: taosdemo rest query output to result file. fix insecure strcat(). * [TD-4073]: taosdemo rest query output to result file. fix postSql command mistake * [TD-4073]: taosdemo rest query output to result file. fix append function bug. * [TD-4073]: taosdemo rest query output to result file. prevent potential null file passed. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 154 ++++++++++++++++++++---------------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 99befbaf31..935aa358c5 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -76,6 +76,7 @@ enum QUERY_MODE { #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) +#define COND_BUF_LEN BUFFER_SIZE - 30 #define MAX_USERNAME_SIZE 64 #define MAX_PASSWORD_SIZE 64 #define MAX_DB_NAME_SIZE 64 @@ -522,6 +523,8 @@ static int taosRandom() static int createDatabasesAndStables(); static void createChildTables(); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); +static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, + char* sqlstr, char *resultFile); /* ************ Global variables ************ */ @@ -1090,27 +1093,33 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { return 0; } -static void getResult(TAOS_RES *res, char* resultFileName) { +static void appendResultBufToFile(char *resultBuf, char *resultFile) +{ + FILE *fp = NULL; + if (resultFile[0] != 0) { + fp = fopen(resultFile, "at"); + if (fp == NULL) { + errorPrint( + "%s() LN%d, failed to open result file: %s, result will not save to file\n", + __func__, __LINE__, resultFile); + return; + } + } + + fprintf(fp, "%s", resultBuf); + tmfclose(fp); +} + +static void appendResultToFile(TAOS_RES *res, char* resultFile) { TAOS_ROW row = NULL; int num_rows = 0; int num_fields = taos_field_count(res); TAOS_FIELD *fields = taos_fetch_fields(res); - FILE *fp = NULL; - if (resultFileName[0] != 0) { - fp = fopen(resultFileName, "at"); - if (fp == NULL) { - errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n", - __func__, __LINE__, resultFileName); - } - } - char* databuf = (char*) calloc(1, 100*1024*1024); if (databuf == NULL) { errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", __func__, __LINE__); - if (fp) - fclose(fp); return ; } @@ -1120,7 +1129,7 @@ static void getResult(TAOS_RES *res, char* resultFileName) { // fetch the records row by row while((row = taos_fetch_row(res))) { if (totalLen >= 100*1024*1024 - 32000) { - if (fp) fprintf(fp, "%s", databuf); + appendResultBufToFile(databuf, resultFile); totalLen = 0; memset(databuf, 0, 100*1024*1024); } @@ -1132,22 +1141,39 @@ static void getResult(TAOS_RES *res, char* resultFileName) { totalLen += len; } - if (fp) fprintf(fp, "%s", databuf); - tmfclose(fp); + appendResultBufToFile(databuf, resultFile); free(databuf); } -static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { - TAOS_RES *res = taos_query(taos, command); - if (res == NULL || taos_errno(res) != 0) { - errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n", - __func__, __LINE__, command, taos_errstr(res)); - taos_free_result(res); - return; - } +static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* resultFile) +{ + if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { + TAOS_RES *res = taos_query(pThreadInfo->taos, command); + if (res == NULL || taos_errno(res) != 0) { + errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n", + __func__, __LINE__, command, taos_errstr(res)); + taos_free_result(res); + return; + } - getResult(res, resultFileName); - taos_free_result(res); + if ((resultFile) && (strlen(resultFile))) { + appendResultToFile(res, resultFile); + } + taos_free_result(res); + + } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { + int retCode = postProceSql( + g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port, + command, + resultFile); + if (0 != retCode) { + printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); + } + + } else { + errorPrint("%s() LN%d, unknown query mode: %s\n", + __func__, __LINE__, g_queryInfo.queryMode); + } } static int32_t rand_bool(){ @@ -1940,13 +1966,13 @@ static void printfQuerySystemInfo(TAOS * taos) { // show variables res = taos_query(taos, "show variables;"); - //getResult(res, filename); + //appendResultToFile(res, filename); xDumpResultToFile(filename, res); // show dnodes res = taos_query(taos, "show dnodes;"); xDumpResultToFile(filename, res); - //getResult(res, filename); + //appendResultToFile(res, filename); // show databases res = taos_query(taos, "show databases;"); @@ -1981,7 +2007,8 @@ static void printfQuerySystemInfo(TAOS * taos) { free(dbInfos); } -static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, char* sqlstr) +static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, + char* sqlstr, char *resultFile) { char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; @@ -2117,6 +2144,10 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port response_buf[RESP_BUF_LEN - 1] = '\0'; printf("Response:\n%s\n", response_buf); + if (resultFile) { + appendResultBufToFile(response_buf, resultFile); + } + free(request_buf); #ifdef WINDOWS closesocket(sockfd); @@ -4688,7 +4719,8 @@ static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k) if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false); } else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) { - if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, buffer)) { + if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, + buffer, NULL /* not set result file */)) { affectedRows = -1; printf("========restful return fail, threadID[%d]\n", pThreadInfo->threadID); @@ -5936,7 +5968,7 @@ static void *readMetric(void *sarg) { fprintf(fp, "Querying On %d records:\n", totalData); for (int j = 0; j < n; j++) { - char condition[BUFFER_SIZE - 30] = "\0"; + char condition[COND_BUF_LEN] = "\0"; char tempS[64] = "\0"; int m = 10 < num_of_tables ? 10 : num_of_tables; @@ -5947,7 +5979,7 @@ static void *readMetric(void *sarg) { } else { sprintf(tempS, " or t1 = %d ", i); } - strcat(condition, tempS); + strncat(condition, tempS, COND_BUF_LEN - 1); sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); @@ -6125,43 +6157,24 @@ static void *specifiedTableQuery(void *sarg) { taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms } - st = taosGetTimestampMs(); - - if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) { - int64_t t1 = taosGetTimestampMs(); - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) { sprintf(tmpFile, "%s-%d", g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], pThreadInfo->threadID); - } - selectAndGetResult(pThreadInfo->taos, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); - int64_t t2 = taosGetTimestampMs(); - printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %10.3f s\n", - taosGetSelfPthreadId(), (t2 - t1)/1000.0); - } else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) { - int64_t t1 = taosGetTimestampMs(); - int retCode = postProceSql(g_queryInfo.host, &(g_queryInfo.serv_addr), - g_queryInfo.port, - g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]); - if (0 != retCode) { - printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID); - return NULL; - } - int64_t t2 = taosGetTimestampMs(); - printf("=[restful] thread[%"PRId64"] complete one sql, Spent %10.3f s\n", - taosGetSelfPthreadId(), (t2 - t1)/1000.0); - - } else { - errorPrint("%s() LN%d, unknown query mode: %s\n", - __func__, __LINE__, g_queryInfo.queryMode); - return NULL; } - totalQueried ++; - g_queryInfo.specifiedQueryInfo.totalQueried ++; + + st = taosGetTimestampMs(); + + selectAndGetResult(pThreadInfo, + g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile); et = taosGetTimestampMs(); + printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n", + taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0); + + totalQueried ++; + g_queryInfo.specifiedQueryInfo.totalQueried ++; uint64_t currentPrintTime = taosGetTimestampMs(); uint64_t endTs = taosGetTimestampMs(); @@ -6194,14 +6207,14 @@ static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) { tstrncpy(outSql, inSql, pos - inSql + 1); //printf("1: %s\n", outSql); - strcat(outSql, subTblName); + strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1); //printf("2: %s\n", outSql); - strcat(outSql, pos+strlen(sourceString)); + strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1); //printf("3: %s\n", outSql); } static void *superTableQuery(void *sarg) { - char sqlstr[1024]; + char sqlstr[MAX_QUERY_SQL_LENGTH]; threadInfo *pThreadInfo = (threadInfo *)sarg; if (pThreadInfo->taos == NULL) { @@ -6246,7 +6259,7 @@ static void *superTableQuery(void *sarg) { g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - selectAndGetResult(pThreadInfo->taos, sqlstr, tmpFile); + selectAndGetResult(pThreadInfo, sqlstr, tmpFile); totalQueried++; g_queryInfo.superQueryInfo.totalQueried ++; @@ -6447,7 +6460,8 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c return; } - getResult(res, (char*)param); + if (param) + appendResultToFile(res, (char*)param); // tao_unscribe() will free result. } @@ -6476,7 +6490,7 @@ static TAOS_SUB* subscribeImpl( static void *superSubscribe(void *sarg) { threadInfo *pThreadInfo = (threadInfo *)sarg; - char subSqlstr[1024]; + char subSqlstr[MAX_QUERY_SQL_LENGTH]; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; if (g_queryInfo.superQueryInfo.sqlCount == 0) @@ -6551,8 +6565,8 @@ static void *superSubscribe(void *sarg) { sprintf(tmpFile, "%s-%d", g_queryInfo.superQueryInfo.result[i], pThreadInfo->threadID); + appendResultToFile(res, tmpFile); } - getResult(res, tmpFile); } } } @@ -6639,8 +6653,8 @@ static void *specifiedSubscribe(void *sarg) { if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { sprintf(tmpFile, "%s-%d", g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); + appendResultToFile(res, tmpFile); } - getResult(res, tmpFile); } } } From edbc123e2e0b4265e2ac89d85f10bb82bb34c39e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 May 2021 10:43:53 +0800 Subject: [PATCH 15/19] [td-225]update the test case. --- .../jdbc/cases/InsertSpecialCharacterRestfulTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java index 0cbbe76716..ea0d1aec41 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java @@ -13,7 +13,7 @@ public class InsertSpecialCharacterRestfulTest { private static String tbname1 = "test"; private static String tbname2 = "weather"; private static String special_character_str_1 = "$asd$$fsfsf$"; - private static String special_character_str_2 = "\\asdfsfsf\\\\"; + private static String special_character_str_2 = "\\\\asdfsfsf\\\\"; private static String special_character_str_3 = "\\\\asdfsfsf\\"; private static String special_character_str_4 = "?asd??fsf?sf?"; private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; @@ -49,7 +49,7 @@ public class InsertSpecialCharacterRestfulTest { @Test public void testCase02() throws SQLException { //TODO: - // Expected :\asdfsfsf\\ + // Expected :\asdfsfsf\ // Actual :\asdfsfsf\ final long now = System.currentTimeMillis(); @@ -71,7 +71,7 @@ public class InsertSpecialCharacterRestfulTest { String f1 = new String(rs.getBytes(2)); //TODO: bug to be fixed // Assert.assertEquals(special_character_str_2, f1); - Assert.assertEquals(special_character_str_2.substring(0, special_character_str_1.length() - 2), f1); + Assert.assertEquals(special_character_str_2.substring(1, special_character_str_1.length() - 1), f1); String f2 = rs.getString(3); Assert.assertNull(f2); } From 055919a7a71f233bbc77517ded850f197061993c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 15:38:05 +0800 Subject: [PATCH 16/19] Hotfix/sangshuduo/td 3992 taosdemo subscribe (#6062) * [TD-3902]: taosdemo subscribe. * [TD-3992]: taosdemo subscribe. refactor sync/async mode. Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 128 ++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 935aa358c5..3c83cb7a5e 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -68,12 +68,6 @@ enum TEST_MODE { INVAID_TEST }; -enum QUERY_MODE { - SYNC_QUERY_MODE, // 0 - ASYNC_QUERY_MODE, // 1 - INVALID_MODE -}; - #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) #define COND_BUF_LEN BUFFER_SIZE - 30 @@ -119,8 +113,8 @@ typedef enum TALBE_EXISTS_EN { } TALBE_EXISTS_EN; enum MODE { - SYNC, - ASYNC, + SYNC_MODE, + ASYNC_MODE, MODE_BUT }; @@ -206,7 +200,7 @@ typedef struct SArguments_S { bool verbose_print; bool performance_print; char * output_file; - uint32_t query_mode; + bool async_mode; char * datatype[MAX_NUM_DATATYPE + 1]; uint32_t len_of_binary; uint32_t num_of_CPR; @@ -344,7 +338,7 @@ typedef struct SDbs_S { bool use_metric; bool insert_only; bool do_aggreFunc; - bool queryMode; + bool asyncMode; uint32_t threadCount; uint32_t threadCountByCreateTbl; @@ -361,7 +355,7 @@ typedef struct SpecifiedQueryInfo_S { uint64_t queryInterval; // 0: unlimit > 0 loop/s uint64_t concurrent; uint64_t sqlCount; - uint32_t mode; // 0: sync, 1: async + uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms uint64_t queryTimes; int subscribeRestart; @@ -376,7 +370,7 @@ typedef struct SuperQueryInfo_S { char sTblName[MAX_TB_NAME_SIZE+1]; uint64_t queryInterval; // 0: unlimit > 0 loop/s uint32_t threadCnt; - uint32_t mode; // 0: sync, 1: async + uint32_t asyncMode; // 0: sync, 1: async uint64_t subscribeInterval; // ms int subscribeRestart; int subscribeKeepProgress; @@ -774,49 +768,48 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) { } arguments->sqlFile = argv[++i]; } else if (strcmp(argv[i], "-q") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n"); exit(EXIT_FAILURE); } - arguments->query_mode = atoi(argv[++i]); + arguments->async_mode = atoi(argv[++i]); } else if (strcmp(argv[i], "-T") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-T need a number following!\n"); exit(EXIT_FAILURE); } arguments->num_of_threads = atoi(argv[++i]); } else if (strcmp(argv[i], "-i") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-i need a number following!\n"); exit(EXIT_FAILURE); } arguments->insert_interval = atoi(argv[++i]); } else if (strcmp(argv[i], "-qt") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1])) - || (atoi(argv[i+1]) <= 0)) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); - errorPrint("%s", "\n\t-qt need a valid (>0) number following!\n"); + errorPrint("%s", "\n\t-qt need a number following!\n"); exit(EXIT_FAILURE); } arguments->query_times = atoi(argv[++i]); } else if (strcmp(argv[i], "-B") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-B need a number following!\n"); exit(EXIT_FAILURE); } arguments->interlace_rows = atoi(argv[++i]); } else if (strcmp(argv[i], "-r") == 0) { - if ((argc == i+1) - || (!isStringNumber(argv[i+1]))) { + if ((argc == i+1) || + (!isStringNumber(argv[i+1]))) { printHelp(); errorPrint("%s", "\n\t-r need a number following!\n"); exit(EXIT_FAILURE); @@ -1076,7 +1069,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) { if (code != 0) { if (!quiet) { debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); - errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); + errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res)); } taos_free_result(res); //taos_close(taos); @@ -1672,7 +1665,7 @@ static void printfQueryMeta() { printf("concurrent: \033[33m%"PRIu64"\033[0m\n", g_queryInfo.specifiedQueryInfo.concurrent); printf("mod: \033[33m%s\033[0m\n", - (g_queryInfo.specifiedQueryInfo.mode)?"async":"sync"); + (g_queryInfo.specifiedQueryInfo.asyncMode)?"async":"sync"); printf("interval: \033[33m%"PRIu64"\033[0m\n", g_queryInfo.specifiedQueryInfo.subscribeInterval); printf("restart: \033[33m%d\033[0m\n", @@ -1704,7 +1697,7 @@ static void printfQueryMeta() { g_queryInfo.superQueryInfo.queryTimes); printf("mod: \033[33m%s\033[0m\n", - (g_queryInfo.superQueryInfo.mode)?"async":"sync"); + (g_queryInfo.superQueryInfo.asyncMode)?"async":"sync"); printf("interval: \033[33m%"PRIu64"\033[0m\n", g_queryInfo.superQueryInfo.subscribeInterval); printf("restart: \033[33m%d\033[0m\n", @@ -4072,9 +4065,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); if (gQueryTimes && gQueryTimes->type == cJSON_Number) { - if (gQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, gQueryTimes->valueint); + if (gQueryTimes->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } g_args.query_times = gQueryTimes->valueint; @@ -4123,9 +4116,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, "query_times"); if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { - if (specifiedQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, specifiedQueryTimes->valueint); + if (specifiedQueryTimes->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } @@ -4152,20 +4145,20 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { g_queryInfo.specifiedQueryInfo.concurrent = 1; } - cJSON* mode = cJSON_GetObjectItem(specifiedQuery, "mode"); - if (mode && mode->type == cJSON_String - && mode->valuestring != NULL) { - if (0 == strcmp("sync", mode->valuestring)) { - g_queryInfo.specifiedQueryInfo.mode = SYNC_QUERY_MODE; - } else if (0 == strcmp("async", mode->valuestring)) { - g_queryInfo.specifiedQueryInfo.mode = ASYNC_QUERY_MODE; + cJSON* specifiedAsyncMode = cJSON_GetObjectItem(specifiedQuery, "mode"); + if (specifiedAsyncMode && specifiedAsyncMode->type == cJSON_String + && specifiedAsyncMode->valuestring != NULL) { + if (0 == strcmp("sync", specifiedAsyncMode->valuestring)) { + g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE; + } else if (0 == strcmp("async", specifiedAsyncMode->valuestring)) { + g_queryInfo.specifiedQueryInfo.asyncMode = ASYNC_MODE; } else { - errorPrint("%s() LN%d, failed to read json, query mode input error\n", + errorPrint("%s() LN%d, failed to read json, async mode input error\n", __func__, __LINE__); goto PARSE_OVER; } } else { - g_queryInfo.specifiedQueryInfo.mode = SYNC_QUERY_MODE; + g_queryInfo.specifiedQueryInfo.asyncMode = SYNC_MODE; } cJSON* interval = cJSON_GetObjectItem(specifiedQuery, "interval"); @@ -4267,9 +4260,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); if (superQueryTimes && superQueryTimes->type == cJSON_Number) { - if (superQueryTimes->valueint <= 0) { - errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", - __func__, __LINE__, superQueryTimes->valueint); + if (superQueryTimes->valueint < 0) { + errorPrint("%s() LN%d, failed to read json, query_times input mistake\n", + __func__, __LINE__); goto PARSE_OVER; } g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; @@ -4312,20 +4305,20 @@ static bool getMetaFromQueryJsonFile(cJSON* root) { goto PARSE_OVER; } - cJSON* submode = cJSON_GetObjectItem(superQuery, "mode"); - if (submode && submode->type == cJSON_String - && submode->valuestring != NULL) { - if (0 == strcmp("sync", submode->valuestring)) { - g_queryInfo.superQueryInfo.mode = SYNC_QUERY_MODE; - } else if (0 == strcmp("async", submode->valuestring)) { - g_queryInfo.superQueryInfo.mode = ASYNC_QUERY_MODE; + cJSON* superAsyncMode = cJSON_GetObjectItem(superQuery, "mode"); + if (superAsyncMode && superAsyncMode->type == cJSON_String + && superAsyncMode->valuestring != NULL) { + if (0 == strcmp("sync", superAsyncMode->valuestring)) { + g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE; + } else if (0 == strcmp("async", superAsyncMode->valuestring)) { + g_queryInfo.superQueryInfo.asyncMode = ASYNC_MODE; } else { - errorPrint("%s() LN%d, failed to read json, query mode input error\n", + errorPrint("%s() LN%d, failed to read json, async mode input error\n", __func__, __LINE__); goto PARSE_OVER; } } else { - g_queryInfo.superQueryInfo.mode = SYNC_QUERY_MODE; + g_queryInfo.superQueryInfo.asyncMode = SYNC_MODE; } cJSON* superInterval = cJSON_GetObjectItem(superQuery, "interval"); @@ -5233,13 +5226,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) { startTs = taosGetTimestampMs(); - if (recOfBatch == 0) { - errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n", - pThreadInfo->threadID, __func__, __LINE__, - recOfBatch); - errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n"); - goto free_of_interlace; - } int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch); endTs = taosGetTimestampMs(); @@ -5780,10 +5766,10 @@ static void startMultiThreadInsertData(int threads, char* db_name, } */ tsem_init(&(t_info->lock_sem), 0, 0); - if (SYNC == g_Dbs.queryMode) { - pthread_create(pids + i, NULL, syncWrite, t_info); - } else { + if (ASYNC_MODE == g_Dbs.asyncMode) { pthread_create(pids + i, NULL, asyncWrite, t_info); + } else { + pthread_create(pids + i, NULL, syncWrite, t_info); } } @@ -6469,7 +6455,7 @@ static TAOS_SUB* subscribeImpl( TAOS *taos, char *sql, char* topic, char* resultFileName) { TAOS_SUB* tsub = NULL; - if (ASYNC_QUERY_MODE == g_queryInfo.specifiedQueryInfo.mode) { + if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { tsub = taos_subscribe(taos, g_queryInfo.specifiedQueryInfo.subscribeRestart, topic, sql, subscribe_callback, (void*)resultFileName, @@ -6554,7 +6540,7 @@ static void *superSubscribe(void *sarg) { TAOS_RES* res = NULL; while(1) { for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (ASYNC_QUERY_MODE == g_queryInfo.superQueryInfo.mode) { + if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { continue; } @@ -6643,7 +6629,7 @@ static void *specifiedSubscribe(void *sarg) { TAOS_RES* res = NULL; while(1) { for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { - if (ASYNC_QUERY_MODE == g_queryInfo.specifiedQueryInfo.mode) { + if (ASYNC_MODE == g_queryInfo.specifiedQueryInfo.asyncMode) { continue; } @@ -6866,7 +6852,7 @@ static void setParaFromArg(){ g_Dbs.db[0].superTbls[0].childTblCount = g_args.num_of_tables; g_Dbs.threadCount = g_args.num_of_threads; g_Dbs.threadCountByCreateTbl = g_args.num_of_threads; - g_Dbs.queryMode = g_args.query_mode; + g_Dbs.asyncMode = g_args.async_mode; g_Dbs.db[0].superTbls[0].autoCreateTable = PRE_CREATE_SUBTBL; g_Dbs.db[0].superTbls[0].childTblExists = TBL_NO_EXISTS; From af33e1184063a047de7eb26805fd4bd314dce579 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 19:06:02 +0800 Subject: [PATCH 17/19] [TD-4136]: taosdemo max records per req < 32767 (#6082) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 12 +++++++++++- .../tools/taosdemoAllTest/speciQueryRestful.json | 4 ++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 3c83cb7a5e..a3a8968017 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -68,6 +68,14 @@ enum TEST_MODE { INVAID_TEST }; +enum QUERY_MODE { + SYNC_QUERY_MODE, // 0 + ASYNC_QUERY_MODE, // 1 + INVALID_MODE +}; + +#define MAX_RECORDS_PER_REQ 32766 + #define MAX_SQL_SIZE 65536 #define BUFFER_SIZE (65536*2) #define COND_BUF_LEN BUFFER_SIZE - 30 @@ -3436,10 +3444,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) { errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n", __func__, __LINE__); goto PARSE_OVER; + } else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) { + numRecPerReq->valueint = MAX_RECORDS_PER_REQ; } g_args.num_of_RPR = numRecPerReq->valueint; } else if (!numRecPerReq) { - g_args.num_of_RPR = UINT64_MAX; + g_args.num_of_RPR = MAX_RECORDS_PER_REQ; } else { errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", __func__, __LINE__); diff --git a/tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json b/tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json index 98e9b7a4e8..bc3a66f43c 100644 --- a/tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json +++ b/tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json @@ -8,7 +8,7 @@ "confirm_parameter_prompt": "no", "databases": "db", "query_times": 2, - "query_mode": "restful", + "query_mode": "rest", "specified_table_query": { "query_interval": 1, "concurrent": 3, @@ -35,4 +35,4 @@ ] } } - \ No newline at end of file + From 95139cf4a1c03cd6f9bb60fe0daca48bbd84e4e9 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Tue, 11 May 2021 21:36:48 +0800 Subject: [PATCH 18/19] [TD-4130]: taosdemo subscribe super table. (#6086) Co-authored-by: Shuduo Sang --- src/kit/taosdemo/taosdemo.c | 134 +++++++++++++++++------------------- 1 file changed, 63 insertions(+), 71 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index a3a8968017..8866bf2607 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -68,12 +68,6 @@ enum TEST_MODE { INVAID_TEST }; -enum QUERY_MODE { - SYNC_QUERY_MODE, // 0 - ASYNC_QUERY_MODE, // 1 - INVALID_MODE -}; - #define MAX_RECORDS_PER_REQ 32766 #define MAX_SQL_SIZE 65536 @@ -1107,6 +1101,7 @@ static void appendResultBufToFile(char *resultBuf, char *resultFile) } } + fprintf(fp, "%s", resultBuf); tmfclose(fp); } @@ -1142,6 +1137,7 @@ static void appendResultToFile(TAOS_RES *res, char* resultFile) { totalLen += len; } + verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", __func__, __LINE__, databuf, resultFile); appendResultBufToFile(databuf, resultFile); free(databuf); } @@ -6517,59 +6513,63 @@ static void *superSubscribe(void *sarg) { return NULL; } - //int64_t st = 0; - //int64_t et = 0; - do { - //if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < g_queryInfo.specifiedQueryInfo.queryInterval) { - // taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval- (et - st)); // ms - // //printf("========sleep duration:%"PRId64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - //} - - //st = taosGetTimestampMs(); - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - sprintf(topic, "taosdemo-subscribe-%d", i); + char topic[32] = {0}; + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + sprintf(topic, "taosdemo-subscribe-%"PRIu64"-%d", i, j); memset(subSqlstr,0,sizeof(subSqlstr)); - replaceChildTblName(g_queryInfo.superQueryInfo.sql[i], subSqlstr, i); + replaceChildTblName(g_queryInfo.superQueryInfo.sql[j], subSqlstr, i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { + if (g_queryInfo.superQueryInfo.result[j][0] != 0) { sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], pThreadInfo->threadID); + g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); } - tsub[i] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); - if (NULL == tsub[i]) { + + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + debugPrint("%s() LN%d, subSeq=%"PRIu64" subSqlstr: %s\n", + __func__, __LINE__, subSeq, subSqlstr); + tsub[subSeq] = subscribeImpl(pThreadInfo->taos, subSqlstr, topic, tmpFile); + if (NULL == tsub[subSeq]) { taos_close(pThreadInfo->taos); return NULL; } } - //et = taosGetTimestampMs(); - //printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); - } while(0); + } // start loop to consume result TAOS_RES* res = NULL; while(1) { - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { - continue; - } + for (uint64_t i = pThreadInfo->start_table_from; i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + if (ASYNC_MODE == g_queryInfo.superQueryInfo.asyncMode) { + continue; + } - res = taos_consume(tsub[i]); - if (res) { - char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; - if (g_queryInfo.superQueryInfo.result[i][0] != 0) { - sprintf(tmpFile, "%s-%d", - g_queryInfo.superQueryInfo.result[i], + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + taosMsleep(100); // ms + res = taos_consume(tsub[subSeq]); + if (res) { + char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; + if (g_queryInfo.superQueryInfo.result[j][0] != 0) { + sprintf(tmpFile, "%s-%d", + g_queryInfo.superQueryInfo.result[j], pThreadInfo->threadID); - appendResultToFile(res, tmpFile); + appendResultToFile(res, tmpFile); + } } } } } taos_free_result(res); - for (int i = 0; i < g_queryInfo.superQueryInfo.sqlCount; i++) { - taos_unsubscribe(tsub[i], g_queryInfo.superQueryInfo.subscribeKeepProgress); + for (uint64_t i = pThreadInfo->start_table_from; + i <= pThreadInfo->end_table_to; i++) { + for (int j = 0; j < g_queryInfo.superQueryInfo.sqlCount; j++) { + uint64_t subSeq = i * g_queryInfo.superQueryInfo.sqlCount + j; + taos_unsubscribe(tsub[subSeq], + g_queryInfo.superQueryInfo.subscribeKeepProgress); + } } taos_close(pThreadInfo->taos); @@ -6607,17 +6607,8 @@ static void *specifiedSubscribe(void *sarg) { return NULL; } - //int64_t st = 0; - //int64_t et = 0; - do { - //if (g_queryInfo.specifiedQueryInfo.queryInterval && (et - st) < g_queryInfo.specifiedQueryInfo.queryInterval) { - // taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval- (et - st)); // ms - // //printf("========sleep duration:%"PRIu64 "========inserted rows:%d, table range:%d - %d\n", (1000 - (et - st)), i, pThreadInfo->start_table_from, pThreadInfo->end_table_to); - //} - - //st = taosGetTimestampMs(); - char topic[32] = {0}; - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { + char topic[32] = {0}; + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.sqlCount; i++) { sprintf(topic, "taosdemo-subscribe-%d", i); char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { @@ -6630,11 +6621,7 @@ static void *specifiedSubscribe(void *sarg) { taos_close(pThreadInfo->taos); return NULL; } - } - //et = taosGetTimestampMs(); - //printf("========thread[%"PRIu64"] complete all sqls to super table once queries duration:%.4fs\n", taosGetSelfPthreadId(), (double)(et - st)/1000.0); - } while(0); - + } // start loop to consume result TAOS_RES* res = NULL; while(1) { @@ -6643,6 +6630,7 @@ static void *specifiedSubscribe(void *sarg) { continue; } + taosMsleep(1000); // ms res = taos_consume(tsub[i]); if (res) { char tmpFile[MAX_FILE_NAME_LEN*2] = {0}; @@ -6699,31 +6687,35 @@ static int subscribeTestProcess() { pthread_t *pids = NULL; threadInfo *infos = NULL; - //==== create sub threads for query from super table - if ((g_queryInfo.specifiedQueryInfo.sqlCount <= 0) || - (g_queryInfo.specifiedQueryInfo.concurrent <= 0)) { - errorPrint("%s() LN%d, query sqlCount %"PRIu64" or concurrent %"PRIu64" is not correct.\n", + //==== create sub threads for query for specified table + if (g_queryInfo.specifiedQueryInfo.sqlCount <= 0) { + printf("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", __func__, __LINE__, - g_queryInfo.specifiedQueryInfo.sqlCount, - g_queryInfo.specifiedQueryInfo.concurrent); - exit(-1); - } + g_queryInfo.specifiedQueryInfo.sqlCount); + } else { + if (g_queryInfo.specifiedQueryInfo.concurrent <= 0) { + errorPrint("%s() LN%d, sepcified query sqlCount %"PRIu64".\n", + __func__, __LINE__, + g_queryInfo.specifiedQueryInfo.sqlCount); + exit(-1); + } - pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); - infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); - if ((NULL == pids) || (NULL == infos)) { - errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); - exit(-1); - } + pids = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(pthread_t)); + infos = malloc(g_queryInfo.specifiedQueryInfo.concurrent * sizeof(threadInfo)); + if ((NULL == pids) || (NULL == infos)) { + errorPrint("%s() LN%d, malloc failed for create threads\n", __func__, __LINE__); + exit(-1); + } - for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { + for (int i = 0; i < g_queryInfo.specifiedQueryInfo.concurrent; i++) { threadInfo *t_info = infos + i; t_info->threadID = i; t_info->taos = NULL; // TODO: workaround to use separate taos connection; pthread_create(pids + i, NULL, specifiedSubscribe, t_info); + } } - //==== create sub threads for query from sub table + //==== create sub threads for super table query pthread_t *pidsOfSub = NULL; threadInfo *infosOfSub = NULL; if ((g_queryInfo.superQueryInfo.sqlCount > 0) From 5131e7b12a664ddcdfc48a1d11125d71f92cf64a Mon Sep 17 00:00:00 2001 From: zyyang <69311263+zyyang-taosdata@users.noreply.github.com> Date: Wed, 12 May 2021 09:41:03 +0800 Subject: [PATCH 19/19] Fix/td 3986 (#6088) * [TD-3986]: fix subscribe test case error * change jdbc version number --- cmake/install.inc | 2 +- src/connector/jdbc/CMakeLists.txt | 2 +- src/connector/jdbc/deploy-pom.xml | 2 +- src/connector/jdbc/pom.xml | 2 +- .../com/taosdata/jdbc/TSDBJNIConnector.java | 15 ++----- .../java/com/taosdata/jdbc/TSDBResultSet.java | 13 +++--- .../java/com/taosdata/jdbc/SubscribeTest.java | 44 +++++++++++++------ 7 files changed, 45 insertions(+), 35 deletions(-) diff --git a/cmake/install.inc b/cmake/install.inc index 9e325531d5..f8b3b7c3c6 100755 --- a/cmake/install.inc +++ b/cmake/install.inc @@ -32,7 +32,7 @@ ELSEIF (TD_WINDOWS) #INSTALL(TARGETS taos RUNTIME DESTINATION driver) #INSTALL(TARGETS shell RUNTIME DESTINATION .) IF (TD_MVN_INSTALLED) - INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.28-dist.jar DESTINATION connector/jdbc) + INSTALL(FILES ${LIBRARY_OUTPUT_PATH}/taos-jdbcdriver-2.0.29.jar DESTINATION connector/jdbc) ENDIF () ELSEIF (TD_DARWIN) SET(TD_MAKE_INSTALL_SH "${TD_COMMUNITY_DIR}/packaging/tools/make_install.sh") diff --git a/src/connector/jdbc/CMakeLists.txt b/src/connector/jdbc/CMakeLists.txt index de4b8f6bfb..61e976cb18 100644 --- a/src/connector/jdbc/CMakeLists.txt +++ b/src/connector/jdbc/CMakeLists.txt @@ -8,7 +8,7 @@ IF (TD_MVN_INSTALLED) ADD_CUSTOM_COMMAND(OUTPUT ${JDBC_CMD_NAME} POST_BUILD COMMAND mvn -Dmaven.test.skip=true install -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml - COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.28-dist.jar ${LIBRARY_OUTPUT_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/target/taos-jdbcdriver-2.0.29.jar ${LIBRARY_OUTPUT_PATH} COMMAND mvn -Dmaven.test.skip=true clean -f ${CMAKE_CURRENT_SOURCE_DIR}/pom.xml COMMENT "build jdbc driver") ADD_CUSTOM_TARGET(${JDBC_TARGET_NAME} ALL WORKING_DIRECTORY ${EXECUTABLE_OUTPUT_PATH} DEPENDS ${JDBC_CMD_NAME}) diff --git a/src/connector/jdbc/deploy-pom.xml b/src/connector/jdbc/deploy-pom.xml index a31796ffde..968a9bf470 100755 --- a/src/connector/jdbc/deploy-pom.xml +++ b/src/connector/jdbc/deploy-pom.xml @@ -5,7 +5,7 @@ com.taosdata.jdbc taos-jdbcdriver - 2.0.28 + 2.0.29 jar JDBCDriver diff --git a/src/connector/jdbc/pom.xml b/src/connector/jdbc/pom.xml index 3400a82e73..d94d28d9fa 100755 --- a/src/connector/jdbc/pom.xml +++ b/src/connector/jdbc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.taosdata.jdbc taos-jdbcdriver - 2.0.28 + 2.0.29 jar JDBCDriver https://github.com/taosdata/TDengine/tree/master/src/connector/jdbc diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java index 5e3ffffa4f..7f0cf7de8d 100755 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java @@ -32,7 +32,7 @@ public class TSDBJNIConnector { // Connection pointer used in C private long taos = TSDBConstants.JNI_NULL_POINTER; // result set status in current connection - private boolean isResultsetClosed = true; + private boolean isResultsetClosed; private int affectedRows = -1; static { @@ -135,6 +135,7 @@ public class TSDBJNIConnector { // Try retrieving result set for the executed SQL using the current connection pointer. pSql = this.getResultSetImp(this.taos, pSql); + // if pSql == 0L that means resultset is closed isResultsetClosed = (pSql == TSDBConstants.JNI_NULL_POINTER); return pSql; @@ -172,16 +173,7 @@ public class TSDBJNIConnector { * Free resultset operation from C to release resultset pointer by JNI */ public int freeResultSet(long pSql) { - int res = TSDBConstants.JNI_SUCCESS; -// if (result != taosResultSetPointer && taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { -// throw new RuntimeException("Invalid result set pointer"); -// } - -// if (taosResultSetPointer != TSDBConstants.JNI_NULL_POINTER) { - res = this.freeResultSetImp(this.taos, pSql); -// taosResultSetPointer = TSDBConstants.JNI_NULL_POINTER; -// } - + int res = this.freeResultSetImp(this.taos, pSql); isResultsetClosed = true; return res; } @@ -199,7 +191,6 @@ public class TSDBJNIConnector { // } // return resCode; // } - private native int freeResultSetImp(long connection, long result); /** diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java index 2576a25f0d..aba29d602b 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBResultSet.java @@ -109,6 +109,8 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { public void close() throws SQLException { if (isClosed) return; + if (this.statement == null) + return; if (this.jniConnector != null) { int code = this.jniConnector.freeResultSet(this.resultSetPointer); if (code == TSDBConstants.JNI_CONNECTION_NULL) { @@ -461,12 +463,13 @@ public class TSDBResultSet extends AbstractResultSet implements ResultSet { } public boolean isClosed() throws SQLException { - if (isClosed) - return true; - if (jniConnector != null) { - isClosed = jniConnector.isResultsetClosed(); - } return isClosed; +// if (isClosed) +// return true; +// if (jniConnector != null) { +// isClosed = jniConnector.isResultsetClosed(); +// } +// return isClosed; } public String getNString(int columnIndex) throws SQLException { diff --git a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java index 3a223ed981..24c73fdd5c 100644 --- a/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java +++ b/src/connector/jdbc/src/test/java/com/taosdata/jdbc/SubscribeTest.java @@ -1,6 +1,7 @@ package com.taosdata.jdbc; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -19,6 +20,7 @@ public class SubscribeTest { String tName = "t0"; String host = "127.0.0.1"; String topic = "test"; + private long ts; @Test public void subscribe() { @@ -27,26 +29,40 @@ public class SubscribeTest { TSDBConnection conn = connection.unwrap(TSDBConnection.class); TSDBSubscribe subscribe = conn.subscribe(topic, rawSql, false); - int a = 0; - while (true) { - TimeUnit.MILLISECONDS.sleep(1000); + for (int j = 0; j < 10; j++) { + TimeUnit.SECONDS.sleep(1); TSDBResultSet resSet = subscribe.consume(); + + int rowCnt = 0; while (resSet.next()) { - for (int i = 1; i <= resSet.getMetaData().getColumnCount(); i++) { - System.out.printf(i + ": " + resSet.getString(i) + "\t"); + if (rowCnt == 0) { + long cur_ts = resSet.getTimestamp(1).getTime(); + int k = resSet.getInt(2); + int v = resSet.getInt(3); + Assert.assertEquals(ts, cur_ts); + Assert.assertEquals(100, k); + Assert.assertEquals(1, v); } - System.out.println("\n======" + a + "=========="); - } - a++; - if (a >= 2) { - break; + if (rowCnt == 1) { + long cur_ts = resSet.getTimestamp(1).getTime(); + int k = resSet.getInt(2); + int v = resSet.getInt(3); + Assert.assertEquals(ts + 1, cur_ts); + Assert.assertEquals(101, k); + Assert.assertEquals(2, v); + + } + rowCnt++; } + if (j == 0) + Assert.assertEquals(2, rowCnt); resSet.close(); } - subscribe.close(true); - } catch (Exception e) { - e.printStackTrace(); + + + } catch (SQLException | InterruptedException throwables) { + throwables.printStackTrace(); } } @@ -62,7 +78,7 @@ public class SubscribeTest { statement.execute("drop database if exists " + dbName); statement.execute("create database if not exists " + dbName); statement.execute("create table if not exists " + dbName + "." + tName + " (ts timestamp, k int, v int)"); - long ts = System.currentTimeMillis(); + ts = System.currentTimeMillis(); statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + ts + ", 100, 1)"); statement.executeUpdate("insert into " + dbName + "." + tName + " values (" + (ts + 1) + ", 101, 2)"); }