xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge from master into develop

This commit is contained in:
Shengliang Guan 2021-05-11 20:58:18 +08:00
parent 4f7a835ad6 9c0e8c9d78
commit 644f41a0bb
32 changed files with 353 additions and 243 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/client/inc/tschemautil.h
View File

@ -21,8 +21,8 @@ extern "C" {
#endif #endif
#include "taosmsg.h" #include "taosmsg.h"
#include "tstoken.h"
#include "tsclient.h" #include "tsclient.h"
#include "ttoken.h"
/** /**
* get the number of tags of this table * get the number of tags of this table

22
src/client/src/tscParseInsert.c
View File

@ -29,8 +29,7 @@
#include "taosdef.h" #include "taosdef.h"
#include "tscLog.h" #include "tscLog.h"
#include "tscSubquery.h" #include "ttoken.h"
#include "tstoken.h"
#include "tdataformat.h" #include "tdataformat.h"
@ -463,23 +462,24 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, SSqlCmd *pCmd, int1
// Remove quotation marks // Remove quotation marks
if (TK_STRING == sToken.type) { if (TK_STRING == sToken.type) {
// delete escape character: \\, \', \" // delete escape character: \\, \', \"
char delim = sToken.z[0]; char delim = sToken.z[0];
int32_t cnt = 0; int32_t cnt = 0;
int32_t j = 0; int32_t j = 0;
for (uint32_t k = 1; k < sToken.n - 1; ++k) { for (uint32_t k = 1; k < sToken.n - 1; ++k) {
if (sToken.z[k] == delim || sToken.z[k] == '\\') { if (sToken.z[k] == '\\' || (sToken.z[k] == delim && sToken.z[k + 1] == delim)) {
if (sToken.z[k + 1] == delim) {
cnt++;
tmpTokenBuf[j] = sToken.z[k + 1]; tmpTokenBuf[j] = sToken.z[k + 1];
j++;
k++; cnt++;
continue; j++;
} k++;
continue;
} }
tmpTokenBuf[j] = sToken.z[k]; tmpTokenBuf[j] = sToken.z[k];
j++; j++;
} }
tmpTokenBuf[j] = 0; tmpTokenBuf[j] = 0;
sToken.z = tmpTokenBuf; sToken.z = tmpTokenBuf;
sToken.n -= 2 + cnt; sToken.n -= 2 + cnt;
@ -1003,7 +1003,7 @@ int validateTableName(char *tblName, int len, SStrToken* psTblToken) {
psTblToken->n = len; psTblToken->n = len;
psTblToken->type = TK_ID; psTblToken->type = TK_ID;
tSQLGetToken(psTblToken->z, &psTblToken->type); tGetToken(psTblToken->z, &psTblToken->type);
return tscValidateName(psTblToken); return tscValidateName(psTblToken);
} }

2
src/client/src/tscPrepare.c
View File

@ -151,7 +151,7 @@ static int normalStmtPrepare(STscStmt* stmt) {
while (sql[i] != 0) { while (sql[i] != 0) {
SStrToken token = {0}; SStrToken token = {0};
token.n = tSQLGetToken(sql + i, &token.type); token.n = tGetToken(sql + i, &token.type);
if (token.type == TK_QUESTION) { if (token.type == TK_QUESTION) {
sql[i] = 0; sql[i] = 0;

27
src/client/src/tscSQLParser.c
View File

@ -21,19 +21,19 @@
#endif // __APPLE__ #endif // __APPLE__
#include "os.h" #include "os.h"
#include "ttype.h"
#include "texpr.h"
#include "taos.h" #include "taos.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tcompare.h" #include "tcompare.h"
#include "texpr.h"
#include "tname.h" #include "tname.h"
#include "tscLog.h" #include "tscLog.h"
#include "tscUtil.h" #include "tscUtil.h"
#include "tschemautil.h" #include "tschemautil.h"
#include "tsclient.h" #include "tsclient.h"
#include "tstoken.h"
#include "tstrbuild.h" #include "tstrbuild.h"
#include "ttoken.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "ttype.h"
#include "qUtil.h" #include "qUtil.h"
#define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0"
@ -432,7 +432,6 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) {
if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
} }
// additional msg has been attached already // additional msg has been attached already
code = tscSetTableFullName(pTableMetaInfo, pToken, pSql); code = tscSetTableFullName(pTableMetaInfo, pToken, pSql);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -983,11 +982,10 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam
const char* msg3 = "no acctId"; const char* msg3 = "no acctId";
const char* msg4 = "db name too long"; const char* msg4 = "db name too long";
const char* msg5 = "table name too long"; const char* msg5 = "table name too long";
SSqlCmd* pCmd = &pSql->cmd; SSqlCmd* pCmd = &pSql->cmd;
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t idx = getDelimiterIndex(pTableName); int32_t idx = getDelimiterIndex(pTableName);
if (idx != -1) { // db has been specified in sql string so we ignore current db path if (idx != -1) { // db has been specified in sql string so we ignore current db path
char* acctId = getAccountId(pSql); char* acctId = getAccountId(pSql);
if (acctId == NULL || strlen(acctId) <= 0) { if (acctId == NULL || strlen(acctId) <= 0) {
@ -1001,9 +999,9 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pTableNam
if (idx >= TSDB_DB_NAME_LEN) { if (idx >= TSDB_DB_NAME_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
} }
if (pTableName->n - 1 - idx >= TSDB_TABLE_NAME_LEN) { if (pTableName->n - 1 - idx >= TSDB_TABLE_NAME_LEN) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5);
} }
char name[TSDB_TABLE_FNAME_LEN] = {0}; char name[TSDB_TABLE_FNAME_LEN] = {0};
@ -1350,8 +1348,8 @@ static char* cloneCurrentDBName(SSqlObj* pSql) {
/* length limitation, strstr cannot be applied */ /* length limitation, strstr cannot be applied */
static int32_t getDelimiterIndex(SStrToken* pTableName) { static int32_t getDelimiterIndex(SStrToken* pTableName) {
for (uint32_t i = 0; i < pTableName->n; ++i) { for (uint32_t i = 0; i < pTableName->n; ++i) {
if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { if (pTableName->z[i] == TS_PATH_DELIMITER[0]) {
return i; return i;
} }
} }
@ -4645,7 +4643,7 @@ int32_t getTimeRange(STimeWindow* win, tSqlExpr* pRight, int32_t optr, int16_t t
} }
} else { } else {
SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID}; SStrToken token = {.z = pRight->value.pz, .n = pRight->value.nLen, .type = TK_ID};
int32_t len = tSQLGetToken(pRight->value.pz, &token.type); int32_t len = tGetToken(pRight->value.pz, &token.type);
if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) { if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->value.nLen) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
@ -5532,13 +5530,13 @@ int32_t validateLocalConfig(SMiscInfo* pOptions) {
} }
int32_t validateColumnName(char* name) { int32_t validateColumnName(char* name) {
bool ret = isKeyWord(name, (int32_t)strlen(name)); bool ret = taosIsKeyWordToken(name, (int32_t)strlen(name));
if (ret) { if (ret) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
SStrToken token = {.z = name}; SStrToken token = {.z = name};
token.n = tSQLGetToken(name, &token.type); token.n = tGetToken(name, &token.type);
if (token.type != TK_STRING && token.type != TK_ID) { if (token.type != TK_STRING && token.type != TK_ID) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
@ -5549,7 +5547,7 @@ int32_t validateColumnName(char* name) {
strntolower(token.z, token.z, token.n); strntolower(token.z, token.z, token.n);
token.n = (uint32_t)strtrim(token.z); token.n = (uint32_t)strtrim(token.z);
int32_t k = tSQLGetToken(token.z, &token.type); int32_t k = tGetToken(token.z, &token.type);
if (k != token.n) { if (k != token.n) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
@ -7526,4 +7524,3 @@ bool hasNormalColumnFilter(SQueryInfo* pQueryInfo) {
return false; return false;
} }

3
src/client/src/tscServer.c
View File

@ -1881,6 +1881,8 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) {
return TSDB_CODE_TSC_INVALID_VALUE; return TSDB_CODE_TSC_INVALID_VALUE;
} }
assert(pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE || pTableMeta->tableType == TSDB_NORMAL_TABLE || pTableMeta->tableType == TSDB_STREAM_TABLE);
if (pTableMeta->tableType == TSDB_CHILD_TABLE) { if (pTableMeta->tableType == TSDB_CHILD_TABLE) {
// check if super table hashmap or not // check if super table hashmap or not
int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN); int32_t len = (int32_t) strnlen(pTableMeta->sTableName, TSDB_TABLE_FNAME_LEN);
@ -2451,6 +2453,7 @@ int32_t tscGetTableMeta(SSqlObj *pSql, STableMetaInfo *pTableMetaInfo) {
return TSDB_CODE_TSC_OUT_OF_MEMORY; return TSDB_CODE_TSC_OUT_OF_MEMORY;
} }
pTableMetaInfo->pTableMeta = (STableMeta *)tmp; pTableMetaInfo->pTableMeta = (STableMeta *)tmp;
memset(pTableMetaInfo->pTableMeta, 0, size);
pTableMetaInfo->tableMetaSize = size; pTableMetaInfo->tableMetaSize = size;
} else { } else {
//uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta); //uint32_t s = tscGetTableMetaSize(pTableMetaInfo->pTableMeta);

2
src/client/src/tscSql.c
View File

@ -962,7 +962,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t
len = (int32_t)strtrim(tblName); len = (int32_t)strtrim(tblName);
SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName};
tSQLGetToken(tblName, &sToken.type); tGetToken(tblName, &sToken.type);
// Check if the table name available or not // Check if the table name available or not
if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) { if (tscValidateName(&sToken) != TSDB_CODE_SUCCESS) {

8
src/client/src/tscUtil.c
View File

@ -1889,7 +1889,7 @@ void tscColumnListDestroy(SArray* pColumnList) {
static int32_t validateQuoteToken(SStrToken* pToken) { static int32_t validateQuoteToken(SStrToken* pToken) {
tscDequoteAndTrimToken(pToken); tscDequoteAndTrimToken(pToken);
int32_t k = tSQLGetToken(pToken->z, &pToken->type); int32_t k = tGetToken(pToken->z, &pToken->type);
if (pToken->type == TK_STRING) { if (pToken->type == TK_STRING) {
return tscValidateName(pToken); return tscValidateName(pToken);
@ -1957,7 +1957,7 @@ int32_t tscValidateName(SStrToken* pToken) {
tscStrToLower(pToken->z, pToken->n); tscStrToLower(pToken->z, pToken->n);
//pToken->n = (uint32_t)strtrim(pToken->z); //pToken->n = (uint32_t)strtrim(pToken->z);
int len = tSQLGetToken(pToken->z, &pToken->type); int len = tGetToken(pToken->z, &pToken->type);
// single token, validate it // single token, validate it
if (len == pToken->n) { if (len == pToken->n) {
@ -1983,7 +1983,7 @@ int32_t tscValidateName(SStrToken* pToken) {
pToken->n = (uint32_t)strtrim(pToken->z); pToken->n = (uint32_t)strtrim(pToken->z);
} }
pToken->n = tSQLGetToken(pToken->z, &pToken->type); pToken->n = tGetToken(pToken->z, &pToken->type);
if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) { if (pToken->z[pToken->n] != TS_PATH_DELIMITER[0]) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }
@ -2000,7 +2000,7 @@ int32_t tscValidateName(SStrToken* pToken) {
pToken->z = sep + 1; pToken->z = sep + 1;
pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1); pToken->n = (uint32_t)(oldLen - (sep - pStr) - 1);
int32_t len = tSQLGetToken(pToken->z, &pToken->type); int32_t len = tGetToken(pToken->z, &pToken->type);
if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) { if (len != pToken->n || (pToken->type != TK_STRING && pToken->type != TK_ID)) {
return TSDB_CODE_TSC_INVALID_SQL; return TSDB_CODE_TSC_INVALID_SQL;
} }

2
src/client/tests/timeParseTest.cpp
View File

@ -4,7 +4,7 @@
#include <iostream> #include <iostream>
#include "taos.h" #include "taos.h"
#include "tstoken.h" #include "ttoken.h"
#include "tutil.h" #include "tutil.h"
int main(int argc, char** argv) { int main(int argc, char** argv) {

2
src/common/inc/tname.h
View File

@ -18,7 +18,7 @@
#include "os.h" #include "os.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tstoken.h" #include "ttoken.h"
#include "tvariant.h" #include "tvariant.h"
typedef struct SDataStatis { typedef struct SDataStatis {

2
src/common/inc/tvariant.h
View File

@ -16,8 +16,8 @@
#ifndef TDENGINE_TVARIANT_H #ifndef TDENGINE_TVARIANT_H
#define TDENGINE_TVARIANT_H #define TDENGINE_TVARIANT_H
#include "tstoken.h"
#include "tarray.h" #include "tarray.h"
#include "ttoken.h"
#ifdef __cplusplus #ifdef __cplusplus
extern "C" { extern "C" {

2
src/common/src/tname.c
View File

@ -2,7 +2,7 @@
#include "tutil.h" #include "tutil.h"
#include "tname.h" #include "tname.h"
#include "tstoken.h" #include "ttoken.h"
#include "tvariant.h" #include "tvariant.h"
#define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS)

14
src/common/src/tvariant.c
View File

@ -14,14 +14,14 @@
*/ */
#include "os.h" #include "os.h"
#include "tvariant.h"
#include "hash.h" #include "hash.h"
#include "taos.h" #include "taos.h"
#include "taosdef.h" #include "taosdef.h"
#include "tstoken.h" #include "ttoken.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tutil.h"
#include "ttype.h" #include "ttype.h"
#include "tutil.h"
#include "tvariant.h"
void tVariantCreate(tVariant *pVar, SStrToken *token) { void tVariantCreate(tVariant *pVar, SStrToken *token) {
int32_t ret = 0; int32_t ret = 0;
@ -49,7 +49,7 @@ void tVariantCreate(tVariant *pVar, SStrToken *token) {
ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true); ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true);
if (ret != 0) { if (ret != 0) {
SStrToken t = {0}; SStrToken t = {0};
tSQLGetToken(token->z, &t.type); tGetToken(token->z, &t.type);
if (t.type == TK_MINUS) { // it is a signed number which is greater than INT64_MAX or less than INT64_MIN if (t.type == TK_MINUS) { // it is a signed number which is greater than INT64_MAX or less than INT64_MIN
pVar->nType = -1; // -1 means error type pVar->nType = -1; // -1 means error type
return; return;
@ -460,7 +460,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
*result = (int64_t) pVariant->dKey; *result = (int64_t) pVariant->dKey;
} else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) { } else if (pVariant->nType == TSDB_DATA_TYPE_BINARY) {
SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen}; SStrToken token = {.z = pVariant->pz, .n = pVariant->nLen};
/*int32_t n = */tSQLGetToken(pVariant->pz, &token.type); /*int32_t n = */tGetToken(pVariant->pz, &token.type);
if (token.type == TK_NULL) { if (token.type == TK_NULL) {
if (releaseVariantPtr) { if (releaseVariantPtr) {
@ -495,10 +495,10 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result
wchar_t *endPtr = NULL; wchar_t *endPtr = NULL;
SStrToken token = {0}; SStrToken token = {0};
token.n = tSQLGetToken(pVariant->pz, &token.type); token.n = tGetToken(pVariant->pz, &token.type);
if (token.type == TK_MINUS || token.type == TK_PLUS) { if (token.type == TK_MINUS || token.type == TK_PLUS) {
token.n = tSQLGetToken(pVariant->pz + token.n, &token.type); token.n = tGetToken(pVariant->pz + token.n, &token.type);
} }
if (token.type == TK_FLOAT) { if (token.type == TK_FLOAT) {

4
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterJniTest.java
View File

@ -12,7 +12,7 @@ public class InsertSpecialCharacterJniTest {
private static String tbname1 = "test"; private static String tbname1 = "test";
private static String tbname2 = "weather"; private static String tbname2 = "weather";
private static String special_character_str_1 = "$asd$$fsfsf$"; private static String special_character_str_1 = "$asd$$fsfsf$";
private static String special_character_str_2 = "\\asdfsfsf\\\\"; private static String special_character_str_2 = "\\\\asdfsfsf\\\\";
private static String special_character_str_3 = "\\\\asdfsfsf\\"; private static String special_character_str_3 = "\\\\asdfsfsf\\";
private static String special_character_str_4 = "?asd??fsf?sf?"; private static String special_character_str_4 = "?asd??fsf?sf?";
private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$";
@ -70,7 +70,7 @@ public class InsertSpecialCharacterJniTest {
String f1 = new String(rs.getBytes(2)); String f1 = new String(rs.getBytes(2));
//TODO: bug to be fixed //TODO: bug to be fixed
// Assert.assertEquals(special_character_str_2, f1); // Assert.assertEquals(special_character_str_2, f1);
Assert.assertEquals(special_character_str_2.substring(0, special_character_str_1.length() - 2), f1); Assert.assertEquals(special_character_str_2.substring(1, special_character_str_1.length() - 1), f1);
String f2 = rs.getString(3); String f2 = rs.getString(3);
Assert.assertNull(f2); Assert.assertNull(f2);
} }

6
src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/InsertSpecialCharacterRestfulTest.java
View File

@ -13,7 +13,7 @@ public class InsertSpecialCharacterRestfulTest {
private static String tbname1 = "test"; private static String tbname1 = "test";
private static String tbname2 = "weather"; private static String tbname2 = "weather";
private static String special_character_str_1 = "$asd$$fsfsf$"; private static String special_character_str_1 = "$asd$$fsfsf$";
private static String special_character_str_2 = "\\asdfsfsf\\\\"; private static String special_character_str_2 = "\\\\asdfsfsf\\\\";
private static String special_character_str_3 = "\\\\asdfsfsf\\"; private static String special_character_str_3 = "\\\\asdfsfsf\\";
private static String special_character_str_4 = "?asd??fsf?sf?"; private static String special_character_str_4 = "?asd??fsf?sf?";
private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$"; private static String special_character_str_5 = "?#sd@$f(('<(s[P)>\"){]}f?s[]{}%vaew|\"fsfs^a&d*jhg)(j))(f@~!?$";
@ -49,7 +49,7 @@ public class InsertSpecialCharacterRestfulTest {
@Test @Test
public void testCase02() throws SQLException { public void testCase02() throws SQLException {
//TODO: //TODO:
// Expected :\asdfsfsf\\ // Expected :\asdfsfsf\
// Actual :\asdfsfsf\ // Actual :\asdfsfsf\
final long now = System.currentTimeMillis(); final long now = System.currentTimeMillis();
@ -71,7 +71,7 @@ public class InsertSpecialCharacterRestfulTest {
String f1 = new String(rs.getBytes(2)); String f1 = new String(rs.getBytes(2));
//TODO: bug to be fixed //TODO: bug to be fixed
// Assert.assertEquals(special_character_str_2, f1); // Assert.assertEquals(special_character_str_2, f1);
Assert.assertEquals(special_character_str_2.substring(0, special_character_str_1.length() - 2), f1); Assert.assertEquals(special_character_str_2.substring(1, special_character_str_1.length() - 1), f1);
String f2 = rs.getString(3); String f2 = rs.getString(3);
Assert.assertNull(f2); Assert.assertNull(f2);
} }

220
src/kit/taosdemo/taosdemo.c
View File

@ -68,8 +68,17 @@ enum TEST_MODE {
INVAID_TEST INVAID_TEST
}; };
enum QUERY_MODE {
SYNC_QUERY_MODE, // 0
ASYNC_QUERY_MODE, // 1
INVALID_MODE
};
#define MAX_RECORDS_PER_REQ 32766
#define MAX_SQL_SIZE 65536 #define MAX_SQL_SIZE 65536
#define BUFFER_SIZE (65536*2) #define BUFFER_SIZE (65536*2)
#define COND_BUF_LEN BUFFER_SIZE - 30
#define MAX_USERNAME_SIZE 64 #define MAX_USERNAME_SIZE 64
#define MAX_PASSWORD_SIZE 64 #define MAX_PASSWORD_SIZE 64
#define MAX_DB_NAME_SIZE 64 #define MAX_DB_NAME_SIZE 64
@ -516,6 +525,8 @@ static int taosRandom()
static int createDatabasesAndStables(); static int createDatabasesAndStables();
static void createChildTables(); static void createChildTables();
static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet); static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet);
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
char* sqlstr, char *resultFile);
/* ************ Global variables ************ */ /* ************ Global variables ************ */
@ -765,49 +776,48 @@ static void parse_args(int argc, char *argv[], SArguments *arguments) {
} }
arguments->sqlFile = argv[++i]; arguments->sqlFile = argv[++i];
} else if (strcmp(argv[i], "-q") == 0) { } else if (strcmp(argv[i], "-q") == 0) {
if ((argc == i+1) if ((argc == i+1) ||
|| (!isStringNumber(argv[i+1]))) { (!isStringNumber(argv[i+1]))) {
printHelp(); printHelp();
errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n"); errorPrint("%s", "\n\t-q need a number following!\nQuery mode -- 0: SYNC, 1: ASYNC. Default is SYNC.\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
arguments->async_mode = atoi(argv[++i]); arguments->async_mode = atoi(argv[++i]);
} else if (strcmp(argv[i], "-T") == 0) { } else if (strcmp(argv[i], "-T") == 0) {
if ((argc == i+1) if ((argc == i+1) ||
|| (!isStringNumber(argv[i+1]))) { (!isStringNumber(argv[i+1]))) {
printHelp(); printHelp();
errorPrint("%s", "\n\t-T need a number following!\n"); errorPrint("%s", "\n\t-T need a number following!\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
arguments->num_of_threads = atoi(argv[++i]); arguments->num_of_threads = atoi(argv[++i]);
} else if (strcmp(argv[i], "-i") == 0) { } else if (strcmp(argv[i], "-i") == 0) {
if ((argc == i+1) if ((argc == i+1) ||
|| (!isStringNumber(argv[i+1]))) { (!isStringNumber(argv[i+1]))) {
printHelp(); printHelp();
errorPrint("%s", "\n\t-i need a number following!\n"); errorPrint("%s", "\n\t-i need a number following!\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
arguments->insert_interval = atoi(argv[++i]); arguments->insert_interval = atoi(argv[++i]);
} else if (strcmp(argv[i], "-qt") == 0) { } else if (strcmp(argv[i], "-qt") == 0) {
if ((argc == i+1) if ((argc == i+1) ||
|| (!isStringNumber(argv[i+1])) (!isStringNumber(argv[i+1]))) {
|| (atoi(argv[i+1]) <= 0)) {
printHelp(); printHelp();
errorPrint("%s", "\n\t-qt need a valid (>0) number following!\n"); errorPrint("%s", "\n\t-qt need a number following!\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
arguments->query_times = atoi(argv[++i]); arguments->query_times = atoi(argv[++i]);
} else if (strcmp(argv[i], "-B") == 0) { } else if (strcmp(argv[i], "-B") == 0) {
if ((argc == i+1) if ((argc == i+1) ||
|| (!isStringNumber(argv[i+1]))) { (!isStringNumber(argv[i+1]))) {
printHelp(); printHelp();
errorPrint("%s", "\n\t-B need a number following!\n"); errorPrint("%s", "\n\t-B need a number following!\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
} }
arguments->interlace_rows = atoi(argv[++i]); arguments->interlace_rows = atoi(argv[++i]);
} else if (strcmp(argv[i], "-r") == 0) { } else if (strcmp(argv[i], "-r") == 0) {
if ((argc == i+1) if ((argc == i+1) ||
|| (!isStringNumber(argv[i+1]))) { (!isStringNumber(argv[i+1]))) {
printHelp(); printHelp();
errorPrint("%s", "\n\t-r need a number following!\n"); errorPrint("%s", "\n\t-r need a number following!\n");
exit(EXIT_FAILURE); exit(EXIT_FAILURE);
@ -1067,7 +1077,7 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
if (code != 0) { if (code != 0) {
if (!quiet) { if (!quiet) {
debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command); debugPrint("%s() LN%d - command: %s\n", __func__, __LINE__, command);
errorPrint("Failed to execute %s, reason: %s\n", command, taos_errstr(res)); errorPrint("Failed to run %s, reason: %s\n", command, taos_errstr(res));
} }
taos_free_result(res); taos_free_result(res);
//taos_close(taos); //taos_close(taos);
@ -1084,27 +1094,33 @@ static int queryDbExec(TAOS *taos, char *command, QUERY_TYPE type, bool quiet) {
return 0; return 0;
} }
static void getResult(TAOS_RES *res, char* resultFileName) { static void appendResultBufToFile(char *resultBuf, char *resultFile)
{
FILE *fp = NULL;
if (resultFile[0] != 0) {
fp = fopen(resultFile, "at");
if (fp == NULL) {
errorPrint(
"%s() LN%d, failed to open result file: %s, result will not save to file\n",
__func__, __LINE__, resultFile);
return;
}
}
fprintf(fp, "%s", resultBuf);
tmfclose(fp);
}
static void appendResultToFile(TAOS_RES *res, char* resultFile) {
TAOS_ROW row = NULL; TAOS_ROW row = NULL;
int num_rows = 0; int num_rows = 0;
int num_fields = taos_field_count(res); int num_fields = taos_field_count(res);
TAOS_FIELD *fields = taos_fetch_fields(res); TAOS_FIELD *fields = taos_fetch_fields(res);
FILE *fp = NULL;
if (resultFileName[0] != 0) {
fp = fopen(resultFileName, "at");
if (fp == NULL) {
errorPrint("%s() LN%d, failed to open result file: %s, result will not save to file\n",
__func__, __LINE__, resultFileName);
}
}
char* databuf = (char*) calloc(1, 100*1024*1024); char* databuf = (char*) calloc(1, 100*1024*1024);
if (databuf == NULL) { if (databuf == NULL) {
errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n", errorPrint("%s() LN%d, failed to malloc, warning: save result to file slowly!\n",
__func__, __LINE__); __func__, __LINE__);
if (fp)
fclose(fp);
return ; return ;
} }
@ -1114,7 +1130,7 @@ static void getResult(TAOS_RES *res, char* resultFileName) {
// fetch the records row by row // fetch the records row by row
while((row = taos_fetch_row(res))) { while((row = taos_fetch_row(res))) {
if (totalLen >= 100*1024*1024 - 32000) { if (totalLen >= 100*1024*1024 - 32000) {
if (fp) fprintf(fp, "%s", databuf); appendResultBufToFile(databuf, resultFile);
totalLen = 0; totalLen = 0;
memset(databuf, 0, 100*1024*1024); memset(databuf, 0, 100*1024*1024);
} }
@ -1126,22 +1142,39 @@ static void getResult(TAOS_RES *res, char* resultFileName) {
totalLen += len; totalLen += len;
} }
if (fp) fprintf(fp, "%s", databuf); appendResultBufToFile(databuf, resultFile);
tmfclose(fp);
free(databuf); free(databuf);
} }
static void selectAndGetResult(TAOS *taos, char *command, char* resultFileName) { static void selectAndGetResult(threadInfo *pThreadInfo, char *command, char* resultFile)
TAOS_RES *res = taos_query(taos, command); {
if (res == NULL || taos_errno(res) != 0) { if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n", TAOS_RES *res = taos_query(pThreadInfo->taos, command);
__func__, __LINE__, command, taos_errstr(res)); if (res == NULL || taos_errno(res) != 0) {
taos_free_result(res); errorPrint("%s() LN%d, failed to execute sql:%s, reason:%s\n",
return; __func__, __LINE__, command, taos_errstr(res));
} taos_free_result(res);
return;
}
getResult(res, resultFileName); if ((resultFile) && (strlen(resultFile))) {
taos_free_result(res); appendResultToFile(res, resultFile);
}
taos_free_result(res);
} else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
int retCode = postProceSql(
g_queryInfo.host, &(g_queryInfo.serv_addr), g_queryInfo.port,
command,
resultFile);
if (0 != retCode) {
printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
}
} else {
errorPrint("%s() LN%d, unknown query mode: %s\n",
__func__, __LINE__, g_queryInfo.queryMode);
}
} }
static int32_t rand_bool(){ static int32_t rand_bool(){
@ -1934,13 +1967,13 @@ static void printfQuerySystemInfo(TAOS * taos) {
// show variables // show variables
res = taos_query(taos, "show variables;"); res = taos_query(taos, "show variables;");
//getResult(res, filename); //appendResultToFile(res, filename);
xDumpResultToFile(filename, res); xDumpResultToFile(filename, res);
// show dnodes // show dnodes
res = taos_query(taos, "show dnodes;"); res = taos_query(taos, "show dnodes;");
xDumpResultToFile(filename, res); xDumpResultToFile(filename, res);
//getResult(res, filename); //appendResultToFile(res, filename);
// show databases // show databases
res = taos_query(taos, "show databases;"); res = taos_query(taos, "show databases;");
@ -1975,7 +2008,8 @@ static void printfQuerySystemInfo(TAOS * taos) {
free(dbInfos); free(dbInfos);
} }
static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port, char* sqlstr) static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port,
char* sqlstr, char *resultFile)
{ {
char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s"; char *req_fmt = "POST %s HTTP/1.1\r\nHost: %s:%d\r\nAccept: */*\r\nAuthorization: Basic %s\r\nContent-Length: %d\r\nContent-Type: application/x-www-form-urlencoded\r\n\r\n%s";
@ -2111,6 +2145,10 @@ static int postProceSql(char *host, struct sockaddr_in *pServAddr, uint16_t port
response_buf[RESP_BUF_LEN - 1] = '\0'; response_buf[RESP_BUF_LEN - 1] = '\0';
printf("Response:\n%s\n", response_buf); printf("Response:\n%s\n", response_buf);
if (resultFile) {
appendResultBufToFile(response_buf, resultFile);
}
free(request_buf); free(request_buf);
#ifdef WINDOWS #ifdef WINDOWS
closesocket(sockfd); closesocket(sockfd);
@ -3406,10 +3444,12 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n", errorPrint("%s() LN%d, failed to read json, num_of_records_per_req input mistake\n",
__func__, __LINE__); __func__, __LINE__);
goto PARSE_OVER; goto PARSE_OVER;
} else if (numRecPerReq->valueint > MAX_RECORDS_PER_REQ) {
numRecPerReq->valueint = MAX_RECORDS_PER_REQ;
} }
g_args.num_of_RPR = numRecPerReq->valueint; g_args.num_of_RPR = numRecPerReq->valueint;
} else if (!numRecPerReq) { } else if (!numRecPerReq) {
g_args.num_of_RPR = UINT64_MAX; g_args.num_of_RPR = MAX_RECORDS_PER_REQ;
} else { } else {
errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n", errorPrint("%s() LN%d, failed to read json, num_of_records_per_req not found\n",
__func__, __LINE__); __func__, __LINE__);
@ -4035,9 +4075,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times"); cJSON* gQueryTimes = cJSON_GetObjectItem(root, "query_times");
if (gQueryTimes && gQueryTimes->type == cJSON_Number) { if (gQueryTimes && gQueryTimes->type == cJSON_Number) {
if (gQueryTimes->valueint <= 0) { if (gQueryTimes->valueint < 0) {
errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
__func__, __LINE__, gQueryTimes->valueint); __func__, __LINE__);
goto PARSE_OVER; goto PARSE_OVER;
} }
g_args.query_times = gQueryTimes->valueint; g_args.query_times = gQueryTimes->valueint;
@ -4086,9 +4126,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery, cJSON* specifiedQueryTimes = cJSON_GetObjectItem(specifiedQuery,
"query_times"); "query_times");
if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) { if (specifiedQueryTimes && specifiedQueryTimes->type == cJSON_Number) {
if (specifiedQueryTimes->valueint <= 0) { if (specifiedQueryTimes->valueint < 0) {
errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
__func__, __LINE__, specifiedQueryTimes->valueint); __func__, __LINE__);
goto PARSE_OVER; goto PARSE_OVER;
} }
@ -4230,9 +4270,9 @@ static bool getMetaFromQueryJsonFile(cJSON* root) {
cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times"); cJSON* superQueryTimes = cJSON_GetObjectItem(superQuery, "query_times");
if (superQueryTimes && superQueryTimes->type == cJSON_Number) { if (superQueryTimes && superQueryTimes->type == cJSON_Number) {
if (superQueryTimes->valueint <= 0) { if (superQueryTimes->valueint < 0) {
errorPrint("%s() LN%d, failed to read json, query_times: %"PRId64", need be a valid (>0) number\n", errorPrint("%s() LN%d, failed to read json, query_times input mistake\n",
__func__, __LINE__, superQueryTimes->valueint); __func__, __LINE__);
goto PARSE_OVER; goto PARSE_OVER;
} }
g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint; g_queryInfo.superQueryInfo.queryTimes = superQueryTimes->valueint;
@ -4682,7 +4722,8 @@ static int64_t execInsert(threadInfo *pThreadInfo, char *buffer, uint64_t k)
if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) { if (0 == strncasecmp(superTblInfo->insertMode, "taosc", strlen("taosc"))) {
affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false); affectedRows = queryDbExec(pThreadInfo->taos, buffer, INSERT_TYPE, false);
} else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) { } else if (0 == strncasecmp(superTblInfo->insertMode, "rest", strlen("rest"))) {
if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port, buffer)) { if (0 != postProceSql(g_Dbs.host, &g_Dbs.serv_addr, g_Dbs.port,
buffer, NULL /* not set result file */)) {
affectedRows = -1; affectedRows = -1;
printf("========restful return fail, threadID[%d]\n", printf("========restful return fail, threadID[%d]\n",
pThreadInfo->threadID); pThreadInfo->threadID);
@ -5195,13 +5236,6 @@ static void* syncWriteInterlace(threadInfo *pThreadInfo) {
startTs = taosGetTimestampMs(); startTs = taosGetTimestampMs();
if (recOfBatch == 0) {
errorPrint("[%d] %s() LN%d try inserting records of batch is %"PRIu64"\n",
pThreadInfo->threadID, __func__, __LINE__,
recOfBatch);
errorPrint("%s\n", "\tPlease check if the batch or the buffer length is proper value!\n");
goto free_of_interlace;
}
int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch); int64_t affectedRows = execInsert(pThreadInfo, buffer, recOfBatch);
endTs = taosGetTimestampMs(); endTs = taosGetTimestampMs();
@ -5930,7 +5964,7 @@ static void *readMetric(void *sarg) {
fprintf(fp, "Querying On %d records:\n", totalData); fprintf(fp, "Querying On %d records:\n", totalData);
for (int j = 0; j < n; j++) { for (int j = 0; j < n; j++) {
char condition[BUFFER_SIZE - 30] = "\0"; char condition[COND_BUF_LEN] = "\0";
char tempS[64] = "\0"; char tempS[64] = "\0";
int m = 10 < num_of_tables ? 10 : num_of_tables; int m = 10 < num_of_tables ? 10 : num_of_tables;
@ -5941,7 +5975,7 @@ static void *readMetric(void *sarg) {
} else { } else {
sprintf(tempS, " or t1 = %d ", i); sprintf(tempS, " or t1 = %d ", i);
} }
strcat(condition, tempS); strncat(condition, tempS, COND_BUF_LEN - 1);
sprintf(command, "select %s from meters where %s", aggreFunc[j], condition); sprintf(command, "select %s from meters where %s", aggreFunc[j], condition);
@ -6119,43 +6153,24 @@ static void *specifiedTableQuery(void *sarg) {
taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms taosMsleep(g_queryInfo.specifiedQueryInfo.queryInterval - (et - st)); // ms
} }
st = taosGetTimestampMs(); char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
if (0 == strncasecmp(g_queryInfo.queryMode, "taosc", strlen("taosc"))) {
int64_t t1 = taosGetTimestampMs();
char tmpFile[MAX_FILE_NAME_LEN*2] = {0};
if (g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq][0] != 0) {
sprintf(tmpFile, "%s-%d", sprintf(tmpFile, "%s-%d",
g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq], g_queryInfo.specifiedQueryInfo.result[pThreadInfo->querySeq],
pThreadInfo->threadID); pThreadInfo->threadID);
}
selectAndGetResult(pThreadInfo->taos,
g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile);
int64_t t2 = taosGetTimestampMs();
printf("=[taosc] thread[%"PRId64"] complete one sql, Spent %10.3f s\n",
taosGetSelfPthreadId(), (t2 - t1)/1000.0);
} else if (0 == strncasecmp(g_queryInfo.queryMode, "rest", strlen("rest"))) {
int64_t t1 = taosGetTimestampMs();
int retCode = postProceSql(g_queryInfo.host, &(g_queryInfo.serv_addr),
g_queryInfo.port,
g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq]);
if (0 != retCode) {
printf("====restful return fail, threadID[%d]\n", pThreadInfo->threadID);
return NULL;
}
int64_t t2 = taosGetTimestampMs();
printf("=[restful] thread[%"PRId64"] complete one sql, Spent %10.3f s\n",
taosGetSelfPthreadId(), (t2 - t1)/1000.0);
} else {
errorPrint("%s() LN%d, unknown query mode: %s\n",
__func__, __LINE__, g_queryInfo.queryMode);
return NULL;
} }
totalQueried ++;
g_queryInfo.specifiedQueryInfo.totalQueried ++; st = taosGetTimestampMs();
selectAndGetResult(pThreadInfo,
g_queryInfo.specifiedQueryInfo.sql[pThreadInfo->querySeq], tmpFile);
et = taosGetTimestampMs(); et = taosGetTimestampMs();
printf("=thread[%"PRId64"] use %s complete one sql, Spent %10.3f s\n",
taosGetSelfPthreadId(), g_queryInfo.queryMode, (et - st)/1000.0);
totalQueried ++;
g_queryInfo.specifiedQueryInfo.totalQueried ++;
uint64_t currentPrintTime = taosGetTimestampMs(); uint64_t currentPrintTime = taosGetTimestampMs();
uint64_t endTs = taosGetTimestampMs(); uint64_t endTs = taosGetTimestampMs();
@ -6188,14 +6203,14 @@ static void replaceChildTblName(char* inSql, char* outSql, int tblIndex) {
tstrncpy(outSql, inSql, pos - inSql + 1); tstrncpy(outSql, inSql, pos - inSql + 1);
//printf("1: %s\n", outSql); //printf("1: %s\n", outSql);
strcat(outSql, subTblName); strncat(outSql, subTblName, MAX_QUERY_SQL_LENGTH - 1);
//printf("2: %s\n", outSql); //printf("2: %s\n", outSql);
strcat(outSql, pos+strlen(sourceString)); strncat(outSql, pos+strlen(sourceString), MAX_QUERY_SQL_LENGTH - 1);
//printf("3: %s\n", outSql); //printf("3: %s\n", outSql);
} }
static void *superTableQuery(void *sarg) { static void *superTableQuery(void *sarg) {
char sqlstr[1024]; char sqlstr[MAX_QUERY_SQL_LENGTH];
threadInfo *pThreadInfo = (threadInfo *)sarg; threadInfo *pThreadInfo = (threadInfo *)sarg;
if (pThreadInfo->taos == NULL) { if (pThreadInfo->taos == NULL) {
@ -6240,7 +6255,7 @@ static void *superTableQuery(void *sarg) {
g_queryInfo.superQueryInfo.result[j], g_queryInfo.superQueryInfo.result[j],
pThreadInfo->threadID); pThreadInfo->threadID);
} }
selectAndGetResult(pThreadInfo->taos, sqlstr, tmpFile); selectAndGetResult(pThreadInfo, sqlstr, tmpFile);
totalQueried++; totalQueried++;
g_queryInfo.superQueryInfo.totalQueried ++; g_queryInfo.superQueryInfo.totalQueried ++;
@ -6441,7 +6456,8 @@ static void subscribe_callback(TAOS_SUB* tsub, TAOS_RES *res, void* param, int c
return; return;
} }
getResult(res, (char*)param); if (param)
appendResultToFile(res, (char*)param);
// tao_unscribe() will free result. // tao_unscribe() will free result.
} }
@ -6470,7 +6486,7 @@ static TAOS_SUB* subscribeImpl(
static void *superSubscribe(void *sarg) { static void *superSubscribe(void *sarg) {
threadInfo *pThreadInfo = (threadInfo *)sarg; threadInfo *pThreadInfo = (threadInfo *)sarg;
char subSqlstr[1024]; char subSqlstr[MAX_QUERY_SQL_LENGTH];
TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0}; TAOS_SUB* tsub[MAX_QUERY_SQL_COUNT] = {0};
if (g_queryInfo.superQueryInfo.sqlCount == 0) if (g_queryInfo.superQueryInfo.sqlCount == 0)
@ -6545,8 +6561,8 @@ static void *superSubscribe(void *sarg) {
sprintf(tmpFile, "%s-%d", sprintf(tmpFile, "%s-%d",
g_queryInfo.superQueryInfo.result[i], g_queryInfo.superQueryInfo.result[i],
pThreadInfo->threadID); pThreadInfo->threadID);
appendResultToFile(res, tmpFile);
} }
getResult(res, tmpFile);
} }
} }
} }
@ -6633,8 +6649,8 @@ static void *specifiedSubscribe(void *sarg) {
if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) { if (g_queryInfo.specifiedQueryInfo.result[i][0] != 0) {
sprintf(tmpFile, "%s-%d", sprintf(tmpFile, "%s-%d",
g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID); g_queryInfo.specifiedQueryInfo.result[i], pThreadInfo->threadID);
appendResultToFile(res, tmpFile);
} }
getResult(res, tmpFile);
} }
} }
} }

115
src/kit/taosdump/taosdump.c
View File

@ -110,14 +110,14 @@ typedef struct {
} SColDes; } SColDes;
typedef struct { typedef struct {
char name[TSDB_COL_NAME_LEN + 1]; char name[TSDB_TABLE_NAME_LEN];
SColDes cols[]; SColDes cols[];
} STableDef; } STableDef;
extern char version[]; extern char version[];
typedef struct { typedef struct {
char name[TSDB_DB_NAME_LEN + 1]; char name[TSDB_DB_NAME_LEN];
char create_time[32]; char create_time[32];
int32_t ntables; int32_t ntables;
int32_t vgroups; int32_t vgroups;
@ -142,8 +142,8 @@ typedef struct {
} SDbInfo; } SDbInfo;
typedef struct { typedef struct {
char name[TSDB_TABLE_NAME_LEN + 1]; char name[TSDB_TABLE_NAME_LEN];
char metric[TSDB_TABLE_NAME_LEN + 1]; char metric[TSDB_TABLE_NAME_LEN];
} STableRecord; } STableRecord;
typedef struct { typedef struct {
@ -155,7 +155,7 @@ typedef struct {
pthread_t threadID; pthread_t threadID;
int32_t threadIndex; int32_t threadIndex;
int32_t totalThreads; int32_t totalThreads;
char dbName[TSDB_TABLE_NAME_LEN + 1]; char dbName[TSDB_DB_NAME_LEN];
void *taosCon; void *taosCon;
int64_t rowsOfDumpOut; int64_t rowsOfDumpOut;
int64_t tablesOfDumpOut; int64_t tablesOfDumpOut;
@ -214,13 +214,13 @@ static struct argp_option options[] = {
{"encode", 'e', "ENCODE", 0, "Input file encoding.", 1}, {"encode", 'e', "ENCODE", 0, "Input file encoding.", 1},
// dump unit options // dump unit options
{"all-databases", 'A', 0, 0, "Dump all databases.", 2}, {"all-databases", 'A', 0, 0, "Dump all databases.", 2},
{"databases", 'B', 0, 0, "Dump assigned databases", 2}, {"databases", 'D', 0, 0, "Dump assigned databases", 2},
// dump format options // dump format options
{"schemaonly", 's', 0, 0, "Only dump schema.", 3}, {"schemaonly", 's', 0, 0, "Only dump schema.", 3},
{"with-property", 'M', 0, 0, "Dump schema with properties.", 3}, {"without-property", 'N', 0, 0, "Dump schema without properties.", 3},
{"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, {"start-time", 'S', "START_TIME", 0, "Start time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3},
{"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3}, {"end-time", 'E', "END_TIME", 0, "End time to dump. Either Epoch or ISO8601/RFC3339 format is acceptable. Epoch precision millisecond. ISO8601 format example: 2017-10-01T18:00:00.000+0800 or 2017-10-0100:00:00.000+0800 or '2017-10-01 00:00:00.000+0800'", 3},
{"data-batch", 'N', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3}, {"data-batch", 'B', "DATA_BATCH", 0, "Number of data point per insert statement. Default is 1.", 3},
{"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3}, {"max-sql-len", 'L', "SQL_LEN", 0, "Max length of one sql. Default is 65480.", 3},
{"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3}, {"table-batch", 't', "TABLE_BATCH", 0, "Number of table dumpout into one output file. Default is 1.", 3},
{"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3}, {"thread_num", 'T', "THREAD_NUM", 0, "Number of thread for dump in file. Default is 5.", 3},
@ -341,15 +341,15 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
case 'A': case 'A':
arguments->all_databases = true; arguments->all_databases = true;
break; break;
case 'B': case 'D':
arguments->databases = true; arguments->databases = true;
break; break;
// dump format option // dump format option
case 's': case 's':
arguments->schemaonly = true; arguments->schemaonly = true;
break; break;
case 'M': case 'N':
arguments->with_property = true; arguments->with_property = false;
break; break;
case 'S': case 'S':
// parse time here. // parse time here.
@ -358,7 +358,7 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
case 'E': case 'E':
arguments->end_time = atol(arg); arguments->end_time = atol(arg);
break; break;
case 'N': case 'B':
arguments->data_batch = atoi(arg); arguments->data_batch = atoi(arg);
if (arguments->data_batch >= INT16_MAX) { if (arguments->data_batch >= INT16_MAX) {
arguments->data_batch = INT16_MAX - 1; arguments->data_batch = INT16_MAX - 1;
@ -402,17 +402,17 @@ static resultStatistics g_resultStatistics = {0};
static FILE *g_fpOfResult = NULL; static FILE *g_fpOfResult = NULL;
static int g_numOfCores = 1; static int g_numOfCores = 1;
int taosDumpOut(struct arguments *arguments); static int taosDumpOut(struct arguments *arguments);
int taosDumpIn(struct arguments *arguments); static int taosDumpIn(struct arguments *arguments);
void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp); static void taosDumpCreateDbClause(SDbInfo *dbInfo, bool isDumpProperty, FILE *fp);
int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon); static int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *taosCon);
int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName); static int32_t taosDumpStable(char *table, FILE *fp, TAOS* taosCon, char* dbName);
void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName); static void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, char* dbName);
void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName); static void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols, FILE *fp, char* dbName);
int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName); static int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName);
int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName); static int taosDumpTableData(FILE *fp, char *tbname, struct arguments *arguments, TAOS* taosCon, char* dbName);
int taosCheckParam(struct arguments *arguments); static int taosCheckParam(struct arguments *arguments);
void taosFreeDbInfos(); static void taosFreeDbInfos();
static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName); static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName);
struct arguments g_args = { struct arguments g_args = {
@ -436,8 +436,8 @@ struct arguments g_args = {
false, false,
false, false,
// dump format option // dump format option
false, false, // schemeonly
false, true, // with_property
0, 0,
INT64_MAX, INT64_MAX,
1, 1,
@ -959,7 +959,8 @@ int taosDumpOut(struct arguments *arguments) {
goto _exit_failure; goto _exit_failure;
} }
strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX], fields[TSDB_SHOW_DB_NAME_INDEX].bytes); strncpy(dbInfos[count]->name, (char *)row[TSDB_SHOW_DB_NAME_INDEX],
fields[TSDB_SHOW_DB_NAME_INDEX].bytes);
if (arguments->with_property) { if (arguments->with_property) {
dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]); dbInfos[count]->ntables = *((int32_t *)row[TSDB_SHOW_DB_NTABLES_INDEX]);
dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]); dbInfos[count]->vgroups = *((int32_t *)row[TSDB_SHOW_DB_VGROUPS_INDEX]);
@ -967,7 +968,8 @@ int taosDumpOut(struct arguments *arguments) {
dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]); dbInfos[count]->quorum = *((int16_t *)row[TSDB_SHOW_DB_QUORUM_INDEX]);
dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]); dbInfos[count]->days = *((int16_t *)row[TSDB_SHOW_DB_DAYS_INDEX]);
strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX], fields[TSDB_SHOW_DB_KEEP_INDEX].bytes); strncpy(dbInfos[count]->keeplist, (char *)row[TSDB_SHOW_DB_KEEP_INDEX],
fields[TSDB_SHOW_DB_KEEP_INDEX].bytes);
//dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]); //dbInfos[count]->daysToKeep = *((int16_t *)row[TSDB_SHOW_DB_KEEP_INDEX]);
//dbInfos[count]->daysToKeep1; //dbInfos[count]->daysToKeep1;
//dbInfos[count]->daysToKeep2; //dbInfos[count]->daysToKeep2;
@ -980,7 +982,8 @@ int taosDumpOut(struct arguments *arguments) {
dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX])); dbInfos[count]->comp = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_COMP_INDEX]));
dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX])); dbInfos[count]->cachelast = (int8_t)(*((int8_t *)row[TSDB_SHOW_DB_CACHELAST_INDEX]));
strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX], fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes); strncpy(dbInfos[count]->precision, (char *)row[TSDB_SHOW_DB_PRECISION_INDEX],
fields[TSDB_SHOW_DB_PRECISION_INDEX].bytes);
//dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]); //dbInfos[count]->precision = *((int8_t *)row[TSDB_SHOW_DB_PRECISION_INDEX]);
dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]); dbInfos[count]->update = *((int8_t *)row[TSDB_SHOW_DB_UPDATE_INDEX]);
} }
@ -1095,7 +1098,9 @@ _exit_failure:
return -1; return -1;
} }
int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCon, bool isSuperTable) { int taosGetTableDes(
char* dbName, char *table,
STableDef *tableDes, TAOS* taosCon, bool isSuperTable) {
TAOS_ROW row = NULL; TAOS_ROW row = NULL;
TAOS_RES* res = NULL; TAOS_RES* res = NULL;
int count = 0; int count = 0;
@ -1113,7 +1118,7 @@ int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCo
TAOS_FIELD *fields = taos_fetch_fields(res); TAOS_FIELD *fields = taos_fetch_fields(res);
tstrncpy(tableDes->name, table, TSDB_COL_NAME_LEN); tstrncpy(tableDes->name, table, TSDB_TABLE_NAME_LEN);
while ((row = taos_fetch_row(res)) != NULL) { while ((row = taos_fetch_row(res)) != NULL) {
strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX],
@ -1232,7 +1237,9 @@ int taosGetTableDes(char* dbName, char *table, STableDef *tableDes, TAOS* taosCo
return count; return count;
} }
int32_t taosDumpTable(char *table, char *metric, struct arguments *arguments, FILE *fp, TAOS* taosCon, char* dbName) { int32_t taosDumpTable(
char *table, char *metric, struct arguments *arguments,
FILE *fp, TAOS* taosCon, char* dbName) {
int count = 0; int count = 0;
STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS); STableDef *tableDes = (STableDef *)calloc(1, sizeof(STableDef) + sizeof(SColDes) * TSDB_MAX_COLUMNS);
@ -1346,14 +1353,17 @@ void* taosDumpOutWorkThreadFp(void *arg)
ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord)); ssize_t readLen = read(fd, &tableRecord, sizeof(STableRecord));
if (readLen <= 0) break; if (readLen <= 0) break;
int ret = taosDumpTable(tableRecord.name, tableRecord.metric, &g_args, fp, pThread->taosCon, pThread->dbName); int ret = taosDumpTable(
tableRecord.name, tableRecord.metric, &g_args,
fp, pThread->taosCon, pThread->dbName);
if (ret >= 0) { if (ret >= 0) {
// TODO: sum table count and table rows by self // TODO: sum table count and table rows by self
pThread->tablesOfDumpOut++; pThread->tablesOfDumpOut++;
pThread->rowsOfDumpOut += ret; pThread->rowsOfDumpOut += ret;
if (pThread->rowsOfDumpOut >= lastRowsPrint) { if (pThread->rowsOfDumpOut >= lastRowsPrint) {
printf(" %"PRId64 " rows already be dumpout from database %s\n", pThread->rowsOfDumpOut, pThread->dbName); printf(" %"PRId64 " rows already be dumpout from database %s\n",
pThread->rowsOfDumpOut, pThread->dbName);
lastRowsPrint += 5000000; lastRowsPrint += 5000000;
} }
@ -1364,9 +1374,12 @@ void* taosDumpOutWorkThreadFp(void *arg)
memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128); memset(tmpBuf, 0, TSDB_FILENAME_LEN + 128);
if (g_args.outpath[0] != 0) { if (g_args.outpath[0] != 0) {
sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql", g_args.outpath, pThread->dbName, pThread->threadIndex, fileNameIndex); sprintf(tmpBuf, "%s/%s.tables.%d-%d.sql",
g_args.outpath, pThread->dbName,
pThread->threadIndex, fileNameIndex);
} else { } else {
sprintf(tmpBuf, "%s.tables.%d-%d.sql", pThread->dbName, pThread->threadIndex, fileNameIndex); sprintf(tmpBuf, "%s.tables.%d-%d.sql",
pThread->dbName, pThread->threadIndex, fileNameIndex);
} }
fileNameIndex++; fileNameIndex++;
@ -1391,14 +1404,15 @@ void* taosDumpOutWorkThreadFp(void *arg)
static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName) static void taosStartDumpOutWorkThreads(void* taosCon, struct arguments* args, int32_t numOfThread, char *dbName)
{ {
pthread_attr_t thattr; pthread_attr_t thattr;
SThreadParaObj *threadObj = (SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj)); SThreadParaObj *threadObj =
(SThreadParaObj *)calloc(numOfThread, sizeof(SThreadParaObj));
for (int t = 0; t < numOfThread; ++t) { for (int t = 0; t < numOfThread; ++t) {
SThreadParaObj *pThread = threadObj + t; SThreadParaObj *pThread = threadObj + t;
pThread->rowsOfDumpOut = 0; pThread->rowsOfDumpOut = 0;
pThread->tablesOfDumpOut = 0; pThread->tablesOfDumpOut = 0;
pThread->threadIndex = t; pThread->threadIndex = t;
pThread->totalThreads = numOfThread; pThread->totalThreads = numOfThread;
tstrncpy(pThread->dbName, dbName, TSDB_TABLE_NAME_LEN); tstrncpy(pThread->dbName, dbName, TSDB_DB_NAME_LEN);
pThread->taosCon = taosCon; pThread->taosCon = taosCon;
pthread_attr_init(&thattr); pthread_attr_init(&thattr);
@ -1487,7 +1501,8 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp)
while ((row = taos_fetch_row(res)) != NULL) { while ((row = taos_fetch_row(res)) != NULL) {
memset(&tableRecord, 0, sizeof(STableRecord)); memset(&tableRecord, 0, sizeof(STableRecord));
strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX],
fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
taosWrite(fd, &tableRecord, sizeof(STableRecord)); taosWrite(fd, &tableRecord, sizeof(STableRecord));
} }
@ -1557,8 +1572,10 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
int32_t numOfTable = 0; int32_t numOfTable = 0;
while ((row = taos_fetch_row(res)) != NULL) { while ((row = taos_fetch_row(res)) != NULL) {
memset(&tableRecord, 0, sizeof(STableRecord)); memset(&tableRecord, 0, sizeof(STableRecord));
tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes); tstrncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX],
tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes); fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
tstrncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX],
fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
taosWrite(fd, &tableRecord, sizeof(STableRecord)); taosWrite(fd, &tableRecord, sizeof(STableRecord));
@ -1643,15 +1660,18 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, cha
char* pstr = sqlstr; char* pstr = sqlstr;
pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s", dbName, tableDes->name); pstr += sprintf(sqlstr, "CREATE TABLE IF NOT EXISTS %s.%s",
dbName, tableDes->name);
for (; counter < numOfCols; counter++) { for (; counter < numOfCols; counter++) {
if (tableDes->cols[counter].note[0] != '\0') break; if (tableDes->cols[counter].note[0] != '\0') break;
if (counter == 0) { if (counter == 0) {
pstr += sprintf(pstr, " (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); pstr += sprintf(pstr, " (%s %s",
tableDes->cols[counter].field, tableDes->cols[counter].type);
} else { } else {
pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); pstr += sprintf(pstr, ", %s %s",
tableDes->cols[counter].field, tableDes->cols[counter].type);
} }
if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
@ -1664,9 +1684,11 @@ void taosDumpCreateTableClause(STableDef *tableDes, int numOfCols, FILE *fp, cha
for (; counter < numOfCols; counter++) { for (; counter < numOfCols; counter++) {
if (counter == count_temp) { if (counter == count_temp) {
pstr += sprintf(pstr, ") TAGS (%s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); pstr += sprintf(pstr, ") TAGS (%s %s",
tableDes->cols[counter].field, tableDes->cols[counter].type);
} else { } else {
pstr += sprintf(pstr, ", %s %s", tableDes->cols[counter].field, tableDes->cols[counter].type); pstr += sprintf(pstr, ", %s %s",
tableDes->cols[counter].field, tableDes->cols[counter].type);
} }
if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 || if (strcasecmp(tableDes->cols[counter].type, "binary") == 0 ||
@ -1693,7 +1715,8 @@ void taosDumpCreateMTableClause(STableDef *tableDes, char *metric, int numOfCols
char *pstr = NULL; char *pstr = NULL;
pstr = tmpBuf; pstr = tmpBuf;
pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (", dbName, tableDes->name, dbName, metric); pstr += sprintf(tmpBuf, "CREATE TABLE IF NOT EXISTS %s.%s USING %s.%s TAGS (",
dbName, tableDes->name, dbName, metric);
for (; counter < numOfCols; counter++) { for (; counter < numOfCols; counter++) {
if (tableDes->cols[counter].note[0] != '\0') break; if (tableDes->cols[counter].note[0] != '\0') break;

2
src/mnode/inc/mnodeDef.h
View File

@ -249,7 +249,7 @@ typedef struct SAcctObj {
} SAcctObj; } SAcctObj;
typedef struct { typedef struct {
char db[TSDB_DB_NAME_LEN]; char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
int8_t type; int8_t type;
int16_t numOfColumns; int16_t numOfColumns;
int32_t index; int32_t index;

2
src/mnode/src/mnodeShow.c
View File

@ -129,7 +129,7 @@ static int32_t mnodeProcessShowMsg(SMnodeMsg *pMsg) {
SShowObj *pShow = calloc(1, showObjSize); SShowObj *pShow = calloc(1, showObjSize);
pShow->type = pShowMsg->type; pShow->type = pShowMsg->type;
pShow->payloadLen = htons(pShowMsg->payloadLen); pShow->payloadLen = htons(pShowMsg->payloadLen);
tstrncpy(pShow->db, pShowMsg->db, TSDB_DB_NAME_LEN); tstrncpy(pShow->db, pShowMsg->db, TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN);
memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen); memcpy(pShow->payload, pShowMsg->payload, pShow->payloadLen);
pShow = mnodePutShowObj(pShow); pShow = mnodePutShowObj(pShow);

2
src/os/tests/test.cpp
View File

@ -4,7 +4,7 @@
#include <iostream> #include <iostream>
#include "taos.h" #include "taos.h"
#include "tstoken.h" #include "ttoken.h"
#include "tutil.h" #include "tutil.h"
int main(int argc, char** argv) { int main(int argc, char** argv) {

2
src/query/inc/qSqlparser.h
View File

@ -22,8 +22,8 @@ extern "C" {
#include "taos.h" #include "taos.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tstoken.h"
#include "tstrbuild.h" #include "tstrbuild.h"
#include "ttoken.h"
#include "tvariant.h" #include "tvariant.h"
#define ParseTOKENTYPE SStrToken #define ParseTOKENTYPE SStrToken

6
src/query/src/qSqlParser.c
View File

@ -13,13 +13,13 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>. * along with this program. If not, see <http://www.gnu.org/licenses/>.
*/ */
#include "os.h"
#include "qSqlparser.h" #include "qSqlparser.h"
#include "os.h"
#include "taosdef.h" #include "taosdef.h"
#include "taosmsg.h" #include "taosmsg.h"
#include "tcmdtype.h" #include "tcmdtype.h"
#include "tstoken.h"
#include "tstrbuild.h" #include "tstrbuild.h"
#include "ttoken.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tutil.h" #include "tutil.h"
@ -38,7 +38,7 @@ SSqlInfo qSqlParse(const char *pStr) {
goto abort_parse; goto abort_parse;
} }
t0.n = tSQLGetToken((char *)&pStr[i], &t0.type); t0.n = tGetToken((char *)&pStr[i], &t0.type);
t0.z = (char *)(pStr + i); t0.z = (char *)(pStr + i);
i += t0.n; i += t0.n;

6
src/query/src/sql.c
View File

@ -25,14 +25,14 @@
#include <stdio.h> #include <stdio.h>
/************ Begin %include sections from the grammar ************************/ /************ Begin %include sections from the grammar ************************/
#include <assert.h>
#include <stdbool.h>
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <assert.h>
#include <stdbool.h>
#include "qSqlparser.h" #include "qSqlparser.h"
#include "tcmdtype.h" #include "tcmdtype.h"
#include "tstoken.h" #include "ttoken.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tutil.h" #include "tutil.h"
#include "tvariant.h" #include "tvariant.h"

4
src/query/tests/tsBufTest.cpp
View File

@ -3,10 +3,10 @@
#include <cassert> #include <cassert>
#include <iostream> #include <iostream>
#include "qTsbuf.h"
#include "taos.h" #include "taos.h"
#include "tsdb.h" #include "tsdb.h"
#include "qTsbuf.h" #include "ttoken.h"
#include "tstoken.h"
#include "tutil.h" #include "tutil.h"
namespace { namespace {

14
src/query/tests/unitTest.cpp
View File

@ -21,7 +21,7 @@ int32_t testValidateName(char* name) {
token.n = strlen(name); token.n = strlen(name);
token.type = 0; token.type = 0;
tSQLGetToken(name, &token.type); tGetToken(name, &token.type);
return tscValidateName(&token); return tscValidateName(&token);
} }
} }
@ -691,32 +691,32 @@ TEST(testCase, tGetToken_Test) {
char* s = ".123 "; char* s = ".123 ";
uint32_t type = 0; uint32_t type = 0;
int32_t len = tSQLGetToken(s, &type); int32_t len = tGetToken(s, &type);
EXPECT_EQ(type, TK_FLOAT); EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s) - 1); EXPECT_EQ(len, strlen(s) - 1);
char s1[] = "1.123e10 "; char s1[] = "1.123e10 ";
len = tSQLGetToken(s1, &type); len = tGetToken(s1, &type);
EXPECT_EQ(type, TK_FLOAT); EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s1) - 1); EXPECT_EQ(len, strlen(s1) - 1);
char s4[] = "0xff "; char s4[] = "0xff ";
len = tSQLGetToken(s4, &type); len = tGetToken(s4, &type);
EXPECT_EQ(type, TK_HEX); EXPECT_EQ(type, TK_HEX);
EXPECT_EQ(len, strlen(s4) - 1); EXPECT_EQ(len, strlen(s4) - 1);
// invalid data type // invalid data type
char s2[] = "e10 "; char s2[] = "e10 ";
len = tSQLGetToken(s2, &type); len = tGetToken(s2, &type);
EXPECT_FALSE(type == TK_FLOAT); EXPECT_FALSE(type == TK_FLOAT);
char s3[] = "1.1.1.1"; char s3[] = "1.1.1.1";
len = tSQLGetToken(s3, &type); len = tGetToken(s3, &type);
EXPECT_EQ(type, TK_IPTOKEN); EXPECT_EQ(type, TK_IPTOKEN);
EXPECT_EQ(len, strlen(s3)); EXPECT_EQ(len, strlen(s3));
char s5[] = "0x "; char s5[] = "0x ";
len = tSQLGetToken(s5, &type); len = tGetToken(s5, &type);
EXPECT_FALSE(type == TK_HEX); EXPECT_FALSE(type == TK_HEX);
} }

29
src/util/inc/tstoken.h → src/util/inc/ttoken.h
View File

@ -37,13 +37,25 @@ typedef struct SStrToken {
char *z; char *z;
} SStrToken; } SStrToken;
extern const char escapeChar[];
/**
* check if it is a number or not
* @param pToken
* @return
*/
#define isNumber(tk) \
((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN)
#define GET_ESCAPE_CHAR(c) (escapeChar[(uint8_t)(c)])
/** /**
* tokenizer for sql string * tokenizer for sql string
* @param z * @param z
* @param tokenType * @param tokenType
* @return * @return
*/ */
uint32_t tSQLGetToken(char *z, uint32_t *tokenType); uint32_t tGetToken(char *z, uint32_t *tokenType);
/** /**
* enhanced tokenizer for sql string. * enhanced tokenizer for sql string.
@ -61,21 +73,12 @@ SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr);
* @param len * @param len
* @return * @return
*/ */
bool isKeyWord(const char *z, int32_t len); bool taosIsKeyWordToken(const char *z, int32_t len);
/**
* check if it is a number or not
* @param pToken
* @return
*/
#define isNumber(tk) \
((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN)
/** /**
* check if it is a token or not * check if it is a token or not
* @param pToken * @param pToken
* @return token type, if it is not a number, TK_ILLEGAL will return * @return token type, if it is not a number, TK_ILLEGAL will return
*/ */
static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) { static FORCE_INLINE int32_t tGetNumericStringType(const SStrToken* pToken) {
const char* z = pToken->z; const char* z = pToken->z;

34
src/query/src/qTokenizer.c → src/util/src/ttokenizer.c
View File

@ -18,7 +18,7 @@
#include "hash.h" #include "hash.h"
#include "hashfunc.h" #include "hashfunc.h"
#include "taosdef.h" #include "taosdef.h"
#include "tstoken.h" #include "ttoken.h"
#include "ttokendef.h" #include "ttokendef.h"
#include "tutil.h" #include "tutil.h"
@ -232,6 +232,18 @@ static const char isIdChar[] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
}; };
const char escapeChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, /* 0x */
0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, 0x1C, 0x1D, 0x1E, 0x1F, /* 1x */
0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x27, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x2D, 0x2E, 0x2F, /* 2x */
0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x3A, 0x3B, 0x3C, 0x3D, 0x3E, 0x3F, /* 3x */
0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F,/* 4x */
0x50, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F,/* 5x */
0x60, 0x07, 0x08, 0x63, 0x64, 0x65, 0x0C, 0x67, 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x0A, 0x6F,/* 6x */
0x70, 0x71, 0x0D, 0x73, 0x09, 0x75, 0x0B, 0x77, 0x78, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F,/* 7x */
};
static void* keywordHashTable = NULL; static void* keywordHashTable = NULL;
static void doInitKeywordsTable(void) { static void doInitKeywordsTable(void) {
@ -247,7 +259,7 @@ static void doInitKeywordsTable(void) {
static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT; static pthread_once_t keywordsHashTableInit = PTHREAD_ONCE_INIT;
int tSQLKeywordCode(const char* z, int n) { static int32_t tKeywordCode(const char* z, int n) {
pthread_once(&keywordsHashTableInit, doInitKeywordsTable); pthread_once(&keywordsHashTableInit, doInitKeywordsTable);
char key[512] = {0}; char key[512] = {0};
@ -271,7 +283,7 @@ int tSQLKeywordCode(const char* z, int n) {
* Return the length of the token that begins at z[0]. * Return the length of the token that begins at z[0].
* Store the token type in *type before returning. * Store the token type in *type before returning.
*/ */
uint32_t tSQLGetToken(char* z, uint32_t* tokenId) { uint32_t tGetToken(char* z, uint32_t* tokenId) {
uint32_t i; uint32_t i;
switch (*z) { switch (*z) {
case ' ': case ' ':
@ -403,12 +415,12 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) {
int delim = z[0]; int delim = z[0];
bool strEnd = false; bool strEnd = false;
for (i = 1; z[i]; i++) { for (i = 1; z[i]; i++) {
if (z[i] == '\\') { if (z[i] == '\\') { // ignore the escaped character that follows this backslash
i++; i++;
continue; continue;
} }
if (z[i] == delim ) { if (z[i] == delim) {
if (z[i + 1] == delim) { if (z[i + 1] == delim) {
i++; i++;
} else { } else {
@ -551,7 +563,7 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenId) {
} }
for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t) z[i]]; i++) { for (i = 1; ((z[i] & 0x80) == 0) && isIdChar[(uint8_t) z[i]]; i++) {
} }
*tokenId = tSQLKeywordCode(z, i); *tokenId = tKeywordCode(z, i);
return i; return i;
} }
} }
@ -584,7 +596,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
t = str[++(*i)]; t = str[++(*i)];
} }
t0.n = tSQLGetToken(&str[*i], &t0.type); t0.n = tGetToken(&str[*i], &t0.type);
break; break;
// not support user specfied ignored symbol list // not support user specfied ignored symbol list
@ -613,7 +625,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
// support parse the 'db.tbl' format, notes: There should be no space on either side of the dot! // support parse the 'db.tbl' format, notes: There should be no space on either side of the dot!
if ('.' == str[*i + t0.n]) { if ('.' == str[*i + t0.n]) {
len = tSQLGetToken(&str[*i + t0.n + 1], &type); len = tGetToken(&str[*i + t0.n + 1], &type);
// only id and string are valid // only id and string are valid
if ((TK_STRING != t0.type) && (TK_ID != t0.type)) { if ((TK_STRING != t0.type) && (TK_ID != t0.type)) {
@ -628,7 +640,7 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
} else { } else {
// support parse the -/+number format // support parse the -/+number format
if ((isPrevOptr) && (t0.type == TK_MINUS || t0.type == TK_PLUS)) { if ((isPrevOptr) && (t0.type == TK_MINUS || t0.type == TK_PLUS)) {
len = tSQLGetToken(&str[*i + t0.n], &type); len = tGetToken(&str[*i + t0.n], &type);
if (type == TK_INTEGER || type == TK_FLOAT) { if (type == TK_INTEGER || type == TK_FLOAT) {
t0.type = type; t0.type = type;
t0.n += len; t0.n += len;
@ -642,7 +654,9 @@ SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr) {
return t0; return t0;
} }
bool isKeyWord(const char* z, int32_t len) { return (tSQLKeywordCode((char*)z, len) != TK_ID); } bool taosIsKeyWordToken(const char* z, int32_t len) {
return (tKeywordCode((char*)z, len) != TK_ID);
}
void taosCleanupKeywordsTable() { void taosCleanupKeywordsTable() {
void* m = keywordHashTable; void* m = keywordHashTable;

1
tests/pytest/fulltest.sh
View File

@ -305,6 +305,7 @@ python3 ./test.py -f functions/function_top.py -r 1
python3 ./test.py -f functions/function_twa.py -r 1 python3 ./test.py -f functions/function_twa.py -r 1
python3 ./test.py -f functions/function_twa_test2.py python3 ./test.py -f functions/function_twa_test2.py
python3 ./test.py -f functions/function_stddev_td2555.py python3 ./test.py -f functions/function_stddev_td2555.py
python3 ./test.py -f functions/showOfflineThresholdIs864000.py
python3 ./test.py -f insert/metadataUpdate.py python3 ./test.py -f insert/metadataUpdate.py
python3 ./test.py -f query/last_cache.py python3 ./test.py -f query/last_cache.py
python3 ./test.py -f query/last_row_cache.py python3 ./test.py -f query/last_row_cache.py

36
tests/pytest/functions/showOfflineThresholdIs864000.py Normal file
View File

@ -0,0 +1,36 @@
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import sys
from util.log import *
from util.cases import *
from util.sql import *
from util.dnodes import *
class TDTestCase:
def init(self, conn, logSql):
tdLog.debug(f"start to execute {__file__}")
tdSql.init(conn.cursor(), logSql)
def run(self):
tdSql.query("show variables")
tdSql.checkData(51, 1, 864000)
def stop(self):
tdSql.close()
tdLog.success(f"{__file__} successfully executed")
tdCases.addWindows(__file__, TDTestCase())
tdCases.addLinux(__file__, TDTestCase())

5
tests/pytest/insert/nchar-unicode.py
View File

@ -57,12 +57,15 @@ class TDTestCase:
# https://www.ltg.ed.ac.uk/~richard/unicode-sample.html # https://www.ltg.ed.ac.uk/~richard/unicode-sample.html
# Basic Latin # Basic Latin
data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~' data = r'! # $ % & ( ) * + , - . / 0 1 2 3 4 5 6 7 8 9 : ; < = > ? @ A B C D E F G H I J K L M N O P Q R S T U V W X Y Z [ \\ ] ^ _ ` a b c d e f g h i j k l m n o p q r s t u v w x y z { | } ~'
tdLog.info("insert Basic Latin %d length data: %s" % (len(data), data)) tdLog.info("insert Basic Latin %d length data: %s" % (len(data), data))
tdSql.execute("insert into tb values (now, '%s')" % data) tdSql.execute("insert into tb values (now, '%s')" % data)
tdSql.query("select * from tb") tdSql.query("select * from tb")
tdSql.checkRows(3) tdSql.checkRows(3)
data = data.replace('\\\\', '\\')
tdSql.checkData(2, 1, data) tdSql.checkData(2, 1, data)
# tdSql.execute("insert into tb values(now, 'abc')")
# Latin-1 Supplement # Latin-1 Supplement
data = ' ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ­ ® ¯ ° ± ² ³ ´ µ ¶ · ¸ ¹ º » ¼ ½ ¾ ¿ À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ' data = ' ¡ ¢ £ ¤ ¥ ¦ § ¨ © ª « ¬ ­ ® ¯ ° ± ² ³ ´ µ ¶ · ¸ ¹ º » ¼ ½ ¾ ¿ À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï Ð Ñ Ò Ó Ô Õ Ö × Ø Ù Ú Û Ü Ý Þ ß à á â ã ä å æ ç è é ê ë ì í î ï ð ñ ò ó ô õ ö ÷ ø ù ú û ü ý þ ÿ'

4
tests/pytest/tools/taosdemoAllTest/speciQueryRestful.json
View File

@ -8,7 +8,7 @@
"confirm_parameter_prompt": "no", "confirm_parameter_prompt": "no",
"databases": "db", "databases": "db",
"query_times": 2, "query_times": 2,
"query_mode": "restful", "query_mode": "rest",
"specified_table_query": { "specified_table_query": {
"query_interval": 1, "query_interval": 1,
"concurrent": 3, "concurrent": 3,
@ -35,4 +35,4 @@
] ]
} }
} }

10
tests/script/general/parser/binary_escapeCharacter.sim
View File

@ -93,5 +93,15 @@ if $data41 != @udp005@ then
print "[ERROR] expect: udp005, act:$data41" print "[ERROR] expect: udp005, act:$data41"
endi endi
print ---------------------> TD-3967
sql insert into tb values(now, '\\abc\\\\');
sql insert into tb values(now, '\\abc\\\\');
sql insert into tb values(now, '\\\\');
print ------------->sim bug
# sql_error insert into tb values(now, '\\\');
sql_error insert into tb values(now, '\');
#sql_error insert into tb values(now, '\\\n');
sql insert into tb values(now, '\n');
system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s stop -x SIGINT

6
tests/script/general/parser/testSuite.sim
View File

@ -54,5 +54,9 @@ run general/parser/timestamp.sim
run general/parser/sliding.sim run general/parser/sliding.sim
run general/parser/function.sim run general/parser/function.sim
run general/parser/stableOp.sim run general/parser/stableOp.sim
run general/parser/having.sim run general/parser/having.sim
run general/parser/having_child.sim run general/parser/having_child.sim
run general/parser/slimit_alter_tags.sim
run general/parser/binary_escapeCharacter.sim