xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/develop' into mac

This commit is contained in:
freemine 2021-01-25 19:44:27 +08:00
parent e3f6230761 3480099198
commit 7c02d0c2ec
175 changed files with 4649 additions and 2948 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
CMakeLists.txt
View File

@ -13,7 +13,7 @@ ENDIF ()
SET(TD_ACCOUNT FALSE)
SET(TD_ADMIN FALSE)
SET(TD_GRANT FALSE)
SET(TD_MQTT TRUE)
SET(TD_MQTT FALSE)
SET(TD_TSDB_PLUGINS FALSE)
SET(TD_COVER FALSE)
@ -29,6 +29,11 @@ MESSAGE(STATUS "Community directory: " ${TD_COMMUNITY_DIR})
INCLUDE(cmake/input.inc)
INCLUDE(cmake/platform.inc)
IF (TD_WINDOWS OR TD_DARWIN)
SET(TD_SOMODE_STATIC TRUE)
ENDIF ()
INCLUDE(cmake/define.inc)
INCLUDE(cmake/env.inc)
INCLUDE(cmake/version.inc)

79
Jenkinsfile vendored
View File

@ -176,5 +176,84 @@ pipeline {
}
}
}
post {
success {
emailext (
subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>提交信息：${CHANGE_TITLE}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "${env.CHANGE_AUTHOR_EMAIL}",
from: "support@taosdata.com"
)
}
failure {
emailext (
subject: "PR-result: Job '${env.JOB_NAME} [${env.BUILD_NUMBER}]'",
body: '''<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
</head>
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4" offset="0">
<table width="95%" cellpadding="0" cellspacing="0" style="font-size: 16pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
<tr>
<td><br />
<b><font color="#0B610B"><font size="6">构建信息</font></font></b>
<hr size="2" width="100%" align="center" /></td>
</tr>
<tr>
<td>
<ul>
<div style="font-size:18px">
<li>构建名称>>分支：${PROJECT_NAME}</li>
<li>构建结果：<span style="color:green"> Successful </span></li>
<li>构建编号：${BUILD_NUMBER}</li>
<li>触发用户：${CAUSE}</li>
<li>提交信息：${CHANGE_TITLE}</li>
<li>构建地址：<a href=${BUILD_URL}>${BUILD_URL}</a></li>
<li>构建日志：<a href=${BUILD_URL}console>${BUILD_URL}console</a></li>
<li>变更集：${JELLY_SCRIPT}</li>
</div>
</ul>
</td>
</tr>
</table></font>
</body>
</html>''',
to: "${env.CHANGE_AUTHOR_EMAIL}",
from: "support@taosdata.com"
)
}
}
}

6
README.md
View File

@ -133,7 +133,9 @@ cmake .. -G "NMake Makefiles"
nmake
```
If you use the Visual Studio 2019, please open a command window by executing "cmd.exe".
If you use the Visual Studio 2019 or 2017:
please open a command window by executing "cmd.exe".
Please specify "x64" for 64 bits Windows or specify "x86" is for 32 bits Windows when you execute vcvarsall.bat.
```
mkdir debug && cd debug
@ -142,7 +144,7 @@ cmake .. -G "NMake Makefiles"
nmake
```
Or, you can open a command window by clicking Visual Studio 2019 menu "Tools -> Command Line -> Developer Command Prompt" or "Tools -> Command Line -> Developer PowerShell" then execute commands as follows:
Or, you can simply open a command window by clicking Windows Start -> "Visual Studio < 2019 | 2017 >" folder -> "x64 Native Tools Command Prompt for VS < 2019 | 2017 >" or "x86 Native Tools Command Prompt for VS < 2019 | 2017 >" depends what architecture your Windows is, then execute commands as follows:
```
mkdir debug && cd debug
cmake .. -G "NMake Makefiles"

3
cmake/define.inc
View File

@ -141,6 +141,9 @@ IF (TD_WINDOWS)
SET(CMAKE_GENERATOR "NMake Makefiles" CACHE INTERNAL "" FORCE)
IF (NOT TD_GODLL)
SET(COMMON_FLAGS "/nologo /WX /wd4018 /wd2220 /Oi /Oy- /Gm- /EHsc /MT /GS /Gy /fp:precise /Zc:wchar_t /Zc:forScope /Gd /errorReport:prompt /analyze-")
IF (MSVC AND (MSVC_VERSION GREATER_EQUAL 1900))
SET(COMMON_FLAGS "${COMMON_FLAGS} /Wv:18")
ENDIF ()
SET(DEBUG_FLAGS "/Zi /W3 /GL")
SET(RELEASE_FLAGS "/W0 /O3 /GL")
ENDIF ()

8
deps/zlib-1.2.11/src/gzlib.c vendored
View File

@ -5,6 +5,10 @@
#include <stdint.h>
#include "gzguts.h"
#ifndef O_BINARY
#define O_BINARY 0
#endif
#if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__)
# define LSEEK _lseeki64
#else
@ -240,9 +244,9 @@ local gzFile gz_open(path, fd, mode)
/* open the file with the appropriate flags (or just use fd) */
state->fd = fd > -1 ? fd : (
#ifdef WIDECHAR
fd == -2 ? _wopen(path, oflag, 0666) :
fd == -2 ? _wopen(path, oflag | O_BINARY, 0666) :
#endif
open((const char *)path, oflag, 0666));
open((const char *)path, oflag | O_BINARY, 0666));
if (state->fd == -1) {
free(state->path);
free(state);

2
packaging/tools/install_client.sh
View File

@ -21,7 +21,7 @@ else
cd ${script_dir}
script_dir="$(pwd)"
data_dir="/var/lib/taos"
log_dir=~/TDengineLog
log_dir=~/TDengine/log
fi
log_link_dir="/usr/local/taos/log"

2
packaging/tools/make_install.sh
View File

@ -24,7 +24,7 @@ data_dir="/var/lib/taos"
if [ "$osType" != "Darwin" ]; then
log_dir="/var/log/taos"
else
log_dir=~/TDengineLog
log_dir=~/TDengine/log
fi
data_link_dir="/usr/local/taos/data"

8
src/balance/CMakeLists.txt
View File

@ -8,10 +8,4 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
IF (TD_LINUX)
ADD_LIBRARY(balance ${SRC})
ENDIF ()
IF (TD_DARWIN)
ADD_LIBRARY(balance ${SRC})
ENDIF ()
ADD_LIBRARY(balance ${SRC})

2
src/balance/inc/bnThread.h
View File

@ -24,7 +24,7 @@ extern "C" {
int32_t bnInitThread();
void bnCleanupThread();
void bnNotify();
void bnStartTimer(int64_t mseconds);
void bnStartTimer(int32_t mseconds);
#ifdef __cplusplus
}

2
src/balance/src/bnMain.c
View File

@ -30,7 +30,7 @@
#include "mnodeVgroup.h"
extern int64_t tsDnodeRid;
extern int64_t tsSdbRid;
extern int32_t tsSdbRid;
static SBnMgmt tsBnMgmt;
static void bnMonitorDnodeModule();

10
src/balance/src/bnScore.c
View File

@ -271,23 +271,23 @@ static int32_t bnRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = systemScore;
*(float *)pWrite = (float)systemScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = pDnode->customScore;
*(float *)pWrite = (float)pDnode->customScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)moduleScore;
*(float *)pWrite = (float)moduleScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)vnodeScore;
*(float *)pWrite = (float)vnodeScore;
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
*(float *)pWrite = (float)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;

6
src/balance/src/bnThread.c
View File

@ -119,13 +119,13 @@ static void bnProcessTimer(void *handle, void *tmrId) {
}
}
void bnStartTimer(int64_t mseconds) {
void bnStartTimer(int32_t mseconds) {
if (tsBnThread.stop) return;
bool updateSoon = (mseconds != -1);
if (updateSoon) {
mTrace("balance function will be called after %" PRId64 " ms", mseconds);
taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
mTrace("balance function will be called after %d ms", mseconds);
taosTmrReset(bnProcessTimer, mseconds, (void *)(int64_t)mseconds, tsMnodeTmr, &tsBnThread.timer);
} else {
taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);
}

4
src/client/inc/tscUtil.h
View File

@ -99,14 +99,14 @@ static FORCE_INLINE SQueryInfo* tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t sub
}
int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOffset, SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks);
void tscDestroyDataBlock(STableDataBlocks* pDataBlock);
void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta);
void tscSortRemoveDataBlockDupRows(STableDataBlocks* dataBuf);
SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, int16_t bytes,
uint32_t offset);
void* tscDestroyBlockArrayList(SArray* pDataBlockList);
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable);
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta);
int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock);
int32_t tscMergeTableDataBlocks(SSqlObj* pSql, bool freeBlockMap);

10
src/client/inc/tsclient.h
View File

@ -22,15 +22,15 @@ extern "C" {
#include "os.h"
#include "qAggMain.h"
#include "taos.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tarray.h"
#include "tglobal.h"
#include "tsqlfunction.h"
#include "tutil.h"
#include "tcache.h"
#include "tglobal.h"
#include "tref.h"
#include "tutil.h"
#include "qExecutor.h"
#include "qSqlparser.h"
@ -223,6 +223,8 @@ typedef struct SQueryInfo {
int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX
int16_t resColumnId; // result column id
bool distinctTag; // distinct tag or not
} SQueryInfo;
typedef struct {
@ -411,7 +413,7 @@ void tscRestoreSQLFuncForSTableQuery(SQueryInfo *pQueryInfo);
int32_t tscCreateResPointerInfo(SSqlRes *pRes, SQueryInfo *pQueryInfo);
void tscSetResRawPtr(SSqlRes* pRes, SQueryInfo* pQueryInfo);
void tscResetSqlCmdObj(SSqlCmd *pCmd);
void tscResetSqlCmd(SSqlCmd *pCmd, bool removeMeta);
/**
* free query result of the sql object

2
src/client/src/tscAsync.c
View File

@ -351,7 +351,7 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) {
if (pCmd->command == TSDB_SQL_SELECT) {
tscDebug("%p redo parse sql string and proceed", pSql);
pCmd->parseFinished = false;
tscResetSqlCmdObj(pCmd);
tscResetSqlCmd(pCmd, true);
code = tsParseSql(pSql, true);
if (code == TSDB_CODE_TSC_ACTION_IN_PROGRESS) {

6
src/client/src/tscLocalMerge.c
View File

@ -16,7 +16,7 @@
#include "tscLocalMerge.h"
#include "tscSubquery.h"
#include "os.h"
#include "qAst.h"
#include "texpr.h"
#include "tlosertree.h"
#include "tscLog.h"
#include "tscUtil.h"
@ -1101,7 +1101,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx)
* the number of output result is decided by main output
*/
int32_t functionId = pCtx[j].functionId;
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ) {
if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG) {
continue;
}
@ -1183,7 +1183,7 @@ bool needToMerge(SQueryInfo *pQueryInfo, SLocalMerger *pLocalMerge, tFilePage *t
int16_t functionId = pLocalMerge->pCtx[0].functionId;
// todo opt performance
if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0))) { // column projection query
if ((/*functionId == TSDB_FUNC_PRJ || */functionId == TSDB_FUNC_ARITHM) || (tscIsProjectionQueryOnSTable(pQueryInfo, 0) && pQueryInfo->distinctTag == false)) { // column projection query
ret = 1; // disable merge procedure
} else {
tOrderDescriptor *pDesc = pLocalMerge->pDesc;

15
src/client/src/tscParseInsert.c
View File

@ -1036,11 +1036,7 @@ static int32_t validateDataSource(SSqlCmd *pCmd, int8_t type, const char *sql) {
}
/**
* usage: insert into table1 values() () table2 values()()
*
* @param str
* @param acct
* @param db
* parse insert sql
* @param pSql
* @return
*/
@ -1343,10 +1339,11 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
// make a backup as tsParseInsertSql may modify the string
char* sqlstr = strdup(pSql->sqlstr);
ret = tsParseInsertSql(pSql);
if (sqlstr == NULL || pSql->parseRetry >= 1 || ret != TSDB_CODE_TSC_INVALID_SQL) {
if ((sqlstr == NULL) || (pSql->parseRetry >= 1) ||
(ret != TSDB_CODE_TSC_SQL_SYNTAX_ERROR && ret != TSDB_CODE_TSC_INVALID_SQL)) {
free(sqlstr);
} else {
tscResetSqlCmdObj(pCmd);
tscResetSqlCmd(pCmd, true);
free(pSql->sqlstr);
pSql->sqlstr = sqlstr;
pSql->parseRetry++;
@ -1358,7 +1355,7 @@ int tsParseSql(SSqlObj *pSql, bool initial) {
SSqlInfo SQLInfo = qSQLParse(pSql->sqlstr);
ret = tscToSQLCmd(pSql, &SQLInfo);
if (ret == TSDB_CODE_TSC_INVALID_SQL && pSql->parseRetry == 0 && SQLInfo.type == TSDB_SQL_NULL) {
tscResetSqlCmdObj(pCmd);
tscResetSqlCmd(pCmd, true);
pSql->parseRetry++;
ret = tscToSQLCmd(pSql, &SQLInfo);
}
@ -1551,7 +1548,7 @@ void tscImportDataFromFile(SSqlObj *pSql) {
SSqlObj *pNew = createSubqueryObj(pSql, 0, parseFileSendDataBlock, pSupporter, TSDB_SQL_INSERT, NULL);
pCmd->count = 1;
FILE *fp = fopen(pCmd->payload, "r");
FILE *fp = fopen(pCmd->payload, "rb");
if (fp == NULL) {
pSql->res.code = TAOS_SYSTEM_ERROR(errno);
tscError("%p failed to open file %s to load data from file, code:%s", pSql, pCmd->payload, tstrerror(pSql->res.code));

84
src/client/src/tscSQLParser.c
View File

@ -22,7 +22,7 @@
#include "os.h"
#include "ttype.h"
#include "qAst.h"
#include "texpr.h"
#include "taos.h"
#include "taosmsg.h"
#include "tcompare.h"
@ -43,7 +43,7 @@
#define COLUMN_INDEX_INITIAL_VAL (-3)
#define COLUMN_INDEX_INITIALIZER \
{ COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL }
#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX))
#define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_BLOCK_DIST_COLUMN_INDEX))
#define TBNAME_LIST_SEP ","
typedef struct SColumnList { // todo refactor
@ -1360,7 +1360,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo, colList, NULL);
if (ret != TSDB_CODE_SUCCESS) {
taosArrayDestroy(colList);
tExprTreeDestroy(&pNode, NULL);
tExprTreeDestroy(pNode, NULL);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
@ -1369,9 +1369,9 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
for(int32_t k = 0; k < numOfNode; ++k) {
SColIndex* pIndex = taosArrayGet(colList, k);
if (TSDB_COL_IS_TAG(pIndex->flag)) {
tExprTreeDestroy(&pNode, NULL);
tExprTreeDestroy(pNode, NULL);
taosArrayDestroy(colList);
tExprTreeDestroy(&pNode, NULL);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
}
@ -1398,7 +1398,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
tbufCloseWriter(&bw);
taosArrayDestroy(colList);
tExprTreeDestroy(&pNode, NULL);
tExprTreeDestroy(pNode, NULL);
} else {
columnList.num = 0;
columnList.ids[0] = (SColumnIndex) {0, 0};
@ -1430,7 +1430,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t
int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo, NULL, &pArithExprInfo->uid);
if (ret != TSDB_CODE_SUCCESS) {
tExprTreeDestroy(&pArithExprInfo->pExpr, NULL);
tExprTreeDestroy(pArithExprInfo->pExpr, NULL);
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause");
}
@ -1507,23 +1507,39 @@ static void addPrimaryTsColIntoResult(SQueryInfo* pQueryInfo) {
pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY;
}
bool isValidDistinctSql(SQueryInfo* pQueryInfo) {
if (pQueryInfo == NULL) {
return false;
}
if ((pQueryInfo->type & TSDB_QUERY_TYPE_STABLE_QUERY) != TSDB_QUERY_TYPE_STABLE_QUERY) {
return false;
}
if (tscQueryTags(pQueryInfo) && tscSqlExprNumOfExprs(pQueryInfo) == 1){
return true;
}
return false;
}
int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery, bool intervalQuery) {
assert(pSelection != NULL && pCmd != NULL);
const char* msg2 = "functions can not be mixed up";
const char* msg3 = "not support query expression";
const char* msg5 = "invalid function name";
const char* msg6 = "only support distinct one tag";
SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, clauseIndex);
if (pQueryInfo->colList == NULL) {
pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES);
}
bool hasDistinct = false;
for (int32_t i = 0; i < pSelection->nExpr; ++i) {
int32_t outputIndex = (int32_t)tscSqlExprNumOfExprs(pQueryInfo);
tSqlExprItem* pItem = &pSelection->a[i];
if (hasDistinct == false) {
hasDistinct = (pItem->distinct == true);
}
// project on all fields
int32_t optr = pItem->pNode->nSQLOptr;
@ -1557,6 +1573,13 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel
}
}
if (hasDistinct == true) {
if (!isValidDistinctSql(pQueryInfo)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
pQueryInfo->distinctTag = true;
}
// there is only one user-defined column in the final result field, add the timestamp column.
size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList);
if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) {
@ -1733,6 +1756,9 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
SSchema colSchema = tGetTableNameColumnSchema();
tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG);
} else if (index.columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) {
SSchema colSchema = tGetBlockDistColumnSchema();
tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_TAG);
} else {
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
@ -2200,6 +2226,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
if (getColumnIndexByName(pCmd, &pParamElem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3);
}
if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6);
}
@ -2408,6 +2435,14 @@ static bool isTablenameToken(SStrToken* token) {
return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L));
}
static bool isTableBlockDistToken(SStrToken* token) {
SStrToken tmpToken = *token;
SStrToken tableToken = {0};
extractTableNameFromToken(&tmpToken, &tableToken);
return (strncasecmp(TSQL_BLOCK_DIST, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_BLOCK_DIST_L));
}
static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) {
STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta;
@ -2437,6 +2472,8 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQu
if (isTablenameToken(pToken)) {
pIndex->columnIndex = TSDB_TBNAME_COLUMN_INDEX;
} else if (isTableBlockDistToken(pToken)) {
pIndex->columnIndex = TSDB_BLOCK_DIST_COLUMN_INDEX;
} else if (strncasecmp(pToken->z, DEFAULT_PRIMARY_TIMESTAMP_COL_NAME, pToken->n) == 0) {
pIndex->columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX;
} else {
@ -2677,8 +2714,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) {
if (!validateIpAddress(pDnodeIp->z, pDnodeIp->n)) {
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4);
}
}
}
return TSDB_CODE_SUCCESS;
}
@ -4309,7 +4345,7 @@ static int32_t getTagQueryCondExpr(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SCondE
doCompactQueryExpr(pExpr);
tSqlExprDestroy(p1);
tExprTreeDestroy(&p, NULL);
tExprTreeDestroy(p, NULL);
taosArrayDestroy(colList);
if (pQueryInfo->tagCond.pCond != NULL && taosArrayGetSize(pQueryInfo->tagCond.pCond) > 0 && !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) {
@ -4530,10 +4566,10 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2);
}
size_t size = tscNumOfFields(pQueryInfo);
size_t numOfFields = tscNumOfFields(pQueryInfo);
if (pQueryInfo->fillVal == NULL) {
pQueryInfo->fillVal = calloc(size, sizeof(int64_t));
pQueryInfo->fillVal = calloc(numOfFields, sizeof(int64_t));
if (pQueryInfo->fillVal == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@ -4543,7 +4579,7 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
pQueryInfo->fillType = TSDB_FILL_NONE;
} else if (strncasecmp(pItem->pVar.pz, "null", 4) == 0 && pItem->pVar.nLen == 4) {
pQueryInfo->fillType = TSDB_FILL_NULL;
for (int32_t i = START_INTERPO_COL_IDX; i < size; ++i) {
for (int32_t i = START_INTERPO_COL_IDX; i < numOfFields; ++i) {
TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
setNull((char*)&pQueryInfo->fillVal[i], pField->type, pField->bytes);
}
@ -4557,7 +4593,7 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
pQueryInfo->fillType = TSDB_FILL_SET_VALUE;
size_t num = taosArrayGetSize(pFillToken);
if (num == 1) {
if (num == 1) { // no actual value, return with error code
return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
}
@ -4568,11 +4604,11 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
if (tscIsPointInterpQuery(pQueryInfo)) {
startPos = 0;
if (numOfFillVal > size) {
numOfFillVal = (int32_t)size;
if (numOfFillVal > numOfFields) {
numOfFillVal = (int32_t)numOfFields;
}
} else {
numOfFillVal = (int16_t)((num > (int32_t)size) ? (int32_t)size : num);
numOfFillVal = (int16_t)((num > (int32_t)numOfFields) ? (int32_t)numOfFields : num);
}
int32_t j = 1;
@ -4592,10 +4628,10 @@ int32_t parseFillClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuery
}
}
if ((num < size) || ((num - 1 < size) && (tscIsPointInterpQuery(pQueryInfo)))) {
if ((num < numOfFields) || ((num - 1 < numOfFields) && (tscIsPointInterpQuery(pQueryInfo)))) {
tVariantListItem* lastItem = taosArrayGetLast(pFillToken);
for (int32_t i = numOfFillVal; i < size; ++i) {
for (int32_t i = numOfFillVal; i < numOfFields; ++i) {
TAOS_FIELD* pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i);
if (pField->type == TSDB_DATA_TYPE_BINARY || pField->type == TSDB_DATA_TYPE_NCHAR) {
@ -4646,6 +4682,12 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu
setDefaultOrderInfo(pQueryInfo);
STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
if (pQueryInfo->distinctTag == true) {
pQueryInfo->order.order = TSDB_ORDER_ASC;
pQueryInfo->order.orderColId = 0;
return TSDB_CODE_SUCCESS;
}
if (pQuerySql->pSortOrder == NULL) {
return TSDB_CODE_SUCCESS;
}

4
src/client/src/tscSql.c
View File

@ -15,7 +15,7 @@
#include "hash.h"
#include "os.h"
#include "qAst.h"
#include "texpr.h"
#include "tkey.h"
#include "tcache.h"
#include "tnote.h"
@ -941,7 +941,7 @@ int taos_validate_sql(TAOS *taos, const char *sql) {
static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t tblListLen) {
// must before clean the sqlcmd object
tscResetSqlCmdObj(&pSql->cmd);
tscResetSqlCmd(&pSql->cmd, false);
SSqlCmd *pCmd = &pSql->cmd;

15
src/client/src/tscStream.c
View File

@ -191,9 +191,10 @@ static void tscProcessStreamQueryCallback(void *param, TAOS_RES *tres, int numOf
STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pStream->pSql->cmd, 0, 0);
assert(0);
// char* name = pTableMetaInfo->name;
// taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pTableMetaInfo->name, name);
taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
pTableMetaInfo->vgroupList = tscVgroupInfoClear(pTableMetaInfo->vgroupList);
tscSetRetryTimer(pStream, pStream->pSql, retryDelay);
@ -292,8 +293,8 @@ static void tscProcessStreamRetrieveResult(void *param, TAOS_RES *res, int numOf
pStream->stime += 1;
}
// tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, pTableMetaInfo->name,
// pStream->numOfRes);
tscDebug("%p stream:%p, query on:%s, fetch result completed, fetched rows:%" PRId64, pSql, pStream, tNameGetTableName(&pTableMetaInfo->name),
pStream->numOfRes);
tfree(pTableMetaInfo->pTableMeta);
@ -556,8 +557,8 @@ static void tscCreateStream(void *param, TAOS_RES *res, int code) {
taosTmrReset(tscProcessStreamTimer, (int32_t)starttime, pStream, tscTmr, &pStream->pTimer);
// tscDebug("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql,
// pStream, pTableMetaInfo->name, pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr);
tscDebug("%p stream:%p is opened, query on:%s, interval:%" PRId64 ", sliding:%" PRId64 ", first launched in:%" PRId64 ", sql:%s", pSql,
pStream, tNameGetTableName(&pTableMetaInfo->name), pStream->interval.interval, pStream->interval.sliding, starttime, pSql->sqlstr);
}
void tscSetStreamDestTable(SSqlStream* pStream, const char* dstTable) {

4
src/client/src/tscSub.c
View File

@ -313,7 +313,7 @@ static int tscLoadSubscriptionProgress(SSub* pSub) {
char buf[TSDB_MAX_SQL_LEN];
sprintf(buf, "%s/subscribe/%s", tsDataDir, pSub->topic);
FILE* fp = fopen(buf, "r");
FILE* fp = fopen(buf, "rb");
if (fp == NULL) {
tscDebug("subscription progress file does not exist: %s", pSub->topic);
return 1;
@ -368,7 +368,7 @@ void tscSaveSubscriptionProgress(void* sub) {
}
sprintf(path, "%s/subscribe/%s", tsDataDir, pSub->topic);
FILE* fp = fopen(path, "w+");
FILE* fp = fopen(path, "wb+");
if (fp == NULL) {
tscError("failed to create progress file for subscription: %s", pSub->topic);
return;

58
src/client/src/tscSubquery.c
View File

@ -16,7 +16,7 @@
#include "os.h"
#include "qAst.h"
#include "texpr.h"
#include "qTsbuf.h"
#include "tcompare.h"
#include "tscLog.h"
@ -582,13 +582,14 @@ void freeJoinSubqueryObj(SSqlObj* pSql) {
pSql->subState.numOfSub = 0;
}
static void quitAllSubquery(SSqlObj* pSqlSub, SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
static int32_t quitAllSubquery(SSqlObj* pSqlSub, SSqlObj* pSqlObj, SJoinSupporter* pSupporter) {
if (subAndCheckDone(pSqlSub, pSqlObj, pSupporter->subqueryIndex)) {
tscError("%p all subquery return and query failed, global code:%s", pSqlObj, tstrerror(pSqlObj->res.code));
freeJoinSubqueryObj(pSqlObj);
return;
return 0;
}
return 1;
//tscDestroyJoinSupporter(pSupporter);
}
@ -835,7 +836,9 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code);
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)) {
return;
}
tscAsyncResultOnError(pParentSql);
@ -850,7 +853,9 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex);
pParentSql->res.code = numOfRows;
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)) {
return;
}
tscAsyncResultOnError(pParentSql);
return;
@ -867,7 +872,9 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
tscError("%p failed to malloc memory", pSql);
pParentSql->res.code = TAOS_SYSTEM_ERROR(errno);
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)) {
return;
}
tscAsyncResultOnError(pParentSql);
return;
@ -985,7 +992,9 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code);
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)){
return;
}
tscAsyncResultOnError(pParentSql);
@ -999,7 +1008,9 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
tscError("%p sub query failed, code:%s, index:%d", pSql, tstrerror(numOfRows), pSupporter->subqueryIndex);
pParentSql->res.code = numOfRows;
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)){
return;
}
tscAsyncResultOnError(pParentSql);
return;
@ -1007,14 +1018,16 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
if (numOfRows > 0) { // write the compressed timestamp to disk file
if(pSupporter->f == NULL) {
pSupporter->f = fopen(pSupporter->path, "w");
pSupporter->f = fopen(pSupporter->path, "wb");
if (pSupporter->f == NULL) {
tscError("%p failed to create tmp file:%s, reason:%s", pSql, pSupporter->path, strerror(errno));
pParentSql->res.code = TAOS_SYSTEM_ERROR(errno);
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)) {
return;
}
tscAsyncResultOnError(pParentSql);
@ -1032,7 +1045,9 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
pParentSql->res.code = TAOS_SYSTEM_ERROR(errno);
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)){
return;
}
tscAsyncResultOnError(pParentSql);
@ -1051,7 +1066,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
// continue to retrieve ts-comp data from vnode
if (!pRes->completed) {
taosGetTmpfilePath("ts-join", pSupporter->path);
pSupporter->f = fopen(pSupporter->path, "w");
pSupporter->f = fopen(pSupporter->path, "wb");
pRes->row = pRes->numOfRows;
taos_fetch_rows_a(tres, tsCompRetrieveCallback, param);
@ -1077,7 +1092,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow
taosGetTmpfilePath("ts-join", pSupporter->path);
// TODO check for failure
pSupporter->f = fopen(pSupporter->path, "w");
pSupporter->f = fopen(pSupporter->path, "wb");
pRes->row = pRes->numOfRows;
// set the callback function
@ -1129,8 +1144,10 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, numOfRows, pParentSql->res.code);
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)) {
return;
}
tscAsyncResultOnError(pParentSql);
return;
@ -1472,7 +1489,9 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
// retrieve actual query results from vnode during the second stage join subquery
if (pParentSql->res.code != TSDB_CODE_SUCCESS) {
tscError("%p abort query due to other subquery failure. code:%d, global code:%d", pSql, code, pParentSql->res.code);
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)) {
return;
}
tscAsyncResultOnError(pParentSql);
@ -1486,7 +1505,10 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) {
tscError("%p abort query, code:%s, global code:%s", pSql, tstrerror(code), tstrerror(pParentSql->res.code));
pParentSql->res.code = code;
quitAllSubquery(pSql, pParentSql, pSupporter);
if (quitAllSubquery(pSql, pParentSql, pSupporter)) {
return;
}
tscAsyncResultOnError(pParentSql);
return;
@ -2441,7 +2463,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
pParentObj->cmd.parseFinished = false;
tscResetSqlCmdObj(&pParentObj->cmd);
tscResetSqlCmd(&pParentObj->cmd, false);
// in case of insert, redo parsing the sql string and build new submit data block for two reasons:
// 1. the table Id(tid & uid) may have been update, the submit block needs to be updated accordingly.

31
src/client/src/tscUtil.c
View File

@ -16,7 +16,7 @@
#include "tscUtil.h"
#include "hash.h"
#include "os.h"
#include "qAst.h"
#include "texpr.h"
#include "taosmsg.h"
#include "tkey.h"
#include "tmd5.h"
@ -381,7 +381,7 @@ void tscFreeQueryInfo(SSqlCmd* pCmd) {
tfree(pCmd->pQueryInfo);
}
void tscResetSqlCmdObj(SSqlCmd* pCmd) {
void tscResetSqlCmd(SSqlCmd* pCmd, bool removeMeta) {
pCmd->command = 0;
pCmd->numOfCols = 0;
pCmd->count = 0;
@ -399,7 +399,7 @@ void tscResetSqlCmdObj(SSqlCmd* pCmd) {
pCmd->numOfTables = 0;
tfree(pCmd->pTableNameList);
pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList);
pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, removeMeta);
pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks);
tscFreeQueryInfo(pCmd);
}
@ -501,7 +501,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
pSql->self = 0;
tscFreeSqlResult(pSql);
tscResetSqlCmdObj(pCmd);
tscResetSqlCmd(pCmd, false);
tfree(pCmd->tagData.data);
pCmd->tagData.dataLen = 0;
@ -515,7 +515,7 @@ void tscFreeSqlObj(SSqlObj* pSql) {
free(pSql);
}
void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
void tscDestroyDataBlock(STableDataBlocks* pDataBlock, bool removeMeta) {
if (pDataBlock == NULL) {
return;
}
@ -528,6 +528,13 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
tfree(pDataBlock->pTableMeta);
}
if (removeMeta) {
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pDataBlock->tableName, name);
taosHashRemove(tscTableMetaInfo, name, strnlen(name, TSDB_TABLE_FNAME_LEN));
}
tfree(pDataBlock);
}
@ -563,21 +570,21 @@ void* tscDestroyBlockArrayList(SArray* pDataBlockList) {
size_t size = taosArrayGetSize(pDataBlockList);
for (int32_t i = 0; i < size; i++) {
void* d = taosArrayGetP(pDataBlockList, i);
tscDestroyDataBlock(d);
tscDestroyDataBlock(d, false);
}
taosArrayDestroy(pDataBlockList);
return NULL;
}
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable) {
void* tscDestroyBlockHashTable(SHashObj* pBlockHashTable, bool removeMeta) {
if (pBlockHashTable == NULL) {
return NULL;
}
STableDataBlocks** p = taosHashIterate(pBlockHashTable, NULL);
while(p) {
tscDestroyDataBlock(*p);
tscDestroyDataBlock(*p, removeMeta);
p = taosHashIterate(pBlockHashTable, p);
}
@ -791,7 +798,7 @@ static void extractTableNameList(SSqlCmd* pCmd, bool freeBlockMap) {
}
if (freeBlockMap) {
pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList);
pCmd->pTableBlockHashList = tscDestroyBlockHashTable(pCmd->pTableBlockHashList, false);
}
}
@ -1047,7 +1054,7 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) {
SInternalField* pInfo = taosArrayGet(pFieldInfo->internalField, i);
if (pInfo->pArithExprInfo != NULL) {
tExprTreeDestroy(&pInfo->pArithExprInfo->pExpr, NULL);
tExprTreeDestroy(pInfo->pArithExprInfo->pExpr, NULL);
SSqlFuncMsg* pFuncMsg = &pInfo->pArithExprInfo->base;
for(int32_t j = 0; j < pFuncMsg->numOfParams; ++j) {
@ -1080,6 +1087,8 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol
// set the correct columnIndex index
if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX;
} else if (pColIndex->columnIndex == TSDB_BLOCK_DIST_COLUMN_INDEX) {
pExpr->colInfo.colId = TSDB_BLOCK_DIST_COLUMN_INDEX;
} else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) {
pExpr->colInfo.colId = pColIndex->columnIndex;
} else {
@ -1496,7 +1505,7 @@ bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t
return false;
}
if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) {
if (colId == TSDB_TBNAME_COLUMN_INDEX || colId == TSDB_BLOCK_DIST_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) {
return true;
}

0
src/query/inc/qArithmeticOperator.h → src/common/inc/tarithoperator.h
View File

26
src/common/inc/tdataformat.h
View File

@ -27,23 +27,23 @@
extern "C" {
#endif
#define STR_TO_VARSTR(x, str) \
do { \
VarDataLenT __len = strlen(str); \
*(VarDataLenT *)(x) = __len; \
memcpy(varDataVal(x), (str), __len); \
#define STR_TO_VARSTR(x, str) \
do { \
VarDataLenT __len = (VarDataLenT)strlen(str); \
*(VarDataLenT *)(x) = __len; \
memcpy(varDataVal(x), (str), __len); \
} while (0);
#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \
do { \
#define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \
do { \
char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \
varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \
varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \
} while (0)
#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \
do { \
*(VarDataLenT *)(x) = (_size); \
memcpy(varDataVal(x), (str), (_size)); \
#define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \
do { \
*(VarDataLenT *)(x) = (VarDataLenT)(_size); \
memcpy(varDataVal(x), (str), (_size)); \
} while (0);
// ----------------- TSDB COLUMN DEFINITION
@ -156,7 +156,7 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
* +----------+----------+---------------------------------+---------------------------------+
* | len | sversion | First part | Second part |
* +----------+----------+---------------------------------+---------------------------------+
*
*
* NOTE: timestamp in this row structure is TKEY instead of TSKEY
*/
typedef void *SDataRow;

38
src/query/inc/qAst.h → src/common/inc/texpr.h
View File

@ -31,6 +31,15 @@ extern "C" {
struct tExprNode;
struct SSchema;
#define QUERY_COND_REL_PREFIX_IN "IN|"
#define QUERY_COND_REL_PREFIX_LIKE "LIKE|"
#define QUERY_COND_REL_PREFIX_IN_LEN 3
#define QUERY_COND_REL_PREFIX_LIKE_LEN 5
typedef bool (*__result_filter_fn_t)(const void *, void *);
typedef void (*__do_filter_suppl_fn_t)(void *, void *);
enum {
TSQL_NODE_DUMMY = 0x0,
TSQL_NODE_EXPR = 0x1,
@ -38,9 +47,6 @@ enum {
TSQL_NODE_VALUE = 0x4,
};
typedef bool (*__result_filter_fn_t)(const void *, void *);
typedef void (*__do_filter_suppl_fn_t)(void *, void *);
/**
* this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required
*/
@ -52,12 +58,6 @@ typedef struct tQueryInfo {
bool indexed; // indexed columns
} tQueryInfo;
typedef struct SExprTraverseSupp {
__result_filter_fn_t nodeFilterFn;
__do_filter_suppl_fn_t setupInfoFn;
void * pExtInfo;
} SExprTraverseSupp;
typedef struct tExprNode {
uint8_t nodeType;
union {
@ -65,7 +65,7 @@ typedef struct tExprNode {
uint8_t optr; // filter operator
uint8_t hasPK; // 0: do not contain primary filter, 1: contain
void * info; // support filter operation on this expression only available for leaf node
struct tExprNode *pLeft; // left child pointer
struct tExprNode *pRight; // right child pointer
} _node;
@ -74,19 +74,27 @@ typedef struct tExprNode {
};
} tExprNode;
void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
char *(*cb)(void *, const char*, int32_t));
typedef struct SExprTraverseSupp {
__result_filter_fn_t nodeFilterFn;
__do_filter_suppl_fn_t setupInfoFn;
void * pExtInfo;
} SExprTraverseSupp;
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
tExprNode* exprTreeFromBinary(const void* data, size_t size);
tExprNode* exprTreeFromTableName(const char* tbnameCond);
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *));
void tExprTreeDestroy(tExprNode **pExprs, void (*fp)(void*));
bool exprTreeApplayFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param);
typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight,
int32_t rightType, void *output, int32_t order);
void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,
char *(*cb)(void *, const char*, int32_t));
#ifdef __cplusplus
}
#endif

4
src/common/inc/tglobal.h
View File

@ -88,8 +88,8 @@ extern int32_t tsMinRowsInFileBlock;
extern int32_t tsMaxRowsInFileBlock;
extern int16_t tsCommitTime; // seconds
extern int32_t tsTimePrecision;
extern int16_t tsCompression;
extern int16_t tsWAL;
extern int8_t tsCompression;
extern int8_t tsWAL;
extern int32_t tsFsyncPeriod;
extern int32_t tsReplications;
extern int32_t tsQuorum;

17
src/common/inc/tname.h
View File

@ -1,3 +1,18 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_NAME_H
#define TDENGINE_NAME_H
@ -45,6 +60,8 @@ void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable);
SSchema tGetTableNameColumnSchema();
SSchema tGetBlockDistColumnSchema();
SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name);
bool tscValidateTableNameLength(size_t len);

2
src/query/src/qArithmeticOperator.c → src/common/src/tarithoperator.c
View File

@ -15,9 +15,9 @@
#include "os.h"
#include "qArithmeticOperator.h"
#include "ttype.h"
#include "tutil.h"
#include "tarithoperator.h"
#define ARRAY_LIST_OP(left, right, _left_type, _right_type, len1, len2, out, op, _res_type, _ord) \
{ \

28
src/query/src/qAst.c → src/common/src/texpr.c
View File

@ -16,18 +16,15 @@
#include "os.h"
#include "exception.h"
#include "qArithmeticOperator.h"
#include "qAst.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tarray.h"
#include "tbuffer.h"
#include "tcompare.h"
#include "tname.h"
#include "tschemautil.h"
#include "tsdb.h"
#include "tskiplist.h"
#include "tsqlfunction.h"
#include "texpr.h"
#include "tarithoperator.h"
static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
if (pLeft->nodeType == TSQL_NODE_COL) {
@ -102,13 +99,15 @@ static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOf
}
}
void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) {
static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *));
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) {
if (pNode == NULL) {
return;
}
if (pNode->nodeType == TSQL_NODE_EXPR) {
tExprTreeDestroy(&pNode, fp);
doExprTreeDestroy(&pNode, fp);
} else if (pNode->nodeType == TSQL_NODE_VALUE) {
tVariantDestroy(pNode->pVal);
} else if (pNode->nodeType == TSQL_NODE_COL) {
@ -118,14 +117,14 @@ void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)) {
free(pNode);
}
void tExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
if (*pExpr == NULL) {
return;
}
if ((*pExpr)->nodeType == TSQL_NODE_EXPR) {
tExprTreeDestroy(&(*pExpr)->_node.pLeft, fp);
tExprTreeDestroy(&(*pExpr)->_node.pRight, fp);
doExprTreeDestroy(&(*pExpr)->_node.pLeft, fp);
doExprTreeDestroy(&(*pExpr)->_node.pRight, fp);
if (fp != NULL) {
fp((*pExpr)->_node.info);
@ -270,8 +269,9 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
}
}
free(pLeftOutput);
free(pRightOutput);
tfree(pdata);
tfree(pLeftOutput);
tfree(pRightOutput);
}
static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
@ -342,7 +342,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
}
tExprNode* pExpr = exception_calloc(1, sizeof(tExprNode));
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, pExpr, NULL);
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, pExpr, NULL);
pExpr->nodeType = tbufReadUint8(br);
if (pExpr->nodeType == TSQL_NODE_VALUE) {
@ -396,7 +396,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
int32_t anchor = CLEANUP_GET_ANCHOR();
tExprNode* expr = exception_calloc(1, sizeof(tExprNode));
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprNodeDestroy, expr, NULL);
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL);
expr->nodeType = TSQL_NODE_EXPR;

8
src/common/src/tglobal.c
View File

@ -121,8 +121,8 @@ int32_t tsMinRowsInFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK;
int32_t tsMaxRowsInFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK;
int16_t tsCommitTime = TSDB_DEFAULT_COMMIT_TIME; // seconds
int32_t tsTimePrecision = TSDB_DEFAULT_PRECISION;
int16_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
int16_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int8_t tsCompression = TSDB_DEFAULT_COMP_LEVEL;
int8_t tsWAL = TSDB_DEFAULT_WAL_LEVEL;
int32_t tsFsyncPeriod = TSDB_DEFAULT_FSYNC_PERIOD;
int32_t tsReplications = TSDB_DEFAULT_DB_REPLICA_OPTION;
int32_t tsQuorum = TSDB_DEFAULT_DB_QUORUM_OPTION;
@ -758,7 +758,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "comp";
cfg.ptr = &tsCompression;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_COMP_LEVEL;
cfg.maxValue = TSDB_MAX_COMP_LEVEL;
@ -768,7 +768,7 @@ static void doInitGlobalConfig(void) {
cfg.option = "walLevel";
cfg.ptr = &tsWAL;
cfg.valType = TAOS_CFG_VTYPE_INT16;
cfg.valType = TAOS_CFG_VTYPE_INT8;
cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
cfg.minValue = TSDB_MIN_WAL_LEVEL;
cfg.maxValue = TSDB_MAX_WAL_LEVEL;

8
src/common/src/tname.c
View File

@ -39,6 +39,14 @@ SSchema tGetTableNameColumnSchema() {
tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN);
return s;
}
SSchema tGetBlockDistColumnSchema() {
SSchema s = {0};
s.bytes = TSDB_MAX_BINARY_LEN;;
s.type = TSDB_DATA_TYPE_BINARY;
s.colId = TSDB_BLOCK_DIST_COLUMN_INDEX;
tstrncpy(s.name, TSQL_BLOCK_DIST_L, TSDB_COL_NAME_LEN);
return s;
}
SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name) {
SSchema s = {0};

4
src/common/src/tvariant.c
View File

@ -775,7 +775,7 @@ int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool inclu
return -1;
}
} else {
wcsncpy((wchar_t *)p, pVariant->wpz, pVariant->nLen);
memcpy(p, pVariant->wpz, pVariant->nLen);
newlen = pVariant->nLen;
}
@ -867,4 +867,4 @@ int32_t tVariantTypeSetType(tVariant *pVariant, char type) {
}
return 0;
}
}

24
src/cq/CMakeLists.txt
View File

@ -6,22 +6,10 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC)
IF (TD_LINUX)
ADD_LIBRARY(tcq ${SRC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(tcq tutil common taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(tcq tutil common taos)
ENDIF ()
ADD_SUBDIRECTORY(test)
ENDIF ()
IF (TD_DARWIN)
ADD_LIBRARY(tcq ${SRC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(tcq tutil common taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(tcq tutil common taos)
ENDIF ()
ADD_SUBDIRECTORY(test)
ADD_LIBRARY(tcq ${SRC})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(tcq tutil common taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(tcq tutil common taos)
ENDIF ()
ADD_SUBDIRECTORY(test)

2
src/cq/src/cqMain.c
View File

@ -343,7 +343,7 @@ static void cqProcessStreamRes(void *param, TAOS_RES *tres, TAOS_ROW row) {
char buf[TSDB_MAX_NCHAR_LEN];
int32_t len = taos_fetch_lengths(tres)[i];
taosMbsToUcs4(val, len, buf, sizeof(buf), &len);
memcpy(val + sizeof(VarDataLenT), buf, len);
memcpy((char *)val + sizeof(VarDataLenT), buf, len);
varDataLen(val) = len;
}
tdAppendColVal(trow, val, c->type, c->bytes, c->offset);

2
src/cq/test/CMakeLists.txt
View File

@ -3,4 +3,4 @@ PROJECT(TDengine)
LIST(APPEND CQTEST_SRC ./cqtest.c)
ADD_EXECUTABLE(cqtest ${CQTEST_SRC})
TARGET_LINK_LIBRARIES(cqtest tcq)
TARGET_LINK_LIBRARIES(cqtest tcq taos_static)

101
src/dnode/CMakeLists.txt
View File

@ -10,79 +10,38 @@ INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
IF (TD_LINUX)
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync)
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosd taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(taosd taos)
ENDIF ()
IF (TD_ACCOUNT)
TARGET_LINK_LIBRARIES(taosd account)
ENDIF ()
IF (TD_GRANT)
TARGET_LINK_LIBRARIES(taosd grant)
ENDIF ()
IF (TD_MQTT)
TARGET_LINK_LIBRARIES(taosd mqtt)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd")
SET(PREPARE_ENV_TARGET "prepare_env_target")
ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
POST_BUILD
COMMAND echo "make test directory"
DEPENDS taosd
COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/
COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/
COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/
COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
COMMENT "prepare taosd environment")
ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD})
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosd taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(taosd taos)
ENDIF ()
IF (TD_DARWIN)
ADD_EXECUTABLE(taosd ${SRC})
TARGET_LINK_LIBRARIES(taosd mnode monitor http tsdb twal vnode cJson lz4 balance sync)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(taosd taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(taosd taos)
ENDIF ()
IF (TD_ACCOUNT)
TARGET_LINK_LIBRARIES(taosd account)
ENDIF ()
IF (TD_GRANT)
TARGET_LINK_LIBRARIES(taosd grant)
ENDIF ()
IF (TD_MQTT)
TARGET_LINK_LIBRARIES(taosd mqtt)
ENDIF ()
# SET(PREPARE_ENV_CMD "prepare_env_cmd")
# SET(PREPARE_ENV_TARGET "prepare_env_target")
# ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
# POST_BUILD
# COMMAND echo "make test directory"
# DEPENDS taosd
# COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/
# COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/
# COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/
# COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
# COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
# COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
# COMMENT "prepare taosd environment")
# ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD})
IF (TD_ACCOUNT)
TARGET_LINK_LIBRARIES(taosd account)
ENDIF ()
IF (TD_GRANT)
TARGET_LINK_LIBRARIES(taosd grant)
ENDIF ()
IF ((TD_LINUX OR TD_WINDOWS) AND TD_MQTT)
TARGET_LINK_LIBRARIES(taosd mqtt)
ENDIF ()
SET(PREPARE_ENV_CMD "prepare_env_cmd")
SET(PREPARE_ENV_TARGET "prepare_env_target")
ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
POST_BUILD
COMMAND echo "make test directory"
DEPENDS taosd
COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/cfg/
COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/log/
COMMAND ${CMAKE_COMMAND} -E make_directory ${TD_TESTS_OUTPUT_DIR}/data/
COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8 >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
COMMENT "prepare taosd environment")
ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD})

4
src/dnode/src/dnodeCfg.c
View File

@ -97,7 +97,7 @@ static int32_t dnodeReadCfg() {
goto PARSE_CFG_OVER;
}
len = fread(content, 1, maxLen, fp);
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s, content is null", file);
goto PARSE_CFG_OVER;
@ -115,7 +115,7 @@ static int32_t dnodeReadCfg() {
dError("failed to read %s, dnodeId not found", file);
goto PARSE_CFG_OVER;
}
cfg.dnodeId = dnodeId->valueint;
cfg.dnodeId = (int32_t)dnodeId->valueint;
cJSON *clusterId = cJSON_GetObjectItem(root, "clusterId");
if (!clusterId || clusterId->type != cJSON_String) {

40
src/dnode/src/dnodeCheck.c
View File

@ -29,8 +29,8 @@ typedef struct {
static SCheckItem tsCheckItem[TSDB_CHECK_ITEM_MAX] = {{0}};
int64_t tsMinFreeMemSizeForStart = 0;
static int bindTcpPort(int port) {
int serverSocket;
static int32_t bindTcpPort(int16_t port) {
SOCKET serverSocket;
struct sockaddr_in server_addr;
if ((serverSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0) {
@ -45,22 +45,22 @@ static int bindTcpPort(int port) {
if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
dError("port:%d tcp bind() fail: %s", port, strerror(errno));
close(serverSocket);
taosCloseSocket(serverSocket);
return -1;
}
if (listen(serverSocket, 5) < 0) {
dError("port:%d listen() fail: %s", port, strerror(errno));
close(serverSocket);
taosCloseSocket(serverSocket);
return -1;
}
close(serverSocket);
taosCloseSocket(serverSocket);
return 0;
}
static int bindUdpPort(int port) {
int serverSocket;
static int32_t bindUdpPort(int16_t port) {
SOCKET serverSocket;
struct sockaddr_in server_addr;
if ((serverSocket = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP)) < 0) {
@ -75,19 +75,19 @@ static int bindUdpPort(int port) {
if (bind(serverSocket, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) {
dError("port:%d udp bind() fail: %s", port, strerror(errno));
close(serverSocket);
taosCloseSocket(serverSocket);
return -1;
}
close(serverSocket);
taosCloseSocket(serverSocket);
return 0;
}
static int dnodeCheckNetwork() {
int ret;
int startPort = tsServerPort;
static int32_t dnodeCheckNetwork() {
int32_t ret;
int16_t startPort = tsServerPort;
for (int port = startPort; port < startPort + 12; port++) {
for (int16_t port = startPort; port < startPort + 12; port++) {
ret = bindTcpPort(port);
if (0 != ret) {
dError("failed to tcp bind port %d, quit", port);
@ -103,7 +103,7 @@ static int dnodeCheckNetwork() {
return 0;
}
static int dnodeCheckMem() {
static int32_t dnodeCheckMem() {
float memoryUsedMB;
float memoryAvailMB;
if (true != taosGetSysMemory(&memoryUsedMB)) {
@ -121,12 +121,12 @@ static int dnodeCheckMem() {
return 0;
}
static int dnodeCheckCpu() {
static int32_t dnodeCheckCpu() {
// TODO:
return 0;
}
static int dnodeCheckDisk() {
static int32_t dnodeCheckDisk() {
taosGetDisk();
if (tsAvailDataDirGB < tsMinimalDataDirGB) {
@ -147,24 +147,24 @@ static int dnodeCheckDisk() {
return 0;
}
static int dnodeCheckOs() {
static int32_t dnodeCheckOs() {
// TODO:
return 0;
}
static int dnodeCheckAccess() {
static int32_t dnodeCheckAccess() {
// TODO:
return 0;
}
static int dnodeCheckVersion() {
static int32_t dnodeCheckVersion() {
// TODO:
return 0;
}
static int dnodeCheckDatafile() {
static int32_t dnodeCheckDatafile() {
// TODO:
return 0;

4
src/dnode/src/dnodeEps.c
View File

@ -152,7 +152,7 @@ static int32_t dnodeReadEps() {
goto PRASE_EPS_OVER;
}
len = fread(content, 1, maxLen, fp);
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s, content is null", file);
goto PRASE_EPS_OVER;
@ -199,7 +199,7 @@ static int32_t dnodeReadEps() {
dError("failed to read %s, dnodeId not found", file);
goto PRASE_EPS_OVER;
}
ep->dnodeId = dnodeId->valueint;
ep->dnodeId = (int32_t)dnodeId->valueint;
cJSON *dnodeFqdn = cJSON_GetObjectItem(dnodeInfo, "dnodeFqdn");
if (!dnodeFqdn || dnodeFqdn->type != cJSON_String || dnodeFqdn->valuestring == NULL) {

16
src/dnode/src/dnodeMInfos.c
View File

@ -80,7 +80,7 @@ void dnodeUpdateEpSetForPeer(SRpcEpSet *ep) {
pthread_mutex_lock(&tsMInfosMutex);
dInfo("minfos is changed, numOfEps:%d inUse:%d", ep->numOfEps, ep->inUse);
for (int i = 0; i < ep->numOfEps; ++i) {
for (int32_t i = 0; i < ep->numOfEps; ++i) {
ep->port[i] -= TSDB_PORT_DNODEDNODE;
dInfo("minfo:%d %s:%u", i, ep->fqdn[i], ep->port[i]);
}
@ -108,7 +108,7 @@ void dnodeGetMInfos(SMInfos *pMinfos) {
void dnodeGetEpSetForPeer(SRpcEpSet *epSet) {
pthread_mutex_lock(&tsMInfosMutex);
*epSet = tsMEpSet;
for (int i = 0; i < epSet->numOfEps; ++i) {
for (int32_t i = 0; i < epSet->numOfEps; ++i) {
epSet->port[i] += TSDB_PORT_DNODEDNODE;
}
pthread_mutex_unlock(&tsMInfosMutex);
@ -171,7 +171,7 @@ static int32_t dnodeReadMInfos() {
goto PARSE_MINFOS_OVER;
}
len = fread(content, 1, maxLen, fp);
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s, content is null", file);
goto PARSE_MINFOS_OVER;
@ -189,14 +189,14 @@ static int32_t dnodeReadMInfos() {
dError("failed to read mnodeEpSet.json, inUse not found");
goto PARSE_MINFOS_OVER;
}
tsMInfos.inUse = inUse->valueint;
tsMInfos.inUse = (int8_t)inUse->valueint;
cJSON *nodeNum = cJSON_GetObjectItem(root, "nodeNum");
if (!nodeNum || nodeNum->type != cJSON_Number) {
dError("failed to read mnodeEpSet.json, nodeNum not found");
goto PARSE_MINFOS_OVER;
}
minfos.mnodeNum = nodeNum->valueint;
minfos.mnodeNum = (int8_t)nodeNum->valueint;
cJSON *nodeInfos = cJSON_GetObjectItem(root, "nodeInfos");
if (!nodeInfos || nodeInfos->type != cJSON_Array) {
@ -204,13 +204,13 @@ static int32_t dnodeReadMInfos() {
goto PARSE_MINFOS_OVER;
}
int size = cJSON_GetArraySize(nodeInfos);
int32_t size = cJSON_GetArraySize(nodeInfos);
if (size != minfos.mnodeNum) {
dError("failed to read mnodeEpSet.json, nodeInfos size not matched");
goto PARSE_MINFOS_OVER;
}
for (int i = 0; i < size; ++i) {
for (int32_t i = 0; i < size; ++i) {
cJSON *nodeInfo = cJSON_GetArrayItem(nodeInfos, i);
if (nodeInfo == NULL) continue;
@ -227,7 +227,7 @@ static int32_t dnodeReadMInfos() {
}
SMInfo *pMinfo = &minfos.mnodeInfos[i];
pMinfo->mnodeId = nodeId->valueint;
pMinfo->mnodeId = (int32_t)nodeId->valueint;
tstrncpy(pMinfo->mnodeEp, nodeEp->valuestring, TSDB_EP_LEN);
bool changed = dnodeCheckEpChanged(pMinfo->mnodeId, pMinfo->mnodeEp);

4
src/dnode/src/dnodeMPeer.c
View File

@ -60,7 +60,7 @@ int32_t dnodeInitMPeer() {
void dnodeCleanupMPeer() {
for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerWP.worker + i;
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
taosQsetThreadResume(tsMPeerQset);
}
dDebug("dnode mpeer worker:%d is closed", i);
@ -69,7 +69,7 @@ void dnodeCleanupMPeer() {
for (int32_t i = 0; i < tsMPeerWP.maxNum; ++i) {
SMPeerWorker *pWorker = tsMPeerWP.worker + i;
dDebug("dnode mpeer worker:%d start to join", i);
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
pthread_join(pWorker->thread, NULL);
}
dDebug("dnode mpeer worker:%d join success", i);

6
src/dnode/src/dnodeMRead.c
View File

@ -40,7 +40,7 @@ static void *dnodeProcessMReadQueue(void *param);
int32_t dnodeInitMRead() {
tsMReadQset = taosOpenQset();
tsMReadWP.maxNum = tsNumOfCores * tsNumOfThreadsPerCore / 2;
tsMReadWP.maxNum = (int32_t)(tsNumOfCores * tsNumOfThreadsPerCore / 2);
tsMReadWP.maxNum = MAX(2, tsMReadWP.maxNum);
tsMReadWP.maxNum = MIN(4, tsMReadWP.maxNum);
tsMReadWP.curNum = 0;
@ -60,7 +60,7 @@ int32_t dnodeInitMRead() {
void dnodeCleanupMRead() {
for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadWP.worker + i;
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
taosQsetThreadResume(tsMReadQset);
}
dDebug("dnode mread worker:%d is closed", i);
@ -69,7 +69,7 @@ void dnodeCleanupMRead() {
for (int32_t i = 0; i < tsMReadWP.maxNum; ++i) {
SMReadWorker *pWorker = tsMReadWP.worker + i;
dDebug("dnode mread worker:%d start to join", i);
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
pthread_join(pWorker->thread, NULL);
}
dDebug("dnode mread worker:%d start to join", i);

4
src/dnode/src/dnodeMWrite.c
View File

@ -60,7 +60,7 @@ int32_t dnodeInitMWrite() {
void dnodeCleanupMWrite() {
for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWriteWP.worker + i;
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
taosQsetThreadResume(tsMWriteQset);
}
dDebug("dnode mwrite worker:%d is closed", i);
@ -69,7 +69,7 @@ void dnodeCleanupMWrite() {
for (int32_t i = 0; i < tsMWriteWP.maxNum; ++i) {
SMWriteWorker *pWorker = tsMWriteWP.worker + i;
dDebug("dnode mwrite worker:%d start to join", i);
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
pthread_join(pWorker->thread, NULL);
}
dDebug("dnode mwrite worker:%d join success", i);

5
src/dnode/src/dnodePeer.c
View File

@ -90,7 +90,10 @@ static void dnodeProcessReqMsgFromDnode(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
};
if (pMsg->pCont == NULL) return;
if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) return dnodeSendStartupStep(pMsg);
if (pMsg->msgType == TSDB_MSG_TYPE_NETWORK_TEST) {
dnodeSendStartupStep(pMsg);
return;
}
if (dnodeGetRunStatus() != TSDB_RUN_STATUS_RUNING) {
rspMsg.code = TSDB_CODE_APP_NOT_READY;

2
src/dnode/src/dnodeShell.c
View File

@ -70,7 +70,7 @@ int32_t dnodeInitShell() {
dnodeProcessShellMsgFp[TSDB_MSG_TYPE_NETWORK_TEST] = dnodeSendStartupStep;
int32_t numOfThreads = (tsNumOfCores * tsNumOfThreadsPerCore) / 2.0;
int32_t numOfThreads = (int32_t)((tsNumOfCores * tsNumOfThreadsPerCore) / 2.0);
if (numOfThreads < 1) {
numOfThreads = 1;
}

2
src/dnode/src/dnodeStep.c
View File

@ -70,5 +70,5 @@ int32_t dnodeStepInit(SStep *pSteps, int32_t stepSize) {
}
void dnodeStepCleanup(SStep *pSteps, int32_t stepSize) {
return taosStepCleanupImp(pSteps, stepSize - 1);
taosStepCleanupImp(pSteps, stepSize - 1);
}

80
src/dnode/src/dnodeSystem.c
View File

@ -19,8 +19,10 @@
#include "tconfig.h"
#include "dnodeMain.h"
static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
static tsem_t exitSem;
static void siguser1Handler(int32_t signum, void *sigInfo, void *context);
static void siguser2Handler(int32_t signum, void *sigInfo, void *context);
static void sigintHandler(int32_t signum, void *sigInfo, void *context);
int32_t main(int32_t argc, char *argv[]) {
int dump_config = 0;
@ -28,7 +30,7 @@ int32_t main(int32_t argc, char *argv[]) {
// Set global configuration file
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= TSDB_FILENAME_LEN) {
printf("config file path overflow");
exit(EXIT_FAILURE);
@ -80,8 +82,8 @@ int32_t main(int32_t argc, char *argv[]) {
taosSetRandomFileFailOutput(NULL);
}
} else if (strcmp(argv[i], "--random-file-fail-factor") == 0) {
if ( (i+1) < argc ) {
int factor = atoi(argv[i+1]);
if ((i + 1) < argc) {
int factor = atoi(argv[i + 1]);
printf("The factor of random failure is %d\n", factor);
taosSetRandomFileFailFactor(factor);
} else {
@ -93,7 +95,7 @@ int32_t main(int32_t argc, char *argv[]) {
}
if (0 != dump_config) {
tscEmbedded = 1;
tscEmbedded = 1;
taosInitGlobalCfg();
taosReadGlobalLogCfg();
@ -112,14 +114,13 @@ int32_t main(int32_t argc, char *argv[]) {
}
/* Set termination handler. */
struct sigaction act = {{0}};
act.sa_flags = SA_SIGINFO;
act.sa_sigaction = signal_handler;
sigaction(SIGTERM, &act, NULL);
sigaction(SIGHUP, &act, NULL);
sigaction(SIGINT, &act, NULL);
sigaction(SIGUSR1, &act, NULL);
sigaction(SIGUSR2, &act, NULL);
taosSetSignal(SIGUSR1, siguser1Handler);
taosSetSignal(SIGUSR2, siguser2Handler);
taosSetSignal(SIGTERM, sigintHandler);
taosSetSignal(SIGHUP, sigintHandler);
taosSetSignal(SIGINT, sigintHandler);
taosSetSignal(SIGABRT, sigintHandler);
taosSetSignal(SIGBREAK, sigintHandler);
// Open /var/log/syslog file to record information.
openlog("TDengine:", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_LOCAL1);
@ -144,37 +145,40 @@ int32_t main(int32_t argc, char *argv[]) {
syslog(LOG_INFO, "Shut down TDengine service successfully");
dInfo("TDengine is shut down!");
closelog();
#ifdef WINDOWS
tsem_post(&exitSem);
#endif
return EXIT_SUCCESS;
}
static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context) {
if (signum == SIGUSR1) {
taosCfgDynamicOptions("debugFlag 143");
return;
}
if (signum == SIGUSR2) {
taosCfgDynamicOptions("resetlog");
return;
}
static void siguser1Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("debugFlag 143"); }
static void siguser2Handler(int32_t signum, void *sigInfo, void *context) { taosCfgDynamicOptions("resetlog"); }
static void sigintHandler(int32_t signum, void *sigInfo, void *context) {
// protect the application from receive another signal
taosIgnSignal(SIGUSR1);
taosIgnSignal(SIGUSR2);
taosIgnSignal(SIGTERM);
taosIgnSignal(SIGHUP);
taosIgnSignal(SIGINT);
taosIgnSignal(SIGABRT);
taosIgnSignal(SIGBREAK);
// clean the system.
dInfo("shut down signal is %d", signum);
#ifndef WINDOWS
dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid));
#endif
syslog(LOG_INFO, "Shut down signal is %d", signum);
syslog(LOG_INFO, "Shutting down TDengine service...");
// clean the system.
#ifdef __APPLE__
dInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid);
#else // __APPLE__
dInfo("shut down signal is %d, sender PID:%d cmdline:%s", signum, sigInfo->si_pid, taosGetCmdlineByPID(sigInfo->si_pid));
#endif // __APPLE__
// protect the application from receive another signal
struct sigaction act = {{0}};
act.sa_handler = SIG_IGN;
sigaction(SIGTERM, &act, NULL);
sigaction(SIGHUP, &act, NULL);
sigaction(SIGINT, &act, NULL);
sigaction(SIGUSR1, &act, NULL);
sigaction(SIGUSR2, &act, NULL);
// inform main thread to exit
tsem_post(&exitSem);
}
#ifdef WINDOWS
tsem_wait(&exitSem);
#endif
}

23
src/dnode/src/dnodeTelemetry.c
View File

@ -93,14 +93,14 @@ static void addStringField(SBufferWriter* bw, const char* k, const char* v) {
static void addCpuInfo(SBufferWriter* bw) {
char * line = NULL;
size_t size = 0;
int done = 0;
int32_t done = 0;
FILE* fp = fopen("/proc/cpuinfo", "r");
if (fp == NULL) {
return;
}
while (done != 3 && (size = getline(&line, &size, fp)) != -1) {
while (done != 3 && (size = tgetline(&line, &size, fp)) != -1) {
line[size - 1] = '\0';
if (((done&1) == 0) && strncmp(line, "model name", 10) == 0) {
const char* v = strchr(line, ':') + 2;
@ -129,7 +129,7 @@ static void addOsInfo(SBufferWriter* bw) {
return;
}
while ((size = getline(&line, &size, fp)) != -1) {
while ((size = tgetline(&line, &size, fp)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "PRETTY_NAME", 11) == 0) {
const char* p = strchr(line, '=') + 1;
@ -155,7 +155,7 @@ static void addMemoryInfo(SBufferWriter* bw) {
return;
}
while ((size = getline(&line, &size, fp)) != -1) {
while ((size = tgetline(&line, &size, fp)) != -1) {
line[size - 1] = '\0';
if (strncmp(line, "MemTotal", 8) == 0) {
const char* p = strchr(line, ':') + 1;
@ -200,7 +200,7 @@ static void sendTelemetryReport() {
dTrace("failed to get IP address of " TELEMETRY_SERVER ", reason:%s", strerror(errno));
return;
}
int fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0);
SOCKET fd = taosOpenTcpClientSocket(ip, TELEMETRY_PORT, 0);
if (fd < 0) {
dTrace("failed to create socket for telemetry, reason:%s", strerror(errno));
return;
@ -222,10 +222,10 @@ static void sendTelemetryReport() {
"Content-Type: application/json\n"
"Content-Length: ";
taosWriteSocket(fd, header, strlen(header));
int contLen = tbufTell(&bw) - 1;
taosWriteSocket(fd, header, (int32_t)strlen(header));
int32_t contLen = (int32_t)(tbufTell(&bw) - 1);
sprintf(buf, "%d\n\n", contLen);
taosWriteSocket(fd, buf, strlen(buf));
taosWriteSocket(fd, buf, (int32_t)strlen(buf));
taosWriteSocket(fd, tbufGetData(&bw, false), contLen);
tbufCloseWriter(&bw);
@ -265,7 +265,7 @@ static void* telemetryThread(void* param) {
}
static void dnodeGetEmail(char* filepath) {
int fd = open(filepath, O_RDONLY);
int32_t fd = open(filepath, O_RDONLY);
if (fd < 0) {
return;
}
@ -274,10 +274,9 @@ static void dnodeGetEmail(char* filepath) {
dError("failed to read %d bytes from file %s since %s", TSDB_FQDN_LEN, filepath, strerror(errno));
}
close(fd);
taosClose(fd);
}
int32_t dnodeInitTelemetry() {
if (!tsEnableTelemetryReporting) {
return 0;
@ -310,7 +309,7 @@ void dnodeCleanupTelemetry() {
return;
}
if (tsTelemetryThread) {
if (taosCheckPthreadValid(tsTelemetryThread)) {
tsem_post(&tsExitSem);
pthread_join(tsTelemetryThread, NULL);
tsem_destroy(&tsExitSem);

4
src/dnode/src/dnodeVWrite.c
View File

@ -52,14 +52,14 @@ int32_t dnodeInitVWrite() {
void dnodeCleanupVWrite() {
for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
SVWriteWorker *pWorker = tsVWriteWP.worker + i;
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
taosQsetThreadResume(pWorker->qset);
}
}
for (int32_t i = 0; i < tsVWriteWP.max; ++i) {
SVWriteWorker *pWorker = tsVWriteWP.worker + i;
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
pthread_join(pWorker->thread, NULL);
taosFreeQall(pWorker->qall);
taosCloseQset(pWorker->qset);

4
src/dnode/src/dnodeVnodes.c
View File

@ -157,7 +157,7 @@ int32_t dnodeInitVnodes() {
int32_t failedVnodes = 0;
for (int32_t t = 0; t < threadNum; ++t) {
SOpenVnodeThread *pThread = &threads[t];
if (pThread->vnodeNum > 0 && pThread->thread) {
if (pThread->vnodeNum > 0 && taosCheckPthreadValid(pThread->thread)) {
pthread_join(pThread->thread, NULL);
}
openVnodes += pThread->opened;
@ -260,7 +260,7 @@ static void dnodeSendStatusMsg(void *handle, void *tmrId) {
tstrncpy(pStatus->clusterCfg.timezone, tsTimezone, 64);
pStatus->clusterCfg.checkTime = 0;
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
(void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
tstrncpy(pStatus->clusterCfg.locale, tsLocale, TSDB_LOCALE_LEN);
tstrncpy(pStatus->clusterCfg.charset, tsCharset, TSDB_LOCALE_LEN);

2
src/inc/taosdef.h
View File

@ -235,7 +235,9 @@ do { \
#define TSDB_MAX_REPLICA 5
#define TSDB_TBNAME_COLUMN_INDEX (-1)
#define TSDB_BLOCK_DIST_COLUMN_INDEX (-2)
#define TSDB_UD_COLUMN_INDEX (-100)
#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta
#define TSDB_MIN_CACHE_BLOCK_SIZE 1

27
src/inc/tsdb.h
View File

@ -114,6 +114,8 @@ void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_
char* tsdbGetTableName(void *pTable);
#define TSDB_TABLEID(_table) ((STableId*) (_table))
#define TSDB_PREV_ROW 0x1
#define TSDB_NEXT_ROW 0x2
STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
@ -141,7 +143,6 @@ typedef struct {
int64_t tableTotalDataSize; // In bytes
int64_t tableTotalDiskSize; // In bytes
} STableInfo;
STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid);
// -- FOR INSERT DATA
/**
@ -160,9 +161,10 @@ typedef void *TsdbQueryHandleT; // Use void to hide implementation details
// query condition to build vnode iterator
typedef struct STsdbQueryCond {
STimeWindow twindow;
int32_t order; // desc|asc order to iterate the data block
int32_t order; // desc|asc order to iterate the data block
int32_t numOfCols;
SColumnInfo *colList;
bool loadExternalRows; // load external rows or not
} STsdbQueryCond;
typedef struct SMemRef {
@ -232,13 +234,32 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
void *qinfo, SMemRef* pRef);
/**
* move to next block if exists
* get num of rows in mem table
*
* @param pHandle
* @return row size
*/
int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle);
/**
* move to next block if exists
*
* @param pQueryHandle
* @return
*/
bool tsdbNextDataBlock(TsdbQueryHandleT *pQueryHandle);
/**
* move to next block if exists but not merge data in memtable
*
* @param pQueryHandle
* @return
*/
bool tsdbNextDataBlockWithoutMerge(TsdbQueryHandleT *pQueryHandle);
SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t type);
/**
* Get current data block information

197
src/inc/ttokendef.h
View File

@ -16,6 +16,7 @@
#ifndef TDENGINE_TTOKENDEF_H
#define TDENGINE_TTOKENDEF_H
#define TK_ID 1
#define TK_BOOL 2
#define TK_TINYINT 3
@ -127,104 +128,104 @@
#define TK_SELECT 109
#define TK_UNION 110
#define TK_ALL 111
#define TK_FROM 112
#define TK_VARIABLE 113
#define TK_INTERVAL 114
#define TK_FILL 115
#define TK_SLIDING 116
#define TK_ORDER 117
#define TK_BY 118
#define TK_ASC 119
#define TK_DESC 120
#define TK_GROUP 121
#define TK_HAVING 122
#define TK_LIMIT 123
#define TK_OFFSET 124
#define TK_SLIMIT 125
#define TK_SOFFSET 126
#define TK_WHERE 127
#define TK_NOW 128
#define TK_RESET 129
#define TK_QUERY 130
#define TK_ADD 131
#define TK_COLUMN 132
#define TK_TAG 133
#define TK_CHANGE 134
#define TK_SET 135
#define TK_KILL 136
#define TK_CONNECTION 137
#define TK_STREAM 138
#define TK_COLON 139
#define TK_ABORT 140
#define TK_AFTER 141
#define TK_ATTACH 142
#define TK_BEFORE 143
#define TK_BEGIN 144
#define TK_CASCADE 145
#define TK_CLUSTER 146
#define TK_CONFLICT 147
#define TK_COPY 148
#define TK_DEFERRED 149
#define TK_DELIMITERS 150
#define TK_DETACH 151
#define TK_EACH 152
#define TK_END 153
#define TK_EXPLAIN 154
#define TK_FAIL 155
#define TK_FOR 156
#define TK_IGNORE 157
#define TK_IMMEDIATE 158
#define TK_INITIALLY 159
#define TK_INSTEAD 160
#define TK_MATCH 161
#define TK_KEY 162
#define TK_OF 163
#define TK_RAISE 164
#define TK_REPLACE 165
#define TK_RESTRICT 166
#define TK_ROW 167
#define TK_STATEMENT 168
#define TK_TRIGGER 169
#define TK_VIEW 170
#define TK_COUNT 171
#define TK_SUM 172
#define TK_AVG 173
#define TK_MIN 174
#define TK_MAX 175
#define TK_FIRST 176
#define TK_LAST 177
#define TK_TOP 178
#define TK_BOTTOM 179
#define TK_STDDEV 180
#define TK_PERCENTILE 181
#define TK_APERCENTILE 182
#define TK_LEASTSQUARES 183
#define TK_HISTOGRAM 184
#define TK_DIFF 185
#define TK_SPREAD 186
#define TK_TWA 187
#define TK_INTERP 188
#define TK_LAST_ROW 189
#define TK_RATE 190
#define TK_IRATE 191
#define TK_SUM_RATE 192
#define TK_SUM_IRATE 193
#define TK_AVG_RATE 194
#define TK_AVG_IRATE 195
#define TK_TBID 196
#define TK_SEMI 197
#define TK_NONE 198
#define TK_PREV 199
#define TK_LINEAR 200
#define TK_IMPORT 201
#define TK_METRIC 202
#define TK_TBNAME 203
#define TK_JOIN 204
#define TK_METRICS 205
#define TK_INSERT 206
#define TK_INTO 207
#define TK_VALUES 208
#define TK_DISTINCT 112
#define TK_FROM 113
#define TK_VARIABLE 114
#define TK_INTERVAL 115
#define TK_FILL 116
#define TK_SLIDING 117
#define TK_ORDER 118
#define TK_BY 119
#define TK_ASC 120
#define TK_DESC 121
#define TK_GROUP 122
#define TK_HAVING 123
#define TK_LIMIT 124
#define TK_OFFSET 125
#define TK_SLIMIT 126
#define TK_SOFFSET 127
#define TK_WHERE 128
#define TK_NOW 129
#define TK_RESET 130
#define TK_QUERY 131
#define TK_ADD 132
#define TK_COLUMN 133
#define TK_TAG 134
#define TK_CHANGE 135
#define TK_SET 136
#define TK_KILL 137
#define TK_CONNECTION 138
#define TK_STREAM 139
#define TK_COLON 140
#define TK_ABORT 141
#define TK_AFTER 142
#define TK_ATTACH 143
#define TK_BEFORE 144
#define TK_BEGIN 145
#define TK_CASCADE 146
#define TK_CLUSTER 147
#define TK_CONFLICT 148
#define TK_COPY 149
#define TK_DEFERRED 150
#define TK_DELIMITERS 151
#define TK_DETACH 152
#define TK_EACH 153
#define TK_END 154
#define TK_EXPLAIN 155
#define TK_FAIL 156
#define TK_FOR 157
#define TK_IGNORE 158
#define TK_IMMEDIATE 159
#define TK_INITIALLY 160
#define TK_INSTEAD 161
#define TK_MATCH 162
#define TK_KEY 163
#define TK_OF 164
#define TK_RAISE 165
#define TK_REPLACE 166
#define TK_RESTRICT 167
#define TK_ROW 168
#define TK_STATEMENT 169
#define TK_TRIGGER 170
#define TK_VIEW 171
#define TK_COUNT 172
#define TK_SUM 173
#define TK_AVG 174
#define TK_MIN 175
#define TK_MAX 176
#define TK_FIRST 177
#define TK_LAST 178
#define TK_TOP 179
#define TK_BOTTOM 180
#define TK_STDDEV 181
#define TK_PERCENTILE 182
#define TK_APERCENTILE 183
#define TK_LEASTSQUARES 184
#define TK_HISTOGRAM 185
#define TK_DIFF 186
#define TK_SPREAD 187
#define TK_TWA 188
#define TK_INTERP 189
#define TK_LAST_ROW 190
#define TK_RATE 191
#define TK_IRATE 192
#define TK_SUM_RATE 193
#define TK_SUM_IRATE 194
#define TK_AVG_RATE 195
#define TK_AVG_IRATE 196
#define TK_TBID 197
#define TK_SEMI 198
#define TK_NONE 199
#define TK_PREV 200
#define TK_LINEAR 201
#define TK_IMPORT 202
#define TK_METRIC 203
#define TK_TBNAME 204
#define TK_JOIN 205
#define TK_METRICS 206
#define TK_INSERT 207
#define TK_INTO 208
#define TK_VALUES 209
#define TK_SPACE 300

38
src/inc/ttype.h
View File

@ -45,6 +45,7 @@ typedef struct tstr {
case TSDB_DATA_TYPE_USMALLINT: \
(_v) = (_finalType)GET_UINT16_VAL(_data); \
break; \
case TSDB_DATA_TYPE_TIMESTAMP:\
case TSDB_DATA_TYPE_BIGINT: \
(_v) = (_finalType)(GET_INT64_VAL(_data)); \
break; \
@ -66,6 +67,43 @@ typedef struct tstr {
} \
} while (0)
#define SET_TYPED_DATA(_v, _type, _data) \
do { \
switch (_type) { \
case TSDB_DATA_TYPE_BOOL: \
case TSDB_DATA_TYPE_TINYINT: \
*(int8_t *)(_v) = (int8_t)(_data); \
break; \
case TSDB_DATA_TYPE_UTINYINT: \
*(uint8_t *)(_v) = (uint8_t)(_data); \
break; \
case TSDB_DATA_TYPE_SMALLINT: \
*(int16_t *)(_v) = (int16_t)(_data); \
break; \
case TSDB_DATA_TYPE_USMALLINT: \
*(uint16_t *)(_v) = (uint16_t)(_data); \
break; \
case TSDB_DATA_TYPE_BIGINT: \
*(int64_t *)(_v) = (int64_t)(_data); \
break; \
case TSDB_DATA_TYPE_UBIGINT: \
*(uint64_t *)(_v) = (uint64_t)(_data); \
break; \
case TSDB_DATA_TYPE_FLOAT: \
*(float *)(_v) = (float)(_data); \
break; \
case TSDB_DATA_TYPE_DOUBLE: \
*(double *)(_v) = (double)(_data); \
break; \
case TSDB_DATA_TYPE_UINT: \
*(uint32_t *)(_v) = (uint32_t)(_data); \
break; \
default: \
*(int32_t *)(_v) = (int32_t)(_data); \
break; \
} \
} while (0)
#define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT)
#define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT)
#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE)

12
src/kit/shell/src/shellMain.c
View File

@ -21,7 +21,7 @@
pthread_t pid;
static tsem_t cancelSem;
void shellQueryInterruptHandler(int signum) {
void shellQueryInterruptHandler(int32_t signum, void *sigInfo, void *context) {
tsem_post(&cancelSem);
}
@ -130,12 +130,10 @@ int main(int argc, char* argv[]) {
pthread_create(&spid, NULL, cancelHandler, NULL);
/* Interrupt handler. */
struct sigaction act;
memset(&act, 0, sizeof(struct sigaction));
act.sa_handler = shellQueryInterruptHandler;
sigaction(SIGTERM, &act, NULL);
sigaction(SIGINT, &act, NULL);
taosSetSignal(SIGTERM, shellQueryInterruptHandler);
taosSetSignal(SIGINT, shellQueryInterruptHandler);
taosSetSignal(SIGHUP, shellQueryInterruptHandler);
taosSetSignal(SIGABRT, shellQueryInterruptHandler);
/* Get grant information */
shellGetGrantInfo(con);

23
src/mnode/CMakeLists.txt
View File

@ -1,23 +1,10 @@
CMAKE_MINIMUM_REQUIRED(VERSION 3.5)
PROJECT(TDengine)
IF (TD_LINUX)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(mnode ${SRC})
ENDIF ()
IF (TD_DARWIN)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(mnode ${SRC})
ENDIF ()
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
ADD_LIBRARY(mnode ${SRC})

8
src/mnode/src/mnodeAcct.c
View File

@ -81,7 +81,7 @@ static int32_t mnodeAcctActionDecode(SSdbRow *pRow) {
}
static int32_t mnodeAcctActionRestored() {
int32_t numOfRows = sdbGetNumOfRows(tsAcctSdb);
int64_t numOfRows = sdbGetNumOfRows(tsAcctSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
mInfo("dnode first deploy, create root acct");
int32_t code = mnodeCreateRootAcct();
@ -97,14 +97,14 @@ static int32_t mnodeAcctActionRestored() {
int32_t mnodeInitAccts() {
SAcctObj tObj;
tsAcctUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsAcctUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
SSdbTableDesc desc = {
.id = SDB_TABLE_ACCOUNT,
.name = "accounts",
.hashSessions = TSDB_DEFAULT_ACCOUNTS_HASH_SIZE,
.maxRowSize = tsAcctUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_STRING,
.fpInsert = mnodeAcctActionInsert,
.fpDelete = mnodeAcctActionDelete,
@ -206,7 +206,7 @@ void mnodeDropUserFromAcct(SAcctObj *pAcct, SUserObj *pUser) {
}
static int32_t mnodeCreateRootAcct() {
int32_t numOfAccts = sdbGetNumOfRows(tsAcctSdb);
int64_t numOfAccts = sdbGetNumOfRows(tsAcctSdb);
if (numOfAccts != 0) return TSDB_CODE_SUCCESS;
SAcctObj *pAcct = malloc(sizeof(SAcctObj));

10
src/mnode/src/mnodeCluster.c
View File

@ -68,7 +68,7 @@ static int32_t mnodeClusterActionDecode(SSdbRow *pRow) {
}
static int32_t mnodeClusterActionRestored() {
int32_t numOfRows = sdbGetNumOfRows(tsClusterSdb);
int64_t numOfRows = sdbGetNumOfRows(tsClusterSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
mInfo("dnode first deploy, create cluster");
int32_t code = mnodeCreateCluster();
@ -84,14 +84,14 @@ static int32_t mnodeClusterActionRestored() {
int32_t mnodeInitCluster() {
SClusterObj tObj;
tsClusterUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsClusterUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
SSdbTableDesc desc = {
.id = SDB_TABLE_CLUSTER,
.name = "cluster",
.hashSessions = TSDB_DEFAULT_CLUSTER_HASH_SIZE,
.maxRowSize = tsClusterUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_STRING,
.fpInsert = mnodeClusterActionInsert,
.fpDelete = mnodeClusterActionDelete,
@ -147,7 +147,7 @@ bool taosGetSystemUid(char *uid) {
#endif // __APPLE__
static int32_t mnodeCreateCluster() {
int32_t numOfClusters = sdbGetNumOfRows(tsClusterSdb);
int64_t numOfClusters = sdbGetNumOfRows(tsClusterSdb);
if (numOfClusters != 0) return TSDB_CODE_SUCCESS;
SClusterObj *pCluster = malloc(sizeof(SClusterObj));
@ -234,7 +234,7 @@ static int32_t mnodeRetrieveClusters(SShowObj *pShow, char *data, int32_t rows,
cols++;
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
*(int32_t *) pWrite = pCluster->createdTime;
*(int64_t *) pWrite = pCluster->createdTime;
cols++;
mnodeDecClusterRef(pCluster);

8
src/mnode/src/mnodeDb.c
View File

@ -143,14 +143,14 @@ static int32_t mnodeDbActionRestored() {
int32_t mnodeInitDbs() {
SDbObj tObj;
tsDbUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsDbUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
SSdbTableDesc desc = {
.id = SDB_TABLE_DB,
.name = "dbs",
.hashSessions = TSDB_DEFAULT_DBS_HASH_SIZE,
.maxRowSize = tsDbUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_STRING,
.fpInsert = mnodeDbActionInsert,
.fpDelete = mnodeDbActionDelete,
@ -192,11 +192,11 @@ SDbObj *mnodeGetDb(char *db) {
}
void mnodeIncDbRef(SDbObj *pDb) {
return sdbIncRef(tsDbSdb, pDb);
sdbIncRef(tsDbSdb, pDb);
}
void mnodeDecDbRef(SDbObj *pDb) {
return sdbDecRef(tsDbSdb, pDb);
sdbDecRef(tsDbSdb, pDb);
}
SDbObj *mnodeGetDbByTableName(char *tableName) {

16
src/mnode/src/mnodeDnode.c
View File

@ -39,7 +39,7 @@
#include "mnodeCluster.h"
int32_t tsAccessSquence = 0;
int64_t tsDnodeRid = -1;
int64_t tsDnodeRid = -1;
static void * tsDnodeSdb = NULL;
static int32_t tsDnodeUpdateSize = 0;
extern void * tsMnodeSdb;
@ -148,7 +148,7 @@ static int32_t mnodeDnodeActionDecode(SSdbRow *pRow) {
}
static int32_t mnodeDnodeActionRestored() {
int32_t numOfRows = sdbGetNumOfRows(tsDnodeSdb);
int64_t numOfRows = sdbGetNumOfRows(tsDnodeSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
mInfo("dnode first deploy, create dnode:%s", tsLocalEp);
mnodeCreateDnode(tsLocalEp, NULL);
@ -165,7 +165,7 @@ static int32_t mnodeDnodeActionRestored() {
int32_t mnodeInitDnodes() {
SDnodeObj tObj;
tsDnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsDnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
pthread_mutex_init(&tsDnodeEpsMutex, NULL);
SSdbTableDesc desc = {
@ -173,7 +173,7 @@ int32_t mnodeInitDnodes() {
.name = "dnodes",
.hashSessions = TSDB_DEFAULT_DNODES_HASH_SIZE,
.maxRowSize = tsDnodeUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_AUTO,
.fpInsert = mnodeDnodeActionInsert,
.fpDelete = mnodeDnodeActionDelete,
@ -227,7 +227,7 @@ void mnodeCancelGetNextDnode(void *pIter) {
}
int32_t mnodeGetDnodesNum() {
return sdbGetNumOfRows(tsDnodeSdb);
return (int32_t)sdbGetNumOfRows(tsDnodeSdb);
}
int32_t mnodeGetOnlinDnodesCpuCoreNum() {
@ -407,7 +407,7 @@ static int32_t mnodeCheckClusterCfgPara(const SClusterCfg *clusterCfg) {
int64_t checkTime = 0;
char timestr[32] = "1970-01-01 00:00:00.00";
(void)taosParseTime(timestr, &checkTime, strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
(void)taosParseTime(timestr, &checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0);
if ((0 != strncasecmp(clusterCfg->timezone, tsTimezone, strlen(tsTimezone))) &&
(checkTime != clusterCfg->checkTime)) {
mError("\"timezone\"[%s - %s] [%" PRId64 " - %" PRId64 "] cfg parameters inconsistent", clusterCfg->timezone,
@ -638,9 +638,9 @@ static int32_t mnodeCreateDnode(char *ep, SMnodeMsg *pMsg) {
char *temp = strchr(dnodeEp, ':');
if (!temp) {
int len = strlen(dnodeEp);
int32_t len = (int32_t)strlen(dnodeEp);
if (dnodeEp[len - 1] == ';') dnodeEp[len - 1] = 0;
len = strlen(dnodeEp);
len = (int32_t)strlen(dnodeEp);
snprintf(dnodeEp + len, TSDB_EP_LEN - len, ":%d", tsServerPort);
}
ep = dnodeEp;

6
src/mnode/src/mnodeMnode.c
View File

@ -136,14 +136,14 @@ int32_t mnodeInitMnodes() {
mnodeMnodeInitLock();
SMnodeObj tObj;
tsMnodeUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsMnodeUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
SSdbTableDesc desc = {
.id = SDB_TABLE_MNODE,
.name = "mnodes",
.hashSessions = TSDB_DEFAULT_MNODES_HASH_SIZE,
.maxRowSize = tsMnodeUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_INT,
.fpInsert = mnodeMnodeActionInsert,
.fpDelete = mnodeMnodeActionDelete,
@ -176,7 +176,7 @@ void mnodeCleanupMnodes() {
}
int32_t mnodeGetMnodesNum() {
return sdbGetNumOfRows(tsMnodeSdb);
return (int32_t)sdbGetNumOfRows(tsMnodeSdb);
}
void *mnodeGetMnode(int32_t mnodeId) {

22
src/mnode/src/mnodeSdb.c
View File

@ -207,7 +207,7 @@ static void sdbRestoreTables() {
(*pTable->fpRestored)();
}
totalRows += pTable->numOfRows;
totalRows += (int32_t)pTable->numOfRows;
numOfTables++;
sdbInfo("vgId:1, sdb:%s is checked, rows:%" PRId64, pTable->name, pTable->numOfRows);
}
@ -475,7 +475,7 @@ void sdbIncRef(void *tparam, void *pRow) {
if (pRow == NULL || tparam == NULL) return;
SSdbTable *pTable = tparam;
int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos);
int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos);
int32_t refCount = atomic_add_fetch_32(pRefCount, 1);
sdbTrace("vgId:1, sdb:%s, inc ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount);
}
@ -484,11 +484,11 @@ void sdbDecRef(void *tparam, void *pRow) {
if (pRow == NULL || tparam == NULL) return;
SSdbTable *pTable = tparam;
int32_t * pRefCount = (int32_t *)(pRow + pTable->refCountPos);
int32_t * pRefCount = (int32_t *)((char *)pRow + pTable->refCountPos);
int32_t refCount = atomic_sub_fetch_32(pRefCount, 1);
sdbTrace("vgId:1, sdb:%s, dec ref to row:%p:%s:%d", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount);
int32_t *updateEnd = pRow + pTable->refCountPos - 4;
int32_t *updateEnd = (int32_t *)((char *)pRow + pTable->refCountPos - 4);
if (refCount <= 0 && *updateEnd) {
sdbTrace("vgId:1, sdb:%s, row:%p:%s:%d destroyed", pTable->name, pRow, sdbGetRowStr(pTable, pRow), refCount);
SSdbRow row = {.pObj = pRow};
@ -501,7 +501,7 @@ static void *sdbGetRowMeta(SSdbTable *pTable, void *key) {
int32_t keySize = sizeof(int32_t);
if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
keySize = strlen((char *)key);
keySize = (int32_t)strlen((char *)key);
}
void **ppRow = (void **)taosHashGet(pTable->iHandle, key, keySize);
@ -534,7 +534,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) {
int32_t keySize = sizeof(int32_t);
if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
keySize = strlen((char *)key);
keySize = (int32_t)strlen((char *)key);
}
pthread_mutex_lock(&pTable->mutex);
@ -564,7 +564,7 @@ static int32_t sdbInsertHash(SSdbTable *pTable, SSdbRow *pRow) {
}
static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) {
int32_t *updateEnd = pRow->pObj + pTable->refCountPos - 4;
int32_t *updateEnd = (int32_t *)((char*)pRow->pObj + pTable->refCountPos - 4);
bool set = atomic_val_compare_exchange_32(updateEnd, 0, 1) == 0;
if (!set) {
sdbError("vgId:1, sdb:%s, failed to delete key:%s from hash, for it already removed", pTable->name,
@ -577,7 +577,7 @@ static int32_t sdbDeleteHash(SSdbTable *pTable, SSdbRow *pRow) {
void * key = sdbGetObjKey(pTable, pRow->pObj);
int32_t keySize = sizeof(int32_t);
if (pTable->keyType == SDB_KEY_STRING || pTable->keyType == SDB_KEY_VAR_STRING) {
keySize = strlen((char *)key);
keySize = (int32_t)strlen((char *)key);
}
pthread_mutex_lock(&pTable->mutex);
@ -764,7 +764,7 @@ bool sdbCheckRowDeleted(void *tparam, void *pRow) {
SSdbTable *pTable = tparam;
if (pTable == NULL) return false;
int32_t *updateEnd = pRow + pTable->refCountPos - 4;
int32_t *updateEnd = (int32_t *)((char*)pRow + pTable->refCountPos - 4);
return atomic_val_compare_exchange_32(updateEnd, 1, 1) == 1;
}
@ -942,14 +942,14 @@ static int32_t sdbInitWorker() {
static void sdbCleanupWorker() {
for (int32_t i = 0; i < tsSdbPool.num; ++i) {
SSdbWorker *pWorker = tsSdbPool.worker + i;
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
taosQsetThreadResume(tsSdbWQset);
}
}
for (int32_t i = 0; i < tsSdbPool.num; ++i) {
SSdbWorker *pWorker = tsSdbPool.worker + i;
if (pWorker->thread) {
if (taosCheckPthreadValid(pWorker->thread)) {
pthread_join(pWorker->thread, NULL);
}
}

2
src/mnode/src/mnodeShow.c
View File

@ -218,7 +218,7 @@ static int32_t mnodeProcessRetrieveMsg(SMnodeMsg *pMsg) {
}
pRsp->numOfRows = htonl(rowsRead);
pRsp->precision = htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision
pRsp->precision = (int16_t)htonl(TSDB_TIME_PRECISION_MILLI); // millisecond time precision
pMsg->rpcRsp.rsp = pRsp;
pMsg->rpcRsp.len = size;

66
src/mnode/src/mnodeTable.c
View File

@ -222,23 +222,23 @@ static int32_t mnodeChildTableActionEncode(SSdbRow *pRow) {
SCTableObj *pTable = pRow->pObj;
assert(pTable != NULL && pRow->rowData != NULL);
int32_t len = strlen(pTable->info.tableId);
int32_t len = (int32_t)strlen(pTable->info.tableId);
if (len >= TSDB_TABLE_FNAME_LEN) return TSDB_CODE_MND_INVALID_TABLE_ID;
memcpy(pRow->rowData, pTable->info.tableId, len);
memset(pRow->rowData + len, 0, 1);
memset((char *)pRow->rowData + len, 0, 1);
len++;
memcpy(pRow->rowData + len, (char*)pTable + sizeof(char *), tsChildTableUpdateSize);
memcpy((char *)pRow->rowData + len, (char *)pTable + sizeof(char *), tsChildTableUpdateSize);
len += tsChildTableUpdateSize;
if (pTable->info.type != TSDB_CHILD_TABLE) {
int32_t schemaSize = pTable->numOfColumns * sizeof(SSchema);
memcpy(pRow->rowData + len, pTable->schema, schemaSize);
memcpy((char *)pRow->rowData + len, pTable->schema, schemaSize);
len += schemaSize;
if (pTable->sqlLen != 0) {
memcpy(pRow->rowData + len, pTable->sql, pTable->sqlLen);
memcpy((char *)pRow->rowData + len, pTable->sql, pTable->sqlLen);
len += pTable->sqlLen;
}
}
@ -253,7 +253,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) {
SCTableObj *pTable = calloc(1, sizeof(SCTableObj));
if (pTable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
int32_t len = strlen(pRow->rowData);
int32_t len = (int32_t)strlen(pRow->rowData);
if (len >= TSDB_TABLE_FNAME_LEN) {
free(pTable);
return TSDB_CODE_MND_INVALID_TABLE_ID;
@ -261,7 +261,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) {
pTable->info.tableId = strdup(pRow->rowData);
len++;
memcpy((char*)pTable + sizeof(char *), pRow->rowData + len, tsChildTableUpdateSize);
memcpy((char *)pTable + sizeof(char *), (char *)pRow->rowData + len, tsChildTableUpdateSize);
len += tsChildTableUpdateSize;
if (pTable->info.type != TSDB_CHILD_TABLE) {
@ -271,7 +271,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) {
mnodeDestroyChildTable(pTable);
return TSDB_CODE_MND_INVALID_TABLE_TYPE;
}
memcpy(pTable->schema, pRow->rowData + len, schemaSize);
memcpy(pTable->schema, (char *)pRow->rowData + len, schemaSize);
len += schemaSize;
if (pTable->sqlLen != 0) {
@ -280,7 +280,7 @@ static int32_t mnodeChildTableActionDecode(SSdbRow *pRow) {
mnodeDestroyChildTable(pTable);
return TSDB_CODE_MND_OUT_OF_MEMORY;
}
memcpy(pTable->sql, pRow->rowData + len, pTable->sqlLen);
memcpy(pTable->sql, (char *)pRow->rowData + len, pTable->sqlLen);
}
}
@ -352,14 +352,14 @@ static int32_t mnodeChildTableActionRestored() {
static int32_t mnodeInitChildTables() {
SCTableObj tObj;
tsChildTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
tsChildTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type);
SSdbTableDesc desc = {
.id = SDB_TABLE_CTABLE,
.name = "ctables",
.hashSessions = TSDB_DEFAULT_CTABLES_HASH_SIZE,
.maxRowSize = sizeof(SCTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN + TSDB_CQ_SQL_SIZE,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_VAR_STRING,
.fpInsert = mnodeChildTableActionInsert,
.fpDelete = mnodeChildTableActionDelete,
@ -501,18 +501,18 @@ static int32_t mnodeSuperTableActionEncode(SSdbRow *pRow) {
SSTableObj *pStable = pRow->pObj;
assert(pRow->pObj != NULL && pRow->rowData != NULL);
int32_t len = strlen(pStable->info.tableId);
int32_t len = (int32_t)strlen(pStable->info.tableId);
if (len >= TSDB_TABLE_FNAME_LEN) len = TSDB_CODE_MND_INVALID_TABLE_ID;
memcpy(pRow->rowData, pStable->info.tableId, len);
memset(pRow->rowData + len, 0, 1);
memset((char *)pRow->rowData + len, 0, 1);
len++;
memcpy(pRow->rowData + len, (char*)pStable + sizeof(char *), tsSuperTableUpdateSize);
memcpy((char *)pRow->rowData + len, (char *)pStable + sizeof(char *), tsSuperTableUpdateSize);
len += tsSuperTableUpdateSize;
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
memcpy(pRow->rowData + len, pStable->schema, schemaSize);
memcpy((char *)pRow->rowData + len, pStable->schema, schemaSize);
len += schemaSize;
pRow->rowSize = len;
@ -525,7 +525,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) {
SSTableObj *pStable = (SSTableObj *) calloc(1, sizeof(SSTableObj));
if (pStable == NULL) return TSDB_CODE_MND_OUT_OF_MEMORY;
int32_t len = strlen(pRow->rowData);
int32_t len = (int32_t)strlen(pRow->rowData);
if (len >= TSDB_TABLE_FNAME_LEN){
free(pStable);
return TSDB_CODE_MND_INVALID_TABLE_ID;
@ -533,7 +533,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) {
pStable->info.tableId = strdup(pRow->rowData);
len++;
memcpy((char*)pStable + sizeof(char *), pRow->rowData + len, tsSuperTableUpdateSize);
memcpy((char *)pStable + sizeof(char *), (char *)pRow->rowData + len, tsSuperTableUpdateSize);
len += tsSuperTableUpdateSize;
int32_t schemaSize = sizeof(SSchema) * (pStable->numOfColumns + pStable->numOfTags);
@ -543,8 +543,7 @@ static int32_t mnodeSuperTableActionDecode(SSdbRow *pRow) {
return TSDB_CODE_MND_NOT_SUPER_TABLE;
}
memcpy(pStable->schema, pRow->rowData + len, schemaSize);
memcpy(pStable->schema, (char *)pRow->rowData + len, schemaSize);
pRow->pObj = pStable;
return TSDB_CODE_SUCCESS;
@ -556,14 +555,14 @@ static int32_t mnodeSuperTableActionRestored() {
static int32_t mnodeInitSuperTables() {
SSTableObj tObj;
tsSuperTableUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type;
tsSuperTableUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj.info.type);
SSdbTableDesc desc = {
.id = SDB_TABLE_STABLE,
.name = "stables",
.hashSessions = TSDB_DEFAULT_STABLES_HASH_SIZE,
.maxRowSize = sizeof(SSTableObj) + sizeof(SSchema) * (TSDB_MAX_TAGS + TSDB_MAX_COLUMNS + 16) + TSDB_TABLE_FNAME_LEN,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_VAR_STRING,
.fpInsert = mnodeSuperTableActionInsert,
.fpDelete = mnodeSuperTableActionDelete,
@ -844,6 +843,7 @@ static int32_t mnodeProcessBatchCreateTableMsg(SMnodeMsg *pMsg) {
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
} else { // batch master replay, reprocess the whole batch
assert(0);
return TSDB_CODE_MND_MSG_NOT_PROCESSED;
}
}
}
@ -1257,7 +1257,7 @@ static int32_t mnodeModifySuperTableTagName(SMnodeMsg *pMsg, char *oldTagName, c
}
// int32_t rowSize = 0;
uint32_t len = strlen(newTagName);
uint32_t len = (int32_t)strlen(newTagName);
if (len >= TSDB_COL_NAME_LEN) {
return TSDB_CODE_MND_COL_NAME_TOO_LONG;
}
@ -1420,7 +1420,7 @@ static int32_t mnodeChangeSuperTableColumn(SMnodeMsg *pMsg, char *oldName, char
}
// int32_t rowSize = 0;
uint32_t len = strlen(newName);
uint32_t len = (uint32_t)strlen(newName);
if (len >= TSDB_COL_NAME_LEN) {
return TSDB_CODE_MND_COL_NAME_TOO_LONG;
}
@ -1525,7 +1525,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
tstrncpy(prefix, pDb->name, 64);
strcat(prefix, TS_PATH_DELIMITER);
prefixLen = strlen(prefix);
prefixLen = (int32_t)strlen(prefix);
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char stableName[TSDB_TABLE_NAME_LEN] = {0};
@ -1550,7 +1550,7 @@ int32_t mnodeRetrieveShowSuperTables(SShowObj *pShow, char *data, int32_t rows,
pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
int16_t len = strnlen(stableName, TSDB_TABLE_NAME_LEN - 1);
int16_t len = (int16_t)strnlen(stableName, TSDB_TABLE_NAME_LEN - 1);
*(int16_t*) pWrite = len;
pWrite += sizeof(int16_t); // todo refactor
@ -1593,7 +1593,7 @@ void mnodeDropAllSuperTables(SDbObj *pDropDb) {
char prefix[64] = {0};
tstrncpy(prefix, pDropDb->name, 64);
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = strlen(prefix);
int32_t prefixLen = (int32_t)strlen(prefix);
mInfo("db:%s, all super tables will be dropped from sdb", pDropDb->name);
@ -1746,9 +1746,9 @@ static int32_t mnodeProcessSuperTableVgroupMsg(SMnodeMsg *pMsg) {
rpcFreeCont(pRsp);
return TSDB_CODE_MND_INVALID_TABLE_NAME;
} else {
pRsp->numOfTables = htonl(pRsp->numOfTables);
pRsp->numOfTables = (int32_t)htonl(pRsp->numOfTables);
pMsg->rpcRsp.rsp = pRsp;
pMsg->rpcRsp.len = msg - (char *)pRsp;
pMsg->rpcRsp.len = (int32_t)((char *)msg - (char *)pRsp);
return TSDB_CODE_SUCCESS;
}
@ -2021,7 +2021,7 @@ static int32_t mnodeDoCreateChildTable(SMnodeMsg *pMsg, int32_t tid) {
static int32_t mnodeProcessCreateChildTableMsg(SMnodeMsg *pMsg) {
//SCMCreateTableMsg* p1 = pMsg->rpcMsg.pCont; // there are several tables here.
SCreateTableMsg* pCreate = (SCreateTableMsg*)(pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg));
SCreateTableMsg* pCreate = (SCreateTableMsg*)((char *)pMsg->rpcMsg.pCont + sizeof(SCMCreateTableMsg));
int32_t code = grantCheck(TSDB_GRANT_TIMESERIES);
if (code != TSDB_CODE_SUCCESS) {
@ -2278,7 +2278,7 @@ static int32_t mnodeChangeNormalTableColumn(SMnodeMsg *pMsg, char *oldName, char
}
// int32_t rowSize = 0;
uint32_t len = strlen(newName);
uint32_t len = (uint32_t)strlen(newName);
if (len >= TSDB_COL_NAME_LEN) {
return TSDB_CODE_MND_COL_NAME_TOO_LONG;
}
@ -2481,7 +2481,7 @@ void mnodeDropAllChildTables(SDbObj *pDropDb) {
char prefix[64] = {0};
tstrncpy(prefix, pDropDb->name, 64);
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = strlen(prefix);
int32_t prefixLen = (int32_t)strlen(prefix);
mInfo("db:%s, all child tables will be dropped from sdb", pDropDb->name);
@ -2897,7 +2897,7 @@ static int32_t mnodeRetrieveShowTables(SShowObj *pShow, char *data, int32_t rows
SPatternCompareInfo info = PATTERN_COMPARE_INFO_INITIALIZER;
char prefix[64] = {0};
int32_t prefixLen = tableIdPrefix(pDb->name, prefix, 64);
int32_t prefixLen = (int32_t)tableIdPrefix(pDb->name, prefix, 64);
char* pattern = NULL;
if (pShow->payloadLen > 0) {
@ -3133,7 +3133,7 @@ static int32_t mnodeRetrieveStreamTables(SShowObj *pShow, char *data, int32_t ro
char prefix[64] = {0};
tstrncpy(prefix, pDb->name, 64);
strcat(prefix, TS_PATH_DELIMITER);
int32_t prefixLen = strlen(prefix);
int32_t prefixLen = (int32_t)strlen(prefix);
while (numOfRows < rows) {
pShow->pIter = mnodeGetNextChildTable(pShow->pIter, &pTable);

12
src/mnode/src/mnodeUser.c
View File

@ -128,7 +128,7 @@ static void mnodePrintUserAuth() {
}
static int32_t mnodeUserActionRestored() {
int32_t numOfRows = sdbGetNumOfRows(tsUserSdb);
int64_t numOfRows = sdbGetNumOfRows(tsUserSdb);
if (numOfRows <= 0 && dnodeIsFirstDeploy()) {
mInfo("dnode first deploy, create root user");
SAcctObj *pAcct = mnodeGetAcct(TSDB_DEFAULT_USER);
@ -148,14 +148,14 @@ static int32_t mnodeUserActionRestored() {
int32_t mnodeInitUsers() {
SUserObj tObj;
tsUserUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsUserUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
SSdbTableDesc desc = {
.id = SDB_TABLE_USER,
.name = "users",
.hashSessions = TSDB_DEFAULT_USERS_HASH_SIZE,
.maxRowSize = tsUserUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_STRING,
.fpInsert = mnodeUserActionInsert,
.fpDelete = mnodeUserActionDelete,
@ -204,11 +204,11 @@ void mnodeCancelGetNextUser(void *pIter) {
}
void mnodeIncUserRef(SUserObj *pUser) {
return sdbIncRef(tsUserSdb, pUser);
sdbIncRef(tsUserSdb, pUser);
}
void mnodeDecUserRef(SUserObj *pUser) {
return sdbDecRef(tsUserSdb, pUser);
sdbDecRef(tsUserSdb, pUser);
}
static int32_t mnodeUpdateUser(SUserObj *pUser, void *pMsg) {
@ -561,7 +561,7 @@ static int32_t mnodeProcessDropUserMsg(SMnodeMsg *pMsg) {
void mnodeDropAllUsers(SAcctObj *pAcct) {
void * pIter = NULL;
int32_t numOfUsers = 0;
int32_t acctNameLen = strlen(pAcct->user);
int32_t acctNameLen = (int32_t)strlen(pAcct->user);
SUserObj *pUser = NULL;
while (1) {

8
src/mnode/src/mnodeVgroup.c
View File

@ -206,14 +206,14 @@ static int32_t mnodeVgroupActionRestored() {
int32_t mnodeInitVgroups() {
SVgObj tObj;
tsVgUpdateSize = (int8_t *)tObj.updateEnd - (int8_t *)&tObj;
tsVgUpdateSize = (int32_t)((int8_t *)tObj.updateEnd - (int8_t *)&tObj);
SSdbTableDesc desc = {
.id = SDB_TABLE_VGROUP,
.name = "vgroups",
.hashSessions = TSDB_DEFAULT_VGROUPS_HASH_SIZE,
.maxRowSize = tsVgUpdateSize,
.refCountPos = (int8_t *)(&tObj.refCount) - (int8_t *)&tObj,
.refCountPos = (int32_t)((int8_t *)(&tObj.refCount) - (int8_t *)&tObj),
.keyType = SDB_KEY_AUTO,
.fpInsert = mnodeVgroupActionInsert,
.fpDelete = mnodeVgroupActionDelete,
@ -245,11 +245,11 @@ int32_t mnodeInitVgroups() {
}
void mnodeIncVgroupRef(SVgObj *pVgroup) {
return sdbIncRef(tsVgroupSdb, pVgroup);
sdbIncRef(tsVgroupSdb, pVgroup);
}
void mnodeDecVgroupRef(SVgObj *pVgroup) {
return sdbDecRef(tsVgroupSdb, pVgroup);
sdbDecRef(tsVgroupSdb, pVgroup);
}
SVgObj *mnodeGetVgroup(int32_t vgId) {

1
src/os/inc/os.h
View File

@ -62,6 +62,7 @@ extern "C" {
#include "osMemory.h"
#include "osRand.h"
#include "osSemphone.h"
#include "osSignal.h"
#include "osSocket.h"
#include "osString.h"
#include "osSysinfo.h"

9
src/os/inc/osDarwin.h
View File

@ -106,10 +106,11 @@ int64_t tsosStr2int64(char *str);
void taos_block_sigalrm(void);
#define TAOS_OS_DEF_EPOLL
#define TAOS_EPOLL_WAIT_TIME 500
typedef int32_t SOCKET;
typedef SOCKET EpollFd;
#define EpollClose(pollFd) epoll_close(pollFd)
#ifdef __cplusplus
}

9
src/os/inc/osDef.h
View File

@ -20,6 +20,12 @@
extern "C" {
#endif
#ifndef WINDOWS
#ifndef O_BINARY
#define O_BINARY 0
#endif
#endif
#ifndef STDERR_FILENO
#define STDERR_FILENO (2)
#endif
@ -94,8 +100,7 @@ extern "C" {
#elif defined(__GNUC__) && !defined(threadlocal)
#define threadlocal __thread
#else
// #define threadlocal
#error please follow with the thread-local implementation on the target platform
#define threadlocal __declspec( thread )
#endif
#ifdef __cplusplus

4
src/os/inc/osFile.h
View File

@ -20,6 +20,8 @@
extern "C" {
#endif
#include "osSocket.h"
int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
@ -37,7 +39,7 @@ int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstP
}
// TAOS_OS_FUNC_FILE_SENDIFLE
int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size);
int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size);
int64_t taosFSendFile(FILE *outfile, FILE *infile, int64_t *offset, int64_t size);
#ifdef TAOS_RANDOM_FILE_FAIL

60
src/os/inc/osSignal.h Normal file
View File

@ -0,0 +1,60 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_OS_SIGNAL_H
#define TDENGINE_OS_SIGNAL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "taosdef.h"
#include <signal.h>
#ifndef SIGALRM
#define SIGALRM 1234
#endif
#ifndef SIGHUP
#define SIGHUP 1230
#endif
#ifndef SIGCHLD
#define SIGCHLD 1234
#endif
#ifndef SIGUSR1
#define SIGUSR1 1234
#endif
#ifndef SIGUSR2
#define SIGUSR2 1234
#endif
#ifndef SIGBREAK
#define SIGBREAK 1234
#endif
typedef void (*FSignalHandler)(int32_t signum, void *sigInfo, void *context);
void taosSetSignal(int32_t signum, FSignalHandler sigfp);
void taosIgnSignal(int32_t signum);
void taosDflSignal(int32_t signum);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TTIME_H

5
src/os/inc/osSocket.h
View File

@ -33,11 +33,13 @@ extern "C" {
x = FD_INITIALIZER; \
} \
}
typedef int32_t SOCKET;
#endif
#ifndef TAOS_OS_DEF_EPOLL
#define TAOS_EPOLL_WAIT_TIME 500
typedef int32_t SOCKET;
typedef SOCKET EpollFd;
#define EpollClose(pollFd) taosCloseSocket(pollFd)
#endif
#ifdef TAOS_RANDOM_NETWORK_FAIL
@ -61,6 +63,7 @@ extern "C" {
int32_t taosSetNonblocking(SOCKET sock, int32_t on);
void taosIgnSIGPIPE();
void taosBlockSIGPIPE();
void taosSetMaskSIGPIPE();
// TAOS_OS_FUNC_SOCKET_SETSOCKETOPT
int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen);

20
src/os/inc/osWindows.h
View File

@ -93,6 +93,12 @@ typedef SOCKET eventfd_t;
#define TAOS_OS_DEF_EPOLL
#define TAOS_EPOLL_WAIT_TIME 100
typedef SOCKET EpollFd;
#define EpollClose(pollFd) epoll_close(pollFd)
#ifndef EPOLLWAKEUP
#define EPOLLWAKEUP (1u << 29)
#endif
#define TAOS_OS_DEF_ZU
#define PRIzu "ld"
@ -191,13 +197,7 @@ int gettimeofday(struct timeval *ptv, void *pTimeZone);
#define PATH_MAX 256
#endif
//for signal, not dispose
#define SIGALRM 1234
typedef int sigset_t;
struct sigaction {
void (*sa_handler)(int);
};
int sigaction(int, struct sigaction *, void *);
#define TAOS_OS_FUNC_SIGNAL
typedef struct {
int we_wordc;
@ -208,6 +208,12 @@ typedef struct {
int wordexp(const char *words, wordexp_t *pwordexp, int flags);
void wordfree(wordexp_t *pwordexp);
#define openlog(a, b, c)
#define closelog()
#define LOG_ERR 0
#define LOG_INFO 1
void syslog(int unused, const char *format, ...);
#define TAOS_OS_FUNC_ATOMIC
#define atomic_load_8(ptr) (*(char volatile*)(ptr))
#define atomic_load_16(ptr) (*(short volatile*)(ptr))

38
src/os/src/darwin/darwinFile.c
View File

@ -51,7 +51,37 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co
return writeLen;
}
int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) {
uError("taosSendFile not implemented yet");
return -1;
}
int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t* offset, int64_t count) {
lseek(sfd, (int32_t)(*offset), 0);
int64_t writeLen = 0;
uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_);
if (rlen <= 0) {
return writeLen;
}
else if (rlen < _SEND_FILE_STEP_) {
taosWriteSocket(dfd, buffer, rlen);
return (int64_t)(writeLen + rlen);
}
else {
taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_);
writeLen += _SEND_FILE_STEP_;
}
}
int64_t remain = count - writeLen;
if (remain > 0) {
int32_t rlen = read(sfd, buffer, (int32_t)remain);
if (rlen <= 0) {
return writeLen;
}
else {
taosWriteSocket(sfd, buffer, (int32_t)remain);
writeLen += remain;
}
}
return writeLen;
}

4
src/os/src/darwin/darwinSysInfo.c
View File

@ -101,4 +101,6 @@ int taosSystem(const char *cmd) {
return -1;
}
void taosSetCoreDump() {}
void taosSetCoreDump() {}
char *taosGetCmdlineByPID(int pid) { return ""; }

5
src/os/src/detail/osDir.c
View File

@ -54,6 +54,9 @@ int taosMkDir(const char *path, mode_t mode) {
void taosRename(char* oldName, char *newName) {
// if newName in not empty, rename return fail.
// the newName must be empty or does not exist
#ifdef WINDOWS
remove(newName);
#endif
if (rename(oldName, newName)) {
uError("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno));
} else {
@ -117,7 +120,7 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
goto cmp_end;
}
int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
int32_t fd = open(destFileName, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO);
if (fd < 0) {
ret = -2;
goto cmp_end;

6
src/os/src/detail/osFile.c
View File

@ -121,15 +121,11 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
#ifndef TAOS_OS_FUNC_FILE_SENDIFLE
int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t *offset, int64_t size) {
int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) {
int64_t leftbytes = size;
int64_t sentbytes;
while (leftbytes > 0) {
/*
* TODO : Think to check if file is larger than 1GB
*/
// if (leftbytes > 1000000000) leftbytes = 1000000000;
sentbytes = sendfile(dfd, sfd, offset, leftbytes);
if (sentbytes == -1) {
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK) {

44
src/os/src/detail/osSignal.c Normal file
View File

@ -0,0 +1,44 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "tconfig.h"
#include "tglobal.h"
#include "tulog.h"
#ifndef TAOS_OS_FUNC_SIGNAL
typedef void (*FLinuxSignalHandler)(int32_t signum, siginfo_t *sigInfo, void *context);
void taosSetSignal(int32_t signum, FSignalHandler sigfp) {
struct sigaction act = {{0}};
#if 1
act.sa_flags = SA_SIGINFO;
act.sa_sigaction = (FLinuxSignalHandler)sigfp;
#else
act.sa_handler = sigfp;
#endif
sigaction(signum, &act, NULL);
}
void taosIgnSignal(int32_t signum) {
signal(signum, SIG_IGN);
}
void taosDflSignal(int32_t signum) {
signal(signum, SIG_DFL);
}
#endif

10
src/os/src/detail/osSocket.c
View File

@ -53,6 +53,16 @@ void taosBlockSIGPIPE() {
}
}
void taosSetMaskSIGPIPE() {
sigset_t signal_mask;
sigemptyset(&signal_mask);
sigaddset(&signal_mask, SIGPIPE);
int32_t rc = pthread_sigmask(SIG_SETMASK, &signal_mask, NULL);
if (rc != 0) {
uError("failed to setmask SIGPIPE");
}
}
#endif
#ifndef TAOS_OS_FUNC_SOCKET_SETSOCKETOPT

117
src/os/src/detail/osSysinfo.c
View File

@ -45,6 +45,21 @@ static char tsProcMemFile[25] = {0};
static char tsProcIOFile[25] = {0};
static float tsPageSizeKB = 0;
static void taosGetProcInfos() {
tsPageSize = sysconf(_SC_PAGESIZE);
tsOpenMax = sysconf(_SC_OPEN_MAX);
tsStreamMax = sysconf(_SC_STREAM_MAX);
tsProcId = (pid_t)syscall(SYS_gettid);
tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024;
snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId);
snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId);
snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId);
}
static int32_t taosGetTotalMemory() { return (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024); }
bool taosGetSysMemory(float *memoryUsedMB) {
float memoryAvailMB = (float)sysconf(_SC_AVPHYS_PAGES) * tsPageSizeKB / 1024;
*memoryUsedMB = (float)tsTotalMemoryMB - memoryAvailMB;
@ -105,7 +120,8 @@ static bool taosGetSysCpuInfo(SysCpuInfo *cpuInfo) {
}
char cpu[10] = {0};
sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system, &cpuInfo->idle);
sscanf(line, "%s %" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, cpu, &cpuInfo->user, &cpuInfo->nice, &cpuInfo->system,
&cpuInfo->idle);
tfree(line);
fclose(fp);
@ -131,7 +147,8 @@ static bool taosGetProcCpuInfo(ProcCpuInfo *cpuInfo) {
for (int i = 0, blank = 0; line[i] != 0; ++i) {
if (line[i] == ' ') blank++;
if (blank == PROCESS_ITEM) {
sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime, &cpuInfo->cutime, &cpuInfo->cstime);
sscanf(line + i + 1, "%" PRIu64 " %" PRIu64 " %" PRIu64 " %" PRIu64, &cpuInfo->utime, &cpuInfo->stime,
&cpuInfo->cutime, &cpuInfo->cstime);
break;
}
}
@ -162,12 +179,12 @@ static void taosGetSystemTimezone() {
char buf[68] = {0};
if (f != NULL) {
int len = fread(buf, 64, 1, f);
if(len < 64 && ferror(f)) {
if (len < 64 && ferror(f)) {
fclose(f);
uError("read /etc/timezone error, reason:%s", strerror(errno));
return;
}
fclose(f);
buf[sizeof(buf) - 1] = 0;
@ -258,6 +275,8 @@ static void taosGetSystemLocale() { // get and set default locale
}
}
static int32_t taosGetCpuCores() { return (int32_t)sysconf(_SC_NPROCESSORS_ONLN); }
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
static uint64_t lastSysUsed = 0;
static uint64_t lastSysTotal = 0;
@ -300,11 +319,9 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
bool taosGetDisk() {
struct statvfs info;
const double unit = 1024 * 1024 * 1024;
if (tscEmbedded) {
if (statvfs(tsDataDir, &info)) {
//tsTotalDataDirGB = 0;
//tsAvailDataDirGB = 0;
uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
return false;
} else {
@ -314,8 +331,6 @@ bool taosGetDisk() {
}
if (statvfs(tsLogDir, &info)) {
//tsTotalLogDirGB = 0;
//tsAvailLogDirGB = 0;
uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno));
return false;
} else {
@ -324,8 +339,6 @@ bool taosGetDisk() {
}
if (statvfs("/tmp", &info)) {
//tsTotalTmpDirGB = 0;
//tsAvailTmpDirectorySpace = 0;
uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno));
return false;
} else {
@ -344,13 +357,12 @@ static bool taosGetCardInfo(int64_t *bytes) {
return false;
}
size_t len = 2048;
char * line = calloc(1, len);
while (!feof(fp)) {
memset(line, 0, len);
int64_t rbytes = 0;
int64_t rpackts = 0;
int64_t tbytes = 0;
@ -465,7 +477,7 @@ bool taosGetProcIO(float *readKB, float *writeKB) {
static int64_t lastReadbyte = -1;
static int64_t lastWritebyte = -1;
int64_t curReadbyte = 0;
int64_t curReadbyte = 0;
int64_t curWritebyte = 0;
if (!taosReadProcIO(&curReadbyte, &curWritebyte)) {
@ -490,18 +502,10 @@ bool taosGetProcIO(float *readKB, float *writeKB) {
}
void taosGetSystemInfo() {
tsNumOfCores = (int32_t)sysconf(_SC_NPROCESSORS_ONLN);
tsPageSize = sysconf(_SC_PAGESIZE);
tsOpenMax = sysconf(_SC_OPEN_MAX);
tsStreamMax = sysconf(_SC_STREAM_MAX);
taosGetProcInfos();
tsProcId = (pid_t)syscall(SYS_gettid);
tsPageSizeKB = (float)(sysconf(_SC_PAGESIZE)) / 1024;
tsTotalMemoryMB = (int32_t)((float)sysconf(_SC_PHYS_PAGES) * tsPageSizeKB / 1024);
snprintf(tsProcMemFile, 25, "/proc/%d/status", tsProcId);
snprintf(tsProcCpuFile, 25, "/proc/%d/stat", tsProcId);
snprintf(tsProcIOFile, 25, "/proc/%d/io", tsProcId);
tsNumOfCores = taosGetCpuCores();
tsTotalMemoryMB = taosGetTotalMemory();
float tmp1, tmp2;
taosGetSysMemory(&tmp1);
@ -573,16 +577,16 @@ void taosSetCoreDump() {
if (0 == tsEnableCoreFile) {
return;
}
// 1. set ulimit -c unlimited
struct rlimit rlim;
struct rlimit rlim_new;
if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
#ifndef _ALPINE
#ifndef _ALPINE
uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
#else
#else
uInfo("the old unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max);
#endif
#endif
rlim_new.rlim_cur = RLIM_INFINITY;
rlim_new.rlim_max = RLIM_INFINITY;
if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
@ -594,57 +598,56 @@ void taosSetCoreDump() {
}
if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
#ifndef _ALPINE
#ifndef _ALPINE
uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
#else
#else
uInfo("the new unlimited para: rlim_cur=%llu, rlim_max=%llu", rlim.rlim_cur, rlim.rlim_max);
#endif
#endif
}
#ifndef _TD_ARM_
// 2. set the path for saving core file
struct __sysctl_args args;
int old_usespid = 0;
size_t old_len = 0;
int new_usespid = 1;
size_t new_len = sizeof(new_usespid);
int old_usespid = 0;
size_t old_len = 0;
int new_usespid = 1;
size_t new_len = sizeof(new_usespid);
int name[] = {CTL_KERN, KERN_CORE_USES_PID};
memset(&args, 0, sizeof(struct __sysctl_args));
args.name = name;
args.nlen = sizeof(name)/sizeof(name[0]);
args.oldval = &old_usespid;
args.name = name;
args.nlen = sizeof(name) / sizeof(name[0]);
args.oldval = &old_usespid;
args.oldlenp = &old_len;
args.newval = &new_usespid;
args.newlen = new_len;
args.newval = &new_usespid;
args.newlen = new_len;
old_len = sizeof(old_usespid);
if (syscall(SYS__sysctl, &args) == -1) {
uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
}
uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
old_usespid = 0;
old_len = 0;
old_len = 0;
memset(&args, 0, sizeof(struct __sysctl_args));
args.name = name;
args.nlen = sizeof(name)/sizeof(name[0]);
args.oldval = &old_usespid;
args.name = name;
args.nlen = sizeof(name) / sizeof(name[0]);
args.oldval = &old_usespid;
args.oldlenp = &old_len;
old_len = sizeof(old_usespid);
if (syscall(SYS__sysctl, &args) == -1) {
uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
}
uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
#endif
}
bool taosGetSystemUid(char *uid) {

20
src/os/src/detail/osTime.c
View File

@ -43,7 +43,7 @@
*/
int64_t user_mktime64(const unsigned int year0, const unsigned int mon0,
const unsigned int day, const unsigned int hour,
const unsigned int min, const unsigned int sec)
const unsigned int min, const unsigned int sec, int64_t timezone)
{
unsigned int mon = mon0, year = year0;
@ -61,12 +61,6 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0,
res = res*24;
res = ((res + hour) * 60 + min) * 60 + sec;
#ifdef _MSC_VER
#if _MSC_VER >= 1900
int64_t timezone = _timezone;
#endif
#endif
return (res + timezone);
}
@ -219,7 +213,7 @@ int32_t parseTimeWithTz(char* timestr, int64_t* time, int32_t timePrec) {
/* mktime will be affected by TZ, set by using taos_options */
#ifdef WINDOWS
int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, 0);
//int64_t seconds = gmtime(&tm);
#else
int64_t seconds = timegm(&tm);
@ -276,7 +270,13 @@ int32_t parseLocaltime(char* timestr, int64_t* time, int32_t timePrec) {
return -1;
}
int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec);
#ifdef _MSC_VER
#if _MSC_VER >= 1900
int64_t timezone = _timezone;
#endif
#endif
int64_t seconds = user_mktime64(tm.tm_year+1900, tm.tm_mon+1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec, timezone);
int64_t fraction = 0;
@ -574,4 +574,4 @@ const char* fmtts(int64_t ts) {
}
return buf;
}
}

15
src/os/src/linux/linuxEnv.c
View File

@ -18,7 +18,6 @@
#include "tglobal.h"
void osInit() {
#ifdef _TD_POWER_
if (configDir[0] == 0) {
strcpy(configDir, "/etc/power");
@ -43,16 +42,14 @@ void osInit() {
char cmdline[1024];
char *taosGetCmdlineByPID(int pid)
{
sprintf(cmdline, "/proc/%d/cmdline",pid);
FILE* f = fopen(cmdline,"r");
if(f){
char* taosGetCmdlineByPID(int pid) {
sprintf(cmdline, "/proc/%d/cmdline", pid);
FILE* f = fopen(cmdline, "r");
if (f) {
size_t size;
size = fread(cmdline, sizeof(char), 1024, f);
if(size>0){
if('\n'==cmdline[size-1])
cmdline[size-1]='\0';
if (size > 0) {
if ('\n' == cmdline[size - 1]) cmdline[size - 1] = '\0';
}
fclose(f);
}

127
src/os/src/windows/wFile.c
View File

@ -15,18 +15,19 @@
#define _DEFAULT_SOURCE
#include "os.h"
#include "tulog.h"
#include "osSocket.h"
#include "tglobal.h"
#include "tulog.h"
void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
const char* tdengineTmpFileNamePrefix = "tdengine-";
char tmpPath[PATH_MAX];
const char *tdengineTmpFileNamePrefix = "tdengine-";
char tmpPath[PATH_MAX];
int32_t len = (int32_t)strlen(tsTempDir);
memcpy(tmpPath, tsTempDir, len);
if (tmpPath[len - 1] != '/' && tmpPath[len - 1] != '\\') {
tmpPath[len++] = '\\';
tmpPath[len++] = '\\';
}
strcpy(tmpPath + len, tdengineTmpFileNamePrefix);
@ -35,7 +36,7 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
strcat(tmpPath, fileNamePrefix);
strcat(tmpPath, "-%d-%s");
}
char rand[8] = {0};
taosRandStr(rand, tListLen(rand) - 1);
snprintf(dstPath, PATH_MAX, tmpPath, getpid(), rand);
@ -46,18 +47,16 @@ void taosGetTmpfilePath(const char *fileNamePrefix, char *dstPath) {
int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t count) {
fseek(in_file, (int32_t)(*offset), 0);
int64_t writeLen = 0;
uint8_t buffer[_SEND_FILE_STEP_] = { 0 };
uint8_t buffer[_SEND_FILE_STEP_] = {0};
for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
size_t rlen = fread(buffer, 1, _SEND_FILE_STEP_, in_file);
if (rlen <= 0) {
return writeLen;
}
else if (rlen < _SEND_FILE_STEP_) {
} else if (rlen < _SEND_FILE_STEP_) {
fwrite(buffer, 1, rlen, out_file);
return (int64_t)(writeLen + rlen);
}
else {
} else {
fwrite(buffer, 1, _SEND_FILE_STEP_, in_file);
writeLen += _SEND_FILE_STEP_;
}
@ -65,12 +64,11 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co
int64_t remain = count - writeLen;
if (remain > 0) {
size_t rlen = fread(buffer, 1, (size_t) remain, in_file);
size_t rlen = fread(buffer, 1, (size_t)remain, in_file);
if (rlen <= 0) {
return writeLen;
}
else {
fwrite(buffer, 1, (size_t) remain, out_file);
} else {
fwrite(buffer, 1, (size_t)remain, out_file);
writeLen += remain;
}
}
@ -78,12 +76,99 @@ int64_t taosFSendFile(FILE *out_file, FILE *in_file, int64_t *offset, int64_t co
return writeLen;
}
int64_t taosSendFile(int32_t dfd, int32_t sfd, int64_t* offset, int64_t size) {
uError("taosSendFile no implemented yet");
int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t count) {
if (offset != NULL) lseek(sfd, (int32_t)(*offset), 0);
int64_t writeLen = 0;
uint8_t buffer[_SEND_FILE_STEP_] = {0};
for (int64_t len = 0; len < (count - _SEND_FILE_STEP_); len += _SEND_FILE_STEP_) {
int32_t rlen = (int32_t)read(sfd, buffer, _SEND_FILE_STEP_);
if (rlen <= 0) {
return writeLen;
} else if (rlen < _SEND_FILE_STEP_) {
taosWriteSocket(dfd, buffer, rlen);
return (int64_t)(writeLen + rlen);
} else {
taosWriteSocket(dfd, buffer, _SEND_FILE_STEP_);
writeLen += _SEND_FILE_STEP_;
}
}
int64_t remain = count - writeLen;
if (remain > 0) {
int32_t rlen = read(sfd, buffer, (int32_t)remain);
if (rlen <= 0) {
return writeLen;
} else {
taosWriteSocket(sfd, buffer, (int32_t)remain);
writeLen += remain;
}
}
return writeLen;
}
int32_t taosFtruncate(int32_t fd, int64_t l_size) {
if (fd < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
HANDLE h = (HANDLE)_get_osfhandle(fd);
LARGE_INTEGER li_0;
li_0.QuadPart = (int64_t)0;
BOOL cur = SetFilePointerEx(h, li_0, NULL, FILE_CURRENT);
if (!cur) {
uError("SetFilePointerEx Error getting current position in file.\n");
return -1;
}
LARGE_INTEGER li_size;
li_size.QuadPart = l_size;
BOOL cur2 = SetFilePointerEx(h, li_size, NULL, FILE_BEGIN);
if (cur2 == 0) {
int error = GetLastError();
uError("SetFilePointerEx GetLastError is: %d\n", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
if (!SetEndOfFile(h)) {
int error = GetLastError();
uError("SetEndOfFile GetLastError is:%d", error);
switch (error) {
case ERROR_INVALID_HANDLE:
errno = EBADF;
break;
default:
errno = EIO;
break;
}
return -1;
}
return 0;
}
int32_t taosFtruncate(int32_t fd, int64_t length) {
uError("taosFtruncate no implemented yet");
return 0;
}
int fsync(int filedes) {
if (filedes < 0) {
errno = EBADF;
uError("%s\n", "fd arg was negative");
return -1;
}
HANDLE h = (HANDLE)_get_osfhandle(filedes);
return FlushFileBuffers(h);
}

42
src/os/src/windows/wSignal.c Normal file
View File

@ -0,0 +1,42 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include <signal.h>
#include <windows.h>
typedef void (*FWinSignalHandler)(int32_t signum);
void taosSetSignal(int32_t signum, FSignalHandler sigfp) {
if (signum == SIGUSR1) return;
// SIGHUP doesn't exist in windows, we handle it in the way of ctrlhandler
if (signum == SIGHUP) {
SetConsoleCtrlHandler((PHANDLER_ROUTINE)sigfp, TRUE);
} else {
signal(signum, (FWinSignalHandler)sigfp);
}
}
void taosIgnSignal(int32_t signum) {
if (signum == SIGUSR1 || signum == SIGHUP) return;
signal(signum, SIG_IGN);
}
void taosDflSignal(int32_t signum) {
if (signum == SIGUSR1 || signum == SIGHUP) return;
signal(signum, SIG_DFL);
}

1
src/os/src/windows/wSocket.c
View File

@ -48,6 +48,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
void taosIgnSIGPIPE() {}
void taosBlockSIGPIPE() {}
void taosSetMaskSIGPIPE() {}
int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {
if (level == SOL_SOCKET && optname == TCP_KEEPCNT) {

19
src/os/src/windows/wSysLog.c Normal file
View File

@ -0,0 +1,19 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
void syslog(int unused, const char *format, ...) {}

178
src/os/src/windows/wSysinfo.c
View File

@ -31,11 +31,54 @@
#pragma comment(lib, "Mswsock.lib ")
#endif
#include <objbase.h>
#pragma warning(push)
#pragma warning(disable:4091)
#pragma warning(disable : 4091)
#include <DbgHelp.h>
#pragma warning(pop)
static int32_t taosGetTotalMemory() {
MEMORYSTATUSEX memsStat;
memsStat.dwLength = sizeof(memsStat);
if (!GlobalMemoryStatusEx(&memsStat)) {
return 0;
}
float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f);
return (int32_t)nMemTotal;
}
bool taosGetSysMemory(float *memoryUsedMB) {
MEMORYSTATUSEX memsStat;
memsStat.dwLength = sizeof(memsStat);
if (!GlobalMemoryStatusEx(&memsStat)) {
return false;
}
float nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f);
float nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f);
*memoryUsedMB = nMemTotal - nMemFree;
return true;
}
bool taosGetProcMemory(float *memoryUsedMB) {
unsigned bytes_used = 0;
#if defined(_WIN64) && defined(_MSC_VER)
PROCESS_MEMORY_COUNTERS pmc;
HANDLE cur_proc = GetCurrentProcess();
if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) {
bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage);
}
#endif
*memoryUsedMB = (float)bytes_used / 1024 / 1024;
return true;
}
static void taosGetSystemTimezone() {
// get and set default timezone
SGlobalCfg *cfg_timezone = taosGetConfigOption("timezone");
@ -71,16 +114,16 @@ static void taosGetSystemLocale() {
}
}
void taosPrintOsInfo() {}
void taosKillSystem() {
uError("function taosKillSystem, exit!");
exit(0);
static int32_t taosGetCpuCores() {
SYSTEM_INFO info;
GetSystemInfo(&info);
return (int32_t)info.dwNumberOfProcessors;
}
void taosGetSystemInfo() {
taosGetSystemTimezone();
taosGetSystemLocale();
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
*sysCpuUsage = 0;
*procCpuUsage = 0;
return true;
}
bool taosGetDisk() {
@ -89,20 +132,35 @@ bool taosGetDisk() {
unsigned _int64 i64FreeBytesToCaller;
unsigned _int64 i64TotalBytes;
unsigned _int64 i64FreeBytes;
char dir[4] = {'C', ':', '\\', '\0'};
int drive_type;
if (tscEmbedded) {
drive_type = GetDriveTypeA(dir);
if (drive_type == DRIVE_FIXED) {
fResult = GetDiskFreeSpaceExA(dir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
(PULARGE_INTEGER)&i64FreeBytes);
if (fResult) {
tsTotalDataDirGB = tsTotalLogDirGB = tsTotalTmpDirGB = (float)(i64TotalBytes / unit);
tsAvailDataDirGB = tsAvailLogDirGB = tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit);
}
fResult = GetDiskFreeSpaceExA(tsDataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
(PULARGE_INTEGER)&i64FreeBytes);
if (fResult) {
tsTotalDataDirGB = (float)(i64TotalBytes / unit);
tsAvailDataDirGB = (float)(i64FreeBytes / unit);
}
}
fResult = GetDiskFreeSpaceExA(tsLogDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
(PULARGE_INTEGER)&i64FreeBytes);
if (fResult) {
tsTotalLogDirGB = (float)(i64TotalBytes / unit);
tsAvailLogDirGB = (float)(i64FreeBytes / unit);
}
fResult = GetDiskFreeSpaceExA(tsTempDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
(PULARGE_INTEGER)&i64FreeBytes);
if (fResult) {
tsTotalTmpDirGB = (float)(i64TotalBytes / unit);
tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit);
}
return true;
}
bool taosGetBandSpeed(float *bandSpeedKb) {
*bandSpeedKb = 0;
return true;
}
@ -144,48 +202,30 @@ bool taosGetProcIO(float *readKB, float *writeKB) {
return true;
}
bool taosGetBandSpeed(float *bandSpeedKb) {
*bandSpeedKb = 0;
return true;
void taosGetSystemInfo() {
tsNumOfCores = taosGetCpuCores();
tsTotalMemoryMB = taosGetTotalMemory();
float tmp1, tmp2;
taosGetDisk();
taosGetBandSpeed(&tmp1);
taosGetCpuUsage(&tmp1, &tmp2);
taosGetProcIO(&tmp1, &tmp2);
taosGetSystemTimezone();
taosGetSystemLocale();
}
bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
*sysCpuUsage = 0;
*procCpuUsage = 0;
return true;
void taosPrintOsInfo() {
uInfo(" os numOfCores: %d", tsNumOfCores);
uInfo(" os totalDisk: %f(GB)", tsTotalDataDirGB);
uInfo(" os totalMemory: %d(MB)", tsTotalMemoryMB);
uInfo("==================================");
}
bool taosGetProcMemory(float *memoryUsedMB) {
unsigned bytes_used = 0;
#if 0
#if defined(_WIN32) && defined(_MSC_VER)
PROCESS_MEMORY_COUNTERS pmc;
HANDLE cur_proc = GetCurrentProcess();
if (GetProcessMemoryInfo(cur_proc, &pmc, sizeof(pmc))) {
bytes_used = (unsigned)(pmc.WorkingSetSize + pmc.PagefileUsage);
}
#endif
#endif
*memoryUsedMB = (float)bytes_used / 1024 / 1024;
return true;
}
bool taosGetSysMemory(float *memoryUsedMB) {
MEMORYSTATUSEX memsStat;
float nMemFree;
float nMemTotal;
memsStat.dwLength = sizeof(memsStat);
if (!GlobalMemoryStatusEx(&memsStat)) {
return false;
}
nMemFree = memsStat.ullAvailPhys / (1024.0f * 1024.0f);
nMemTotal = memsStat.ullTotalPhys / (1024.0f * 1024.0f);
*memoryUsedMB = nMemTotal - nMemFree;
return true;
void taosKillSystem() {
uError("function taosKillSystem, exit!");
exit(0);
}
int taosSystem(const char *cmd) {
@ -195,10 +235,6 @@ int taosSystem(const char *cmd) {
int flock(int fd, int option) { return 0; }
int fsync(int filedes) { return 0; }
int sigaction(int sig, struct sigaction *d, void *p) { return 0; }
LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) {
typedef BOOL(WINAPI * FxMiniDumpWriteDump)(IN HANDLE hProcess, IN DWORD ProcessId, IN HANDLE hFile,
IN MINIDUMP_TYPE DumpType,
@ -235,4 +271,22 @@ LONG WINAPI FlCrashDump(PEXCEPTION_POINTERS ep) {
return EXCEPTION_CONTINUE_SEARCH;
}
void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); }
void taosSetCoreDump() { SetUnhandledExceptionFilter(&FlCrashDump); }
bool taosGetSystemUid(char *uid) {
GUID guid;
CoCreateGuid(&guid);
sprintf(
uid,
"%08X-%04X-%04X-%02X%02X-%02X%02X%02X%02X%02X%02X",
guid.Data1, guid.Data2, guid.Data3,
guid.Data4[0], guid.Data4[1],
guid.Data4[2], guid.Data4[3],
guid.Data4[4], guid.Data4[5],
guid.Data4[6], guid.Data4[7]);
return true;
}
char *taosGetCmdlineByPID(int pid) { return ""; }

2
src/plugins/CMakeLists.txt
View File

@ -3,6 +3,6 @@ PROJECT(TDengine)
ADD_SUBDIRECTORY(monitor)
ADD_SUBDIRECTORY(http)
IF (TD_MQTT)
IF (TD_LINUX AND TD_MQTT)
ADD_SUBDIRECTORY(mqtt)
ENDIF ()

33
src/plugins/http/CMakeLists.txt
View File

@ -9,32 +9,15 @@ INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
INCLUDE_DIRECTORIES(inc)
AUX_SOURCE_DIRECTORY(src SRC)
IF (TD_LINUX)
ADD_LIBRARY(http ${SRC})
TARGET_LINK_LIBRARIES(http z)
ADD_LIBRARY(http ${SRC})
TARGET_LINK_LIBRARIES(http z)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(http taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(http taos)
ENDIF ()
IF (TD_ADMIN)
TARGET_LINK_LIBRARIES(http admin)
ENDIF ()
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(http taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(http taos)
ENDIF ()
IF (TD_DARWIN)
ADD_LIBRARY(http ${SRC})
TARGET_LINK_LIBRARIES(http z)
IF (TD_SOMODE_STATIC)
TARGET_LINK_LIBRARIES(http taos_static)
ELSE ()
TARGET_LINK_LIBRARIES(http taos)
ENDIF ()
IF (TD_ADMIN)
TARGET_LINK_LIBRARIES(http admin)
ENDIF ()
IF (TD_ADMIN)
TARGET_LINK_LIBRARIES(http admin)
ENDIF ()

2
src/plugins/http/inc/httpContext.h
View File

@ -22,7 +22,7 @@ bool httpInitContexts();
void httpCleanupContexts();
const char *httpContextStateStr(HttpContextState state);
HttpContext *httpCreateContext(int32_t fd);
HttpContext *httpCreateContext(SOCKET fd);
bool httpInitContext(HttpContext *pContext);
HttpContext *httpGetContext(void * pContext);
void httpReleaseContext(HttpContext *pContext, bool clearRes);

9
src/plugins/http/inc/httpInt.h
View File

@ -16,6 +16,7 @@
#ifndef TDENGINE_HTTP_INT_H
#define TDENGINE_HTTP_INT_H
#include "os.h"
#include <stdbool.h>
#include "pthread.h"
#include "semaphore.h"
@ -140,7 +141,7 @@ typedef enum {
typedef struct HttpContext {
int32_t refCount;
int32_t fd;
SOCKET fd;
uint32_t accessTimes;
uint32_t lastAccessTime;
int32_t state;
@ -167,7 +168,7 @@ typedef struct HttpThread {
HttpContext * pHead;
pthread_mutex_t threadMutex;
bool stop;
int32_t pollFd;
EpollFd pollFd;
int32_t numOfContexts;
int32_t threadId;
char label[HTTP_LABEL_SIZE];
@ -178,7 +179,9 @@ typedef struct HttpServer {
char label[HTTP_LABEL_SIZE];
uint32_t serverIp;
uint16_t serverPort;
int32_t fd;
int8_t stop;
int8_t reserve;
SOCKET fd;
int32_t numOfThreads;
int32_t methodScannerLen;
int32_t requestNum;

8
src/plugins/http/src/httpAuth.c
View File

@ -26,7 +26,7 @@
int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len) {
token[len] = '\0';
int32_t outlen = 0;
char *base64 = (char *)base64_decode(token, len, &outlen);
char * base64 = (char *)base64_decode(token, len, &outlen);
if (base64 == NULL || outlen == 0) {
httpError("context:%p, fd:%d, basic token:%s parsed error", pContext, pContext->fd, token);
free(base64);
@ -49,7 +49,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len)
strncpy(pContext->user, base64, (size_t)user_len);
pContext->user[user_len] = 0;
char *password = user + 1;
char * password = user + 1;
int32_t pass_len = (int32_t)((base64 + outlen) - password);
if (pass_len < 1 || pass_len >= HTTP_PASSWORD_LEN) {
httpError("context:%p, fd:%d, basic token:%s parse password error", pContext, pContext->fd, token);
@ -66,7 +66,7 @@ int32_t httpParseBasicAuthToken(HttpContext *pContext, char *token, int32_t len)
int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len) {
token[len] = '\0';
int32_t outlen = 0;
int32_t outlen = 0;
unsigned char *base64 = base64_decode(token, len, &outlen);
if (base64 == NULL || outlen == 0) {
httpError("context:%p, fd:%d, taosd token:%s parsed error", pContext, pContext->fd, token);
@ -97,7 +97,7 @@ int32_t httpParseTaosdAuthToken(HttpContext *pContext, char *token, int32_t len)
}
int32_t httpGenTaosdAuthToken(HttpContext *pContext, char *token, int32_t maxLen) {
char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0};
char buffer[sizeof(pContext->user) + sizeof(pContext->pass)] = {0};
size_t size = sizeof(pContext->user);
tstrncpy(buffer, pContext->user, size);
size = sizeof(pContext->pass);

21
src/plugins/http/src/httpContext.c
View File

@ -35,20 +35,24 @@ static void httpRemoveContextFromEpoll(HttpContext *pContext) {
HttpThread *pThread = pContext->pThread;
if (pContext->fd >= 0) {
epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pContext->fd, NULL);
int32_t fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1);
#ifdef WINDOWS
SOCKET fd = atomic_val_compare_exchange_32(&pContext->fd, pContext->fd, -1);
#else
SOCKET fd = atomic_val_compare_exchange_64(&pContext->fd, pContext->fd, -1);
#endif
taosCloseSocket(fd);
}
}
static void httpDestroyContext(void *data) {
HttpContext *pContext = *(HttpContext **)data;
if (pContext->fd > 0) taosClose(pContext->fd);
if (pContext->fd > 0) taosCloseSocket(pContext->fd);
HttpThread *pThread = pContext->pThread;
httpRemoveContextFromEpoll(pContext);
httpReleaseSession(pContext);
atomic_sub_fetch_32(&pThread->numOfContexts, 1);
httpDebug("context:%p, is destroyed, refCount:%d data:%p thread:%s numOfContexts:%d", pContext, pContext->refCount,
data, pContext->pThread->label, pContext->pThread->numOfContexts);
pContext->pThread = 0;
@ -100,15 +104,13 @@ const char *httpContextStateStr(HttpContextState state) {
}
}
void httpNotifyContextClose(HttpContext *pContext) {
shutdown(pContext->fd, SHUT_WR);
}
void httpNotifyContextClose(HttpContext *pContext) { shutdown(pContext->fd, SHUT_WR); }
bool httpAlterContextState(HttpContext *pContext, HttpContextState srcState, HttpContextState destState) {
return (atomic_val_compare_exchange_32(&pContext->state, srcState, destState) == srcState);
}
HttpContext *httpCreateContext(int32_t fd) {
HttpContext *httpCreateContext(SOCKET fd) {
HttpContext *pContext = calloc(1, sizeof(HttpContext));
if (pContext == NULL) return NULL;
@ -123,8 +125,8 @@ HttpContext *httpCreateContext(int32_t fd) {
pContext->ppContext = ppContext;
httpDebug("context:%p, fd:%d, is created, data:%p", pContext, fd, ppContext);
// set the ref to 0
taosCacheRelease(tsHttpServer.contextCache, (void**)&ppContext, false);
// set the ref to 0
taosCacheRelease(tsHttpServer.contextCache, (void **)&ppContext, false);
return pContext;
}
@ -174,7 +176,6 @@ bool httpInitContext(HttpContext *pContext) {
pContext->encodeMethod = NULL;
memset(&pContext->singleCmd, 0, sizeof(HttpSqlCmd));
httpTrace("context:%p, fd:%d, parsed:%d", pContext, pContext->fd, pContext->parsed);
return true;
}

2
src/plugins/http/src/httpGcHandle.c
View File

@ -192,7 +192,7 @@ bool gcProcessQueryRequest(HttpContext* pContext) {
break;
}
cJSON* alias = cJSON_GetObjectItem(query, "alias");
cJSON* alias = cJSON_GetObjectItem(query, "alias");
int32_t aliasBuffer = -1;
if (!(alias == NULL || alias->valuestring == NULL || strlen(alias->valuestring) == 0)) {
aliasBuffer = httpAddToSqlCmdBuffer(pContext, alias->valuestring);

29
src/plugins/http/src/httpGcJson.c
View File

@ -86,7 +86,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
JsonBuf *jsonBuf = httpMallocJsonBuf(pContext);
if (jsonBuf == NULL) return false;
int32_t num_fields = taos_num_fields(result);
int32_t num_fields = taos_num_fields(result);
TAOS_FIELD *fields = taos_fetch_fields(result);
if (num_fields == 0) {
return false;
@ -101,7 +101,7 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
// such as select count(*) count(*) from sys.cpu group by ipaddr interval(1d)
int32_t dataFields = -1;
int32_t groupFields = -1;
bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP;
bool hasTimestamp = fields[0].type == TSDB_DATA_TYPE_TIMESTAMP;
if (hasTimestamp) {
dataFields = 1;
if (num_fields > 2) groupFields = num_fields - 1;
@ -125,15 +125,15 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
cmd->numOfRows--;
continue;
}
int32_t* length = taos_fetch_lengths(result);
int32_t *length = taos_fetch_lengths(result);
// for group by
if (groupFields != -1) {
char target[HTTP_GC_TARGET_SIZE] = {0};
char target[HTTP_GC_TARGET_SIZE] = {0};
int32_t len;
len = snprintf(target,HTTP_GC_TARGET_SIZE,"%s{",aliasBuffer);
for (int32_t i = dataFields + 1; i<num_fields; i++){
switch (fields[i].type) {
len = snprintf(target, HTTP_GC_TARGET_SIZE, "%s{", aliasBuffer);
for (int32_t i = dataFields + 1; i < num_fields; i++) {
switch (fields[i].type) {
case TSDB_DATA_TYPE_BOOL:
case TSDB_DATA_TYPE_TINYINT:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%d", fields[i].name, *((int8_t *)row[i]));
@ -155,20 +155,19 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
if (row[i]!= NULL){
if (row[i] != NULL) {
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:", fields[i].name);
memcpy(target + len, (char *) row[i], length[i]);
len = strlen(target);
memcpy(target + len, (char *)row[i], length[i]);
len = (int32_t)strlen(target);
}
break;
default:
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "%s:%s", fields[i].name, "-");
break;
}
if(i < num_fields - 1 ){
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, ", ");
if (i < num_fields - 1) {
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, ", ");
}
}
len += snprintf(target + len, HTTP_GC_TARGET_SIZE - len, "}");
@ -217,10 +216,10 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result,
break;
case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR:
httpJsonStringForTransMean(jsonBuf, (char*)row[i], fields[i].bytes);
httpJsonStringForTransMean(jsonBuf, (char *)row[i], fields[i].bytes);
break;
case TSDB_DATA_TYPE_TIMESTAMP:
if (precision == TSDB_TIME_PRECISION_MILLI) { //ms
if (precision == TSDB_TIME_PRECISION_MILLI) { // ms
httpJsonInt64(jsonBuf, *((int64_t *)row[i]));
} else {
httpJsonInt64(jsonBuf, *((int64_t *)row[i]) / 1000);

92
src/plugins/http/src/httpGzip.c
View File

@ -25,25 +25,24 @@ typedef enum {
} EHTTP_GZIP_STATE;
struct ehttp_gzip_s {
ehttp_gzip_conf_t conf;
ehttp_gzip_callbacks_t callbacks;
void *arg;
z_stream *gzip;
gz_header *header;
char *chunk;
int32_t state;
ehttp_gzip_conf_t conf;
ehttp_gzip_callbacks_t callbacks;
void * arg;
z_stream * gzip;
gz_header * header;
char * chunk;
int32_t state;
};
static void dummy_on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) {
}
static void dummy_on_data(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) {}
static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) {
switch(gzip->state) {
switch (gzip->state) {
case EHTTP_GZIP_READY: {
inflateEnd(gzip->gzip);
} break;
default: break;
default:
break;
}
if (gzip->gzip) {
free(gzip->gzip);
@ -60,43 +59,43 @@ static void ehttp_gzip_cleanup(ehttp_gzip_t *gzip) {
gzip->state = EHTTP_GZIP_CLOSED;
}
ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) {
ehttp_gzip_t *gzip = (ehttp_gzip_t*)calloc(1, sizeof(*gzip));
ehttp_gzip_t *ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg) {
ehttp_gzip_t *gzip = (ehttp_gzip_t *)calloc(1, sizeof(*gzip));
if (!gzip) return NULL;
do {
gzip->conf = conf;
gzip->callbacks = callbacks;
gzip->arg = arg;
gzip->conf = conf;
gzip->callbacks = callbacks;
gzip->arg = arg;
if (gzip->callbacks.on_data == NULL) gzip->callbacks.on_data = dummy_on_data;
gzip->gzip = (z_stream*)calloc(1, sizeof(*gzip->gzip));
gzip->gzip = (z_stream *)calloc(1, sizeof(*gzip->gzip));
if (gzip->conf.get_header) {
gzip->header = (gz_header*)calloc(1, sizeof(*gzip->header));
gzip->header = (gz_header *)calloc(1, sizeof(*gzip->header));
}
if (gzip->conf.chunk_size<=0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT;
gzip->chunk = (char*)malloc(gzip->conf.chunk_size);
if (gzip->conf.chunk_size <= 0) gzip->conf.chunk_size = EHTTP_GZIP_CHUNK_SIZE_DEFAULT;
gzip->chunk = (char *)malloc(gzip->conf.chunk_size);
if (!gzip->gzip || (gzip->conf.get_header && !gzip->header) || !gzip->chunk) break;
gzip->gzip->zalloc = Z_NULL;
gzip->gzip->zfree = Z_NULL;
gzip->gzip->opaque = Z_NULL;
gzip->gzip->zalloc = Z_NULL;
gzip->gzip->zfree = Z_NULL;
gzip->gzip->opaque = Z_NULL;
// 863 windowBits can also be greater than 15 for optional gzip decoding. Add
// 864 32 to windowBits to enable zlib and gzip decoding with automatic header
// 865 detection, or add 16 to decode only the gzip format (the zlib format will
// 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a
// 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see
// 868 below), inflate() will not automatically decode concatenated gzip streams.
// 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state
// 870 would need to be reset to continue decoding a subsequent gzip stream.
int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS
// 863 windowBits can also be greater than 15 for optional gzip decoding. Add
// 864 32 to windowBits to enable zlib and gzip decoding with automatic header
// 865 detection, or add 16 to decode only the gzip format (the zlib format will
// 866 return a Z_DATA_ERROR). If a gzip stream is being decoded, strm->adler is a
// 867 CRC-32 instead of an Adler-32. Unlike the gunzip utility and gzread() (see
// 868 below), inflate() will not automatically decode concatenated gzip streams.
// 869 inflate() will return Z_STREAM_END at the end of the gzip stream. The state
// 870 would need to be reset to continue decoding a subsequent gzip stream.
int32_t ret = inflateInit2(gzip->gzip, 32); // 32/16? 32/16 + MAX_WBITS
if (ret != Z_OK) break;
if (gzip->header) {
ret = inflateGetHeader(gzip->gzip, gzip->header);
}
if (ret != Z_OK) break;
gzip->gzip->next_out = (z_const Bytef*)gzip->chunk;
gzip->gzip->avail_out = gzip->conf.chunk_size;
gzip->gzip->next_out = (z_const Bytef *)gzip->chunk;
gzip->gzip->avail_out = gzip->conf.chunk_size;
gzip->state = EHTTP_GZIP_READY;
return gzip;
} while (0);
@ -105,7 +104,7 @@ ehttp_gzip_t* ehttp_gzip_create_decompressor(ehttp_gzip_conf_t conf, ehttp_gzip_
return NULL;
}
ehttp_gzip_t* ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg);
ehttp_gzip_t *ehttp_gzip_create_compressor(ehttp_gzip_conf_t conf, ehttp_gzip_callbacks_t callbacks, void *arg);
void ehttp_gzip_destroy(ehttp_gzip_t *gzip) {
ehttp_gzip_cleanup(gzip);
@ -129,16 +128,16 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) {
}
if (ret != Z_OK && ret != Z_STREAM_END) return -1;
if (gzip->gzip->avail_out>0) {
if (ret!=Z_STREAM_END) continue;
if (gzip->gzip->avail_out > 0) {
if (ret != Z_STREAM_END) continue;
}
int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk;
int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk);
gzip->gzip->next_out[0] = '\0';
gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len);
gzip->gzip->next_out = (z_const Bytef*)gzip->chunk;
gzip->gzip->avail_out = gzip->conf.chunk_size;
gzip->gzip->next_out = (z_const Bytef *)gzip->chunk;
gzip->gzip->avail_out = gzip->conf.chunk_size;
}
return 0;
@ -147,21 +146,20 @@ int32_t ehttp_gzip_write(ehttp_gzip_t *gzip, const char *buf, int32_t len) {
int32_t ehttp_gzip_finish(ehttp_gzip_t *gzip) {
if (gzip->state != EHTTP_GZIP_READY) return -1;
gzip->gzip->next_in = NULL;
gzip->gzip->avail_in = 0;
gzip->gzip->next_in = NULL;
gzip->gzip->avail_in = 0;
int32_t ret;
ret = inflate(gzip->gzip, Z_FINISH);
if (ret != Z_STREAM_END) return -1;
int32_t len = gzip->gzip->next_out - (z_const Bytef*)gzip->chunk;
int32_t len = (int32_t)(gzip->gzip->next_out - (z_const Bytef *)gzip->chunk);
gzip->gzip->next_out[0] = '\0';
gzip->callbacks.on_data(gzip, gzip->arg, gzip->chunk, len);
gzip->gzip->next_out = NULL;
gzip->gzip->avail_out = 0;
gzip->gzip->next_out = NULL;
gzip->gzip->avail_out = 0;
return 0;
}

2
src/plugins/http/src/httpHandle.c
View File

@ -48,7 +48,7 @@ bool httpProcessData(HttpContext* pContext) {
/*
* httpCloseContextByApp has been called when parsing the error
*/
//httpCloseContextByApp(pContext);
// httpCloseContextByApp(pContext);
} else {
httpProcessRequest(pContext);
}

51
src/plugins/http/src/httpJson.c
View File

@ -44,20 +44,21 @@ int32_t httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int32_t
int32_t writeLen = 0;
do {
if (pContext->fd > 2){
if (pContext->fd > 2) {
len = (int32_t)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
}
else {
} else {
return sz;
}
if (len < 0) {
httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno), countWait);
httpDebug("context:%p, fd:%d, socket write errno:%d:%s, times:%d", pContext, pContext->fd, errno, strerror(errno),
countWait);
if (++countWait > HTTP_WRITE_RETRY_TIMES) break;
taosMsleep(HTTP_WRITE_WAIT_TIME_MS);
continue;
} else if (len == 0) {
httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno, strerror(errno));
httpDebug("context:%p, fd:%d, socket write errno:%d:%s, connect already closed", pContext, pContext->fd, errno,
strerror(errno));
break;
} else {
countWait = 0;
@ -80,7 +81,7 @@ int32_t httpWriteBuf(struct HttpContext* pContext, const char* buf, int32_t sz)
return writeSz;
}
int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32_t sz) {
int32_t httpWriteBufNoTrace(struct HttpContext* pContext, const char* buf, int32_t sz) {
int32_t writeSz = httpWriteBufByFd(pContext, buf, sz);
if (writeSz != sz) {
httpError("context:%p, fd:%d, dataSize:%d, writeSize:%d, failed to send response", pContext, pContext->fd, sz,
@ -92,8 +93,8 @@ int32_t httpWriteBufNoTrace(struct HttpContext *pContext, const char *buf, int32
int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
int32_t remain = 0;
char sLen[24];
uint64_t srcLen = (uint64_t) (buf->lst - buf->buf);
char sLen[24];
int32_t srcLen = (int32_t)(buf->lst - buf->buf);
if (buf->pContext->fd <= 0) {
httpTrace("context:%p, fd:%d, write json body error", buf->pContext, buf->pContext->fd);
@ -113,21 +114,21 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
httpTrace("context:%p, fd:%d, no data need dump", buf->pContext, buf->pContext->fd);
return 0; // there is no data to dump.
} else {
int32_t len = sprintf(sLen, "%" PRIx64 "\r\n", srcLen);
httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", response:\n%s", buf->pContext, buf->pContext->fd,
srcLen, buf->buf);
int32_t len = sprintf(sLen, "%x\r\n", srcLen);
httpTrace("context:%p, fd:%d, write body, chunkSize:%d, response:\n%s", buf->pContext, buf->pContext->fd, srcLen,
buf->buf);
httpWriteBufNoTrace(buf->pContext, sLen, len);
remain = httpWriteBufNoTrace(buf->pContext, buf->buf, (int32_t)srcLen);
remain = httpWriteBufNoTrace(buf->pContext, buf->buf, srcLen);
}
} else {
char compressBuf[JSON_BUFFER_SIZE] = {0};
char compressBuf[JSON_BUFFER_SIZE] = {0};
int32_t compressBufLen = JSON_BUFFER_SIZE;
int32_t ret = httpGzipCompress(buf->pContext, buf->buf, srcLen, compressBuf, &compressBufLen, isTheLast);
if (ret == 0) {
if (compressBufLen > 0) {
int32_t len = sprintf(sLen, "%x\r\n", compressBufLen);
httpTrace("context:%p, fd:%d, write body, chunkSize:%" PRIu64 ", compressSize:%d, last:%d, response:\n%s",
buf->pContext, buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf);
httpTrace("context:%p, fd:%d, write body, chunkSize:%d, compressSize:%d, last:%d, response:\n%s", buf->pContext,
buf->pContext->fd, srcLen, compressBufLen, isTheLast, buf->buf);
httpWriteBufNoTrace(buf->pContext, sLen, len);
remain = httpWriteBufNoTrace(buf->pContext, (const char*)compressBuf, compressBufLen);
} else {
@ -136,7 +137,7 @@ int32_t httpWriteJsonBufBody(JsonBuf* buf, bool isTheLast) {
remain = 0; // there is no data to dump.
}
} else {
httpError("context:%p, fd:%d, failed to compress data, chunkSize:%" PRIu64 ", last:%d, error:%d, response:\n%s",
httpError("context:%p, fd:%d, failed to compress data, chunkSize:%d, last:%d, error:%d, response:\n%s",
buf->pContext, buf->pContext->fd, srcLen, isTheLast, ret, buf->buf);
remain = 0;
}
@ -154,8 +155,8 @@ void httpWriteJsonBufHead(JsonBuf* buf) {
buf->pContext->fd = -1;
}
char msg[1024] = {0};
int32_t len = -1;
char msg[1024] = {0};
int32_t len = -1;
if (buf->pContext->parser->acceptEncodingGzip == 0 || !tsHttpEnableCompress) {
len = sprintf(msg, httpRespTemplate[HTTP_RESPONSE_CHUNKED_UN_COMPRESS], httpVersionStr[buf->pContext->parser->httpVersion],
@ -256,16 +257,16 @@ void httpJsonInt64(JsonBuf* buf, int64_t num) {
}
void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) {
char ts[35] = {0};
struct tm *ptm;
int32_t precision = 1000;
char ts[35] = {0};
struct tm* ptm;
int32_t precision = 1000;
if (us) {
precision = 1000000;
}
time_t tt = t / precision;
ptm = localtime(&tt);
int32_t length = (int32_t) strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm);
int32_t length = (int32_t)strftime(ts, 35, "%Y-%m-%d %H:%M:%S", ptm);
if (us) {
length += snprintf(ts + length, 8, ".%06" PRId64, t % precision);
} else {
@ -276,9 +277,9 @@ void httpJsonTimestamp(JsonBuf* buf, int64_t t, bool us) {
}
void httpJsonUtcTimestamp(JsonBuf* buf, int64_t t, bool us) {
char ts[40] = {0};
struct tm *ptm;
int32_t precision = 1000;
char ts[40] = {0};
struct tm* ptm;
int32_t precision = 1000;
if (us) {
precision = 1000000;
}

134
src/plugins/http/src/httpParser.c
View File

@ -130,7 +130,7 @@ static int32_t httpAppendString(HttpString *str, const char *s, int32_t len) {
static void httpClearString(HttpString *str) {
if (str->str) {
str->str[0] = '\0';
str->pos = 0;
str->pos = 0;
}
}
@ -153,7 +153,7 @@ static int32_t httpOnRequestLine(HttpParser *pParser, char *method, char *target
for (int32_t i = 0; i < HTTP_MAX_URL; i++) {
char *pSeek = strchr(pStart, '/');
if (pSeek == NULL) {
(void)httpAppendString(pParser->path + i, pStart, strlen(pStart));
(void)httpAppendString(pParser->path + i, pStart, (int32_t)strlen(pStart));
break;
} else {
(void)httpAppendString(pParser->path + i, pStart, (int32_t)(pSeek - pStart));
@ -237,7 +237,6 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const
}
httpTrace("context:%p, fd:%d, keepAlive:%d", pContext, pContext->fd, pContext->parser->keepAlive);
}
#if 0
else if (0 == strcasecmp(key, "Content-Encoding")) {
if (0 == strcmp(val, "gzip")) {
@ -246,7 +245,7 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const
}
return 0;
}
#endif
#endif
else if (0 == strcasecmp(key, "Transfer-Encoding") || 0 == strcasecmp(key, "Content-Encoding")) {
if (strstr(val, "gzip")) {
@ -272,20 +271,17 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const
}
else if (0 == strcasecmp(key, "Authorization")) {
char * t = NULL;
char * s = NULL;
char t[6] = {0};
char s[129] = {0};
int32_t bytes = 0;
int32_t n = sscanf(val, "%ms %ms%n", &t, &s, &bytes);
if (n == 2 && t && s && bytes == strlen(val)) {
int32_t n = sscanf(val, "%5s %128s%n", t, s, &bytes);
if (n == 2 && t[0] && s[0] && bytes == strlen(val)) {
if (strcmp(t, "Basic") == 0) {
free(parser->authContent);
parser->authContent = s;
parser->authContent = strdup(s);
parser->authType = HTTP_BASIC_AUTH;
s = NULL;
free(t);
free(s);
httpTrace("context:%p, fd:%d, basic auth:%s", pContext, pContext->fd, parser->authContent);
int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, strlen(parser->authContent));
int32_t ok = httpParseBasicAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent));
if (ok != 0) {
httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_BASIC_AUTH);
return -1;
@ -293,13 +289,10 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const
return 0;
} else if (strcmp(t, "Taosd") == 0) {
free(parser->authContent);
parser->authContent = s;
parser->authContent = strdup(s);
parser->authType = HTTP_TAOSD_AUTH;
s = NULL;
free(t);
free(s);
httpTrace("context:%p, fd:%d, taosd auth:%s", pContext, pContext->fd, parser->authContent);
int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, strlen(parser->authContent));
int32_t ok = httpParseTaosdAuthToken(pContext, parser->authContent, (int32_t)strlen(parser->authContent));
if (ok != 0) {
httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_TAOSD_AUTH);
return -1;
@ -309,16 +302,12 @@ static int32_t httpOnParseHeaderField(HttpParser *parser, const char *key, const
parser->authType = HTTP_INVALID_AUTH;
httpError("context:%p, fd:%d, invalid auth, t:%s s:%s", pContext, pContext->fd, t, s);
httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_TYPE);
free(t);
free(s);
return -1;
}
} else {
parser->authType = HTTP_INVALID_AUTH;
httpError("context:%p, fd:%d, parse auth failed, t:%s s:%s", pContext, pContext->fd, t, s);
httpOnError(parser, 0, TSDB_CODE_HTTP_INVALID_AUTH_FORMAT);
free(t);
free(s);
return -1;
}
}
@ -349,7 +338,7 @@ static int32_t httpOnBody(HttpParser *parser, const char *chunk, int32_t len) {
newSize = MIN(newSize, HTTP_BUFFER_SIZE);
buf->str = realloc(buf->str, newSize);
buf->size = newSize;
if (buf->str == NULL) {
httpError("context:%p, fd:%d, failed parse body, realloc %d failed", pContext, pContext->fd, buf->size);
httpOnError(parser, 0, TSDB_CODE_HTTP_NO_ENOUGH_MEMORY);
@ -410,9 +399,7 @@ static int32_t httpPopStack(HttpParser *parser) {
return 0;
}
static void httpClearStack(HttpStack *stack) {
stack->pos = 0;
}
static void httpClearStack(HttpStack *stack) { stack->pos = 0; }
static int32_t httpCleanupStack(HttpStack *stack) {
free(stack->stacks);
@ -451,7 +438,7 @@ void httpInitParser(HttpParser *parser) {
free(parser->key); parser->key = NULL;
free(parser->val); parser->val = NULL;
free(parser->authContent); parser->authContent = NULL;
httpClearStack(&parser->stacks);
httpClearString(&parser->str);
httpClearString(&parser->body);
@ -497,7 +484,7 @@ void httpDestroyParser(HttpParser *parser) {
free(parser->key); parser->key = NULL;
free(parser->val); parser->val = NULL;
free(parser->authContent); parser->authContent = NULL;
httpCleanupStack(&parser->stacks);
httpCleanupString(&parser->str);
httpCleanupString(&parser->body);
@ -513,42 +500,53 @@ void httpDestroyParser(HttpParser *parser) {
free(parser);
}
#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c))
#define is_token(c) (strchr("!#$%&'*+-.^_`|~", c) || isdigit(c) || isalpha(c))
char *httpDecodeUrl(const char *enc) {
int32_t ok = 1;
int32_t ok = 1;
HttpString str = {0};
while (*enc) {
char *p = strchr(enc, '%');
if (!p) break;
int32_t hex, cnt;
int32_t n = sscanf(p+1, "%2x%n", &hex, &cnt);
if (n!=1 && cnt !=2) { ok = 0; break; }
if (httpAppendString(&str, enc, p-enc)) { ok = 0; break; }
int32_t n = sscanf(p + 1, "%2x%n", &hex, &cnt);
if (n != 1 && cnt != 2) {
ok = 0;
break;
}
if (httpAppendString(&str, enc, (int32_t)(p - enc))) {
ok = 0;
break;
}
char c = (char)hex;
if (httpAppendString(&str, &c, 1)) { ok = 0; break; }
enc = p+3;
if (httpAppendString(&str, &c, 1)) {
ok = 0;
break;
}
enc = p + 3;
}
char *dec = NULL;
if (ok && *enc) {
if (httpAppendString(&str, enc, strlen(enc))) { ok = 0; }
if (httpAppendString(&str, enc, (int32_t)strlen(enc))) {
ok = 0;
}
}
if (ok) {
dec = str.str;
str.str = NULL;
}
httpCleanupString(&str);
//httpCleanupString(&str);
return dec;
}
static void httpOnData(ehttp_gzip_t *gzip, void *arg, const char *buf, int32_t len) {
HttpParser *parser = (HttpParser*)arg;
HttpParser *parser = (HttpParser *)arg;
httpOnBody(parser, buf, len);
}
static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (c == 'G' || c == 'P' || c == 'H' || c == 'D' || c == 'C' || c == 'O' || c == 'T') {
if (httpAppendString(&parser->str, &c, 1)) {
@ -570,7 +568,7 @@ static int32_t httpParserOnBegin(HttpParser *parser, HTTP_PARSER_STATE state, co
static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (parser->str.pos == 1) {
if (c == 'T' && parser->str.str[0] == 'H') {
@ -608,7 +606,7 @@ static int32_t httpParserOnRquestOrResponse(HttpParser *parser, HTTP_PARSER_STAT
static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) {
if (httpAppendString(&parser->str, &c, 1)) {
@ -637,7 +635,7 @@ static int32_t httpParserOnMethod(HttpParser *parser, HTTP_PARSER_STATE state, c
static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (!isspace(c) && c != '\r' && c != '\n') {
if (httpAppendString(&parser->str, &c, 1)) {
@ -648,7 +646,7 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c
}
break;
}
parser->target = strdup(parser->str.str);
parser->target = httpDecodeUrl(parser->str.str);
if (!parser->target) {
httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c);
ok = -1;
@ -664,10 +662,10 @@ static int32_t httpParserOnTarget(HttpParser *parser, HTTP_PARSER_STATE state, c
static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
const char *prefix = "HTTP/1.";
int32_t len = strlen(prefix);
int32_t len = (int32_t)strlen(prefix);
if (parser->str.pos < len) {
if (prefix[parser->str.pos] != c) {
httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c);
@ -717,6 +715,10 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state,
if (parser->method) {
ok = httpOnRequestLine(parser, parser->method, parser->target, parser->version);
if (parser->target) {
free(parser->target);
parser->target = NULL;
}
}
httpClearString(&parser->str);
@ -727,7 +729,7 @@ static int32_t httpParserOnVersion(HttpParser *parser, HTTP_PARSER_STATE state,
static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (c == ' ') {
httpPopStack(parser);
@ -742,7 +744,7 @@ static int32_t httpParserOnSp(HttpParser *parser, HTTP_PARSER_STATE state, const
static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (isdigit(c)) {
if (httpAppendString(&parser->str, &c, 1)) {
@ -767,7 +769,7 @@ static int32_t httpParserOnStatusCode(HttpParser *parser, HTTP_PARSER_STATE stat
static int32_t httpParserOnReasonPhrase(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (c == '\r') {
parser->reasonPhrase = strdup(parser->str.str);
@ -808,10 +810,10 @@ static int32_t httpParserPostProcess(HttpParser *parser) {
static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
const char *s = "\r\n";
int32_t len = strlen(s);
const char *s = "\r\n";
int32_t len = (int32_t)strlen(s);
if (s[parser->str.pos] != c) {
httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c);
ok = -1;
@ -838,7 +840,7 @@ static int32_t httpParserOnCrlf(HttpParser *parser, HTTP_PARSER_STATE state, con
static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (c == '\r') {
httpPopStack(parser);
@ -876,7 +878,7 @@ static int32_t httpParserOnHeader(HttpParser *parser, HTTP_PARSER_STATE state, c
static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (isalnum(c) || strchr("!#$%&'*+-.^_`|~", c)) {
if (httpAppendString(&parser->str, &c, 1)) {
@ -888,7 +890,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state
break;
}
if (c == ':') {
parser->key = strdup(parser->str.str);
parser->key = strdup(parser->str.str);
if (!parser->key) {
httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c);
ok = -1;
@ -908,7 +910,7 @@ static int32_t httpParserOnHeaderKey(HttpParser *parser, HTTP_PARSER_STATE state
static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (c != '\r' && c != '\n' && (!isspace(c) || parser->str.pos > 0)) {
if (httpAppendString(&parser->str, &c, 1)) {
@ -935,10 +937,10 @@ static int32_t httpParserOnHeaderVal(HttpParser *parser, HTTP_PARSER_STATE state
static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t bytes;
int32_t len;
int32_t n;
int32_t ok = 0;
int32_t bytes;
int32_t len;
int32_t n;
do {
if (isxdigit(c)) {
if (httpAppendString(&parser->str, &c, 1)) {
@ -985,7 +987,7 @@ static int32_t httpParserOnChunkSize(HttpParser *parser, HTTP_PARSER_STATE state
static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
if (httpAppendString(&parser->str, &c, 1)) {
httpError("context:%p, fd:%d, parser state:%d, char:[%c]%02x, oom", pContext, pContext->fd, state, c, c);
@ -1019,7 +1021,7 @@ static int32_t httpParserOnChunk(HttpParser *parser, HTTP_PARSER_STATE state, co
static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
int32_t ok = 0;
do {
ok = -1;
httpError("context:%p, fd:%d, parser state:%d, unexpected char:[%c]%02x", pContext, pContext->fd, state, c, c);
@ -1029,8 +1031,8 @@ static int32_t httpParserOnEnd(HttpParser *parser, HTTP_PARSER_STATE state, cons
}
static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) {
HttpContext *pContext = parser->pContext;
int32_t ok = 0;
HttpContext * pContext = parser->pContext;
int32_t ok = 0;
HTTP_PARSER_STATE state = httpTopStack(parser);
do {
if (state == HTTP_PARSER_BEGIN) {
@ -1119,9 +1121,9 @@ static int32_t httpParseChar(HttpParser *parser, const char c, int32_t *again) {
int32_t httpParseBuf(HttpParser *parser, const char *buf, int32_t len) {
HttpContext *pContext = parser->pContext;
const char *p = buf;
int32_t ret = 0;
int32_t i = 0;
const char * p = buf;
int32_t ret = 0;
int32_t i = 0;
while (i < len) {
int32_t again = 0;

Some files were not shown because too many files have changed in this diff Show More