xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

s3/evict: fetch atime from stat file

This commit is contained in:
Minglei Jin 2023-08-07 15:59:37 +08:00
parent 65d8af19ed
commit fac7e521e9
19 changed files with 300 additions and 225 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/os/osFile.h
View File

@ -76,7 +76,7 @@ int32_t taosUnLockFile(TdFilePtr pFile);
int32_t taosUmaskFile(int32_t maskVal); int32_t taosUmaskFile(int32_t maskVal);
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime); int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime);
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno); int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno);
int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime); int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime);
bool taosCheckExistFile(const char *pathname); bool taosCheckExistFile(const char *pathname);

4
source/dnode/mgmt/mgmt_mnode/src/mmFile.c
View File

@ -46,7 +46,7 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) {
if (code < 0) return -1; if (code < 0) return -1;
tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code); tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code);
if (code < 0) return -1; if (code < 0) return -1;
if(pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER){ if (pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER) {
pOption->numOfReplicas++; pOption->numOfReplicas++;
} }
} }
@ -65,7 +65,7 @@ int32_t mmReadFile(const char *path, SMnodeOpt *pOption) {
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP); snprintf(file, sizeof(file), "%s%smnode.json", path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("mnode file:%s not exist", file); dInfo("mnode file:%s not exist", file);
return 0; return 0;
} }

2
source/dnode/mgmt/mgmt_vnode/src/vmFile.c
View File

@ -97,7 +97,7 @@ int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t
SWrapperCfg *pCfgs = NULL; SWrapperCfg *pCfgs = NULL;
snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP); snprintf(file, sizeof(file), "%s%svnodes.json", pMgmt->path, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("vnode file:%s not exist", file); dInfo("vnode file:%s not exist", file);
return 0; return 0;
} }

6
source/dnode/mgmt/node_util/src/dmEps.c
View File

@ -100,7 +100,7 @@ int32_t dmReadEps(SDnodeData *pData) {
goto _OVER; goto _OVER;
} }
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("dnode file:%s not exist", file); dInfo("dnode file:%s not exist", file);
code = 0; code = 0;
goto _OVER; goto _OVER;
@ -350,7 +350,7 @@ void dmRotateMnodeEpSet(SDnodeData *pData) {
} }
void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) { void dmGetMnodeEpSetForRedirect(SDnodeData *pData, SRpcMsg *pMsg, SEpSet *pEpSet) {
if(!pData->validMnodeEps) return; if (!pData->validMnodeEps) return;
dmGetMnodeEpSet(pData, pEpSet); dmGetMnodeEpSet(pData, pEpSet);
dTrace("msg is redirected, handle:%p num:%d use:%d", pMsg->info.handle, pEpSet->numOfEps, pEpSet->inUse); dTrace("msg is redirected, handle:%p num:%d use:%d", pMsg->info.handle, pEpSet->numOfEps, pEpSet->inUse);
for (int32_t i = 0; i < pEpSet->numOfEps; ++i) { for (int32_t i = 0; i < pEpSet->numOfEps; ++i) {
@ -469,7 +469,7 @@ static int32_t dmReadDnodePairs(SDnodeData *pData) {
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP); snprintf(file, sizeof(file), "%s%sdnode%sep.json", tsDataDir, TD_DIRSEP, TD_DIRSEP);
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dDebug("dnode file:%s not exist", file); dDebug("dnode file:%s not exist", file);
code = 0; code = 0;
goto _OVER; goto _OVER;

2
source/dnode/mgmt/node_util/src/dmFile.c
View File

@ -38,7 +38,7 @@ int32_t dmReadFile(const char *path, const char *name, bool *pDeployed) {
char file[PATH_MAX] = {0}; char file[PATH_MAX] = {0};
snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name); snprintf(file, sizeof(file), "%s%s%s.json", path, TD_DIRSEP, name);
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
dInfo("file:%s not exist", file); dInfo("file:%s not exist", file);
code = 0; code = 0;
goto _OVER; goto _OVER;

3
source/dnode/vnode/src/inc/vndCos.h
View File

@ -30,8 +30,9 @@ void s3PutObjectFromFile(const char *file, const char *object);
void s3DeleteObjects(const char *object_name[], int nobject); void s3DeleteObjects(const char *object_name[], int nobject);
bool s3Exists(const char *object_name); bool s3Exists(const char *object_name);
bool s3Get(const char *object_name, const char *path); bool s3Get(const char *object_name, const char *path);
void s3EvictCache(); void s3EvictCache(const char *path, long object_size);
long s3Size(const char *object_name); long s3Size(const char *object_name);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

2
source/dnode/vnode/src/tq/tqOffsetSnapshot.c
View File

@ -60,7 +60,7 @@ int32_t tqOffsetSnapRead(STqOffsetReader* pReader, uint8_t** ppData) {
} }
int64_t sz = 0; int64_t sz = 0;
if (taosStatFile(fname, &sz, NULL) < 0) { if (taosStatFile(fname, &sz, NULL, NULL) < 0) {
taosCloseFile(&pFile); taosCloseFile(&pFile);
taosMemoryFree(fname); taosMemoryFree(fname);
return -1; return -1;

10
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -176,7 +176,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// SDelFile // SDelFile
if (pTsdb->fs.pDelFile) { if (pTsdb->fs.pDelFile) {
tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname); tsdbDelFileName(pTsdb, pTsdb->fs.pDelFile, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -195,7 +195,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// head ========= // head =========
tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -206,7 +206,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// data ========= // data =========
tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -221,7 +221,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// sma ============= // sma =============
tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }
@ -237,7 +237,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
// stt =========== // stt ===========
for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
if (taosStatFile(fname, &size, NULL)) { if (taosStatFile(fname, &size, NULL, NULL)) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
TSDB_CHECK_CODE(code, lino, _exit); TSDB_CHECK_CODE(code, lino, _exit);
} }

4
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
View File

@ -38,7 +38,7 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
const char *object_name = taosDirEntryBaseName((char *)path); const char *object_name = taosDirEntryBaseName((char *)path);
long s3_size = s3Size(object_name); long s3_size = s3Size(object_name);
if (!strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) { if (!strncmp(path + strlen(path) - 5, ".data", 5) && s3_size > 0) {
s3EvictCache(); s3EvictCache(path, s3_size);
s3Get(object_name, path); s3Get(object_name, path);
pFD->pFD = taosOpenFile(path, flag); pFD->pFD = taosOpenFile(path, flag);
@ -66,7 +66,7 @@ int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **p
// not check file size when reading data files. // not check file size when reading data files.
if (flag != TD_FILE_READ) { if (flag != TD_FILE_READ) {
if (taosStatFile(path, &pFD->szFile, NULL) < 0) { if (taosStatFile(path, &pFD->szFile, NULL, NULL) < 0) {
code = TAOS_SYSTEM_ERROR(errno); code = TAOS_SYSTEM_ERROR(errno);
taosMemoryFree(pFD->pBuf); taosMemoryFree(pFD->pBuf);
taosCloseFile(&pFD->pFD); taosCloseFile(&pFD->pFD);

66
source/dnode/vnode/src/vnd/vnodeCos.c
View File

@ -190,7 +190,71 @@ bool s3Get(const char *object_name, const char *path) {
return ret; return ret;
} }
void s3EvictCache() {} typedef struct {
int64_t size;
int32_t atime;
char name[TSDB_FILENAME_LEN];
} SEvictFile;
static int32_t evictFileCompareAsce(const void *pLeft, const void *pRight) {
SEvictFile *lhs = (SEvictFile *)pLeft;
SEvictFile *rhs = (SEvictFile *)pRight;
return lhs->atime < rhs->atime ? -1 : 1;
}
void s3EvictCache(const char *path, long object_size) {
SDiskSize disk_size = {0};
if (taosGetDiskSize((char *)path, &disk_size) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
vError("failed to get disk:%s size since %s", path, terrstr());
return;
}
if (object_size >= disk_size.avail + 1 << 30) {
// evict too old files
// 1, list data files' atime under dir(path)
char dir_name[TSDB_FILENAME_LEN] = "\0";
tstrncpy(dir_name, path, TSDB_FILENAME_LEN);
taosDirName(dir_name);
tdbDirPtr pDir = taosOpenDir(dir_name);
if (pDir == NULL) {
terrno = TAOS_SYSTEM_ERROR(errno);
vError("failed to open %s since %s", dir_name, terrstr());
}
SArray *evict_files = taosArrayInit(16, sizeof(SEvictFile));
tdbDirEntryPtr pDirEntry;
while ((pDirEntry = taosReadDir(pDir)) != NULL) {
char *name = taosGetDirEntryName(pDirEntry);
if (!strncmp(name + strlen(name) - 5, ".data", 5)) {
SEvictFile e_file = {0};
tstrncpy(e_file.name, name, TSDB_FILENAME_LEN);
taosStatFile(name, &e_file.size, NULL, &e_file.atime);
taosArrayPush(evict_files, &e_file);
}
}
taosCloseDir(&pDir);
// 2, sort by atime
taosArraySort(evict_files, evictFileCompareAsce);
// 3, remove files ascendingly until we get enough object_size space
long evict_size = 0;
size_t ef_size = TARRAY_SIZE(evict_files);
for (size_t i = 0; i < ef_size; ++i) {
SEvictFile *evict_file = taosArrayGet(evict_files, i);
taosRemoveFile(evict_file->name);
evict_size += evict_file->size;
if (evict_size >= object_size) {
break;
}
}
taosArrayDestroy(evict_files);
}
}
long s3Size(const char *object_name) { long s3Size(const char *object_name) {
long size = 0; long size = 0;

8
source/libs/function/src/udfd.c
View File

@ -378,9 +378,9 @@ int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) {
"pyUdfDestroy", "pyUdfScalarProc", "pyUdfAggStart", "pyUdfDestroy", "pyUdfScalarProc", "pyUdfAggStart",
"pyUdfAggFinish", "pyUdfAggProc", "pyUdfAggMerge"}; "pyUdfAggFinish", "pyUdfAggProc", "pyUdfAggMerge"};
void **funcs[UDFD_MAX_PLUGIN_FUNCS] = { void **funcs[UDFD_MAX_PLUGIN_FUNCS] = {
(void **)&plugin->openFunc, (void **)&plugin->closeFunc, (void **)&plugin->udfInitFunc, (void **)&plugin->openFunc, (void **)&plugin->closeFunc, (void **)&plugin->udfInitFunc,
(void **)&plugin->udfDestroyFunc, (void **)&plugin->udfScalarProcFunc, (void **)&plugin->udfAggStartFunc, (void **)&plugin->udfDestroyFunc, (void **)&plugin->udfScalarProcFunc, (void **)&plugin->udfAggStartFunc,
(void **)&plugin->udfAggFinishFunc, (void **)&plugin->udfAggProcFunc, (void **)&plugin->udfAggMergeFunc}; (void **)&plugin->udfAggFinishFunc, (void **)&plugin->udfAggProcFunc, (void **)&plugin->udfAggMergeFunc};
int32_t err = udfdLoadSharedLib(plugin->libPath, &plugin->lib, funcName, funcs, UDFD_MAX_PLUGIN_FUNCS); int32_t err = udfdLoadSharedLib(plugin->libPath, &plugin->lib, funcName, funcs, UDFD_MAX_PLUGIN_FUNCS);
if (err != 0) { if (err != 0) {
fnError("can not load python plugin. lib path %s", plugin->libPath); fnError("can not load python plugin. lib path %s", plugin->libPath);
@ -848,7 +848,7 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
char path[PATH_MAX] = {0}; char path[PATH_MAX] = {0};
udfdGetFuncBodyPath(udf, path); udfdGetFuncBodyPath(udf, path);
bool fileExist = !(taosStatFile(path, NULL, NULL) < 0); bool fileExist = !(taosStatFile(path, NULL, NULL, NULL) < 0);
if (fileExist) { if (fileExist) {
strncpy(udf->path, path, PATH_MAX); strncpy(udf->path, path, PATH_MAX);
fnInfo("udfd func body file. reuse existing file %s", path); fnInfo("udfd func body file. reuse existing file %s", path);

4
source/libs/index/src/indexFstFile.c
View File

@ -162,7 +162,7 @@ static FORCE_INLINE int idxFileCtxGetSize(IFileCtx* ctx) {
return ctx->offset; return ctx->offset;
} else { } else {
int64_t file_size = 0; int64_t file_size = 0;
taosStatFile(ctx->file.buf, &file_size, NULL); taosStatFile(ctx->file.buf, &file_size, NULL, NULL);
return (int)file_size; return (int)file_size;
} }
} }
@ -199,7 +199,7 @@ IFileCtx* idxFileCtxCreate(WriterType type, const char* path, bool readOnly, int
code = taosFtruncateFile(ctx->file.pFile, 0); code = taosFtruncateFile(ctx->file.pFile, 0);
UNUSED(code); UNUSED(code);
code = taosStatFile(path, &ctx->file.size, NULL); code = taosStatFile(path, &ctx->file.size, NULL, NULL);
UNUSED(code); UNUSED(code);
ctx->file.wBufOffset = 0; ctx->file.wBufOffset = 0;

179
source/libs/parser/src/parTranslater.c
View File

@ -31,8 +31,8 @@
#define SYSTABLE_SHOW_TYPE_OFFSET QUERY_NODE_SHOW_DNODES_STMT #define SYSTABLE_SHOW_TYPE_OFFSET QUERY_NODE_SHOW_DNODES_STMT
typedef struct SRewriteTbNameContext { typedef struct SRewriteTbNameContext {
int32_t errCode; int32_t errCode;
char* pTbName; char* pTbName;
} SRewriteTbNameContext; } SRewriteTbNameContext;
typedef struct STranslateContext { typedef struct STranslateContext {
@ -278,10 +278,11 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = {
static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode); static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode);
static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode); static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode);
static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal); static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal);
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt); static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery); SSelectStmt** pStmt);
static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery); static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta, SNode** pQuery);
static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery); static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery);
static int32_t setRefreshMate(STranslateContext* pCxt, SQuery* pQuery);
static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_BY; } static bool afterGroupBy(ESqlClause clause) { return clause > SQL_CLAUSE_GROUP_BY; }
@ -772,7 +773,8 @@ static SNodeList* getProjectList(const SNode* pNode) {
static bool isTimeLineQuery(SNode* pStmt) { static bool isTimeLineQuery(SNode* pStmt) {
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) { if (QUERY_NODE_SELECT_STMT == nodeType(pStmt)) {
return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) || (TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode); return (TIME_LINE_MULTI == ((SSelectStmt*)pStmt)->timeLineResMode) ||
(TIME_LINE_GLOBAL == ((SSelectStmt*)pStmt)->timeLineResMode);
} else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) { } else if (QUERY_NODE_SET_OPERATOR == nodeType(pStmt)) {
return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode; return TIME_LINE_GLOBAL == ((SSetOperator*)pStmt)->timeLineResMode;
} else { } else {
@ -791,7 +793,7 @@ static bool isGlobalTimeLineQuery(SNode* pStmt) {
} }
static bool isTimeLineAlignedQuery(SNode* pStmt) { static bool isTimeLineAlignedQuery(SNode* pStmt) {
SSelectStmt *pSelect = (SSelectStmt *)pStmt; SSelectStmt* pSelect = (SSelectStmt*)pStmt;
if (isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { if (isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
return true; return true;
} }
@ -801,7 +803,7 @@ static bool isTimeLineAlignedQuery(SNode* pStmt) {
if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) { if (QUERY_NODE_SELECT_STMT != nodeType(((STempTableNode*)pSelect->pFromTable)->pSubquery)) {
return false; return false;
} }
SSelectStmt *pSub = (SSelectStmt *)((STempTableNode*)pSelect->pFromTable)->pSubquery; SSelectStmt* pSub = (SSelectStmt*)((STempTableNode*)pSelect->pFromTable)->pSubquery;
if (nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) { if (nodesListMatch(pSelect->pPartitionByList, pSub->pPartitionByList)) {
return true; return true;
} }
@ -1394,7 +1396,7 @@ static bool isCountStar(SFunctionNode* pFunc) {
} }
static int32_t rewriteCountStarAsCount1(STranslateContext* pCxt, SFunctionNode* pCount) { static int32_t rewriteCountStarAsCount1(STranslateContext* pCxt, SFunctionNode* pCount) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == pVal) { if (NULL == pVal) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -1596,9 +1598,11 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC);
} }
if (pSelect->hasInterpFunc && (FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) { if (pSelect->hasInterpFunc &&
(FUNC_RETURN_ROWS_INDEFINITE == pSelect->returnRows || pSelect->returnRows != fmGetFuncReturnRows(pFunc))) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
"%s ignoring null value options cannot be used when applying to multiple columns", pFunc->functionName); "%s ignoring null value options cannot be used when applying to multiple columns",
pFunc->functionName);
} }
if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) { if (NULL != pSelect->pWindow || NULL != pSelect->pGroupByList) {
@ -1636,7 +1640,8 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu
} }
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) && if (NULL != pSelect->pFromTable && QUERY_NODE_TEMP_TABLE == nodeType(pSelect->pFromTable) &&
!isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) && !isTimeLineAlignedQuery(pCxt->pCurrStmt)) { !isGlobalTimeLineQuery(((STempTableNode*)pSelect->pFromTable)->pSubquery) &&
!isTimeLineAlignedQuery(pCxt->pCurrStmt)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC,
"%s function requires valid time series input", pFunc->functionName); "%s function requires valid time series input", pFunc->functionName);
} }
@ -1706,8 +1711,8 @@ static int32_t translateForbidSysTableFunc(STranslateContext* pCxt, SFunctionNod
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt;
SNode* pTable = pSelect->pFromTable; SNode* pTable = pSelect->pFromTable;
if (NULL != pTable && QUERY_NODE_REAL_TABLE == nodeType(pTable) && if (NULL != pTable && QUERY_NODE_REAL_TABLE == nodeType(pTable) &&
TSDB_SYSTEM_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType) { TSDB_SYSTEM_TABLE == ((SRealTableNode*)pTable)->pMeta->tableType) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC, pFunc->functionName); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_SYSTABLE_NOT_ALLOWED_FUNC, pFunc->functionName);
@ -2296,7 +2301,8 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) {
} }
} }
if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) {
if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc || (isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) { if (pSelect->selectFuncNum > 1 || pSelect->hasOtherVectorFunc || !pSelect->hasSelectFunc ||
(isDistinctOrderBy(pCxt) && pCxt->currClause == SQL_CLAUSE_ORDER_BY)) {
return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias); return generateDealNodeErrMsg(pCxt, getGroupByErrorCode(pCxt), ((SExprNode*)(*pNode))->userAlias);
} else { } else {
return rewriteColToSelectValFunc(pCxt, pNode); return rewriteColToSelectValFunc(pCxt, pNode);
@ -2391,14 +2397,14 @@ static int32_t checkHavingGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect)
if (NULL != pSelect->pHaving) { if (NULL != pSelect->pHaving) {
code = checkExprForGroupBy(pCxt, &pSelect->pHaving); code = checkExprForGroupBy(pCxt, &pSelect->pHaving);
} }
/* /*
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pProjectionList) { if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pProjectionList) {
code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pProjectionList); code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pProjectionList);
} }
if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pOrderByList) { if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pOrderByList) {
code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pOrderByList); code = checkExprListForGroupBy(pCxt, pSelect, pSelect->pOrderByList);
} }
*/ */
return code; return code;
} }
@ -2657,10 +2663,10 @@ static int32_t setTableCacheLastMode(STranslateContext* pCxt, SSelectStmt* pSele
static EDealRes doTranslateTbName(SNode** pNode, void* pContext) { static EDealRes doTranslateTbName(SNode** pNode, void* pContext) {
switch (nodeType(*pNode)) { switch (nodeType(*pNode)) {
case QUERY_NODE_FUNCTION: { case QUERY_NODE_FUNCTION: {
SFunctionNode *pFunc = (SFunctionNode *)*pNode; SFunctionNode* pFunc = (SFunctionNode*)*pNode;
if (FUNCTION_TYPE_TBNAME == pFunc->funcType) { if (FUNCTION_TYPE_TBNAME == pFunc->funcType) {
SRewriteTbNameContext *pCxt = (SRewriteTbNameContext*)pContext; SRewriteTbNameContext* pCxt = (SRewriteTbNameContext*)pContext;
SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == pVal) { if (NULL == pVal) {
pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY;
return DEAL_RES_ERROR; return DEAL_RES_ERROR;
@ -2699,11 +2705,12 @@ static int32_t replaceTbName(STranslateContext* pCxt, SSelectStmt* pSelect) {
} }
SRealTableNode* pTable = (SRealTableNode*)pSelect->pFromTable; SRealTableNode* pTable = (SRealTableNode*)pSelect->pFromTable;
if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType && TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) { if (TSDB_CHILD_TABLE != pTable->pMeta->tableType && TSDB_NORMAL_TABLE != pTable->pMeta->tableType &&
TSDB_SYSTEM_TABLE != pTable->pMeta->tableType) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
SNode** pNode = NULL; SNode** pNode = NULL;
SRewriteTbNameContext pRewriteCxt = {0}; SRewriteTbNameContext pRewriteCxt = {0};
pRewriteCxt.pTbName = pTable->table.tableName; pRewriteCxt.pTbName = pTable->table.tableName;
@ -3110,7 +3117,8 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList
code = scalarCalculateConstants(pCastFunc, &pCell->pNode); code = scalarCalculateConstants(pCastFunc, &pCell->pNode);
} }
if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) { if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) {
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant"); code =
generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value can only accept constant");
} else if (TSDB_CODE_SUCCESS != code) { } else if (TSDB_CODE_SUCCESS != code) {
code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch");
} }
@ -3576,7 +3584,6 @@ static int32_t createDefaultEveryNode(STranslateContext* pCxt, SNode** pOutput)
pEvery->isDuration = true; pEvery->isDuration = true;
pEvery->literal = taosStrdup("1s"); pEvery->literal = taosStrdup("1s");
*pOutput = (SNode*)pEvery; *pOutput = (SNode*)pEvery;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -3675,8 +3682,8 @@ static int32_t translatePartitionBy(STranslateContext* pCxt, SSelectStmt* pSelec
if (pSelect->pPartitionByList) { if (pSelect->pPartitionByList) {
int8_t typeType = getTableTypeFromTableNode(pSelect->pFromTable); int8_t typeType = getTableTypeFromTableNode(pSelect->pFromTable);
SNode* pPar = nodesListGetNode(pSelect->pPartitionByList, 0); SNode* pPar = nodesListGetNode(pSelect->pPartitionByList, 0);
if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) && if (!((TSDB_NORMAL_TABLE == typeType || TSDB_CHILD_TABLE == typeType) && 1 == pSelect->pPartitionByList->length &&
1 == pSelect->pPartitionByList->length && (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) { (QUERY_NODE_FUNCTION == nodeType(pPar) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPar)->funcType))) {
pSelect->timeLineResMode = TIME_LINE_MULTI; pSelect->timeLineResMode = TIME_LINE_MULTI;
} }
@ -3943,9 +3950,9 @@ static int32_t translateSetOperProject(STranslateContext* pCxt, SSetOperator* pS
} }
snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName); snprintf(pRightExpr->aliasName, sizeof(pRightExpr->aliasName), "%s", pLeftExpr->aliasName);
SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft); SNode* pProj = createSetOperProject(pSetOperator->stmtName, pLeft);
if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight) if (QUERY_NODE_COLUMN == nodeType(pLeft) && QUERY_NODE_COLUMN == nodeType(pRight) &&
&& ((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID ((SColumnNode*)pLeft)->colId == PRIMARYKEY_TIMESTAMP_COL_ID &&
&& ((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { ((SColumnNode*)pRight)->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
((SColumnNode*)pProj)->colId = PRIMARYKEY_TIMESTAMP_COL_ID; ((SColumnNode*)pProj)->colId = PRIMARYKEY_TIMESTAMP_COL_ID;
} }
if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) { if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pSetOperator->pProjectionList, pProj)) {
@ -5725,7 +5732,6 @@ static int32_t translateRestoreDnode(STranslateContext* pCxt, SRestoreComponentN
return buildCmdMsg(pCxt, TDMT_MND_RESTORE_DNODE, (FSerializeFunc)tSerializeSRestoreDnodeReq, &restoreReq); return buildCmdMsg(pCxt, TDMT_MND_RESTORE_DNODE, (FSerializeFunc)tSerializeSRestoreDnodeReq, &restoreReq);
} }
static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, const char* pDbName, const char* pTableName, static int32_t getSmaIndexDstVgId(STranslateContext* pCxt, const char* pDbName, const char* pTableName,
int32_t* pVgId) { int32_t* pVgId) {
SVgroupInfo vg = {0}; SVgroupInfo vg = {0};
@ -5853,7 +5859,7 @@ static int32_t checkCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pS
} }
static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) { static int32_t translateCreateSmaIndex(STranslateContext* pCxt, SCreateIndexStmt* pStmt) {
int32_t code = checkCreateSmaIndex(pCxt, pStmt); int32_t code = checkCreateSmaIndex(pCxt, pStmt);
pStmt->pReq = taosMemoryCalloc(1, sizeof(SMCreateSmaReq)); pStmt->pReq = taosMemoryCalloc(1, sizeof(SMCreateSmaReq));
if (pStmt->pReq == NULL) code = TSDB_CODE_OUT_OF_MEMORY; if (pStmt->pReq == NULL) code = TSDB_CODE_OUT_OF_MEMORY;
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -5867,13 +5873,15 @@ int32_t createIntervalFromCreateSmaIndexStmt(SCreateIndexStmt* pStmt, SInterval*
pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i; pInterval->interval = ((SValueNode*)pStmt->pOptions->pInterval)->datum.i;
pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit; pInterval->intervalUnit = ((SValueNode*)pStmt->pOptions->pInterval)->unit;
pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0; pInterval->offset = NULL != pStmt->pOptions->pOffset ? ((SValueNode*)pStmt->pOptions->pOffset)->datum.i : 0;
pInterval->sliding = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval; pInterval->sliding =
pInterval->slidingUnit = NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit; NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->datum.i : pInterval->interval;
pInterval->slidingUnit =
NULL != pStmt->pOptions->pSliding ? ((SValueNode*)pStmt->pOptions->pSliding)->unit : pInterval->intervalUnit;
pInterval->precision = pStmt->pOptions->tsPrecision; pInterval->precision = pStmt->pOptions->tsPrecision;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void ** pResRow) { int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot; SCreateIndexStmt* pStmt = (SCreateIndexStmt*)pQuery->pRoot;
int64_t lastTs = 0; int64_t lastTs = 0;
@ -6041,7 +6049,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS
toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name); toName(pCxt->pParseCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name);
tNameGetFullDbName(&name, pReq->subDbName); tNameGetFullDbName(&name, pReq->subDbName);
tNameExtractFullName(&name, pReq->subStbName); tNameExtractFullName(&name, pReq->subStbName);
if(pStmt->pQuery != NULL) { if (pStmt->pQuery != NULL) {
code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL);
} }
} else if ('\0' != pStmt->subDbName[0]) { } else if ('\0' != pStmt->subDbName[0]) {
@ -6100,7 +6108,8 @@ static EDealRes checkColumnTagsInCond(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE; return DEAL_RES_CONTINUE;
} }
static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta, SNodeList** ppProjection) { static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt* pStmt, STableMeta* pMeta,
SNodeList** ppProjection) {
SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL}; SBuildTopicContext colCxt = {.colExists = false, .colNotFound = false, .pMeta = pMeta, .pTags = NULL};
nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt); nodesWalkExprPostOrder(pStmt->pWhere, checkColumnTagsInCond, &colCxt);
if (colCxt.colNotFound) { if (colCxt.colNotFound) {
@ -6110,18 +6119,18 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt*
nodesDestroyList(colCxt.pTags); nodesDestroyList(colCxt.pTags);
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Columns are forbidden in where clause"); return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, "Columns are forbidden in where clause");
} }
if (NULL == colCxt.pTags) { // put one column to select if (NULL == colCxt.pTags) { // put one column to select
// for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) { // for (int32_t i = 0; i < pMeta->tableInfo.numOfColumns; ++i) {
SSchema* column = &pMeta->schema[0]; SSchema* column = &pMeta->schema[0];
SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); SColumnNode* col = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN);
if (NULL == col) { if (NULL == col) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
strcpy(col->colName, column->name); strcpy(col->colName, column->name);
strcpy(col->node.aliasName, col->colName); strcpy(col->node.aliasName, col->colName);
strcpy(col->node.userAlias, col->colName); strcpy(col->node.userAlias, col->colName);
addTagList(&colCxt.pTags, (SNode*)col); addTagList(&colCxt.pTags, (SNode*)col);
// } // }
} }
*ppProjection = colCxt.pTags; *ppProjection = colCxt.pTags;
@ -6129,13 +6138,13 @@ static int32_t checkCollectTopicTags(STranslateContext* pCxt, SCreateTopicStmt*
} }
static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SNode** pSelect) { static int32_t buildQueryForTableTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SNode** pSelect) {
SParseContext* pParCxt = pCxt->pParseCxt; SParseContext* pParCxt = pCxt->pParseCxt;
SRequestConnInfo connInfo = {.pTrans = pParCxt->pTransporter, SRequestConnInfo connInfo = {.pTrans = pParCxt->pTransporter,
.requestId = pParCxt->requestId, .requestId = pParCxt->requestId,
.requestObjRefId = pParCxt->requestRid, .requestObjRefId = pParCxt->requestRid,
.mgmtEps = pParCxt->mgmtEpSet}; .mgmtEps = pParCxt->mgmtEpSet};
SName name; SName name;
STableMeta* pMeta = NULL; STableMeta* pMeta = NULL;
int32_t code = getTableMetaImpl(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta); int32_t code = getTableMetaImpl(pCxt, toName(pParCxt->acctId, pStmt->subDbName, pStmt->subSTbName, &name), &pMeta);
if (code) { if (code) {
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
@ -6542,7 +6551,8 @@ static int32_t checkStreamQuery(STranslateContext* pCxt, SCreateStreamStmt* pStm
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"SUBTABLE expression must be of VARCHAR type"); "SUBTABLE expression must be of VARCHAR type");
} }
if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) && subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) { if (NULL != pSelect->pSubtable && 0 == LIST_LENGTH(pSelect->pPartitionByList) &&
subtableExprHasColumnOrPseudoColumn(pSelect->pSubtable)) {
return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY, return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_STREAM_QUERY,
"SUBTABLE expression must not has column when no partition by clause"); "SUBTABLE expression must not has column when no partition by clause");
} }
@ -6900,13 +6910,13 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta
strcpy(col->colName, pMeta->schema[0].name); strcpy(col->colName, pMeta->schema[0].name);
SNodeList* pParamterList = nodesMakeList(); SNodeList* pParamterList = nodesMakeList();
if (NULL == pParamterList) { if (NULL == pParamterList) {
nodesDestroyNode((SNode *)col); nodesDestroyNode((SNode*)col);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
int32_t code = nodesListStrictAppend(pParamterList, (SNode *)col); int32_t code = nodesListStrictAppend(pParamterList, (SNode*)col);
if (code) { if (code) {
nodesDestroyNode((SNode *)col); nodesDestroyNode((SNode*)col);
nodesDestroyList(pParamterList); nodesDestroyList(pParamterList);
return code; return code;
} }
@ -6929,7 +6939,7 @@ static int32_t createLastTsSelectStmt(char* pDb, char* pTable, STableMeta* pMeta
return code; return code;
} }
code = createSimpleSelectStmtFromProjList(pDb, pTable, pProjectionList, (SSelectStmt **)pQuery); code = createSimpleSelectStmtFromProjList(pDb, pTable, pProjectionList, (SSelectStmt**)pQuery);
if (code) { if (code) {
nodesDestroyList(pProjectionList); nodesDestroyList(pProjectionList);
return code; return code;
@ -6967,14 +6977,14 @@ static int32_t buildCreateStreamQuery(STranslateContext* pCxt, SCreateStreamStmt
if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory) { if (TSDB_CODE_SUCCESS == code && pStmt->pOptions->fillHistory) {
SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable); SRealTableNode* pTable = (SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable);
code = createLastTsSelectStmt(pTable->table.dbName, pTable->table.tableName, pTable->pMeta, &pStmt->pPrevQuery); code = createLastTsSelectStmt(pTable->table.dbName, pTable->table.tableName, pTable->pMeta, &pStmt->pPrevQuery);
/* /*
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
STranslateContext cxt = {0}; STranslateContext cxt = {0};
int32_t code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt); int32_t code = initTranslateContext(pCxt->pParseCxt, pCxt->pMetaCache, &cxt);
code = translateQuery(&cxt, pStmt->pPrevQuery); code = translateQuery(&cxt, pStmt->pPrevQuery);
destroyTranslateContext(&cxt); destroyTranslateContext(&cxt);
} }
*/ */
} }
taosMemoryFree(pMeta); taosMemoryFree(pMeta);
return code; return code;
@ -7084,9 +7094,9 @@ static int32_t buildIntervalForCreateStream(SCreateStreamStmt* pStmt, SInterval*
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow) { int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void** pResRow) {
SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pQuery->pRoot; SCreateStreamStmt* pStmt = (SCreateStreamStmt*)pQuery->pRoot;
STranslateContext cxt = {0}; STranslateContext cxt = {0};
SInterval interval = {0}; SInterval interval = {0};
int64_t lastTs = 0; int64_t lastTs = 0;
int32_t code = initTranslateContext(pParseCxt, NULL, &cxt); int32_t code = initTranslateContext(pParseCxt, NULL, &cxt);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
@ -7121,7 +7131,6 @@ int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, void
return code; return code;
} }
static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) { static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) {
SMDropStreamReq dropReq = {0}; SMDropStreamReq dropReq = {0};
SName name; SName name;
@ -7152,7 +7161,7 @@ static int32_t translateResumeStream(STranslateContext* pCxt, SResumeStreamStmt*
static int32_t readFromFile(char* pName, int32_t* len, char** buf) { static int32_t readFromFile(char* pName, int32_t* len, char** buf) {
int64_t filesize = 0; int64_t filesize = 0;
if (taosStatFile(pName, &filesize, NULL) < 0) { if (taosStatFile(pName, &filesize, NULL, NULL) < 0) {
return TAOS_SYSTEM_ERROR(errno); return TAOS_SYSTEM_ERROR(errno);
} }
@ -7246,7 +7255,7 @@ static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt,
} }
} }
int32_t code = createRealTableForGrantTable(pStmt, &pTable); int32_t code = createRealTableForGrantTable(pStmt, &pTable);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
SName name; SName name;
code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name),
@ -7806,7 +7815,8 @@ static SNodeList* createProjectCols(int32_t ncols, const char* const pCols[]) {
return pProjections; return pProjections;
} }
static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) { static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, SNodeList* pProjectionList,
SSelectStmt** pStmt) {
SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT); SSelectStmt* pSelect = (SSelectStmt*)nodesMakeNode(QUERY_NODE_SELECT_STMT);
if (NULL == pSelect) { if (NULL == pSelect) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
@ -7829,9 +7839,8 @@ static int32_t createSimpleSelectStmtImpl(const char* pDb, const char* pTable, S
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTable, int32_t numOfProjs, static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTable, int32_t numOfProjs,
const char* const pProjCol[], SSelectStmt** pStmt) { const char* const pProjCol[], SSelectStmt** pStmt) {
SNodeList* pProjectionList = NULL; SNodeList* pProjectionList = NULL;
if (numOfProjs >= 0) { if (numOfProjs >= 0) {
pProjectionList = createProjectCols(numOfProjs, pProjCol); pProjectionList = createProjectCols(numOfProjs, pProjCol);
@ -7843,13 +7852,15 @@ static int32_t createSimpleSelectStmtFromCols(const char* pDb, const char* pTabl
return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt);
} }
static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList, SSelectStmt** pStmt) { static int32_t createSimpleSelectStmtFromProjList(const char* pDb, const char* pTable, SNodeList* pProjectionList,
SSelectStmt** pStmt) {
return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt); return createSimpleSelectStmtImpl(pDb, pTable, pProjectionList, pStmt);
} }
static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) { static int32_t createSelectStmtForShow(ENodeType showType, SSelectStmt** pStmt) {
const SSysTableShowAdapter* pShow = &sysTableShowAdapter[showType - SYSTABLE_SHOW_TYPE_OFFSET]; const SSysTableShowAdapter* pShow = &sysTableShowAdapter[showType - SYSTABLE_SHOW_TYPE_OFFSET];
return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols, pStmt); return createSimpleSelectStmtFromCols(pShow->pDbName, pShow->pTableName, pShow->numOfShowCols, pShow->pShowCols,
pStmt);
} }
static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) { static int32_t createSelectStmtForShowTableDist(SShowTableDistributedStmt* pStmt, SSelectStmt** pOutput) {
@ -7987,8 +7998,8 @@ static int32_t createShowTableTagsProjections(SNodeList** pProjections, SNodeLis
static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) { static int32_t rewriteShowStableTags(STranslateContext* pCxt, SQuery* pQuery) {
SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot; SShowTableTagsStmt* pShow = (SShowTableTagsStmt*)pQuery->pRoot;
SSelectStmt* pSelect = NULL; SSelectStmt* pSelect = NULL;
int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal, ((SValueNode*)pShow->pTbName)->literal, int32_t code = createSimpleSelectStmtFromCols(((SValueNode*)pShow->pDbName)->literal,
-1, NULL, &pSelect); ((SValueNode*)pShow->pTbName)->literal, -1, NULL, &pSelect);
if (TSDB_CODE_SUCCESS == code) { if (TSDB_CODE_SUCCESS == code) {
code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags); code = createShowTableTagsProjections(&pSelect->pProjectionList, &pShow->pTags);
} }

2
source/libs/sync/src/syncRaftStore.c
View File

@ -42,7 +42,7 @@ int32_t raftStoreReadFile(SSyncNode *pNode) {
const char *file = pNode->raftStorePath; const char *file = pNode->raftStorePath;
SRaftStore *pStore = &pNode->raftStore; SRaftStore *pStore = &pNode->raftStore;
if (taosStatFile(file, NULL, NULL) < 0) { if (taosStatFile(file, NULL, NULL, NULL) < 0) {
sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file); sInfo("vgId:%d, raft store file:%s not exist, use default value", pNode->vgId, file);
pStore->currentTerm = 0; pStore->currentTerm = 0;
pStore->voteFor.addr = 0; pStore->voteFor.addr = 0;

12
source/libs/wal/src/walMeta.c
View File

@ -53,7 +53,7 @@ static FORCE_INLINE int64_t walScanLogGetLastVer(SWal* pWal, int32_t fileIdx) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0; int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL); taosStatFile(fnameStr, &fileSize, NULL, NULL);
TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE); TdFilePtr pFile = taosOpenFile(fnameStr, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) { if (pFile == NULL) {
@ -304,7 +304,7 @@ int walRepairLogFileTs(SWal* pWal, bool* updateMeta) {
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t mtime = 0; int32_t mtime = 0;
if (taosStatFile(fnameStr, NULL, &mtime) < 0) { if (taosStatFile(fnameStr, NULL, &mtime, NULL) < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); wError("vgId:%d, failed to stat file due to %s, file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
return -1; return -1;
@ -353,7 +353,7 @@ int walTrimIdxFile(SWal* pWal, int32_t fileIdx) {
walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr);
int64_t fileSize = 0; int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL); taosStatFile(fnameStr, &fileSize, NULL, NULL);
int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1); int64_t records = TMAX(0, pFileInfo->lastVer - pFileInfo->firstVer + 1);
int64_t lastEndOffset = records * sizeof(SWalIdxEntry); int64_t lastEndOffset = records * sizeof(SWalIdxEntry);
@ -436,7 +436,7 @@ int walCheckAndRepairMeta(SWal* pWal) {
SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx); SWalFileInfo* pFileInfo = taosArrayGet(pWal->fileInfoSet, fileIdx);
walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); walBuildLogName(pWal, pFileInfo->firstVer, fnameStr);
int32_t code = taosStatFile(fnameStr, &fileSize, NULL); int32_t code = taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (code < 0) { if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
wError("failed to stat file since %s. file:%s", terrstr(), fnameStr); wError("failed to stat file since %s. file:%s", terrstr(), fnameStr);
@ -522,7 +522,7 @@ int walCheckAndRepairIdxFile(SWal* pWal, int32_t fileIdx) {
walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr); walBuildLogName(pWal, pFileInfo->firstVer, fLogNameStr);
int64_t fileSize = 0; int64_t fileSize = 0;
if (taosStatFile(fnameStr, &fileSize, NULL) < 0 && errno != ENOENT) { if (taosStatFile(fnameStr, &fileSize, NULL, NULL) < 0 && errno != ENOENT) {
wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr); wError("vgId:%d, failed to stat file due to %s. file:%s", pWal->cfg.vgId, strerror(errno), fnameStr);
terrno = TAOS_SYSTEM_ERROR(errno); terrno = TAOS_SYSTEM_ERROR(errno);
return -1; return -1;
@ -935,7 +935,7 @@ int walLoadMeta(SWal* pWal) {
walBuildMetaName(pWal, metaVer, fnameStr); walBuildMetaName(pWal, metaVer, fnameStr);
// read metafile // read metafile
int64_t fileSize = 0; int64_t fileSize = 0;
taosStatFile(fnameStr, &fileSize, NULL); taosStatFile(fnameStr, &fileSize, NULL, NULL);
if (fileSize == 0) { if (fileSize == 0) {
(void)taosRemoveFile(fnameStr); (void)taosRemoveFile(fnameStr);
wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer); wDebug("vgId:%d, wal find empty meta ver %d", pWal->cfg.vgId, metaVer);

28
source/os/src/osFile.c
View File

@ -191,7 +191,7 @@ int32_t taosRenameFile(const char *oldName, const char *newName) {
#endif #endif
} }
int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) { int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime, int32_t *atime) {
#ifdef WINDOWS #ifdef WINDOWS
struct _stati64 fileStat; struct _stati64 fileStat;
int32_t code = _stati64(path, &fileStat); int32_t code = _stati64(path, &fileStat);
@ -211,6 +211,10 @@ int32_t taosStatFile(const char *path, int64_t *size, int32_t *mtime) {
*mtime = fileStat.st_mtime; *mtime = fileStat.st_mtime;
} }
if (atime != NULL) {
*atime = fileStat.st_mtime;
}
return 0; return 0;
} }
int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) { int32_t taosDevInoFile(TdFilePtr pFile, int64_t *stDev, int64_t *stIno) {
@ -540,7 +544,7 @@ int32_t taosFStatFile(TdFilePtr pFile, int64_t *size, int32_t *mtime) {
#ifdef WINDOWS #ifdef WINDOWS
struct __stat64 fileStat; struct __stat64 fileStat;
int32_t code = _fstat64(pFile->fd, &fileStat); int32_t code = _fstat64(pFile->fd, &fileStat);
#else #else
struct stat fileStat; struct stat fileStat;
int32_t code = fstat(pFile->fd, &fileStat); int32_t code = fstat(pFile->fd, &fileStat);
@ -897,17 +901,17 @@ int32_t taosCompressFile(char *srcFileName, char *destFileName) {
goto cmp_end; goto cmp_end;
} }
dstFp = gzdopen(pFile->fd, "wb6f"); dstFp = gzdopen(pFile->fd, "wb6f");
if (dstFp == NULL) { if (dstFp == NULL) {
ret = -3; ret = -3;
taosCloseFile(&pFile); taosCloseFile(&pFile);
goto cmp_end; goto cmp_end;
} }
while (!feof(pSrcFile->fp)) { while (!feof(pSrcFile->fp)) {
len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp); len = (int32_t)fread(data, 1, compressSize, pSrcFile->fp);
(void)gzwrite(dstFp, data, len); (void)gzwrite(dstFp, data, len);
} }
cmp_end: cmp_end:
if (pSrcFile) { if (pSrcFile) {

40
source/util/src/tlog.c
View File

@ -17,8 +17,8 @@
#include "tlog.h" #include "tlog.h"
#include "os.h" #include "os.h"
#include "tconfig.h" #include "tconfig.h"
#include "tjson.h"
#include "tglobal.h" #include "tglobal.h"
#include "tjson.h"
#define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_SIZE (10024)
#define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3)
@ -74,12 +74,12 @@ static SLogObj tsLogObj = {.fileNum = 1};
static int64_t tsAsyncLogLostLines = 0; static int64_t tsAsyncLogLostLines = 0;
static int32_t tsDaylightActive; /* Currently in daylight saving time. */ static int32_t tsDaylightActive; /* Currently in daylight saving time. */
bool tsLogEmbedded = 0; bool tsLogEmbedded = 0;
bool tsAsyncLog = true; bool tsAsyncLog = true;
#ifdef ASSERT_NOT_CORE #ifdef ASSERT_NOT_CORE
bool tsAssert = false; bool tsAssert = false;
#else #else
bool tsAssert = true; bool tsAssert = true;
#endif #endif
int32_t tsNumOfLogLines = 10000000; int32_t tsNumOfLogLines = 10000000;
int32_t tsLogKeepDays = 0; int32_t tsLogKeepDays = 0;
@ -403,13 +403,13 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
strcpy(name, fn); strcpy(name, fn);
strcat(name, ".0"); strcat(name, ".0");
} }
bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime) >= 0; bool log0Exist = taosStatFile(name, NULL, &logstat0_mtime, NULL) >= 0;
if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) { if (strlen(fn) < LOG_FILE_NAME_LEN + 50 - 2) {
strcpy(name, fn); strcpy(name, fn);
strcat(name, ".1"); strcat(name, ".1");
} }
bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime) >= 0; bool log1Exist = taosStatFile(name, NULL, &logstat1_mtime, NULL) >= 0;
// if none of the log files exist, open 0, if both exists, open the old one // if none of the log files exist, open 0, if both exists, open the old one
if (!log0Exist && !log1Exist) { if (!log0Exist && !log1Exist) {
@ -834,12 +834,12 @@ bool taosAssertDebug(bool condition, const char *file, int32_t line, const char
return true; return true;
} }
void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, void *sigInfo) { void taosLogCrashInfo(char *nodeType, char *pMsg, int64_t msgLen, int signum, void *sigInfo) {
const char *flags = "UTL FATAL "; const char *flags = "UTL FATAL ";
ELogLevel level = DEBUG_FATAL; ELogLevel level = DEBUG_FATAL;
int32_t dflag = 255; int32_t dflag = 255;
char filepath[PATH_MAX] = {0}; char filepath[PATH_MAX] = {0};
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
if (pMsg && msgLen > 0) { if (pMsg && msgLen > 0) {
snprintf(filepath, sizeof(filepath), "%s%s.%sCrashLog", tsLogDir, TD_DIRSEP, nodeType); snprintf(filepath, sizeof(filepath), "%s%s.%sCrashLog", tsLogDir, TD_DIRSEP, nodeType);
@ -857,7 +857,7 @@ void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, vo
if (sizeof(msgLen) != writeSize) { if (sizeof(msgLen) != writeSize) {
taosUnLockFile(pFile); taosUnLockFile(pFile);
taosPrintLog(flags, level, dflag, "failed to write len to file:%s,%p wlen:%" PRId64 " tlen:%lu since %s", taosPrintLog(flags, level, dflag, "failed to write len to file:%s,%p wlen:%" PRId64 " tlen:%lu since %s",
filepath, pFile, writeSize, sizeof(msgLen), terrstr()); filepath, pFile, writeSize, sizeof(msgLen), terrstr());
goto _return; goto _return;
} }
@ -865,7 +865,7 @@ void taosLogCrashInfo(char* nodeType, char* pMsg, int64_t msgLen, int signum, vo
if (msgLen != writeSize) { if (msgLen != writeSize) {
taosUnLockFile(pFile); taosUnLockFile(pFile);
taosPrintLog(flags, level, dflag, "failed to write file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", taosPrintLog(flags, level, dflag, "failed to write file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s",
filepath, pFile, writeSize, msgLen, terrstr()); filepath, pFile, writeSize, msgLen, terrstr());
goto _return; goto _return;
} }
@ -883,7 +883,7 @@ _return:
taosPrintTrace(flags, level, dflag, 4); taosPrintTrace(flags, level, dflag, 4);
#elif !defined(WINDOWS) #elif !defined(WINDOWS)
taosPrintLog(flags, level, dflag, "sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, taosPrintLog(flags, level, dflag, "sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid,
taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid));
taosPrintTrace(flags, level, dflag, 3); taosPrintTrace(flags, level, dflag, 3);
#else #else
taosPrintTrace(flags, level, dflag, 8); taosPrintTrace(flags, level, dflag, 8);
@ -892,17 +892,17 @@ _return:
taosMemoryFree(pMsg); taosMemoryFree(pMsg);
} }
void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr* pFd) { void taosReadCrashInfo(char *filepath, char **pMsg, int64_t *pMsgLen, TdFilePtr *pFd) {
const char *flags = "UTL FATAL "; const char *flags = "UTL FATAL ";
ELogLevel level = DEBUG_FATAL; ELogLevel level = DEBUG_FATAL;
int32_t dflag = 255; int32_t dflag = 255;
TdFilePtr pFile = NULL; TdFilePtr pFile = NULL;
bool truncateFile = false; bool truncateFile = false;
char* buf = NULL; char *buf = NULL;
if (NULL == *pFd) { if (NULL == *pFd) {
int64_t filesize = 0; int64_t filesize = 0;
if (taosStatFile(filepath, &filesize, NULL) < 0) { if (taosStatFile(filepath, &filesize, NULL, NULL) < 0) {
if (ENOENT == errno) { if (ENOENT == errno) {
return; return;
} }
@ -916,7 +916,7 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
return; return;
} }
pFile = taosOpenFile(filepath, TD_FILE_READ|TD_FILE_WRITE); pFile = taosOpenFile(filepath, TD_FILE_READ | TD_FILE_WRITE);
if (pFile == NULL) { if (pFile == NULL) {
if (ENOENT == errno) { if (ENOENT == errno) {
return; return;
@ -938,7 +938,7 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
truncateFile = true; truncateFile = true;
if (readSize < 0) { if (readSize < 0) {
taosPrintLog(flags, level, dflag, "failed to read len from file:%s,%p wlen:%" PRId64 " tlen:%lu since %s", taosPrintLog(flags, level, dflag, "failed to read len from file:%s,%p wlen:%" PRId64 " tlen:%lu since %s",
filepath, pFile, readSize, sizeof(msgLen), terrstr()); filepath, pFile, readSize, sizeof(msgLen), terrstr());
} }
goto _return; goto _return;
} }
@ -952,8 +952,8 @@ void taosReadCrashInfo(char* filepath, char** pMsg, int64_t* pMsgLen, TdFilePtr*
readSize = taosReadFile(pFile, buf, msgLen); readSize = taosReadFile(pFile, buf, msgLen);
if (msgLen != readSize) { if (msgLen != readSize) {
truncateFile = true; truncateFile = true;
taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", taosPrintLog(flags, level, dflag, "failed to read file:%s,%p wlen:%" PRId64 " tlen:%" PRId64 " since %s", filepath,
filepath, pFile, readSize, msgLen, terrstr()); pFile, readSize, msgLen, terrstr());
goto _return; goto _return;
} }

94
tools/shell/src/shellEngine.c
View File

@ -18,9 +18,9 @@
#define _GNU_SOURCE #define _GNU_SOURCE
#define _XOPEN_SOURCE #define _XOPEN_SOURCE
#define _DEFAULT_SOURCE #define _DEFAULT_SOURCE
#include "shellInt.h"
#include "shellAuto.h"
#include "geosWrapper.h" #include "geosWrapper.h"
#include "shellAuto.h"
#include "shellInt.h"
static bool shellIsEmptyCommand(const char *cmd); static bool shellIsEmptyCommand(const char *cmd);
static int32_t shellRunSingleCommand(char *command); static int32_t shellRunSingleCommand(char *command);
@ -41,9 +41,9 @@ static bool shellIsCommentLine(char *line);
static void shellSourceFile(const char *file); static void shellSourceFile(const char *file);
static void shellGetGrantInfo(); static void shellGetGrantInfo();
static void shellCleanup(void *arg); static void shellCleanup(void *arg);
static void *shellCancelHandler(void *arg); static void *shellCancelHandler(void *arg);
static void *shellThreadLoop(void *arg); static void *shellThreadLoop(void *arg);
bool shellIsEmptyCommand(const char *cmd) { bool shellIsEmptyCommand(const char *cmd) {
for (char c = *cmd++; c != 0; c = *cmd++) { for (char c = *cmd++; c != 0; c = *cmd++) {
@ -66,7 +66,7 @@ int32_t shellRunSingleCommand(char *command) {
if (shellRegexMatch(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) { if (shellRegexMatch(command, "^[\t ]*clear[ \t;]*$", REG_EXTENDED | REG_ICASE)) {
#pragma GCC diagnostic push #pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-result" #pragma GCC diagnostic ignored "-Wunused-result"
system("clear"); system("clear");
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
return 0; return 0;
} }
@ -143,7 +143,7 @@ int32_t shellRunCommand(char *command, bool recordHistory) {
} }
// add help or help; // add help or help;
if(strncasecmp(command, "help;", 5) == 0) { if (strncasecmp(command, "help;", 5) == 0) {
showHelp(); showHelp();
return 0; return 0;
} }
@ -223,14 +223,14 @@ void shellRunSingleCommandImp(char *command) {
} }
// pre string // pre string
char * pre = "Query OK"; char *pre = "Query OK";
if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) { if (shellRegexMatch(command, "^\\s*delete\\s*from\\s*.*", REG_EXTENDED | REG_ICASE)) {
pre = "Delete OK"; pre = "Delete OK";
} else if(shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) { } else if (shellRegexMatch(command, "^\\s*insert\\s*into\\s*.*", REG_EXTENDED | REG_ICASE)) {
pre = "Insert OK"; pre = "Insert OK";
} else if(shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) { } else if (shellRegexMatch(command, "^\\s*create\\s*.*", REG_EXTENDED | REG_ICASE)) {
pre = "Create OK"; pre = "Create OK";
} else if(shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) { } else if (shellRegexMatch(command, "^\\s*drop\\s*.*", REG_EXTENDED | REG_ICASE)) {
pre = "Drop OK"; pre = "Drop OK";
} }
@ -295,7 +295,7 @@ char *shellFormatTimestamp(char *buf, int64_t val, int32_t precision) {
if (taosLocalTime(&tt, &ptm, buf) == NULL) { if (taosLocalTime(&tt, &ptm, buf) == NULL) {
return buf; return buf;
} }
size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm);
if (precision == TSDB_TIME_PRECISION_NANO) { if (precision == TSDB_TIME_PRECISION_NANO) {
sprintf(buf + pos, ".%09d", ms); sprintf(buf + pos, ".%09d", ms);
@ -387,22 +387,20 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i
break; break;
case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_BINARY:
case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_NCHAR:
case TSDB_DATA_TYPE_JSON: case TSDB_DATA_TYPE_JSON: {
{ int32_t bufIndex = 0;
int32_t bufIndex = 0; for (int32_t i = 0; i < length; i++) {
for (int32_t i = 0; i < length; i++) { buf[bufIndex] = val[i];
bufIndex++;
if (val[i] == '\"') {
buf[bufIndex] = val[i]; buf[bufIndex] = val[i];
bufIndex++; bufIndex++;
if (val[i] == '\"') {
buf[bufIndex] = val[i];
bufIndex++;
}
} }
buf[bufIndex] = 0;
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
} }
break; buf[bufIndex] = 0;
taosFprintfFile(pFile, "%s%s%s", quotationStr, buf, quotationStr);
} break;
case TSDB_DATA_TYPE_GEOMETRY: case TSDB_DATA_TYPE_GEOMETRY:
shellDumpHexValue(buf, val, length); shellDumpHexValue(buf, val, length);
taosFprintfFile(pFile, "%s", buf); taosFprintfFile(pFile, "%s", buf);
@ -535,12 +533,10 @@ void shellPrintString(const char *str, int32_t width) {
if (width == 0) { if (width == 0) {
printf("%s", str); printf("%s", str);
} } else if (len > width) {
else if (len > width) {
if (width <= 3) { if (width <= 3) {
printf("%.*s.", width - 1, str); printf("%.*s.", width - 1, str);
} } else {
else {
printf("%.*s...", width - 3, str); printf("%.*s...", width - 3, str);
} }
} else { } else {
@ -549,7 +545,7 @@ void shellPrintString(const char *str, int32_t width) {
} }
void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width) { void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width) {
if (length == 0) { //empty value if (length == 0) { // empty value
shellPrintString("", width); shellPrintString("", width);
return; return;
} }
@ -565,7 +561,7 @@ void shellPrintGeometry(const unsigned char *val, int32_t length, int32_t width)
char *outputWKT = NULL; char *outputWKT = NULL;
code = doAsText(val, length, &outputWKT); code = doAsText(val, length, &outputWKT);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
shellPrintString(getThreadLocalGeosCtx()->errMsg, width); //should NOT happen shellPrintString(getThreadLocalGeosCtx()->errMsg, width); // should NOT happen
return; return;
} }
@ -612,27 +608,26 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t
break; break;
case TSDB_DATA_TYPE_FLOAT: case TSDB_DATA_TYPE_FLOAT:
if (tsEnableScience) { if (tsEnableScience) {
printf("%*.7e",width,GET_FLOAT_VAL(val)); printf("%*.7e", width, GET_FLOAT_VAL(val));
} else { } else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val)); n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.7f", width, GET_FLOAT_VAL(val));
if (n > SHELL_FLOAT_WIDTH) { if (n > SHELL_FLOAT_WIDTH) {
printf("%*.7e", width, GET_FLOAT_VAL(val));
printf("%*.7e", width,GET_FLOAT_VAL(val));
} else { } else {
printf("%s", buf); printf("%s", buf);
} }
} }
break; break;
case TSDB_DATA_TYPE_DOUBLE: case TSDB_DATA_TYPE_DOUBLE:
if (tsEnableScience) { if (tsEnableScience) {
snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width,GET_DOUBLE_VAL(val)); snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15e", width, GET_DOUBLE_VAL(val));
printf("%s", buf); printf("%s", buf);
} else { } else {
n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val)); n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.15f", width, GET_DOUBLE_VAL(val));
if (n > SHELL_DOUBLE_WIDTH) { if (n > SHELL_DOUBLE_WIDTH) {
printf("%*.15e", width, GET_DOUBLE_VAL(val)); printf("%*.15e", width, GET_DOUBLE_VAL(val));
} else { } else {
printf("%*s", width,buf); printf("%*s", width, buf);
} }
} }
break; break;
@ -905,7 +900,7 @@ void shellReadHistory() {
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM); TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_READ | TD_FILE_STREAM);
if (pFile == NULL) return; if (pFile == NULL) return;
char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1); char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
int32_t read_size = 0; int32_t read_size = 0;
while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { while ((read_size = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) {
line[read_size - 1] = '\0'; line[read_size - 1] = '\0';
@ -922,8 +917,8 @@ void shellReadHistory() {
taosMemoryFreeClear(line); taosMemoryFreeClear(line);
taosCloseFile(&pFile); taosCloseFile(&pFile);
int64_t file_size; int64_t file_size;
if (taosStatFile(pHistory->file, &file_size, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) { if (taosStatFile(pHistory->file, &file_size, NULL, NULL) == 0 && file_size > SHELL_MAX_COMMAND_SIZE) {
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC); TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_TRUNC);
if (pFile == NULL) return; if (pFile == NULL) return;
int32_t endIndex = pHistory->hstart; int32_t endIndex = pHistory->hstart;
if (endIndex != 0) { if (endIndex != 0) {
@ -945,7 +940,7 @@ void shellReadHistory() {
void shellWriteHistory() { void shellWriteHistory() {
SShellHistory *pHistory = &shell.history; SShellHistory *pHistory = &shell.history;
if (pHistory->hend == pHistory->hstart) return; if (pHistory->hend == pHistory->hstart) return;
TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND); TdFilePtr pFile = taosOpenFile(pHistory->file, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_STREAM | TD_FILE_APPEND);
if (pFile == NULL) return; if (pFile == NULL) return;
for (int32_t i = pHistory->hstart; i != pHistory->hend;) { for (int32_t i = pHistory->hstart; i != pHistory->hend;) {
@ -991,7 +986,7 @@ void shellSourceFile(const char *file) {
tstrncpy(fullname, file, PATH_MAX); tstrncpy(fullname, file, PATH_MAX);
} }
sprintf(sourceFileCommand, "source %s;",fullname); sprintf(sourceFileCommand, "source %s;", fullname);
shellRecordCommandToHistory(sourceFileCommand); shellRecordCommandToHistory(sourceFileCommand);
TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_READ | TD_FILE_STREAM); TdFilePtr pFile = taosOpenFile(fullname, TD_FILE_READ | TD_FILE_STREAM);
@ -1001,7 +996,7 @@ void shellSourceFile(const char *file) {
return; return;
} }
char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1); char *line = taosMemoryMalloc(TSDB_MAX_ALLOWED_SQL_LEN + 1);
while ((read_len = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) { while ((read_len = taosGetsFile(pFile, TSDB_MAX_ALLOWED_SQL_LEN, line)) != -1) {
if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue; if (read_len >= TSDB_MAX_ALLOWED_SQL_LEN) continue;
line[--read_len] = '\0'; line[--read_len] = '\0';
@ -1044,7 +1039,8 @@ void shellGetGrantInfo() {
int32_t code = taos_errno(tres); int32_t code = taos_errno(tres);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS && code != TSDB_CODE_PAR_PERMISSION_DENIED) { if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS &&
code != TSDB_CODE_PAR_PERMISSION_DENIED) {
fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\r\n\r\n", code, taos_errstr(tres)); fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\r\n\r\n", code, taos_errstr(tres));
} }
return; return;
@ -1080,7 +1076,8 @@ void shellGetGrantInfo() {
} else if (strcmp(expiretime, "unlimited") == 0) { } else if (strcmp(expiretime, "unlimited") == 0) {
fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\r\n", serverVersion, sinfo); fprintf(stdout, "Server is Enterprise %s Edition, %s and will never expire.\r\n", serverVersion, sinfo);
} else { } else {
fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo, expiretime); fprintf(stdout, "Server is Enterprise %s Edition, %s and will expire at %s.\r\n", serverVersion, sinfo,
expiretime);
} }
taos_free_result(tres); taos_free_result(tres);
@ -1123,9 +1120,9 @@ void *shellCancelHandler(void *arg) {
#ifdef WEBSOCKET #ifdef WEBSOCKET
} }
#endif #endif
#ifdef WINDOWS #ifdef WINDOWS
printf("\n%s", shell.info.promptHeader); printf("\n%s", shell.info.promptHeader);
#endif #endif
} }
return NULL; return NULL;
@ -1165,8 +1162,7 @@ void *shellThreadLoop(void *arg) {
} }
int32_t shellExecute() { int32_t shellExecute() {
printf(shell.info.clientVersion, shell.info.cusName, printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName);
taos_get_client_info(), shell.info.cusName);
fflush(stdout); fflush(stdout);
SShellArgs *pArgs = &shell.args; SShellArgs *pArgs = &shell.args;

3
utils/test/c/tmqDemo.c
View File

@ -221,7 +221,7 @@ int64_t getDirectorySize(char* dir) {
totalSize += subDirSize; totalSize += subDirSize;
} else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file } else if (0 == strcmp(strchr(fileName, '.'), ".log")) { // only calc .log file size, and not include .idx file
int64_t file_size = 0; int64_t file_size = 0;
taosStatFile(subdir, &file_size, NULL); taosStatFile(subdir, &file_size, NULL, NULL);
totalSize += file_size; totalSize += file_size;
} }
} }
@ -702,4 +702,3 @@ int main(int32_t argc, char* argv[]) {
taosCloseFile(&g_fp); taosCloseFile(&g_fp);
return 0; return 0;
} }