diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1123f5f020..31b316861b 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2017,7 +2017,6 @@ typedef struct { int8_t slidingUnit; // MACRO: TIME_UNIT_XXX int8_t timezoneInt; // sma data expired if timezone changes. char indexName[TSDB_INDEX_NAME_LEN]; - char timezone[TD_TIMEZONE_LEN]; int32_t exprLen; int32_t tagsFilterLen; int64_t indexUid; @@ -2055,32 +2054,6 @@ void* tDeserializeSVCreateTSmaReq(void* buf, SVCreateTSmaReq* pReq); int32_t tSerializeSVDropTSmaReq(void** buf, SVDropTSmaReq* pReq); void* tDeserializeSVDropTSmaReq(void* buf, SVDropTSmaReq* pReq); -typedef struct { - col_id_t colId; - uint16_t blockSize; // sma data block size - char data[]; -} STSmaColData; - -typedef struct { - tb_uid_t tableUid; // super/child/normal table uid - int32_t dataLen; // not including head - char data[]; -} STSmaTbData; - -typedef struct { - int64_t indexUid; - TSKEY skey; // startKey of one interval/sliding window - int64_t interval; - int32_t dataLen; // not including head - int8_t intervalUnit; - char data[]; -} STSmaDataWrapper; // sma data for a interval/sliding window - -// interval/sliding => window - -// => window->table->colId -// => 当一个window下所有的表均计算完成时，流计算告知tsdb清除window的过期标记 - // RSma: Rollup SMA typedef struct { int64_t interval; diff --git a/include/common/trow.h b/include/common/trow.h index df28bc9962..abbe55c3a3 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -937,9 +937,8 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col STColumn *pCol = NULL; STSchema *pSchema = pIter->pSchema; while (pIter->colIdx <= pSchema->numOfCols) { - pCol = &pSchema->columns[pIter->colIdx]; + pCol = &pSchema->columns[pIter->colIdx]; // 1st column of schema is primary TS key if (colId == pCol->colId) { - ++pIter->colIdx; break; } else if (colId < pCol->colId) { ++pIter->colIdx; @@ -948,7 +947,8 @@ static FORCE_INLINE bool tdSTSRowIterNext(STSRowIter *pIter, col_id_t colId, col return false; } } - return tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + tdGetTpRowDataOfCol(pIter, pCol->type, pCol->offset - sizeof(TSKEY), pVal); + ++pIter->colIdx; } else if (TD_IS_KV_ROW(pIter->pRow)) { return tdGetKvRowValOfColEx(pIter, colId, colType, &pIter->kvIdx, pVal); } else { diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index fbdb1d66b4..d3cd828cf5 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -37,6 +37,11 @@ typedef struct SReadHandle { #define STREAM_DATA_TYPE_SUBMIT_BLOCK 0x1 #define STREAM_DATA_TYPE_SSDATA_BLOCK 0x2 +typedef enum { + OPTR_EXEC_MODEL_BATCH = 0x1, + OPTR_EXEC_MODEL_STREAM = 0x2, +} EOPTR_EXEC_MODEL; + /** * Create the exec task for streaming mode * @param pMsg @@ -84,7 +89,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, SArray* tableIdList, bool isA * @return */ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle); + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model); /** * The main task execution function, including query on both table and multiple tables, diff --git a/include/os/os.h b/include/os/os.h index 0d0c308134..3ea94d0094 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -82,6 +82,7 @@ extern "C" { #include "osLz4.h" #include "osMath.h" #include "osMemory.h" +#include "osProc.h" #include "osRand.h" #include "osThread.h" #include "osSemaphore.h" diff --git a/include/os/osProc.h b/include/os/osProc.h new file mode 100644 index 0000000000..0b22105e5e --- /dev/null +++ b/include/os/osProc.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_OS_PROC_H_ +#define _TD_OS_PROC_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +int32_t taosNewProc(char **args); +void taosSetProcName(int32_t argc, char **argv, const char *name); +void taosSetProcPath(int32_t argc, char **argv); +bool taosProcExists(int32_t pid); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_OS_PROC_H_*/ diff --git a/include/os/osShm.h b/include/os/osShm.h index 82ee2339f2..d26a99e277 100644 --- a/include/os/osShm.h +++ b/include/os/osShm.h @@ -29,7 +29,6 @@ typedef struct { int32_t taosCreateShm(SShm *pShm, int32_t shmsize) ; void taosDropShm(SShm *pShm); int32_t taosAttachShm(SShm *pShm); -void taosDetachShm(SShm *pShm); #ifdef __cplusplus } diff --git a/include/os/osSignal.h b/include/os/osSignal.h index ab4b6fa242..e22c43684c 100644 --- a/include/os/osSignal.h +++ b/include/os/osSignal.h @@ -49,6 +49,8 @@ void taosSetSignal(int32_t signum, FSignalHandler sigfp); void taosIgnSignal(int32_t signum); void taosDflSignal(int32_t signum); +void taosKillChildOnParentStopped(); + #ifdef __cplusplus } #endif diff --git a/include/os/osThread.h b/include/os/osThread.h index 6d8ff1de3e..a145b0da07 100644 --- a/include/os/osThread.h +++ b/include/os/osThread.h @@ -32,6 +32,9 @@ typedef pthread_once_t TdThreadOnce; typedef pthread_rwlockattr_t TdThreadRwlockAttr; typedef pthread_cond_t TdThreadCond; typedef pthread_condattr_t TdThreadCondAttr; +typedef pthread_key_t TdThreadKey; +typedef pthread_barrier_t TdThreadBarrier; +typedef pthread_barrierattr_t TdThreadBarrierAttr; #define taosThreadCleanupPush pthread_cleanup_push #define taosThreadCleanupPop pthread_cleanup_pop @@ -49,68 +52,180 @@ typedef pthread_condattr_t TdThreadCondAttr; #define pthread_rwlockattr_t PTHREAD_RWLOCKATTR_T_TYPE_TAOS_FORBID #define pthread_cond_t PTHREAD_COND_T_TYPE_TAOS_FORBID #define pthread_condattr_t PTHREAD_CONDATTR_T_TYPE_TAOS_FORBID - #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID - #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID - #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID - #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID - #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID - #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID - #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID - #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID - #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID - #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID - #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID - #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID - #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID + #define pthread_key_t PTHREAD_KEY_T_TYPE_TAOS_FORBID + #define pthread_barrier_t PTHREAD_BARRIER_T_TYPE_TAOS_FORBID + #define pthread_barrierattr_t PTHREAD_BARRIERATTR_T_TYPE_TAOS_FORBID #define pthread_create PTHREAD_CREATE_FUNC_TAOS_FORBID - #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID - #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID #define pthread_attr_destroy PTHREAD_ATTR_DESTROY_FUNC_TAOS_FORBID - #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID - #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID - #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID - #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID - #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID - #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID - #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID - #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID - #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID - #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID - #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID + #define pthread_attr_getdetachstate PTHREAD_ATTR_GETDETACHSTATE_FUNC_TAOS_FORBID + #define pthread_attr_getinheritsched PTHREAD_ATTR_GETINHERITSCHED_FUNC_TAOS_FORBID + #define pthread_attr_getschedparam PTHREAD_ATTR_GETSCHEDPARAM_FUNC_TAOS_FORBID + #define pthread_attr_getschedpolicy PTHREAD_ATTR_GETSCHEDPOLICY_FUNC_TAOS_FORBID + #define pthread_attr_getscope PTHREAD_ATTR_GETSCOPE_FUNC_TAOS_FORBID + #define pthread_attr_getstacksize PTHREAD_ATTR_GETSTACKSIZE_FUNC_TAOS_FORBID + #define pthread_attr_init PTHREAD_ATTR_INIT_FUNC_TAOS_FORBID + #define pthread_attr_setdetachstate PTHREAD_ATTR_SETDETACHSTATE_FUNC_TAOS_FORBID + #define pthread_attr_setinheritsched PTHREAD_ATTR_SETINHERITSCHED_FUNC_TAOS_FORBID + #define pthread_attr_setschedparam PTHREAD_ATTR_SETSCHEDPARAM_FUNC_TAOS_FORBID + #define pthread_attr_setschedpolicy PTHREAD_ATTR_SETSCHEDPOLICY_FUNC_TAOS_FORBID + #define pthread_attr_setscope PTHREAD_ATTR_SETSCOPE_FUNC_TAOS_FORBID + #define pthread_attr_setstacksize PTHREAD_ATTR_SETSTACKSIZE_FUNC_TAOS_FORBID + #define pthread_barrier_destroy PTHREAD_BARRIER_DESTROY_FUNC_TAOS_FORBID + #define pthread_barrier_init PTHREAD_BARRIER_INIT_FUNC_TAOS_FORBID + #define pthread_barrier_wait PTHREAD_BARRIER_WAIT_FUNC_TAOS_FORBID + #define pthread_barrierattr_destroy PTHREAD_BARRIERATTR_DESTROY_FUNC_TAOS_FORBID + #define pthread_barrierattr_getpshared PTHREAD_BARRIERATTR_GETPSHARED_FUNC_TAOS_FORBID + #define pthread_barrierattr_init PTHREAD_BARRIERATTR_INIT_FUNC_TAOS_FORBID + #define pthread_barrierattr_setpshared PTHREAD_BARRIERATTR_SETPSHARED_FUNC_TAOS_FORBID #define pthread_cancel PTHREAD_CANCEL_FUNC_TAOS_FORBID + #define pthread_cond_destroy PTHREAD_COND_DESTROY_FUNC_TAOS_FORBID + #define pthread_cond_init PTHREAD_COND_INIT_FUNC_TAOS_FORBID + #define pthread_cond_signal PTHREAD_COND_SIGNAL_FUNC_TAOS_FORBID + #define pthread_cond_broadcast PTHREAD_COND_BROADCAST_FUNC_TAOS_FORBID + #define pthread_cond_wait PTHREAD_COND_WAIT_FUNC_TAOS_FORBID + #define pthread_cond_timedwait PTHREAD_COND_TIMEDWAIT_FUNC_TAOS_FORBID + #define pthread_condattr_destroy PTHREAD_CONDATTR_DESTROY_FUNC_TAOS_FORBID + #define pthread_condattr_getpshared PTHREAD_CONDATTR_GETPSHARED_FUNC_TAOS_FORBID + #define pthread_condattr_init PTHREAD_CONDATTR_INIT_FUNC_TAOS_FORBID + #define pthread_condattr_setpshared PTHREAD_CONDATTR_SETPSHARED_FUNC_TAOS_FORBID + #define pthread_detach PTHREAD_DETACH_FUNC_TAOS_FORBID + #define pthread_equal PTHREAD_EQUAL_FUNC_TAOS_FORBID + #define pthread_exit PTHREAD_EXIT_FUNC_TAOS_FORBID + #define pthread_getschedparam PTHREAD_GETSCHEDPARAM_FUNC_TAOS_FORBID + #define pthread_getspecific PTHREAD_GETSPECIFIC_FUNC_TAOS_FORBID + #define pthread_join PTHREAD_JOIN_FUNC_TAOS_FORBID + #define pthread_key_create PTHREAD_KEY_CREATE_FUNC_TAOS_FORBID + #define pthread_key_delete PTHREAD_KEY_DELETE_FUNC_TAOS_FORBID #define pthread_kill PTHREAD_KILL_FUNC_TAOS_FORBID + #define pthread_mutex_consistent PTHREAD_MUTEX_CONSISTENT_FUNC_TAOS_FORBID + #define pthread_mutex_destroy PTHREAD_MUTEX_DESTROY_FUNC_TAOS_FORBID + #define pthread_mutex_init PTHREAD_MUTEX_INIT_FUNC_TAOS_FORBID + #define pthread_mutex_lock PTHREAD_MUTEX_LOCK_FUNC_TAOS_FORBID + #define pthread_mutex_timedlock PTHREAD_MUTEX_TIMEDLOCK_FUNC_TAOS_FORBID + #define pthread_mutex_trylock PTHREAD_MUTEX_TRYLOCK_FUNC_TAOS_FORBID + #define pthread_mutex_unlock PTHREAD_MUTEX_UNLOCK_FUNC_TAOS_FORBID + #define pthread_mutexattr_destroy PTHREAD_MUTEXATTR_DESTROY_FUNC_TAOS_FORBID + #define pthread_mutexattr_getpshared PTHREAD_MUTEXATTR_GETPSHARED_FUNC_TAOS_FORBID + #define pthread_mutexattr_getrobust PTHREAD_MUTEXATTR_GETROBUST_FUNC_TAOS_FORBID + #define pthread_mutexattr_gettype PTHREAD_MUTEXATTR_GETTYPE_FUNC_TAOS_FORBID + #define pthread_mutexattr_init PTHREAD_MUTEXATTR_INIT_FUNC_TAOS_FORBID + #define pthread_mutexattr_setpshared PTHREAD_MUTEXATTR_SETPSHARED_FUNC_TAOS_FORBID + #define pthread_mutexattr_setrobust PTHREAD_MUTEXATTR_SETROBUST_FUNC_TAOS_FORBID + #define pthread_mutexattr_settype PTHREAD_MUTEXATTR_SETTYPE_FUNC_TAOS_FORBID + #define pthread_once PTHREAD_ONCE_FUNC_TAOS_FORBID + #define pthread_rwlock_destroy PTHREAD_RWLOCK_DESTROY_FUNC_TAOS_FORBID + #define pthread_rwlock_init PTHREAD_RWLOCK_INIT_FUNC_TAOS_FORBID + #define pthread_rwlock_rdlock PTHREAD_RWLOCK_RDLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_timedrdlock PTHREAD_RWLOCK_TIMEDRDLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_timedwrlock PTHREAD_RWLOCK_TIMEDWRLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_tryrdlock PTHREAD_RWLOCK_TRYRDLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_trywrlock PTHREAD_RWLOCK_TRYWRLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_unlock PTHREAD_RWLOCK_UNLOCK_FUNC_TAOS_FORBID + #define pthread_rwlock_wrlock PTHREAD_RWLOCK_WRLOCK_FUNC_TAOS_FORBID + #define pthread_rwlockattr_destroy PTHREAD_RWLOCKATTR_DESTROY_FUNC_TAOS_FORBID + #define pthread_rwlockattr_getpshared PTHREAD_RWLOCKATTR_GETPSHARED_FUNC_TAOS_FORBID + #define pthread_rwlockattr_init PTHREAD_RWLOCKATTR_INIT_FUNC_TAOS_FORBID + #define pthread_rwlockattr_setpshared PTHREAD_RWLOCKATTR_SETPSHARED_FUNC_TAOS_FORBID + #define pthread_self PTHREAD_SELF_FUNC_TAOS_FORBID + #define pthread_setcancelstate PTHREAD_SETCANCELSTATE_FUNC_TAOS_FORBID + #define pthread_setcanceltype PTHREAD_SETCANCELTYPE_FUNC_TAOS_FORBID + #define pthread_setschedparam PTHREAD_SETSCHEDPARAM_FUNC_TAOS_FORBID + #define pthread_setspecific PTHREAD_SETSPECIFIC_FUNC_TAOS_FORBID + #define pthread_spin_destroy PTHREAD_SPIN_DESTROY_FUNC_TAOS_FORBID + #define pthread_spin_init PTHREAD_SPIN_INIT_FUNC_TAOS_FORBID + #define pthread_spin_lock PTHREAD_SPIN_LOCK_FUNC_TAOS_FORBID + #define pthread_spin_trylock PTHREAD_SPIN_TRYLOCK_FUNC_TAOS_FORBID + #define pthread_spin_unlock PTHREAD_SPIN_UNLOCK_FUNC_TAOS_FORBID + #define pthread_testcancel PTHREAD_TESTCANCEL_FUNC_TAOS_FORBID + #define pthread_sigmask PTHREAD_SIGMASK_FUNC_TAOS_FORBID + #define sigwait SIGWAIT_FUNC_TAOS_FORBID #endif -int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared); -int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr); -int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock); -int32_t taosThreadMutexDestroy(TdThreadMutex * mutex); -int32_t taosThreadSpinLock(TdThreadSpinlock *lock); -int32_t taosThreadMutexLock(TdThreadMutex *mutex); -int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock); -int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock); -int32_t taosThreadMutexUnlock(TdThreadMutex *mutex); -int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock); -int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock); -void taosThreadTestCancel(void); -int32_t taosThreadAttrInit(TdThreadAttr *attr); -int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg); -int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void)); -int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState); -int32_t taosThreadAttrDestroy(TdThreadAttr *attr); -int32_t taosThreadJoin(TdThread thread, void **pValue); -int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr); -int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock); -int32_t taosThreadCondSignal(TdThreadCond *cond); -int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr); -int32_t taosThreadCondBroadcast(TdThreadCond *cond); -int32_t taosThreadCondDestroy(TdThreadCond *cond); -int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex); -TdThread taosThreadSelf(void); -int32_t taosThreadEqual(TdThread t1, TdThread t2); -int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset); +int32_t taosThreadCreate(TdThread * tid, const TdThreadAttr * attr, void *(*start)(void *), void *arg); +int32_t taosThreadAttrDestroy(TdThreadAttr * attr); +int32_t taosThreadAttrGetDetachState(const TdThreadAttr * attr, int32_t *detachstate); +int32_t taosThreadAttrGetInheritSched(const TdThreadAttr * attr, int32_t *inheritsched); +int32_t taosThreadAttrGetSchedParam(const TdThreadAttr * attr, struct sched_param *param); +int32_t taosThreadAttrGetSchedPolicy(const TdThreadAttr * attr, int32_t *policy); +int32_t taosThreadAttrGetScope(const TdThreadAttr * attr, int32_t *contentionscope); +int32_t taosThreadAttrGetStackSize(const TdThreadAttr * attr, size_t * stacksize); +int32_t taosThreadAttrInit(TdThreadAttr * attr); +int32_t taosThreadAttrSetDetachState(TdThreadAttr * attr, int32_t detachstate); +int32_t taosThreadAttrSetInheritSched(TdThreadAttr * attr, int32_t inheritsched); +int32_t taosThreadAttrSetSchedParam(TdThreadAttr * attr, const struct sched_param *param); +int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr * attr, int32_t policy); +int32_t taosThreadAttrSetScope(TdThreadAttr * attr, int32_t contentionscope); +int32_t taosThreadAttrSetStackSize(TdThreadAttr * attr, size_t stacksize); +int32_t taosThreadBarrierDestroy(TdThreadBarrier * barrier); +int32_t taosThreadBarrierInit(TdThreadBarrier * barrier, const TdThreadBarrierAttr * attr, uint32_t count); +int32_t taosThreadBarrierWait(TdThreadBarrier * barrier); +int32_t taosThreadBarrierAttrDestroy(TdThreadBarrierAttr * attr); +int32_t taosThreadBarrierAttrGetPshared(const TdThreadBarrierAttr * attr, int32_t *pshared); +int32_t taosThreadBarrierAttrInit(TdThreadBarrierAttr * attr); +int32_t taosThreadBarrierAttrSetPshared(TdThreadBarrierAttr * attr, int32_t pshared); int32_t taosThreadCancel(TdThread thread); -int32_t taosThreadKill(TdThread thread, int sig); +int32_t taosThreadCondDestroy(TdThreadCond * cond); +int32_t taosThreadCondInit(TdThreadCond * cond, const TdThreadCondAttr * attr); +int32_t taosThreadCondSignal(TdThreadCond * cond); +int32_t taosThreadCondBroadcast(TdThreadCond * cond); +int32_t taosThreadCondWait(TdThreadCond * cond, TdThreadMutex * mutex); +int32_t taosThreadCondTimedWait(TdThreadCond * cond, TdThreadMutex * mutex, const struct timespec *abstime); +int32_t taosThreadCondAttrDestroy(TdThreadCondAttr * attr); +int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr * attr, int32_t *pshared); +int32_t taosThreadCondAttrInit(TdThreadCondAttr * attr); +int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr * attr, int32_t pshared); +int32_t taosThreadDetach(TdThread thread); +int32_t taosThreadEqual(TdThread t1, TdThread t2); +void taosThreadExit(void *valuePtr); +int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_param *param); +void *taosThreadGetSpecific(TdThreadKey key); +int32_t taosThreadJoin(TdThread thread, void **valuePtr); +int32_t taosThreadKeyCreate(TdThreadKey * key, void(*destructor)(void *)); +int32_t taosThreadKeyDelete(TdThreadKey key); +int32_t taosThreadKill(TdThread thread, int32_t sig); +int32_t taosThreadMutexConsistent(TdThreadMutex* mutex); +int32_t taosThreadMutexDestroy(TdThreadMutex * mutex); +int32_t taosThreadMutexInit(TdThreadMutex * mutex, const TdThreadMutexAttr * attr); +int32_t taosThreadMutexLock(TdThreadMutex * mutex); +int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime); +int32_t taosThreadMutexTryLock(TdThreadMutex * mutex); +int32_t taosThreadMutexUnlock(TdThreadMutex * mutex); +int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr * attr); +int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *pshared); +int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust); +int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind); +int32_t taosThreadMutexAttrInit(TdThreadMutexAttr * attr); +int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared); +int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust); +int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind); +int32_t taosThreadOnce(TdThreadOnce * onceControl, void(*initRoutine)(void)); +int32_t taosThreadRwlockDestroy(TdThreadRwlock * rwlock); +int32_t taosThreadRwlockInit(TdThreadRwlock * rwlock, const TdThreadRwlockAttr * attr); +int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock); +int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime); +int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime); +int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock); +int32_t taosThreadRwlockTryWrlock(TdThreadRwlock * rwlock); +int32_t taosThreadRwlockUnlock(TdThreadRwlock * rwlock); +int32_t taosThreadRwlockWrlock(TdThreadRwlock * rwlock); +int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr * attr); +int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr * attr, int32_t *pshared); +int32_t taosThreadRwlockAttrInit(TdThreadRwlockAttr * attr); +int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr * attr, int32_t pshared); +TdThread taosThreadSelf(void); +int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate); +int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype); +int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sched_param *param); +int32_t taosThreadSetSpecific(TdThreadKey key, const void *value); +int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock); +int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared); +int32_t taosThreadSpinLock(TdThreadSpinlock * lock); +int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock); +int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock); +void taosThreadTestCancel(void); +int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset); +int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig); + #ifdef __cplusplus } #endif diff --git a/include/os/osTime.h b/include/os/osTime.h index 031b9d28f9..9e426455dc 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -20,6 +20,8 @@ extern "C" { #endif +#include + // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. #ifndef ALLOW_FORBID_FUNC diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 42f4490051..7c6ecd6d9a 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -78,6 +78,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x010C) #define TSDB_CODE_INVALID_CFG TAOS_DEF_ERROR_CODE(0, 0x010D) #define TSDB_CODE_OUT_OF_SHM_MEM TAOS_DEF_ERROR_CODE(0, 0x010E) +#define TSDB_CODE_INVALID_SHM_ID TAOS_DEF_ERROR_CODE(0, 0x010F) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) diff --git a/include/util/tprocess.h b/include/util/tprocess.h index 51ce0243b7..3a47450eec 100644 --- a/include/util/tprocess.h +++ b/include/util/tprocess.h @@ -32,29 +32,25 @@ typedef void *(*ProcConsumeFp)(void *pParent, void *pHead, int16_t headLen, void ProcFuncType ftype); typedef struct { - int32_t childQueueSize; ProcConsumeFp childConsumeFp; ProcMallocFp childMallocHeadFp; ProcFreeFp childFreeHeadFp; ProcMallocFp childMallocBodyFp; ProcFreeFp childFreeBodyFp; - int32_t parentQueueSize; ProcConsumeFp parentConsumeFp; - ProcMallocFp parentdMallocHeadFp; + ProcMallocFp parentMallocHeadFp; ProcFreeFp parentFreeHeadFp; ProcMallocFp parentMallocBodyFp; ProcFreeFp parentFreeBodyFp; - bool testFlag; + SShm shm; void *pParent; const char *name; + bool isChild; } SProcCfg; SProcObj *taosProcInit(const SProcCfg *pCfg); void taosProcCleanup(SProcObj *pProc); int32_t taosProcRun(SProcObj *pProc); -void taosProcStop(SProcObj *pProc); -bool taosProcIsChild(SProcObj *pProc); -int32_t taosProcChildId(SProcObj *pProc); int32_t taosProcPutToChildQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, ProcFuncType ftype); int32_t taosProcPutToParentQ(SProcObj *pProc, const void *pHead, int16_t headLen, const void *pBody, int32_t bodyLen, diff --git a/source/dnode/mgmt/dm/src/dmFile.c b/source/dnode/mgmt/dm/src/dmFile.c index d44b1222a3..444f18e6e0 100644 --- a/source/dnode/mgmt/dm/src/dmFile.c +++ b/source/dnode/mgmt/dm/src/dmFile.c @@ -130,7 +130,7 @@ int32_t dmReadFile(SDnodeMgmt *pMgmt) { } code = 0; - dInfo("succcessed to read file %s", file); + dDebug("succcessed to read file %s", file); dmPrintDnodes(pMgmt); PRASE_DNODE_OVER: diff --git a/source/dnode/mgmt/dm/src/dmInt.c b/source/dnode/mgmt/dm/src/dmInt.c index b729888a72..3c5f394d5e 100644 --- a/source/dnode/mgmt/dm/src/dmInt.c +++ b/source/dnode/mgmt/dm/src/dmInt.c @@ -112,6 +112,16 @@ int32_t dmInit(SMgmtWrapper *pWrapper) { return -1; } + if (dndInitServer(pDnode) != 0) { + dError("failed to init trans server since %s", terrstr()); + return -1; + } + + if (dndInitClient(pDnode) != 0) { + dError("failed to init trans client since %s", terrstr()); + return -1; + } + pWrapper->pMgmt = pMgmt; dInfo("dnode-mgmt is initialized"); return 0; @@ -122,6 +132,7 @@ void dmCleanup(SMgmtWrapper *pWrapper) { if (pMgmt == NULL) return; dInfo("dnode-mgmt start to clean up"); + SDnode *pDnode = pMgmt->pDnode; dmStopWorker(pMgmt); taosWLockLatch(&pMgmt->latch); @@ -140,6 +151,9 @@ void dmCleanup(SMgmtWrapper *pWrapper) { taosMemoryFree(pMgmt); pWrapper->pMgmt = NULL; + dndCleanupServer(pDnode); + dndCleanupClient(pDnode); + dInfo("dnode-mgmt is cleaned up"); } diff --git a/source/dnode/mgmt/dm/src/dmMsg.c b/source/dnode/mgmt/dm/src/dmMsg.c index b301ef478b..3795cffbfa 100644 --- a/source/dnode/mgmt/dm/src/dmMsg.c +++ b/source/dnode/mgmt/dm/src/dmMsg.c @@ -53,7 +53,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { tSerializeSStatusReq(pHead, contLen, &req); taosArrayDestroy(req.pVloads); - SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; + SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)0x9527}; pMgmt->statusSent = 1; dTrace("send req:%s to mnode, app:%p", TMSG_INFO(rpcMsg.msgType), rpcMsg.ahandle); diff --git a/source/dnode/mgmt/main/exe/dndMain.c b/source/dnode/mgmt/main/exe/dndMain.c index 525b26d967..61b480990d 100644 --- a/source/dnode/mgmt/main/exe/dndMain.c +++ b/source/dnode/mgmt/main/exe/dndMain.c @@ -30,7 +30,7 @@ static struct { } global = {0}; static void dndStopDnode(int signum, void *info, void *ctx) { - dInfo("signal:%d is received", signum); + dInfo("system signal:%d received", signum); SDnode *pDnode = atomic_val_compare_exchange_ptr(&global.pDnode, 0, global.pDnode); if (pDnode != NULL) { dndHandleEvent(pDnode, DND_EVENT_STOP); @@ -38,8 +38,10 @@ static void dndStopDnode(int signum, void *info, void *ctx) { } static void dndHandleChild(int signum, void *info, void *ctx) { - dInfo("signal:%d is received", signum); - dndHandleEvent(global.pDnode, DND_EVENT_CHILD); + dInfo("sigchild received"); + if (global.pDnode != NULL) { + dndHandleEvent(global.pDnode, DND_EVENT_CHILD); + } } static void dndSetSignalHandle() { @@ -50,15 +52,10 @@ static void dndSetSignalHandle() { taosSetSignal(SIGBREAK, dndStopDnode); if (!tsMultiProcess) { - // Set the single process signal - } else if (global.ntype == DNODE) { - // Set the parent process signal - // When the child process exits, the parent process receives a signal + } else if (global.ntype == DNODE || global.ntype == NODE_MAX) { taosSetSignal(SIGCHLD, dndHandleChild); } else { - // Set child process signal - // When the parent process exits, the child process will receive the SIGKILL signal - prctl(PR_SET_PDEATHSIG, SIGKILL); + taosKillChildOnParentStopped(); } } @@ -79,10 +76,14 @@ static int32_t dndParseArgs(int32_t argc, char const *argv[]) { tstrncpy(global.apolloUrl, argv[++i], PATH_MAX); } else if (strcmp(argv[i], "-e") == 0) { tstrncpy(global.envFile, argv[++i], PATH_MAX); - } else if (strcmp(argv[i], "-k") == 0) { - global.generateGrant = true; } else if (strcmp(argv[i], "-n") == 0) { global.ntype = atoi(argv[++i]); + if (global.ntype <= DNODE || global.ntype > NODE_MAX) { + printf("'-n' range is [1-5], default is 0\n"); + return -1; + } + } else if (strcmp(argv[i], "-k") == 0) { + global.generateGrant = true; } else if (strcmp(argv[i], "-C") == 0) { global.dumpConfig = true; } else if (strcmp(argv[i], "-V") == 0) { @@ -138,24 +139,24 @@ static int32_t dndInitLog() { return taosCreateLog(logName, 1, configDir, global.envFile, global.apolloUrl, global.pArgs, 0); } -static void dndSetProcName(char **argv) { - if (global.ntype != DNODE) { +static void dndSetProcInfo(int32_t argc, char **argv) { + taosSetProcPath(argc, argv); + if (global.ntype != DNODE && global.ntype != NODE_MAX) { const char *name = dndNodeProcStr(global.ntype); - prctl(PR_SET_NAME, name); - strcpy(argv[0], name); + taosSetProcName(argc, argv, name); } } static int32_t dndRunDnode() { if (dndInit() != 0) { - dError("failed to initialize environment since %s", terrstr()); + dError("failed to init environment since %s", terrstr()); return -1; } SDnodeOpt option = dndGetOpt(); SDnode *pDnode = dndCreate(&option); if (pDnode == NULL) { - dError("failed to to create dnode object since %s", terrstr()); + dError("failed to to create dnode since %s", terrstr()); return -1; } else { global.pDnode = pDnode; @@ -212,6 +213,6 @@ int main(int argc, char const *argv[]) { return 0; } - dndSetProcName((char **)argv); + dndSetProcInfo(argc, (char **)argv); return dndRunDnode(); } diff --git a/source/dnode/mgmt/main/inc/dnd.h b/source/dnode/mgmt/main/inc/dnd.h index b416ee4f7a..b9c760c980 100644 --- a/source/dnode/mgmt/main/inc/dnd.h +++ b/source/dnode/mgmt/main/inc/dnd.h @@ -95,13 +95,14 @@ typedef struct SMgmtWrapper { bool deployed; bool required; EProcType procType; + int32_t procId; SProcObj *pProc; SShm shm; void *pMgmt; SDnode *pDnode; - NodeMsgFp msgFps[TDMT_MAX]; - int8_t msgVgIds[TDMT_MAX]; // Handle the case where the same message type is distributed to qnode or vnode SMgmtFp fp; + int8_t msgVgIds[TDMT_MAX]; // Handle the case where the same message type is distributed to qnode or vnode + NodeMsgFp msgFps[TDMT_MAX]; } SMgmtWrapper; typedef struct { @@ -128,26 +129,31 @@ typedef struct SDnode { EDndStatus status; EDndEvent event; SStartupReq startup; - TdFilePtr runtimeFile; + TdFilePtr lockfile; STransMgmt trans; SMgmtWrapper wrappers[NODE_MAX]; } SDnode; const char *dndNodeLogStr(ENodeType ntype); const char *dndNodeProcStr(ENodeType ntype); +const char *dndEventStr(EDndEvent ev); EDndStatus dndGetStatus(SDnode *pDnode); void dndSetStatus(SDnode *pDnode, EDndStatus stat); void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId); void dndReportStartup(SDnode *pDnode, const char *pName, const char *pDesc); void dndSendMonitorReport(SDnode *pDnode); +int32_t dndInitServer(SDnode *pDnode); +void dndCleanupServer(SDnode *pDnode); +int32_t dndInitClient(SDnode *pDnode); +void dndCleanupClient(SDnode *pDnode); +int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); int32_t dndSendReqToMnode(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); int32_t dndSendReqToDnode(SMgmtWrapper *pWrapper, const SEpSet *pEpSet, SRpcMsg *pMsg); void dndSendRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); void dndRegisterBrokenLinkArg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg); SMsgCb dndCreateMsgcb(SMgmtWrapper *pWrapper); -int32_t dndProcessNodeMsg(SDnode *pDnode, SNodeMsg *pMsg); int32_t dndReadFile(SMgmtWrapper *pWrapper, bool *pDeployed); int32_t dndWriteFile(SMgmtWrapper *pWrapper, bool deployed); diff --git a/source/dnode/mgmt/main/inc/dndInt.h b/source/dnode/mgmt/main/inc/dndInt.h index 56782f872b..612d35d513 100644 --- a/source/dnode/mgmt/main/inc/dndInt.h +++ b/source/dnode/mgmt/main/inc/dndInt.h @@ -50,17 +50,13 @@ void dndClose(SDnode *pDnode); void dndHandleEvent(SDnode *pDnode, EDndEvent event); // dndTransport.c -int32_t dndInitServer(SDnode *pDnode); -void dndCleanupServer(SDnode *pDnode); -int32_t dndInitClient(SDnode *pDnode); -void dndCleanupClient(SDnode *pDnode); int32_t dndInitMsgHandle(SDnode *pDnode); void dndSendRpcRsp(SMgmtWrapper *pWrapper, const SRpcMsg *pRsp); // dndFile.c -int32_t dndOpenRuntimeFile(SDnode *pDnode); -int32_t dndWriteRuntimeFile(SDnode *pDnode); -void dndCloseRuntimeFile(SDnode *pDnode); +TdFilePtr dndCheckRunning(const char *dataDir); +int32_t dndReadShmFile(SDnode *pDnode); +int32_t dndWriteShmFile(SDnode *pDnode); #ifdef __cplusplus } diff --git a/source/dnode/mgmt/main/src/dndExec.c b/source/dnode/mgmt/main/src/dndExec.c index c41d4f28e4..2999a30d67 100644 --- a/source/dnode/mgmt/main/src/dndExec.c +++ b/source/dnode/mgmt/main/src/dndExec.c @@ -16,18 +16,9 @@ #define _DEFAULT_SOURCE #include "dndInt.h" -static void dndResetLog(SMgmtWrapper *pMgmt) { - char logname[24] = {0}; - snprintf(logname, sizeof(logname), "%slog", pMgmt->name); - - dInfo("node:%s, reset log to %s in child process", pMgmt->name, logname); - taosCloseLog(); - taosInitLog(logname, 1); -} - static bool dndRequireNode(SMgmtWrapper *pWrapper) { - bool required = false; - int32_t code =(*pWrapper->fp.requiredFp)(pWrapper, &required); + bool required = false; + int32_t code = (*pWrapper->fp.requiredFp)(pWrapper, &required); if (!required) { dDebug("node:%s, no need to start", pWrapper->name); } else { @@ -37,14 +28,18 @@ static bool dndRequireNode(SMgmtWrapper *pWrapper) { } int32_t dndOpenNode(SMgmtWrapper *pWrapper) { - int32_t code = (*pWrapper->fp.openFp)(pWrapper); - if (code != 0) { - dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + if (taosMkDir(pWrapper->path) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("node:%s, failed to create dir:%s since %s", pWrapper->name, pWrapper->path, terrstr()); return -1; - } else { - dDebug("node:%s, has been opened", pWrapper->name); } + if ((*pWrapper->fp.openFp)(pWrapper) != 0) { + dError("node:%s, failed to open since %s", pWrapper->name, terrstr()); + return -1; + } + + dDebug("node:%s, has been opened", pWrapper->name); pWrapper->deployed = true; return 0; } @@ -70,54 +65,6 @@ void dndCloseNode(SMgmtWrapper *pWrapper) { dDebug("node:%s, has been closed", pWrapper->name); } -static int32_t dndRunInSingleProcess(SDnode *pDnode) { - dInfo("dnode run in single process mode"); - - for (ENodeType n = 0; n < NODE_MAX; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - pWrapper->required = dndRequireNode(pWrapper); - if (!pWrapper->required) continue; - SMsgCb msgCb = dndCreateMsgcb(pWrapper); - tmsgSetDefaultMsgCb(&msgCb); - - if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); - return -1; - } - - dInfo("node:%s, will start in single process", pWrapper->name); - pWrapper->procType = PROC_SINGLE; - if (dndOpenNode(pWrapper) != 0) { - dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); - return -1; - } - } - - dndSetStatus(pDnode, DND_STAT_RUNNING); - - for (ENodeType n = 0; n < NODE_MAX; ++n) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - if (!pWrapper->required) continue; - if (pWrapper->fp.startFp == NULL) continue; - if ((*pWrapper->fp.startFp)(pWrapper) != 0) { - dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); - return -1; - } - } - - return 0; -} - -static void dndClearNodesExecpt(SDnode *pDnode, ENodeType except) { - // dndCleanupServer(pDnode); - for (ENodeType n = 0; n < NODE_MAX; ++n) { - if (except == n) continue; - SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; - pWrapper->required = false; - } -} - static void dndConsumeChildQueue(SMgmtWrapper *pWrapper, SNodeMsg *pMsg, int16_t msgLen, void *pCont, int32_t contLen, ProcFuncType ftype) { SRpcMsg *pRpc = &pMsg->rpcMsg; @@ -163,82 +110,57 @@ static void dndConsumeParentQueue(SMgmtWrapper *pWrapper, SRpcMsg *pMsg, int16_t taosMemoryFree(pMsg); } -static int32_t dndRunInMultiProcess(SDnode *pDnode) { - dInfo("dnode run in multi process mode"); +static int32_t dndNewProc(SMgmtWrapper *pWrapper, ENodeType n) { + char tstr[8] = {0}; + char *args[6] = {0}; + snprintf(tstr, sizeof(tstr), "%d", n); + args[1] = "-c"; + args[2] = configDir; + args[3] = "-n"; + args[4] = tstr; + args[5] = NULL; - for (ENodeType n = 0; n < NODE_MAX; ++n) { + int32_t pid = taosNewProc(args); + if (pid <= 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("node:%s, failed to exec in new process since %s", pWrapper->name, terrstr()); + return -1; + } + + pWrapper->procId = pid; + dInfo("node:%s, run in new process, pid:%d", pWrapper->name, pid); + return 0; +} + +static SProcCfg dndGenProcCfg(SMgmtWrapper *pWrapper) { + SProcCfg cfg = {.childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, + .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, + .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, + .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, + .parentMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, + .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, + .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, + .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, + .shm = pWrapper->shm, + .pParent = pWrapper, + .name = pWrapper->name}; + return cfg; +} + +static int32_t dndRunInSingleProcess(SDnode *pDnode) { + dInfo("dnode run in single process"); + + for (ENodeType n = DNODE; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; pWrapper->required = dndRequireNode(pWrapper); if (!pWrapper->required) continue; - SMsgCb msgCb = dndCreateMsgcb(pWrapper); - tmsgSetDefaultMsgCb(&msgCb); - - if (taosMkDir(pWrapper->path) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + if (dndOpenNode(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; } - - if (n == DNODE) { - dInfo("node:%s, will start in parent process", pWrapper->name); - pWrapper->procType = PROC_SINGLE; - if (dndOpenNode(pWrapper) != 0) { - dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); - return -1; - } - continue; - } - - SProcCfg cfg = {.childQueueSize = 1024 * 1024 * 2, // size will be a configuration item - .childConsumeFp = (ProcConsumeFp)dndConsumeChildQueue, - .childMallocHeadFp = (ProcMallocFp)taosAllocateQitem, - .childFreeHeadFp = (ProcFreeFp)taosFreeQitem, - .childMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .childFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .parentQueueSize = 1024 * 1024 * 2, // size will be a configuration item - .parentConsumeFp = (ProcConsumeFp)dndConsumeParentQueue, - .parentdMallocHeadFp = (ProcMallocFp)taosMemoryMalloc, - .parentFreeHeadFp = (ProcFreeFp)taosMemoryFree, - .parentMallocBodyFp = (ProcMallocFp)rpcMallocCont, - .parentFreeBodyFp = (ProcFreeFp)rpcFreeCont, - .pParent = pWrapper, - .name = pWrapper->name}; - SProcObj *pProc = taosProcInit(&cfg); - if (pProc == NULL) { - dError("node:%s, failed to fork since %s", pWrapper->name, terrstr()); - return -1; - } - - pWrapper->pProc = pProc; - - if (taosProcIsChild(pProc)) { - dInfo("node:%s, will start in child process", pWrapper->name); - pWrapper->procType = PROC_CHILD; - dndResetLog(pWrapper); - - dInfo("node:%s, clean up resources inherited from parent", pWrapper->name); - dndClearNodesExecpt(pDnode, n); - - dInfo("node:%s, will be initialized in child process", pWrapper->name); - if (dndOpenNode(pWrapper) != 0) { - dInfo("node:%s, failed to init in child process since %s", pWrapper->name, terrstr()); - return -1; - } - - if (taosProcRun(pProc) != 0) { - dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); - return -1; - } - break; - } else { - dInfo("node:%s, will not start in parent process, child pid:%d", pWrapper->name, taosProcChildId(pProc)); - pWrapper->procType = PROC_PARENT; - if (taosProcRun(pProc) != 0) { - dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); - return -1; - } - } } dndSetStatus(pDnode, DND_STAT_RUNNING); @@ -247,32 +169,14 @@ static int32_t dndRunInMultiProcess(SDnode *pDnode) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; if (!pWrapper->required) continue; if (pWrapper->fp.startFp == NULL) continue; - if (pWrapper->procType == PROC_PARENT && n != DNODE) continue; - if (pWrapper->procType == PROC_CHILD && n == DNODE) continue; if ((*pWrapper->fp.startFp)(pWrapper) != 0) { dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); return -1; } } - return 0; -} - -int32_t dndRun(SDnode *pDnode) { - if (!tsMultiProcess) { - if (dndRunInSingleProcess(pDnode) != 0) { - dError("failed to run dnode in single process mode since %s", terrstr()); - return -1; - } - } else { - if (dndRunInMultiProcess(pDnode) != 0) { - dError("failed to run dnode in multi process mode since %s", terrstr()); - return -1; - } - } - + dInfo("TDengine initialized successfully"); dndReportStartup(pDnode, "TDengine", "initialized successfully"); - while (1) { if (pDnode->event == DND_EVENT_STOP) { dInfo("dnode is about to stop"); @@ -283,3 +187,146 @@ int32_t dndRun(SDnode *pDnode) { return 0; } + +static int32_t dndRunInParentProcess(SDnode *pDnode) { + dInfo("dnode run in parent process"); + SMgmtWrapper *pDWrapper = &pDnode->wrappers[DNODE]; + if (dndOpenNode(pDWrapper) != 0) { + dError("node:%s, failed to start since %s", pDWrapper->name, terrstr()); + return -1; + } + + for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + pWrapper->required = dndRequireNode(pWrapper); + if (!pWrapper->required) continue; + + int32_t shmsize = 1024 * 1024 * 2; // size will be a configuration item + if (taosCreateShm(&pWrapper->shm, shmsize) != 0) { + terrno = TAOS_SYSTEM_ERROR(terrno); + dError("node:%s, failed to create shm size:%d since %s", pWrapper->name, shmsize, terrstr()); + return -1; + } + dInfo("node:%s, shm:%d is created, size:%d", pWrapper->name, pWrapper->shm.id, shmsize); + + SProcCfg cfg = dndGenProcCfg(pWrapper); + cfg.isChild = false; + pWrapper->procType = PROC_PARENT; + pWrapper->pProc = taosProcInit(&cfg); + if (pWrapper->pProc == NULL) { + dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr()); + return -1; + } + } + + if (dndWriteShmFile(pDnode) != 0) { + dError("failed to write runtime file since %s", terrstr()); + return -1; + } + + for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + if (!pWrapper->required) continue; + + if (pDnode->ntype == NODE_MAX) { + dInfo("node:%s, should be started manually", pWrapper->name); + } else { + if (dndNewProc(pWrapper, n) != 0) { + return -1; + } + } + + if (taosProcRun(pWrapper->pProc) != 0) { + dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); + return -1; + } + } + + dndSetStatus(pDnode, DND_STAT_RUNNING); + + if ((*pDWrapper->fp.startFp)(pDWrapper) != 0) { + dError("node:%s, failed to start since %s", pDWrapper->name, terrstr()); + return -1; + } + + dInfo("TDengine initialized successfully"); + dndReportStartup(pDnode, "TDengine", "initialized successfully"); + + while (1) { + if (pDnode->event == DND_EVENT_STOP) { + dInfo("dnode is about to stop"); + break; + } + + for (ENodeType n = DNODE + 1; n < NODE_MAX; ++n) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; + if (!pWrapper->required) continue; + if (pDnode->ntype == NODE_MAX) continue; + + if (pWrapper->procId != 0 && !taosProcExists(pWrapper->procId)) { + dInfo("node:%s, process not exist, pid:%d", pWrapper->name, pWrapper->procId); + dndNewProc(pWrapper, n); + } + + taosMsleep(100); + } + } + + return 0; +} + +static int32_t dndRunInChildProcess(SDnode *pDnode) { + dInfo("dnode run in child process"); + SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; + + SMsgCb msgCb = dndCreateMsgcb(pWrapper); + tmsgSetDefaultMsgCb(&msgCb); + pWrapper->procType = PROC_CHILD; + + if (dndOpenNode(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + + SProcCfg cfg = dndGenProcCfg(pWrapper); + cfg.isChild = true; + pWrapper->pProc = taosProcInit(&cfg); + if (pWrapper->pProc == NULL) { + dError("node:%s, failed to create proc since %s", pWrapper->name, terrstr()); + return -1; + } + + if (pWrapper->fp.startFp != NULL) { + if ((*pWrapper->fp.startFp)(pWrapper) != 0) { + dError("node:%s, failed to start since %s", pWrapper->name, terrstr()); + return -1; + } + } + + if (taosProcRun(pWrapper->pProc) != 0) { + dError("node:%s, failed to run proc since %s", pWrapper->name, terrstr()); + return -1; + } + + dInfo("TDengine initialized successfully"); + dndReportStartup(pDnode, "TDengine", "initialized successfully"); + while (1) { + if (pDnode->event == DND_EVENT_STOP) { + dInfo("dnode is about to stop"); + break; + } + taosMsleep(100); + } +} + +int32_t dndRun(SDnode *pDnode) { + if (!tsMultiProcess) { + return dndRunInSingleProcess(pDnode); + } else if (pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) { + return dndRunInParentProcess(pDnode); + } else { + return dndRunInChildProcess(pDnode); + } + + return 0; +} diff --git a/source/dnode/mgmt/main/src/dndFile.c b/source/dnode/mgmt/main/src/dndFile.c index 51d4ff3902..92e6cea3e1 100644 --- a/source/dnode/mgmt/main/src/dndFile.c +++ b/source/dnode/mgmt/main/src/dndFile.c @@ -117,7 +117,30 @@ _OVER: return code; } -int32_t dndOpenRuntimeFile(SDnode *pDnode) { +TdFilePtr dndCheckRunning(const char *dataDir) { + char filepath[PATH_MAX] = {0}; + snprintf(filepath, sizeof(filepath), "%s%s.running", dataDir, TD_DIRSEP); + + TdFilePtr pFile = taosOpenFile(filepath, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s", filepath, terrstr()); + return NULL; + } + + int32_t ret = taosLockFile(pFile); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to lock file:%s since %s", filepath, terrstr()); + taosCloseFile(&pFile); + return NULL; + } + + dDebug("file:%s is locked", filepath); + return pFile; +} + +int32_t dndReadShmFile(SDnode *pDnode) { int32_t code = -1; char itemName[24] = {0}; char content[MAXLEN + 1] = {0}; @@ -125,17 +148,11 @@ int32_t dndOpenRuntimeFile(SDnode *pDnode) { cJSON *root = NULL; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%s.running", pDnode->dataDir, TD_DIRSEP); - pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); + snprintf(file, sizeof(file), "%s%s.shmfile", pDnode->dataDir, TD_DIRSEP); + pFile = taosOpenFile(file, TD_FILE_READ); if (pFile == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to open file:%s since %s", file, terrstr()); - goto _OVER; - } - - if (taosLockFile(pFile) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s", file, terrstr()); + dDebug("file %s not exist", file); + code = 0; goto _OVER; } @@ -162,10 +179,10 @@ int32_t dndOpenRuntimeFile(SDnode *pDnode) { } } - if (tsMultiProcess || pDnode->ntype == DNODE) { + if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) { for (ENodeType ntype = DNODE; ntype < NODE_MAX; ++ntype) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; - if (pWrapper->shm.id > 0) { + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + if (pWrapper->shm.id >= 0) { dDebug("shmid:%d, is closed, size:%d", pWrapper->shm.id, pWrapper->shm.size); taosDropShm(&pWrapper->shm); } @@ -174,27 +191,23 @@ int32_t dndOpenRuntimeFile(SDnode *pDnode) { SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; if (taosAttachShm(&pWrapper->shm) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("shmid:%d, failed to attach since %s", pWrapper->shm.id, terrstr()); + dError("shmid:%d, failed to attach shm since %s", pWrapper->shm.id, terrstr()); goto _OVER; } - dDebug("shmid:%d, is attached, size:%d", pWrapper->shm.id, pWrapper->shm.size); + dInfo("node:%s, shmid:%d is attached, size:%d", pWrapper->name, pWrapper->shm.id, pWrapper->shm.size); } - dDebug("successed to open %s", file); + dDebug("successed to load %s", file); code = 0; _OVER: if (root != NULL) cJSON_Delete(root); - if (code != 0) { - if (pFile != NULL) taosCloseFile(&pFile); - } else { - pDnode->runtimeFile = pFile; - } + if (pFile != NULL) taosCloseFile(&pFile); return code; } -int32_t dndWriteRuntimeFile(SDnode *pDnode) { +int32_t dndWriteShmFile(SDnode *pDnode) { int32_t code = -1; int32_t len = 0; char content[MAXLEN + 1] = {0}; @@ -202,8 +215,8 @@ int32_t dndWriteRuntimeFile(SDnode *pDnode) { char realfile[PATH_MAX] = {0}; TdFilePtr pFile = NULL; - snprintf(file, sizeof(file), "%s%s.running.bak", pDnode->dataDir, TD_DIRSEP); - snprintf(realfile, sizeof(realfile), "%s%s.running", pDnode->dataDir, TD_DIRSEP); + snprintf(file, sizeof(file), "%s%s.shmfile.bak", pDnode->dataDir, TD_DIRSEP); + snprintf(realfile, sizeof(realfile), "%s%s.shmfile", pDnode->dataDir, TD_DIRSEP); pFile = taosOpenFile(file, TD_FILE_CTEATE | TD_FILE_WRITE | TD_FILE_TRUNC); if (pFile == NULL) { @@ -214,12 +227,12 @@ int32_t dndWriteRuntimeFile(SDnode *pDnode) { len += snprintf(content + len, MAXLEN - len, "{\n"); for (ENodeType ntype = DNODE + 1; ntype < NODE_MAX; ++ntype) { - SMgmtWrapper *pWrapper = &pDnode->wrappers[pDnode->ntype]; - len += snprintf(content + len, MAXLEN - len, " \"%s_shmid\": %d,\n", dndNodeProcStr(ntype), pWrapper->shm.id); + SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; + len += snprintf(content + len, MAXLEN - len, " \"%s_shmid\":%d,\n", dndNodeProcStr(ntype), pWrapper->shm.id); if (ntype == NODE_MAX - 1) { - len += snprintf(content + len, MAXLEN - len, " \"%s_shmsize\": %d\n", dndNodeProcStr(ntype), pWrapper->shm.size); + len += snprintf(content + len, MAXLEN - len, " \"%s_shmsize\":%d\n", dndNodeProcStr(ntype), pWrapper->shm.size); } else { - len += snprintf(content + len, MAXLEN - len, " \"%s_shmsize\": %d,\n", dndNodeProcStr(ntype), pWrapper->shm.size); + len += snprintf(content + len, MAXLEN - len, " \"%s_shmsize\":%d,\n", dndNodeProcStr(ntype), pWrapper->shm.size); } } len += snprintf(content + len, MAXLEN - len, "}\n"); @@ -244,7 +257,7 @@ int32_t dndWriteRuntimeFile(SDnode *pDnode) { return -1; } - dDebug("successed to write %s", realfile); + dInfo("successed to write %s", realfile); code = 0; _OVER: @@ -254,11 +267,3 @@ _OVER: return code; } - -void dndCloseRuntimeFile(SDnode *pDnode) { - if (pDnode->runtimeFile) { - taosUnLockFile(pDnode->runtimeFile); - taosCloseFile(&pDnode->runtimeFile); - pDnode->runtimeFile = NULL; - } -} \ No newline at end of file diff --git a/source/dnode/mgmt/main/src/dndInt.c b/source/dnode/mgmt/main/src/dndInt.c index 7dde3561fb..8792147822 100644 --- a/source/dnode/mgmt/main/src/dndInt.c +++ b/source/dnode/mgmt/main/src/dndInt.c @@ -41,7 +41,7 @@ int32_t dndInit() { return -1; } - dDebug("dnode env is initialized"); + dInfo("dnode env is initialized"); return 0; } @@ -55,7 +55,7 @@ void dndCleanup() { monCleanup(); walCleanUp(); taosStopCacheRefreshWorker(); - dDebug("dnode env is cleaned up"); + dInfo("dnode env is cleaned up"); } void dndSetMsgHandle(SMgmtWrapper *pWrapper, tmsg_t msgType, NodeMsgFp nodeMsgFp, int8_t vgId) { diff --git a/source/dnode/mgmt/main/src/dndMsg.c b/source/dnode/mgmt/main/src/dndMsg.c index 3aafa5a5e3..2ab1e401c7 100644 --- a/source/dnode/mgmt/main/src/dndMsg.c +++ b/source/dnode/mgmt/main/src/dndMsg.c @@ -66,7 +66,7 @@ void dndProcessRpcMsg(SMgmtWrapper *pWrapper, SRpcMsg *pRpc, SEpSet *pEpSet) { dTrace("msg:%p, is created, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); code = (*msgFp)(pWrapper, pMsg); } else if (pWrapper->procType == PROC_PARENT) { - dTrace("msg:%p, is created and will put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, + dTrace("msg:%p, is created and put into child queue, handle:%p app:%p user:%s", pMsg, pRpc->handle, pRpc->ahandle, pMsg->user); code = taosProcPutToChildQ(pWrapper->pProc, pMsg, sizeof(SNodeMsg), pRpc->pCont, pRpc->contLen, PROC_REQ); } else { diff --git a/source/dnode/mgmt/main/src/dndObj.c b/source/dnode/mgmt/main/src/dndObj.c index 99dc782a9b..602ebc6b3c 100644 --- a/source/dnode/mgmt/main/src/dndObj.c +++ b/source/dnode/mgmt/main/src/dndObj.c @@ -34,6 +34,14 @@ static int32_t dndInitVars(SDnode *pDnode, const SDnodeOpt *pOption) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } + + if (!tsMultiProcess || pDnode->ntype == DNODE || pDnode->ntype == NODE_MAX) { + pDnode->lockfile = dndCheckRunning(pDnode->dataDir); + if (pDnode->lockfile == NULL) { + return -1; + } + } + return 0; } @@ -42,18 +50,22 @@ static void dndClearVars(SDnode *pDnode) { SMgmtWrapper *pMgmt = &pDnode->wrappers[n]; taosMemoryFreeClear(pMgmt->path); } - dndCloseRuntimeFile(pDnode); + if (pDnode->lockfile != NULL) { + taosUnLockFile(pDnode->lockfile); + taosCloseFile(&pDnode->lockfile); + pDnode->lockfile = NULL; + } taosMemoryFreeClear(pDnode->localEp); taosMemoryFreeClear(pDnode->localFqdn); taosMemoryFreeClear(pDnode->firstEp); taosMemoryFreeClear(pDnode->secondEp); taosMemoryFreeClear(pDnode->dataDir); taosMemoryFree(pDnode); - dDebug("dnode object memory is cleared, data:%p", pDnode); + dDebug("dnode memory is cleared, data:%p", pDnode); } SDnode *dndCreate(const SDnodeOpt *pOption) { - dInfo("start to create dnode object"); + dDebug("start to create dnode object"); int32_t code = -1; char path[PATH_MAX] = {0}; SDnode *pDnode = NULL; @@ -77,29 +89,11 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { smGetMgmtFp(&pDnode->wrappers[SNODE]); bmGetMgmtFp(&pDnode->wrappers[BNODE]); - if (dndOpenRuntimeFile(pDnode) != 0) { - dError("failed to open runtime file since %s", terrstr()); - goto _OVER; - } - - if (dndInitServer(pDnode) != 0) { - dError("failed to init trans server since %s", terrstr()); - goto _OVER; - } - - if (dndInitClient(pDnode) != 0) { - dError("failed to init trans client since %s", terrstr()); - goto _OVER; - } - - if (dndInitMsgHandle(pDnode) != 0) { - goto _OVER; - } - for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; snprintf(path, sizeof(path), "%s%s%s", pDnode->dataDir, TD_DIRSEP, pWrapper->name); pWrapper->path = strdup(path); + pWrapper->shm.id = -1; pWrapper->pDnode = pDnode; if (pWrapper->path == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -110,15 +104,27 @@ SDnode *dndCreate(const SDnodeOpt *pOption) { taosInitRWLatch(&pWrapper->latch); } + if (dndInitMsgHandle(pDnode) != 0) { + dError("failed to msg handles since %s", terrstr()); + goto _OVER; + } + + if (dndReadShmFile(pDnode) != 0) { + dError("failed to read shm file since %s", terrstr()); + goto _OVER; + } + + SMsgCb msgCb = dndCreateMsgcb(&pDnode->wrappers[0]); + tmsgSetDefaultMsgCb(&msgCb); + + dInfo("dnode is created, data:%p", pDnode); code = 0; _OVER: if (code != 0 && pDnode) { dndClearVars(pDnode); pDnode = NULL; - dError("failed to create dnode object since %s", terrstr()); - } else { - dInfo("dnode object is created, data:%p", pDnode); + dError("failed to create dnode since %s", terrstr()); } return pDnode; @@ -135,21 +141,20 @@ void dndClose(SDnode *pDnode) { dInfo("start to close dnode, data:%p", pDnode); dndSetStatus(pDnode, DND_STAT_STOPPED); - dndCleanupServer(pDnode); - dndCleanupClient(pDnode); - for (ENodeType n = 0; n < NODE_MAX; ++n) { SMgmtWrapper *pWrapper = &pDnode->wrappers[n]; dndCloseNode(pWrapper); } dndClearVars(pDnode); - dInfo("dnode object is closed, data:%p", pDnode); + dInfo("dnode is closed, data:%p", pDnode); } void dndHandleEvent(SDnode *pDnode, EDndEvent event) { - dInfo("dnode object receive event %d, data:%p", event, pDnode); - pDnode->event = event; + dInfo("dnode receive %s event, data:%p", dndEventStr(event), pDnode); + if (event == DND_EVENT_STOP) { + pDnode->event = event; + } } SMgmtWrapper *dndAcquireWrapper(SDnode *pDnode, ENodeType ntype) { diff --git a/source/dnode/mgmt/main/src/dndStr.c b/source/dnode/mgmt/main/src/dndStr.c index 00d8b0d6e0..8a5af68b4f 100644 --- a/source/dnode/mgmt/main/src/dndStr.c +++ b/source/dnode/mgmt/main/src/dndStr.c @@ -62,3 +62,16 @@ const char *dndNodeProcStr(ENodeType ntype) { return "taosd"; } } + +const char *dndEventStr(EDndEvent ev) { + switch (ev) { + case DND_EVENT_START: + return "start"; + case DND_EVENT_STOP: + return "stop"; + case DND_EVENT_CHILD: + return "child"; + default: + return "UNKNOWN"; + } +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index dff918f135..5c3167dd79 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -310,6 +310,8 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); return -1; } + + pShow->numOfReads = 0; } ShowRetrieveFp retrieveFp = pMgmt->retrieveFps[pShow->type]; @@ -374,7 +376,7 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { pReq->pRsp = pRsp; pReq->rspLen = size; - if (rowsRead == 0 || rowsToRead == 0 || (rowsRead == rowsToRead && pShow->numOfRows == pShow->numOfReads)) { + if (rowsRead == 0 || rowsToRead == 0 || (rowsRead < rowsToRead)) { pRsp->completed = 1; mDebug("show:0x%" PRIx64 ", retrieve completed", pShow->id); mndReleaseShowObj((SShowObj*) pShow, true); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 7799ac7562..02e09252f9 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1608,7 +1608,6 @@ static int32_t mndRetrieveStb(SNodeMsg *pReq, SShowObj *pShow, char *data, int32 SStbObj *pStb = NULL; int32_t cols = 0; char *pWrite; - char prefix[TSDB_DB_FNAME_LEN] = {0}; SDbObj* pDb = NULL; if (strlen(pShow->db) > 0) { diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index f1083c0d91..0d3fcffe7d 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -100,10 +100,11 @@ int32_t tsdbUpdateSmaWindow(STsdb *pTsdb, const char *msg); * @brief Insert tSma(Time-range-wise SMA) data from stream computing engine * * @param pTsdb + * @param indexUid * @param msg * @return int32_t */ -int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg); +int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg); /** * @brief Drop tSma data and local cache. @@ -130,16 +131,11 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg); * @param pTsdb * @param pData * @param indexUid - * @param interval - * @param intervalUnit - * @param tableUid - * @param colId * @param querySKey * @param nMaxResult * @return int32_t */ -int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, - tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult); +int32_t tsdbGetTSmaData(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); // STsdbCfg int tsdbOptionsInit(STsdbCfg *); diff --git a/source/dnode/vnode/src/inc/tsdbSma.h b/source/dnode/vnode/src/inc/tsdbSma.h index e0170c90e7..da0a6856ab 100644 --- a/source/dnode/vnode/src/inc/tsdbSma.h +++ b/source/dnode/vnode/src/inc/tsdbSma.h @@ -44,15 +44,14 @@ int32_t tsdbRemoveTSmaData(STsdb *pTsdb, STSma *param, STimeWindow *pWin); #endif // internal func -static FORCE_INLINE int32_t tsdbEncodeTSmaKey(tb_uid_t tableUid, col_id_t colId, TSKEY tsKey, void **pData) { +static FORCE_INLINE int32_t tsdbEncodeTSmaKey(int64_t groupId, TSKEY tsKey, void **pData) { int32_t len = 0; - len += taosEncodeFixedI64(pData, tableUid); - len += taosEncodeFixedU16(pData, colId); len += taosEncodeFixedI64(pData, tsKey); + len += taosEncodeFixedI64(pData, groupId); return len; } -static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { +static FORCE_INLINE int32_t tsdbRLockSma(SSmaEnv *pEnv) { int code = taosThreadRwlockRdlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); @@ -61,7 +60,7 @@ static FORCE_INLINE int tsdbRLockSma(SSmaEnv *pEnv) { return 0; } -static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { +static FORCE_INLINE int32_t tsdbWLockSma(SSmaEnv *pEnv) { int code = taosThreadRwlockWrlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); @@ -70,7 +69,7 @@ static FORCE_INLINE int tsdbWLockSma(SSmaEnv *pEnv) { return 0; } -static FORCE_INLINE int tsdbUnLockSma(SSmaEnv *pEnv) { +static FORCE_INLINE int32_t tsdbUnLockSma(SSmaEnv *pEnv) { int code = taosThreadRwlockUnlock(&(pEnv->lock)); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 9c0b0802ab..344b4fa655 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -167,7 +167,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; } - if (colDataAppend(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { + if (colDataAppend(pColData, curRow, sVal.val, false) < 0) { taosArrayDestroyEx(pArray, (void (*)(void*))tDeleteSSDataBlock); return NULL; } diff --git a/source/dnode/vnode/src/tsdb/tsdbSma.c b/source/dnode/vnode/src/tsdb/tsdbSma.c index 07eafd6df0..f04f4791b5 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSma.c +++ b/source/dnode/vnode/src/tsdb/tsdbSma.c @@ -24,7 +24,7 @@ static const char *TSDB_SMA_DNAME[] = { #define SMA_STORAGE_TSDB_DAYS 30 #define SMA_STORAGE_TSDB_TIMES 10 #define SMA_STORAGE_SPLIT_HOURS 24 -#define SMA_KEY_LEN 18 // tableUid_colId_TSKEY 8+2+8 +#define SMA_KEY_LEN 16 // TSKEY+groupId 8+8 #define SMA_DROP_EXPIRED_TIME 10 // default is 10 seconds #define SMA_STATE_HASH_SLOT 4 @@ -38,10 +38,10 @@ typedef enum { } ESmaStorageLevel; typedef struct { - STsdb *pTsdb; - SDBFile dFile; - SSDataBlock *pData; // sma data - int32_t interval; // interval with the precision of DB + STsdb *pTsdb; + SDBFile dFile; + const SArray *pDataBlocks; // sma data + int32_t interval; // interval with the precision of DB } STSmaWriteH; typedef struct { @@ -94,26 +94,24 @@ static int32_t tsdbUnRefSmaStat(STsdb *pTsdb, SSmaStat *pStat); // read data // TODO: This is the basic params, and should wrap the params to a queryHandle. -static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, - int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, - int32_t nMaxResult); +static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult); // insert data -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, SSDataBlock *pData, int64_t interval, +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, const SArray *pDataBlocks, int64_t interval, int8_t intervalUnit); static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH); static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit); static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit); -static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData); +static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid); static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen); -static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision); +static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted); static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLevel); -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid); +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid); static int32_t tsdbInitTSmaFile(STSmaReadH *pSmaH, int64_t indexUid, TSKEY skey); static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey); static void tsdbGetSmaDir(int32_t vgId, ETsdbSmaType smaType, char dirName[]); -static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg); -static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg); +static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg); +static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg); // mgmt interface static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid); @@ -387,7 +385,6 @@ static int32_t tsdbCheckAndInitSmaEnv(STsdb *pTsdb, int8_t smaType) { return TSDB_CODE_SUCCESS; }; - static int32_t tsdbSetExpiredWindow(STsdb *pTsdb, SHashObj *pItemsHash, int64_t indexUid, int64_t winSKey) { SSmaStatItem *pItem = taosHashGet(pItemsHash, &indexUid, sizeof(indexUid)); if (pItem == NULL) { @@ -480,18 +477,15 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg) { TSKEY expiredWindows[SMA_TEST_EXPIRED_WINDOW_SIZE]; #endif - // Firstly, assume that tSma can only be created on super table/normal table. // getActiveTimeWindow - - SSmaEnv *pEnv = REPO_SMA_ENV(pTsdb, TSDB_SMA_TYPE_TIME_RANGE); + SSmaEnv *pEnv = REPO_SMA_ENV(pTsdb, TSDB_SMA_TYPE_TIME_RANGE); SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *pItemsHash = SMA_ENV_STAT_ITEMS(pEnv); TASSERT(pEnv != NULL && pStat != NULL && pItemsHash != NULL); - // basic procedure // TODO: optimization tsdbRefSmaStat(pTsdb, pStat); @@ -523,11 +517,11 @@ int32_t tsdbUpdateExpiredWindowImpl(STsdb *pTsdb, const char *msg) { tdFreeTSmaWrapper(pSW); break; } - if(pSW == NULL) { - if((pSW =metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) { + if (pSW == NULL) { + if ((pSW = metaGetSmaInfoByTable(REPO_META(pTsdb), pBlock->suid)) == NULL) { break; } - if((pSW->number) <= 0 || (pSW->tSma == NULL)) { + if ((pSW->number) <= 0 || (pSW->tSma == NULL)) { tdFreeTSmaWrapper(pSW); break; } @@ -657,9 +651,8 @@ static int32_t tsdbGetSmaStorageLevel(int64_t interval, int8_t intervalUnit) { */ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t keyLen, void *pData, uint32_t dataLen) { SDBFile *pDBFile = &pSmaH->dFile; - tsdbDebug("vgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", dataLen %d", - REPO_ID(pSmaH->pTsdb), pDBFile->path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), - *(int64_t *)POINTER_SHIFT(smaKey, 10), dataLen); + printf("\nvgId:%d insert sma data blocks into %s: smaKey %" PRIx64 "-%" PRIx64 ", dataLen %" PRIu32 "\n", + REPO_ID(pSmaH->pTsdb), pDBFile->path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), dataLen); // TODO: insert sma data blocks into B+Tree(TDB) if (tsdbSaveSmaToDB(pDBFile, smaKey, keyLen, pData, dataLen) != 0) { @@ -683,9 +676,15 @@ static int32_t tsdbInsertTSmaBlocks(STSmaWriteH *pSmaH, void *smaKey, uint32_t k * @param interval * @param intervalUnit * @param precision + * @param adjusted Interval already adjusted according to DB precision * @return int64_t */ -static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision) { +static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit, int8_t precision, bool adjusted) { + + if (adjusted) { + return interval; + } + switch (intervalUnit) { case TIME_UNIT_YEAR: // approximate value interval *= 365 * 86400 * 1e3; @@ -753,59 +752,12 @@ static int64_t tsdbGetIntervalByPrecision(int64_t interval, uint8_t intervalUnit return interval; } -/** - * @brief Split the TSma data blocks into expected size and insert into B+Tree. - * - * @param pSmaH - * @param pData - * @param nOffset The nOffset of blocks since fid changes. - * @param nBlocks The nBlocks with the same fid since nOffset. - * @return int32_t - */ -static int32_t tsdbInsertTSmaDataSection(STSmaWriteH *pSmaH, STSmaDataWrapper *pData) { - STsdb *pTsdb = pSmaH->pTsdb; - - tsdbDebug("tsdbInsertTSmaDataSection: index %" PRIi64 ", skey %" PRIi64, pData->indexUid, pData->skey); - - // TODO: check the data integrity - - int32_t len = 0; - while (true) { - if (len >= pData->dataLen) { - break; - } - assert(pData->dataLen > 0); - STSmaTbData *pTbData = (STSmaTbData *)POINTER_SHIFT(pData->data, len); - - int32_t tbLen = 0; - while (true) { - if (tbLen >= pTbData->dataLen) { - break; - } - assert(pTbData->dataLen > 0); - STSmaColData *pColData = (STSmaColData *)POINTER_SHIFT(pTbData->data, tbLen); - char smaKey[SMA_KEY_LEN] = {0}; - void *pSmaKey = &smaKey; -#if 0 - printf("tsdbInsertTSmaDataSection: index %" PRIi64 ", skey %" PRIi64 " table[%" PRIi64 "]col[%" PRIu16 "]\n", - pData->indexUid, pData->skey, pTbData->tableUid, pColData->colId); -#endif - tsdbEncodeTSmaKey(pTbData->tableUid, pColData->colId, pData->skey, (void **)&pSmaKey); - if (tsdbInsertTSmaBlocks(pSmaH, smaKey, SMA_KEY_LEN, pColData->data, pColData->blockSize) < 0) { - tsdbWarn("vgId:%d insert tSma blocks failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - } - tbLen += (sizeof(STSmaColData) + pColData->blockSize); - } - len += (sizeof(STSmaTbData) + pTbData->dataLen); - } - - return TSDB_CODE_SUCCESS; -} - -static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, SSDataBlock *pData, int64_t interval, int8_t intervalUnit) { +static int32_t tsdbInitTSmaWriteH(STSmaWriteH *pSmaH, STsdb *pTsdb, const SArray *pDataBlocks, int64_t interval, + int8_t intervalUnit) { pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision); - pSmaH->pData = pData; + pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision, true); + pSmaH->pDataBlocks = pDataBlocks; + pSmaH->dFile.fid = TSDB_IVLD_FID; return TSDB_CODE_SUCCESS; } @@ -815,7 +767,7 @@ static void tsdbDestroyTSmaWriteH(STSmaWriteH *pSmaH) { } } -static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int64_t indexUid, int32_t fid) { +static int32_t tsdbSetTSmaDataFile(STSmaWriteH *pSmaH, int64_t indexUid, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; ASSERT(pSmaH->dFile.path == NULL && pSmaH->dFile.pDB == NULL); @@ -859,11 +811,10 @@ static int32_t tsdbGetTSmaDays(STsdb *pTsdb, int64_t interval, int32_t storageLe * @param msg * @return int32_t */ -static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - SSDataBlock *pData = (SSDataBlock *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); - int64_t indexUid = SMA_TEST_INDEX_UID; +static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, int64_t indexUid, const char *msg) { + STsdbCfg *pCfg = REPO_CFG(pTsdb); + const SArray *pDataBlocks = (const SArray *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; @@ -871,15 +822,15 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { return terrno; } - if (pData == NULL) { + if (pDataBlocks == NULL) { terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert tSma data failed since pData is NULL", REPO_ID(pTsdb)); + tsdbWarn("vgId:%d insert tSma data failed since pDataBlocks is NULL", REPO_ID(pTsdb)); return terrno; } - if (taosArrayGetSize(pData->pDataBlock) <= 0) { + if (taosArrayGetSize(pDataBlocks) <= 0) { terrno = TSDB_CODE_INVALID_PARA; - tsdbWarn("vgId:%d insert tSma data failed since pDataBlock is empty", REPO_ID(pTsdb)); + tsdbWarn("vgId:%d insert tSma data failed since pDataBlocks is empty", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } @@ -899,10 +850,9 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { } STSma *pSma = pItem->pSma; - STSmaWriteH tSmaH = {0}; - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData, pSma->interval, pSma->intervalUnit) != 0) { + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pDataBlocks, pSma->interval, pSma->intervalUnit) != 0) { return TSDB_CODE_FAILED; } @@ -921,33 +871,134 @@ static int32_t tsdbInsertTSmaDataImpl(STsdb *pTsdb, char *msg) { int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); + // key: skey + groupId + char smaKey[SMA_KEY_LEN] = {0}; + char dataBuf[512] = {0}; + void *pDataBuf = &dataBuf; + int32_t sz = taosArrayGetSize(pDataBlocks); + for (int32_t i = 0; i < sz; ++i) { + SSDataBlock *pDataBlock = *(SSDataBlock **)taosArrayGet(pDataBlocks, i); + int32_t colNum = pDataBlock->info.numOfCols; + int32_t rows = pDataBlock->info.rows; + int32_t rowSize = pDataBlock->info.rowSize; + int64_t groupId = pDataBlock->info.groupId; + for (int32_t j = 0; j < rows; ++j) { + printf("|"); + TSKEY skey = TSKEY_INITIAL_VAL; // the start key of TS window by interval + void *pSmaKey = &smaKey; + int32_t tlen = 0; + for (int32_t k = 0; k < colNum; ++k) { + SColumnInfoData *pColInfoData = *(SColumnInfoData **)taosArrayGet(pDataBlock->pDataBlock, k); + void *var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + skey = *(TSKEY *)var; + printf("==> skey = %" PRIi64 " groupId = %" PRId64 "|", skey, groupId); + tsdbEncodeTSmaKey(groupId, skey, &pSmaKey); + break; + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_UTINYINT: + printf(" %15d |", *(uint8_t *)var); + tlen += taosEncodeFixedU8(&pDataBuf, *(uint8_t *)var); + break; + case TSDB_DATA_TYPE_TINYINT: + printf(" %15d |", *(int8_t *)var); + tlen += taosEncodeFixedI8(&pDataBuf, *(int8_t *)var); + break; + case TSDB_DATA_TYPE_SMALLINT: + printf(" %15d |", *(int16_t *)var); + tlen += taosEncodeFixedI16(&pDataBuf, *(int16_t *)var); + break; + case TSDB_DATA_TYPE_USMALLINT: + printf(" %15d |", *(uint16_t *)var); + tlen += taosEncodeFixedU16(&pDataBuf, *(uint16_t *)var); + break; + case TSDB_DATA_TYPE_INT: + printf(" %15d |", *(int32_t *)var); + tlen += taosEncodeFixedI32(&pDataBuf, *(int32_t *)var); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_UINT: + printf(" %15u |", *(uint32_t *)var); + tlen += taosEncodeFixedU32(&pDataBuf, *(uint32_t *)var); + break; + case TSDB_DATA_TYPE_BIGINT: + printf(" %15ld |", *(int64_t *)var); + tlen += taosEncodeFixedI64(&pDataBuf, *(int64_t *)var); + break; + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_UBIGINT: + printf(" %15lu |", *(uint64_t *)var); + tlen += taosEncodeFixedU64(&pDataBuf, *(uint64_t *)var); + break; + case TSDB_DATA_TYPE_NCHAR: { + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARCHAR: { // TSDB_DATA_TYPE_BINARY + char tmpChar[100] = {0}; + strncpy(tmpChar, varDataVal(var), varDataLen(var)); + printf(" %s |", tmpChar); + tlen += taosEncodeBinary(&pDataBuf, varDataVal(var), varDataLen(var)); + break; + } + case TSDB_DATA_TYPE_VARBINARY: + // TODO: add binary/varbinary + TASSERT(0); + default: + printf("the column type %" PRIi16 " is undefined\n", pColInfoData->info.type); + TASSERT(0); + break; + } + } + if ((tlen > 0) && (skey != TSKEY_INITIAL_VAL)) { + int32_t fid = (int32_t)(TSDB_KEY_FID(skey, daysPerFile, pCfg->precision)); -#if 0 - int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); + // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index + // file + // - Set and open the DFile or the B+Tree file + // TODO: tsdbStartTSmaCommit(); + if (fid != tSmaH.dFile.fid) { + if (tSmaH.dFile.fid != TSDB_IVLD_FID) { + tsdbCloseDBF(&tSmaH.dFile); + } + tsdbSetTSmaDataFile(&tSmaH, indexUid, fid); + if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { + tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), + tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } + } - // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file - // - Set and open the DFile or the B+Tree file - // TODO: tsdbStartTSmaCommit(); - tsdbSetTSmaDataFile(&tSmaH, pData, indexUid, fid); - if (tsdbOpenDBF(pTsdb->pTSmaEnv->dbEnv, &tSmaH.dFile) != 0) { - tsdbWarn("vgId:%d open DB file %s failed since %s", REPO_ID(pTsdb), - tSmaH.dFile.path ? tSmaH.dFile.path : "path is NULL", tstrerror(terrno)); - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; + if (tsdbInsertTSmaBlocks(&tSmaH, &smaKey, SMA_KEY_LEN, pDataBuf, tlen) != 0) { + tsdbWarn("vgId:%d insert tSma data blocks failed for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64 + " since %s", + REPO_ID(pTsdb), indexUid, skey, groupId, tstrerror(terrno)); + tsdbDestroyTSmaWriteH(&tSmaH); + tsdbUnRefSmaStat(pTsdb, pStat); + return TSDB_CODE_FAILED; + } else { + tsdbWarn("vgId:%d insert tSma data blocks success for index %" PRIi64 ", skey %" PRIi64 ", groupId %" PRIi64, + REPO_ID(pTsdb), indexUid, skey, groupId); + } + // TODO:tsdbEndTSmaCommit(); + + // Step 3: reset the SSmaStat + tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), indexUid, skey); + } else { + tsdbWarn("vgId:%d invalid data skey:%" PRIi64 ", tlen %" PRIi32 " during insert tSma data for %" PRIi64, + REPO_ID(pTsdb), skey, tlen, indexUid); + } + + printf("\n"); + } } - if (tsdbInsertTSmaDataSection(&tSmaH, pData) != 0) { - tsdbWarn("vgId:%d insert tSma data section failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); - tsdbDestroyTSmaWriteH(&tSmaH); - tsdbUnRefSmaStat(pTsdb, pStat); - return TSDB_CODE_FAILED; - } - // TODO:tsdbEndTSmaCommit(); - - // Step 3: reset the SSmaStat - tsdbResetExpiredWindow(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv), pData->indexUid, pData->skey); -#endif tsdbDestroyTSmaWriteH(&tSmaH); tsdbUnRefSmaStat(pTsdb, pStat); return TSDB_CODE_SUCCESS; @@ -1002,7 +1053,7 @@ static int32_t tsdbDropTSmaDataImpl(STsdb *pTsdb, int64_t indexUid) { // TODO: } -static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, int32_t fid) { +static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, int32_t fid) { STsdb *pTsdb = pSmaH->pTsdb; char tSmaFile[TSDB_FILENAME_LEN] = {0}; @@ -1012,11 +1063,11 @@ static int32_t tsdbSetRSmaDataFile(STSmaWriteH *pSmaH, STSmaDataWrapper *pData, return TSDB_CODE_SUCCESS; } -static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { - STsdbCfg *pCfg = REPO_CFG(pTsdb); - SSDataBlock *pData = (SSDataBlock *)msg; - SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); - int64_t indexUid = SMA_TEST_INDEX_UID; +static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, const char *msg) { + STsdbCfg *pCfg = REPO_CFG(pTsdb); + const SArray *pDataBlocks = (const SArray *)msg; + SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pRSmaEnv); + int64_t indexUid = SMA_TEST_INDEX_UID; if (pEnv == NULL) { terrno = TSDB_CODE_INVALID_PTR; @@ -1030,15 +1081,15 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { return terrno; } - if (pData == NULL) { + if (pDataBlocks == NULL) { terrno = TSDB_CODE_INVALID_PTR; - tsdbWarn("vgId:%d insert rSma data failed since pData is NULL", REPO_ID(pTsdb)); + tsdbWarn("vgId:%d insert rSma data failed since pDataBlocks is NULL", REPO_ID(pTsdb)); return terrno; } - if (taosArrayGetSize(pData->pDataBlock) <= 0) { + if (taosArrayGetSize(pDataBlocks) <= 0) { terrno = TSDB_CODE_INVALID_PARA; - tsdbWarn("vgId:%d insert rSma data failed since pDataBlock is empty", REPO_ID(pTsdb)); + tsdbWarn("vgId:%d insert rSma data failed since pDataBlocks is empty", REPO_ID(pTsdb)); return TSDB_CODE_FAILED; } @@ -1061,12 +1112,12 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { STSmaWriteH tSmaH = {0}; - if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pData, pSma->interval, pSma->intervalUnit) != 0) { + if (tsdbInitTSmaWriteH(&tSmaH, pTsdb, pDataBlocks, pSma->interval, pSma->intervalUnit) != 0) { return TSDB_CODE_FAILED; } - char rPath[TSDB_FILENAME_LEN] = {0}; - char aPath[TSDB_FILENAME_LEN] = {0}; + char rPath[TSDB_FILENAME_LEN] = {0}; + char aPath[TSDB_FILENAME_LEN] = {0}; snprintf(rPath, TSDB_FILENAME_LEN, "%s%s%" PRIi64, SMA_ENV_PATH(pEnv), TD_DIRSEP, indexUid); tfsAbsoluteName(REPO_TFS(pTsdb), SMA_ENV_DID(pEnv), rPath, aPath); if (!taosCheckExistFile(aPath)) { @@ -1078,7 +1129,7 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { // Step 1: Judge the storage level and days int32_t storageLevel = tsdbGetSmaStorageLevel(pSma->interval, pSma->intervalUnit); int32_t daysPerFile = tsdbGetTSmaDays(pTsdb, tSmaH.interval, storageLevel); - #if 0 +#if 0 int32_t fid = (int32_t)(TSDB_KEY_FID(pData->skey, daysPerFile, pCfg->precision)); // Step 2: Set the DFile for storage of SMA index, and iterate/split the TSma data and store to B+Tree index file @@ -1119,7 +1170,7 @@ static int32_t tsdbInsertRSmaDataImpl(STsdb *pTsdb, char *msg) { */ static int32_t tsdbInitTSmaReadH(STSmaReadH *pSmaH, STsdb *pTsdb, int64_t interval, int8_t intervalUnit) { pSmaH->pTsdb = pTsdb; - pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision); + pSmaH->interval = tsdbGetIntervalByPrecision(interval, intervalUnit, REPO_CFG(pTsdb)->precision, true); pSmaH->storageLevel = tsdbGetSmaStorageLevel(interval, intervalUnit); pSmaH->days = tsdbGetTSmaDays(pTsdb, pSmaH->interval, pSmaH->storageLevel); } @@ -1185,17 +1236,11 @@ static bool tsdbSetAndOpenTSmaFile(STSmaReadH *pReadH, TSKEY *queryKey) { * @param pTsdb Return the data between queryWin and fill the pData. * @param pData * @param indexUid - * @param interval - * @param intervalUnit - * @param tableUid - * @param colId * @param pQuerySKey * @param nMaxResult The query invoker should control the nMaxResult need to return to avoid OOM. * @return int32_t */ -static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, - int8_t intervalUnit, tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, - int32_t nMaxResult) { +static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, char *pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { SSmaEnv *pEnv = atomic_load_ptr(&pTsdb->pTSmaEnv); if (!pEnv) { @@ -1243,13 +1288,18 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ tsdbDebug("vgId:%d skey %" PRIi64 " of window not in expired window for index %" PRIi64, REPO_ID(pTsdb), querySKey, indexUid); } - tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); + + STSma *pTSma = pItem->pSma; + + #endif STSmaReadH tReadH = {0}; - tsdbInitTSmaReadH(&tReadH, pTsdb, interval, intervalUnit); + tsdbInitTSmaReadH(&tReadH, pTsdb, pTSma->interval, pTSma->intervalUnit); tsdbCloseDBF(&tReadH.dFile); + + tsdbUnRefSmaStat(pTsdb, SMA_ENV_STAT(pTsdb->pTSmaEnv)); tsdbInitTSmaFile(&tReadH, indexUid, querySKey); if (tsdbOpenDBF(SMA_ENV_ENV(pTsdb->pTSmaEnv), &tReadH.dFile) != 0) { @@ -1257,21 +1307,19 @@ static int32_t tsdbGetTSmaDataImpl(STsdb *pTsdb, STSmaDataWrapper *pData, int64_ return TSDB_CODE_FAILED; } - char smaKey[SMA_KEY_LEN] = {0}; - void *pSmaKey = &smaKey; - tsdbEncodeTSmaKey(tableUid, colId, querySKey, (void **)&pSmaKey); + char smaKey[SMA_KEY_LEN] = {0}; + void *pSmaKey = &smaKey; + int64_t queryGroupId = 1; + tsdbEncodeTSmaKey(queryGroupId, querySKey, (void **)&pSmaKey); - tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), - tReadH.dFile.path, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), - *(int64_t *)POINTER_SHIFT(smaKey, 10), SMA_KEY_LEN); + tsdbDebug("vgId:%d get sma data from %s: smaKey %" PRIx64 "-%" PRIx64 ", keyLen %d", REPO_ID(pTsdb), + tReadH.dFile.path, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), SMA_KEY_LEN); void *result = NULL; uint32_t valueSize = 0; if ((result = tsdbGetSmaDataByKey(&tReadH.dFile, smaKey, SMA_KEY_LEN, &valueSize)) == NULL) { - tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIu16 "-%" PRIx64 - " since %s", - REPO_ID(pTsdb), indexUid, *(tb_uid_t *)smaKey, *(uint16_t *)POINTER_SHIFT(smaKey, 8), - *(int64_t *)POINTER_SHIFT(smaKey, 10), tstrerror(terrno)); + tsdbWarn("vgId:%d get sma data failed from smaIndex %" PRIi64 ", smaKey %" PRIx64 "-%" PRIx64 " since %s", + REPO_ID(pTsdb), indexUid, *(int64_t *)smaKey, *(int64_t *)POINTER_SHIFT(smaKey, 8), tstrerror(terrno)); tsdbCloseDBF(&tReadH.dFile); return TSDB_CODE_FAILED; } @@ -1347,11 +1395,10 @@ int32_t tsdbRemoveTSmaData(STsdb *pTsdb, void *smaIndex, STimeWindow *pWin) { } #endif - // TODO: Who is responsible for resource allocate and release? -int32_t tsdbInsertTSmaData(STsdb *pTsdb, char *msg) { +int32_t tsdbInsertTSmaData(STsdb *pTsdb, int64_t indexUid, const char *msg) { int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbInsertTSmaDataImpl(pTsdb, msg)) < 0) { + if ((code = tsdbInsertTSmaDataImpl(pTsdb, indexUid, msg)) < 0) { tsdbWarn("vgId:%d insert tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; @@ -1373,18 +1420,14 @@ int32_t tsdbInsertRSmaData(STsdb *pTsdb, char *msg) { return code; } - -int32_t tsdbGetTSmaData(STsdb *pTsdb, STSmaDataWrapper *pData, int64_t indexUid, int64_t interval, int8_t intervalUnit, - tb_uid_t tableUid, col_id_t colId, TSKEY querySKey, int32_t nMaxResult) { +int32_t tsdbGetTSmaData(STsdb *pTsdb, char*pData, int64_t indexUid, TSKEY querySKey, int32_t nMaxResult) { int32_t code = TSDB_CODE_SUCCESS; - if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, interval, intervalUnit, tableUid, colId, querySKey, - nMaxResult)) < 0) { + if ((code = tsdbGetTSmaDataImpl(pTsdb, pData, indexUid, querySKey, nMaxResult)) < 0) { tsdbWarn("vgId:%d get tSma data failed since %s", REPO_ID(pTsdb), tstrerror(terrno)); } return code; } - int32_t tsdbDropTSmaData(STsdb *pTsdb, int64_t indexUid) { int32_t code = TSDB_CODE_SUCCESS; if ((code = tsdbDropTSmaDataImpl(pTsdb, indexUid)) < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 598647f797..d377ec2039 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -69,9 +69,9 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return tqProcessPollReq(pVnode->pTq, pMsg); case TDMT_VND_TASK_PIPE_EXEC: case TDMT_VND_TASK_MERGE_EXEC: - return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, pInfo->workerId); + return tqProcessTaskExec(pVnode->pTq, msgstr, msgLen, 0); case TDMT_VND_STREAM_TRIGGER: - return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, pInfo->workerId); + return tqProcessStreamTrigger(pVnode->pTq, pMsg->pCont, pMsg->contLen, 0); case TDMT_VND_QUERY_HEARTBEAT: return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg); default: diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 8875090b43..d6f9f0da0b 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -17,7 +17,9 @@ void smaHandleRes(void *pVnode, int64_t smaId, const SArray *data) { // TODO + blockDebugShowData(data); + tsdbInsertTSmaData(((SVnode *)pVnode)->pTsdb, smaId, (const char *)data); } void vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { @@ -202,7 +204,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vCreateSmaReq.tSma.indexUid); // record current timezone of server side - tstrncpy(vCreateSmaReq.tSma.timezone, tsTimezoneStr, TD_TIMEZONE_LEN); + vCreateSmaReq.tSma.timezoneInt = tsTimezone; if (metaCreateTSma(pVnode->pMeta, &vCreateSmaReq) < 0) { // TODO: handle error diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index d010ea4437..db31b9f4a3 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -15,6 +15,7 @@ #include #include + #include #include #include @@ -58,20 +59,21 @@ TEST(testCase, unionEncodeDecodeTest) { void *buf = taosMemoryMalloc(1024); void *pBuf = buf; + void *qBuf = buf; int32_t tlen = 0; - tlen += taosEncodeFixedU8(&buf, sut.info); - tlen += taosEncodeFixedI16(&buf, sut.nBSmaCols); + tlen += taosEncodeFixedU8(&pBuf, sut.info); + tlen += taosEncodeFixedI16(&pBuf, sut.nBSmaCols); for (col_id_t i = 0; i < sut.nBSmaCols; ++i) { - tlen += taosEncodeFixedI16(&buf, sut.pBSmaCols[i]); + tlen += taosEncodeFixedI16(&pBuf, sut.pBSmaCols[i]); } SUnionTest dut = {0}; - pBuf = taosDecodeFixedU8(pBuf, &dut.info); - pBuf = taosDecodeFixedI16(pBuf, &dut.nBSmaCols); + qBuf = taosDecodeFixedU8(qBuf, &dut.info); + qBuf = taosDecodeFixedI16(qBuf, &dut.nBSmaCols); if (dut.nBSmaCols > 0) { dut.pBSmaCols = (col_id_t *)taosMemoryMalloc(dut.nBSmaCols * sizeof(col_id_t)); for (col_id_t i = 0; i < dut.nBSmaCols; ++i) { - pBuf = taosDecodeFixedI16(pBuf, dut.pBSmaCols + i); + qBuf = taosDecodeFixedI16(qBuf, dut.pBSmaCols + i); } } else { dut.pBSmaCols = NULL; @@ -80,13 +82,17 @@ TEST(testCase, unionEncodeDecodeTest) { printf("sut.rollup=%" PRIu8 ", type=%" PRIu8 ", info=%" PRIu8 "\n", sut.rollup, sut.type, sut.info); printf("dut.rollup=%" PRIu8 ", type=%" PRIu8 ", info=%" PRIu8 "\n", dut.rollup, dut.type, dut.info); - ASSERT_EQ(sut.rollup, dut.rollup); - ASSERT_EQ(sut.type, dut.type); - ASSERT_EQ(sut.nBSmaCols, dut.nBSmaCols); + EXPECT_EQ(sut.rollup, dut.rollup); + EXPECT_EQ(sut.type, dut.type); + EXPECT_EQ(sut.nBSmaCols, dut.nBSmaCols); for (col_id_t i = 0; i < sut.nBSmaCols; ++i) { - ASSERT_EQ(*(col_id_t *)(sut.pBSmaCols + i), sut.pBSmaCols[i]); - ASSERT_EQ(*(col_id_t *)(sut.pBSmaCols + i), dut.pBSmaCols[i]); + EXPECT_EQ(*(col_id_t *)(sut.pBSmaCols + i), sut.pBSmaCols[i]); + EXPECT_EQ(*(col_id_t *)(sut.pBSmaCols + i), dut.pBSmaCols[i]); } + + taosMemoryFreeClear(buf); + taosMemoryFreeClear(dut.pBSmaCols); + taosMemoryFreeClear(sut.pBSmaCols); } #if 1 TEST(testCase, tSma_Meta_Encode_Decode_Test) { @@ -106,37 +112,37 @@ TEST(testCase, tSma_Meta_Encode_Decode_Test) { uint32_t bufLen = tEncodeTSmaWrapper(NULL, &tSmaWrapper); void *buf = taosMemoryCalloc(1, bufLen); - ASSERT_NE(buf, nullptr); + EXPECT_NE(buf, nullptr); STSmaWrapper *pSW = (STSmaWrapper *)buf; uint32_t len = tEncodeTSmaWrapper(&buf, &tSmaWrapper); - ASSERT_EQ(len, bufLen); + EXPECT_EQ(len, bufLen); // decode STSmaWrapper dstTSmaWrapper = {0}; void *result = tDecodeTSmaWrapper(pSW, &dstTSmaWrapper); - ASSERT_NE(result, nullptr); + EXPECT_NE(result, nullptr); - ASSERT_EQ(tSmaWrapper.number, dstTSmaWrapper.number); + EXPECT_EQ(tSmaWrapper.number, dstTSmaWrapper.number); for (int i = 0; i < tSmaWrapper.number; ++i) { STSma *pSma = tSmaWrapper.tSma + i; STSma *qSma = dstTSmaWrapper.tSma + i; - ASSERT_EQ(pSma->version, qSma->version); - ASSERT_EQ(pSma->intervalUnit, qSma->intervalUnit); - ASSERT_EQ(pSma->slidingUnit, qSma->slidingUnit); - ASSERT_STRCASEEQ(pSma->indexName, qSma->indexName); - ASSERT_EQ(pSma->timezoneInt, qSma->timezoneInt); - ASSERT_EQ(pSma->indexUid, qSma->indexUid); - ASSERT_EQ(pSma->tableUid, qSma->tableUid); - ASSERT_EQ(pSma->interval, qSma->interval); - ASSERT_EQ(pSma->sliding, qSma->sliding); - ASSERT_EQ(pSma->exprLen, qSma->exprLen); - ASSERT_STRCASEEQ(pSma->expr, qSma->expr); - ASSERT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen); - ASSERT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter); + EXPECT_EQ(pSma->version, qSma->version); + EXPECT_EQ(pSma->intervalUnit, qSma->intervalUnit); + EXPECT_EQ(pSma->slidingUnit, qSma->slidingUnit); + EXPECT_STRCASEEQ(pSma->indexName, qSma->indexName); + EXPECT_EQ(pSma->timezoneInt, qSma->timezoneInt); + EXPECT_EQ(pSma->indexUid, qSma->indexUid); + EXPECT_EQ(pSma->tableUid, qSma->tableUid); + EXPECT_EQ(pSma->interval, qSma->interval); + EXPECT_EQ(pSma->sliding, qSma->sliding); + EXPECT_EQ(pSma->exprLen, qSma->exprLen); + EXPECT_STRCASEEQ(pSma->expr, qSma->expr); + EXPECT_EQ(pSma->tagsFilterLen, qSma->tagsFilterLen); + EXPECT_STRCASEEQ(pSma->tagsFilter, qSma->tagsFilter); } // resource release @@ -172,12 +178,12 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { tSma.exprLen = strlen(expr); tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1); - ASSERT_NE(tSma.expr, nullptr); + EXPECT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); tSma.tagsFilter = (char *)taosMemoryCalloc(tSma.tagsFilterLen + 1, 1); - ASSERT_NE(tSma.tagsFilter, nullptr); + EXPECT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); SMeta *pMeta = NULL; @@ -189,7 +195,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { pMeta = metaOpen(smaTestDir, pMetaCfg, NULL); assert(pMeta != NULL); // save index 1 - ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); + EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); pSmaCfg->indexUid = indexUid2; tstrncpy(pSmaCfg->indexName, smaIndexName2, TSDB_INDEX_NAME_LEN); @@ -200,7 +206,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { pSmaCfg->sliding = 5; // save index 2 - ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); + EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); // get value by indexName STSma *qSmaCfg = NULL; @@ -210,8 +216,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("timezone1 = %" PRIi8 "\n", qSmaCfg->timezoneInt); printf("expr1 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("tagsFilter1 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); - ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); - ASSERT_EQ(qSmaCfg->tableUid, tSma.tableUid); + EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName1); + EXPECT_EQ(qSmaCfg->tableUid, tSma.tableUid); tdDestroyTSma(qSmaCfg); taosMemoryFreeClear(qSmaCfg); @@ -221,8 +227,8 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("timezone2 = %" PRIi8 "\n", qSmaCfg->timezoneInt); printf("expr2 = %s\n", qSmaCfg->expr != NULL ? qSmaCfg->expr : ""); printf("tagsFilter2 = %s\n", qSmaCfg->tagsFilter != NULL ? qSmaCfg->tagsFilter : ""); - ASSERT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); - ASSERT_EQ(qSmaCfg->interval, tSma.interval); + EXPECT_STRCASEEQ(qSmaCfg->indexName, smaIndexName2); + EXPECT_EQ(qSmaCfg->interval, tSma.interval); tdDestroyTSma(qSmaCfg); taosMemoryFreeClear(qSmaCfg); @@ -238,25 +244,25 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("indexName = %s\n", indexName); ++indexCnt; } - ASSERT_EQ(indexCnt, nCntTSma); + EXPECT_EQ(indexCnt, nCntTSma); metaCloseSmaCurosr(pSmaCur); // get wrapper by table uid STSmaWrapper *pSW = metaGetSmaInfoByTable(pMeta, tbUid); assert(pSW != NULL); - ASSERT_EQ(pSW->number, nCntTSma); - ASSERT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); - ASSERT_EQ(pSW->tSma->timezoneInt, timezone); - ASSERT_STRCASEEQ(pSW->tSma->expr, expr); - ASSERT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter); - ASSERT_EQ(pSW->tSma->indexUid, indexUid1); - ASSERT_EQ(pSW->tSma->tableUid, tbUid); - ASSERT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); - ASSERT_EQ((pSW->tSma + 1)->timezoneInt, timezone); - ASSERT_STRCASEEQ((pSW->tSma + 1)->expr, expr); - ASSERT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter); - ASSERT_EQ((pSW->tSma + 1)->indexUid, indexUid2); - ASSERT_EQ((pSW->tSma + 1)->tableUid, tbUid); + EXPECT_EQ(pSW->number, nCntTSma); + EXPECT_STRCASEEQ(pSW->tSma->indexName, smaIndexName1); + EXPECT_EQ(pSW->tSma->timezoneInt, timezone); + EXPECT_STRCASEEQ(pSW->tSma->expr, expr); + EXPECT_STRCASEEQ(pSW->tSma->tagsFilter, tagsFilter); + EXPECT_EQ(pSW->tSma->indexUid, indexUid1); + EXPECT_EQ(pSW->tSma->tableUid, tbUid); + EXPECT_STRCASEEQ((pSW->tSma + 1)->indexName, smaIndexName2); + EXPECT_EQ((pSW->tSma + 1)->timezoneInt, timezone); + EXPECT_STRCASEEQ((pSW->tSma + 1)->expr, expr); + EXPECT_STRCASEEQ((pSW->tSma + 1)->tagsFilter, tagsFilter); + EXPECT_EQ((pSW->tSma + 1)->indexUid, indexUid2); + EXPECT_EQ((pSW->tSma + 1)->tableUid, tbUid); tdDestroyTSmaWrapper(pSW); taosMemoryFreeClear(pSW); @@ -268,7 +274,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { printf("metaGetSmaTbUids: uid[%" PRIu32 "] = %" PRIi64 "\n", i, *(tb_uid_t *)taosArrayGet(pUids, i)); // printf("metaGetSmaTbUids: index[%" PRIu32 "] = %s", i, (char *)taosArrayGet(pUids, i)); } - ASSERT_EQ(taosArrayGetSize(pUids), 1); + EXPECT_EQ(taosArrayGetSize(pUids), 1); taosArrayDestroy(pUids); // resource release @@ -280,7 +286,7 @@ TEST(testCase, tSma_metaDB_Put_Get_Del_Test) { } #endif -#if 0 +#if 1 TEST(testCase, tSma_Data_Insert_Query_Test) { // step 1: prepare meta const char *smaIndexName1 = "sma_index_test_1"; @@ -299,9 +305,9 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { // encode STSma tSma = {0}; tSma.version = 0; - tSma.intervalUnit = TIME_UNIT_DAY; + tSma.intervalUnit = TIME_UNIT_MINUTE; tSma.interval = 1; - tSma.slidingUnit = TIME_UNIT_HOUR; + tSma.slidingUnit = TIME_UNIT_MINUTE; tSma.sliding = 1; // sliding = interval when it's convert window tSma.indexUid = indexUid1; tstrncpy(tSma.indexName, smaIndexName1, TSDB_INDEX_NAME_LEN); @@ -310,12 +316,12 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { tSma.exprLen = strlen(expr); tSma.expr = (char *)taosMemoryCalloc(1, tSma.exprLen + 1); - ASSERT_NE(tSma.expr, nullptr); + EXPECT_NE(tSma.expr, nullptr); tstrncpy(tSma.expr, expr, tSma.exprLen + 1); tSma.tagsFilterLen = strlen(tagsFilter); tSma.tagsFilter = (char *)taosMemoryCalloc(1, tSma.tagsFilterLen + 1); - ASSERT_NE(tSma.tagsFilter, nullptr); + EXPECT_NE(tSma.tagsFilter, nullptr); tstrncpy(tSma.tagsFilter, tagsFilter, tSma.tagsFilterLen + 1); SMeta *pMeta = NULL; @@ -327,11 +333,11 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { pMeta = metaOpen(smaTestDir, pMetaCfg, NULL); assert(pMeta != NULL); // save index 1 - ASSERT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); + EXPECT_EQ(metaSaveSmaToDB(pMeta, pSmaCfg), 0); // step 2: insert data - STsdb *pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); - STsdbCfg *pCfg = &pTsdb->config; + STsdb *pTsdb = (STsdb *)taosMemoryCalloc(1, sizeof(STsdb)); + STsdbCfg *pCfg = &pTsdb->config; pTsdb->pMeta = pMeta; pTsdb->vgId = 2; @@ -364,7 +370,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { strncpy(pDisks.dir, "/var/lib/taos", TSDB_FILENAME_LEN); int32_t numOfDisks = 1; pTsdb->pTfs = tfsOpen(&pDisks, numOfDisks); - ASSERT_NE(pTsdb->pTfs, nullptr); + EXPECT_NE(pTsdb->pTfs, nullptr); // generate SSubmitReq msg and update expired window int16_t schemaVer = 0; @@ -374,7 +380,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { uint32_t msgLen = sizeof(SSubmitReq) + mockBlkNum * sizeof(SSubmitBlk) + mockBlkNum * mockRowNum * mockRowLen; SSubmitReq *pMsg = (SSubmitReq *)taosMemoryCalloc(1, msgLen); - ASSERT_NE(pMsg, nullptr); + EXPECT_NE(pMsg, nullptr); pMsg->version = htobe64(schemaVer); pMsg->numOfBlocks = htonl(mockBlkNum); pMsg->length = htonl(msgLen); @@ -400,20 +406,99 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { } } - ASSERT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS); + EXPECT_EQ(tdScanAndConvertSubmitMsg(pMsg), TSDB_CODE_SUCCESS); - ASSERT_EQ(tsdbUpdateSmaWindow(pTsdb, (const char *)pMsg), 0); + EXPECT_EQ(tsdbUpdateSmaWindow(pTsdb, (const char *)pMsg), 0); // init - int32_t allocCnt = 0; - int32_t allocStep = 16384; - int32_t buffer = 1024; - void *buf = NULL; - ASSERT_EQ(tsdbMakeRoom(&buf, allocStep), 0); - int32_t bufSize = taosTSizeof(buf); - int32_t numOfTables = 10; - col_id_t numOfCols = 4096; - ASSERT_GT(numOfCols, 0); + const int32_t tSmaGroupSize = 4; + const int32_t tSmaNumOfTags = 2; + const int64_t tSmaGroupId = 12345670; + const col_id_t tSmaNumOfCols = 9; // binary/nchar/varbinary/varchar are only used for tags for group by conditions. + const int32_t tSmaNumOfRows = 2; + + SArray *pDataBlocks = taosArrayInit(tSmaGroupSize, sizeof(SSDataBlock *)); + EXPECT_NE(pDataBlocks, nullptr); + int32_t tSmaTypeArray[tSmaNumOfCols] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL, TSDB_DATA_TYPE_INT, + TSDB_DATA_TYPE_UBIGINT, TSDB_DATA_TYPE_SMALLINT, TSDB_DATA_TYPE_FLOAT, + TSDB_DATA_TYPE_DOUBLE, TSDB_DATA_TYPE_VARCHAR, TSDB_DATA_TYPE_NCHAR}; + // last 2 columns for group by tags + // int32_t tSmaTypeArray[tSmaNumOfCols] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_BOOL}; + const char *tSmaGroupbyTags[tSmaGroupSize * tSmaNumOfTags] = {"BeiJing", "HaiDian", "BeiJing", "ChaoYang", + "ShangHai", "PuDong", "ShangHai", "MinHang"}; + TSKEY tSmaSKeyMs = (int64_t)1648535332 * 1000; + int64_t tSmaIntervalMs = tSma.interval * 60 * 1000; + int64_t tSmaInitVal = 0; + + for (int32_t g = 0; g < tSmaGroupSize; ++g) { + SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); + EXPECT_NE(pDataBlock, nullptr); + pDataBlock->pBlockAgg = NULL; + pDataBlock->info.numOfCols = tSmaNumOfCols; + pDataBlock->info.rows = tSmaNumOfRows; + pDataBlock->info.groupId = tSmaGroupId + g; + + pDataBlock->pDataBlock = taosArrayInit(tSmaNumOfCols, sizeof(SColumnInfoData *)); + EXPECT_NE(pDataBlock->pDataBlock, nullptr); + for (int32_t c = 0; c < tSmaNumOfCols; ++c) { + + SColumnInfoData *pColInfoData = (SColumnInfoData *)taosMemoryCalloc(1, sizeof(SColumnInfoData)); + EXPECT_NE(pColInfoData, nullptr); + + pColInfoData->info.type = tSmaTypeArray[c]; + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pColInfoData->info.bytes = 100; // update accordingly + } else { + pColInfoData->info.bytes = TYPE_BYTES[pColInfoData->info.type]; + } + pColInfoData->pData = (char *)taosMemoryCalloc(1, tSmaNumOfRows * pColInfoData->info.bytes); + + for (int32_t r = 0; r < tSmaNumOfRows; ++r) { + void *pCellData = pColInfoData->pData + r * pColInfoData->info.bytes; + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + *(TSKEY *)pCellData = tSmaSKeyMs + tSmaIntervalMs * r; + break; + case TSDB_DATA_TYPE_BOOL: + *(bool *)pCellData = (bool)tSmaInitVal++; + break; + case TSDB_DATA_TYPE_INT: + *(int *)pCellData = (int)tSmaInitVal++; + break; + case TSDB_DATA_TYPE_UBIGINT: + *(uint64_t *)pCellData = (uint64_t)tSmaInitVal++; + break; + case TSDB_DATA_TYPE_SMALLINT: + *(int16_t *)pCellData = (int16_t)tSmaInitVal++; + break; + case TSDB_DATA_TYPE_FLOAT: + *(float *)pCellData = (float)tSmaInitVal++; + break; + case TSDB_DATA_TYPE_DOUBLE: + *(double *)pCellData = (double)tSmaInitVal++; + break; + case TSDB_DATA_TYPE_VARCHAR: // city + varDataSetLen(pCellData, strlen(tSmaGroupbyTags[g * 2])); + memcpy(varDataVal(pCellData), tSmaGroupbyTags[g * 2], varDataLen(pCellData)); + break; + case TSDB_DATA_TYPE_NCHAR: // district + varDataSetLen(pCellData, strlen(tSmaGroupbyTags[g * 2 + 1])); + memcpy(varDataVal(pCellData), tSmaGroupbyTags[g * 2 + 1], varDataLen(pCellData)); + break; + default: + EXPECT_EQ(0, 1); // add definition + break; + } + } + // push SColumnInfoData + taosArrayPush(pDataBlock->pDataBlock, &pColInfoData); + } + // push SSDataBlock + taosArrayPush(pDataBlocks, &pDataBlock); + } + + // execute + EXPECT_EQ(tsdbInsertTSmaData(pTsdb, tSma.indexUid, (const char *)pDataBlocks), TSDB_CODE_SUCCESS); #if 0 STSmaDataWrapper *pSmaData = NULL; @@ -432,7 +517,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { int32_t tableDataLen = sizeof(STSmaTbData); for (col_id_t c = 0; c < numOfCols; ++c) { if (bufSize - len - tableDataLen < buffer) { - ASSERT_EQ(tsdbMakeRoom(&buf, bufSize + allocStep), 0); + EXPECT_EQ(tsdbMakeRoom(&buf, bufSize + allocStep), 0); pSmaData = (STSmaDataWrapper *)buf; pTbData = (STSmaTbData *)POINTER_SHIFT(pSmaData, len); bufSize = taosTSizeof(buf); @@ -459,31 +544,36 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { } pSmaData->dataLen = (len - sizeof(STSmaDataWrapper)); - ASSERT_GE(bufSize, pSmaData->dataLen); + EXPECT_GE(bufSize, pSmaData->dataLen); // execute - ASSERT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); + EXPECT_EQ(tsdbInsertTSmaData(pTsdb, (char *)pSmaData), TSDB_CODE_SUCCESS); #endif - SSDataBlock *pSmaData = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - - - // step 3: query uint32_t checkDataCnt = 0; - for (int32_t t = 0; t < numOfTables; ++t) { - for (col_id_t c = 0; c < numOfCols; ++c) { - ASSERT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, interval1, intervalUnit1, tbUid + t, - c + PRIMARYKEY_TIMESTAMP_COL_ID, skey1, 1), - TSDB_CODE_SUCCESS); - ++checkDataCnt; - } - } + EXPECT_EQ(tsdbGetTSmaData(pTsdb, NULL, indexUid1, skey1, 1), TSDB_CODE_SUCCESS); + ++checkDataCnt; printf("%s:%d The sma data check count for insert and query is %" PRIu32 "\n", __FILE__, __LINE__, checkDataCnt); // release data taosMemoryFreeClear(pMsg); - taosTZfree(buf); + + for (int32_t i = 0; i < taosArrayGetSize(pDataBlocks); ++i) { + SSDataBlock *pDataBlock = *(SSDataBlock **)taosArrayGet(pDataBlocks, i); + int32_t numOfOutput = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t j = 0; j < numOfOutput; ++j) { + SColumnInfoData *pColInfoData = *(SColumnInfoData **)taosArrayGet(pDataBlock->pDataBlock, j); + colDataDestroy(pColInfoData); + taosMemoryFreeClear(pColInfoData); + } + + taosArrayDestroy(pDataBlock->pDataBlock); + taosMemoryFreeClear(pDataBlock->pBlockAgg); + taosMemoryFreeClear(pDataBlock); + } + taosArrayDestroy(pDataBlocks); + // release meta tdDestroyTSma(&tSma); tfsClose(pTsdb->pTfs); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index bcbfeb7015..5d5e0cae88 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -156,6 +156,8 @@ SArray* interResFromBinary(const char* data, int32_t len); void freeInterResult(void* param); void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo); +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); + void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo); bool hasRemainData(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index bc55f46986..ce0069fdfa 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -233,8 +233,8 @@ struct SOperatorInfo; typedef void (*__optr_encode_fn_t)(struct SOperatorInfo* pOperator, char **result, int32_t *length); typedef bool (*__optr_decode_fn_t)(struct SOperatorInfo* pOperator, char *result, int32_t length); -typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* param); -typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* param, bool* newgroup); +typedef int32_t (*__optr_open_fn_t)(struct SOperatorInfo* pOptr); +typedef SSDataBlock* (*__optr_fn_t)(struct SOperatorInfo* pOptr, bool* newgroup); typedef void (*__optr_close_fn_t)(void* param, int32_t num); typedef struct STaskIdInfo { @@ -255,7 +255,8 @@ typedef struct SExecTaskInfo { uint64_t totalRows; // total number of rows STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure char* sql; // query sql string - jmp_buf env; // + jmp_buf env; // jump to this position when error happens. + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] struct SOperatorInfo* pRoot; } SExecTaskInfo; @@ -421,6 +422,7 @@ typedef struct SStreamBlockScanInfo { uint64_t numOfRows; // total scanned rows uint64_t numOfExec; // execution times void* readerHandle; // stream block reader handle + SArray* pColMatchInfo; // } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -466,11 +468,6 @@ typedef struct SAggSupporter { int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row } SAggSupporter; -typedef enum { - OPTR_EXEC_MODEL_BATCH = 0x1, - OPTR_EXEC_MODEL_STREAM = 0x2, -} OPTR_EXEC_MODEL; - typedef struct STableIntervalOperatorInfo { SOptrBasicInfo binfo; // basic info SGroupResInfo groupResInfo; // multiple results build supporter @@ -481,9 +478,9 @@ typedef struct STableIntervalOperatorInfo { SAggSupporter aggSup; // aggregate supporter STableQueryInfo *pCurrent; // current tableQueryInfo struct int32_t order; // current SSDataBlock scan order - OPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] SArray *pUpdatedWindow; // updated time window due to the input data block from the downstream operator. - SColumnInfoData timeWindowData; // query time window info for scalar function execution. + SColumnInfoData timeWindowData; // query time window info for scalar function execution. } STableIntervalOperatorInfo; typedef struct SAggOperatorInfo { @@ -718,7 +715,7 @@ int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx* pCtx, int32_t idx, int32_t type); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId); +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, EOPTR_EXEC_MODEL model); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2c6468a13f..93fbfe43ce 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -222,6 +222,16 @@ void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList) { + if (pGroupResInfo->pRows != NULL) { + taosArrayDestroy(pGroupResInfo->pRows); + } + + pGroupResInfo->pRows = pArrayList; + pGroupResInfo->index = 0; + ASSERT(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); +} + bool hasRemainDataInCurrentGroup(SGroupResInfo* pGroupResInfo) { if (pGroupResInfo->pRows == NULL) { return false; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 615bcdcdbb..15203d91ca 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -113,7 +113,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL); + code = qCreateExecTask(streamReadHandle, 0, 0, plan, &pTaskInfo, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { // TODO: destroy SSubplan & pTaskInfo terrno = code; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index cc9921ce73..6d78fa835a 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -51,11 +51,12 @@ static void freeqinfoFn(void *qhandle) { qDestroyTask(*handle); } -int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo, DataSinkHandle* handle) { +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, EOPTR_EXEC_MODEL model) { assert(readHandle != NULL && pSubplan != NULL); SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId); + int32_t code = createExecTaskInfoImpl(pSubplan, pTask, readHandle, taskId, model); if (code != TSDB_CODE_SUCCESS) { goto _error; } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 55def79958..6265fe6123 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -352,7 +352,7 @@ SSDataBlock* createOutputBuf_rv1(SDataBlockDescNode* pNode) { continue; } - idata.info.type = pDescNode->dataType.type; + idata.info.type = pDescNode->dataType.type; idata.info.bytes = pDescNode->dataType.bytes; idata.info.scale = pDescNode->dataType.scale; idata.info.slotId = pDescNode->slotId; @@ -1551,8 +1551,8 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, 0); tsCols = (int64_t*)pColDataInfo->pData; - assert(tsCols[0] == pSDataBlock->info.window.skey && - tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); +// assert(tsCols[0] == pSDataBlock->info.window.skey && +// tsCols[pSDataBlock->info.rows - 1] == pSDataBlock->info.window.ekey); } int32_t startPos = ascScan? 0 : (pSDataBlock->info.rows - 1); @@ -3660,6 +3660,7 @@ void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbased if (pCtx[j].fpSet.process) { // TODO set the dummy function. pCtx[j].fpSet.finalize(&pCtx[j]); + pResInfo->initialized = true; } if (pRow->numOfRows < pResInfo->numOfRes) { @@ -3778,10 +3779,6 @@ void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf* pBuf, SResultRow* pResult, S continue; } - // int32_t functionId = pCtx[i].functionId; - // if (functionId < 0) { - // continue; - // } // if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { // if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput; // } @@ -4971,7 +4968,20 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup) return NULL; } - pInfo->pRes->pDataBlock = tqRetrieveDataBlock(pInfo->readerHandle); + SArray* pCols = tqRetrieveDataBlock(pInfo->readerHandle); + + int32_t numOfCols = pInfo->pRes->info.numOfCols; + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pCols, i); + SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); + if (!pColMatchInfo->output) { + continue; + } + + ASSERT(pColMatchInfo->colId == p->info.colId); + taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, p); + } + if (pInfo->pRes->pDataBlock == NULL) { // TODO add log pTaskInfo->code = terrno; @@ -5625,8 +5635,18 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock* return NULL; } + int32_t numOfOutput = taosArrayGetSize(pColList); + + SArray* pColIds = taosArrayInit(4, sizeof(int16_t)); + for(int32_t i = 0; i < numOfOutput; ++i) { + int16_t* id = taosArrayGet(pColList, i); + taosArrayPush(pColIds, id); + } + + pInfo->pColMatchInfo = pColList; + // set the extract column id to streamHandle - tqReadHandleSetColIdList((STqReadHandle*)streamReadHandle, pColList); + tqReadHandleSetColIdList((STqReadHandle*)streamReadHandle, pColIds); int32_t code = tqReadHandleSetTbUidList(streamReadHandle, pTableIdList); if (code != 0) { taosMemoryFreeClear(pInfo); @@ -5665,9 +5685,9 @@ static int32_t loadSysTableContentCb(void* param, const SDataBuf* pMsg, int32_t SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->handle = htobe64(pRsp->handle); - pRsp->compLen = htonl(pRsp->compLen); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->handle = htobe64(pRsp->handle); + pRsp->compLen = htonl(pRsp->compLen); } else { operator->pTaskInfo->code = code; } @@ -5824,6 +5844,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { // pInfo->totalBytes; return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; } else { // load the meta from mnode of the given epset + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + int64_t startTs = taosGetTimestampUs(); pInfo->req.type = pInfo->type; @@ -5863,6 +5887,10 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; pInfo->req.showId = pRsp->handle; + if (pRsp->numOfRows == 0 || pRsp->completed) { + pOperator->status = OP_EXEC_DONE; + } + if (pRsp->numOfRows == 0) { // qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" // try next", @@ -6930,6 +6958,10 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro return NULL; } + if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { + return pOperator->getStreamResFn(pOperator, newgroup); + } + pTaskInfo->code = pOperator->_openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { return NULL; @@ -6946,7 +6978,7 @@ static SSDataBlock* doBuildIntervalResult(SOperatorInfo* pOperator, bool* newgro return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; } -static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator) { +static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator, bool* newgroup) { STableIntervalOperatorInfo* pInfo = pOperator->info; int32_t order = TSDB_ORDER_ASC; @@ -6960,18 +6992,18 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator) { if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } - return pInfo->binfo.pRes; + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; } // STimeWindow win = {0}; - bool newgroup = false; + *newgroup = false; SOperatorInfo* downstream = pOperator->pDownstream[0]; SArray* pUpdated = NULL; while (1) { publishOperatorProfEvent(downstream, QUERY_PROF_BEFORE_OPERATOR_EXEC); - SSDataBlock* pBlock = downstream->getNextFn(downstream, &newgroup); + SSDataBlock* pBlock = downstream->getNextFn(downstream, newgroup); publishOperatorProfEvent(downstream, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pBlock == NULL) { @@ -6989,6 +7021,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo *pOperator) { finalizeUpdatedResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, pUpdated, pInfo->binfo.rowCellInfoOffset); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->binfo.capacity); toSDatablock(&pInfo->groupResInfo, pInfo->aggSup.pResultBuf, pInfo->binfo.pRes, pInfo->binfo.capacity, pInfo->binfo.rowCellInfoOffset); @@ -7882,9 +7915,10 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* } pInfo->order = TSDB_ORDER_ASC; - pInfo->win = pTaskInfo->window; pInfo->interval = *pInterval; - pInfo->execModel = OPTR_EXEC_MODEL_BATCH; + pInfo->execModel = pTaskInfo->execModel; + + pInfo->win = pTaskInfo->window; pInfo->win.skey = 0; pInfo->win.ekey = INT64_MAX; @@ -7909,6 +7943,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->info = pInfo; pOperator->_openFn = doOpenIntervalAgg; pOperator->getNextFn = doBuildIntervalResult; + pOperator->getStreamResFn= doStreamIntervalAgg; pOperator->closeFn = destroyIntervalOperatorInfo; code = appendDownstream(pOperator, &downstream, 1); @@ -8736,12 +8771,13 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* return pExprs; } -static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { +static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPTR_EXEC_MODEL model) { SExecTaskInfo* pTaskInfo = taosMemoryCalloc(1, sizeof(SExecTaskInfo)); setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->cost.created = taosGetTimestampMs(); pTaskInfo->id.queryId = queryId; + pTaskInfo->execModel = model; char* p = taosMemoryCalloc(1, 128); snprintf(p, 128, "TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, queryId); @@ -8785,10 +8821,10 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createOutputBuf_rv1(pScanPhyNode->node.pOutputDataBlockDesc); - SArray* colList = extractScanColumnId(pScanPhyNode->pScanCols); - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle->reader, pResBlock, colList, tableIdList, pTaskInfo); + int32_t numOfCols = 0; + SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pColList, tableIdList, pTaskInfo); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == nodeType(pPhyNode)) { @@ -9102,11 +9138,11 @@ _error: return NULL; } -int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId) { +int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, EOPTR_EXEC_MODEL model) { uint64_t queryId = pPlan->id.queryId; int32_t code = TSDB_CODE_SUCCESS; - *pTaskInfo = createExecTaskInfo(queryId, taskId); + *pTaskInfo = createExecTaskInfo(queryId, taskId, model); if (*pTaskInfo == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index c9b0b62013..870a0b71fc 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -944,7 +944,7 @@ TEST(testCase, build_executor_tree_Test) { int32_t code = qStringToSubplan(msg, &plan); ASSERT_EQ(code, 0); - code = qCreateExecTask(&handle, 2, 1, plan, (void**) &pTaskInfo, &sinkHandle); + code = qCreateExecTask(&handle, 2, 1, plan, (void**) &pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); ASSERT_EQ(code, 0); } diff --git a/source/libs/index/inc/indexFstDfa.h b/source/libs/index/inc/indexFstDfa.h new file mode 100644 index 0000000000..f6c220bcb7 --- /dev/null +++ b/source/libs/index/inc/indexFstDfa.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef __INDEX_FST_DFA_H__ +#define __INDEX_FST_DFA_H__ + +#include "indexFstRegex.h" +#include "indexFstSparse.h" +#include "tarray.h" +#include "thash.h" + +#ifdef __cplusplus + +extern "C" { +#endif + +typedef struct FstDfa FstDfa; + +typedef struct { + SArray * insts; + uint32_t next[256]; + bool isMatch; +} State; + +/* + * dfa builder related func + **/ +typedef struct FstDfaBuilder { + FstDfa * dfa; + SHashObj *cache; +} FstDfaBuilder; + +FstDfaBuilder *dfaBuilderCreate(SArray *insts); + +void dfaBuilderDestroy(FstDfaBuilder *builder); + +FstDfa *dfaBuilderBuild(FstDfaBuilder *builder); + +bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t bytes, + uint32_t *result); + +bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result); + +/* + * dfa related func + **/ +typedef struct FstDfa { + SArray *insts; + SArray *states; +} FstDfa; + +FstDfa *dfaCreate(SArray *insts, SArray *states); +bool dfaIsMatch(FstDfa *dfa, uint32_t si); +bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result); +void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip); +bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/index/inc/indexFstRegex.h b/source/libs/index/inc/indexFstRegex.h new file mode 100644 index 0000000000..50b9cae7ff --- /dev/null +++ b/source/libs/index/inc/indexFstRegex.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_INDEX_FST_REGEX_H_ +#define _TD_INDEX_FST_REGEX_H_ + +//#include "indexFstDfa.h" +#include "taos.h" +#include "tarray.h" +#include "tchecksum.h" +#include "thash.h" +#include "tlog.h" +#include "tutil.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef enum { MATCH, JUMP, SPLIT, RANGE } InstType; + +typedef struct MatchValue { +} MatchValue; +typedef struct JumpValue { + uint32_t step; +} JumpValue; + +typedef struct SplitValue { + uint32_t len1; + uint32_t len2; +} SplitValue; + +typedef struct RangeValue { + uint8_t start; + uint8_t end; +} RangeValue; + +typedef struct { + InstType ty; + union { + MatchValue mv; + JumpValue jv; + SplitValue sv; + RangeValue rv; + }; +} Inst; + +typedef struct { + char *orig; + void *dfa; +} FstRegex; + +FstRegex *regexCreate(const char *str); + +void regexSetup(FstRegex *regex, uint32_t size, const char *str); + +// uint32_t regexStart() + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/source/libs/index/inc/indexSparse.h b/source/libs/index/inc/indexFstSparse.h similarity index 94% rename from source/libs/index/inc/indexSparse.h rename to source/libs/index/inc/indexFstSparse.h index 8035f6e08d..69b33c82d9 100644 --- a/source/libs/index/inc/indexSparse.h +++ b/source/libs/index/inc/indexFstSparse.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_INDEX_SPARSE_H_ -#define _TD_INDEX_SPARSE_H_ +#ifndef _TD_INDEX_FST_SPARSE_H_ +#define _TD_INDEX_FST_SPARSE_H_ #include "tarray.h" diff --git a/source/libs/index/src/indexFstDfa.c b/source/libs/index/src/indexFstDfa.c new file mode 100644 index 0000000000..ff6b154c54 --- /dev/null +++ b/source/libs/index/src/indexFstDfa.c @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "indexFstDfa.h" +#include "thash.h" + +const static uint32_t STATE_LIMIT = 1000; + +static int dfaInstsEqual(const void *a, const void *b, size_t size) { + SArray *ar = (SArray *)a; + SArray *br = (SArray *)b; + size_t al = ar != NULL ? taosArrayGetSize(ar) : 0; + size_t bl = br != NULL ? taosArrayGetSize(br) : 0; + if (al != bl) { + return -1; + } + for (int i = 0; i < al; i++) { + uint32_t v1 = *(uint32_t *)taosArrayGet(ar, i); + uint32_t v2 = *(uint32_t *)taosArrayGet(br, i); + if (v1 != v2) { + return -1; + } + } + return 0; +} +FstDfaBuilder *dfaBuilderCreate(SArray *insts) { + FstDfaBuilder *builder = taosMemoryCalloc(1, sizeof(FstDfaBuilder)); + if (builder == NULL) { + return NULL; + } + + SArray *states = taosArrayInit(4, sizeof(State)); + + builder->dfa = dfaCreate(insts, states); + builder->cache = taosHashInit( + 4, taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), + false, HASH_NO_LOCK); + taosHashSetEqualFp(builder->cache, dfaInstsEqual); + return builder; +} +void dfaBuilderDestroy(FstDfaBuilder *builder) { + if (builder == NULL) { + return; + } + void *pIter = builder->cache != NULL ? taosHashIterate(builder->cache, NULL) : NULL; + while (pIter) { + SArray **key = pIter; + taosArrayDestroy(*key); + pIter = taosHashIterate(builder->cache, pIter); + } + taosHashCleanup(builder->cache); +} + +FstDfa *dfaBuilderBuild(FstDfaBuilder *builder) { + uint32_t sz = taosArrayGetSize(builder->dfa->insts); + FstSparseSet *cur = sparSetCreate(sz); + FstSparseSet *nxt = sparSetCreate(sz); + + dfaAdd(builder->dfa, cur, 0); + + SArray * states = taosArrayInit(0, sizeof(uint32_t)); + uint32_t result; + if (dfaBuilderCachedState(builder, cur, &result)) { + taosArrayPush(states, &result); + } + SHashObj *seen = taosHashInit(12, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); + while (taosArrayGetSize(states) != 0) { + result = *(uint32_t *)taosArrayPop(states); + for (int i = 0; i < 256; i++) { + uint32_t ns, dummpy = 0; + if (dfaBuilderRunState(builder, cur, nxt, result, i, &ns)) { + if (taosHashGet(seen, &ns, sizeof(ns)) == NULL) { + taosHashPut(seen, &ns, sizeof(ns), &dummpy, sizeof(dummpy)); + taosArrayPush(states, &ns); + } + } + if (taosArrayGetSize(builder->dfa->states) > STATE_LIMIT) { + // Too many state; + // + } + } + } + taosArrayDestroy(states); + taosHashCleanup(seen); + return builder->dfa; +} + +bool dfaBuilderRunState(FstDfaBuilder *builder, FstSparseSet *cur, FstSparseSet *next, uint32_t state, uint8_t byte, + uint32_t *result) { + sparSetClear(cur); + State *t = taosArrayGet(builder->dfa->states, state); + for (int i = 0; i < taosArrayGetSize(t->insts); i++) { + uint32_t ip = *(int32_t *)taosArrayGet(t->insts, i); + sparSetAdd(cur, ip); + } + dfaRun(builder->dfa, cur, next, byte); + + t = taosArrayGet(builder->dfa->states, state); + + uint32_t nxtState; + if (dfaBuilderCachedState(builder, next, &nxtState)) { + t->next[byte] = nxtState; + *result = nxtState; + return true; + } + return false; +} + +bool dfaBuilderCachedState(FstDfaBuilder *builder, FstSparseSet *set, uint32_t *result) { + SArray *tinsts = taosArrayInit(4, sizeof(uint32_t)); + bool isMatch = false; + + for (int i = 0; i < sparSetLen(set); i++) { + uint32_t ip = sparSetGet(set, i); + + Inst *inst = taosArrayGet(builder->dfa->insts, ip); + if (inst->ty == JUMP || inst->ty == SPLIT) { + continue; + } else if (inst->ty == RANGE) { + taosArrayPush(tinsts, &ip); + } else if (inst->ty == MATCH) { + isMatch = true; + taosArrayPush(tinsts, &ip); + } + } + if (taosArrayGetSize(tinsts) == 0) { + return false; + } + uint32_t *v = taosHashGet(builder->cache, &tinsts, sizeof(POINTER_BYTES)); + if (v != NULL) { + *result = *v; + taosArrayDestroy(tinsts); + } else { + State st; + st.insts = tinsts; + st.isMatch = isMatch; + taosArrayPush(builder->dfa->states, &st); + int32_t sz = taosArrayGetSize(builder->dfa->states) - 1; + taosHashPut(builder->cache, &tinsts, sizeof(POINTER_BYTES), &sz, sizeof(sz)); + *result = sz; + } + return true; +} + +FstDfa *dfaCreate(SArray *insts, SArray *states) { + FstDfa *dfa = taosMemoryCalloc(1, sizeof(FstDfa)); + if (dfa == NULL) { + return NULL; + } + + dfa->insts = insts; + dfa->states = states; + return dfa; +} +bool dfaIsMatch(FstDfa *dfa, uint32_t si) { + if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) { + return false; + } + State *st = taosArrayGet(dfa->states, si); + return st != NULL ? st->isMatch : false; +} +bool dfaAccept(FstDfa *dfa, uint32_t si, uint8_t byte, uint32_t *result) { + if (dfa->states == NULL || si < taosArrayGetSize(dfa->states)) { + return false; + } + State *st = taosArrayGet(dfa->states, si); + *result = st->next[byte]; + return true; +} +void dfaAdd(FstDfa *dfa, FstSparseSet *set, uint32_t ip) { + if (sparSetContains(set, ip)) { + return; + } + sparSetAdd(set, ip); + Inst *inst = taosArrayGet(dfa->insts, ip); + if (inst->ty == MATCH || inst->ty == RANGE) { + // do nothing + } else if (inst->ty == JUMP) { + dfaAdd(dfa, set, inst->jv.step); + } else if (inst->ty == SPLIT) { + dfaAdd(dfa, set, inst->sv.len1); + dfaAdd(dfa, set, inst->sv.len2); + } + + return; +} +bool dfaRun(FstDfa *dfa, FstSparseSet *from, FstSparseSet *to, uint8_t byte) { + bool isMatch = false; + sparSetClear(to); + for (int i = 0; i < sparSetLen(from); i++) { + uint32_t ip = sparSetGet(from, i); + + Inst *inst = taosArrayGet(dfa->insts, ip); + if (inst->ty == JUMP || inst->ty == SPLIT) { + continue; + } else if (inst->ty == MATCH) { + isMatch = true; + } else if (inst->ty == RANGE) { + if (inst->rv.start <= byte && byte <= inst->rv.end) { + dfaAdd(dfa, to, ip + 1); + } + } + } + + return isMatch; +} diff --git a/source/libs/index/src/indexFstRegex.c b/source/libs/index/src/indexFstRegex.c new file mode 100644 index 0000000000..ec41a7f58e --- /dev/null +++ b/source/libs/index/src/indexFstRegex.c @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "indexFstRegex.h" +#include "indexFstSparse.h" + +FstRegex *regexCreate(const char *str) { + FstRegex *regex = taosMemoryCalloc(1, sizeof(FstRegex)); + if (regex == NULL) { + return NULL; + } + int32_t sz = (int32_t)strlen(str); + char * orig = taosMemoryCalloc(1, sz); + memcpy(orig, str, sz); + + regex->orig = orig; +} + +void regexSetup(FstRegex *regex, uint32_t size, const char *str) { + // return + // return; +} diff --git a/source/libs/index/src/indexSparse.c b/source/libs/index/src/indexSparse.c index 8bcf04602f..9d228e71ff 100644 --- a/source/libs/index/src/indexSparse.c +++ b/source/libs/index/src/indexSparse.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "indexSparse.h" +#include "indexFstSparse.h" FstSparseSet *sparSetCreate(int32_t sz) { FstSparseSet *ss = taosMemoryCalloc(1, sizeof(FstSparseSet)); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 34159a1355..42d10a2361 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -959,7 +959,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, int8_t taskType) { QW_ERR_JRET(code); } - code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle); + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, OPTR_EXEC_MODEL_BATCH); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); diff --git a/source/libs/stream/src/tstream.c b/source/libs/stream/src/tstream.c index fb6c0f6c12..5223aa1e3a 100644 --- a/source/libs/stream/src/tstream.c +++ b/source/libs/stream/src/tstream.c @@ -152,6 +152,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in // sink if (pTask->sinkType == TASK_SINK__TABLE) { // + blockDebugShowData(pRes); } else if (pTask->sinkType == TASK_SINK__SMA) { pTask->smaSink.smaHandle(pTask->ahandle, pTask->smaSink.smaId, pRes); // diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index b133e1fcbc..fc27e88cf1 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -18,12 +18,6 @@ #define TDB_BTREE_ROOT 0x1 #define TDB_BTREE_LEAF 0x2 -#define TDB_BTREE_PAGE_IS_ROOT(flags) TDB_FLAG_HAS(flags, TDB_BTREE_ROOT) -#define TDB_BTREE_PAGE_IS_LEAF(flags) TDB_FLAG_HAS(flags, TDB_BTREE_LEAF) -#define TDB_BTREE_ASSERT_FLAG(flags) \ - ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ - TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0)) - struct SBTree { SPgno root; int keyLen; @@ -40,8 +34,13 @@ struct SBTree { #define TDB_BTREE_PAGE_COMMON_HDR u8 flags; -#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0] +#define TDB_BTREE_PAGE_GET_FLAGS(PAGE) (PAGE)->pData[0] #define TDB_BTREE_PAGE_SET_FLAGS(PAGE, flags) ((PAGE)->pData[0] = (flags)) +#define TDB_BTREE_PAGE_IS_ROOT(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_ROOT) +#define TDB_BTREE_PAGE_IS_LEAF(PAGE) (TDB_BTREE_PAGE_GET_FLAGS(PAGE) & TDB_BTREE_LEAF) +#define TDB_BTREE_ASSERT_FLAG(flags) \ + ASSERT(TDB_FLAG_IS(flags, TDB_BTREE_ROOT) || TDB_FLAG_IS(flags, TDB_BTREE_LEAF) || \ + TDB_FLAG_IS(flags, TDB_BTREE_ROOT | TDB_BTREE_LEAF) || TDB_FLAG_IS(flags, 0)) typedef struct __attribute__((__packed__)) { TDB_BTREE_PAGE_COMMON_HDR @@ -58,15 +57,15 @@ typedef struct { } SBtreeInitPageArg; typedef struct { - int kLen; - u8 *pKey; - int vLen; - u8 *pVal; - SPgno pgno; - u8 *pTmpSpace; + int kLen; + const u8 *pKey; + int vLen; + const u8 *pVal; + SPgno pgno; + u8 *pBuf; } SCellDecoder; -static int tdbBtCursorMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); +static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst); static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); static int tdbBtreeZeroPage(SPage *pPage, void *arg); @@ -77,7 +76,7 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD static int tdbBtreeBalance(SBTC *pBtc); static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell); static int tdbBtcMoveToNext(SBTC *pBtc); -static int tdbBtcMoveDownward(SBTC *pBtc, SPgno pgno); +static int tdbBtcMoveDownward(SBTC *pBtc); static int tdbBtcMoveUpward(SBTC *pBtc); int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, SBTree **ppBt) { @@ -94,9 +93,9 @@ int tdbBtreeOpen(int keyLen, int valLen, SPager *pPager, FKeyComparator kcmpr, S } // pBt->keyLen - pBt->keyLen = keyLen; + pBt->keyLen = keyLen < 0 ? TDB_VARIANT_LEN : keyLen; // pBt->valLen - pBt->valLen = valLen; + pBt->valLen = valLen < 0 ? TDB_VARIANT_LEN : valLen; // pBt->pPager pBt->pPager = pPager; // pBt->kcmpr @@ -137,7 +136,7 @@ int tdbBtCursorInsert(SBTC *pBtc, const void *pKey, int kLen, const void *pVal, int cret; SBTree *pBt; - ret = tdbBtCursorMoveTo(pBtc, pKey, kLen, &cret); + ret = tdbBtcMoveTo(pBtc, pKey, kLen, &cret); if (ret < 0) { // TODO: handle error return -1; @@ -200,7 +199,7 @@ int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen tdbBtcOpen(&btc, pBt); - tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); + tdbBtcMoveTo(&btc, pKey, kLen, &cret); if (cret) { return cret; @@ -230,7 +229,7 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL tdbBtcOpen(&btc, pBt); - tdbBtCursorMoveTo(&btc, pKey, kLen, &cret); + tdbBtcMoveTo(&btc, pKey, kLen, &cret); if (cret) { return cret; } @@ -257,106 +256,6 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL return 0; } -static int tdbBtCursorMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { - int ret; - SBTree *pBt; - SPager *pPager; - - pBt = pBtc->pBt; - pPager = pBt->pPager; - - if (pBtc->iPage < 0) { - ASSERT(pBtc->iPage == -1); - ASSERT(pBtc->idx == -1); - - // Move from the root - ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); - if (ret < 0) { - ASSERT(0); - return -1; - } - - pBtc->iPage = 0; - - if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) { - // Current page is empty - // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pBtc->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); - return 0; - } - - for (;;) { - int lidx, ridx, midx, c, nCells; - SCell *pCell; - SPage *pPage; - SCellDecoder cd = {0}; - - pPage = pBtc->pPage; - nCells = TDB_PAGE_TOTAL_CELLS(pPage); - lidx = 0; - ridx = nCells - 1; - - ASSERT(nCells > 0); - - for (;;) { - if (lidx > ridx) break; - - midx = (lidx + ridx) >> 1; - - pCell = tdbPageGetCell(pPage, midx); - ret = tdbBtreeDecodeCell(pPage, pCell, &cd); - if (ret < 0) { - // TODO: handle error - ASSERT(0); - return -1; - } - - // Compare the key values - c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); - if (c < 0) { - /* input-key < cell-key */ - ridx = midx - 1; - } else if (c > 0) { - /* input-key > cell-key */ - lidx = midx + 1; - } else { - /* input-key == cell-key */ - break; - } - } - - // Move downward or break - u8 flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); - u8 leaf = TDB_BTREE_PAGE_IS_LEAF(flags); - if (leaf) { - pBtc->idx = midx; - *pCRst = c; - break; - } else { - if (c <= 0) { - pBtc->idx = midx; - tdbBtcMoveDownward(pBtc, cd.pgno); - } else { - pBtc->idx = midx + 1; - if (midx == nCells - 1) { - /* Move to right-most child */ - tdbBtcMoveDownward(pBtc, ((SIntHdr *)pBtc->pPage->pData)->pgno); - } else { - pCell = tdbPageGetCell(pPage, pBtc->idx); - tdbBtreeDecodeCell(pPage, pCell, &cd); - tdbBtcMoveDownward(pBtc, cd.pgno); - } - } - } - } - - } else { - // TODO: Move the cursor from a some position instead of a clear state - ASSERT(0); - } - - return 0; -} - static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2) { int mlen; int cret; @@ -416,7 +315,7 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { pBt = (SBTree *)arg; flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); - isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + isLeaf = TDB_BTREE_PAGE_IS_LEAF(pPage); ASSERT(flags == TDB_BTREE_PAGE_GET_FLAGS(pPage)); @@ -442,15 +341,15 @@ static int tdbBtreeInitPage(SPage *pPage, void *arg) { static int tdbBtreeZeroPage(SPage *pPage, void *arg) { u8 flags; SBTree *pBt; - u8 isLeaf; + u8 leaf; flags = ((SBtreeInitPageArg *)arg)->flags; pBt = ((SBtreeInitPageArg *)arg)->pBt; - isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + leaf = flags & TDB_BTREE_LEAF; - tdbPageZero(pPage, isLeaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); + tdbPageZero(pPage, leaf ? sizeof(SLeafHdr) : sizeof(SIntHdr), tdbBtreeCellSize); - if (isLeaf) { + if (leaf) { SLeafHdr *pLeafHdr = (SLeafHdr *)(pPage->pData); pLeafHdr->flags = flags; @@ -495,7 +394,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild) { pPager = pRoot->pPager; flags = TDB_BTREE_PAGE_GET_FLAGS(pRoot); - leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + leaf = TDB_BTREE_PAGE_IS_LEAF(pRoot); // Allocate a new child page zArg.flags = TDB_FLAG_REMOVE(flags, TDB_BTREE_ROOT); @@ -561,7 +460,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { SPgno pgno; if (sIdx + i == nCells) { - ASSERT(!TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pParent))); + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pParent)); pgno = ((SIntHdr *)(pParent->pData))->pgno; } else { pCell = tdbPageGetCell(pParent, sIdx + i); @@ -575,7 +474,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx) { } } // copy the parent key out if child pages are not leaf page - childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pOlds[0])); + childNotLeaf = !TDB_BTREE_PAGE_IS_LEAF(pOlds[0]); if (childNotLeaf) { for (int i = 0; i < nOlds; i++) { if (sIdx + i < TDB_PAGE_TOTAL_CELLS(pParent)) { @@ -849,9 +748,8 @@ static int tdbBtreeBalance(SBTC *pBtc) { for (;;) { iPage = pBtc->iPage; pPage = pBtc->pPage; - flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); - leaf = TDB_BTREE_PAGE_IS_LEAF(flags); - root = TDB_BTREE_PAGE_IS_ROOT(flags); + leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); + root = TDB_BTREE_PAGE_IS_ROOT(pPage); // when the page is not overflow and not too empty, the balance work // is finished. Just break out the balance loop. @@ -892,23 +790,17 @@ static int tdbBtreeBalance(SBTC *pBtc) { } #endif -#ifndef TDB_BTREE_CELL // ========================================================= -static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, int kLen, const void *pVal, int vLen, - int *szPayload) { +// TDB_BTREE_CELL ===================== +static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const void *pKey, int kLen, const void *pVal, + int vLen, int *szPayload) { int nPayload; - ASSERT(pKey != NULL); - - if (pVal == NULL) { - vLen = 0; - } - nPayload = kLen + vLen; - if (nPayload <= pPage->maxLocal) { - // General case without overflow - memcpy(pPayload, pKey, kLen); + if (nPayload + nHeader <= pPage->maxLocal) { + // no overflow page is needed + memcpy(pCell + nHeader, pKey, kLen); if (pVal) { - memcpy(pPayload + kLen, pVal, vLen); + memcpy(pCell + nHeader + kLen, pVal, vLen); } *szPayload = nPayload; @@ -923,10 +815,8 @@ static int tdbBtreeEncodePayload(SPage *pPage, u8 *pPayload, const void *pKey, i return 0; } -// TODO: allow vLen = 0 static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell) { - u8 flags; u8 leaf; int nHeader; int nPayload; @@ -934,11 +824,11 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo ASSERT(pPage->kLen == TDB_VARIANT_LEN || pPage->kLen == kLen); ASSERT(pPage->vLen == TDB_VARIANT_LEN || pPage->vLen == vLen); + ASSERT(pKey != NULL && kLen > 0); nPayload = 0; nHeader = 0; - flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); - leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); // 1. Encode Header part /* Encode SPgno if interior page */ @@ -960,38 +850,42 @@ static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const vo } // 2. Encode payload part - if (leaf && vLen > 0) { - ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, pVal, vLen, &nPayload); - } else { - ret = tdbBtreeEncodePayload(pPage, pCell + nHeader, pKey, kLen, NULL, 0, &nPayload); + if ((!leaf) || pPage->vLen == 0) { + pVal = NULL; + vLen = 0; } + + ret = tdbBtreeEncodePayload(pPage, pCell, nHeader, pKey, kLen, pVal, vLen, &nPayload); if (ret < 0) { - // TODO: handle error - return -1; + // TODO + ASSERT(0); + return 0; } *szCell = nHeader + nPayload; return 0; } -static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder *pDecoder) { +static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, SCellDecoder *pDecoder) { int nPayload; - ASSERT(pDecoder->pKey == NULL); - if (pDecoder->pVal) { - nPayload = pDecoder->kLen + pDecoder->vLen; - } else { + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pPage)); nPayload = pDecoder->kLen; + } else { + nPayload = pDecoder->kLen + pDecoder->vLen; } - if (nPayload <= pPage->maxLocal) { - // General case without overflow - pDecoder->pKey = (void *)pPayload; - if (!pDecoder->pVal) { - pDecoder->pVal = (void *)(pPayload + pDecoder->kLen); + if (nHeader + nPayload <= pPage->maxLocal) { + // no over flow case + pDecoder->pKey = pCell + nHeader; + if (pDecoder->pVal == NULL && pDecoder->vLen > 0) { + pDecoder->pVal = pCell + nHeader + pDecoder->kLen; } - } else { + return 0; + } + + { // TODO: handle overflow case ASSERT(0); } @@ -999,16 +893,13 @@ static int tdbBtreeDecodePayload(SPage *pPage, const u8 *pPayload, SCellDecoder return 0; } -// TODO: here has problem static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder) { - u8 flags; u8 leaf; int nHeader; int ret; nHeader = 0; - flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); - leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); // Clear the state of decoder pDecoder->kLen = -1; @@ -1033,13 +924,14 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD } if (pPage->vLen == TDB_VARIANT_LEN) { + ASSERT(leaf); nHeader += tdbGetVarInt(pCell + nHeader, &(pDecoder->vLen)); } else { pDecoder->vLen = pPage->vLen; } // 2. Decode payload part - ret = tdbBtreeDecodePayload(pPage, pCell + nHeader, pDecoder); + ret = tdbBtreeDecodePayload(pPage, pCell, nHeader, pDecoder); if (ret < 0) { return -1; } @@ -1048,16 +940,14 @@ static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pD } static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { - u8 flags; - u8 isLeaf; + u8 leaf; int szCell; int kLen = 0, vLen = 0; - flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); - isLeaf = TDB_BTREE_PAGE_IS_LEAF(flags); + leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); szCell = 0; - if (!isLeaf) { + if (!leaf) { szCell += sizeof(SPgno); } @@ -1067,21 +957,28 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell) { kLen = pPage->kLen; } - if (isLeaf) { - if (pPage->vLen == TDB_VARIANT_LEN) { - szCell += tdbGetVarInt(pCell + szCell, &vLen); - } else { - vLen = pPage->vLen; - } + if (pPage->vLen == TDB_VARIANT_LEN) { + ASSERT(leaf); + szCell += tdbGetVarInt(pCell + szCell, &vLen); + } else if (leaf) { + vLen = pPage->vLen; } szCell = szCell + kLen + vLen; - return szCell; + if (szCell <= pPage->maxLocal) { + return szCell; + } + + { + // TODO + ASSERT(0); + return 0; + } } +// TDB_BTREE_CELL -#endif - +// TDB_BTREE_CURSOR ===================== int tdbBtcOpen(SBTC *pBtc, SBTree *pBt) { pBtc->pBt = pBt; pBtc->iPage = -1; @@ -1095,7 +992,6 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { int ret; SBTree *pBt; SPager *pPager; - u8 flags; SCell *pCell; SPgno pgno; @@ -1110,23 +1006,43 @@ int tdbBtcMoveToFirst(SBTC *pBtc) { return -1; } + ASSERT(TDB_BTREE_PAGE_IS_ROOT(pBtc->pPage)); + pBtc->iPage = 0; - pBtc->idx = 0; + if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0) { + pBtc->idx = 0; + } else { + // no any data, point to an invalid position + ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + pBtc->idx = -1; + return 0; + } } else { // move from a position - ASSERT(0); + int iPage = 0; + + for (; iPage < pBtc->iPage; iPage++) { + ASSERT(pBtc->idxStack[iPage] >= 0); + if (pBtc->idxStack[iPage]) break; + } + + // move upward + for (;;) { + if (pBtc->iPage == 0) { + pBtc->idx = 0; + break; + } + + if (pBtc->iPage < iPage) break; + tdbBtcMoveUpward(pBtc); + } } // move downward for (;;) { - flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) break; - if (TDB_BTREE_PAGE_IS_LEAF(flags)) break; - - pCell = tdbPageGetCell(pBtc->pPage, 0); - pgno = *(SPgno *)pCell; - - ret = tdbBtcMoveDownward(pBtc, pgno); + ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { ASSERT(0); return -1; @@ -1142,7 +1058,6 @@ int tdbBtcMoveToLast(SBTC *pBtc) { int ret; SBTree *pBt; SPager *pPager; - u8 flags; SPgno pgno; pBt = pBtc->pBt; @@ -1164,18 +1079,15 @@ int tdbBtcMoveToLast(SBTC *pBtc) { // move downward for (;;) { - flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); - - if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { // TODO: handle empty case ASSERT(TDB_PAGE_TOTAL_CELLS(pBtc->pPage) > 0); pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage) - 1; break; } else { pBtc->idx = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); - pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; - ret = tdbBtcMoveDownward(pBtc, pgno); + ret = tdbBtcMoveDownward(pBtc); if (ret < 0) { ASSERT(0); return -1; @@ -1186,11 +1098,6 @@ int tdbBtcMoveToLast(SBTC *pBtc) { return 0; } -int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen) { - // TODO - return 0; -} - int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { SCell *pCell; SCellDecoder cd; @@ -1233,11 +1140,9 @@ int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen) { static int tdbBtcMoveToNext(SBTC *pBtc) { int nCells; - SPgno pgno; SCell *pCell; - u8 flags; - ASSERT(TDB_BTREE_PAGE_IS_LEAF(TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage))); + ASSERT(TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); if (pBtc->idx < 0) return -1; @@ -1270,18 +1175,11 @@ static int tdbBtcMoveToNext(SBTC *pBtc) { // Move downward for (;;) { nCells = TDB_PAGE_TOTAL_CELLS(pBtc->pPage); - if (pBtc->idx < nCells) { - pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); - pgno = *(SPgno *)pCell; - } else { - pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; - } - tdbBtcMoveDownward(pBtc, pgno); + tdbBtcMoveDownward(pBtc); pBtc->idx = 0; - flags = TDB_BTREE_PAGE_GET_FLAGS(pBtc->pPage); - if (TDB_BTREE_PAGE_IS_LEAF(flags)) { + if (TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)) { break; } } @@ -1289,13 +1187,20 @@ static int tdbBtcMoveToNext(SBTC *pBtc) { return 0; } -int tdbBtcClose(SBTC *pBtc) { - // TODO - return 0; -} +static int tdbBtcMoveDownward(SBTC *pBtc) { + int ret; + SPgno pgno; + SCell *pCell; -static int tdbBtcMoveDownward(SBTC *pBtc, SPgno pgno) { - int ret; + ASSERT(pBtc->idx >= 0); + ASSERT(!TDB_BTREE_PAGE_IS_LEAF(pBtc->pPage)); + + if (pBtc->idx < TDB_PAGE_TOTAL_CELLS(pBtc->pPage)) { + pCell = tdbPageGetCell(pBtc->pPage, pBtc->idx); + pgno = ((SPgno *)pCell)[0]; + } else { + pgno = ((SIntHdr *)pBtc->pPage->pData)->pgno; + } pBtc->pgStack[pBtc->iPage] = pBtc->pPage; pBtc->idxStack[pBtc->iPage] = pBtc->idx; @@ -1306,6 +1211,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc, SPgno pgno) { ret = tdbPagerFetchPage(pBtc->pBt->pPager, pgno, &pBtc->pPage, tdbBtreeInitPage, pBtc->pBt); if (ret < 0) { ASSERT(0); + return -1; } return 0; @@ -1314,7 +1220,7 @@ static int tdbBtcMoveDownward(SBTC *pBtc, SPgno pgno) { static int tdbBtcMoveUpward(SBTC *pBtc) { if (pBtc->iPage == 0) return -1; - // tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); + tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); pBtc->iPage--; pBtc->pPage = pBtc->pgStack[pBtc->iPage]; @@ -1323,6 +1229,117 @@ static int tdbBtcMoveUpward(SBTC *pBtc) { return 0; } +static int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { + int ret; + SBTree *pBt; + SPager *pPager; + + pBt = pBtc->pBt; + pPager = pBt->pPager; + + if (pBtc->iPage < 0) { + ASSERT(pBtc->iPage == -1); + ASSERT(pBtc->idx == -1); + + // Move from the root + ret = tdbPagerFetchPage(pPager, pBt->root, &(pBtc->pPage), tdbBtreeInitPage, pBt); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pBtc->iPage = 0; + + if (TDB_PAGE_TOTAL_CELLS(pBtc->pPage) == 0) { + // Current page is empty + // ASSERT(TDB_FLAG_IS(TDB_PAGE_FLAGS(pBtc->pPage), TDB_BTREE_ROOT | TDB_BTREE_LEAF)); + return 0; + } + + for (;;) { + int lidx, ridx, midx, c, nCells; + SCell *pCell; + SPage *pPage; + SCellDecoder cd = {0}; + + pPage = pBtc->pPage; + nCells = TDB_PAGE_TOTAL_CELLS(pPage); + lidx = 0; + ridx = nCells - 1; + + ASSERT(nCells > 0); + + for (;;) { + if (lidx > ridx) break; + + midx = (lidx + ridx) >> 1; + + pCell = tdbPageGetCell(pPage, midx); + ret = tdbBtreeDecodeCell(pPage, pCell, &cd); + if (ret < 0) { + // TODO: handle error + ASSERT(0); + return -1; + } + + // Compare the key values + c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); + if (c < 0) { + /* input-key < cell-key */ + ridx = midx - 1; + } else if (c > 0) { + /* input-key > cell-key */ + lidx = midx + 1; + } else { + /* input-key == cell-key */ + break; + } + } + + // Move downward or break + u8 leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); + if (leaf) { + pBtc->idx = midx; + *pCRst = c; + break; + } else { + if (c <= 0) { + pBtc->idx = midx; + } else { + pBtc->idx = midx + 1; + } + tdbBtcMoveDownward(pBtc); + } + } + + } else { + // TODO: Move the cursor from a some position instead of a clear state + ASSERT(0); + } + + return 0; +} + +int tdbBtcClose(SBTC *pBtc) { + if (pBtc->iPage < 0) return 0; + + for (;;) { + ASSERT(pBtc->pPage); + + tdbPagerReturnPage(pBtc->pBt->pPager, pBtc->pPage); + + pBtc->iPage--; + if (pBtc->iPage < 0) break; + + pBtc->pPage = pBtc->pgStack[pBtc->iPage]; + pBtc->idx = pBtc->idxStack[pBtc->iPage]; + } + + return 0; +} +// TDB_BTREE_CURSOR + +// TDB_BTREE_DEBUG ===================== #ifndef NODEBUG typedef struct { SPgno pgno; @@ -1336,17 +1353,14 @@ typedef struct { SBtPageInfo btPageInfos[20]; void tdbBtPageInfo(SPage *pPage, int idx) { - u8 flags; SBtPageInfo *pBtPageInfo; pBtPageInfo = btPageInfos + idx; pBtPageInfo->pgno = TDB_PAGE_PGNO(pPage); - flags = TDB_BTREE_PAGE_GET_FLAGS(pPage); - - pBtPageInfo->root = TDB_BTREE_PAGE_IS_ROOT(flags); - pBtPageInfo->leaf = TDB_BTREE_PAGE_IS_LEAF(flags); + pBtPageInfo->root = TDB_BTREE_PAGE_IS_ROOT(pPage); + pBtPageInfo->leaf = TDB_BTREE_PAGE_IS_LEAF(pPage); pBtPageInfo->rChild = 0; if (!pBtPageInfo->leaf) { @@ -1356,4 +1370,5 @@ void tdbBtPageInfo(SPage *pPage, int idx) { pBtPageInfo->nCells = TDB_PAGE_TOTAL_CELLS(pPage) - pPage->nOverflow; pBtPageInfo->nOvfl = pPage->nOverflow; } -#endif \ No newline at end of file +#endif +// TDB_BTREE_DEBUG \ No newline at end of file diff --git a/source/libs/tdb/src/inc/tdbBtree.h b/source/libs/tdb/src/inc/tdbBtree.h index 0e8ca0c803..2797b5ea5e 100644 --- a/source/libs/tdb/src/inc/tdbBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -35,7 +35,6 @@ struct SBTC { int idx; int idxStack[BTREE_MAX_DEPTH + 1]; SPage *pgStack[BTREE_MAX_DEPTH + 1]; - void *pBuf; }; // SBTree @@ -49,7 +48,6 @@ int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkL int tdbBtcOpen(SBTC *pCur, SBTree *pBt); int tdbBtcMoveToFirst(SBTC *pBtc); int tdbBtcMoveToLast(SBTC *pBtc); -int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen); int tdbBtreeNext(SBTC *pBtc, void **ppKey, int *kLen, void **ppVal, int *vLen); int tdbBtcClose(SBTC *pBtc); diff --git a/source/libs/tdb/src/inc/tdbOs.h b/source/libs/tdb/src/inc/tdbOs.h index 794d4c502a..196bbdafc7 100644 --- a/source/libs/tdb/src/inc/tdbOs.h +++ b/source/libs/tdb/src/inc/tdbOs.h @@ -64,7 +64,7 @@ typedef TdThreadSpinlock tdb_spinlock_t; #define tdbSpinlockDestroy taosThreadSpinDestroy #define tdbSpinlockLock taosThreadSpinLock #define tdbSpinlockUnlock taosThreadSpinUnlock -#define tdbSpinlockTrylock pthread_spin_trylock +#define tdbSpinlockTrylock taosThreadSpinTrylock /* mutex lock */ typedef TdThreadMutex tdb_mutex_t; diff --git a/source/os/src/osMemory.c b/source/os/src/osMemory.c index 81ee259ff1..f4d4c74d1d 100644 --- a/source/os/src/osMemory.c +++ b/source/os/src/osMemory.c @@ -17,6 +17,8 @@ #include #include "os.h" +#ifdef USE_TD_MEMORY + #define TD_MEMORY_SYMBOL ('T'<<24|'A'<<16|'O'<<8|'S') #define TD_MEMORY_STACK_TRACE_DEPTH 10 @@ -70,6 +72,8 @@ int32_t taosBackTrace(void **buffer, int32_t size) { // return backtrace_symbols(buffer, *size); // } +#endif + void *taosMemoryMalloc(int32_t size) { #ifdef USE_TD_MEMORY void *tmp = malloc(size + sizeof(TdMemoryInfo)); @@ -126,9 +130,9 @@ void *taosMemoryRealloc(void *ptr, int32_t size) { } void taosMemoryFree(const void *ptr) { -#ifdef USE_TD_MEMORY if (ptr == NULL) return; +#ifdef USE_TD_MEMORY TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); if(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL) { pTdMemoryInfo->memorySize = 0; @@ -143,9 +147,9 @@ void taosMemoryFree(const void *ptr) { } int32_t taosMemorySize(void *ptr) { -#ifdef USE_TD_MEMORY if (ptr == NULL) return 0; - + +#ifdef USE_TD_MEMORY TdMemoryInfoPtr pTdMemoryInfo = (TdMemoryInfoPtr)((char*)ptr - sizeof(TdMemoryInfo)); assert(pTdMemoryInfo->symbol == TD_MEMORY_SYMBOL); diff --git a/source/os/src/osProc.c b/source/os/src/osProc.c new file mode 100644 index 0000000000..00ed650cbb --- /dev/null +++ b/source/os/src/osProc.c @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#define ALLOW_FORBID_FUNC +#define _DEFAULT_SOURCE +#include "os.h" + +static char *tsProcPath = NULL; + +int32_t taosNewProc(char **args) { + int32_t pid = fork(); + if (pid == 0) { + args[0] = tsProcPath; + return execvp(tsProcPath, args); + } else { + return pid; + } +} + +// the length of the new name must be less than the original name to take effect +void taosSetProcName(int32_t argc, char **argv, const char *name) { + prctl(PR_SET_NAME, name); + + for (int32_t i = 0; i < argc; ++i) { + int32_t len = strlen(argv[i]); + for (int32_t j = 0; j < len; ++j) { + argv[i][j] = 0; + } + if (i == 0) { + tstrncpy(argv[0], name, len + 1); + } + } +} + +void taosSetProcPath(int32_t argc, char **argv) { tsProcPath = argv[0]; } + +bool taosProcExists(int32_t pid) { + int32_t p = getpgid(pid); + return p >= 0; +} diff --git a/source/os/src/osShm.c b/source/os/src/osShm.c index e7a22c3da1..ba184c1f5d 100644 --- a/source/os/src/osShm.c +++ b/source/os/src/osShm.c @@ -18,7 +18,9 @@ #include "os.h" int32_t taosCreateShm(SShm* pShm, int32_t shmsize) { - int32_t shmid = shmget(IPC_PRIVATE, shmsize, IPC_CREAT | 0600); + pShm->id = -1; + + int32_t shmid = shmget(0X95279527, shmsize, IPC_CREAT | 0600); if (shmid < 0) { return -1; } @@ -35,37 +37,23 @@ int32_t taosCreateShm(SShm* pShm, int32_t shmsize) { } void taosDropShm(SShm* pShm) { - if (pShm->id > 0) { + if (pShm->id >= 0) { if (pShm->ptr != NULL) { shmdt(pShm->ptr); } shmctl(pShm->id, IPC_RMID, NULL); } - pShm->id = 0; + pShm->id = -1; pShm->size = 0; pShm->ptr = NULL; } int32_t taosAttachShm(SShm* pShm) { - if (pShm->id > 0 && pShm->size > 0) { - pShm->ptr = shmat(pShm->id, NULL, 0); - if (pShm->ptr != NULL) { - return 0; - } + errno = 0; + + void* ptr = shmat(pShm->id, NULL, 0); + if (errno == 0) { + pShm->ptr = ptr; } - - return -1; -} - -void taosDetachShm(SShm* pShm) { - if (pShm->id > 0) { - if (pShm->ptr != NULL) { - shmdt(pShm->ptr); - pShm->ptr = NULL; - } - } - - pShm->id = 0; - pShm->size = 0; - pShm->ptr = NULL; + return errno; } diff --git a/source/os/src/osSignal.c b/source/os/src/osSignal.c index 12721a17f5..d4e6cb3318 100644 --- a/source/os/src/osSignal.c +++ b/source/os/src/osSignal.c @@ -71,4 +71,6 @@ void taosIgnSignal(int32_t signum) { signal(signum, SIG_IGN); } void taosDflSignal(int32_t signum) { signal(signum, SIG_DFL); } +void taosKillChildOnParentStopped() { prctl(PR_SET_PDEATHSIG, SIGKILL); } + #endif diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index b130689d4b..d79d7c3d58 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -758,7 +758,7 @@ void taosBlockSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = taosThreadSigmask(SIG_BLOCK, &signal_mask, NULL); + int32_t rc = taosThreadSigMask(SIG_BLOCK, &signal_mask, NULL); if (rc != 0) { // printf("failed to block SIGPIPE"); } @@ -876,7 +876,7 @@ void taosSetMaskSIGPIPE() { sigset_t signal_mask; sigemptyset(&signal_mask); sigaddset(&signal_mask, SIGPIPE); - int32_t rc = taosThreadSigmask(SIG_SETMASK, &signal_mask, NULL); + int32_t rc = taosThreadSigMask(SIG_SETMASK, &signal_mask, NULL); if (rc != 0) { // printf("failed to setmask SIGPIPE"); } diff --git a/source/os/src/osThread.c b/source/os/src/osThread.c index 716080274b..436614ad6a 100644 --- a/source/os/src/osThread.c +++ b/source/os/src/osThread.c @@ -14,132 +14,341 @@ */ #define ALLOW_FORBID_FUNC +#include #include "os.h" -// int32_t taosThreadSetnameNp(TdThread thread, const char *name) { -// return pthread_setname_np(thread,name); -// } - -int32_t taosThreadSpinInit(TdThreadSpinlock *lock, int pshared) { - return pthread_spin_init(lock, pshared); +int32_t taosThreadCreate(TdThread * tid, const TdThreadAttr * attr, void *(*start)(void *), void *arg) { + return pthread_create(tid, attr, start, arg); } -int32_t taosThreadMutexInit(TdThreadMutex *mutex, const TdThreadMutexAttr *attr) { - return pthread_mutex_init(mutex, attr); +int32_t taosThreadAttrDestroy(TdThreadAttr * attr) { + return pthread_attr_destroy(attr); } -int32_t taosThreadSpinDestroy(TdThreadSpinlock *lock) { - return pthread_spin_destroy(lock); +int32_t taosThreadAttrGetDetachState(const TdThreadAttr * attr, int32_t *detachstate) { + return pthread_attr_getdetachstate(attr, detachstate); } -int32_t taosThreadMutexDestroy(TdThreadMutex * mutex) { - return pthread_mutex_destroy(mutex); +int32_t taosThreadAttrGetInheritSched(const TdThreadAttr * attr, int32_t *inheritsched) { + return pthread_attr_getinheritsched(attr, inheritsched); } -int32_t taosThreadSpinLock(TdThreadSpinlock *lock) { - return pthread_spin_lock(lock); +int32_t taosThreadAttrGetSchedParam(const TdThreadAttr * attr, struct sched_param *param) { + return pthread_attr_getschedparam(attr, param); } -int32_t taosThreadMutexLock(TdThreadMutex *mutex) { - return pthread_mutex_lock(mutex); +int32_t taosThreadAttrGetSchedPolicy(const TdThreadAttr * attr, int32_t *policy) { + return pthread_attr_getschedpolicy(attr, policy); } -int32_t taosThreadSpinUnlock(TdThreadSpinlock *lock) { - return pthread_spin_unlock(lock); +int32_t taosThreadAttrGetScope(const TdThreadAttr * attr, int32_t *contentionscope) { + return pthread_attr_getscope(attr, contentionscope); } -int32_t taosThreadMutexUnlock(TdThreadMutex *mutex) { - return pthread_mutex_unlock(mutex); +int32_t taosThreadAttrGetStackSize(const TdThreadAttr * attr, size_t * stacksize) { + return pthread_attr_getstacksize(attr, stacksize); } -int32_t taosThreadRwlockRdlock(TdThreadRwlock *rwlock) { - return pthread_rwlock_rdlock(rwlock); +int32_t taosThreadAttrInit(TdThreadAttr * attr) { + return pthread_attr_init(attr); } -int32_t taosThreadRwlockWrlock(TdThreadRwlock *rwlock) { - return pthread_rwlock_wrlock(rwlock); +int32_t taosThreadAttrSetDetachState(TdThreadAttr * attr, int32_t detachstate) { + return pthread_attr_setdetachstate(attr, detachstate); } -int32_t taosThreadRwlockUnlock(TdThreadRwlock *rwlock) { - return pthread_rwlock_unlock(rwlock); +int32_t taosThreadAttrSetInheritSched(TdThreadAttr * attr, int32_t inheritsched) { + return pthread_attr_setinheritsched(attr, inheritsched); } -void taosThreadTestCancel(void) { - return pthread_testcancel(); +int32_t taosThreadAttrSetSchedParam(TdThreadAttr * attr, const struct sched_param *param) { + return pthread_attr_setschedparam(attr, param); } -int32_t taosThreadAttrInit(TdThreadAttr *attr) { - return pthread_attr_init(attr); +int32_t taosThreadAttrSetSchedPolicy(TdThreadAttr * attr, int32_t policy) { + return pthread_attr_setschedpolicy(attr, policy); } -int32_t taosThreadCreate(TdThread *tid, const TdThreadAttr *attr, void*(*start)(void*), void *arg) { - return pthread_create(tid, attr, start, arg); +int32_t taosThreadAttrSetScope(TdThreadAttr * attr, int32_t contentionscope) { + return pthread_attr_setscope(attr, contentionscope); } -int32_t taosThreadOnce(TdThreadOnce *onceControl, void(*initRoutine)(void)) { - return pthread_once(onceControl, initRoutine); +int32_t taosThreadAttrSetStackSize(TdThreadAttr * attr, size_t stacksize) { + return pthread_attr_setstacksize(attr, stacksize); } -int32_t taosThreadAttrSetDetachState(TdThreadAttr *attr, int32_t detachState) { - return pthread_attr_setdetachstate(attr, detachState); +int32_t taosThreadBarrierDestroy(TdThreadBarrier * barrier) { + return pthread_barrier_destroy(barrier); } -int32_t taosThreadAttrDestroy(TdThreadAttr *attr) { - return pthread_attr_destroy(attr); +int32_t taosThreadBarrierInit(TdThreadBarrier * barrier, const TdThreadBarrierAttr * attr, uint32_t count) { + return pthread_barrier_init(barrier, attr, count); } -int32_t taosThreadJoin(TdThread thread, void **pValue) { - return pthread_join(thread, pValue); +int32_t taosThreadBarrierWait(TdThreadBarrier * barrier) { + return pthread_barrier_wait(barrier); } -int32_t taosThreadRwlockInit(TdThreadRwlock *rwlock, const TdThreadRwlockAttr *attr) { - return pthread_rwlock_init(rwlock, attr); +int32_t taosThreadBarrierAttrDestroy(TdThreadBarrierAttr * attr) { + return pthread_barrierattr_destroy(attr); } -int32_t taosThreadRwlockDestroy(TdThreadRwlock *rwlock) { - return pthread_rwlock_destroy(rwlock); +int32_t taosThreadBarrierAttrGetPshared(const TdThreadBarrierAttr * attr, int32_t *pshared) { + return pthread_barrierattr_getpshared(attr, pshared); } -int32_t taosThreadCondSignal(TdThreadCond *cond) { - return pthread_cond_signal(cond); +int32_t taosThreadBarrierAttrInit(TdThreadBarrierAttr * attr) { + return pthread_barrierattr_init(attr); } -int32_t taosThreadCondInit(TdThreadCond *cond, const TdThreadCondAttr *attr) { - return pthread_cond_init(cond, attr); -} - -int32_t taosThreadCondBroadcast(TdThreadCond *cond) { - return pthread_cond_broadcast(cond); -} - -int32_t taosThreadCondDestroy(TdThreadCond *cond) { - return pthread_cond_destroy(cond); -} - -int32_t taosThreadCondWait(TdThreadCond *cond, TdThreadMutex *mutex) { - return pthread_cond_wait(cond, mutex); -} - -TdThread taosThreadSelf(void) { - return pthread_self(); -} - -// int32_t taosThreadGetW32ThreadIdNp(TdThread thread) { -// return pthread_getw32threadid_np(thread); -// } - -int32_t taosThreadEqual(TdThread t1, TdThread t2) { - return pthread_equal(t1, t2); -} - -int32_t taosThreadSigmask(int how, sigset_t const *set, sigset_t *oset) { - return pthread_sigmask(how, set, oset); +int32_t taosThreadBarrierAttrSetPshared(TdThreadBarrierAttr * attr, int32_t pshared) { + return pthread_barrierattr_setpshared(attr, pshared); } int32_t taosThreadCancel(TdThread thread) { - return pthread_cancel(thread); + return pthread_cancel(thread); } -int32_t taosThreadKill(TdThread thread, int sig) { - return pthread_kill(thread, sig); +int32_t taosThreadCondDestroy(TdThreadCond * cond) { + return pthread_cond_destroy(cond); +} + +int32_t taosThreadCondInit(TdThreadCond * cond, const TdThreadCondAttr * attr) { + return pthread_cond_init(cond, attr); +} + +int32_t taosThreadCondSignal(TdThreadCond * cond) { + return pthread_cond_signal(cond); +} + +int32_t taosThreadCondBroadcast(TdThreadCond * cond) { + return pthread_cond_broadcast(cond); +} + +int32_t taosThreadCondWait(TdThreadCond * cond, TdThreadMutex * mutex) { + return pthread_cond_wait(cond, mutex); +} + +int32_t taosThreadCondTimedWait(TdThreadCond * cond, TdThreadMutex * mutex, const struct timespec *abstime) { + return pthread_cond_timedwait(cond, mutex, abstime); +} + +int32_t taosThreadCondAttrDestroy(TdThreadCondAttr * attr) { + return pthread_condattr_destroy(attr); +} + +int32_t taosThreadCondAttrGetPshared(const TdThreadCondAttr * attr, int32_t *pshared) { + return pthread_condattr_getpshared(attr, pshared); +} + +int32_t taosThreadCondAttrInit(TdThreadCondAttr * attr) { + return pthread_condattr_init(attr); +} + +int32_t taosThreadCondAttrSetPshared(TdThreadCondAttr * attr, int32_t pshared) { + return pthread_condattr_setpshared(attr, pshared); +} + +int32_t taosThreadDetach(TdThread thread) { + return pthread_detach(thread); +} + +int32_t taosThreadEqual(TdThread t1, TdThread t2) { + return pthread_equal(t1, t2); +} + +void taosThreadExit(void *valuePtr) { + return pthread_exit(valuePtr); +} + +int32_t taosThreadGetSchedParam(TdThread thread, int32_t *policy, struct sched_param *param) { + return pthread_getschedparam(thread, policy, param); +} + +void *taosThreadGetSpecific(TdThreadKey key) { + return pthread_getspecific(key); +} + +int32_t taosThreadJoin(TdThread thread, void **valuePtr) { + return pthread_join(thread, valuePtr); +} + +int32_t taosThreadKeyCreate(TdThreadKey * key, void(*destructor)(void *)) { + return pthread_key_create(key, destructor); +} + +int32_t taosThreadKeyDelete(TdThreadKey key) { + return pthread_key_delete(key); +} + +int32_t taosThreadKill(TdThread thread, int32_t sig) { + return pthread_kill(thread, sig); +} + +int32_t taosThreadMutexConsistent(TdThreadMutex* mutex) { + return pthread_mutex_consistent(mutex); +} + +int32_t taosThreadMutexDestroy(TdThreadMutex * mutex) { + return pthread_mutex_destroy(mutex); +} + +int32_t taosThreadMutexInit(TdThreadMutex * mutex, const TdThreadMutexAttr * attr) { + return pthread_mutex_init(mutex, attr); +} + +int32_t taosThreadMutexLock(TdThreadMutex * mutex) { + return pthread_mutex_lock(mutex); +} + +int32_t taosThreadMutexTimedLock(TdThreadMutex * mutex, const struct timespec *abstime) { + return pthread_mutex_timedlock(mutex, abstime); +} + +int32_t taosThreadMutexTryLock(TdThreadMutex * mutex) { + return pthread_mutex_trylock(mutex); +} + +int32_t taosThreadMutexUnlock(TdThreadMutex * mutex) { + return pthread_mutex_unlock(mutex); +} + +int32_t taosThreadMutexAttrDestroy(TdThreadMutexAttr * attr) { + return pthread_mutexattr_destroy(attr); +} + +int32_t taosThreadMutexAttrGetPshared(const TdThreadMutexAttr * attr, int32_t *pshared) { + return pthread_mutexattr_getpshared(attr, pshared); +} + +int32_t taosThreadMutexAttrGetRobust(const TdThreadMutexAttr * attr, int32_t * robust) { + return pthread_mutexattr_getrobust(attr, robust); +} + +int32_t taosThreadMutexAttrGetType(const TdThreadMutexAttr * attr, int32_t *kind) { + return pthread_mutexattr_gettype(attr, kind); +} + +int32_t taosThreadMutexAttrInit(TdThreadMutexAttr * attr) { + return pthread_mutexattr_init(attr); +} + +int32_t taosThreadMutexAttrSetPshared(TdThreadMutexAttr * attr, int32_t pshared) { + return pthread_mutexattr_setpshared(attr, pshared); +} + +int32_t taosThreadMutexAttrSetRobust(TdThreadMutexAttr * attr, int32_t robust) { + return pthread_mutexattr_setrobust(attr, robust); +} + +int32_t taosThreadMutexAttrSetType(TdThreadMutexAttr * attr, int32_t kind) { + return pthread_mutexattr_settype(attr, kind); +} + +int32_t taosThreadOnce(TdThreadOnce * onceControl, void(*initRoutine)(void)) { + return pthread_once(onceControl, initRoutine); +} + +int32_t taosThreadRwlockDestroy(TdThreadRwlock * rwlock) { + return pthread_rwlock_destroy(rwlock); +} + +int32_t taosThreadRwlockInit(TdThreadRwlock * rwlock, const TdThreadRwlockAttr * attr) { + return pthread_rwlock_init(rwlock, attr); +} + +int32_t taosThreadRwlockRdlock(TdThreadRwlock * rwlock) { + return pthread_rwlock_rdlock(rwlock); +} + +int32_t taosThreadRwlockTimedRdlock(TdThreadRwlock * rwlock, const struct timespec *abstime) { + return pthread_rwlock_timedrdlock(rwlock, abstime); +} + +int32_t taosThreadRwlockTimedWrlock(TdThreadRwlock * rwlock, const struct timespec *abstime) { + return pthread_rwlock_timedwrlock(rwlock, abstime); +} + +int32_t taosThreadRwlockTryRdlock(TdThreadRwlock * rwlock) { + return pthread_rwlock_tryrdlock(rwlock); +} + +int32_t taosThreadRwlockTryWrlock(TdThreadRwlock * rwlock) { + return pthread_rwlock_trywrlock(rwlock); +} + +int32_t taosThreadRwlockUnlock(TdThreadRwlock * rwlock) { + return pthread_rwlock_unlock(rwlock); +} + +int32_t taosThreadRwlockWrlock(TdThreadRwlock * rwlock) { + return pthread_rwlock_wrlock(rwlock); +} + +int32_t taosThreadRwlockAttrDestroy(TdThreadRwlockAttr * attr) { + return pthread_rwlockattr_destroy(attr); +} + +int32_t taosThreadRwlockAttrGetPshared(const TdThreadRwlockAttr * attr, int32_t *pshared) { + return pthread_rwlockattr_getpshared(attr, pshared); +} + +int32_t taosThreadRwlockAttrInit(TdThreadRwlockAttr * attr) { + return pthread_rwlockattr_init(attr); +} + +int32_t taosThreadRwlockAttrSetPshared(TdThreadRwlockAttr * attr, int32_t pshared) { + return pthread_rwlockattr_setpshared(attr, pshared); +} + +TdThread taosThreadSelf(void) { + return pthread_self(); +} + +int32_t taosThreadSetCancelState(int32_t state, int32_t *oldstate) { + return pthread_setcancelstate(state, oldstate); +} + +int32_t taosThreadSetCancelType(int32_t type, int32_t *oldtype) { + return pthread_setcanceltype(type, oldtype); +} + +int32_t taosThreadSetSchedParam(TdThread thread, int32_t policy, const struct sched_param *param) { + return pthread_setschedparam(thread, policy, param); +} + +int32_t taosThreadSetSpecific(TdThreadKey key, const void *value) { + return pthread_setspecific(key, value); +} + +int32_t taosThreadSpinDestroy(TdThreadSpinlock * lock) { + return pthread_spin_destroy(lock); +} + +int32_t taosThreadSpinInit(TdThreadSpinlock * lock, int32_t pshared) { + return pthread_spin_init(lock, pshared); +} + +int32_t taosThreadSpinLock(TdThreadSpinlock * lock) { + return pthread_spin_lock(lock); +} + +int32_t taosThreadSpinTrylock(TdThreadSpinlock * lock) { + return pthread_spin_trylock(lock); +} + +int32_t taosThreadSpinUnlock(TdThreadSpinlock * lock) { + return pthread_spin_unlock(lock); +} + +void taosThreadTestCancel(void) { + return pthread_testcancel(); +} + +int32_t taosThreadSigMask(int32_t how, sigset_t const *set, sigset_t * oset) { + return pthread_sigmask(how, set, oset); +} + +int32_t taosThreadSigWait(const sigset_t * set, int32_t *sig) { + return sigwait(set, sig); } \ No newline at end of file diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 96bb638b95..04061cbaf1 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -590,12 +590,12 @@ void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { } int32_t cfgLoadFromEnvVar(SConfig *pConfig) { - uInfo("load from global env variables"); + uInfo("load from global env variables success"); return 0; } int32_t cfgLoadFromEnvFile(SConfig *pConfig, const char *filepath) { - uInfo("load from env file %s", filepath); + uInfo("load from env file [%s] success", filepath); return 0; } @@ -649,11 +649,11 @@ int32_t cfgLoadFromCfgFile(SConfig *pConfig, const char *filepath) { taosCloseFile(&pFile); if (line != NULL) taosMemoryFreeClear(line); - uInfo("load from cfg file %s success", filepath); + uInfo("load from cfg file [%s] success", filepath); return 0; } int32_t cfgLoadFromApollUrl(SConfig *pConfig, const char *url) { - uInfo("load from apoll url %s", url); + uInfo("load from apoll url [%s] success", url); return 0; } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 005995c8e2..90fa624a8d 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -85,7 +85,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization TAOS_DEFINE_ERROR(TSDB_CODE_CFG_NOT_FOUND, "Config not found") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG, "Invalid config option") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_SHM_MEM, "Out of Share memory") - +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_SHM_ID, "Invalid SHM ID") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") diff --git a/source/util/src/tprocess.c b/source/util/src/tprocess.c index 86cec2d271..f18e7d5777 100644 --- a/source/util/src/tprocess.c +++ b/source/util/src/tprocess.c @@ -23,34 +23,36 @@ typedef void *(*ProcThreadFp)(void *param); typedef struct SProcQueue { - int32_t head; - int32_t tail; - int32_t total; - int32_t avail; - int32_t items; - char *pBuffer; - ProcMallocFp mallocHeadFp; - ProcFreeFp freeHeadFp; - ProcMallocFp mallocBodyFp; - ProcFreeFp freeBodyFp; - ProcConsumeFp consumeFp; - void *pParent; - tsem_t sem; - TdThreadMutex *mutex; - int32_t mutexShmid; - int32_t bufferShmid; - const char *name; + int32_t head; + int32_t tail; + int32_t total; + int32_t avail; + int32_t items; + char name[8]; + TdThreadMutex mutex; + tsem_t sem; + char pBuffer[]; } SProcQueue; typedef struct SProcObj { - TdThread childThread; - SProcQueue *pChildQueue; - TdThread parentThread; - SProcQueue *pParentQueue; - const char *name; - int32_t pid; - bool isChild; - bool stopFlag; + TdThread thread; + SProcQueue *pChildQueue; + SProcQueue *pParentQueue; + ProcConsumeFp childConsumeFp; + ProcMallocFp childMallocHeadFp; + ProcFreeFp childFreeHeadFp; + ProcMallocFp childMallocBodyFp; + ProcFreeFp childFreeBodyFp; + ProcConsumeFp parentConsumeFp; + ProcMallocFp parentMallocHeadFp; + ProcFreeFp parentFreeHeadFp; + ProcMallocFp parentMallocBodyFp; + ProcFreeFp parentFreeBodyFp; + void *pParent; + const char *name; + int32_t pid; + bool isChild; + bool stopFlag; } SProcObj; static inline int32_t CEIL8(int32_t v) { @@ -58,150 +60,94 @@ static inline int32_t CEIL8(int32_t v) { return c < 8 ? 8 : c; } -static int32_t taosProcInitMutex(TdThreadMutex **ppMutex, int32_t *pShmid) { - TdThreadMutex *pMutex = NULL; +static int32_t taosProcInitMutex(SProcQueue *pQueue) { TdThreadMutexAttr mattr = {0}; - int32_t shmid = -1; - int32_t code = -1; - if (pthread_mutexattr_init(&mattr) != 0) { + if (taosThreadMutexAttrInit(&mattr) != 0) { terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to init mutex while init attr since %s", terrstr()); - goto _OVER; + return -1; } - if (pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED) != 0) { + if (taosThreadMutexAttrSetPshared(&mattr, PTHREAD_PROCESS_SHARED) != 0) { + taosThreadMutexAttrDestroy(&mattr); terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to init mutex while set shared since %s", terrstr()); - goto _OVER; + return -1; } - shmid = shmget(IPC_PRIVATE, sizeof(TdThreadMutex), IPC_CREAT | 0600); - if (shmid <= 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to init mutex while shmget since %s", terrstr()); - goto _OVER; - } - - pMutex = (TdThreadMutex *)shmat(shmid, NULL, 0); - if (pMutex == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to init mutex while shmat since %s", terrstr()); - goto _OVER; - } - - if (taosThreadMutexInit(pMutex, &mattr) != 0) { + if (taosThreadMutexInit(&pQueue->mutex, &mattr) != 0) { + taosThreadMutexDestroy(&pQueue->mutex); terrno = TAOS_SYSTEM_ERROR(errno); uError("failed to init mutex since %s", terrstr()); - goto _OVER; - } - - code = 0; - -_OVER: - if (code != 0) { - if (pMutex != NULL) { - taosThreadMutexDestroy(pMutex); - shmdt(pMutex); - } - if (shmid >= 0) { - shmctl(shmid, IPC_RMID, NULL); - } - } else { - *ppMutex = pMutex; - *pShmid = shmid; - } - - pthread_mutexattr_destroy(&mattr); - return code; -} - -static void taosProcDestroyMutex(TdThreadMutex *pMutex, int32_t shmid) { - if (pMutex != NULL) { - taosThreadMutexDestroy(pMutex); - } - if (shmid >= 0) { - shmctl(shmid, IPC_RMID, NULL); - } -} - -static int32_t taosProcInitBuffer(void **ppBuffer, int32_t size) { - int32_t shmid = shmget(IPC_PRIVATE, size, IPC_CREAT | 0600); - if (shmid <= 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to init buffer while shmget since %s", terrstr()); return -1; } - void *shmptr = shmat(shmid, NULL, 0); - if (shmptr == NULL) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to init buffer while shmat since %s", terrstr()); - shmctl(shmid, IPC_RMID, NULL); - return -1; - } - - *ppBuffer = shmptr; - return shmid; + taosThreadMutexAttrDestroy(&mattr); + return 0; } -static void taosProcDestroyBuffer(void *pBuffer, int32_t shmid) { - if (shmid > 0) { - shmdt(pBuffer); - shmctl(shmid, IPC_RMID, NULL); - } -} - -static SProcQueue *taosProcInitQueue(int32_t size) { - if (size <= 0) size = SHM_DEFAULT_SIZE; - - int32_t bufSize = CEIL8(size); - int32_t headSize = CEIL8(sizeof(SProcQueue)); - - SProcQueue *pQueue = NULL; - int32_t shmId = taosProcInitBuffer((void **)&pQueue, bufSize + headSize); - if (shmId < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - pQueue->bufferShmid = shmId; - - if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { - taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); - return NULL; - } - +static int32_t taosProcInitSem(SProcQueue *pQueue) { if (tsem_init(&pQueue->sem, 1, 0) != 0) { - taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); - taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to init sem"); + return -1; + } + + return 0; +} + +static SProcQueue *taosProcInitQueue(const char *name, bool isChild, char *ptr, int32_t size) { + int32_t bufSize = size - CEIL8(sizeof(SProcQueue)); + if (bufSize <= 1024) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - if (taosProcInitMutex(&pQueue->mutex, &pQueue->mutexShmid) != 0) { - tsem_destroy(&pQueue->sem); - taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); - taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); - return NULL; + SProcQueue *pQueue = (SProcQueue *)(ptr); + + if (!isChild) { + if (taosProcInitMutex(pQueue) != 0) { + return NULL; + } + + if (taosProcInitSem(pQueue) != 0) { + return NULL; + } + + tstrncpy(pQueue->name, name, sizeof(pQueue->name)); + pQueue->head = 0; + pQueue->tail = 0; + pQueue->total = bufSize; + pQueue->avail = bufSize; + pQueue->items = 0; } - pQueue->head = 0; - pQueue->tail = 0; - pQueue->total = bufSize; - pQueue->avail = bufSize; - pQueue->items = 0; - pQueue->pBuffer = (char *)pQueue + headSize; return pQueue; } +#if 0 +static void taosProcDestroyMutex(SProcQueue *pQueue) { + if (pQueue->mutex != NULL) { + taosThreadMutexDestroy(pQueue->mutex); + pQueue->mutex = NULL; + } +} + +static void taosProcDestroySem(SProcQueue *pQueue) { + if (pQueue->sem != NULL) { + tsem_destroy(pQueue->sem); + pQueue->sem = NULL; + } +} + static void taosProcCleanupQueue(SProcQueue *pQueue) { if (pQueue != NULL) { - uDebug("proc:%s, queue:%p clean up", pQueue->name, pQueue); - tsem_destroy(&pQueue->sem); - taosProcDestroyMutex(pQueue->mutex, pQueue->mutexShmid); - taosProcDestroyBuffer(pQueue, pQueue->bufferShmid); + taosProcDestroyMutex(pQueue); + taosProcDestroySem(pQueue); } } +#endif static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t rawHeadLen, const char *pBody, int32_t rawBodyLen, ProcFuncType ftype) { @@ -209,9 +155,9 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t const int32_t bodyLen = CEIL8(rawBodyLen); const int32_t fullLen = headLen + bodyLen + 8; - taosThreadMutexLock(pQueue->mutex); + taosThreadMutexLock(&pQueue->mutex); if (fullLen > pQueue->avail) { - taosThreadMutexUnlock(pQueue->mutex); + taosThreadMutexUnlock(&pQueue->mutex); terrno = TSDB_CODE_OUT_OF_SHM_MEM; return -1; } @@ -260,7 +206,7 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t pQueue->avail -= fullLen; pQueue->items++; - taosThreadMutexUnlock(pQueue->mutex); + taosThreadMutexUnlock(&pQueue->mutex); tsem_post(&pQueue->sem); uTrace("proc:%s, push msg at pos:%d ftype:%d remain:%d, head:%d %p body:%d %p", pQueue->name, pos, ftype, @@ -268,13 +214,14 @@ static int32_t taosProcQueuePush(SProcQueue *pQueue, const char *pHead, int16_t return 0; } -static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHeadLen, void **ppBody, - int32_t *pBodyLen, ProcFuncType *pFuncType) { +static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHeadLen, void **ppBody, int32_t *pBodyLen, + ProcFuncType *pFuncType, ProcMallocFp mallocHeadFp, ProcFreeFp freeHeadFp, + ProcMallocFp mallocBodyFp, ProcFreeFp freeBodyFp) { tsem_wait(&pQueue->sem); - taosThreadMutexLock(pQueue->mutex); + taosThreadMutexLock(&pQueue->mutex); if (pQueue->total - pQueue->avail <= 0) { - taosThreadMutexUnlock(pQueue->mutex); + taosThreadMutexUnlock(&pQueue->mutex); tsem_post(&pQueue->sem); terrno = TSDB_CODE_OUT_OF_SHM_MEM; return 0; @@ -293,13 +240,13 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea bodyLen = *(int32_t *)(pQueue->pBuffer + 4); } - void *pHead = (*pQueue->mallocHeadFp)(headLen); - void *pBody = (*pQueue->mallocBodyFp)(bodyLen); + void *pHead = (*mallocHeadFp)(headLen); + void *pBody = (*mallocBodyFp)(bodyLen); if (pHead == NULL || pBody == NULL) { - taosThreadMutexUnlock(pQueue->mutex); + taosThreadMutexUnlock(&pQueue->mutex); tsem_post(&pQueue->sem); - (*pQueue->freeHeadFp)(pHead); - (*pQueue->freeBodyFp)(pBody); + (*freeHeadFp)(pHead); + (*freeBodyFp)(pBody); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -338,7 +285,7 @@ static int32_t taosProcQueuePop(SProcQueue *pQueue, void **ppHead, int16_t *pHea pQueue->avail = pQueue->avail + headLen + bodyLen + 8; pQueue->items--; - taosThreadMutexUnlock(pQueue->mutex); + taosThreadMutexUnlock(&pQueue->mutex); *ppHead = pHead; *ppBody = pBody; @@ -358,65 +305,85 @@ SProcObj *taosProcInit(const SProcCfg *pCfg) { return NULL; } + int32_t cstart = 0; + int32_t csize = CEIL8(pCfg->shm.size / 2); + int32_t pstart = csize; + int32_t psize = CEIL8(pCfg->shm.size - pstart); + if (pstart + psize > pCfg->shm.size) { + psize -= 8; + } + pProc->name = pCfg->name; - pProc->pChildQueue = taosProcInitQueue(pCfg->childQueueSize); - pProc->pParentQueue = taosProcInitQueue(pCfg->parentQueueSize); + pProc->pChildQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + cstart, csize); + pProc->pParentQueue = taosProcInitQueue(pCfg->name, pCfg->isChild, (char *)pCfg->shm.ptr + pstart, psize); if (pProc->pChildQueue == NULL || pProc->pParentQueue == NULL) { - taosProcCleanupQueue(pProc->pChildQueue); + // taosProcCleanupQueue(pProc->pChildQueue); taosMemoryFree(pProc); return NULL; } - pProc->pChildQueue->name = pCfg->name; - pProc->pChildQueue->pParent = pCfg->pParent; - pProc->pChildQueue->mallocHeadFp = pCfg->childMallocHeadFp; - pProc->pChildQueue->freeHeadFp = pCfg->childFreeHeadFp; - pProc->pChildQueue->mallocBodyFp = pCfg->childMallocBodyFp; - pProc->pChildQueue->freeBodyFp = pCfg->childFreeBodyFp; - pProc->pChildQueue->consumeFp = pCfg->childConsumeFp; - pProc->pParentQueue->name = pCfg->name; - pProc->pParentQueue->pParent = pCfg->pParent; - pProc->pParentQueue->mallocHeadFp = pCfg->parentdMallocHeadFp; - pProc->pParentQueue->freeHeadFp = pCfg->parentFreeHeadFp; - pProc->pParentQueue->mallocBodyFp = pCfg->parentMallocBodyFp; - pProc->pParentQueue->freeBodyFp = pCfg->parentFreeBodyFp; - pProc->pParentQueue->consumeFp = pCfg->parentConsumeFp; + pProc->name = pCfg->name; + pProc->pParent = pCfg->pParent; + pProc->childMallocHeadFp = pCfg->childMallocHeadFp; + pProc->childFreeHeadFp = pCfg->childFreeHeadFp; + pProc->childMallocBodyFp = pCfg->childMallocBodyFp; + pProc->childFreeBodyFp = pCfg->childFreeBodyFp; + pProc->childConsumeFp = pCfg->childConsumeFp; + pProc->parentMallocHeadFp = pCfg->parentMallocHeadFp; + pProc->parentFreeHeadFp = pCfg->parentFreeHeadFp; + pProc->parentMallocBodyFp = pCfg->parentMallocBodyFp; + pProc->parentFreeBodyFp = pCfg->parentFreeBodyFp; + pProc->parentConsumeFp = pCfg->parentConsumeFp; + pProc->isChild = pCfg->isChild; - uDebug("proc:%s, is initialized, child queue:%p parent queue:%p", pProc->name, pProc->pChildQueue, pProc->pParentQueue); - - pProc->pid = fork(); - if (pProc->pid == 0) { - pProc->isChild = 1; - prctl(PR_SET_NAME, pProc->name, NULL, NULL, NULL); - } else { - pProc->isChild = 0; - uInfo("this is parent process, child pid:%d", pProc->pid); - } + uDebug("proc:%s, is initialized, isChild:%d child queue:%p parent queue:%p", pProc->name, pProc->isChild, + pProc->pChildQueue, pProc->pParentQueue); return pProc; } -static void taosProcThreadLoop(SProcQueue *pQueue) { - ProcConsumeFp consumeFp = pQueue->consumeFp; - void *pParent = pQueue->pParent; +static void taosProcThreadLoop(SProcObj *pProc) { void *pHead, *pBody; int16_t headLen; ProcFuncType ftype; int32_t bodyLen; + SProcQueue *pQueue; + ProcConsumeFp consumeFp; + ProcMallocFp mallocHeadFp; + ProcFreeFp freeHeadFp; + ProcMallocFp mallocBodyFp; + ProcFreeFp freeBodyFp; - uDebug("proc:%s, start to get msg from queue:%p", pQueue->name, pQueue); + if (pProc->isChild) { + pQueue = pProc->pChildQueue; + consumeFp = pProc->childConsumeFp; + mallocHeadFp = pProc->childMallocHeadFp; + freeHeadFp = pProc->childFreeHeadFp; + mallocBodyFp = pProc->childMallocBodyFp; + freeBodyFp = pProc->childFreeBodyFp; + } else { + pQueue = pProc->pParentQueue; + consumeFp = pProc->parentConsumeFp; + mallocHeadFp = pProc->parentMallocHeadFp; + freeHeadFp = pProc->parentFreeHeadFp; + mallocBodyFp = pProc->parentMallocBodyFp; + freeBodyFp = pProc->parentFreeBodyFp; + } + + uDebug("proc:%s, start to get msg from queue:%p, isChild:%d", pProc->name, pQueue, pProc->isChild); while (1) { - int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen, &ftype); + int32_t numOfMsgs = taosProcQueuePop(pQueue, &pHead, &headLen, &pBody, &bodyLen, &ftype, mallocHeadFp, freeHeadFp, + mallocBodyFp, freeBodyFp); if (numOfMsgs == 0) { - uDebug("proc:%s, get no msg from queue:%p and exit the proc thread", pQueue->name, pQueue); + uInfo("proc:%s, get no msg from queue:%p and exit the proc thread", pProc->name, pQueue); break; } else if (numOfMsgs < 0) { - uTrace("proc:%s, get no msg from queue:%p since %s", pQueue->name, pQueue, terrstr()); + uTrace("proc:%s, get no msg from queue:%p since %s", pProc->name, pQueue, terrstr()); taosMsleep(1); continue; } else { - (*consumeFp)(pParent, pHead, headLen, pBody, bodyLen, ftype); + (*consumeFp)(pProc->pParent, pHead, headLen, pBody, bodyLen, ftype); } } } @@ -426,40 +393,37 @@ int32_t taosProcRun(SProcObj *pProc) { taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); - if (pProc->isChild) { - if (taosThreadCreate(&pProc->childThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pChildQueue) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to create thread since %s", terrstr()); - return -1; - } - uDebug("proc:%s, child start to consume queue:%p", pProc->name, pProc->pChildQueue); - } else { - if (taosThreadCreate(&pProc->parentThread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc->pParentQueue) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - uError("failed to create thread since %s", terrstr()); - return -1; - } - uDebug("proc:%s, parent start to consume queue:%p", pProc->name, pProc->pParentQueue); + if (taosThreadCreate(&pProc->thread, &thAttr, (ProcThreadFp)taosProcThreadLoop, pProc) != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + uError("failed to create thread since %s", terrstr()); + return -1; } + uDebug("proc:%s, start to consume queue:%p, thread:%" PRId64, pProc->name, pProc->pChildQueue, pProc->thread); return 0; } -void taosProcStop(SProcObj *pProc) { - pProc->stopFlag = true; - // todo join +static void taosProcStop(SProcObj *pProc) { + if (!taosCheckPthreadValid(pProc->thread)) return; + + uDebug("proc:%s, start to join thread:%" PRId64 ", isChild:%d", pProc->name, pProc->thread, pProc->isChild); + SProcQueue *pQueue; + if (pProc->isChild) { + pQueue = pProc->pChildQueue; + } else { + pQueue = pProc->pParentQueue; + } + tsem_post(&pQueue->sem); + taosThreadJoin(pProc->thread, NULL); } -bool taosProcIsChild(SProcObj *pProc) { return pProc->isChild; } - -int32_t taosProcChildId(SProcObj *pProc) { return pProc->pid; } - void taosProcCleanup(SProcObj *pProc) { if (pProc != NULL) { - uDebug("proc:%s, clean up", pProc->name); + uDebug("proc:%s, start to clean up", pProc->name); taosProcStop(pProc); - taosProcCleanupQueue(pProc->pChildQueue); - taosProcCleanupQueue(pProc->pParentQueue); + uDebug("proc:%s, is cleaned up", pProc->name); + // taosProcCleanupQueue(pProc->pChildQueue); + // taosProcCleanupQueue(pProc->pParentQueue); taosMemoryFree(pProc); } } diff --git a/source/util/test/queueTest.cpp b/source/util/test/queueTest.cpp index 09c544df9f..0c4bcf84ad 100644 --- a/source/util/test/queueTest.cpp +++ b/source/util/test/queueTest.cpp @@ -26,108 +26,3 @@ class UtilTestQueue : public ::testing::Test { static void SetUpTestSuite() {} static void TearDownTestSuite() {} }; - -#if 0 -TEST_F(UtilTestQueue, 01_fork) { - pid_t pid; - int shmid; - int* shmptr; - int* tmp; - - int err; - pthread_mutexattr_t mattr; - if ((err = pthread_mutexattr_init(&mattr)) < 0) { - printf("mutex addr init error:%s\n", strerror(err)); - exit(1); - } - - if ((err = pthread_mutexattr_setpshared(&mattr, PTHREAD_PROCESS_SHARED)) < 0) { - printf("mutex addr get shared error:%s\n", strerror(err)); - exit(1); - } - - pthread_mutex_t* m; - int mid = shmget(IPC_PRIVATE, sizeof(pthread_mutex_t), 0600); - m = (pthread_mutex_t*)shmat(mid, NULL, 0); - - if ((err = pthread_mutex_init(m, &mattr)) < 0) { - printf("mutex mutex init error:%s\n", strerror(err)); - exit(1); - } - - if ((shmid = shmget(IPC_PRIVATE, 1000, IPC_CREAT | 0600)) < 0) { - perror("shmget error"); - exit(1); - } - - if ((shmptr = (int*)shmat(shmid, 0, 0)) == (void*)-1) { - perror("shmat error"); - exit(1); - } - - tmp = shmptr; - - int shmid2; - int** shmptr2; - if ((shmid2 = shmget(IPC_PRIVATE, 20, IPC_CREAT | 0600)) < 0) { - perror("shmget2 error"); - exit(1); - } - - if ((shmptr2 = (int**)shmat(shmid2, 0, 0)) == (void*)-1) { - perror("shmat2 error"); - exit(1); - } - - *shmptr2 = shmptr; - - if ((pid = fork()) < 0) { - perror("fork error"); - exit(1); - } else if (pid == 0) { - if ((err = taosThreadMutexLock(m)) < 0) { - printf("lock error:%s\n", strerror(err)); - exit(1); - } - for (int i = 0; i < 30; ++i) { - **shmptr2 = i; - (*shmptr2)++; - } - - if ((err = taosThreadMutexUnlock(m)) < 0) { - printf("unlock error:%s\n", strerror(err)); - exit(1); - } - exit(0); - - } else { - if ((err = taosThreadMutexLock(m)) < 0) { - printf("lock error:%s\n", strerror(err)); - exit(1); - } - for (int i = 10; i < 42; ++i) { - **shmptr2 = i; - (*shmptr2)++; - } - if ((err = taosThreadMutexUnlock(m)) < 0) { - printf("unlock error:%s\n", strerror(err)); - exit(1); - } - } - - wait(NULL); - - for (int i = 0; i < 70; ++i) { - printf("%d ", tmp[i]); - } - - printf("\n"); - - taosThreadAttrDestroy(&mattr); - //销毁mutex - pthread_mutex_destroy(m); - - exit(0); -} - -#endif \ No newline at end of file diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 2b654a3793..9fe4ffa0d4 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -50,7 +50,10 @@ class TDSql: def prepare(self): tdLog.info("prepare database:db") s = 'reset query cache' - self.cursor.execute(s) + try: + self.cursor.execute(s) + except: + tdLog.notice("'reset query cache' is not supported") s = 'drop database if exists db' self.cursor.execute(s) s = 'create database db' @@ -341,4 +344,4 @@ class TDSql: tdLog.info("dir: %s is created" %dir) pass -tdSql = TDSql() \ No newline at end of file +tdSql = TDSql() diff --git a/tests/script/general/stable/disk.sim b/tests/script/general/stable/disk.sim deleted file mode 100644 index 1faae78e89..0000000000 --- a/tests/script/general/stable/disk.sim +++ /dev/null @@ -1,199 +0,0 @@ -system sh/stop_dnodes.sh - - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = d_db -$tbPrefix = d_tb -$mtPrefix = d_mt -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i - -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = 0 - while $x < $rowNum - $val = $x * 60000 - $ms = 1519833600000 + $val - sql insert into $tb values ($ms , $x ) - $x = $x + 1 - endw - - $i = $i + 1 -endw - -sql show vgroups -print vgroups ==> $rows -if $rows != 3 then - return -1 -endi - -sql select count(*) from $mt -print select count(*) from $mt ===> $data00 -if $data00 != $totalNum then - return -1 -endi - -sleep 1000 -system sh/exec.sh -n dnode1 -s stop -x SIGINT -sleep 3000 -system sh/exec.sh -n dnode1 -s start -sleep 6000 - -sql use $db -sql show vgroups -print vgroups ==> $rows -if $rows != 3 then - return -1 -endi - -print =============== step2 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*) from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -sql select count(tbcol) from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -print =============== step3 -sql select count(tbcol) from $tb where ts <= 1519833840000 -print ===> $data00 -if $data00 != 5 then - return -1 -endi - -print =============== step4 -sql select count(tbcol) as b from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -print =============== step5 -sql select count(tbcol) as b from $tb interval(1m) -print ===> $data01 -if $data01 != 1 then - return -1 -endi - -sql select count(tbcol) as b from $tb interval(1d) -print ===> $data01 -if $data01 != $rowNum then - return -1 -endi - -print =============== step6 -sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -print ===> $data01 -if $data01 != 1 then - return -1 -endi -if $rows != 5 then - return -1 -endi - -print =============== step7 -sql select count(*) from $mt -print select count(*) from $mt ===> $data00 -if $data00 != $totalNum then - return -1 -endi - -sql select count(tbcol) from $mt -print ===> $data00 -if $data00 != $totalNum then - return -1 -endi - -print =============== step8 -sql select count(tbcol) as c from $mt where ts <= 1519833840000 -print ===> $data00 -if $data00 != 50 then - return -1 -endi - -sql select count(tbcol) as c from $mt where tgcol < 5 -print ===> $data00 -if $data00 != 100 then - return -1 -endi - -sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -print ===> $data00 -if $data00 != 25 then - return -1 -endi - -print =============== step9 -sql select count(tbcol) as b from $mt interval(1m) -print select count(tbcol) as b from $mt interval(1m) ===> $data01 -if $data01 != 10 then - return -1 -endi - -sql select count(tbcol) as b from $mt interval(1d) -print ===> $data02 -if $data01 != 200 then - return -1 -endi - -print =============== step10 -print select count(tbcol) as b from $mt group by tgcol -sql select count(tbcol) as b from $mt group by tgcol -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -if $rows != $tbNum then - return -1 -endi - -print =============== step11 -sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -print ===> $data01 -if $data01 != 1 then - return -1 -endi -if $rows != 50 then - return -1 -endi - -print =============== clear -sql drop database $db -sql show databases -if $rows != 0 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stable/dnode3.sim b/tests/script/general/stable/dnode3.sim deleted file mode 100644 index 872cfb8d2e..0000000000 --- a/tests/script/general/stable/dnode3.sim +++ /dev/null @@ -1,212 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/deploy.sh -n dnode2 -i 2 -system sh/deploy.sh -n dnode3 -i 3 -system sh/deploy.sh -n dnode4 -i 4 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode2 -c walLevel -v 1 -system sh/cfg.sh -n dnode3 -c walLevel -v 1 -system sh/cfg.sh -n dnode4 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start - -sql connect - -sql create dnode $hostname2 -sql create dnode $hostname3 -sql create dnode $hostname4 -system sh/exec.sh -n dnode2 -s start -system sh/exec.sh -n dnode3 -s start -system sh/exec.sh -n dnode4 -s start -$x = 0 -createDnode: - $x = $x + 1 - sleep 1000 - if $x == 20 then - return -1 - endi -sql show dnodes; -if $data4_2 == offline then - goto createDnode -endi -if $data4_3 == offline then - goto createDnode -endi -if $data4_4 == offline then - goto createDnode -endi - -print ======================== dnode1 start - -$dbPrefix = r3v3_db -$tbPrefix = r3v3_tb -$mtPrefix = r3v3_mt -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i - -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = 0 - while $x < $rowNum - $val = $x * 60000 - $ms = 1519833600000 + $val - sql insert into $tb values ($ms , $x ) - $x = $x + 1 - endw - - $i = $i + 1 -endw - -sql show vgroups -print vgroups ==> $rows -if $rows != 3 then - return -1 -endi - -sleep 100 - -print =============== step2 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*) from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -sql select count(tbcol) from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -print =============== step3 -sql select count(tbcol) from $tb where ts <= 1519833840000 -print ===> $data00 -if $data00 != 5 then - return -1 -endi - -print =============== step4 -sql select count(tbcol) as b from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -print =============== step5 -sql select count(tbcol) as b from $tb interval(1m) -print ===> $data01 -if $data01 != 1 then - return -1 -endi - -sql select count(tbcol) as b from $tb interval(1d) -print ===> $data01 -if $data01 != $rowNum then - return -1 -endi - -print =============== step6 -sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -print ===> $data01 -if $data01 != 1 then - return -1 -endi -if $rows != 5 then - return -1 -endi - -print =============== step7 -print select count(*) from $mt -sql select count(*) from $mt -print ===> $data00 -if $data00 != $totalNum then - return -1 -endi - -sql select count(tbcol) from $mt -print ===> $data00 -if $data00 != $totalNum then - return -1 -endi - -print =============== step8 -sql select count(tbcol) as c from $mt where ts <= 1519833840000 -print ===> $data00 -if $data00 != 50 then - return -1 -endi - -sql select count(tbcol) as c from $mt where tgcol < 5 -print ===> $data00 -if $data00 != 100 then - return -1 -endi - -sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -print ===> $data00 -if $data00 != 25 then - return -1 -endi - -print =============== step9 -sql select count(tbcol) as b from $mt interval(1m) -print ===> $data01 -if $data01 != 10 then - return -1 -endi - -sql select count(tbcol) as b from $mt interval(1d) -print ===> $data01 -if $data01 != 200 then - return -1 -endi - -print =============== step10 -sql select count(tbcol) as b from $mt group by tgcol -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -if $rows != $tbNum then - return -1 -endi - -print =============== step11 -sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -print ===> $data01 -if $data01 != 1 then - return -1 -endi -if $rows != 50 then - return -1 -endi - -print =============== clear -sql drop database $db -sql show databases -if $rows != 0 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stable/values.sim b/tests/script/general/stable/values.sim deleted file mode 100644 index fb2c908da2..0000000000 --- a/tests/script/general/stable/values.sim +++ /dev/null @@ -1,113 +0,0 @@ -system sh/stop_dnodes.sh - - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ======================== dnode1 start - -sql create database vdb0 -sql create table vdb0.mt (ts timestamp, tbcol int) TAGS(tgcol int) - -sql create table vdb0.vtb00 using vdb0.mt tags( 0 ) -sql create table vdb0.vtb01 using vdb0.mt tags( 0 ) - -sql create database vdb1 -sql create table vdb1.mt (ts timestamp, tbcol int) TAGS(tgcol int) -sql_error create table vdb1.vtb10 using vdb0.mt tags( 1 ) -sql_error create table vdb1.vtb11 using vdb0.mt tags( 1 ) -sql create table vdb1.vtb10 using vdb1.mt tags( 1 ) -sql create table vdb1.vtb11 using vdb1.mt tags( 1 ) - -sql create database vdb2 -sql create table vdb2.mt (ts timestamp, tbcol int) TAGS(tgcol int) -sql_error create table vdb2.vtb20 using vdb0.mt tags( 2 ) -sql_error create table vdb2.vtb21 using vdb0.mt tags( 2 ) -sql create table vdb2.vtb20 using vdb2.mt tags( 2 ) -sql create table vdb2.vtb21 using vdb2.mt tags( 2 ) - -sql create database vdb3 -sql create table vdb3.mt (ts timestamp, tbcol int) TAGS(tgcol int) -sql_error create table vdb3.vtb20 using vdb0.mt tags( 2 ) -sql_error create table vdb3.vtb21 using vdb0.mt tags( 2 ) -sql create table vdb3.vtb30 using vdb3.mt tags( 3 ) -sql create table vdb3.vtb31 using vdb3.mt tags( 3 ) - -print =============== step2 -sql insert into vdb0.vtb00 values (1519833600000 , 10) (1519833600001, 20) (1519833600002, 30) -sql insert into vdb0.vtb01 values (1519833600000 , 10) (1519833600001, 20) (1519833600002, 30) -sql insert into vdb1.vtb10 values (1519833600000 , 11) (1519833600001, 21) (1519833600002, 31) -sql insert into vdb1.vtb11 values (1519833600000 , 11) (1519833600001, 21) (1519833600002, 31) -sql insert into vdb2.vtb20 values (1519833600000 , 12) (1519833600001, 22) (1519833600002, 32) -sql insert into vdb2.vtb21 values (1519833600000 , 12) (1519833600001, 22) (1519833600002, 32) -sql insert into vdb3.vtb30 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) -sql insert into vdb3.vtb31 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) -sql select * from vdb0.mt - -if $rows != 6 then - return -1 -endi - -print =============== step3 -sql insert into vdb0.vtb00 values (1519833600003 , 40) (1519833600005, 50) (1519833600004, 60) -sql insert into vdb0.vtb01 values (1519833600003 , 40) (1519833600005, 50) (1519833600004, 60) -sql insert into vdb1.vtb10 values (1519833600003 , 41) (1519833600005, 51) (1519833600004, 61) -sql insert into vdb1.vtb11 values (1519833600003 , 41) (1519833600005, 51) (1519833600004, 61) -sql insert into vdb2.vtb20 values (1519833600003 , 42) (1519833600005, 52) (1519833600004, 62) -sql insert into vdb2.vtb21 values (1519833600003 , 42) (1519833600005, 52) (1519833600004, 62) -sql insert into vdb3.vtb30 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) -sql insert into vdb3.vtb31 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) -sql select * from vdb0.mt - -if $rows != 12 then - return -1 -endi - -print =============== step4 -sql insert into vdb0.vtb00 values(1519833600006, 60) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 60) (1519833600007, 70) -sql insert into vdb1.vtb10 values(1519833600006, 61) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 61) (1519833600007, 71) -sql insert into vdb2.vtb20 values(1519833600006, 62) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 62) (1519833600007, 72) -sql insert into vdb3.vtb30 values(1519833600006, 63) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 63) (1519833600007, 73) -sql select * from vdb0.mt - -if $rows != 16 then - return -1 -endi - -print =============== step5 -sql insert into vdb0.vtb00 values(1519833600008, 80) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 80) (1519833600007, 70) -sql insert into vdb1.vtb10 values(1519833600008, 81) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 81) (1519833600007, 71) -sql insert into vdb2.vtb20 values(1519833600008, 82) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 82) (1519833600007, 72) -sql insert into vdb3.vtb30 values(1519833600008, 83) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 83) (1519833600007, 73) -sql select * from vdb0.mt - -if $rows != 17 then - return -1 -endi - -print =============== step6 -sql insert into vdb0.vtb00 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb10 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb20 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb30 values(1519833600009, 90) (1519833600010, 100) -sql insert into vdb0.vtb01 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb11 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb21 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb31 values(1519833600009, 90) (1519833600010, 100) - -sql select * from vdb0.mt - -if $rows != 21 then - return -1 -endi - -print =============== step7 -sql insert into vdb0.vtb00 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb10 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb20 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb30 values(1519833600012, 120) (1519833600011, 110) -sql insert into vdb0.vtb01 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb11 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb21 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb31 values(1519833600012, 120) (1519833600011, 110) - -sql select * from vdb0.mt - -if $rows != 25 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/stable/vnode3.sim b/tests/script/general/stable/vnode3.sim deleted file mode 100644 index 61948b5063..0000000000 --- a/tests/script/general/stable/vnode3.sim +++ /dev/null @@ -1,179 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 1 -system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 -system sh/exec.sh -n dnode1 -s start - -sleep 2000 -sql connect - -print ======================== dnode1 start - -$dbPrefix = v3_db -$tbPrefix = v3_tb -$mtPrefix = v3_mt -$tbNum = 10 -$rowNum = 20 -$totalNum = 200 - -print =============== step1 -$i = 0 -$db = $dbPrefix . $i -$mt = $mtPrefix . $i - -sql create database $db -sql use $db -sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) - -$i = 0 -while $i < $tbNum - $tb = $tbPrefix . $i - sql create table $tb using $mt tags( $i ) - - $x = 0 - while $x < $rowNum - $val = $x * 60000 - $ms = 1519833600000 + $val - sql insert into $tb values ($ms , $x ) - $x = $x + 1 - endw - - $i = $i + 1 -endw - -sql show vgroups -print vgroups ==> $rows -if $rows != 3 then - return -1 -endi - - -print =============== step2 -$i = 1 -$tb = $tbPrefix . $i - -sql select count(*) from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -sql select count(tbcol) from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -print =============== step3 -sql select count(tbcol) from $tb where ts <= 1519833840000 -print ===> $data00 -if $data00 != 5 then - return -1 -endi - -print =============== step4 -sql select count(tbcol) as b from $tb -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -print =============== step5 -sql select count(tbcol) as b from $tb interval(1m) -print ===> $data01 -if $data01 != 1 then - return -1 -endi - -sql select count(tbcol) as b from $tb interval(1d) -print ===> $data01 -if $data01 != $rowNum then - return -1 -endi - -print =============== step6 -sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) -print ===> $data01 -if $data01 != 1 then - return -1 -endi -if $rows != 5 then - return -1 -endi - -print =============== step7 -sql select count(*) from $mt -print ===> $data00 -if $data00 != $totalNum then - return -1 -endi - -sql select count(tbcol) from $mt -print ===> $data00 -if $data00 != $totalNum then - return -1 -endi - -print =============== step8 -sql select count(tbcol) as c from $mt where ts <= 1519833840000 -print ===> $data00 -if $data00 != 50 then - return -1 -endi - -sql select count(tbcol) as c from $mt where tgcol < 5 -print ===> $data00 -if $data00 != 100 then - return -1 -endi - -sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 -print ===> $data00 -if $data00 != 25 then - return -1 -endi - -print =============== step9 -sql select count(tbcol) as b from $mt interval(1m) -print ===> $data01 -if $data01 != 10 then - return -1 -endi - -sql select count(tbcol) as b from $mt interval(1d) -print ===> $data01 -if $data01 != 200 then - return -1 -endi - -print =============== step10 -sql select count(tbcol) as b from $mt group by tgcol -print ===> $data00 -if $data00 != $rowNum then - return -1 -endi - -if $rows != $tbNum then - return -1 -endi - -print =============== step11 -sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol -print ===> $data01 -if $data01 != 1 then - return -1 -endi -if $rows != 50 then - return -1 -endi - -print =============== clear -sql drop database $db -sql show databases -if $rows != 0 then - return -1 -endi - -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 2cad518128..811c9941f7 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -2,9 +2,10 @@ #======================b1-start=============== # ---- user -./test.sh -f tsim/user/basic1.sim +./test.sh -f tsim/user/basic1.sim # ---- db +./test.sh -f tsim/db/alter_option.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic3.sim @@ -28,15 +29,25 @@ # ---- query ./test.sh -f tsim/query/interval.sim ./test.sh -f tsim/query/interval-offset.sim +./test.sh -f tsim/query/scalarFunction.sim # ---- show -./test.sh -f tsim/show/basic.sim +./test.sh -f tsim/show/basic.sim # ---- table -./test.sh -f tsim/table/basic1.sim +./test.sh -f tsim/table/basic1.sim # ---- tmq ./test.sh -f tsim/tmq/basic.sim ./test.sh -f tsim/tmq/basic1.sim +# --- stable +./test.sh -f tsim/stable/disk.sim +./test.sh -f tsim/stable/dnode3.sim +./test.sh -f tsim/stable/metrics.sim +./test.sh -f tsim/stable/refcount.sim +# ./test.sh -f tsim/stable/show.sim +./test.sh -f tsim/stable/values.sim +./test.sh -f tsim/stable/vnode3.sim + #======================b1-end=============== diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim new file mode 100644 index 0000000000..0fc7175cb1 --- /dev/null +++ b/tests/script/tsim/db/alter_option.sim @@ -0,0 +1,353 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$loop_cnt = 0 +check_dnode_ready_1: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> rows: $rows +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +print ===> $data20 $data21 $data22 $data23 $data24 $data25 +if $data00 != 1 then + return -1 +endi +if $data01 != localhost:7100 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready_1 +endi +if $data14 != ready then + goto check_dnode_ready_1 +endi +if $data24 != ready then + goto check_dnode_ready_1 +endi + +print ============= create database +#database_option: { +# BLOCKS value [3~1000, default: 6] +# | CACHELAST value [0, 1, 2, 3] +# | FSYNC value [0 ~ 180000 ms] +# | KEEP value [days, 365000] +# | QUORUM value [1 | 2] +# | REPLICA value [1 | 3] +# | WAL value [1 | 2] + +sql create database db BLOCKS 7 CACHE 3 CACHELAST 3 COMP 0 DAYS 240 FSYNC 1000 MAXROWS 8000 MINROWS 10 KEEP 1000 PRECISION 'ns' QUORUM 1 REPLICA 3 TTL 7 WAL 2 VGROUPS 6 SINGLE_STABLE 1 STREAM_MODE 1 +sql show databases +print rows: $rows +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +print ====> dataX_db +print $data0_db $data1_db $data2_db $data3_db $data4_db $data5_db $data6_db $data7_db $data8_db $data9_db $data10_db $data11_db $data12_db $data13_db $data14_db $data15_db $data16_db $data17_db + +if $rows != 2 then + return -1 +endi +if $data0_db != db then # name + return -1 +endi +if $data2_db != 6 then # vgroups + return -1 +endi +if $data3_db != 0 then # ntables + return -1 +endi +if $data4_db != 3 then # replica + return -1 +endi +if $data5_db != 1 then # quorum + return -1 +endi +if $data6_db != 240 then # days + return -1 +endi +if $data7_db != 1000,1000,1000 then # keep + return -1 +endi +if $data8_db != 3 then # cache + return -1 +endi +if $data9_db != 7 then # blocks + return -1 +endi +if $data10_db != 10 then # minrows + return -1 +endi +if $data11_db != 8000 then # maxrows + return -1 +endi +if $data12_db != 2 then # wal + return -1 +endi +if $data13_db != 1000 then # fsync + return -1 +endi +if $data14_db != 0 then # comp + return -1 +endi +if $data15_db != 3 then # cachelast + return -1 +endi +if $data16_db != ns then # precision + return -1 +endi + +print ============== not support modify options: name, create_time, vgroups, ntables +sql_error alter database db name dba +sql_error alter database db create_time "2022-03-03 15:08:13.329" +sql_error alter database db vgroups -1 +sql_error alter database db vgroups 0 +sql_error alter database db vgroups 2 +sql_error alter database db vgroups 20 +sql_error alter database db ntables -1 +sql_error alter database db ntables 0 +sql_error alter database db ntables 1 +sql_error alter database db ntables 10 + +#print ============== modify replica +sql_error alter database db replica 2 +sql_error alter database db replica 5 +sql_error alter database db replica -1 +sql_error alter database db replica 0 +#sql alter database db replica 1 +#sql show databases +#print replica: $data4_db +#if $data4_db != 1 then +# return -1 +#endi +#sql alter database db replica 3 +#sql show databases +#print replica: $data4_db +#if $data4_db != 3 then +# return -1 +#endi + +print ============== modify quorum +sql alter database db quorum 2 +sql show databases +print quorum $data5_db +if $data5_db != 2 then + return -1 +endi +sql alter database db quorum 1 +sql show databases +print quorum $data5_db +if $data5_db != 1 then + return -1 +endi + +sql_error alter database db quorum -1 +sql_error alter database db quorum 0 +sql_error alter database db quorum 3 +sql_error alter database db quorum 4 +sql_error alter database db quorum 5 + +#print ============== modify days +sql_error alter database db days 480 +sql_error alter database db days 360 +sql_error alter database db days 0 +sql_error alter database db days 14400 # set over than keep + +print ============== modify keep +sql alter database db keep 2000 +sql show databases +print keep $data7_db +if $data7_db != 1000,1000,2000 then + return -1 +endi + +#sql alter database db keep 1000,2000 +#sql show databases +#print keep $data7_db +#if $data7_db != 500,500,500 then +# return -1 +#endi + +#sql alter database db keep 40,50 +#sql alter database db keep 30,31 +#sql alter database db keep 20 +#sql_error alter database db keep 10.0 +#sql_error alter database db keep 9 +#sql_error alter database db keep 1 +sql_error alter database db keep 0 +sql_error alter database db keep -1 +#sql_error alter database db keep 365001 + +print ============== modify cache +sql_error alter database db cache 12 +sql_error alter database db cache 1 +sql_error alter database db cache 60 +sql_error alter database db cache 50 +sql_error alter database db cache 20 +sql_error alter database db cache 3 +sql_error alter database db cache 129 +sql_error alter database db cache 300 +sql_error alter database db cache 0 +sql_error alter database db cache -1 + +print ============== modify blocks +sql alter database db blocks 3 +sql show databases +print blocks $data9_db +if $data9_db != 3 then + return -1 +endi +sql alter database db blocks 11 +sql show databases +print blocks $data9_db +if $data9_db != 11 then + return -1 +endi + +sql alter database db blocks 40 +sql alter database db blocks 30 +sql alter database db blocks 20 +sql alter database db blocks 10 +sql_error alter database db blocks 2 +sql_error alter database db blocks 1 +sql_error alter database db blocks 0 +sql_error alter database db blocks -1 +sql_error alter database db blocks 10001 + +print ============== modify minrows +sql_error alter database db minrows 8 +sql_error alter database db minrows 200 +sql_error alter database db minrows 11 +sql_error alter database db minrows 8000 +sql_error alter database db minrows 8001 + +print ============== modify maxrows +sql_error alter database db maxrows 1000 +sql_error alter database db maxrows 2000 +sql_error alter database db maxrows 11 # equal minrows +sql_error alter database db maxrows 10 # little than minrows + +print ============== step wal +sql alter database db wal 1 +sql show databases +print wal $data12_db +if $data12_db != 1 then + return -1 +endi +sql alter database db wal 2 +sql show databases +print wal $data12_db +if $data12_db != 2 then + return -1 +endi + +#sql_error alter database db wal 0 # TD-14436 +sql_error alter database db wal 3 +sql_error alter database db wal 100 +sql_error alter database db wal -1 + +print ============== modify fsync +sql alter database db fsync 2000 +sql show databases +print fsync $data13_db +if $data13_db != 2000 then + return -1 +endi +sql alter database db fsync 500 +sql show databases +print fsync $data13_db +if $data13_db != 500 then + return -1 +endi +sql alter database db fsync 0 +sql show databases +print fsync $data13_db +if $data13_db != 0 then + return -1 +endi +sql_error alter database db fsync 180001 +sql_error alter database db fsync -1 + +print ============== modify comp +sql_error alter database db comp 1 +sql_error alter database db comp 2 +sql_error alter database db comp 1 +sql_error alter database db comp 0 +sql_error alter database db comp 3 +sql_error alter database db comp 4 +sql_error alter database db comp 5 +sql_error alter database db comp -1 + +print ============== modify cachelast [0, 1, 2, 3] +sql alter database db cachelast 2 +sql show databases +print cachelast $data15_db +if $data15_db != 2 then + return -1 +endi +sql alter database db cachelast 1 +sql show databases +print cachelast $data15_db +if $data15_db != 1 then + return -1 +endi +sql alter database db cachelast 0 +sql show databases +print cachelast $data15_db +if $data15_db != 0 then + return -1 +endi +sql alter database db cachelast 2 +sql show databases +print cachelast $data15_db +if $data15_db != 2 then + return -1 +endi +sql alter database db cachelast 3 +sql show databases +print cachelast $data15_db +if $data15_db != 3 then + return -1 +endi + +sql_error alter database db cachelast 4 +sql_error alter database db cachelast 10 +sql_error alter database db cachelast -1 + +print ============== modify precision +sql_error alter database db precision 'ms' +sql_error alter database db precision 'us' +sql_error alter database db precision 'ns' +sql_error alter database db precision 'ys' +sql_error alter database db prec 'xs' + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/interval-offset.sim b/tests/script/tsim/query/interval-offset.sim index 4c4ebc6670..ffb13979cd 100644 --- a/tests/script/tsim/query/interval-offset.sim +++ b/tests/script/tsim/query/interval-offset.sim @@ -7,7 +7,7 @@ sql connect print =============== create database sql create database d0 sql show databases -if $rows != 2 then +if $rows != 2 then return -1 endi @@ -17,7 +17,7 @@ print =============== create super table and child table sql create table stb (ts timestamp, tbcol int) tags (t1 int) sql show stables print $rows $data00 $data01 $data02 -if $rows != 1 then +if $rows != 1 then return -1 endi @@ -29,7 +29,7 @@ sql show tables print $rows $data00 $data10 $data20 if $rows != 4 then return -1 -endi +endi print =============== insert data into child table ct1 (s) sql insert into ct1 values ( '2022-01-01 01:01:01.000', 1 ) @@ -83,13 +83,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 if $rows != 5 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data40 != 1 then return -1 -endi +endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(10s) @@ -101,13 +101,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 if $rows != 5 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data40 != 1 then return -1 -endi +endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(5s) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct1 interval(10s, 2s) sliding(5s) @@ -123,16 +123,16 @@ print ===> rows7: $data70 $data71 $data72 $data73 $data74 print ===> rows8: $data80 $data81 $data82 $data83 $data84 if $rows != 9 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data70 != 2 then return -1 -endi +endi if $data80 != 1 then return -1 -endi +endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) @@ -144,10 +144,10 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 if $rows != 4 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data10 != 2 then return -1 endi @@ -155,27 +155,54 @@ endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) print ===> rows: $rows -print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05 -print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15 -print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25 -print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35 -print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45 -print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 -print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 -print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 +print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 if $rows != 8 then return -1 -endi +endi if $data00 != 1 then return -1 -endi +endi if $data10 != 2 then return -1 -endi +endi if $data70 != 1 then return -1 endi - + +sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from ct2 interval(1d, 2h) sliding(12h) +print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct2 interval(1d, 2h) sliding(12h) +print ===> rows: $rows +print ===> rows0: $data00 $data01 $data02 $data03 $data04 $data05 +print ===> rows1: $data10 $data11 $data12 $data13 $data14 $data15 +print ===> rows2: $data20 $data21 $data22 $data23 $data24 $data25 +print ===> rows3: $data30 $data31 $data32 $data33 $data34 $data35 +print ===> rows4: $data40 $data41 $data42 $data43 $data44 $data45 +print ===> rows5: $data50 $data51 $data52 $data53 $data54 $data55 +print ===> rows6: $data60 $data61 $data62 $data63 $data64 $data65 +print ===> rows7: $data70 $data71 $data72 $data73 $data74 $data75 +if $rows != 8 then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data15 != 2 then + return -1 +endi +if $data75 != 1 then + return -1 +endi +if $data02 != 86400000 then + return -1 +endi + sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) print ===> rows: $rows @@ -184,15 +211,15 @@ print ===> rows1: $data10 $data11 $data12 $data13 $data14 print ===> rows2: $data20 $data21 $data22 $data23 $data24 print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 -#if $rows != 5 then -# return -1 -#endi -#if $data00 != 1 then -# return -1 -#endi -#if $data40 != 1 then -# return -1 -#endi +# if $rows != 5 then +# return -1 +# endi +# f $data00 != 1 then +# return -1 +# endi +# if $data40 != 1 then +# return -1 +# endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(2w) @@ -204,13 +231,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 #if $rows != 5 then # return -1 -#endi +#endi #if $data00 != 1 then # return -1 -#endi +#endi #if $data40 != 1 then # return -1 -#endi +#endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct3 interval(1n, 1w) sliding(4w) @@ -225,13 +252,13 @@ print ===> rows6: $data60 $data61 $data62 $data63 $data64 print ===> rows7: $data70 $data71 $data72 $data73 $data74 #if $rows != 8 then # return -1 -#endi +#endi #if $data00 != 2 then # return -1 -#endi +#endi #if $data70 != 1 then # return -1 -#endi +#endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) @@ -243,13 +270,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 #if $rows != 5 then # return -1 -#endi +#endi #if $data00 != 1 then # return -1 -#endi +#endi #if $data40 != 1 then # return -1 -#endi +#endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(6n) @@ -261,13 +288,13 @@ print ===> rows3: $data30 $data31 $data32 $data33 $data34 print ===> rows4: $data40 $data41 $data42 $data43 $data44 #if $rows != 5 then # return -1 -#endi +#endi #if $data00 != 1 then # return -1 -#endi +#endi #if $data40 != 1 then # return -1 -#endi +#endi sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(*) from ct4 interval(1y, 6n) sliding(12n) @@ -282,13 +309,13 @@ print ===> rows6: $data60 $data61 $data62 $data63 $data64 print ===> rows7: $data70 $data71 $data72 $data73 $data74 #if $rows != 8 then # return -1 -#endi +#endi #if $data00 != 2 then # return -1 -#endi +#endi #if $data70 != 1 then # return -1 -#endi +#endi #================================================= print =============== stop and restart taosd @@ -322,9 +349,4 @@ endi #sql select count(*) from car where ts > '2019-05-14 00:00:00' interval(1y, 5d) - - - - - #system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/query/interval.sim b/tests/script/tsim/query/interval.sim index 6dd0a9537e..384008c887 100644 --- a/tests/script/tsim/query/interval.sim +++ b/tests/script/tsim/query/interval.sim @@ -29,18 +29,18 @@ $i = 0 while $i < $tbNum $tb = $tbPrefix . $i sql create table $tb using $mt tags( $i ) - + $x = 0 while $x < $rowNum $cc = $x * 60000 $ms = 1601481600000 + $cc - sql insert into $tb values ($ms , $x ) + sql insert into $tb values ($ms , $x ) $x = $x + 1 - endw - + endw + $i = $i + 1 -endw +endw print =============== step2 $i = 1 @@ -49,51 +49,51 @@ $tb = $tbPrefix . $i sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb interval(1m) print ===> $rows $data01 $data05 -if $rows != $rowNum then +if $rows != $rowNum then return -1 endi -if $data00 != 1 then +if $data00 != 1 then return -1 endi -if $data04 != 1 then +if $data04 != 1 then return -1 endi -#print =============== step3 +print =============== step3 #$cc = 4 * 60000 #$ms = 1601481600000 + $cc #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms interval(1m) #print ===> $rows $data01 $data05 -#if $rows != 5 then +#if $rows != 5 then # return -1 #endi -#if $data00 != 1 then +#if $data00 != 1 then # return -1 #endi -#if $data04 != 1 then +#if $data04 != 1 then # return -1 #endi -#print =============== step4 +print =============== step4 #$cc = 40 * 60000 #$ms = 1601481600000 + $cc #$cc = 1 * 60000 #$ms2 = 1601481600000 - $cc -#sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) -#print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) -#print ===> $rows $data01 $data05 -#if $rows != 20 then -# return -1 -#endi -#if $data00 != 1 then -# return -1 -#endi -#if $data04 != 1 then -# return -1 -#endi +sql select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from $tb interval(1m) +print ===> select _wstartts, _wendts, _wduration, _qstartts, _qendts, count(tbcol) from $tb interval(1m) +print ===> $rows $data01 $data05 +if $rows != $rowNum then + return -1 +endi +if $data05 != 1 then + return -1 +endi +if $data02 != 60000 then + return -1 +endi #print =============== step5 #$cc = 40 * 60000 @@ -105,13 +105,13 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) #print ===> $rows $data21 $data25 -#if $rows != 42 then +#if $rows != 42 then # return -1 #endi -#if $data20 != 1 then +#if $data20 != 1 then # return -1 #endi -#if $data24 != 1 then +#if $data24 != 1 then # return -1 #endi @@ -119,10 +119,10 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt interval(1m) #print ===> $rows $data11 -#if $rows != 20 then +#if $rows != 20 then # return -1 #endi -#if $data11 != 10 then +#if $data11 != 10 then # return -1 #endi @@ -132,10 +132,10 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms interval(1m) #print ===> $rows $data11 -#if $rows != 5 then +#if $rows != 5 then # return -1 #endi -#if $data11 != 10 then +#if $data11 != 10 then # return -1 #endi @@ -149,10 +149,10 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) #print ===> $rows $data11 -#if $rows != 20 then +#if $rows != 20 then # return -1 #endi -#if $data11 != 10 then +#if $data11 != 10 then # return -1 #endi # @@ -166,18 +166,18 @@ endi #sql select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) #print ===> select count(tbcol), sum(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) #print ===> $rows $data11 -#if $rows != 42 then +#if $rows != 42 then # return -1 #endi -#if $data11 != 10 then +#if $data11 != 10 then # return -1 #endi print =============== clear #sql drop database $db #sql show databases -#if $rows != 0 then +#if $rows != 0 then # return -1 #endi -#system sh/exec.sh -n dnode1 -s stop -x SIGINT +#system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/query/scalarFunction.sim b/tests/script/tsim/query/scalarFunction.sim new file mode 100644 index 0000000000..6a2a602703 --- /dev/null +++ b/tests/script/tsim/query/scalarFunction.sim @@ -0,0 +1,484 @@ +#### abs, log, pow, sqrt, sin, cos, tan, asin, acos, atan, ceil, floor, round + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 +system sh/exec.sh -n dnode1 -s start + +$loop_cnt = 0 +check_dnode_ready: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 +if $data00 != 1 then + return -1 +endi +if $data04 != ready then + goto check_dnode_ready +endi + +sql connect + +$vgroups = 4 +$dbNamme = d0 + +print =============== create database $dbNamme vgroups $vgroups +sql create database $dbNamme vgroups $vgroups +sql show databases +print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09 +print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17 $data18 $data19 +#print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27 $data28 $data29 + +sql use $dbNamme + +print =============== create super table +sql create table stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int) + +print =============== create child table +$tbPrefix = ct +$tbNum = 100 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using stb tags( $i ) + $i = $i + 1 +endw + +print =============== create normal table +sql create table ntb (ts timestamp, c1 int, c2 float, c3 double) + +sql show tables +if $rows != 101 then + return -1 +endi + +print =============== insert data +$rowNum = 20 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + + $x = 0 + $c1 = 0 + while $x < $rowNum + $c2 = 0 - $c1 + $c3 = $c1 + 100 + + sql insert into $tb values ($tstart , $c1 , $c2 , $c3 ) + sql insert into ntb values ($tstart , $c1 , $c2 , $c3 ) + $tstart = $tstart + 1 + $c1 = $c1 + 5 + $x = $x + 1 + endw + + $i = $i + 1 + $tstart = 1640966400000 +endw + +$totalRows = $rowNum * $tbNum +print ====> totalRows of stb: $totalRows + +$loop_test = 0 +loop_test_pos: + +print ====> abs +sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ct1 +print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb +sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb +sql select c1, abs(c1), c2, abs(c2), c3, abs(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> log +sql select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1 +print ====> select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from stb +sql select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ntb +sql select c1, log(c1, 10), c2, log(c2, 10), c3, log(c3, 10) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> pow +sql select c1, pow(c1, 2), c2, pow(c2, 2), c3, pow(c3, 2) from ct1 +print ====> select c1, pow(c1, 2), c2, pow(c2, 2), c3, pow(c3, 2) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, pow(c1, 2), c2, pow(c2, 2), c3, pow(c3, 2) from stb +sql select c1, pow(c1, 2), c2, pow(c2, 2), c3, pow(c3, 2) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, pow(c1, 2), c2, pow(c2, 2), c3, pow(c3, 2) from ntb +sql select c1, pow(c1, 2), c2, pow(c2, 2), c3, pow(c3, 2) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> sqrt +sql select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ct1 +print ====> select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from stb +sql select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ntb +sql select c1, sqrt(c1), c2, sqrt(c2), c3, sqrt(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> sin +#sql select c1, sin(c1), sin(c1) * 3.14159265 / 180 from ct1 # TD-14426 +sql select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1 +print ====> select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, sin(c1), c2, sin(c2), c3, sin(c3) from stb +sql select c1, sin(c1), c2, sin(c2), c3, sin(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ntb +sql select c1, sin(c1), c2, sin(c2), c3, sin(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> cos +sql select c1, cos(c1), c2, cos(c2), c3, cos(c3) from ct1 +print ====> select c1, cos(c1), c2, cos(c2), c3, cos(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, cos(c1), c2, cos(c2), c3, cos(c3) from stb +sql select c1, cos(c1), c2, cos(c2), c3, cos(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, cos(c1), c2, cos(c2), c3, cos(c3) from ntb +sql select c1, cos(c1), c2, cos(c2), c3, cos(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> tan +sql select c1, tan(c1), c2, tan(c2), c3, tan(c3) from ct1 +print ====> select c1, tan(c1), c2, tan(c2), c3, tan(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, tan(c1), c2, tan(c2), c3, tan(c3) from stb +sql select c1, tan(c1), c2, tan(c2), c3, tan(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, tan(c1), c2, tan(c2), c3, tan(c3) from ntb +sql select c1, tan(c1), c2, tan(c2), c3, tan(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> asin +sql select c1, asin(c1), c2, asin(c2), c3, asin(c3) from ct1 +print ====> select c1, asin(c1), c2, asin(c2), c3, asin(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, asin(c1), c2, asin(c2), c3, asin(c3) from stb +sql select c1, asin(c1), c2, asin(c2), c3, asin(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, asin(c1), c2, asin(c2), c3, asin(c3) from ntb +sql select c1, asin(c1), c2, asin(c2), c3, asin(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> acos +sql select c1, acos(c1), c2, acos(c2), c3, acos(c3) from ct1 +print ====> select c1, acos(c1), c2, acos(c2), c3, acos(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, acos(c1), c2, acos(c2), c3, acos(c3) from stb +sql select c1, acos(c1), c2, acos(c2), c3, acos(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, acos(c1), c2, acos(c2), c3, acos(c3) from ntb +sql select c1, acos(c1), c2, acos(c2), c3, acos(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> atan +sql select c1, atan(c1), c2, atan(c2), c3, atan(c3) from ct1 +print ====> select c1, atan(c1), c2, atan(c2), c3, atan(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, atan(c1), c2, atan(c2), c3, atan(c3) from stb +sql select c1, atan(c1), c2, atan(c2), c3, atan(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, atan(c1), c2, atan(c2), c3, atan(c3) from ntb +sql select c1, atan(c1), c2, atan(c2), c3, atan(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> ceil +sql select c1, ceil(c1), c2, ceil(c2), c3, ceil(c3) from ct1 +print ====> select c1, ceil(c1), c2, ceil(c2), c3, ceil(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, ceil(c1), c2, ceil(c2), c3, ceil(c3) from stb +sql select c1, ceil(c1), c2, ceil(c2), c3, ceil(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, ceil(c1), c2, ceil(c2), c3, ceil(c3) from ntb +sql select c1, ceil(c1), c2, ceil(c2), c3, ceil(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +print ====> floor +sql select c1, floor(c1), c2, floor(c2), c3, floor(c3) from ct1 +print ====> select c1, floor(c1), c2, floor(c2), c3, floor(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, floor(c1), c2, floor(c2), c3, floor(c3) from stb +sql select c1, floor(c1), c2, floor(c2), c3, floor(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, floor(c1), c2, floor(c2), c3, floor(c3) from ntb +sql select c1, floor(c1), c2, floor(c2), c3, floor(c3) from ntb +if $rows != $rowNum then + return -1 +endi + + +print ====> round +sql select c1, round(c1), c2, round(c2), c3, round(c3) from ct1 +print ====> select c1, round(c1), c2, round(c2), c3, round(c3) from ct1 +print ====> rows: $rows +print ====> $data00 $data01 $data02 $data03 $data04 $data05 +print ====> $data10 $data11 $data12 $data13 $data14 $data15 +print ====> $data20 $data21 $data22 $data23 $data24 $data25 +print ====> $data30 $data31 $data32 $data33 $data34 $data35 +print ====> $data40 $data41 $data42 $data43 $data44 $data45 +print ====> $data50 $data51 $data52 $data53 $data54 $data55 +print ====> $data60 $data61 $data62 $data63 $data64 $data65 +print ====> $data70 $data71 $data72 $data73 $data74 $data75 +print ====> $data80 $data81 $data82 $data83 $data84 $data85 +print ====> $data90 $data91 $data92 $data93 $data94 $data95 +if $rows != $rowNum then + return -1 +endi +print ====> select c1, round(c1), c2, round(c2), c3, round(c3) from stb +sql select c1, round(c1), c2, round(c2), c3, round(c3) from stb +if $rows != $totalRows then + return -1 +endi +print ====> select c1, round(c1), c2, round(c2), c3, round(c3) from ntb +sql select c1, round(c1), c2, round(c2), c3, round(c3) from ntb +if $rows != $rowNum then + return -1 +endi + +if $loop_test == 0 then + print =============== stop and restart taosd + system sh/exec.sh -n dnode1 -s stop -x SIGINT + system sh/exec.sh -n dnode1 -s start + + $loop_cnt = 0 + check_dnode_ready_0: + $loop_cnt = $loop_cnt + 1 + sleep 200 + if $loop_cnt == 10 then + print ====> dnode not ready! + return -1 + endi + sql show dnodes + print ===> $rows $data00 $data01 $data02 $data03 $data04 $data05 + if $data00 != 1 then + return -1 + endi + if $data04 != ready then + goto check_dnode_ready_0 + endi + + $loop_test = 1 + goto loop_test_pos +endi + +#system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stable/disk.sim b/tests/script/tsim/stable/disk.sim new file mode 100644 index 0000000000..0e33c2066d --- /dev/null +++ b/tests/script/tsim/stable/disk.sim @@ -0,0 +1,206 @@ +system sh/stop_dnodes.sh + + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +print ======================== dnode1 start + +$dbPrefix = d_db +$tbPrefix = d_tb +$mtPrefix = d_mt +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql create database $db +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $val = $x * 60000 + $ms = 1519833600000 + $val + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql show vgroups +print vgroups ==> $rows +if $rows != 2 then + return -1 +endi + +sql select count(tbcol) from $mt +print select count(tbcol) from $mt ===> $data00 +if $data00 != $totalNum then + return -1 +endi + +sleep 1000 +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 3000 +system sh/exec.sh -n dnode1 -s start +sleep 6000 + +sql use $db +sql show vgroups +print vgroups ==> $rows +if $rows != 2 then + return -1 +endi + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select count(tbcol) from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +sql select count(tbcol) from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step3 +# TODO : where condition +# sql select count(tbcol) from $tb where ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 5 then +# return -1 +# endi + +print =============== step4 +sql select count(tbcol) as b from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step5 +sql select count(tbcol) as b from $tb interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi + +sql select count(tbcol) as b from $tb interval(1d) +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step6 +# TODO +# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +# print ===> $data00 +# if $data00 != 1 then +# return -1 +# endi +# if $rows != 5 then +# return -1 +# endi + +print =============== step7 +# TODO +# sql select count(*) from $mt +# print select count(*) from $mt ===> $data00 +# if $data00 != $totalNum then +# return -1 +# endi + +sql select count(tbcol) from $mt +print ===> $data00 +if $data00 != $totalNum then + return -1 +endi + +print =============== step8 +# TODO +# sql select count(tbcol) as c from $mt where ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 50 then +# return -1 +# endi +# +# sql select count(tbcol) as c from $mt where tgcol < 5 +# print ===> $data00 +# if $data00 != 100 then +# return -1 +# endi +# +# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 25 then +# return -1 +# endi + +print =============== step9 +# TODO : count from stable +# sql select count(tbcol) as b from $mt interval(1m) +# print select count(tbcol) as b from $mt interval(1m) ===> $data01 +# if $data01 != 10 then +# return -1 +# endi + +# sql select count(tbcol) as b from $mt interval(1d) +# print ===> $data02 +# if $data01 != 200 then +# return -1 +# endi + +print =============== step10 +# TODO +# print select count(tbcol) as b from $mt group by tgcol +# sql select count(tbcol) as b from $mt group by tgcol +# print ===> $data00 +# if $data00 != $rowNum then +# return -1 +# endi + +# if $rows != $tbNum then +# return -1 +# endi +# +print =============== step11 +# TODO : where condition +# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol +# print ===> $data01 +# if $data01 != 1 then +# return -1 +# endi +# if $rows != 50 then +# return -1 +# endi + +print =============== clear +sql drop database $db +sql show databases +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/tsim/stable/dnode3.sim b/tests/script/tsim/stable/dnode3.sim new file mode 100644 index 0000000000..c2243b1ac8 --- /dev/null +++ b/tests/script/tsim/stable/dnode3.sim @@ -0,0 +1,213 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode2 -c walLevel -v 1 +system sh/cfg.sh -n dnode3 -c walLevel -v 1 +system sh/cfg.sh -n dnode4 -c walLevel -v 1 +# system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +# system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4 +# system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4 +# system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sql connect + +sql create dnode $hostname PORT 7200 +sql create dnode $hostname PORT 7300 +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +$x = 0 +createDnode: + $x = $x + 1 + sleep 1000 + if $x == 20 then + return -1 + endi +sql show dnodes; +if $data4_2 == offline then + goto createDnode +endi +if $data4_3 == offline then + goto createDnode +endi +if $data4_4 == offline then + goto createDnode +endi + +print ======================== dnode1 start + +$dbPrefix = r3v3_db +$tbPrefix = r3v3_tb +$mtPrefix = r3v3_mt +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql create database $db +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $val = $x * 60000 + $ms = 1519833600000 + $val + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql show vgroups +print vgroups ==> $rows +if $rows != 2 then + return -1 +endi + +sleep 100 + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select count(*) from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +sql select count(tbcol) from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step3 +# TODO : where condition +# sql select count(tbcol) from $tb where ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 5 then +# return -1 +# endi + +print =============== step4 +sql select count(tbcol) as b from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step5 +sql select count(tbcol) as b from $tb interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi + +sql select count(tbcol) as b from $tb interval(1d) +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step6 +# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +# print ===> $data00 +# if $data00 != 1 then +# return -1 +# endi +# if $rows != 5 then +# return -1 +# endi + +print =============== step7 +# print select count(*) from $mt +# sql select count(*) from $mt +# print ===> $data00 +# if $data00 != $totalNum then +# return -1 +# endi +# +# sql select count(tbcol) from $mt +# print ===> $data00 +# if $data00 != $totalNum then +# return -1 +# endi + +print =============== step8 +# sql select count(tbcol) as c from $mt where ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 50 then +# return -1 +# endi +# +# sql select count(tbcol) as c from $mt where tgcol < 5 +# print ===> $data00 +# if $data00 != 100 then +# return -1 +# endi +# +# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 25 then +# return -1 +# endi + +print =============== step9 +# TODO : group by in stable +# sql select count(tbcol) as b from $mt interval(1m) +# print ===> $data00 +# if $data00 != 10 then +# return -1 +# endi +# +# sql select count(tbcol) as b from $mt interval(1d) +# print ===> $data00 +# if $data00 != 200 then +# return -1 +# endi + +print =============== step10 +# sql select count(tbcol) as b from $mt group by tgcol +# print ===> $data00 +# if $data00 != $rowNum then +# return -1 +# endi +# +# if $rows != $tbNum then +# return -1 +# endi + +print =============== step11 +# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol +# print ===> $data00 +# if $data00 != 1 then +# return -1 +# endi +# if $rows != 50 then +# return -1 +# endi + +print =============== clear +sql drop database $db +sql show databases +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/stable/metrics.sim b/tests/script/tsim/stable/metrics.sim similarity index 64% rename from tests/script/general/stable/metrics.sim rename to tests/script/tsim/stable/metrics.sim index a3dca3f1a5..948af72d09 100644 --- a/tests/script/general/stable/metrics.sim +++ b/tests/script/tsim/stable/metrics.sim @@ -24,14 +24,14 @@ sql use $db sql create table $mt (ts timestamp, speed int) TAGS(sp int) sql show stables -if $rows != 1 then +if $rows != 1 then return -1 endi print =============== step2 sql drop table $mt sql show stables -if $rows != 0 then +if $rows != 0 then return -1 endi @@ -39,97 +39,98 @@ print =============== step3 sql create table $mt (ts timestamp, speed int) TAGS(sp int) sql show stables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $mt then +if $data00 != $mt then return -1 endi -if $data04 != 0 then - return -1 -endi -sql select * from $mt -if $rows != 0 then +if $data04 != 1 then return -1 endi +# TODO : select * from stable +# sql select * from $mt +# if $rows != 0 then +# return -1 +# endi print =============== step4 $i = 0 $tb = $tbPrefix . $i sql create table $tb using $mt tags(1) -$i = 1 +$i = 1 $tb = $tbPrefix . $i sql create table $tb using $mt tags(2) -$i = 2 +$i = 2 $tb = $tbPrefix . $i sql create table $tb using $mt tags(3) sql show tables -if $rows != 3 then - return -1 -endi -if $data03 != $mt then +if $rows != 3 then return -1 endi +# if $data03 != $mt then +# return -1 +# endi sql show stables -if $rows != 1 then +if $rows != 1 then return -1 endi -if $data00 != $mt then - return -1 -endi -if $data04 != 3 then +if $data00 != $mt then return -1 endi +# if $data04 != 3 then +# return -1 +# endi print =============== step5 $i = 0 $tb = $tbPrefix . $i -sql insert into $tb values (now + 1m , 1 ) +sql insert into $tb values (now + 1m , 1 ) $i = 1 $tb = $tbPrefix . $i -sql insert into $tb values (now + 1m , 1 ) +sql insert into $tb values (now + 1m , 1 ) $i = 2 $tb = $tbPrefix . $i -sql insert into $tb values (now + 1m , 1 ) +sql insert into $tb values (now + 1m , 1 ) print sleep 8000 sleep 8000 print =============== step6 -sql select * from $mt -print select * from $mt ==> $rows $data00 -if $rows != 3 then - return -1 -endi +# sql select * from $mt +# print select * from $mt ==> $rows $data00 +# if $rows != 3 then +# return -1 +# endi print =============== step7 -sql select * from $mt where sp = 1 -print select * from $mt where sp = 1 ==> $rows $data00 -if $rows != 1 then - return -1 -endi +# sql select * from $mt where sp = 1 +# print select * from $mt where sp = 1 ==> $rows $data00 +# if $rows != 1 then +# return -1 +# endi print =============== step8 sql drop table $mt print =============== step9 -sql show tables -if $rows != 0 then - return -1 -endi +#sql show tables +#if $rows != 0 then +# return -1 +#endi sql show stables -if $rows != 0 then +if $rows != 0 then return -1 endi sql drop database $db sql show databases -if $rows != 0 then +if $rows != 1 then return -1 endi diff --git a/tests/script/general/stable/refcount.sim b/tests/script/tsim/stable/refcount.sim similarity index 79% rename from tests/script/general/stable/refcount.sim rename to tests/script/tsim/stable/refcount.sim index 6629dc1177..1f00483090 100644 --- a/tests/script/general/stable/refcount.sim +++ b/tests/script/tsim/stable/refcount.sim @@ -16,11 +16,11 @@ sql create table d1.t2 (ts timestamp, i int); sql create table d1.t3 (ts timestamp, i int); sql insert into d1.t1 values(now, 1); sql insert into d1.t2 values(now, 1); -sql drop table d1.t1; +# sql drop table d1.t1; sql drop database d1; sql show databases; -if $rows != 0 then +if $rows != 1 then return -1 endi @@ -32,24 +32,24 @@ sql create table d2.t2 (ts timestamp, i int); sql create table d2.t3 (ts timestamp, i int); sql insert into d2.t1 values(now, 1); sql insert into d2.t2 values(now, 1); -sql drop table d2.t1; -sql drop table d2.t2; -sql drop table d2.t3; - -sql show d2.tables; -if $rows != 0 then - return -1 -endi +# sql drop table d2.t1; +# sql drop table d2.t2; +# sql drop table d2.t3; +# +# sql show d2.tables; +# if $rows != 0 then +# return -1 +# endi sql show d2.vgroups; -if $rows != 0 then +if $rows != 2 then return -1 endi sql drop database d2; sql show databases; -if $rows != 0 then +if $rows != 1 then return -1 endi @@ -61,24 +61,24 @@ sql create table d3.t1 using d3.st tags(1); sql create table d3.t2 using d3.st tags(1); sql create table d3.t3 using d3.st tags(1); sql insert into d3.t1 values(now, 1); -sql drop table d3.t1; -sql drop table d3.t2; -sql drop table d3.t3; - -sql show d3.tables; -if $rows != 0 then - return -1 -endi +# sql drop table d3.t1; +# sql drop table d3.t2; +# sql drop table d3.t3; +# +# sql show d3.tables; +# if $rows != 0 then +# return -1 +# endi sql show d3.vgroups; -if $rows != 0 then +if $rows != 2 then return -1 endi sql drop database d3; sql show databases; -if $rows != 0 then +if $rows != 1 then return -1 endi @@ -90,23 +90,23 @@ sql create table d4.t1 using d4.st tags(1); sql create table d4.t2 using d4.st tags(1); sql create table d4.t3 using d4.st tags(1); sql insert into d4.t1 values(now, 1); -sql drop table d4.t1; +# sql drop table d4.t1; sql drop table d4.st; - -sql show d4.tables; -if $rows != 0 then - return -1 -endi +# +# sql show d4.tables; +# if $rows != 0 then +# return -1 +# endi sql show d4.stables; -if $rows != 0 then +if $rows != 0 then return -1 endi sql drop database d4; sql show databases; -if $rows != 0 then +if $rows != 1 then return -1 endi @@ -118,12 +118,12 @@ sql create table d5.t1 using d5.st tags(1); sql create table d5.t2 using d5.st tags(1); sql create table d5.t3 using d5.st tags(1); sql insert into d5.t1 values(now, 1); -sql drop table d5.t1; +# sql drop table d5.t1; sql drop database d5; sql show databases; -if $rows != 0 then +if $rows != 1 then return -1 endi diff --git a/tests/script/general/stable/show.sim b/tests/script/tsim/stable/show.sim similarity index 63% rename from tests/script/general/stable/show.sim rename to tests/script/tsim/stable/show.sim index 5fe05b41eb..8ebb765a78 100644 --- a/tests/script/general/stable/show.sim +++ b/tests/script/tsim/stable/show.sim @@ -10,20 +10,37 @@ sql connect print ======================== create stable sql create database d1 +sql use d1 $x = 0 while $x < 128 $tb = d1.s . $x sql create table $tb (ts timestamp, i int) tags (j int) $x = $x + 1 -endw +endw + +print ======================== describe stables +# TODO : create stable error +$m = 0 +while $m < 128 + $tb = s . $m + $filter = ' . $tb + $filter = $filter . ' + sql show stables like $filter + # print sql : show stables like $filter ==> $rows + if $rows != 1 then + return -1 + endi + $m = $m + 1 +endw + print ======================== show stables sql show d1.stables print num of stables is $rows -if $rows != 128 then +if $rows != 128 then return -1 endi @@ -34,15 +51,15 @@ while $x < 424 $tb = d1.t . $x sql create table $tb using d1.s0 tags( $x ) $x = $x + 1 -endw +endw print ======================== show stables sql show d1.tables print num of tables is $rows -if $rows != 424 then +if $rows != 424 then return -1 endi -system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stable/values.sim b/tests/script/tsim/stable/values.sim new file mode 100644 index 0000000000..e5e3118e12 --- /dev/null +++ b/tests/script/tsim/stable/values.sim @@ -0,0 +1,121 @@ +system sh/stop_dnodes.sh + + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +print ======================== dnode1 start + +sql create database vdb0 +sql create table vdb0.mt (ts timestamp, tbcol int) TAGS(tgcol int) + +sql create table vdb0.vtb00 using vdb0.mt tags( 0 ) +sql create table vdb0.vtb01 using vdb0.mt tags( 0 ) + +sql create database vdb1 +sql create table vdb1.mt (ts timestamp, tbcol int) TAGS(tgcol int) +# sql_error create table vdb1.vtb10 using vdb0.mt tags( 1 ) +# sql_error create table vdb1.vtb11 using vdb0.mt tags( 1 ) +sql create table vdb1.vtb10 using vdb1.mt tags( 1 ) +sql create table vdb1.vtb11 using vdb1.mt tags( 1 ) + +sql create database vdb2 +sql create table vdb2.mt (ts timestamp, tbcol int) TAGS(tgcol int) +# sql_error create table vdb2.vtb20 using vdb0.mt tags( 2 ) +# sql_error create table vdb2.vtb21 using vdb0.mt tags( 2 ) +sql create table vdb2.vtb20 using vdb2.mt tags( 2 ) +sql create table vdb2.vtb21 using vdb2.mt tags( 2 ) + +sql create database vdb3 +sql create table vdb3.mt (ts timestamp, tbcol int) TAGS(tgcol int) +# sql_error create table vdb3.vtb20 using vdb0.mt tags( 2 ) +# sql_error create table vdb3.vtb21 using vdb0.mt tags( 2 ) +sql create table vdb3.vtb30 using vdb3.mt tags( 3 ) +sql create table vdb3.vtb31 using vdb3.mt tags( 3 ) + +print =============== step2 +sql insert into vdb0.vtb00 values (1519833600000 , 10) (1519833600001, 20) (1519833600002, 30) +sql insert into vdb0.vtb01 values (1519833600000 , 10) (1519833600001, 20) (1519833600002, 30) +sql insert into vdb1.vtb10 values (1519833600000 , 11) (1519833600001, 21) (1519833600002, 31) +sql insert into vdb1.vtb11 values (1519833600000 , 11) (1519833600001, 21) (1519833600002, 31) +sql insert into vdb2.vtb20 values (1519833600000 , 12) (1519833600001, 22) (1519833600002, 32) +sql insert into vdb2.vtb21 values (1519833600000 , 12) (1519833600001, 22) (1519833600002, 32) +sql insert into vdb3.vtb30 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) +sql insert into vdb3.vtb31 values (1519833600000 , 13) (1519833600001, 23) (1519833600002, 33) +# sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 6 then + return -1 +endi + +print =============== step3 +sql insert into vdb0.vtb00 values (1519833600003 , 40) (1519833600005, 50) (1519833600004, 60) +sql insert into vdb0.vtb01 values (1519833600003 , 40) (1519833600005, 50) (1519833600004, 60) +sql insert into vdb1.vtb10 values (1519833600003 , 41) (1519833600005, 51) (1519833600004, 61) +sql insert into vdb1.vtb11 values (1519833600003 , 41) (1519833600005, 51) (1519833600004, 61) +sql insert into vdb2.vtb20 values (1519833600003 , 42) (1519833600005, 52) (1519833600004, 62) +sql insert into vdb2.vtb21 values (1519833600003 , 42) (1519833600005, 52) (1519833600004, 62) +sql insert into vdb3.vtb30 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) +sql insert into vdb3.vtb31 values (1519833600003 , 43) (1519833600005, 53) (1519833600004, 63) +# TODO : select * from stable +# sql select * from vdb0.mt +sql select ts from vdb0.mt + +if $rows != 12 then + return -1 +endi + +print =============== step4 +# TODO : insert into diffrent table +# sql insert into vdb0.vtb00 values(1519833600006, 60) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 60) (1519833600007, 70) +# sql insert into vdb1.vtb10 values(1519833600006, 61) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 61) (1519833600007, 71) +# sql insert into vdb2.vtb20 values(1519833600006, 62) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 62) (1519833600007, 72) +# sql insert into vdb3.vtb30 values(1519833600006, 63) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 63) (1519833600007, 73) +# # sql select * from vdb0.mt +# sql select ts from vdb0.mt +# +# if $rows != 16 then +# return -1 +# endi + +print =============== step5 +# sql insert into vdb0.vtb00 values(1519833600008, 80) (1519833600007, 70) vdb0.vtb01 values(1519833600006, 80) (1519833600007, 70) +# sql insert into vdb1.vtb10 values(1519833600008, 81) (1519833600007, 71) vdb1.vtb11 values(1519833600006, 81) (1519833600007, 71) +# sql insert into vdb2.vtb20 values(1519833600008, 82) (1519833600007, 72) vdb2.vtb21 values(1519833600006, 82) (1519833600007, 72) +# sql insert into vdb3.vtb30 values(1519833600008, 83) (1519833600007, 73) vdb3.vtb31 values(1519833600006, 83) (1519833600007, 73) +# # sql select * from vdb0.mt +# sql select ts from vdb0.mt +# +# if $rows != 17 then +# return -1 +# endi + +print =============== step6 +# sql insert into vdb0.vtb00 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb10 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb20 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb30 values(1519833600009, 90) (1519833600010, 100) +# sql insert into vdb0.vtb01 values(1519833600009, 90) (1519833600010, 100) vdb1.vtb11 values(1519833600009, 90) (1519833600010, 100) vdb2.vtb21 values(1519833600009, 90) (1519833600010, 100) vdb3.vtb31 values(1519833600009, 90) (1519833600010, 100) +# +# # sql select * from vdb0.mt +# sql select ts from vdb0.mt +# +# if $rows != 21 then +# return -1 +# endi + +print =============== step7 +# sql insert into vdb0.vtb00 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb10 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb20 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb30 values(1519833600012, 120) (1519833600011, 110) +# sql insert into vdb0.vtb01 values(1519833600012, 120) (1519833600011, 110) vdb1.vtb11 values(1519833600012, 120) (1519833600011, 110) vdb2.vtb21 values(1519833600012, 120) (1519833600011, 110) vdb3.vtb31 values(1519833600012, 120) (1519833600011, 110) +# +# # sql select * from vdb0.mt +# sql select ts from vdb0.mt +# +# if $rows != 25 then +# return -1 +# endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stable/vnode3.sim b/tests/script/tsim/stable/vnode3.sim new file mode 100644 index 0000000000..2d408b4191 --- /dev/null +++ b/tests/script/tsim/stable/vnode3.sim @@ -0,0 +1,181 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 1 +system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4 +system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 4 +system sh/exec.sh -n dnode1 -s start + +sleep 2000 +sql connect + +print ======================== dnode1 start + +$dbPrefix = v3_db +$tbPrefix = v3_tb +$mtPrefix = v3_mt +$tbNum = 10 +$rowNum = 20 +$totalNum = 200 + +print =============== step1 +$i = 0 +$db = $dbPrefix . $i +$mt = $mtPrefix . $i + +sql create database $db +sql use $db +sql create table $mt (ts timestamp, tbcol int) TAGS(tgcol int) + +$i = 0 +while $i < $tbNum + $tb = $tbPrefix . $i + sql create table $tb using $mt tags( $i ) + + $x = 0 + while $x < $rowNum + $val = $x * 60000 + $ms = 1519833600000 + $val + sql insert into $tb values ($ms , $x ) + $x = $x + 1 + endw + + $i = $i + 1 +endw + +sql show vgroups +print vgroups ==> $rows +if $rows != 2 then + return -1 +endi + + +print =============== step2 +$i = 1 +$tb = $tbPrefix . $i + +sql select count(*) from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +sql select count(tbcol) from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step3 +# TODO : where condition +# sql select count(tbcol) from $tb where ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 5 then +# return -1 +# endi + +print =============== step4 +sql select count(tbcol) as b from $tb +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step5 +sql select count(tbcol) as b from $tb interval(1m) +print ===> $data00 +if $data00 != 1 then + return -1 +endi + +sql select count(tbcol) as b from $tb interval(1d) +print ===> $data00 +if $data00 != $rowNum then + return -1 +endi + +print =============== step6 +# sql select count(tbcol) as b from $tb where ts <= 1519833840000 interval(1m) +# print ===> $data00 +# if $data00 != 1 then +# return -1 +# endi +# if $rows != 5 then +# return -1 +#endi + +print =============== step7 +# TODO : count(*) err +# sql select count(*) from $mt +# print ===> $data00 +# if $data00 != $totalNum then +# return -1 +# endi +# +# sql select count(tbcol) from $mt +# print ===> $data00 +# if $data00 != $totalNum then +# return -1 +# endi + +print =============== step8 +# sql select count(tbcol) as c from $mt where ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 50 then +# return -1 +# endi + +# sql select count(tbcol) as c from $mt where tgcol < 5 +# print ===> $data00 +# if $data00 != 100 then +# return -1 +# endi + +# sql select count(tbcol) as c from $mt where tgcol < 5 and ts <= 1519833840000 +# print ===> $data00 +# if $data00 != 25 then +# return -1 +# endi + +print =============== step9 +# sql select count(tbcol) as b from $mt interval(1m) +# print ===> $data00 +# if $data00 != 10 then +# return -1 +# endi +# +# sql select count(tbcol) as b from $mt interval(1d) +# print ===> $data00 +# if $data00 != 200 then +# return -1 +# endi + +print =============== step10 +# sql select count(tbcol) as b from $mt group by tgcol +# print ===> $data00 +# if $data00 != $rowNum then +# return -1 +# endi +# +# if $rows != $tbNum then +# return -1 +# endi + +print =============== step11 +# sql select count(tbcol) as b from $mt where ts <= 1519833840000 interval(1m) group by tgcol +# print ===> $data01 +# if $data01 != 1 then +# return -1 +# endi +# if $rows != 50 then +# return -1 +# endi + +print =============== clear +sql drop database $db +sql show databases +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/testCaseSuite.sim b/tests/script/tsim/testCaseSuite.sim index bf184f8794..4245529343 100644 --- a/tests/script/tsim/testCaseSuite.sim +++ b/tests/script/tsim/testCaseSuite.sim @@ -1,6 +1,7 @@ run tsim/user/basic1.sim +run tsim/db/alter_option.sim run tsim/db/basic1.sim run tsim/db/basic2.sim run tsim/db/basic3.sim @@ -20,6 +21,7 @@ run tsim/insert/null.sim run tsim/query/interval.sim run tsim/query/interval-offset.sim +run tsim/query/scalarFunction.sim run tsim/show/basic.sim diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index fe6a7a0660..cfcb1ac992 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -66,15 +66,15 @@ print =============== will support: * from stb; function from stb/ctb sql create topic topic_stb_column as select ts, c1, c3 from stb #sql create topic topic_stb_all as select * from stb -#sql create topic topic_stb_function as select ts, abs(c1), sina(c2) from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb sql create topic topic_ctb_column as select ts, c1, c3 from ct0 sql create topic topic_ctb_all as select * from ct0 -#sql create topic topic_ctb_function as select ts, abs(c1), sina(c2) from ct0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ct0 sql create topic topic_ntb_column as select ts, c1, c3 from ntb sql create topic topic_ntb_all as select * from ntb -#sql create topic topic_ntb_function as select ts, abs(c1), sina(c2) from ntb +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb sql show tables if $rows != 3 then @@ -147,6 +147,13 @@ endi # return -1 #endi +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_stb_function" -k "group.id:tg2" +print cmd result----> $system_content +if $system_content != @{consume success: 20, 0}@ then + return -1 +endi + print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_column" -k "group.id:tg2" print cmd result----> $system_content @@ -161,6 +168,13 @@ if $system_content != @{consume success: 10, 0}@ then return -1 endi +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ctb_function" -k "group.id:tg2" +print cmd result----> $system_content +if $system_content != @{consume success: 10, 0}@ then + return -1 +endi + print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_column" -k "group.id:tg2" print cmd result----> $system_content @@ -175,6 +189,13 @@ if $system_content != @{consume success: 20, 0}@ then return -1 endi +print cmd===> system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" +system_content ../../debug/tests/test/c/tmq_sim -c ../../sim/tsim/cfg -d $dbNamme -t "topic_ntb_function" -k "group.id:tg2" +print cmd result----> $system_content +if $system_content != @{consume success: 20, 0}@ then + return -1 +endi + print =============== create database , vgroup 4 $dbNamme = d1 sql create database $dbNamme vgroups 4 diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 4d3108500e..78f1c7e015 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -226,7 +226,7 @@ void loop_consume(tmq_t* tmq) { int32_t totalRows = 0; int32_t skipLogNum = 0; while (running) { - tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 1); + tmq_message_t* tmqMsg = tmq_consumer_poll(tmq, 6000); if (tmqMsg) { totalMsgs++; diff --git a/tools/taos-tools b/tools/taos-tools index f36b07f710..33cdfe4f90 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit f36b07f710d661dca88fdd70e73b5e3e16a960e0 +Subproject commit 33cdfe4f90a209f105c1b6091439798a9cde1e93