diff --git a/cmake/version.inc b/cmake/version.inc index 8c02520142..ad399bfa86 100644 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -1,42 +1,65 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8) PROJECT(TDengine) -SET(TD_VER_1 "2") -SET(TD_VER_2 "0") -SET(TD_VER_3 "2") -SET(TD_VER_4 "3") -SET(TD_VER_GIT "d711657139620f6c50f362597020705b8ad26bd2") -SET(TD_VER_GIT_INTERNAL "1d74ae24c541ffbb280e8630883c0236cd45f8c7") +IF (DEFINED VERNUMBER) + SET(TD_VER_NUMBER ${VERNUMBER}) +ELSE () + SET(TD_VER_NUMBER "2.0.2.0") +ENDIF () -SET(TD_VER_VERTYPE "stable") -SET(TD_VER_CPUTYPE "x64") -SET(TD_VER_OSTYPE "Linux") +IF (DEFINED VERCOMPATIBLE) + SET(TD_VER_COMPATIBLE ${VERCOMPATIBLE}) +ELSE () + SET(TD_VER_COMPATIBLE "2.0.0.0") +ENDIF () -SET(TD_VER_COMPATIBLE "2.0.0.0") -STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") +IF (DEFINED GITINFO) + SET(TD_VER_GIT ${GITINFO}) +ELSE () + SET(TD_VER_GIT "community") +ENDIF () -IF (TD_LINUX_64) +IF (DEFINED GITINFOI) + SET(TD_VER_GIT_INTERNAL ${GITINFOI}) +ELSE () + SET(TD_VER_GIT_INTERNAL "internal") +ENDIF () + +IF (DEFINED VERDATE) + SET(TD_VER_DATE ${VERDATE}) +ELSE () + STRING(TIMESTAMP TD_VER_DATE "%Y-%m-%d %H:%M:%S") +ENDIF () + +IF (DEFINED VERTYPE) + SET(TD_VER_VERTYPE ${VERTYPE}) +ELSE () + SET(TD_VER_VERTYPE "stable") +ENDIF () + +IF (DEFINED CPUTYPE) + SET(TD_VER_CPUTYPE ${CPUTYPE}) +ELSE () SET(TD_VER_CPUTYPE "x64") ENDIF () -IF (TD_LINUX_32) - SET(TD_VER_CPUTYPE "x86") +IF (DEFINED OSTYPE) + SET(TD_VER_OSTYPE ${OSTYPE}) +ELSE () + SET(TD_VER_OSTYPE "Linux") ENDIF () -IF (TD_ARM_64) - SET(TD_VER_CPUTYPE "aarch64") -ENDIF () +MESSAGE(STATUS "============= compile version parameter information start ============= ") +MESSAGE(STATUS "ver number:" ${TD_VER_NUMBER}) +MESSAGE(STATUS "compatible ver number:" ${TD_VER_COMPATIBLE}) +MESSAGE(STATUS "communit commit id:" ${TD_VER_GIT}) +MESSAGE(STATUS "internal commit id:" ${TD_VER_GIT_INTERNAL}) +MESSAGE(STATUS "build date:" ${TD_VER_DATE}) +MESSAGE(STATUS "ver type:" ${TD_VER_VERTYPE}) +MESSAGE(STATUS "ver cpu:" ${TD_VER_CPUTYPE}) +MESSAGE(STATUS "os type:" ${TD_VER_OSTYPE}) +MESSAGE(STATUS "============= compile version parameter information end ============= ") -IF (TD_ARM_32) - SET(TD_VER_CPUTYPE "aarch32") -ENDIF () - -IF (TD_WINDOWS_64) - SET(TD_VER_CPUTYPE "x64") -ENDIF () - -IF (TD_WINDOWS_32) - SET(TD_VER_CPUTYPE "x86") -ENDIF () +STRING(REPLACE "." "_" TD_LIB_VER_NUMBER ${TD_VER_NUMBER}) CONFIGURE_FILE("${TD_COMMUNITY_DIR}/src/util/src/version.c.in" "${TD_COMMUNITY_DIR}/src/util/src/version.c") diff --git a/documentation20/webdocs/markdowndocs/administrator-ch.md b/documentation20/webdocs/markdowndocs/administrator-ch.md index c5ecb862e3..64cadf69cd 100644 --- a/documentation20/webdocs/markdowndocs/administrator-ch.md +++ b/documentation20/webdocs/markdowndocs/administrator-ch.md @@ -153,10 +153,10 @@ TDengine系统的前台交互客户端应用程序为taos,它与taosd共享同 系统管理员可以在CLI界面里添加、删除用户,也可以修改密码。CLI里SQL语法如下: ``` -CREATE USER PASS <‘password’>; +CREATE USER PASS <'password'>; ``` -创建用户,并指定用户名和密码,密码需要用单引号引起来 +创建用户,并指定用户名和密码,密码需要用单引号引起来,单引号为英文半角 ``` DROP USER ; @@ -165,10 +165,10 @@ DROP USER ; 删除用户,限root用户使用 ``` -ALTER USER PASS <‘password’>; +ALTER USER PASS <'password'>; ``` -修改用户密码, 为避免被转换为小写,密码需要用单引号引用 +修改用户密码, 为避免被转换为小写,密码需要用单引号引用,单引号为英文半角 ``` SHOW USERS; diff --git a/documentation20/webdocs/markdowndocs/architecture-ch.md b/documentation20/webdocs/markdowndocs/architecture-ch.md index bfe3b55bd2..7ab4b5d096 100644 --- a/documentation20/webdocs/markdowndocs/architecture-ch.md +++ b/documentation20/webdocs/markdowndocs/architecture-ch.md @@ -82,7 +82,7 @@ TDengine 分布式架构的逻辑结构图如下: ### 节点之间的通讯 **通讯方式:**TDengine系统的各个节点之间的通讯是通过TCP/UDP进行的。因为考虑到物联网场景,数据写入的包一般不大,因此TDengine 除采用TCP做传输之外,还采用UDP方式,因为UDP 更加高效,而且不受连接数的限制。TDengine实现了自己的超时、重传、确认等机制,以确保UDP的可靠传输。对于数据量不到15K的数据包,采取UDP的方式进行传输,超过15K的,或者是查询类的操作,自动采取TCP的方式进行传输。同时,TDengine根据配置和数据包,会自动对数据进行压缩/解压缩,数字签名/认证等处理。对于数据节点之间的数据复制,只采用TCP方式进行数据传输。 -**FQDN配置**:一个数据节点有一个或多个FQDN,可以在系统配置文件taos.cfg通过选项“fqdn"进行指定,如果没有指定,系统将自动获取FQDN。如果节点没有配置FQDN,可以直接使用IP地址作为FQDN,但不建议使用,因为IP地址可变,一旦变化,将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。 +**FQDN配置**:一个数据节点有一个或多个FQDN,可以在系统配置文件taos.cfg通过参数“fqdn"进行指定,如果没有指定,系统将自动获取FQDN。如果节点没有配置FQDN,可以直接将该节点的配置参数fqdn设置为它的IP地址。但不建议使用IP,因为IP地址可变,一旦变化,将让集群无法正常工作。一个数据节点的EP(End Point)由FQDN + Port组成。采用FQDN,需要保证DNS服务正常工作,或者在节点以及应用所在的节点配置好hosts文件。 **端口配置:**一个数据节点对外的端口由TDengine的系统配置参数serverPort决定,对集群内部通讯的端口是serverPort+5。集群内数据节点之间的数据复制操作还占有一个TCP端口,是serverPort+10. 为支持多线程高效的处理UDP数据,每个对内和对外的UDP链接,都需要占用5个连续的端口。因此一个数据节点总的端口范围为serverPort到serverPort + 10,总共11个TCP/UDP端口。使用时,需要确保防火墙将这些端口打开。每个数据节点可以配置不同的serverPort。 diff --git a/packaging/release.sh b/packaging/release.sh index bf355bdcd4..3c0378042d 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -12,6 +12,7 @@ set -e # -l [full | lite] # -s [static | dynamic] # -n [2.0.0.3] +# -m [2.0.0.0] # set parameters by default value verMode=edge # [cluster, edge] @@ -21,8 +22,9 @@ osType=Linux # [Linux | Kylin | Alpine | Raspberrypi | Darwin | Windows | Ni pagMode=full # [full | lite] soMode=dynamic # [static | dynamic] verNumber="" +verNumberComp="2.0.0.0" -while getopts "hv:V:c:o:l:s:n:" arg +while getopts "hv:V:c:o:l:s:n:m:" arg do case $arg in v) @@ -49,6 +51,10 @@ do #echo "verNumber=$OPTARG" verNumber=$(echo $OPTARG) ;; + m) + #echo "verNumberComp=$OPTARG" + verNumberComp=$(echo $OPTARG) + ;; o) #echo "osType=$OPTARG" osType=$(echo $OPTARG) @@ -61,6 +67,7 @@ do echo " -l [full | lite] " echo " -s [static | dynamic] " echo " -n [version number] " + echo " -m [compatible version number] " exit 0 ;; ?) #unknow option @@ -70,216 +77,142 @@ do esac done -echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber}" +echo "verMode=${verMode} verType=${verType} cpuType=${cpuType} osType=${osType} pagMode=${pagMode} soMode=${soMode} verNumber=${verNumber} verNumberComp=${verNumberComp}" curr_dir=$(pwd) if [ "$osType" != "Darwin" ]; then - script_dir="$(dirname $(readlink -f $0))" - top_dir="$(readlink -f ${script_dir}/..)" + script_dir="$(dirname $(readlink -f $0))" + top_dir="$(readlink -f ${script_dir}/..)" else - script_dir=`dirname $0` - cd ${script_dir} - script_dir="$(pwd)" - top_dir=${script_dir}/.. + script_dir=`dirname $0` + cd ${script_dir} + script_dir="$(pwd)" + top_dir=${script_dir}/.. fi -versioninfo="${top_dir}/src/util/src/version.c" - csudo="" #if command -v sudo > /dev/null; then -# csudo="sudo" +# csudo="sudo" #fi function is_valid_version() { - [ -z $1 ] && return 1 || : + [ -z $1 ] && return 1 || : - rx='^([0-9]+\.){3}(\*|[0-9]+)$' - if [[ $1 =~ $rx ]]; then - return 0 - fi - return 1 + rx='^([0-9]+\.){3}(\*|[0-9]+)$' + if [[ $1 =~ $rx ]]; then + return 0 + fi + return 1 } function vercomp () { - if [[ $1 == $2 ]]; then - echo 0 - exit 0 - fi - - local IFS=. - local i ver1=($1) ver2=($2) - - # fill empty fields in ver1 with zeros - for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do - ver1[i]=0 - done - - for ((i=0; i<${#ver1[@]}; i++)); do - if [[ -z ${ver2[i]} ]]; then - # fill empty fields in ver2 with zeros - ver2[i]=0 - fi - if ((10#${ver1[i]} > 10#${ver2[i]})); then - echo 1 - exit 0 - fi - if ((10#${ver1[i]} < 10#${ver2[i]})); then - echo 2 - exit 0 - fi - done + if [[ $1 == $2 ]]; then echo 0 -} + exit 0 + fi + + local IFS=. + local i ver1=($1) ver2=($2) -# 1. Read version information -version=$(cat ${versioninfo} | grep " version" | cut -d '"' -f2) -compatible_version=$(cat ${versioninfo} | grep " compatible_version" | cut -d '"' -f2) + # fill empty fields in ver1 with zeros + for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do + ver1[i]=0 + done -if [ -z ${verNumber} ]; then - while true; do - read -p "Do you want to release a new version? [y/N]: " is_version_change - - if [[ ( "${is_version_change}" == "y") || ( "${is_version_change}" == "Y") ]]; then - read -p "Please enter the new version: " tversion - while true; do - if (! is_valid_version $tversion) || [ "$(vercomp $tversion $version)" = '2' ]; then - read -p "Please enter a correct version: " tversion - continue - fi - version=${tversion} - break - done - - echo - - read -p "Enter the oldest compatible version: " tversion - while true; do - - if [ -z $tversion ]; then - break - fi - - if (! is_valid_version $tversion) || [ "$(vercomp $version $tversion)" = '2' ]; then - read -p "enter correct compatible version: " tversion - else - compatible_version=$tversion - break - fi - done - - break - elif [[ ( "${is_version_change}" == "n") || ( "${is_version_change}" == "N") ]]; then - echo "Use old version: ${version} compatible version: ${compatible_version}." - break - else - continue + for ((i=0; i<${#ver1[@]}; i++)); do + if [[ -z ${ver2[i]} ]]; then + # fill empty fields in ver2 with zeros + ver2[i]=0 + fi + if ((10#${ver1[i]} > 10#${ver2[i]})); then + echo 1 + exit 0 + fi + if ((10#${ver1[i]} < 10#${ver2[i]})); then + echo 2 + exit 0 fi done -else - echo "old version: $version, new version: $verNumber" - #if ( ! is_valid_version $verNumber ) || [[ "$(vercomp $version $verNumber)" == '2' ]]; then - # echo "please enter correct version" - # exit 0 - #else - version=${verNumber} - #fi -fi + echo 0 +} -echo "=======================new version number: ${version}======================================" +# 1. check version information +if (( ! is_valid_version $verNumber ) || ( ! is_valid_version $verNumberComp ) || [[ "$(vercomp $verNumber $verNumberComp)" == '2' ]]); then + echo "please enter correct version" + exit 0 +fi + +echo "=======================new version number: ${verNumber}, compatible version: ${verNumberComp}======================================" -# output the version info to the buildinfo file. build_time=$(date +"%F %R") -echo "char version[12] = \"${version}\";" > ${versioninfo} -echo "char compatible_version[12] = \"${compatible_version}\";" >> ${versioninfo} -echo "char gitinfo[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} -if [ "$verMode" != "cluster" ]; then - echo "char gitinfoOfInternal[48] = \"\";" >> ${versioninfo} -else - enterprise_dir="${top_dir}/../enterprise" - cd ${enterprise_dir} - echo "char gitinfoOfInternal[48] = \"$(git rev-parse --verify HEAD)\";" >> ${versioninfo} - cd ${curr_dir} -fi -echo "char buildinfo[64] = \"Built by ${USER} at ${build_time}\";" >> ${versioninfo} -echo "" >> ${versioninfo} -tmp_version=$(echo $version | tr -s "." "_") -if [ "$verMode" == "cluster" ]; then - libtaos_info=${tmp_version}_${osType}_${cpuType} -else - libtaos_info=edge_${tmp_version}_${osType}_${cpuType} -fi -if [ "$verType" == "beta" ]; then - libtaos_info=${libtaos_info}_${verType} -fi -echo "void libtaos_${libtaos_info}() {};" >> ${versioninfo} + +# get commint id from git +gitinfo=$(git rev-parse --verify HEAD) +enterprise_dir="${top_dir}/../enterprise" +cd ${enterprise_dir} +gitinfoOfInternal=$(git rev-parse --verify HEAD) +cd ${curr_dir} # 2. cmake executable file compile_dir="${top_dir}/debug" if [ -d ${compile_dir} ]; then - ${csudo} rm -rf ${compile_dir} + ${csudo} rm -rf ${compile_dir} fi if [ "$osType" != "Darwin" ]; then - ${csudo} mkdir -p ${compile_dir} + ${csudo} mkdir -p ${compile_dir} else - mkdir -p ${compile_dir} + mkdir -p ${compile_dir} fi cd ${compile_dir} # check support cpu type if [[ "$cpuType" == "x64" ]] || [[ "$cpuType" == "aarch64" ]] || [[ "$cpuType" == "aarch32" ]] || [[ "$cpuType" == "mips64" ]] ; then - if [ "$verMode" != "cluster" ]; then - cmake ../ -DCPUTYPE=${cpuType} -DPAGMODE=${pagMode} -DOSTYPE=${osType} -DSOMODE=${soMode} - else - cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} - fi + if [ "$verMode" != "cluster" ]; then + cmake ../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} -DPAGMODE=${pagMode} + else + cmake ../../ -DCPUTYPE=${cpuType} -DOSTYPE=${osType} -DSOMODE=${soMode} -DVERTYPE=${verType} -DVERDATE="${build_time}" -DGITINFO=${gitinfo} -DGITINFOI=${gitinfoOfInternal} -DVERNUMBER=${verNumber} -DVERCOMPATIBLE=${verNumberComp} + fi else - echo "input cpuType=${cpuType} error!!!" - exit 1 + echo "input cpuType=${cpuType} error!!!" + exit 1 fi make cd ${curr_dir} -# 3. judge the operating system type, then Call the corresponding script for packaging -#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -#osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) -#echo "osinfo: ${osinfo}" - +# 3. Call the corresponding script for packaging if [ "$osType" != "Darwin" ]; then - if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then - echo "====do deb package for the ubuntu system====" - output_dir="${top_dir}/debs" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/deb - ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} - - echo "====do rpm package for the centos system====" - output_dir="${top_dir}/rpms" - if [ -d ${output_dir} ]; then - ${csudo} rm -rf ${output_dir} - fi - ${csudo} mkdir -p ${output_dir} - cd ${script_dir}/rpm - ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${version} ${cpuType} ${osType} ${verMode} ${verType} + if [[ "$verMode" != "cluster" ]] && [[ "$cpuType" == "x64" ]]; then + echo "====do deb package for the ubuntu system====" + output_dir="${top_dir}/debs" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/deb + ${csudo} ./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + + echo "====do rpm package for the centos system====" + output_dir="${top_dir}/rpms" + if [ -d ${output_dir} ]; then + ${csudo} rm -rf ${output_dir} + fi + ${csudo} mkdir -p ${output_dir} + cd ${script_dir}/rpm + ${csudo} ./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} + fi - echo "====do tar.gz package for all systems====" - cd ${script_dir}/tools - - ${csudo} ./makepkg.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} - ${csudo} ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} - ${csudo} ./makearbi.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + echo "====do tar.gz package for all systems====" + cd ${script_dir}/tools + + ${csudo} ./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo} ./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else - cd ${script_dir}/tools - ./makeclient.sh ${compile_dir} ${version} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} + cd ${script_dir}/tools + ./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} fi -# 4. Clean up temporary compile directories -#${csudo} rm -rf ${compile_dir} - diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index f38677f7ac..1cbb3ead9c 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -705,6 +705,7 @@ function install_TDengine() { echo echo -e "\033[44;32;1mTDengine client is installed successfully!${NC}" fi + touch ~/.taos_history rm -rf $(tar -tf taos.tar.gz) } diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 397a60d140..4e579b0729 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -87,7 +87,6 @@ typedef struct SRetrieveSupport { SSqlObj * pParentSql; tFilePage * localBuffer; // temp buffer, there is a buffer for each vnode to uint32_t numOfRetry; // record the number of retry times - pthread_mutex_t queryMutex; } SRetrieveSupport; int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOrderDescriptor **pDesc, diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index cb285c8df1..f77897a74b 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -138,10 +138,10 @@ bool tscIsProjectionQuery(SQueryInfo* pQueryInfo); bool tscIsTwoStageSTableQuery(SQueryInfo* pQueryInfo, int32_t tableIndex); bool tscQueryTags(SQueryInfo* pQueryInfo); -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, - SSchema* pColSchema, int16_t colType); +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, + SColumnIndex* pIndex, SSchema* pColSchema, int16_t colType); -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql); +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql); void tscClearInterpInfo(SQueryInfo* pQueryInfo); bool tscIsInsertData(char* sqlstr); @@ -194,11 +194,11 @@ SColumn* tscColumnListInsert(SArray* pColList, SColumnIndex* colIndex); SArray* tscColumnListClone(const SArray* src, int16_t tableIndex); void tscColumnListDestroy(SArray* pColList); -int32_t tscValidateName(SSQLToken* pToken); +int32_t tscValidateName(SStrToken* pToken); void tscIncStreamExecutionCount(void* pStream); -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId); +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams); // get starter position of metric query condition (query on tags) in SSqlCmd.payload SCond* tsGetSTableQueryCond(STagCond* pCond, uint64_t uid); @@ -217,7 +217,7 @@ STableMetaInfo* tscGetTableMetaInfoFromCmd(SSqlCmd *pCmd, int32_t subClauseIndex STableMetaInfo* tscGetMetaInfo(SQueryInfo *pQueryInfo, int32_t tableIndex); SQueryInfo *tscGetQueryInfoDetail(SSqlCmd* pCmd, int32_t subClauseIndex); -int32_t tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo); +SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index e117f544bf..d2c52e972a 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -35,6 +35,40 @@ extern "C" { #include "qTsbuf.h" #include "tcmdtype.h" +#if 0 +static UNUSED_FUNC void *u_malloc (size_t __size) { + uint32_t v = rand(); + + if (v % 5000 <= 0) { + return NULL; + } else { + return malloc(__size); + } +} + +static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return calloc(num, __size); + } +} + +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5000 <= 0) { + return NULL; + } else { + return realloc(p, __size); + } +} + +#define calloc u_calloc +#define malloc u_malloc +#define realloc u_realloc +#endif + // forward declaration struct SSqlInfo; struct SLocalReducer; @@ -195,9 +229,9 @@ typedef struct STableDataBlocks { typedef struct SQueryInfo { int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert/import type + uint32_t type; // query/insert type char slidingTimeUnit; - STimeWindow window; + STimeWindow window; // query time window int64_t intervalTime; // aggregation time interval int64_t slidingTime; // sliding window in mseconds SSqlGroupbyExpr groupbyExpr; // group by tags info @@ -216,6 +250,7 @@ typedef struct SQueryInfo { char * msg; // pointer to the pCmd->payload to keep error message temporarily int64_t clauseLimit; // limit for current sub clause int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX } SQueryInfo; typedef struct { @@ -431,31 +466,36 @@ static FORCE_INLINE void tscGetResultColumnChr(SSqlRes* pRes, SFieldInfo* pField int32_t bytes = pInfo->pSqlExpr->resBytes; char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + bytes * pRes->row; - if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { - int32_t realLen = varDataLen(pData); - assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + // user defined constant value output columns + if (pInfo->pSqlExpr->colInfo.flag == TSDB_COL_UDC) { + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + pData = pInfo->pSqlExpr->param[1].pz; + pRes->length[columnIndex] = pInfo->pSqlExpr->param[1].nLen; + pRes->tsrow[columnIndex] = (pInfo->pSqlExpr->param[1].nType == TSDB_DATA_TYPE_NULL) ? NULL : pData; } else { - pRes->tsrow[columnIndex] = ((tstr*)pData)->data; - } + assert(bytes == tDataTypeDesc[type].nSize); - if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor - *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : &pInfo->pSqlExpr->param[1].i64Key; + pRes->length[columnIndex] = bytes; } - - pRes->length[columnIndex] = realLen; } else { - assert(bytes == tDataTypeDesc[type].nSize); + if (type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BINARY) { + int32_t realLen = varDataLen(pData); + assert(realLen <= bytes - VARSTR_HEADER_SIZE); - if (isNull(pData, type)) { - pRes->tsrow[columnIndex] = NULL; + pRes->tsrow[columnIndex] = (isNull(pData, type)) ? NULL : ((tstr *)pData)->data; + if (realLen < pInfo->pSqlExpr->resBytes - VARSTR_HEADER_SIZE) { // todo refactor + *(pData + realLen + VARSTR_HEADER_SIZE) = 0; + } + + pRes->length[columnIndex] = realLen; } else { - pRes->tsrow[columnIndex] = pData; - } + assert(bytes == tDataTypeDesc[type].nSize); - pRes->length[columnIndex] = bytes; + pRes->tsrow[columnIndex] = isNull(pData, type) ? NULL : pData; + pRes->length[columnIndex] = bytes; + } } } diff --git a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h index d81bbd9b6d..eaea91d1bf 100644 --- a/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h +++ b/src/client/jni/com_taosdata_jdbc_TSDBJNIConnector.h @@ -89,6 +89,14 @@ JNIEXPORT jstring JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getErrMsgImp JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_getResultSetImp (JNIEnv *env, jobject jobj, jlong con, jlong tres); +/* + * Class: com_taosdata_jdbc_TSDBJNIConnector + * Method: isUpdateQueryImp + * Signature: (J)J + */ +JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_isUpdateQueryImp + (JNIEnv *env, jobject jobj, jlong con, jlong tres); + /* * Class: com_taosdata_jdbc_TSDBJNIConnector * Method: freeResultSetImp diff --git a/src/client/src/taos.rc.in b/src/client/src/taos.rc.in index 05dbd9bb7d..751be85fd0 100644 --- a/src/client/src/taos.rc.in +++ b/src/client/src/taos.rc.in @@ -1,6 +1,6 @@ 1 VERSIONINFO - FILEVERSION ${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3} - PRODUCTVERSION ${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3} + FILEVERSION ${TD_VER_NUMBER} + PRODUCTVERSION ${TD_VER_NUMBER} FILEFLAGSMASK 0x17L #ifdef _DEBUG FILEFLAGS 0x1L @@ -16,12 +16,12 @@ BEGIN BLOCK "040904b0" BEGIN VALUE "FileDescription", "Native C Driver for TDengine" - VALUE "FileVersion", "${TD_VER_1}, ${TD_VER_2}, ${TD_VER_3}" + VALUE "FileVersion", "${TD_VER_NUMBER}" VALUE "InternalName", "taos.dll(${TD_VER_CPUTYPE})" VALUE "LegalCopyright", "Copyright (C) 2020 TAOS Data" VALUE "OriginalFilename", "" VALUE "ProductName", "taos.dll(${TD_VER_CPUTYPE})" - VALUE "ProductVersion", "${TD_VER_1}.${TD_VER_2}.${TD_VER_3}.${TD_VER_4}" + VALUE "ProductVersion", "${TD_VER_NUMBER}" END END BLOCK "VarFileInfo" diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index b05aef76eb..4643d255dc 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -433,7 +433,8 @@ void tscTableMetaCallBack(void *param, TAOS_RES *res, int code) { tscError("%p get tableMeta failed, code:%s", pSql, tstrerror(code)); goto _error; } else { - tscDebug("%p get tableMeta successfully", pSql); + const char* msg = (pCmd->command == TSDB_SQL_STABLEVGROUP)? "vgroup-list":"table-meta"; + tscDebug("%p get %s successfully", pSql, msg); } if (pSql->pStream == NULL) { diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 04c6ef47a5..e74fcba246 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -1813,23 +1813,19 @@ static void last_dist_func_second_merge(SQLFunctionCtx *pCtx) { * NOTE: last_row does not use the interResultBuf to keep the result */ static void last_row_function(SQLFunctionCtx *pCtx) { - assert(pCtx->size == 1); - + assert(pCtx->size >= 1); char *pData = GET_INPUT_CHAR(pCtx); - assignVal(pCtx->aOutputBuf, pData, pCtx->inputBytes, pCtx->inputType); + + // assign the last element in current data block + assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); SResultInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; - SLastrowInfo *pInfo = (SLastrowInfo *)pResInfo->interResultBuf; - pInfo->ts = pCtx->ptsList[0]; - - pInfo->hasResult = DATA_SET_FLAG; - - // set the result to final result buffer + // set the result to final result buffer in case of super table query if (pResInfo->superTableQ) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); - pInfo1->ts = pCtx->ptsList[0]; + pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; pInfo1->hasResult = DATA_SET_FLAG; DO_UPDATE_TAG_COLUMNS(pCtx, pInfo1->ts); @@ -2038,7 +2034,7 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { tValuePair **tvp = pRes->res; int32_t step = QUERY_ASC_FORWARD_STEP; - int32_t len = GET_RES_INFO(pCtx)->numOfRes; + int32_t len = (int32_t)(GET_RES_INFO(pCtx)->numOfRes); switch (type) { case TSDB_DATA_TYPE_INT: { @@ -2412,10 +2408,10 @@ static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { // user specify the order of output by sort the result according to timestamp if (pCtx->param[1].i64Key == PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resAscComparFn : resDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } else if (pCtx->param[1].i64Key > PRIMARYKEY_TIMESTAMP_COL_INDEX) { __compar_fn_t comparator = (pCtx->param[2].i64Key == TSDB_ORDER_ASC) ? resDataAscComparFn : resDataDescComparFn; - qsort(tvp, pResInfo->numOfRes, POINTER_BYTES, comparator); + qsort(tvp, (size_t)pResInfo->numOfRes, POINTER_BYTES, comparator); } GET_TRUE_DATA_TYPE(); @@ -2909,33 +2905,41 @@ static FORCE_INLINE void date_col_output_function_f(SQLFunctionCtx *pCtx, int32_ } static void col_project_function(SQLFunctionCtx *pCtx) { + // the number of output rows should not affect the final number of rows, so set it to be 0 + if (pCtx->numOfParams == 2) { + return; + } + INC_INIT_VAL(pCtx, pCtx->size); - + char *pData = GET_INPUT_CHAR(pCtx); if (pCtx->order == TSDB_ORDER_ASC) { - memcpy(pCtx->aOutputBuf, pData, (size_t)pCtx->size * pCtx->inputBytes); + memcpy(pCtx->aOutputBuf, pData, (size_t) pCtx->size * pCtx->inputBytes); } else { for(int32_t i = 0; i < pCtx->size; ++i) { memcpy(pCtx->aOutputBuf + (pCtx->size - 1 - i) * pCtx->inputBytes, pData + i * pCtx->inputBytes, pCtx->inputBytes); } } - + pCtx->aOutputBuf += pCtx->size * pCtx->outputBytes; } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { SResultInfo *pResInfo = GET_RES_INFO(pCtx); - + if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 + return; + } + // only one output if (pCtx->param[0].i64Key == 1 && pResInfo->numOfRes >= 1) { return; } - + INC_INIT_VAL(pCtx, 1); char *pData = GET_INPUT_CHAR_INDEX(pCtx, index); memcpy(pCtx->aOutputBuf, pData, pCtx->inputBytes); - + pCtx->aOutputBuf += pCtx->inputBytes; } @@ -3903,11 +3907,11 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, d, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE); } } @@ -3926,7 +3930,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, 0, pCtx->tag.i64Key, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 9dc2210ab6..563c9fa84e 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -370,7 +370,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, pCmd->clauseIndex, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - TSKEY stime = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY stime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey : pQueryInfo->window.ekey; int64_t revisedSTime = taosGetIntervalStartTimestamp(stime, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, tinfo.precision); @@ -843,28 +843,6 @@ void savePrevRecordAndSetupFillInfo(SLocalReducer *pLocalReducer, SQueryInfo *pQ tColModelAppend(pModel, pLocalReducer->discardData, pLocalReducer->prevRowOfInput, 0, 1, 1); } -static void reversedCopyFromInterpolationToDstBuf(SQueryInfo *pQueryInfo, SSqlRes *pRes, tFilePage **pResPages, - SLocalReducer *pLocalReducer) { - assert(0); - size_t size = tscSqlExprNumOfExprs(pQueryInfo); - - for (int32_t i = 0; i < size; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - - int32_t offset = tscFieldInfoGetOffset(pQueryInfo, i); - assert(offset == getColumnModelOffset(pLocalReducer->resColModel, i)); - - char *src = pResPages[i]->data + (pRes->numOfRows - 1) * pField->bytes; - char *dst = pRes->data + pRes->numOfRows * offset; - - for (int32_t j = 0; j < pRes->numOfRows; ++j) { - memcpy(dst, src, (size_t)pField->bytes); - dst += pField->bytes; - src -= pField->bytes; - } - } -} - static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer, SQueryInfo* pQueryInfo) { assert(pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE); @@ -925,7 +903,8 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); SFillInfo *pFillInfo = pLocalReducer->pFillInfo; - int64_t actualETime = MAX(pQueryInfo->window.skey, pQueryInfo->window.ekey); + // todo extract function + int64_t actualETime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; tFilePage **pResPages = malloc(POINTER_BYTES * pQueryInfo->fieldsInfo.numOfOutput); for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { @@ -984,14 +963,10 @@ static void doFillResult(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool doneO savePrevRecordAndSetupFillInfo(pLocalReducer, pQueryInfo, pFillInfo); } - if (pQueryInfo->order.order == TSDB_ORDER_ASC) { - for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { - TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); - int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); - memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); - } - } else { // todo bug?? - reversedCopyFromInterpolationToDstBuf(pQueryInfo, pRes, pResPages, pLocalReducer); + for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { + TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, i); + int16_t offset = getColumnModelOffset(pLocalReducer->resColModel, i); + memcpy(pRes->data + offset * pRes->numOfRows, pResPages[i]->data, (size_t)(pField->bytes * pRes->numOfRows)); } pRes->numOfRowsGroup += pRes->numOfRows; @@ -1248,8 +1223,6 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur printf("final result before interpo:\n"); // tColModelDisplay(pLocalReducer->resColModel, pLocalReducer->pBufForInterpo, pResBuf->num, pResBuf->num); #endif - - // no interval query, no fill operation if (pQueryInfo->intervalTime == 0 || pQueryInfo->fillType == TSDB_FILL_NONE) { @@ -1257,7 +1230,9 @@ bool genFinalResults(SSqlObj *pSql, SLocalReducer *pLocalReducer, bool noMoreCur } else { SFillInfo* pFillInfo = pLocalReducer->pFillInfo; if (pFillInfo != NULL) { - taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, pQueryInfo->window.ekey); + TSKEY ekey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey: pQueryInfo->window.skey; + + taosFillSetStartInfo(pFillInfo, (int32_t)pResBuf->num, ekey); taosFillCopyInputDataFromOneFilePage(pFillInfo, pResBuf); } @@ -1292,7 +1267,7 @@ static void resetEnvForNewResultset(SSqlRes *pRes, SSqlCmd *pCmd, SLocalReducer // for group result interpolation, do not return if not data is generated if (pQueryInfo->fillType != TSDB_FILL_NONE) { - TSKEY skey = MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); + TSKEY skey = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.skey:pQueryInfo->window.ekey;//MIN(pQueryInfo->window.skey, pQueryInfo->window.ekey); int64_t newTime = taosGetIntervalStartTimestamp(skey, pQueryInfo->slidingTime, pQueryInfo->intervalTime, pQueryInfo->slidingTimeUnit, precision); taosResetFillInfo(pLocalReducer->pFillInfo, newTime); @@ -1345,7 +1320,7 @@ static bool doHandleLastRemainData(SSqlObj *pSql) { // if fillType == TSDB_FILL_NONE, return directly if (pQueryInfo->fillType != TSDB_FILL_NONE && ((pRes->numOfRowsGroup < pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) || (pQueryInfo->limit.limit < 0))) { - int64_t etime = (pQueryInfo->window.skey < pQueryInfo->window.ekey) ? pQueryInfo->window.ekey : pQueryInfo->window.skey; + int64_t etime = (pQueryInfo->order.order == TSDB_ORDER_ASC)? pQueryInfo->window.ekey : pQueryInfo->window.skey; int32_t rows = (int32_t)getFilledNumOfRes(pFillInfo, etime, pLocalReducer->resColModel->capacity); if (rows > 0) { @@ -1402,13 +1377,12 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { tscResetForNextRetrieve(pRes); if (pSql->signature != pSql || pRes == NULL || pRes->pLocalReducer == NULL) { // all data has been processed - tscDebug("%p %s call the drop local reducer", pSql, __FUNCTION__); - tscDestroyLocalReducer(pSql); - return 0; + tscError("%p local merge abort due to error occurs, code:%s", pSql, tstrerror(pRes->code)); + return pRes->code; } SLocalReducer *pLocalReducer = pRes->pLocalReducer; - SQueryInfo * pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); // set the data merge in progress int32_t prevStatus = @@ -1503,8 +1477,8 @@ int32_t tscDoLocalMerge(SSqlObj *pSql) { * so the processing of previous group is completed. */ int32_t numOfRes = finalizeRes(pQueryInfo, pLocalReducer); + bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); - bool sameGroup = isSameGroup(pCmd, pLocalReducer, pLocalReducer->prevRowOfInput, tmpBuffer); tFilePage *pResBuf = pLocalReducer->pResultBuf; /* diff --git a/src/client/src/tscParseInsert.c b/src/client/src/tscParseInsert.c index cb49bd80b7..47bfe0fcdc 100644 --- a/src/client/src/tscParseInsert.c +++ b/src/client/src/tscParseInsert.c @@ -40,7 +40,7 @@ enum { static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows); -static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { +static int32_t tscToInteger(SStrToken *pToken, int64_t *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -73,7 +73,7 @@ static int32_t tscToInteger(SSQLToken *pToken, int64_t *value, char **endPtr) { return pToken->type; } -static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { +static int32_t tscToDouble(SStrToken *pToken, double *value, char **endPtr) { if (pToken->n == 0) { return TK_ILLEGAL; } @@ -89,9 +89,9 @@ static int32_t tscToDouble(SSQLToken *pToken, double *value, char **endPtr) { return pToken->type; } -int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { +int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int64_t interval; int64_t useconds = 0; char * pTokenEnd = *next; @@ -128,7 +128,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 * time expression: * e.g., now+12a, now-5h */ - SSQLToken valueToken; + SStrToken valueToken; index = 0; sToken = tStrGetToken(pTokenEnd, &index, false, 0, NULL); pTokenEnd += index; @@ -163,7 +163,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1 return TSDB_CODE_SUCCESS; } -int32_t tsParseOneColumnData(SSchema *pSchema, SSQLToken *pToken, char *payload, char *msg, char **str, bool primaryKey, +int32_t tsParseOneColumnData(SSchema *pSchema, SStrToken *pToken, char *payload, char *msg, char **str, bool primaryKey, int16_t timePrec) { int64_t iv; int32_t numType; @@ -409,7 +409,7 @@ static int32_t tsCheckTimestamp(STableDataBlocks *pDataBlocks, const char *start int tsParseOneRowData(char **str, STableDataBlocks *pDataBlocks, SSchema schema[], SParsedDataColInfo *spd, char *error, int16_t timePrec, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken = {0}; + SStrToken sToken = {0}; char * payload = pDataBlocks->pData + pDataBlocks->size; // 1. set the parsed value from sql string @@ -524,7 +524,7 @@ static int32_t rowDataCompar(const void *lhs, const void *rhs) { int tsParseValues(char **str, STableDataBlocks *pDataBlock, STableMeta *pTableMeta, int maxRows, SParsedDataColInfo *spd, char *error, int32_t *code, char *tmpTokenBuf) { int32_t index = 0; - SSQLToken sToken; + SStrToken sToken; int16_t numOfRows = 0; @@ -734,8 +734,8 @@ static int32_t doParseInsertStatement(SSqlObj *pSql, void *pTableList, char **st static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { int32_t index = 0; - SSQLToken sToken = {0}; - SSQLToken tableToken = {0}; + SStrToken sToken = {0}; + SStrToken tableToken = {0}; int32_t code = TSDB_CODE_SUCCESS; const int32_t TABLE_INDEX = 0; @@ -993,7 +993,7 @@ static int32_t tscCheckIfCreateTable(char **sqlstr, SSqlObj *pSql) { return code; } -int validateTableName(char *tblName, int len, SSQLToken* psTblToken) { +int validateTableName(char *tblName, int len, SStrToken* psTblToken) { tstrncpy(psTblToken->z, tblName, TSDB_TABLE_FNAME_LEN); psTblToken->n = len; @@ -1031,11 +1031,11 @@ int tsParseInsertSql(SSqlObj *pSql) { SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); assert(pQueryInfo != NULL); - STableMetaInfo *pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo):tscGetMetaInfo(pQueryInfo, 0); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + code = terrno; + return code; } if ((code = tscAllocPayload(pCmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { @@ -1057,7 +1057,7 @@ int tsParseInsertSql(SSqlObj *pSql) { while (1) { int32_t index = 0; - SSQLToken sToken = tStrGetToken(str, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(str, &index, false, 0, NULL); // no data in the sql string anymore. if (sToken.n == 0) { @@ -1083,7 +1083,7 @@ int tsParseInsertSql(SSqlObj *pSql) { pCmd->curSql = sToken.z; char buf[TSDB_TABLE_FNAME_LEN]; - SSQLToken sTblToken; + SStrToken sTblToken; sTblToken.z = buf; // Check if the table name available or not if (validateTableName(sToken.z, sToken.n, &sTblToken) != TSDB_CODE_SUCCESS) { @@ -1285,15 +1285,14 @@ int tsInsertInitialCheck(SSqlObj *pSql) { int32_t index = 0; SSqlCmd *pCmd = &pSql->cmd; - SSQLToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); + SStrToken sToken = tStrGetToken(pSql->sqlstr, &index, false, 0, NULL); assert(sToken.type == TK_INSERT || sToken.type == TK_IMPORT); pCmd->count = 0; pCmd->command = TSDB_SQL_INSERT; pSql->res.numOfRows = 0; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_INSERT | pCmd->insertType); diff --git a/src/client/src/tscPrepare.c b/src/client/src/tscPrepare.c index b996dd958a..620e8ea57a 100644 --- a/src/client/src/tscPrepare.c +++ b/src/client/src/tscPrepare.c @@ -155,7 +155,7 @@ static int normalStmtPrepare(STscStmt* stmt) { uint32_t i = 0, start = 0; while (sql[i] != 0) { - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(sql + i, &token.type); if (token.type == TK_QUESTION) { diff --git a/src/client/src/tscProfile.c b/src/client/src/tscProfile.c index f48e7b7691..6ff97e9d00 100644 --- a/src/client/src/tscProfile.c +++ b/src/client/src/tscProfile.c @@ -285,9 +285,9 @@ void tscKillConnection(STscObj *pObj) { SSqlObj *pSql = pObj->sqlList; while (pSql) { - //taosStopRpcConn(pSql->thandle); pSql = pSql->next; } + SSqlStream *pStream = pObj->streamList; while (pStream) { diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index d108349085..5ce4c7125f 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -33,8 +33,8 @@ #define DEFAULT_PRIMARY_TIMESTAMP_COL_NAME "_c0" -// -1 is tbname column index, so here use the -2 as the initial value -#define COLUMN_INDEX_INITIAL_VAL (-2) +// -1 is tbname column index, so here use the -3 as the initial value +#define COLUMN_INDEX_INITIAL_VAL (-3) #define COLUMN_INDEX_INITIALIZER \ { COLUMN_INDEX_INITIAL_VAL, COLUMN_INDEX_INITIAL_VAL } #define COLUMN_INDEX_VALIDE(index) (((index).tableIndex >= 0) && ((index).columnIndex >= TSDB_TBNAME_COLUMN_INDEX)) @@ -51,12 +51,12 @@ static int32_t setShowInfo(SSqlObj* pSql, SSqlInfo* pInfo); static char* getAccountId(SSqlObj* pSql); static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name); -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken); -static bool hasSpecifyDB(SSQLToken* pTableName); +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken); +static bool hasSpecifyDB(SStrToken* pTableName); static bool validateTableColumnInfo(tFieldList* pFieldList, SSqlCmd* pCmd); static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSqlCmd* pCmd); -static int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* len); +static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); @@ -65,11 +65,11 @@ static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int3 static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, int8_t type, char* fieldName, SSqlExpr* pSqlExpr); static int32_t changeFunctionID(int32_t optr, int16_t* functionId); -static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable); +static int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery); static bool validateIpAddress(const char* ip, size_t size); static bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo); +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery); static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); @@ -100,11 +100,11 @@ static bool hasTimestampForPointInterpQuery(SQueryInfo* pQueryInfo); static int32_t parseLimitClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t index, SQuerySQL* pQuerySql, SSqlObj* pSql); static int32_t parseCreateDBOptions(SSqlCmd* pCmd, SCreateDBInfo* pCreateDbSql); -static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); -static int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t optrToString(tSQLExpr* pExpr, char** exprString); -static int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); +static int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex); static int32_t doFunctionsCompatibleCheck(SSqlCmd* pCmd, SQueryInfo* pQueryInfo); static int32_t doLocalQueryProcess(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t tscCheckCreateDbParams(SSqlCmd* pCmd, SCMCreateDbMsg* pCreate); @@ -151,7 +151,7 @@ static int setColumnFilterInfoForTimestamp(SSqlCmd* pCmd, SQueryInfo* pQueryInfo return TSDB_CODE_SUCCESS; } -static int32_t handlePassword(SSqlCmd* pCmd, SSQLToken* pPwd) { +static int32_t handlePassword(SSqlCmd* pCmd, SStrToken* pPwd) { const char* msg1 = "password can not be empty"; const char* msg2 = "name or password too long"; const char* msg3 = "password needs single quote marks enclosed"; @@ -179,20 +179,24 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return TSDB_CODE_TSC_APP_ERROR; } - SSqlCmd* pCmd = &(pSql->cmd); - SQueryInfo* pQueryInfo = NULL; + SSqlCmd* pCmd = &pSql->cmd; + SSqlRes* pRes = &pSql->res; + int32_t code = TSDB_CODE_SUCCESS; if (!pInfo->valid) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), pInfo->pzErrMsg); } - int32_t code = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pRes->code = terrno; + return pRes->code; + } - STableMetaInfo* pTableMetaInfo = NULL; - if (pQueryInfo->numOfTables == 0) { - pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); - } else { - pTableMetaInfo = pQueryInfo->pTableMetaInfo[0]; + STableMetaInfo* pTableMetaInfo = (pQueryInfo->numOfTables == 0)? tscAddEmptyMetaInfo(pQueryInfo) : pQueryInfo->pTableMetaInfo[0]; + if (pTableMetaInfo == NULL) { + pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pRes->code; } pCmd->command = pInfo->type; @@ -206,7 +210,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid name"; const char* msg3 = "param name too long"; - SSQLToken* pzName = &pInfo->pDCLInfo->a[0]; + SStrToken* pzName = &pInfo->pDCLInfo->a[0]; if ((pInfo->type != TSDB_SQL_DROP_DNODE) && (tscValidateName(pzName) != TSDB_CODE_SUCCESS)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -241,7 +245,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { case TSDB_SQL_USE_DB: { const char* msg = "invalid db name"; - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); @@ -296,7 +300,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg); } - SSQLToken* pIpAddr = &pInfo->pDCLInfo->a[0]; + SStrToken* pIpAddr = &pInfo->pDCLInfo->a[0]; pIpAddr->n = strdequote(pIpAddr->z); break; } @@ -307,8 +311,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg2 = "invalid user/account name"; const char* msg3 = "name too long"; - SSQLToken* pName = &pInfo->pDCLInfo->user.user; - SSQLToken* pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken* pName = &pInfo->pDCLInfo->user.user; + SStrToken* pPwd = &pInfo->pDCLInfo->user.passwd; if (handlePassword(pCmd, pPwd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -337,7 +341,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } case TSDB_SQL_DESCRIBE_TABLE: { - SSQLToken* pToken = &pInfo->pDCLInfo->a[0]; + SStrToken* pToken = &pInfo->pDCLInfo->a[0]; const char* msg2 = "table name is too long"; const char* msg1 = "invalid table name"; @@ -400,8 +404,8 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { // tDCLSQL* pDCL = pInfo->pDCLInfo; SUserInfo* pUser = &pInfo->pDCLInfo->user; - SSQLToken* pName = &pUser->user; - SSQLToken* pPwd = &pUser->passwd; + SStrToken* pName = &pUser->user; + SStrToken* pPwd = &pUser->passwd; if (pName->n >= TSDB_USER_LEN) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -423,7 +427,7 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { } else if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { assert(pPwd->type == TSDB_DATA_TYPE_NULL); - SSQLToken* pPrivilege = &pUser->privilege; + SStrToken* pPrivilege = &pUser->privilege; if (strncasecmp(pPrivilege->z, "super", 5) == 0 && pPrivilege->n == 5) { pCmd->count = 1; @@ -487,9 +491,10 @@ int32_t tscToSQLCmd(SSqlObj* pSql, struct SSqlInfo* pInfo) { const char* msg1 = "columns in select clause not identical"; for (int32_t i = pCmd->numOfClause; i < pInfo->subclauseInfo.numOfClause; ++i) { - SQueryInfo* pqi = NULL; - if ((code = tscGetQueryInfoDetailSafely(pCmd, i, &pqi)) != TSDB_CODE_SUCCESS) { - return code; + SQueryInfo* pqi = tscGetQueryInfoDetailSafely(pCmd, i); + if (pqi == NULL) { + pRes->code = terrno; + return pRes->code; } } @@ -581,7 +586,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ } // interval is not null - SSQLToken* t = &pQuerySql->interval; + SStrToken* t = &pQuerySql->interval; if (getTimestampInUsFromStr(t->z, t->n, &pQueryInfo->intervalTime) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -667,7 +672,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); - SSQLToken* pSliding = &pQuerySql->sliding; + SStrToken* pSliding = &pQuerySql->sliding; if (pSliding->n != 0) { getTimestampInUsFromStr(pSliding->z, pSliding->n, &pQueryInfo->slidingTime); if (tinfo.precision == TSDB_TIME_PRECISION_MILLI) { @@ -692,7 +697,7 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } -int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableName, SSqlObj* pSql) { +int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SStrToken* pzTableName, SSqlObj* pSql) { const char* msg = "name too long"; SSqlCmd* pCmd = &pSql->cmd; @@ -709,7 +714,7 @@ int32_t tscSetTableFullName(STableMetaInfo* pTableMetaInfo, SSQLToken* pzTableNa // db has been specified in sql string so we ignore current db path code = setObjFullName(pTableMetaInfo->name, getAccountId(pSql), NULL, pzTableName, NULL); } else { // get current DB name first, then set it into path - SSQLToken t = {0}; + SStrToken t = {0}; getCurrentDBName(pSql, &t); code = setObjFullName(pTableMetaInfo->name, NULL, &t, pzTableName, NULL); @@ -1027,13 +1032,13 @@ static bool has(tFieldList* pFieldList, int32_t startIdx, const char* name) { static char* getAccountId(SSqlObj* pSql) { return pSql->pTscObj->acctId; } -static void getCurrentDBName(SSqlObj* pSql, SSQLToken* pDBToken) { +static void getCurrentDBName(SSqlObj* pSql, SStrToken* pDBToken) { pDBToken->z = pSql->pTscObj->db; pDBToken->n = (uint32_t)strlen(pSql->pTscObj->db); } /* length limitation, strstr cannot be applied */ -static bool hasSpecifyDB(SSQLToken* pTableName) { +static bool hasSpecifyDB(SStrToken* pTableName) { for (uint32_t i = 0; i < pTableName->n; ++i) { if (pTableName->z[i] == TS_PATH_DELIMITER[0]) { return true; @@ -1043,7 +1048,7 @@ static bool hasSpecifyDB(SSQLToken* pTableName) { return false; } -int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQLToken* tableName, int32_t* xlen) { +int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* xlen) { int32_t totalLen = 0; if (account != NULL) { @@ -1094,18 +1099,6 @@ int32_t setObjFullName(char* fullName, const char* account, SSQLToken* pDB, SSQL return (totalLen < TSDB_TABLE_FNAME_LEN) ? TSDB_CODE_SUCCESS : TSDB_CODE_TSC_INVALID_SQL; } -static void extractColumnNameFromString(tSQLExprItem* pItem) { - if (pItem->pNode->nSQLOptr == TK_STRING) { - pItem->pNode->val.nLen = strdequote(pItem->pNode->val.pz); - pItem->pNode->nSQLOptr = TK_ID; - - SSQLToken* pIdToken = &pItem->pNode->colInfo; - pIdToken->type = TK_ID; - pIdToken->z = pItem->pNode->val.pz; - pIdToken->n = pItem->pNode->val.nLen; - } -} - static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { const char* msg1 = "invalid column name, or illegal column type"; const char* msg2 = "invalid arithmetic expression in select clause"; @@ -1123,42 +1116,31 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } int32_t tableIndex = columnList.ids[0].tableIndex; - - // todo potential data overflow - char* arithmeticExprStr = malloc(1024*1024); - char* p = arithmeticExprStr; - if (arithmeticType == NORMAL_ARITHMETIC) { pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; // all columns in arithmetic expression must belong to the same table for (int32_t f = 1; f < columnList.num; ++f) { if (columnList.ids[f].tableIndex != tableIndex) { - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); } } - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - // expr string is set as the parameter of function SColumnIndex index = {.tableIndex = tableIndex}; SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, TSDB_FUNC_ARITHM, &index, TSDB_DATA_TYPE_DOUBLE, sizeof(double), sizeof(double), false); - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - tstrncpy(pExpr->aliasName, name, sizeof(pExpr->aliasName)); + char* name = (pItem->aliasName != NULL)? pItem->aliasName:pItem->pNode->token.z; + size_t len = MIN(sizeof(pExpr->aliasName), pItem->pNode->token.n + 1); + tstrncpy(pExpr->aliasName, name, len); tExprNode* pNode = NULL; SArray* colList = taosArrayInit(10, sizeof(SColIndex)); int32_t ret = exprTreeFromSqlExpr(pCmd, &pNode, pItem->pNode, pQueryInfo->exprList, pQueryInfo, colList); if (ret != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); @@ -1167,8 +1149,8 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t size_t numOfNode = taosArrayGetSize(colList); for(int32_t k = 0; k < numOfNode; ++k) { SColIndex* pIndex = taosArrayGet(colList, k); - if (pIndex->flag == 1) { - taosTFree(arithmeticExprStr); + if (TSDB_COL_IS_TAG(pIndex->flag)) { + tExprTreeDestroy(&pNode, NULL); taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); @@ -1185,7 +1167,7 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t // TODO: other error handling } END_TRY - size_t len = tbufTell(&bw); + len = tbufTell(&bw); char* c = tbufGetData(&bw, true); // set the serialized binary string as the parameter of arithmetic expression @@ -1196,16 +1178,18 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t taosArrayDestroy(colList); tExprTreeDestroy(&pNode, NULL); } else { - if (arithmeticExprToString(pItem->pNode, &p) != TSDB_CODE_SUCCESS) { - taosTFree(arithmeticExprStr); - return TSDB_CODE_TSC_INVALID_SQL; - } - columnList.num = 0; columnList.ids[0] = (SColumnIndex) {0, 0}; - char* name = (pItem->aliasName != NULL)? pItem->aliasName:arithmeticExprStr; - insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, name, NULL); + char aliasName[TSDB_COL_NAME_LEN] = {0}; + if (pItem->aliasName != NULL) { + tstrncpy(aliasName, pItem->aliasName, TSDB_COL_NAME_LEN); + } else { + int32_t nameLen = MIN(TSDB_COL_NAME_LEN, pItem->pNode->token.n + 1); + tstrncpy(aliasName, pItem->pNode->token.z, nameLen); + } + + insertResultField(pQueryInfo, exprIndex, &columnList, sizeof(double), TSDB_DATA_TYPE_DOUBLE, aliasName, NULL); int32_t slot = tscNumOfFields(pQueryInfo) - 1; SFieldSupInfo* pInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, slot); @@ -1221,7 +1205,6 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t int32_t ret = exprTreeFromSqlExpr(pCmd, &pArithExprInfo->pExpr, pItem->pNode, pQueryInfo->exprList, pQueryInfo, NULL); if (ret != TSDB_CODE_SUCCESS) { tExprTreeDestroy(&pArithExprInfo->pExpr, NULL); - taosTFree(arithmeticExprStr); return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "invalid expression in select clause"); } @@ -1229,11 +1212,38 @@ static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t } } - taosTFree(arithmeticExprStr); return TSDB_CODE_SUCCESS; } -int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable) { +static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; + tscColumnListInsert(pQueryInfo->colList, &tsCol); +} + +static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { + SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); + STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); + + char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; + tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); + + SColumnList ids = {0}; + ids.num = 1; + ids.ids[0] = *pIndex; + + if (pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX || pIndex->columnIndex == TSDB_UD_COLUMN_INDEX || + pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta)) { + ids.num = 0; + } + + insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); +} + +int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSelection, bool isSTable, bool joinQuery) { assert(pSelection != NULL && pCmd != NULL); const char* msg2 = "functions can not be mixed up"; @@ -1251,16 +1261,14 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel tSQLExprItem* pItem = &pSelection->a[i]; // project on all fields - if (pItem->pNode->nSQLOptr == TK_ALL || pItem->pNode->nSQLOptr == TK_ID || pItem->pNode->nSQLOptr == TK_STRING) { + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL || optr == TK_ID || optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // it is actually a function, but the function name is invalid if (pItem->pNode->nSQLOptr == TK_ID && (pItem->pNode->colInfo.z == NULL && pItem->pNode->colInfo.n == 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg5); } - // if the name of column is quoted, remove it and set the right information for later process - extractColumnNameFromString(pItem); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); - // select table_name1.field_name1, table_name2.field_name2 from table_name1, table_name2 if (addProjectionExprAndResultField(pCmd, pQueryInfo, pItem) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -1276,12 +1284,7 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel if (code != TSDB_CODE_SUCCESS) { return code; } - } else { - /* - * not support such expression - * e.g., select 12+5 from table_name - */ return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1290,7 +1293,26 @@ int32_t parseSelectClause(SSqlCmd* pCmd, int32_t clauseIndex, tSQLExprList* pSel } } - if (!functionCompatibleCheck(pQueryInfo)) { + // there is only one user-defined column in the final result field, add the timestamp column. + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + SColumnIndex index = {0}; + + // set the constant column value always attached to first table. + STableMetaInfo* pTableMetaInfo = tscGetTableMetaInfoFromCmd(pCmd, clauseIndex, 0); + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, PRIMARYKEY_TIMESTAMP_COL_INDEX); + + // add the timestamp column into the output columns + int32_t numOfCols = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + tscAddSpecialColumnForSelect(pQueryInfo, numOfCols, TSDB_FUNC_PRJ, &index, pSchema, TSDB_COL_NORMAL); + + SFieldSupInfo* pSupInfo = tscFieldInfoGetSupp(&pQueryInfo->fieldsInfo, numOfCols); + pSupInfo->visible = false; + + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; + } + + if (!functionCompatibleCheck(pQueryInfo, joinQuery)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } @@ -1353,32 +1375,10 @@ SSqlExpr* doAddProjectCol(SQueryInfo* pQueryInfo, int32_t outputIndex, int32_t c pSchema->bytes, functionId == TSDB_FUNC_TAGPRJ); } -static void addProjectQueryCol(SQueryInfo* pQueryInfo, int32_t startPos, SColumnIndex* pIndex, tSQLExprItem* pItem) { - SSqlExpr* pExpr = doAddProjectCol(pQueryInfo, startPos, pIndex->columnIndex, pIndex->tableIndex); - - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pIndex->tableIndex); - STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; - - SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, pIndex->columnIndex); - - char* colName = (pItem->aliasName == NULL) ? pSchema->name : pItem->aliasName; - tstrncpy(pExpr->aliasName, colName, sizeof(pExpr->aliasName)); - - SColumnList ids = {0}; - ids.num = 1; - ids.ids[0] = *pIndex; - - if (pIndex->columnIndex >= tscGetNumOfColumns(pTableMeta) || pIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - ids.num = 0; - } - - insertResultField(pQueryInfo, startPos, &ids, pExpr->resBytes, (int8_t)pExpr->resType, pExpr->aliasName, pExpr); -} - -void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, +SSqlExpr* tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex, int16_t functionId, SColumnIndex* pIndex, SSchema* pColSchema, int16_t flag) { SSqlExpr* pExpr = tscSqlExprInsert(pQueryInfo, outputColIndex, functionId, pIndex, pColSchema->type, - pColSchema->bytes, pColSchema->bytes, flag); + pColSchema->bytes, pColSchema->bytes, TSDB_COL_IS_TAG(flag)); tstrncpy(pExpr->aliasName, pColSchema->name, sizeof(pExpr->aliasName)); SColumnList ids = getColumnList(1, pIndex->tableIndex, pIndex->columnIndex); @@ -1394,6 +1394,8 @@ void tscAddSpecialColumnForSelect(SQueryInfo* pQueryInfo, int32_t outputColIndex if (TSDB_COL_IS_TAG(flag)) { tscColumnListInsert(pTableMetaInfo->tagColList, pIndex); } + + return pExpr; } static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColumnIndex* pIndex, int32_t startPos) { @@ -1426,18 +1428,16 @@ static int32_t doAddProjectionExprAndResultFields(SQueryInfo* pQueryInfo, SColum return numOfTotalColumns; } -static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SColumnIndex tsCol = {.tableIndex = pIndex->tableIndex, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; - tscColumnListInsert(pQueryInfo->colList, &tsCol); -} - int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem) { const char* msg0 = "invalid column name"; const char* msg1 = "tag for normal table query is not allowed"; - - int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); - if (pItem->pNode->nSQLOptr == TK_ALL) { // project on all fields + int32_t startPos = (int32_t)tscSqlExprNumOfExprs(pQueryInfo); + int32_t optr = pItem->pNode->nSQLOptr; + + if (optr == TK_ALL) { // project on all fields + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_PROJECTION_QUERY); + SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getTableIndexByName(&pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); @@ -1453,7 +1453,24 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } else { doAddProjectionExprAndResultFields(pQueryInfo, &index, startPos); } - } else if (pItem->pNode->nSQLOptr == TK_ID) { // simple column projection query + + // add the primary timestamp column even though it is not required by user + tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); + } else if (optr == TK_STRING || optr == TK_INTEGER || optr == TK_FLOAT) { // simple column projection query + SColumnIndex index = COLUMN_INDEX_INITIALIZER; + + // user-specified constant value as a new result column + index.columnIndex = (pQueryInfo->udColumnId--); + index.tableIndex = 0; + + SSchema colSchema = tGetUserSpecifiedColumnSchema(&pItem->pNode->val, &pItem->pNode->token, pItem->aliasName); + SSqlExpr* pExpr = + tscAddSpecialColumnForSelect(pQueryInfo, startPos, TSDB_FUNC_PRJ, &index, &colSchema, TSDB_COL_UDC); + + // NOTE: the first parameter is reserved for the tag column id during join query process. + pExpr->numOfParams = 2; + tVariantAssign(&pExpr->param[1], &pItem->pNode->val); + } else if (optr == TK_ID) { SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &pItem->pNode->colInfo, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1472,8 +1489,10 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } addProjectQueryCol(pQueryInfo, startPos, &index, pItem); + pQueryInfo->type |= TSDB_QUERY_TYPE_PROJECTION_QUERY; } + // add the primary timestamp column even though it is not required by user tscInsertPrimaryTSSourceColumn(pQueryInfo, &index); } else { return TSDB_CODE_TSC_INVALID_SQL; @@ -1561,7 +1580,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (pItem->pNode->pParam != NULL) { - SSQLToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; + SStrToken* pToken = &pItem->pNode->pParam->a[0].pNode->colInfo; if (pToken->z == NULL || pToken->n == 0) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -1570,7 +1589,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* // check if the table name is valid or not - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -1768,7 +1787,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* - SSQLToken tmpToken = pParamElem->pNode->colInfo; + SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); @@ -1834,10 +1853,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex + i + j, &index) != - 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, functionID, pItem->aliasName, colIndex, &index) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } + + colIndex++; } numOfFields += tscGetNumOfColumns(pTableMetaInfo->pTableMeta); @@ -2045,7 +2065,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col // todo refactor static SColumnList getColumnList(int32_t num, int16_t tableIndex, int32_t columnIndex) { - assert(num == 1 && columnIndex >= -1 && tableIndex >= 0); + assert(num == 1 && tableIndex >= 0); SColumnList columnList = {0}; columnList.num = num; @@ -2070,16 +2090,16 @@ void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, c snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); } -static bool isTablenameToken(SSQLToken* token) { - SSQLToken tmpToken = *token; - SSQLToken tableToken = {0}; +static bool isTablenameToken(SStrToken* token) { + SStrToken tmpToken = *token; + SStrToken tableToken = {0}; extractTableNameFromToken(&tmpToken, &tableToken); return (strncasecmp(TSQL_TBNAME_L, tmpToken.z, tmpToken.n) == 0 && tmpToken.n == strlen(TSQL_TBNAME_L)); } -static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken* pToken) { +static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SStrToken* pToken) { STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, index)->pTableMeta; int32_t numOfCols = tscGetNumOfColumns(pTableMeta) + tscGetNumOfTags(pTableMeta); @@ -2101,7 +2121,7 @@ static int16_t doGetColumnIndex(SQueryInfo* pQueryInfo, int32_t index, SSQLToken return columnIndex; } -int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { const char* msg0 = "ambiguous column name"; const char* msg1 = "invalid column name"; @@ -2143,7 +2163,7 @@ int32_t doGetColumnIndexByName(SSqlCmd* pCmd, SSQLToken* pToken, SQueryInfo* pQu } } -int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getTableIndexImpl(SStrToken* pTableToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pTableToken->n == 0) { // only one table and no table name prefix in column name if (pQueryInfo->numOfTables == 1) { pIndex->tableIndex = 0; @@ -2169,8 +2189,8 @@ int32_t getTableIndexImpl(SSQLToken* pTableToken, SQueryInfo* pQueryInfo, SColum return TSDB_CODE_SUCCESS; } -int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { - SSQLToken tableToken = {0}; +int32_t getTableIndexByName(SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); if (getTableIndexImpl(&tableToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { @@ -2180,12 +2200,12 @@ int32_t getTableIndexByName(SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIn return TSDB_CODE_SUCCESS; } -int32_t getColumnIndexByName(SSqlCmd* pCmd, const SSQLToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { +int32_t getColumnIndexByName(SSqlCmd* pCmd, const SStrToken* pToken, SQueryInfo* pQueryInfo, SColumnIndex* pIndex) { if (pQueryInfo->pTableMetaInfo == NULL || pQueryInfo->numOfTables == 0) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken tmpToken = *pToken; + SStrToken tmpToken = *pToken; if (getTableIndexByName(&tmpToken, pQueryInfo, pIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -2297,7 +2317,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t showType = pShowInfo->showType; if (showType == TSDB_MGMT_TABLE_TABLE || showType == TSDB_MGMT_TABLE_METRIC || showType == TSDB_MGMT_TABLE_VGROUP) { // db prefix in tagCond, show table conds in payload - SSQLToken* pDbPrefixToken = &pShowInfo->prefix; + SStrToken* pDbPrefixToken = &pShowInfo->prefix; if (pDbPrefixToken->type != 0) { if (pDbPrefixToken->n >= TSDB_DB_NAME_LEN) { // db name is too long @@ -2319,7 +2339,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show table/stable like 'xxxx', set the like pattern for show tables - SSQLToken* pPattern = &pShowInfo->pattern; + SStrToken* pPattern = &pShowInfo->pattern; if (pPattern->type != 0) { pPattern->n = strdequote(pPattern->z); @@ -2337,7 +2357,7 @@ int32_t setShowInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } // show vnodes may be ip addr of dnode in payload - SSQLToken* pDnodeIp = &pShowInfo->prefix; + SStrToken* pDnodeIp = &pShowInfo->prefix; if (pDnodeIp->n >= TSDB_IPv4ADDR_LEN) { // ip addr is too long return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } @@ -2358,7 +2378,7 @@ int32_t setKillInfo(SSqlObj* pSql, struct SSqlInfo* pInfo, int32_t killType) { SSqlCmd* pCmd = &pSql->cmd; pCmd->command = pInfo->type; - SSQLToken* idStr = &(pInfo->pDCLInfo->ip); + SStrToken* idStr = &(pInfo->pDCLInfo->ip); if (idStr->n > TSDB_KILL_MSG_LEN) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2510,7 +2530,7 @@ bool hasUnsupportFunctionsForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo) return false; } -static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { +static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t startIdx = 0; SSqlExpr* pExpr = tscSqlExprGet(pQueryInfo, startIdx); @@ -2542,6 +2562,10 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo) { if (functionCompatList[functionId] != factor) { return false; } + + if (functionId == TSDB_FUNC_LAST_ROW && joinQuery) { + return false; + } } return true; @@ -2579,7 +2603,7 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* for (int32_t i = 0; i < pList->nExpr; ++i) { tVariant* pVar = &pList->a[i].pVar; - SSQLToken token = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken token = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = COLUMN_INDEX_INITIALIZER; if (getColumnIndexByName(pCmd, &token, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -2659,9 +2683,12 @@ static SColumnFilterInfo* addColumnFilterInfo(SColumn* pColumn) { } int32_t size = pColumn->numOfFilters + 1; - char* tmp = (char*)realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); + + char* tmp = (char*) realloc((void*)(pColumn->filterInfo), sizeof(SColumnFilterInfo) * (size)); if (tmp != NULL) { pColumn->filterInfo = (SColumnFilterInfo*)tmp; + } else { + return NULL; } pColumn->numOfFilters++; @@ -2945,9 +2972,16 @@ static int32_t extractColumnFilterInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SC } else { // update the existed column filter information, find the filter info here pColFilter = &pColumn->filterInfo[0]; } + + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else if (sqlOptr == TK_OR) { // TODO fixme: failed to invalid the filter expression: "col1 = 1 OR col2 = 2" pColFilter = addColumnFilterInfo(pColumn); + if (pColFilter == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } } else { // error; return TSDB_CODE_TSC_INVALID_SQL; } @@ -3072,7 +3106,6 @@ static int32_t getColumnQueryCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pExpr) { const char* msg1 = "invalid join query condition"; - const char* msg2 = "join on binary/nchar not supported"; const char* msg3 = "type of join columns must be identical"; const char* msg4 = "invalid column name in join condition"; @@ -3116,10 +3149,6 @@ static int32_t getJoinCondInfo(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); } - if (pTagSchema1->type == TSDB_DATA_TYPE_BINARY || pTagSchema1->type == TSDB_DATA_TYPE_NCHAR) { - return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); - } - pTagCond->joinInfo.hasJoin = true; return TSDB_CODE_SUCCESS; } @@ -3156,7 +3185,7 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { +static UNUSED_FUNC int32_t arithmeticExprToString(tSQLExpr* pExpr, char** str) { char* start = *str; int32_t code = doArithmeticExprToString(pExpr, str); @@ -3610,7 +3639,7 @@ static void doExtractExprForSTable(SSqlCmd* pCmd, tSQLExpr** pExpr, SQueryInfo* return; } - SSQLToken t = {0}; + SStrToken t = {0}; extractTableNameFromToken(&pLeft->colInfo, &t); *pOut = *pExpr; @@ -3690,7 +3719,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, num = j; char* name = extractDBName(pTableMetaInfo->name, db); - SSQLToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; + SStrToken dbToken = { .type = TK_STRING, .z = name, .n = (uint32_t)strlen(name) }; for (int32_t i = 0; i < num; ++i) { if (i >= 1) { @@ -3699,7 +3728,7 @@ static int32_t setTableCondForSTableQuery(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, char idBuf[TSDB_TABLE_FNAME_LEN] = {0}; int32_t xlen = (int32_t)strlen(segments[i]); - SSQLToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; + SStrToken t = {.z = segments[i], .n = xlen, .type = TK_STRING}; int32_t ret = setObjFullName(idBuf, account, &dbToken, &t, &xlen); if (ret != TSDB_CODE_SUCCESS) { @@ -4020,7 +4049,7 @@ int32_t getTimeRange(STimeWindow* win, tSQLExpr* pRight, int32_t optr, int16_t t return TSDB_CODE_TSC_INVALID_SQL; } } else { - SSQLToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; + SStrToken token = {.z = pRight->val.pz, .n = pRight->val.nLen, .type = TK_ID}; int32_t len = tSQLGetToken(pRight->val.pz, &token.type); if ((token.type != TK_INTEGER && token.type != TK_FLOAT) || len != pRight->val.nLen) { @@ -4266,7 +4295,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return TSDB_CODE_SUCCESS; } - SSQLToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; + SStrToken columnName = {pVar->nLen, pVar->nType, pVar->pz}; SColumnIndex index = {0}; if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { // super table query @@ -4335,7 +4364,7 @@ int32_t parseOrderbyClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu } tVariant* pVar2 = &pSortorder->a[1].pVar; - SSQLToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; + SStrToken cname = {pVar2->nLen, pVar2->nType, pVar2->pz}; if (getColumnIndexByName(pCmd, &cname, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -4464,7 +4493,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { } SColumnIndex index = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; + SStrToken name = {.z = pItem->pVar.pz, .n = pItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; @@ -4502,12 +4531,12 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { SColumnIndex srcIndex = COLUMN_INDEX_INITIALIZER; SColumnIndex destIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pSrcItem->pVar.pz, .n = pSrcItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &srcToken, pQueryInfo, &srcIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } - SSQLToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; + SStrToken destToken = {.z = pDstItem->pVar.pz, .n = pDstItem->pVar.nLen, .type = TK_STRING}; if (getColumnIndexByName(pCmd, &destToken, pQueryInfo, &destIndex) == TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg19); } @@ -4531,7 +4560,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { int16_t numOfTags = tscGetNumOfTags(pTableMeta); SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; + SStrToken name = {.type = TK_STRING, .z = pTagName->pz, .n = pTagName->nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -4624,7 +4653,7 @@ int32_t setAlterTableInfo(SSqlObj* pSql, struct SSqlInfo* pInfo) { tVariantListItem* pItem = &pAlterSQL->varList->a[0]; SColumnIndex columnIndex = COLUMN_INDEX_INITIALIZER; - SSQLToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; + SStrToken name = {.type = TK_STRING, .z = pItem->pVar.pz, .n = pItem->pVar.nLen}; if (getColumnIndexByName(pCmd, &name, pQueryInfo, &columnIndex) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg17); } @@ -4687,7 +4716,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd, SQueryInfo* pQu } } - if (pExpr->functionId == TSDB_FUNC_PRJ || pExpr->functionId == TSDB_FUNC_DIFF || + if ((pExpr->functionId == TSDB_FUNC_PRJ && pExpr->numOfParams == 0) || pExpr->functionId == TSDB_FUNC_DIFF || pExpr->functionId == TSDB_FUNC_ARITHM) { isProjectionFunction = true; } @@ -4744,7 +4773,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { {"dDebugFlag", 10}, {"mqttDebugFlag", 13}, {"wDebugFlag", 10}, {"tmrDebugFlag", 12}, }; - SSQLToken* pOptionToken = &pOptions->a[1]; + SStrToken* pOptionToken = &pOptions->a[1]; if (pOptions->nTokens == 2) { // reset log and reset query cache does not need value @@ -4756,7 +4785,7 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { } } else if ((strncasecmp(cfgOptions[tokenBalance].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenBalance].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t vnodeId = 0; int32_t dnodeId = 0; strdequote(pValToken->z); @@ -4767,14 +4796,14 @@ int32_t validateDNodeConfig(tDCLSQL* pOptions) { return TSDB_CODE_SUCCESS; } else if ((strncasecmp(cfgOptions[tokenMonitor].name, pOptionToken->z, pOptionToken->n) == 0) && (cfgOptions[tokenMonitor].len == pOptionToken->n)) { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val != 0 && val != 1) { return TSDB_CODE_TSC_INVALID_SQL; // options value is invalid } return TSDB_CODE_SUCCESS; } else { - SSQLToken* pValToken = &pOptions->a[2]; + SStrToken* pValToken = &pOptions->a[2]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 0 || val > 256) { @@ -4803,7 +4832,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { SDNodeDynConfOption LOCAL_DYNAMIC_CFG_OPTIONS[6] = {{"resetLog", 8}, {"rpcDebugFlag", 12}, {"tmrDebugFlag", 12}, {"cDebugFlag", 10}, {"uDebugFlag", 10}, {"debugFlag", 9}}; - SSQLToken* pOptionToken = &pOptions->a[0]; + SStrToken* pOptionToken = &pOptions->a[0]; if (pOptions->nTokens == 1) { // reset log does not need value @@ -4814,7 +4843,7 @@ int32_t validateLocalConfig(tDCLSQL* pOptions) { } } } else { - SSQLToken* pValToken = &pOptions->a[1]; + SStrToken* pValToken = &pOptions->a[1]; int32_t val = strtol(pValToken->z, NULL, 10); if (val < 131 || val > 199) { @@ -4839,7 +4868,7 @@ int32_t validateColumnName(char* name) { return TSDB_CODE_TSC_INVALID_SQL; } - SSQLToken token = {.z = name}; + SStrToken token = {.z = name}; token.n = tSQLGetToken(name, &token.type); if (token.type != TK_STRING && token.type != TK_ID) { @@ -5018,7 +5047,7 @@ static int32_t setTimePrecision(SSqlCmd* pCmd, SCMCreateDbMsg* pMsg, SCreateDBIn pMsg->precision = TSDB_TIME_PRECISION_MILLI; // millisecond by default - SSQLToken* pToken = &pCreateDbInfo->precision; + SStrToken* pToken = &pCreateDbInfo->precision; if (pToken->n > 0) { pToken->n = strdequote(pToken->z); @@ -5689,7 +5718,7 @@ int32_t doCheckForCreateTable(SSqlObj* pSql, int32_t subClauseIndex, SSqlInfo* p assert(pFieldList != NULL); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5744,7 +5773,7 @@ int32_t doCheckForCreateFromStable(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pStableMeterMetaInfo = tscGetMetaInfo(pQueryInfo, STABLE_INDEX); // super table name, create table by using dst - SSQLToken* pToken = &(pCreateTable->usingInfo.stableName); + SStrToken* pToken = &(pCreateTable->usingInfo.stableName); if (tscValidateName(pToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); @@ -5837,7 +5866,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); // if sql specifies db, use it, otherwise use default db - SSQLToken* pzTableName = &(pCreateTable->name); + SStrToken* pzTableName = &(pCreateTable->name); SQuerySQL* pQuerySql = pCreateTable->pSelect; if (tscValidateName(pzTableName) != TSDB_CODE_SUCCESS) { @@ -5850,7 +5879,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } tVariant* pVar = &pSrcMeterName->a[0].pVar; - SSQLToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; + SStrToken srcToken = {.z = pVar->pz, .n = pVar->nLen, .type = TK_STRING}; if (tscValidateName(&srcToken) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } @@ -5865,7 +5894,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } bool isSTable = UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo); - if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + if (parseSelectClause(&pSql->cmd, 0, pQuerySql->pSelection, isSTable, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -5991,7 +6020,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { pTableItem->nLen = strdequote(pTableItem->pz); - SSQLToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; + SStrToken tableName = {.z = pTableItem->pz, .n = pTableItem->nLen, .type = TK_STRING}; if (tscValidateName(&tableName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg0); } @@ -6002,13 +6031,13 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { STableMetaInfo* pTableMetaInfo1 = tscGetMetaInfo(pQueryInfo, i/2); - SSQLToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; + SStrToken t = {.type = TSDB_DATA_TYPE_BINARY, .n = pTableItem->nLen, .z = pTableItem->pz}; if (tscSetTableFullName(pTableMetaInfo1, &t, pSql) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); } tVariant* pTableItem1 = &pQuerySql->from->a[i + 1].pVar; - SSQLToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; + SStrToken aliasName = {.z = pTableItem1->pz, .n = pTableItem1->nLen, .type = TK_STRING}; if (tscValidateName(&aliasName) != TSDB_CODE_SUCCESS) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg11); } @@ -6048,7 +6077,8 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable) != TSDB_CODE_SUCCESS) { + int32_t joinQuery = (pQuerySql->from != NULL && pQuerySql->from->nExpr > 2); + if (parseSelectClause(pCmd, index, pQuerySql->pSelection, isSTable, joinQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -6082,6 +6112,9 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } } else { // set the time rang pQueryInfo->window = TSWINDOW_INITIALIZER; + if (pQuerySql->from->nExpr > 2) { // it is a join query, no wher clause is not allowed. + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), "condition missing for join query "); + } } // user does not specified the query time window, twa is not allowed in such case. @@ -6131,7 +6164,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { if (pQueryInfo->intervalTime > 0) { int64_t timeRange = ABS(pQueryInfo->window.skey - pQueryInfo->window.ekey); // number of result is not greater than 10,000,000 - if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY) { + if ((timeRange == 0) || (timeRange / pQueryInfo->intervalTime) > MAX_INTERVAL_TIME_WINDOW) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } } @@ -6230,7 +6263,7 @@ int32_t exprTreeFromSqlExpr(SSqlCmd* pCmd, tExprNode **pExpr, const tSQLExpr* pS (*pExpr)->_node.pLeft = pLeft; (*pExpr)->_node.pRight = pRight; - SSQLToken t = {.type = pSqlExpr->nSQLOptr}; + SStrToken t = {.type = pSqlExpr->nSQLOptr}; (*pExpr)->_node.optr = getBinaryExprOptr(&t); assert((*pExpr)->_node.optr != 0); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 9282fa74fb..ecb85472fc 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -226,13 +226,17 @@ int tscSendMsgToServer(SSqlObj *pSql) { .handle = &pSql->pRpcCtx, .code = 0 }; - // NOTE: the rpc context should be acquired before sending data to server. // Otherwise, the pSql object may have been released already during the response function, which is // processMsgFromServer function. In the meanwhile, the assignment of the rpc context to sql object will absolutely // cause crash. - rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); - return TSDB_CODE_SUCCESS; + if (pObj != NULL && pObj->signature == pObj) { + rpcSendRequest(pObj->pDnodeConn, &pSql->epSet, &rpcMsg); + return TSDB_CODE_SUCCESS; + } else { + //pObj->signature has been reset by other thread, ignore concurrency problem + return TSDB_CODE_TSC_CONN_KILLED; + } } void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { @@ -314,10 +318,10 @@ void tscProcessMsgFromServer(SRpcMsg *rpcMsg, SRpcEpSet *pEpSet) { pRes->rspLen = 0; - if (pRes->code != TSDB_CODE_TSC_QUERY_CANCELLED) { - pRes->code = (rpcMsg->code != TSDB_CODE_SUCCESS) ? rpcMsg->code : TSDB_CODE_RPC_NETWORK_UNAVAIL; - } else { + if (pRes->code == TSDB_CODE_TSC_QUERY_CANCELLED) { tscDebug("%p query is cancelled, code:%s", pSql, tstrerror(pRes->code)); + } else { + pRes->code = rpcMsg->code; } if (pRes->code == TSDB_CODE_SUCCESS) { @@ -454,35 +458,21 @@ void tscKillSTableQuery(SSqlObj *pSql) { return; } + pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + for (int i = 0; i < pSql->numOfSubs; ++i) { + // NOTE: pSub may have been released already here SSqlObj *pSub = pSql->pSubs[i]; if (pSub == NULL) { continue; } - /* - * here, we cannot set the command = TSDB_SQL_KILL_QUERY. Otherwise, it may cause - * sub-queries not correctly released and master sql object of super table query reaches an abnormal state. - */ - rpcCancelRequest(pSub->pRpcCtx); pSub->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - tscQueueAsyncRes(pSub); - } - - /* - * 1. if the subqueries are not launched or partially launched, we need to waiting the launched - * query return to successfully free allocated resources. - * 2. if no any subqueries are launched yet, which means the super table query only in parse sql stage, - * set the res.code, and return. - */ - const int64_t MAX_WAITING_TIME = 10000; // 10 Sec. - int64_t stime = taosGetTimestampMs(); - - while (pCmd->command != TSDB_SQL_RETRIEVE_LOCALMERGE && pCmd->command != TSDB_SQL_RETRIEVE_EMPTY_RESULT) { - taosMsleep(100); - if (taosGetTimestampMs() - stime > MAX_WAITING_TIME) { - break; + if (pSub->pRpcCtx != NULL) { + rpcCancelRequest(pSub->pRpcCtx); } + + tscQueueAsyncRes(pSub); // async res? not other functions? } tscDebug("%p super table query cancelled", pSql); @@ -638,26 +628,29 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for query msg", pSql); - return -1; // todo add test for this + return TSDB_CODE_TSC_INVALID_SQL; // todo add test for this } SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableMeta * pTableMeta = pTableMetaInfo->pTableMeta; - - if (taosArrayGetSize(pQueryInfo->colList) <= 0 && !tscQueryTags(pQueryInfo)) { - tscError("%p illegal value of numOfCols in query msg: %d", pSql, tscGetNumOfColumns(pTableMeta)); - return -1; + + size_t numOfSrcCols = taosArrayGetSize(pQueryInfo->colList); + if (numOfSrcCols <= 0 && !tscQueryTags(pQueryInfo)) { + tscError("%p illegal value of numOfCols in query msg: %"PRIu64", table cols:%d", pSql, numOfSrcCols, + tscGetNumOfColumns(pTableMeta)); + + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->intervalTime < 0) { tscError("%p illegal value of aggregation time interval in query msg: %ld", pSql, pQueryInfo->intervalTime); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } if (pQueryInfo->groupbyExpr.numOfGroupCols < 0) { tscError("%p illegal value of numOfGroupCols in query msg: %d", pSql, pQueryInfo->groupbyExpr.numOfGroupCols); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } SQueryTableMsg *pQueryMsg = (SQueryTableMsg *)pCmd->payload; @@ -735,7 +728,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (pColFilter->lowerRelOptr == TSDB_RELATION_INVALID && pColFilter->upperRelOptr == TSDB_RELATION_INVALID) { tscError("invalid filter info"); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } } } @@ -744,10 +737,10 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { for (int32_t i = 0; i < tscSqlExprNumOfExprs(pQueryInfo); ++i) { SSqlExpr *pExpr = tscSqlExprGet(pQueryInfo, i); - if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId)) { + if (!tscValidateColumnId(pTableMetaInfo, pExpr->colInfo.colId, pExpr->numOfParams)) { /* column id is not valid according to the cached table meta, the table meta is expired */ tscError("%p table schema is not matched with parsed sql", pSql); - return -1; + return TSDB_CODE_TSC_INVALID_SQL; } pSqlFuncExpr->colInfo.colId = htons(pExpr->colInfo.colId); @@ -944,8 +937,8 @@ int32_t tscBuildAcctMsg(SSqlObj *pSql, SSqlInfo *pInfo) { SCMCreateAcctMsg *pAlterMsg = (SCMCreateAcctMsg *)pCmd->payload; - SSQLToken *pName = &pInfo->pDCLInfo->user.user; - SSQLToken *pPwd = &pInfo->pDCLInfo->user.passwd; + SStrToken *pName = &pInfo->pDCLInfo->user.user; + SStrToken *pPwd = &pInfo->pDCLInfo->user.passwd; strncpy(pAlterMsg->user, pName->z, pName->n); strncpy(pAlterMsg->pass, pPwd->z, pPwd->n); @@ -1147,13 +1140,13 @@ int32_t tscBuildShowMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { - SSQLToken *pPattern = &pShowInfo->pattern; + SStrToken *pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query strncpy(pShowMsg->payload, pPattern->z, pPattern->n); pShowMsg->payloadLen = htons(pPattern->n); } } else { - SSQLToken *pEpAddr = &pShowInfo->prefix; + SStrToken *pEpAddr = &pShowInfo->prefix; assert(pEpAddr->n > 0 && pEpAddr->type > 0); strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); @@ -1295,7 +1288,7 @@ int tscBuildAlterTableMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int size = tscEstimateAlterTableMsgLength(pCmd); if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { tscError("%p failed to malloc for alter table msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMAlterTableMsg *pAlterTableMsg = (SCMAlterTableMsg *)pCmd->payload; @@ -1443,6 +1436,12 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { SSqlRes *pRes = &pSql->res; SSqlCmd *pCmd = &pSql->cmd; + int32_t code = pRes->code; + if (pRes->code != TSDB_CODE_SUCCESS) { + tscQueueAsyncRes(pSql); + return code; + } + pRes->code = tscDoLocalMerge(pSql); SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); @@ -1453,7 +1452,7 @@ int tscProcessRetrieveLocalMergeRsp(SSqlObj *pSql) { pRes->row = 0; pRes->completed = (pRes->numOfRows == 0); - int32_t code = pRes->code; + code = pRes->code; if (pRes->code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); } else { @@ -1647,7 +1646,7 @@ int tscBuildHeartBeatMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (TSDB_CODE_SUCCESS != tscAllocPayload(pCmd, size)) { pthread_mutex_unlock(&pObj->mutex); tscError("%p failed to malloc for heartbeat msg", pSql); - return -1; + return TSDB_CODE_TSC_OUT_OF_MEMORY; } SCMHeartBeatMsg *pHeartbeat = (SCMHeartBeatMsg *)pCmd->payload; @@ -1717,7 +1716,7 @@ int tscProcessTableMetaRsp(SSqlObj *pSql) { assert(pTableMetaInfo->pTableMeta == NULL); pTableMetaInfo->pTableMeta = (STableMeta *) taosCachePut(tscCacheHandle, pTableMetaInfo->name, - strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer); + strlen(pTableMetaInfo->name), pTableMeta, size, tsTableMetaKeepTimer * 1000); // todo handle out of memory case if (pTableMetaInfo->pTableMeta == NULL) { @@ -1921,7 +1920,7 @@ int tscProcessShowRsp(SSqlObj *pSql) { STableMeta* pTableMeta = tscCreateTableMetaFromMsg(pMetaMsg, &size); pTableMetaInfo->pTableMeta = taosCachePut(tscCacheHandle, key, strlen(key), (char *)pTableMeta, size, - tsTableMetaKeepTimer); + tsTableMetaKeepTimer * 1000); SSchema *pTableSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); if (pQueryInfo->colList == NULL) { @@ -1961,8 +1960,12 @@ static void createHBObj(STscObj* pObj) { pSql->fp = tscProcessHeartBeatRsp; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pSql->cmd, 0, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(&pSql->cmd, 0); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return; + } + pQueryInfo->command = TSDB_SQL_HB; pSql->cmd.command = pQueryInfo->command; @@ -2147,8 +2150,7 @@ static int32_t getTableMetaFromMgmt(SSqlObj *pSql, STableMetaInfo *pTableMetaInf tscAddSubqueryInfo(&pNew->cmd); - SQueryInfo *pNewQueryInfo = NULL; - tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo); + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); pNew->cmd.autoCreated = pSql->cmd.autoCreated; // create table if not exists if (TSDB_CODE_SUCCESS != tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE + pSql->cmd.payloadLen)) { @@ -2251,8 +2253,8 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { pNew->cmd.command = TSDB_SQL_STABLEVGROUP; - SQueryInfo *pNewQueryInfo = NULL; - if ((code = tscGetQueryInfoDetailSafely(&pNew->cmd, 0, &pNewQueryInfo)) != TSDB_CODE_SUCCESS) { + SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetailSafely(&pNew->cmd, 0); + if (pNewQueryInfo == NULL) { tscFreeSqlObj(pNew); return code; } diff --git a/src/client/src/tscSql.c b/src/client/src/tscSql.c index de998a6b59..f63923e046 100644 --- a/src/client/src/tscSql.c +++ b/src/client/src/tscSql.c @@ -655,27 +655,30 @@ int* taos_fetch_lengths(TAOS_RES *res) { char *taos_get_client_info() { return version; } void taos_stop_query(TAOS_RES *res) { - if (res == NULL) { + SSqlObj *pSql = (SSqlObj *)res; + if (pSql == NULL || pSql->signature != pSql) { return; } - SSqlObj *pSql = (SSqlObj *)res; + tscDebug("%p start to cancel query", res); SSqlCmd *pCmd = &pSql->cmd; - if (pSql->signature != pSql) return; - tscDebug("%p start to cancel query", res); - - + // TODO there are multi-thread problem. + // It may have been released by the other thread already. + // The ref count may fix this problem. SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); - if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { - tscKillSTableQuery(pSql); - } - if (pSql->cmd.command < TSDB_SQL_LOCAL) { - rpcCancelRequest(pSql->pRpcCtx); - } + // set the error code for master pSqlObj firstly pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; - tscQueueAsyncRes(pSql); + + if (tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + assert(pSql->pRpcCtx == NULL); + tscKillSTableQuery(pSql); + } else { + if (pSql->cmd.command < TSDB_SQL_LOCAL) { + rpcCancelRequest(pSql->pRpcCtx); + } + } tscDebug("%p query is cancelled", res); } @@ -824,8 +827,11 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t int code = TSDB_CODE_TSC_INVALID_TABLE_ID_LENGTH; char *str = (char *)tblNameList; - SQueryInfo *pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex, &pQueryInfo); + SQueryInfo *pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, pCmd->clauseIndex); + if (pQueryInfo == NULL) { + pSql->res.code = terrno; + return terrno; + } STableMetaInfo *pTableMetaInfo = tscAddEmptyMetaInfo(pQueryInfo); @@ -850,7 +856,7 @@ static int tscParseTblNameList(SSqlObj *pSql, const char *tblNameList, int32_t t str = nextStr + 1; len = (int32_t)strtrim(tblName); - SSQLToken sToken = {.n = len, .type = TK_ID, .z = tblName}; + SStrToken sToken = {.n = len, .type = TK_ID, .z = tblName}; tSQLGetToken(tblName, &sToken.type); // Check if the table name available or not diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c022e320a3..2fb264c756 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -95,13 +95,14 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tscInfo("%" PRId64 ", tags:%d \t %" PRId64 ", tags:%d", elem1.ts, elem1.tag, elem2.ts, elem2.tag); #endif - if (elem1.tag < elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem1.ts, elem2.ts))) { + int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); + if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { if (!tsBufNextPos(pSupporter1->pTSBuf)) { break; } numOfInput1++; - } else if (elem1.tag > elem2.tag || (elem1.tag == elem2.tag && tsCompare(order, elem2.ts, elem1.ts))) { + } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) { if (!tsBufNextPos(pSupporter2->pTSBuf)) { break; } @@ -121,8 +122,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ win->ekey = elem1.ts; } - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); } else { pLimit->offset -= 1; } @@ -354,11 +355,12 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pExpr = tscSqlExprGet(pQueryInfo, 0); } - // set the join condition tag column info, to do extract method + // set the join condition tag column info, todo extract method if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { assert(pQueryInfo->tagCond.joinInfo.hasJoin); int16_t colId = tscGetJoinTagColIdByUid(&pQueryInfo->tagCond, pTableMetaInfo->pTableMeta->id.uid); + // set the tag column id for executor to extract correct tag value pExpr->param[0].i64Key = colId; pExpr->numOfParams = 1; } @@ -435,6 +437,7 @@ int32_t tscCompareTidTags(const void* p1, const void* p2) { if (t1->vgId != t2->vgId) { return (t1->vgId > t2->vgId) ? 1 : -1; } + if (t1->tid != t2->tid) { return (t1->tid > t2->tid) ? 1 : -1; } @@ -541,6 +544,7 @@ static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, for(int32_t i = 1; i < p1->num; ++i) { STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); + assert(prev->vgId >= 1 && p->vgId >= 1); if (doCompare(prev->tag, p->tag, pColSchema->type, pColSchema->bytes) == 0) { tscError("%p join tags have same value for different table, free all sub SqlObj and quit", pPSqlObj); @@ -577,6 +581,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar while(i < p1->num && j < p2->num) { STidTags* pp1 = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); STidTags* pp2 = (STidTags*) varDataVal(p2->pIdTagList + j * p2->tagSize); + assert(pp1->tid != 0 && pp2->tid != 0); int32_t ret = doCompare(pp1->tag, pp2->tag, pColSchema->type, pColSchema->bytes); if (ret == 0) { @@ -686,6 +691,7 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow freeJoinSubqueryObj(pParentSql); pParentSql->res.code = code; tscQueueAsyncRes(pParentSql); + taosArrayDestroy(s1); taosArrayDestroy(s2); return; @@ -1145,7 +1151,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { ///////////////////////////////////////////////////////////////////////////////////////// static void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code); -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj); int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter *pSupporter) { SSqlCmd * pCmd = &pSql->cmd; @@ -1218,6 +1224,16 @@ int32_t tscLaunchJoinSubquery(SSqlObj *pSql, int16_t tableIndex, SJoinSupporter int32_t tagColId = tscGetJoinTagColIdByUid(pTagCond, pTableMetaInfo->pTableMeta->id.uid); SSchema* s = tscGetTableColumnSchemaById(pTableMetaInfo->pTableMeta, tagColId); + // get the tag colId column index + int32_t numOfTags = tscGetNumOfTags(pTableMetaInfo->pTableMeta); + SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); + for(int32_t i = 0; i < numOfTags; ++i) { + if (pSchema[i].colId == tagColId) { + index.columnIndex = i; + break; + } + } + int16_t bytes = 0; int16_t type = 0; int32_t inter = 0; @@ -1287,8 +1303,14 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(pCmd, pCmd->clauseIndex); assert((pQueryInfo->type & TSDB_QUERY_TYPE_SUBQUERY) == 0); - + + // todo add test SSubqueryState *pState = calloc(1, sizeof(SSubqueryState)); + if (pState == NULL) { + pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; + return pSql->res.code; + } + pState->numOfTotal = pQueryInfo->numOfTables; pState->numOfRemain = pState->numOfTotal; @@ -1302,7 +1324,7 @@ int32_t tscHandleMasterJoinQuery(SSqlObj* pSql) { pSql->res.code = TSDB_CODE_TSC_OUT_OF_MEMORY; if (0 == i) { taosTFree(pState); - } + } return pSql->res.code; } @@ -1332,10 +1354,6 @@ static void doCleanupSubqueries(SSqlObj *pSql, int32_t numOfSubs, SSubqueryState SRetrieveSupport* pSupport = pSub->param; taosTFree(pSupport->localBuffer); - - pthread_mutex_unlock(&pSupport->queryMutex); - pthread_mutex_destroy(&pSupport->queryMutex); - taosTFree(pSupport); tscFreeSqlObj(pSub); @@ -1408,14 +1426,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { trs->pParentSql = pSql; trs->pFinalColModel = pModel; - pthread_mutexattr_t mutexattr; - memset(&mutexattr, 0, sizeof(pthread_mutexattr_t)); - - pthread_mutexattr_settype(&mutexattr, PTHREAD_MUTEX_RECURSIVE_NP); - pthread_mutex_init(&trs->queryMutex, &mutexattr); - pthread_mutexattr_destroy(&mutexattr); - - SSqlObj *pNew = tscCreateSqlObjForSubquery(pSql, trs, NULL); + SSqlObj *pNew = tscCreateSTableSubquery(pSql, trs, NULL); if (pNew == NULL) { tscError("%p failed to malloc buffer for subObj, orderOfSub:%d, reason:%s", pSql, i, strerror(errno)); taosTFree(trs->localBuffer); @@ -1460,15 +1471,16 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { } static void tscFreeSubSqlObj(SRetrieveSupport *trsupport, SSqlObj *pSql) { - tscDebug("%p start to free subquery result", pSql); - + tscDebug("%p start to free subquery obj", pSql); + + int32_t index = trsupport->subqueryIndex; + SSqlObj *pParentSql = trsupport->pParentSql; + + assert(pSql == pParentSql->pSubs[index]); + pParentSql->pSubs[index] = NULL; + taos_free_result(pSql); - taosTFree(trsupport->localBuffer); - - pthread_mutex_unlock(&trsupport->queryMutex); - pthread_mutex_destroy(&trsupport->queryMutex); - taosTFree(trsupport); } @@ -1477,23 +1489,11 @@ static void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, i static void tscAbortFurtherRetryRetrieval(SRetrieveSupport *trsupport, TAOS_RES *tres, int32_t code) { // set no disk space error info -#ifdef WINDOWS - LPVOID lpMsgBuf; - FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, - GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), // Default language - (LPTSTR)&lpMsgBuf, 0, NULL); - tscError("sub:%p failed to flush data to disk:reason:%s", tres, lpMsgBuf); - LocalFree(lpMsgBuf); -#else tscError("sub:%p failed to flush data to disk, reason:%s", tres, tstrerror(code)); -#endif - SSqlObj* pParentSql = trsupport->pParentSql; pParentSql->res.code = code; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - - pthread_mutex_unlock(&trsupport->queryMutex); tscHandleSubqueryError(trsupport, tres, pParentSql->res.code); } @@ -1512,13 +1512,10 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in // clear local saved number of results trsupport->localBuffer->num = 0; - pthread_mutex_unlock(&trsupport->queryMutex); - - tscTrace("%p sub:%p retrieve failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, + tscError("%p sub:%p retrieve/query failed, code:%s, orderOfSub:%d, retry:%d", trsupport->pParentSql, pSql, tstrerror(code), subqueryIndex, trsupport->numOfRetry); - SSqlObj *pNew = tscCreateSqlObjForSubquery(trsupport->pParentSql, trsupport, pSql); - + SSqlObj *pNew = tscCreateSTableSubquery(trsupport->pParentSql, trsupport, pSql); if (pNew == NULL) { tscError("%p sub:%p failed to create new subquery due to error:%s, abort retry, vgId:%d, orderOfSub:%d", trsupport->pParentSql, pSql, tstrerror(terrno), pVgroup->vgId, trsupport->subqueryIndex); @@ -1529,8 +1526,15 @@ static int32_t tscReissueSubquery(SRetrieveSupport *trsupport, SSqlObj *pSql, in return pParentSql->res.code; } - taos_free_result(pSql); - return tscProcessSql(pNew); + int32_t ret = tscProcessSql(pNew); + + // if failed to process sql, let following code handle the pSql + if (ret == TSDB_CODE_SUCCESS) { + taos_free_result(pSql); + return ret; + } else { + return ret; + } } void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numOfRows) { @@ -1550,14 +1554,14 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO */ pSql->res.numOfRows = 0; trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; // disable retry efforts - tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscDebug("%p query is cancelled, sub:%p, orderOfSub:%d abort retrieve, code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } if (numOfRows >= 0) { // current query is successful, but other sub query failed, still abort current query. tscDebug("%p sub:%p retrieve numOfRows:%d,orderOfSub:%d", pParentSql, pSql, numOfRows, subqueryIndex); - tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%d", pParentSql, pSql, - subqueryIndex, pParentSql->res.code); + tscError("%p sub:%p abort further retrieval due to other queries failure,orderOfSub:%d,code:%s", pParentSql, pSql, + subqueryIndex, tstrerror(pParentSql->res.code)); } else { if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY && pParentSql->res.code == TSDB_CODE_SUCCESS) { if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { @@ -1589,10 +1593,10 @@ void tscHandleSubqueryError(SRetrieveSupport *trsupport, SSqlObj *pSql, int numO taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - + // in case of second stage join subquery, invoke its callback function instead of regular QueueAsyncRes SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, 0); - + if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE)) { (*pParentSql->fp)(pParentSql->param, pParentSql, pParentSql->res.code); } else { // regular super table query @@ -1671,7 +1675,7 @@ static void tscAllDataRetrievedFromDnode(SRetrieveSupport *trsupport, SSqlObj* p // only free once taosTFree(trsupport->pState); tscFreeSubSqlObj(trsupport, pSql); - + // set the command flag must be after the semaphore been correctly set. pParentSql->cmd.command = TSDB_SQL_RETRIEVE_LOCALMERGE; if (pParentSql->res.code == TSDB_CODE_SUCCESS) { @@ -1687,24 +1691,22 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR int32_t idx = trsupport->subqueryIndex; SSqlObj * pParentSql = trsupport->pParentSql; + assert(tres != NULL); SSqlObj *pSql = (SSqlObj *)tres; - if (pSql == NULL) { // sql object has been released in error process, return immediately - tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); - return; - } - +// if (pSql == NULL) { // sql object has been released in error process, return immediately +// tscDebug("%p subquery has been released, idx:%d, abort", pParentSql, idx); +// return; +// } + SSubqueryState* pState = trsupport->pState; assert(pState->numOfRemain <= pState->numOfTotal && pState->numOfRemain >= 0 && pParentSql->numOfSubs == pState->numOfTotal); - // query process and cancel query process may execute at the same time - pthread_mutex_lock(&trsupport->queryMutex); - STableMetaInfo *pTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0); - SCMVgroupInfo* pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; + SCMVgroupInfo *pVgroup = &pTableMetaInfo->vgroupList->vgroups[0]; if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscDebug("%p query cancelled/failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(numOfRows), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, numOfRows); @@ -1715,13 +1717,13 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR assert(numOfRows == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(numOfRows), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, numOfRows) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); + tscDebug("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(numOfRows)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, numOfRows); // set global code and abort } @@ -1766,13 +1768,9 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR (int32_t)pRes->numOfRows, pQueryInfo->groupbyExpr.orderType); if (ret != 0) { // set no disk space error info, and abort retry tscAbortFurtherRetryRetrieval(trsupport, tres, TSDB_CODE_TSC_NO_DISKSPACE); - } else if (pRes->completed) { tscAllDataRetrievedFromDnode(trsupport, pSql); - return; - } else { // continue fetch data from dnode - pthread_mutex_unlock(&trsupport->queryMutex); taos_fetch_rows_a(tres, tscRetrieveFromDnodeCallBack, param); } @@ -1781,15 +1779,15 @@ static void tscRetrieveFromDnodeCallBack(void *param, TAOS_RES *tres, int numOfR } } -static SSqlObj *tscCreateSqlObjForSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { +static SSqlObj *tscCreateSTableSubquery(SSqlObj *pSql, SRetrieveSupport *trsupport, SSqlObj *prevSqlObj) { const int32_t table_index = 0; SSqlObj *pNew = createSubqueryObj(pSql, table_index, tscRetrieveDataRes, trsupport, TSDB_SQL_SELECT, prevSqlObj); if (pNew != NULL) { // the sub query of two-stage super table query SQueryInfo *pQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); + pQueryInfo->type |= TSDB_QUERY_TYPE_STABLE_SUBQUERY; - - assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1); + assert(pQueryInfo->numOfTables == 1 && pNew->cmd.numOfClause == 1 && trsupport->subqueryIndex < pSql->numOfSubs); // launch subquery for each vnode, so the subquery index equals to the vgroupIndex. STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, table_index); @@ -1806,7 +1804,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { SSqlObj* pParentSql = trsupport->pParentSql; SSqlObj* pSql = (SSqlObj *) tres; - + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, 0); assert(pSql->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); @@ -1816,7 +1814,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { // stable query killed or other subquery failed, all query stopped if (pParentSql->res.code != TSDB_CODE_SUCCESS) { trsupport->numOfRetry = MAX_NUM_OF_SUBQUERY_RETRY; - tscTrace("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", + tscError("%p query cancelled or failed, sub:%p, vgId:%d, orderOfSub:%d, code:%s, global code:%s", pParentSql, pSql, pVgroup->vgId, trsupport->subqueryIndex, tstrerror(code), tstrerror(pParentSql->res.code)); tscHandleSubqueryError(param, tres, code); @@ -1834,12 +1832,12 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { assert(code == taos_errno(pSql)); if (trsupport->numOfRetry++ < MAX_NUM_OF_SUBQUERY_RETRY) { - tscTrace("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); + tscError("%p sub:%p failed code:%s, retry:%d", pParentSql, pSql, tstrerror(code), trsupport->numOfRetry); if (tscReissueSubquery(trsupport, pSql, code) == TSDB_CODE_SUCCESS) { return; } } else { - tscTrace("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); + tscError("%p sub:%p reach the max retry times, set global code:%s", pParentSql, pSql, tstrerror(code)); atomic_val_compare_exchange_32(&pParentSql->res.code, TSDB_CODE_SUCCESS, code); // set global code and abort } @@ -1847,7 +1845,7 @@ void tscRetrieveDataRes(void *param, TAOS_RES *tres, int code) { return; } - tscTrace("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, + tscDebug("%p sub:%p query complete, ep:%s, vgId:%d, orderOfSub:%d, retrieve data", trsupport->pParentSql, pSql, pVgroup->epAddr[0].fqdn, pVgroup->vgId, trsupport->subqueryIndex); if (pSql->res.qhandle == 0) { // qhandle is NULL, code is TSDB_CODE_SUCCESS means no results generated from this vnode @@ -1926,8 +1924,14 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { size_t size = taosArrayGetSize(pCmd->pDataBlocks); assert(size > 0); - pSql->pSubs = calloc(size, POINTER_BYTES); + // the number of already initialized subqueries + int32_t numOfSub = 0; + pSql->numOfSubs = (uint16_t)size; + pSql->pSubs = calloc(size, POINTER_BYTES); + if (pSql->pSubs == NULL) { + goto _error; + } tscDebug("%p submit data to %" PRIzu " vnode(s)", pSql, size); @@ -1936,10 +1940,13 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { pState->numOfRemain = pSql->numOfSubs; pRes->code = TSDB_CODE_SUCCESS; - int32_t numOfSub = 0; while(numOfSub < pSql->numOfSubs) { SInsertSupporter* pSupporter = calloc(1, sizeof(SInsertSupporter)); + if (pSupporter == NULL) { + goto _error; + } + pSupporter->pSql = pSql; pSupporter->pState = pState; pSupporter->index = numOfSub; @@ -1972,7 +1979,7 @@ int32_t tscHandleMultivnodeInsert(SSqlObj *pSql) { if (numOfSub < pSql->numOfSubs) { tscError("%p failed to prepare subObj structure and launch sub-insertion", pSql); pRes->code = TSDB_CODE_TSC_OUT_OF_MEMORY; - return pRes->code; // free all allocated resource + goto _error; } pCmd->pDataBlocks = tscDestroyBlockArrayList(pCmd->pDataBlocks); @@ -2070,46 +2077,8 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { doBuildResFromSubqueries(pSql); tsem_post(&pSql->rspSem); - return; - - // continue retrieve data from vnode -// if (!tscHasRemainDataInSubqueryResultSet(pSql)) { -// tscDebug("%p at least one subquery exhausted, free all other %d subqueries", pSql, pSql->numOfSubs - 1); -// SSubqueryState* pState = NULL; -// -// // free all sub sqlobj -// for (int32_t i = 0; i < pSql->numOfSubs; ++i) { -// SSqlObj* pChildObj = pSql->pSubs[i]; -// if (pChildObj == NULL) { -// continue; -// } -// -// SJoinSupporter* pSupporter = (SJoinSupporter*)pChildObj->param; -// pState = pSupporter->pState; -// -// tscDestroyJoinSupporter(pChildObj->param); -// taos_free_result(pChildObj); -// } -// -// free(pState); -// -// pRes->completed = true; // set query completed -// tsem_post(&pSql->rspSem); -// return; -// } - -// tscFetchDatablockFromSubquery(pSql); -// if (pRes->code != TSDB_CODE_SUCCESS) { -// return; -// } } - -// if (pSql->res.code == TSDB_CODE_SUCCESS) { -// (*pSql->fp)(pSql->param, pSql, pRes->numOfRows); -// } else { -// tscQueueAsyncRes(pSql); -// } } static void transferNcharData(SSqlObj *pSql, int32_t columnIndex, TAOS_FIELD *pField) { @@ -2159,7 +2128,6 @@ void **doSetResultRowData(SSqlObj *pSql, bool finalResult) { SSqlRes *pRes = &pSql->res; assert(pRes->row >= 0 && pRes->row <= pRes->numOfRows); - if (pRes->row >= pRes->numOfRows) { // all the results has returned to invoker taosTFree(pRes->tsrow); return pRes->tsrow; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index fa148ebb2a..b61fd7e8c9 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -373,7 +373,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { if (pSql == NULL || pSql->signature != pSql) { return; } - + tscDebug("%p start to free sql object", pSql); tscPartiallyFreeSqlObj(pSql); @@ -388,6 +388,7 @@ void tscFreeSqlObj(SSqlObj* pSql) { taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); + free(pSql); } @@ -485,15 +486,6 @@ int32_t tscCopyDataBlockToPayload(SSqlObj* pSql, STableDataBlocks* pDataBlock) { return TSDB_CODE_SUCCESS; } -//void tscFreeUnusedDataBlocks(SDataBlockList* pList) { -// /* release additional memory consumption */ -// for (int32_t i = 0; i < pList->nSize; ++i) { -// STableDataBlocks* pDataBlock = pList->pData[i]; -// pDataBlock->pData = realloc(pDataBlock->pData, pDataBlock->size); -// pDataBlock->nAllocSize = (uint32_t)pDataBlock->size; -// } -//} - /** * create the in-memory buffer for each table to keep the submitted data block * @param initialSize @@ -518,6 +510,11 @@ int32_t tscCreateDataBlock(size_t initialSize, int32_t rowSize, int32_t startOff } dataBuf->pData = calloc(1, dataBuf->nAllocSize); + if (dataBuf->pData == NULL) { + tscError("failed to allocated memory, reason:%s", strerror(errno)); + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + dataBuf->ordered = true; dataBuf->prevTS = INT64_MIN; @@ -742,7 +739,7 @@ bool tscIsInsertData(char* sqlstr) { int32_t index = 0; do { - SSQLToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); + SStrToken t0 = tStrGetToken(sqlstr, &index, false, 0, NULL); if (t0.type != TK_LP) { return t0.type == TK_INSERT || t0.type == TK_IMPORT; } @@ -926,17 +923,23 @@ void tscFieldInfoClear(SFieldInfo* pFieldInfo) { } static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type, - int16_t size, int16_t interSize, bool isTagCol) { + int16_t size, int16_t interSize, int32_t colType) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex); SSqlExpr* pExpr = calloc(1, sizeof(SSqlExpr)); + if (pExpr == NULL) { + return NULL; + } + pExpr->functionId = functionId; - + // set the correct columnIndex index if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) { pExpr->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX; + } else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX) { + pExpr->colInfo.colId = pColIndex->columnIndex; } else { - if (isTagCol) { + if (TSDB_COL_IS_TAG(colType)) { SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta); pExpr->colInfo.colId = pSchema[pColIndex->columnIndex].colId; tstrncpy(pExpr->colInfo.name, pSchema[pColIndex->columnIndex].name, sizeof(pExpr->colInfo.name)); @@ -948,9 +951,9 @@ static SSqlExpr* doBuildSqlExpr(SQueryInfo* pQueryInfo, int16_t functionId, SCol } } - pExpr->colInfo.flag = isTagCol? TSDB_COL_TAG:TSDB_COL_NORMAL; - + pExpr->colInfo.flag = colType; pExpr->colInfo.colIndex = pColIndex->columnIndex; + pExpr->resType = type; pExpr->resBytes = size; pExpr->interBytes = interSize; @@ -1060,8 +1063,11 @@ void tscSqlExprCopy(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) if (deepcopy) { SSqlExpr* p1 = calloc(1, sizeof(SSqlExpr)); + if (p1 == NULL) { + assert(0); + } + *p1 = *pExpr; - for (int32_t j = 0; j < pExpr->numOfParams; ++j) { tVariantAssign(&p1->param[j], &pExpr->param[j]); } @@ -1097,16 +1103,22 @@ SColumn* tscColumnListInsert(SArray* pColumnList, SColumnIndex* pColIndex) { if (i >= numOfCols || numOfCols == 0) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } else { SColumn* pCol = taosArrayGetP(pColumnList, i); if (i < numOfCols && (pCol->colIndex.columnIndex > col || pCol->colIndex.tableIndex != pColIndex->tableIndex)) { SColumn* b = calloc(1, sizeof(SColumn)); + if (b == NULL) { + return NULL; + } + b->colIndex = *pColIndex; - taosArrayInsert(pColumnList, i, &b); } } @@ -1128,7 +1140,10 @@ SColumn* tscColumnClone(const SColumn* src) { assert(src != NULL); SColumn* dst = calloc(1, sizeof(SColumn)); - + if (dst == NULL) { + return NULL; + } + dst->colIndex = src->colIndex; dst->numOfFilters = src->numOfFilters; dst->filterInfo = tscFilterInfoClone(src->filterInfo, src->numOfFilters); @@ -1183,7 +1198,7 @@ void tscColumnListDestroy(SArray* pColumnList) { * 'first_part.second_part' * */ -static int32_t validateQuoteToken(SSQLToken* pToken) { +static int32_t validateQuoteToken(SStrToken* pToken) { strdequote(pToken->z); pToken->n = (uint32_t)strtrim(pToken->z); @@ -1199,7 +1214,7 @@ static int32_t validateQuoteToken(SSQLToken* pToken) { return TSDB_CODE_SUCCESS; } -int32_t tscValidateName(SSQLToken* pToken) { +int32_t tscValidateName(SStrToken* pToken) { if (pToken->type != TK_STRING && pToken->type != TK_ID) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -1286,12 +1301,12 @@ void tscIncStreamExecutionCount(void* pStream) { ps->num += 1; } -bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId) { +bool tscValidateColumnId(STableMetaInfo* pTableMetaInfo, int32_t colId, int32_t numOfParams) { if (pTableMetaInfo->pTableMeta == NULL) { return false; } - if (colId == TSDB_TBNAME_COLUMN_INDEX) { + if (colId == TSDB_TBNAME_COLUMN_INDEX || (colId <= TSDB_UD_COLUMN_INDEX && numOfParams == 2)) { return true; } @@ -1338,6 +1353,10 @@ void tscTagCondCopy(STagCond* dest, const STagCond* src) { if (pCond->len > 0) { assert(pCond->cond != NULL); c.cond = malloc(c.len); + if (c.cond == NULL) { + assert(0); + } + memcpy(c.cond, pCond->cond, c.len); } @@ -1463,20 +1482,20 @@ STableMetaInfo* tscGetMetaInfo(SQueryInfo* pQueryInfo, int32_t tableIndex) { return pQueryInfo->pTableMetaInfo[tableIndex]; } -int32_t tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex, SQueryInfo** pQueryInfo) { +SQueryInfo* tscGetQueryInfoDetailSafely(SSqlCmd* pCmd, int32_t subClauseIndex) { + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); int32_t ret = TSDB_CODE_SUCCESS; - *pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); - - while ((*pQueryInfo) == NULL) { + while ((pQueryInfo) == NULL) { if ((ret = tscAddSubqueryInfo(pCmd)) != TSDB_CODE_SUCCESS) { - return ret; + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; } - (*pQueryInfo) = tscGetQueryInfoDetail(pCmd, subClauseIndex); + pQueryInfo = tscGetQueryInfoDetail(pCmd, subClauseIndex); } - return TSDB_CODE_SUCCESS; + return pQueryInfo; } STableMetaInfo* tscGetTableMetaInfoByUid(SQueryInfo* pQueryInfo, uint64_t uid, int32_t* index) { @@ -1507,6 +1526,7 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo) { assert(pQueryInfo->exprList == NULL); pQueryInfo->exprList = taosArrayInit(4, POINTER_BYTES); pQueryInfo->colList = taosArrayInit(4, POINTER_BYTES); + pQueryInfo->udColumnId = TSDB_UD_COLUMN_INDEX; } int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { @@ -1522,8 +1542,11 @@ int32_t tscAddSubqueryInfo(SSqlCmd* pCmd) { pCmd->pQueryInfo = (SQueryInfo**)tmp; SQueryInfo* pQueryInfo = calloc(1, sizeof(SQueryInfo)); + if (pQueryInfo == NULL) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + tscInitQueryInfo(pQueryInfo); - pQueryInfo->msg = pCmd->payload; // pointer to the parent error message buffer pCmd->pQueryInfo[pCmd->numOfClause++] = pQueryInfo; @@ -1584,14 +1607,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST SVgroupsInfo* vgroupList, SArray* pTagCols) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } pQueryInfo->pTableMetaInfo = pAlloc; - pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = calloc(1, sizeof(STableMetaInfo)); + STableMetaInfo* pTableMetaInfo = calloc(1, sizeof(STableMetaInfo)); + if (pTableMetaInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + return NULL; + } - STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables]; - assert(pTableMetaInfo != NULL); + pQueryInfo->pTableMetaInfo[pQueryInfo->numOfTables] = pTableMetaInfo; if (name != NULL) { tstrncpy(pTableMetaInfo->name, name, sizeof(pTableMetaInfo->name)); @@ -1602,10 +1629,18 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST if (vgroupList != NULL) { size_t size = sizeof(SVgroupsInfo) + sizeof(SCMVgroupInfo) * vgroupList->numOfVgroups; pTableMetaInfo->vgroupList = malloc(size); + if (pTableMetaInfo->vgroupList == NULL) { + return NULL; + } + memcpy(pTableMetaInfo->vgroupList, vgroupList, size); } pTableMetaInfo->tagColList = taosArrayInit(4, POINTER_BYTES); + if (pTableMetaInfo->tagColList == NULL) { + return NULL; + } + if (pTagCols != NULL) { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } @@ -1671,8 +1706,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm return NULL; } - SQueryInfo* pQueryInfo = NULL; - tscGetQueryInfoDetailSafely(pCmd, 0, &pQueryInfo); + SQueryInfo* pQueryInfo = tscGetQueryInfoDetailSafely(pCmd, 0); assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); @@ -1754,6 +1788,7 @@ static void doSetSqlExprAndResultFieldInfo(SQueryInfo* pQueryInfo, SQueryInfo* p SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void* param, int32_t cmd, SSqlObj* pPrevSql) { SSqlCmd* pCmd = &pSql->cmd; + SSqlObj* pNew = (SSqlObj*)calloc(1, sizeof(SSqlObj)); if (pNew == NULL) { tscError("%p new subquery failed, tableIndex:%d", pSql, tableIndex); @@ -1769,10 +1804,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNew->sqlstr = strdup(pSql->sqlstr); if (pNew->sqlstr == NULL) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - - free(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SSqlCmd* pnCmd = &pNew->cmd; @@ -1789,9 +1822,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pnCmd->parseFinished = 1; if (tscAddSubqueryInfo(pnCmd) != TSDB_CODE_SUCCESS) { - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } SQueryInfo* pNewQueryInfo = tscGetQueryInfoDetail(pnCmd, 0); @@ -1816,20 +1848,28 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void pNewQueryInfo->groupbyExpr = pQueryInfo->groupbyExpr; if (pQueryInfo->groupbyExpr.columnInfo != NULL) { pNewQueryInfo->groupbyExpr.columnInfo = taosArrayClone(pQueryInfo->groupbyExpr.columnInfo); + if (pNewQueryInfo->groupbyExpr.columnInfo == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } } tscTagCondCopy(&pNewQueryInfo->tagCond, &pQueryInfo->tagCond); if (pQueryInfo->fillType != TSDB_FILL_NONE) { pNewQueryInfo->fillVal = malloc(pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); + if (pNewQueryInfo->fillVal == NULL) { + terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto _error; + } + memcpy(pNewQueryInfo->fillVal, pQueryInfo->fillVal, pQueryInfo->fieldsInfo.numOfOutput * sizeof(int64_t)); } if (tscAllocPayload(pnCmd, TSDB_DEFAULT_PAYLOAD_SIZE) != TSDB_CODE_SUCCESS) { tscError("%p new subquery failed, tableIndex:%d, vgroupIndex:%d", pSql, tableIndex, pTableMetaInfo->vgroupIndex); - tscFreeSqlObj(pNew); terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; + goto _error; } tscColumnListCopy(pNewQueryInfo->colList, pQueryInfo->colList, (int16_t)tableIndex); @@ -1872,16 +1912,15 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void if (pFinalInfo->pTableMeta == NULL) { tscError("%p new subquery failed since no tableMeta in cache, name:%s", pSql, name); - tscFreeSqlObj(pNew); - if (pPrevSql != NULL) { + if (pPrevSql != NULL) { // pass the previous error to client assert(pPrevSql->res.code != TSDB_CODE_SUCCESS); terrno = pPrevSql->res.code; } else { terrno = TSDB_CODE_TSC_APP_ERROR; } - return NULL; + goto _error; } assert(pNewQueryInfo->numOfTables == 1); @@ -1906,6 +1945,10 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void } return pNew; + +_error: + tscFreeSqlObj(pNew); + return NULL; } /** diff --git a/src/common/inc/tname.h b/src/common/inc/tname.h index 2263a5dae1..2a4ac3fc40 100644 --- a/src/common/inc/tname.h +++ b/src/common/inc/tname.h @@ -4,6 +4,7 @@ #include "os.h" #include "taosmsg.h" #include "tstoken.h" +#include "tvariant.h" typedef struct SDataStatis { int16_t colId; @@ -24,10 +25,12 @@ void extractTableName(const char *tableId, char *name); char* extractDBName(const char *tableId, char *name); -void extractTableNameFromToken(SSQLToken *pToken, SSQLToken* pTable); +void extractTableNameFromToken(SStrToken *pToken, SStrToken* pTable); SSchema tGetTableNameColumnSchema(); +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name); + bool tscValidateTableNameLength(size_t len); SColumnFilterInfo* tscFilterInfoClone(const SColumnFilterInfo* src, int32_t numOfFilters); diff --git a/src/common/inc/tvariant.h b/src/common/inc/tvariant.h index 4fd6ea5541..e9973bcb95 100644 --- a/src/common/inc/tvariant.h +++ b/src/common/inc/tvariant.h @@ -36,7 +36,7 @@ typedef struct tVariant { }; } tVariant; -void tVariantCreate(tVariant *pVar, SSQLToken *token); +void tVariantCreate(tVariant *pVar, SStrToken *token); void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type); @@ -46,6 +46,8 @@ void tVariantDestroy(tVariant *pV); void tVariantAssign(tVariant *pDst, const tVariant *pSrc); +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2); + int32_t tVariantToString(tVariant *pVar, char *dst); int32_t tVariantDump(tVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix); diff --git a/src/common/src/tname.c b/src/common/src/tname.c index d80ddb0ee2..01945dbb00 100644 --- a/src/common/src/tname.c +++ b/src/common/src/tname.c @@ -4,6 +4,7 @@ #include "tname.h" #include "tstoken.h" #include "ttokendef.h" +#include "tvariant.h" // todo refactor UNUSED_FUNC static FORCE_INLINE const char* skipSegments(const char* input, char delim, int32_t num) { @@ -43,7 +44,30 @@ SSchema tGetTableNameColumnSchema() { s.bytes = TSDB_TABLE_NAME_LEN - 1 + VARSTR_HEADER_SIZE; s.type = TSDB_DATA_TYPE_BINARY; s.colId = TSDB_TBNAME_COLUMN_INDEX; - strncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + tstrncpy(s.name, TSQL_TBNAME_L, TSDB_COL_NAME_LEN); + return s; +} + +SSchema tGetUserSpecifiedColumnSchema(tVariant* pVal, SStrToken* exprStr, const char* name) { + SSchema s = {0}; + + s.type = pVal->nType; + if (s.type == TSDB_DATA_TYPE_BINARY || s.type == TSDB_DATA_TYPE_NCHAR) { + s.bytes = (int16_t)(pVal->nLen + VARSTR_HEADER_SIZE); + } else { + s.bytes = tDataTypeDesc[pVal->nType].nSize; + } + + s.colId = TSDB_UD_COLUMN_INDEX; + if (name != NULL) { + tstrncpy(s.name, name, sizeof(s.name)); + } else { + size_t len = strdequote(exprStr->z); + size_t tlen = MIN(sizeof(s.name), len + 1); + + tstrncpy(s.name, exprStr->z, tlen); + } + return s; } @@ -110,7 +134,7 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in * tablePrefix.columnName * extract table name and save it in pTable, with only column name in pToken */ -void extractTableNameFromToken(SSQLToken* pToken, SSQLToken* pTable) { +void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) { const char sep = TS_PATH_DELIMITER[0]; if (pToken == pTable || pToken == NULL || pTable == NULL) { diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 5460d21252..069d8eb251 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -12,12 +12,10 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#include "os.h" #include "tvariant.h" #include "hash.h" -#include "hashfunc.h" -#include "os.h" -#include "hash.h" #include "taos.h" #include "taosdef.h" #include "tstoken.h" @@ -25,7 +23,7 @@ #include "tutil.h" // todo support scientific expression number and oct number -void tVariantCreate(tVariant *pVar, SSQLToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } +void tVariantCreate(tVariant *pVar, SStrToken *token) { tVariantCreateFromString(pVar, token->z, token->n, token->type); } void tVariantCreateFromString(tVariant *pVar, char *pz, uint32_t len, uint32_t type) { memset(pVar, 0, sizeof(tVariant)); @@ -102,10 +100,9 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 } case TSDB_DATA_TYPE_NCHAR: { // here we get the nchar length from raw binary bits length size_t lenInwchar = len / TSDB_NCHAR_SIZE; + pVar->wpz = calloc(1, (lenInwchar + 1) * TSDB_NCHAR_SIZE); - - wcsncpy(pVar->wpz, (wchar_t *)pz, lenInwchar); - pVar->wpz[lenInwchar] = 0; + memcpy(pVar->wpz, pz, lenInwchar * TSDB_NCHAR_SIZE); pVar->nLen = (int32_t)len; break; @@ -169,6 +166,50 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { char* n = strdup(p); taosArrayPush(pDst->arr, &n); } + + return; + } + + pDst->nLen = tDataTypeDesc[pDst->nType].nSize; +} + +int32_t tVariantCompare(const tVariant* p1, const tVariant* p2) { + if (p1->nType == TSDB_DATA_TYPE_NULL && p2->nType == TSDB_DATA_TYPE_NULL) { + return 0; + } + + if (p1->nType == TSDB_DATA_TYPE_NULL) { + return -1; + } + + if (p2->nType == TSDB_DATA_TYPE_NULL) { + return 1; + } + + switch (p1->nType) { + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: { + if (p1->nLen == p2->nLen) { + return memcmp(p1->pz, p2->pz, p1->nLen); + } else { + return p1->nLen > p2->nLen? 1:-1; + } + }; + + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + if (p1->dKey == p2->dKey) { + return 0; + } else { + return p1->dKey > p2->dKey? 1:-1; + } + + default: + if (p1->i64Key == p2->i64Key) { + return 0; + } else { + return p1->i64Key > p2->i64Key? 1:-1; + } } } @@ -228,7 +269,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -277,7 +318,7 @@ static int32_t doConvertToInteger(tVariant *pVariant, char *pDest, int32_t type, errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -436,7 +477,7 @@ static int32_t toNchar(tVariant *pVariant, char **pDest, int32_t *pDestSize) { } static FORCE_INLINE int32_t convertToDouble(char *pStr, int32_t len, double *value) { - SSQLToken stoken = {.z = pStr, .n = len}; + SStrToken stoken = {.z = pStr, .n = len}; if (TK_ILLEGAL == isValidNumber(&stoken)) { return -1; @@ -462,7 +503,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; char *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { @@ -479,7 +520,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result return 0; } - SSQLToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; + SStrToken sToken = {.z = pVariant->pz, .n = pVariant->nLen}; if (TK_ILLEGAL == isValidNumber(&sToken)) { return -1; } @@ -515,7 +556,7 @@ static FORCE_INLINE int32_t convertToInteger(tVariant *pVariant, int64_t *result errno = 0; wchar_t *endPtr = NULL; - SSQLToken token = {0}; + SStrToken token = {0}; token.n = tSQLGetToken(pVariant->pz, &token.type); if (token.type == TK_MINUS || token.type == TK_PLUS) { diff --git a/src/connector/jdbc/.classpath b/src/connector/jdbc/.classpath index 39abf1c5e9..a5d95095cc 100644 --- a/src/connector/jdbc/.classpath +++ b/src/connector/jdbc/.classpath @@ -28,22 +28,5 @@ - - - - - - - - - - - - - - - - - diff --git a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java index a36f7c67a0..06ae449596 100644 --- a/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java +++ b/src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java @@ -52,7 +52,7 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + // TODO make sure it is not a update query pSql = this.connecter.executeQuery(sql); @@ -68,21 +68,21 @@ public class TSDBStatement implements Statement { this.connecter.freeResultSet(pSql); return null; } - + if (!this.connecter.isUpdateQuery(pSql)) { return new TSDBResultSet(this.connecter, resultSetPointer); } else { this.connecter.freeResultSet(pSql); return null; } - + } public int executeUpdate(String sql) throws SQLException { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + // TODO check if current query is update query pSql = this.connecter.executeQuery(sql); long resultSetPointer = this.connecter.getResultSet(); @@ -94,7 +94,7 @@ public class TSDBStatement implements Statement { this.affectedRows = this.connecter.getAffectedRows(pSql); this.connecter.freeResultSet(pSql); - + return this.affectedRows; } @@ -192,7 +192,7 @@ public class TSDBStatement implements Statement { if (isClosed) { throw new SQLException("Invalid method call on a closed statement."); } - + return this.affectedRows; } diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index 6f32bc0f7a..56316e9619 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -119,11 +119,8 @@ int32_t main(int32_t argc, char *argv[]) { syslog(LOG_INFO, "Started TDengine service successfully."); - for (int res = tsem_wait(&exitSem); res != 0; res = tsem_wait(&exitSem)) { - if (res != EINTR) { - syslog(LOG_ERR, "failed to wait exit semphore: %d", res); - break; - } + if (tsem_wait(&exitSem) != 0) { + syslog(LOG_ERR, "failed to wait exit semphore: %s", strerror(errno)); } dnodeCleanUpSystem(); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index b6d185db9c..b61890347f 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -286,7 +286,8 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TSDB_MAX_REPLICA 5 -#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-100) #define TSDB_MULTI_METERMETA_MAX_NUM 100000 // maximum batch size allowed to load metermeta #define TSDB_MIN_CACHE_BLOCK_SIZE 1 diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index df69d1d19a..a789d002bf 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -97,6 +97,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_APP_ERROR, 0, 0x0211, "Applicatio TAOS_DEFINE_ERROR(TSDB_CODE_TSC_ACTION_IN_PROGRESS, 0, 0x0212, "Action in progress") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DISCONNECTED, 0, 0x0213, "Disconnected from service") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_WRITE_AUTH, 0, 0x0214, "No write permission") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, 0, 0x0215, "Connection killed") // mnode TAOS_DEFINE_ERROR(TSDB_CODE_MND_MSG_NOT_PROCESSED, 0, 0x0300, "Message not processed") @@ -221,6 +222,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_EXCEED_TAGS_LIMIT, 0, 0x0706, "Tag condit TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not ready") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/inc/taosmsg.h b/src/inc/taosmsg.h index 5e7da9bbdc..761a267ce5 100644 --- a/src/inc/taosmsg.h +++ b/src/inc/taosmsg.h @@ -133,7 +133,7 @@ enum _mgmt_table { TSDB_MGMT_TABLE_MODULE, TSDB_MGMT_TABLE_QUERIES, TSDB_MGMT_TABLE_STREAMS, - TSDB_MGMT_TABLE_CONFIGS, + TSDB_MGMT_TABLE_VARIABLES, TSDB_MGMT_TABLE_CONNS, TSDB_MGMT_TABLE_SCORES, TSDB_MGMT_TABLE_GRANTS, @@ -167,9 +167,9 @@ enum _mgmt_table { #define TSDB_VN_WRITE_ACCCESS ((char)0x2) #define TSDB_VN_ALL_ACCCESS (TSDB_VN_READ_ACCCESS | TSDB_VN_WRITE_ACCCESS) -#define TSDB_COL_NORMAL 0x0u -#define TSDB_COL_TAG 0x1u -#define TSDB_COL_JOIN 0x2u +#define TSDB_COL_NORMAL 0x0u // the normal column of the table +#define TSDB_COL_TAG 0x1u // the tag column type +#define TSDB_COL_UDC 0x2u // the user specified normal string column, it is a dummy column extern char *taosMsg[]; diff --git a/src/inc/ttokendef.h b/src/inc/ttokendef.h index e77f5741d3..c1579fe9e7 100644 --- a/src/inc/ttokendef.h +++ b/src/inc/ttokendef.h @@ -69,7 +69,7 @@ #define TK_QUERIES 51 #define TK_CONNECTIONS 52 #define TK_STREAMS 53 -#define TK_CONFIGS 54 +#define TK_VARIABLES 54 #define TK_SCORES 55 #define TK_GRANTS 56 #define TK_VNODES 57 diff --git a/src/kit/shell/src/shellEngine.c b/src/kit/shell/src/shellEngine.c index d82ddf8871..8f7f4d9324 100644 --- a/src/kit/shell/src/shellEngine.c +++ b/src/kit/shell/src/shellEngine.c @@ -25,6 +25,8 @@ #include "taosdef.h" #include "taoserror.h" #include "tglobal.h" +#include "tsclient.h" + #include /**************** Global variables ****************/ @@ -64,11 +66,6 @@ TAOS *shellInit(SShellArguments *args) { } taos_init(); - /* - * set tsTableMetaKeepTimer = 3000ms - * means not save cache in shell - */ - tsTableMetaKeepTimer = 3000; // Connect to the database. TAOS *con = NULL; @@ -287,7 +284,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { st = taosGetTimestampUs(); TAOS_RES* pSql = taos_query(con, command); - result = pSql; // set it into the global variable + atomic_store_ptr(&result, pSql); // set the global TAOS_RES pointer if (taos_errno(pSql)) { taos_error(pSql); @@ -298,17 +295,16 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { fprintf(stdout, "Database changed.\n\n"); fflush(stdout); - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } - int num_fields = taos_field_count(pSql); - if (num_fields != 0) { // select and show kinds of commands + if (!tscIsUpdateQuery(pSql)) { // select and show kinds of commands int error_no = 0; int numOfRows = shellDumpResult(pSql, fname, &error_no, printMode); if (numOfRows < 0) { - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); return; } @@ -331,7 +327,7 @@ void shellRunCommandOnServer(TAOS *con, char command[]) { wordfree(&full_path); } - result = NULL; + atomic_store_ptr(&result, 0); taos_free_result(pSql); } @@ -497,7 +493,6 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) { } while( row != NULL); result = NULL; - //taos_free_result(tres); fclose(fp); return numOfRows; @@ -802,8 +797,8 @@ void write_history() { } void taos_error(TAOS_RES *tres) { + atomic_store_ptr(&result, 0); fprintf(stderr, "\nDB error: %s\n", taos_errstr(tres)); - result = NULL; taos_free_result(tres); } diff --git a/src/kit/shell/src/shellMain.c b/src/kit/shell/src/shellMain.c index 8481f498dd..44de6641f6 100644 --- a/src/kit/shell/src/shellMain.c +++ b/src/kit/shell/src/shellMain.c @@ -18,11 +18,10 @@ pthread_t pid; -// TODO: IMPLEMENT INTERRUPT HANDLER. -void interruptHandler(int signum) { +void shellQueryInterruptHandler(int signum) { #ifdef LINUX - taos_stop_query(result); - result = NULL; + void* pResHandle = atomic_val_compare_exchange_64(&result, result, 0); + taos_stop_query(pResHandle); #else printf("\nReceive ctrl+c or other signal, quit shell.\n"); exit(0); @@ -86,7 +85,7 @@ int main(int argc, char* argv[]) { struct sigaction act; memset(&act, 0, sizeof(struct sigaction)); - act.sa_handler = interruptHandler; + act.sa_handler = shellQueryInterruptHandler; sigaction(SIGTERM, &act, NULL); sigaction(SIGINT, &act, NULL); diff --git a/src/mnode/src/mnodeDnode.c b/src/mnode/src/mnodeDnode.c index ccb85c3301..d5b9d18e37 100644 --- a/src/mnode/src/mnodeDnode.c +++ b/src/mnode/src/mnodeDnode.c @@ -49,7 +49,7 @@ static int32_t mnodeProcessCreateDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessDropDnodeMsg(SMnodeMsg *pMsg); static int32_t mnodeProcessCfgDnodeMsg(SMnodeMsg *pMsg); static void mnodeProcessCfgDnodeMsgRsp(SRpcMsg *rpcMsg) ; -static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *rpcMsg); +static int32_t mnodeProcessDnodeStatusMsg(SMnodeMsg *pMsg); static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); static int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn); static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn); @@ -161,8 +161,8 @@ int32_t mnodeInitDnodes() { mnodeAddPeerMsgHandle(TSDB_MSG_TYPE_DM_STATUS, mnodeProcessDnodeStatusMsg); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_MODULE, mnodeGetModuleMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_MODULE, mnodeRetrieveModules); - mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeGetConfigMeta); - mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_CONFIGS, mnodeRetrieveConfigs); + mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeGetConfigMeta); + mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VARIABLES, mnodeRetrieveConfigs); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_VNODES, mnodeGetVnodeMeta); mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_VNODES, mnodeRetrieveVnodes); mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_DNODE, mnodeGetDnodeMeta); @@ -733,7 +733,7 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo } static bool mnodeCheckModuleInDnode(SDnodeObj *pDnode, int32_t moduleType) { - uint32_t status = pDnode->moduleStatus & (1 << moduleType); + uint32_t status = pDnode->moduleStatus & (1u << moduleType); return status > 0; } @@ -758,7 +758,7 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = 40 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "end point"); + strcpy(pSchema[cols].name, "end_point"); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -792,7 +792,9 @@ static int32_t mnodeGetModuleMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pConn) { int32_t numOfRows = 0; - char * pWrite; + + char* pWrite; + char* moduleName[5] = { "MNODE", "HTTP", "MONITOR", "MQTT", "UNKNOWN" }; while (numOfRows < rows) { SDnodeObj *pDnode = NULL; @@ -807,28 +809,18 @@ int32_t mnodeRetrieveModules(SShowObj *pShow, char *data, int32_t rows, void *pC cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - strncpy(pWrite, pDnode->dnodeEp, pShow->bytes[cols]-1); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pDnode->dnodeEp, pShow->bytes[cols] - 1); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - switch (moduleType) { - case TSDB_MOD_MNODE: - strcpy(pWrite, "mnode"); - break; - case TSDB_MOD_HTTP: - strcpy(pWrite, "http"); - break; - case TSDB_MOD_MONITOR: - strcpy(pWrite, "monitor"); - break; - default: - strcpy(pWrite, "unknown"); - } + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, moduleName[moduleType], pShow->bytes[cols]); cols++; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; bool enable = mnodeCheckModuleInDnode(pDnode, moduleType); - strcpy(pWrite, enable ? "enable" : "disable"); + + char* v = enable? "enable":"disable"; + STR_TO_VARSTR(pWrite, v); cols++; numOfRows++; @@ -862,13 +854,13 @@ static int32_t mnodeGetConfigMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p pShow->bytes[cols] = TSDB_CFG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config name"); + tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; pShow->bytes[cols] = TSDB_CFG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; - strcpy(pSchema[cols].name, "config value"); + tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); pSchema[cols].bytes = htons(pShow->bytes[cols]); cols++; @@ -903,27 +895,32 @@ static int32_t mnodeRetrieveConfigs(SShowObj *pShow, char *data, int32_t rows, v int32_t cols = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; - snprintf(pWrite, TSDB_CFG_OPTION_LEN, "%s", cfg->option); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->option, TSDB_CFG_OPTION_LEN); + cols++; + int32_t t = 0; pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; switch (cfg->valType) { case TAOS_CFG_VTYPE_INT16: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int16_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_INT32: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%d", *((int32_t *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_FLOAT: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + t = snprintf(varDataVal(pWrite), TSDB_CFG_VALUE_LEN, "%f", *((float *)cfg->ptr)); + varDataSetLen(pWrite, t); numOfRows++; break; case TAOS_CFG_VTYPE_STRING: case TAOS_CFG_VTYPE_IPSTR: case TAOS_CFG_VTYPE_DIRECTORY: - snprintf(pWrite, TSDB_CFG_VALUE_LEN, "%s", (char *)cfg->ptr); + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, cfg->ptr, TSDB_CFG_VALUE_LEN); numOfRows++; break; default: diff --git a/src/mnode/src/mnodeProfile.c b/src/mnode/src/mnodeProfile.c index 34147f2e4b..06f992c26a 100644 --- a/src/mnode/src/mnodeProfile.c +++ b/src/mnode/src/mnodeProfile.c @@ -100,7 +100,7 @@ SConnObj *mnodeCreateConn(char *user, uint32_t ip, uint16_t port) { }; tstrncpy(connObj.user, user, sizeof(connObj.user)); - SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME*1000); + SConnObj *pConn = taosCachePut(tsMnodeConnCache, &connId, sizeof(int32_t), &connObj, sizeof(connObj), CONN_KEEP_TIME * 1000); mDebug("connId:%d, is created, user:%s ip:%s:%u", connId, user, taosIpStr(ip), port); return pConn; diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 8a84b66a33..e553853658 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -98,7 +98,7 @@ static char *mnodeGetShowType(int32_t showType) { case TSDB_MGMT_TABLE_MODULE: return "show modules"; case TSDB_MGMT_TABLE_QUERIES: return "show queries"; case TSDB_MGMT_TABLE_STREAMS: return "show streams"; - case TSDB_MGMT_TABLE_CONFIGS: return "show configs"; + case TSDB_MGMT_TABLE_VARIABLES: return "show configs"; case TSDB_MGMT_TABLE_CONNS: return "show connections"; case TSDB_MGMT_TABLE_SCORES: return "show scores"; case TSDB_MGMT_TABLE_GRANTS: return "show grants"; diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemphone.h index fd88d2d798..4280b458a6 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemphone.h @@ -23,7 +23,7 @@ extern "C" { #ifndef TAOS_OS_FUNC_SEMPHONE #define tsem_t sem_t #define tsem_init sem_init - #define tsem_wait sem_wait + int tsem_wait(tsem_t* sem); #define tsem_post sem_post #define tsem_destroy sem_destroy #endif diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemphone.c index 74f8859029..b91888845e 100644 --- a/src/os/src/detail/osSemphone.c +++ b/src/os/src/detail/osSemphone.c @@ -16,6 +16,18 @@ #define _DEFAULT_SOURCE #include "os.h" +#ifndef TAOS_OS_FUNC_SEMPHONE + +int tsem_wait(tsem_t* sem) { + int ret = 0; + do { + ret = sem_wait(sem); + } while (ret != 0 && errno == EINTR); + return ret; +} + +#endif + #ifndef TAOS_OS_FUNC_SEMPHONE_PTHREAD bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; } diff --git a/src/query/inc/qAst.h b/src/query/inc/qAst.h index ec568a6cdb..547616dee6 100644 --- a/src/query/inc/qAst.h +++ b/src/query/inc/qAst.h @@ -80,7 +80,7 @@ void tExprTreeTraverse(tExprNode *pExpr, SSkipList *pSkipList, SArray *result, S void tExprTreeCalcTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order, char *(*cb)(void *, const char*, int32_t)); -uint8_t getBinaryExprOptr(SSQLToken *pToken); +uint8_t getBinaryExprOptr(SStrToken *pToken); void tExprNodeDestroy(tExprNode *pNode, void (*fp)(void *)); void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b5487561b2..7093495763 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -33,6 +33,17 @@ struct SColumnFilterElem; typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2); typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); +typedef struct SPosInfo { + int32_t pageId:20; + int32_t rowId:12; +} SPosInfo; + +typedef struct SGroupResInfo { + int32_t groupId; + int32_t numOfDataPages; + SPosInfo pos; +} SGroupResInfo; + typedef struct SSqlGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information @@ -41,21 +52,12 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SPosInfo { - int32_t pageId; - int32_t rowId; -} SPosInfo; - -typedef struct SWindowStatus { - bool closed; -} SWindowStatus; - typedef struct SWindowResult { - uint16_t numOfRows; // number of rows of current time window - SWindowStatus status; // this result status: closed or opened SPosInfo pos; // Position of current result in disk-based output buffer + uint16_t numOfRows; // number of rows of current time window + bool closed; // this result status: closed or opened SResultInfo* resultInfo; // For each result column, there is a resultInfo - STimeWindow window; // The time window that current result covers. + TSKEY skey; // start key of current time window } SWindowResult; /** @@ -79,6 +81,7 @@ typedef struct SWindowResInfo { int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key int64_t threshold; // threshold to halt query and return the generated results. + int64_t interval; // time window interval } SWindowResInfo; typedef struct SColumnFilterElem { @@ -98,7 +101,7 @@ typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct TSKEY lastKey; int32_t groupIndex; // group id in table list int16_t queryRangeSet; // denote if the query range is set, only available for interval query - int64_t tag; + tVariant tag; STimeWindow win; STSCursor cur; void* pTable; // for retrieve the page id list @@ -121,8 +124,9 @@ typedef struct SQueryCostInfo { uint32_t loadBlockStatis; uint32_t discardBlocks; uint64_t elapsedTime; - uint64_t computTime; + uint64_t firstStageMergeTime; uint64_t internalSupSize; + uint64_t numOfTimeWindows; } SQueryCostInfo; typedef struct SQuery { @@ -189,21 +193,18 @@ typedef struct SQInfo { int64_t owner; // if it is in execution void* tsdb; int32_t vgId; - STableGroupInfo tableGroupInfo; // table id list < only includes the STable list> + STableGroupInfo tableGroupInfo; // table list SArray STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure SQueryRuntimeEnv runtimeEnv; - int32_t groupIndex; - int32_t offset; // offset in group result set of subgroup, todo refactor SArray* arrTableIdInfo; + int32_t groupIndex; /* * the query is executed position on which meter of the whole list. * when the index reaches the last one of the list, it means the query is completed. - * We later may refactor to remove this attribution by using another flag to denote - * whether a multimeter query is completed or not. */ int32_t tableIndex; - int32_t numOfGroupResultPages; + SGroupResInfo groupResInfo; void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; pthread_mutex_t lock; // used to synchronize the rsp/query threads diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index cd9618b223..69fc0bc7ef 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -26,7 +26,7 @@ extern "C" { #include "tstoken.h" #include "tvariant.h" -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken extern char tTokenTypeSwitcher[13]; #define toTSDBType(x) \ @@ -71,16 +71,16 @@ typedef struct SQuerySQL { struct tSQLExpr * pWhere; // where clause [optional] tVariantList * pGroupby; // groupby clause, only for tags[optional] tVariantList * pSortOrder; // orderby [optional] - SSQLToken interval; // interval [optional] - SSQLToken sliding; // sliding window [optional] + SStrToken interval; // interval [optional] + SStrToken sliding; // sliding window [optional] SLimitVal limit; // limit offset [optional] SLimitVal slimit; // group limit offset [optional] tVariantList * fillType; // fill type[optional] - SSQLToken selectToken; // sql string + SStrToken selectToken; // sql string } SQuerySQL; typedef struct SCreateTableSQL { - struct SSQLToken name; // meter name, create table [meterName] xxx + struct SStrToken name; // meter name, create table [meterName] xxx bool existCheck; int8_t type; // create normal table/from super table/ stream @@ -90,7 +90,7 @@ typedef struct SCreateTableSQL { } colInfo; struct { - SSQLToken stableName; // super table name, for using clause + SStrToken stableName; // super table name, for using clause tVariantList *pTagVals; // create by using metric, tag value STagData tagdata; } usingInfo; @@ -99,7 +99,7 @@ typedef struct SCreateTableSQL { } SCreateTableSQL; typedef struct SAlterTableSQL { - SSQLToken name; + SStrToken name; int16_t type; STagData tagData; @@ -108,7 +108,7 @@ typedef struct SAlterTableSQL { } SAlterTableSQL; typedef struct SCreateDBInfo { - SSQLToken dbname; + SStrToken dbname; int32_t replica; int32_t cacheBlockSize; int32_t maxTablesPerVnode; @@ -121,7 +121,7 @@ typedef struct SCreateDBInfo { int32_t walLevel; int32_t quorum; int32_t compressionLevel; - SSQLToken precision; + SStrToken precision; bool ignoreExists; tVariantList *keep; @@ -136,33 +136,33 @@ typedef struct SCreateAcctSQL { int64_t maxStorage; int64_t maxQueryTime; int32_t maxConnections; - SSQLToken stat; + SStrToken stat; } SCreateAcctSQL; typedef struct SShowInfo { uint8_t showType; - SSQLToken prefix; - SSQLToken pattern; + SStrToken prefix; + SStrToken pattern; } SShowInfo; typedef struct SUserInfo { - SSQLToken user; - SSQLToken passwd; - SSQLToken privilege; + SStrToken user; + SStrToken passwd; + SStrToken privilege; int16_t type; } SUserInfo; typedef struct tDCLSQL { int32_t nTokens; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ - SSQLToken *a; /* one entry for element */ + SStrToken *a; /* one entry for element */ bool existsCheck; union { SCreateDBInfo dbOpt; SCreateAcctSQL acctOpt; SShowInfo showOpt; - SSQLToken ip; + SStrToken ip; }; SUserInfo user; @@ -194,14 +194,14 @@ typedef struct tSQLExpr { // the full sql string of function(col, param), which is actually the raw // field name, since the function name is kept in nSQLOptr already - SSQLToken operand; - SSQLToken colInfo; // field id + SStrToken operand; + SStrToken colInfo; // field id tVariant val; // value only for string, float, int struct tSQLExpr *pLeft; // left child struct tSQLExpr *pRight; // right child - struct tSQLExprList *pParam; // function parameters + SStrToken token; // original sql expr string } tSQLExpr; // used in select clause. select from xxx @@ -210,6 +210,7 @@ typedef struct tSQLExprItem { char * aliasName; // alias name, null-terminated string } tSQLExprItem; +// todo refactor by using SArray typedef struct tSQLExprList { int32_t nExpr; /* Number of expressions on the list */ int32_t nAlloc; /* Number of entries allocated below */ @@ -242,7 +243,7 @@ tVariantList *tVariantListAppend(tVariantList *pList, tVariant *pVar, uint8_t so tVariantList *tVariantListInsert(tVariantList *pList, tVariant *pVar, uint8_t sortOrder, int32_t index); -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder); +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pAliasToken, uint8_t sortOrder); void tVariantListDestroy(tVariantList *pList); tFieldList *tFieldListAppend(tFieldList *pList, TAOS_FIELD *pField); @@ -253,61 +254,61 @@ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optType); void tSQLExprDestroy(tSQLExpr *); -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken); +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken); void tSQLExprListDestroy(tSQLExprList *pList); -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit); -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pMetricName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pMetricName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type); void tSQLExprNodeDestroy(tSQLExpr *pExpr); tSQLExpr *tSQLExprNodeClone(tSQLExpr *pExpr); -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type); tSQLExprListList *tSQLListListAppend(tSQLExprListList *pList, tSQLExprList *pExprList); void destroyAllSelectClause(SSubclauseInfo *pSql); void doDestroyQuerySql(SQuerySQL *pSql); -SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type); +SSqlInfo * setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type); SSubclauseInfo *setSubclause(SSubclauseInfo *pClause, void *pSqlExprInfo); SSubclauseInfo *appendSelectClause(SSubclauseInfo *pInfo, void *pSubclause); -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists); +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists); void SQLInfoDestroy(SSqlInfo *pInfo); void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck); -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns); +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns); -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken); +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken); -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists); +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists); -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo); -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd); -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip); -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege); +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo); +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd); +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip); +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege); void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo); // prefix show db.tables; -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB); +void setDBName(SStrToken *pCpxName, SStrToken *pDB); -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pToken, int32_t optType); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optType); -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType); +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType); -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType); +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType); -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *pToken); +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *pToken); void *ParseAlloc(void *(*mallocProc)(size_t)); diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 59b224e096..46e6f79014 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -22,6 +22,7 @@ extern "C" { #include "os.h" #include "taosdef.h" +#include "tvariant.h" #define MEM_BUF_SIZE (1 << 20) #define TS_COMP_FILE_MAGIC 0x87F5EC4C @@ -42,9 +43,9 @@ typedef struct STSRawBlock { } STSRawBlock; typedef struct STSElem { - TSKEY ts; - int64_t tag; - int32_t vnode; + TSKEY ts; + tVariant tag; + int32_t vnode; } STSElem; typedef struct STSCursor { @@ -55,11 +56,11 @@ typedef struct STSCursor { } STSCursor; typedef struct STSBlock { - int64_t tag; // tag value - int32_t numOfElem; // number of elements - int32_t compLen; // size after compressed - int32_t padding; // 0xFFFFFFFF by default, after the payload - char* payload; // actual data that is compressed + tVariant tag; // tag value + int32_t numOfElem; // number of elements + int32_t compLen; // size after compressed + int32_t padding; // 0xFFFFFFFF by default, after the payload + char* payload; // actual data that is compressed } STSBlock; /* @@ -84,8 +85,8 @@ typedef struct STSBuf { uint32_t fileSize; STSVnodeBlockInfoEx* pData; - int32_t numOfAlloc; - int32_t numOfVnodes; + uint32_t numOfAlloc; + uint32_t numOfVnodes; char* assistBuf; int32_t bufSize; @@ -109,7 +110,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ void* tsBufDestroy(STSBuf* pTSBuf); -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len); +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len); int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); STSBuf* tsBufClone(STSBuf* pTSBuf); @@ -122,7 +123,7 @@ void tsBufResetPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); bool tsBufNextPos(STSBuf* pTSBuf); -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag); +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag); STSCursor tsBufGetCursor(STSBuf* pTSBuf); void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index b7f57bf1b5..6de3c7c0e5 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -38,7 +38,8 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf return &pWindowResInfo->pResult[slot]; } -#define curTimeWindow(_winres) ((_winres)->curIndex) +#define curTimeWindowIndex(_winres) ((_winres)->curIndex) +#define GET_TIMEWINDOW(_winresInfo, _win) (STimeWindow) {(_win)->skey, ((_win)->skey + (_winresInfo)->interval - 1)} #define GET_ROW_PARAM_FOR_MULTIOUTPUT(_q, tbq, sq) (((tbq) && (!sq))? (_q)->pSelectExpr[1].base.arg->argValue.i64:1) bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); diff --git a/src/query/inc/sql.y b/src/query/inc/sql.y index 2b2a967edf..7500dcba56 100644 --- a/src/query/inc/sql.y +++ b/src/query/inc/sql.y @@ -2,8 +2,8 @@ //usage: lemon sql.y %token_prefix TK_ -%token_type {SSQLToken} -%default_type {SSQLToken} +%token_type {SStrToken} +%default_type {SStrToken} %extra_argument {SSqlInfo* pInfo} %fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR. @@ -73,18 +73,18 @@ cmd ::= SHOW MODULES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_MODULE, 0, 0); cmd ::= SHOW QUERIES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_QUERIES, 0, 0); } cmd ::= SHOW CONNECTIONS.{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONNS, 0, 0);} cmd ::= SHOW STREAMS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } -cmd ::= SHOW CONFIGS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } +cmd ::= SHOW VARIABLES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } cmd ::= SHOW SCORES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } cmd ::= SHOW GRANTS. { setShowOptions(pInfo, TSDB_MGMT_TABLE_GRANTS, 0, 0); } cmd ::= SHOW VNODES. { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, 0, 0); } cmd ::= SHOW VNODES IPTOKEN(X). { setShowOptions(pInfo, TSDB_MGMT_TABLE_VNODES, &X, 0); } -%type dbPrefix {SSQLToken} +%type dbPrefix {SStrToken} dbPrefix(A) ::=. {A.n = 0; A.type = 0;} dbPrefix(A) ::= ids(X) DOT. {A = X; } -%type cpxName {SSQLToken} +%type cpxName {SStrToken} cpxName(A) ::= . {A.n = 0; } cpxName(A) ::= DOT ids(Y). {A = Y; A.n += 1; } @@ -101,19 +101,19 @@ cmd ::= SHOW dbPrefix(X) STABLES. { } cmd ::= SHOW dbPrefix(X) STABLES LIKE ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &Y); } cmd ::= SHOW dbPrefix(X) VGROUPS. { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } cmd ::= SHOW dbPrefix(X) VGROUPS ids(Y). { - SSQLToken token; + SStrToken token; setDBName(&token, &X); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &Y); } @@ -145,7 +145,7 @@ cmd ::= ALTER DNODE ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL cmd ::= ALTER DNODE ids(X) ids(Y) ids(Z). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_DNODE, 3, &X, &Y, &Z); } cmd ::= ALTER LOCAL ids(X). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 1, &X); } cmd ::= ALTER LOCAL ids(X) ids(Y). { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &X, &Y); } -cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} +cmd ::= ALTER DATABASE ids(X) alter_db_optr(Y). { SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &X, &Y, &t);} cmd ::= ALTER ACCOUNT ids(X) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, NULL, &Z);} cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &X, &Y, &Z);} @@ -153,15 +153,15 @@ cmd ::= ALTER ACCOUNT ids(X) PASS ids(Y) acct_optr(Z). { setCreateAcctSQL(p // An IDENTIFIER can be a generic identifier, or one of several keywords. // Any non-standard keyword can also be an identifier. // And "ids" is an identifer-or-string. -%type ids {SSQLToken} +%type ids {SStrToken} ids(A) ::= ID(X). {A = X; } ids(A) ::= STRING(X). {A = X; } -%type ifexists {SSQLToken} +%type ifexists {SStrToken} ifexists(X) ::= IF EXISTS. {X.n = 1;} ifexists(X) ::= . {X.n = 0;} -%type ifnotexists {SSQLToken} +%type ifnotexists {SStrToken} ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} ifnotexists(X) ::= . {X.n = 0;} @@ -416,7 +416,7 @@ selcollist(A) ::= sclp(P) STAR. { // An option "AS " phrase that can follow one of the expressions that // define the result set, or one of the tables in the FROM clause. // -%type as {SSQLToken} +%type as {SStrToken} as(X) ::= AS ids(Y). { X = Y; } as(X) ::= ids(Y). { X = Y; } as(X) ::= . { X.n = 0; } @@ -458,10 +458,10 @@ tablelist(A) ::= tablelist(Y) COMMA ids(X) cpxName(Z) ids(F). { } // The value of interval should be the form of "number+[a,s,m,h,d,n,y]" or "now" -%type tmvar {SSQLToken} +%type tmvar {SStrToken} tmvar(A) ::= VARIABLE(X). {A = X;} -%type interval_opt {SSQLToken} +%type interval_opt {SStrToken} interval_opt(N) ::= INTERVAL LP tmvar(E) RP. {N = E; } interval_opt(N) ::= . {N.n = 0; N.z = NULL; N.type = 0; } @@ -482,7 +482,7 @@ fill_opt(N) ::= FILL LP ID(Y) RP. { N = tVariantListAppendToken(NULL, &Y, -1); } -%type sliding_opt {SSQLToken} +%type sliding_opt {SStrToken} sliding_opt(K) ::= SLIDING LP tmvar(E) RP. {K = E; } sliding_opt(K) ::= . {K.n = 0; K.z = NULL; K.type = 0; } diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 58d2ff260e..65ab82883b 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -91,16 +91,16 @@ extern "C" { #define QUERY_COND_REL_PREFIX_IN "IN|" #define QUERY_COND_REL_PREFIX_LIKE "LIKE|" -#define QUERY_COND_REL_PREFIX_IN_LEN 3 +#define QUERY_COND_REL_PREFIX_IN_LEN 3 #define QUERY_COND_REL_PREFIX_LIKE_LEN 5 -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) -#define MAX_RETRIEVE_ROWS_IN_INTERVAL_QUERY 10000000 -#define TOP_BOTTOM_QUERY_LIMIT 100 +#define MAX_INTERVAL_TIME_WINDOW 10000000 +#define TOP_BOTTOM_QUERY_LIMIT 100 enum { MASTER_SCAN = 0x0u, @@ -137,13 +137,13 @@ typedef struct SInterpInfoDetail { } SInterpInfoDetail; typedef struct SResultInfo { - int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - bool superTableQ; // is super table query - int32_t numOfRes; // num of output result in current buffer - int32_t bufLen; // buffer size - void* interResultBuf; // output result buffer + int8_t hasResult; // result generated, not NULL value + bool initialized; // output buffer has been initialized + bool complete; // query has completed + bool superTableQ; // is super table query + uint32_t bufLen; // buffer size + uint64_t numOfRes; // num of output result in current buffer + void* interResultBuf; // output result buffer } SResultInfo; struct SQLFunctionCtx; diff --git a/src/query/src/qAst.c b/src/query/src/qAst.c index afa3618a96..634f014d97 100644 --- a/src/query/src/qAst.c +++ b/src/query/src/qAst.c @@ -44,7 +44,7 @@ * ver 0.3, pipeline filter in the form of: (a+2)/9 > 14 * */ -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken); +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken); static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i); static void destroySyntaxTree(tExprNode *); @@ -103,7 +103,7 @@ static void reviseBinaryExprIfNecessary(tExprNode **pLeft, tExprNode **pRight, u } } -static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken *pToken) { +static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SStrToken *pToken) { /* if the token is not a value, return false */ if (pToken->type == TK_RP || (pToken->type != TK_INTEGER && pToken->type != TK_FLOAT && pToken->type != TK_ID && pToken->type != TK_TBNAME && pToken->type != TK_STRING && pToken->type != TK_BOOL)) { @@ -117,7 +117,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken int32_t i = 0; if (pToken->type == TK_ID) { do { - SSQLToken tableToken = {0}; + SStrToken tableToken = {0}; extractTableNameFromToken(pToken, &tableToken); size_t len = strlen(pSchema[i].name); @@ -157,7 +157,7 @@ static tExprNode *tExprNodeCreate(SSchema *pSchema, int32_t numOfCols, SSQLToken return pNode; } -uint8_t getBinaryExprOptr(SSQLToken *pToken) { +uint8_t getBinaryExprOptr(SStrToken *pToken) { switch (pToken->type) { case TK_LT: return TSDB_RELATION_LESS; @@ -234,7 +234,7 @@ uint8_t isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLef } static tExprNode *createSyntaxTree(SSchema *pSchema, int32_t numOfCols, char *str, int32_t *i) { - SSQLToken t0 = tStrGetToken(str, i, false, 0, NULL); + SStrToken t0 = tStrGetToken(str, i, false, 0, NULL); if (t0.n == 0) { return NULL; } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 03b699c1b8..4e2e31d269 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -27,7 +27,8 @@ #include "query.h" #include "queryLog.h" #include "tlosertree.h" -#include "tscompression.h" + +#define MAX_ROWS_PER_RESBUF_PAGE ((1u<<12) - 1) /** * check if the primary column is load by default, otherwise, the program will @@ -35,6 +36,9 @@ */ #define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0) #define TSDB_COL_IS_TAG(f) (((f)&TSDB_COL_TAG) != 0) +#define TSDB_COL_IS_NORMAL_COL(f) ((f) == TSDB_COL_NORMAL) +#define TSDB_COL_IS_UD_COL(f) ((f) == TSDB_COL_UDC) + #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) @@ -93,7 +97,8 @@ typedef struct { #if 0 static UNUSED_FUNC void *u_malloc (size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + + if (v % 1000 <= 0) { return NULL; } else { return malloc(__size); @@ -102,15 +107,25 @@ static UNUSED_FUNC void *u_malloc (size_t __size) { static UNUSED_FUNC void* u_calloc(size_t num, size_t __size) { uint32_t v = rand(); - if (v % 5 <= 1) { + if (v % 1000 <= 0) { return NULL; } else { return calloc(num, __size); } } +static UNUSED_FUNC void* u_realloc(void* p, size_t __size) { + uint32_t v = rand(); + if (v % 5 <= 1) { + return NULL; + } else { + return realloc(p, __size); + } +} + #define calloc u_calloc #define malloc u_malloc +#define realloc u_realloc #endif #define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st))) @@ -151,7 +166,7 @@ static bool hasMainOutput(SQuery *pQuery); static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); -static int32_t flushFromResultBuf(SQInfo *pQInfo); +static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -212,22 +227,22 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { */ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { SQuery *pQuery = pRuntimeEnv->pQuery; - + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); - + int16_t functionId = pRuntimeEnv->pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TS_DUMMY) { continue; } - + assert(pResInfo->numOfRes > numOfRes); pResInfo->numOfRes = numOfRes; } } -static int32_t getGroupResultId(int32_t groupIndex) { +static UNUSED_FUNC int32_t getGroupResultId(int32_t groupIndex) { int32_t base = 20000000; return base + (groupIndex * 10000); } @@ -315,10 +330,10 @@ bool isTSCompQuery(SQuery *pQuery) { return pQuery->pSelectExpr[0].base.function static bool limitResults(SQueryRuntimeEnv* pRuntimeEnv) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery* pQuery = pRuntimeEnv->pQuery; - + if ((pQuery->limit.limit > 0) && (pQuery->rec.total + pQuery->rec.rows > pQuery->limit.limit)) { pQuery->rec.rows = pQuery->limit.limit - pQuery->rec.total; - + qDebug("QInfo:%p discard remain data due to result limitation, limit:%"PRId64", current return:%" PRId64 ", total:%"PRId64, pQInfo, pQuery->limit.limit, pQuery->rec.rows, pQuery->rec.total + pQuery->rec.rows); assert(pQuery->rec.rows >= 0); @@ -371,14 +386,14 @@ static bool hasTagValOutput(SQuery* pQuery) { * @return */ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis **pColStatis) { - if (pStatis != NULL && !TSDB_COL_IS_TAG(pColIndex->flag)) { + if (pStatis != NULL && TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { *pColStatis = &pStatis[pColIndex->colIndex]; assert((*pColStatis)->colId == pColIndex->colId); } else { *pColStatis = NULL; } - if (TSDB_COL_IS_TAG(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (TSDB_COL_IS_TAG(pColIndex->flag) || TSDB_COL_IS_UD_COL(pColIndex->flag) || pColIndex->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { return false; } @@ -412,6 +427,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult))); pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); + pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -425,7 +441,10 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { - createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } } pWindowResInfo->capacity = (int32_t)newCap; @@ -436,6 +455,11 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } + // too many time window in query + if (pWindowResInfo->size > MAX_INTERVAL_TIME_WINDOW) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); + } + return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); } @@ -447,8 +471,9 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t w.skey = pWindowResInfo->prevSKey; w.ekey = w.skey + pQuery->intervalTime - 1; } else { - int32_t slot = curTimeWindow(pWindowResInfo); - w = getWindowResult(pWindowResInfo, slot)->window; + int32_t slot = curTimeWindowIndex(pWindowResInfo); + SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); + w = GET_TIMEWINDOW(pWindowResInfo, pWindowRes); } if (w.skey > ts || w.ekey < ts) { @@ -549,15 +574,15 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes } // set time window for current result - pWindowRes->window = *win; + pWindowRes->skey = win->skey; setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); return TSDB_CODE_SUCCESS; } -static SWindowStatus *getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { +static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); - return &pWindowResInfo->pResult[slot].status; + return pWindowResInfo->pResult[slot].closed; } static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, @@ -599,7 +624,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe // no qualified results exist, abort check int32_t numOfClosed = 0; - + if (pWindowResInfo->size == 0) { return pWindowResInfo->size; } @@ -617,16 +642,17 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe for (i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { + if (pResult->closed) { numOfClosed += 1; continue; } - if ((pResult->window.ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || - (pResult->window.skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { + TSKEY ekey = pResult->skey + pWindowResInfo->interval; + if ((ekey <= lastKey && QUERY_IS_ASC_QUERY(pQuery)) || + (pResult->skey >= lastKey && !QUERY_IS_ASC_QUERY(pQuery))) { closeTimeWindow(pWindowResInfo, i); } else { - skey = pResult->window.skey; + skey = pResult->skey; break; } } @@ -639,26 +665,26 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].window.skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold); - + setQueryStatus(pQuery, QUERY_RESBUF_FULL); } else { qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed); } } - + // output has reached the limitation, set query completed if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed && pRuntimeEnv->scanFlag == MASTER_SCAN) { setQueryStatus(pQuery, QUERY_COMPLETED); } - + assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL); return numOfClosed; } @@ -672,7 +698,7 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo int32_t step = GET_FORWARD_DIRECTION_FACTOR(order); STableQueryInfo* item = pQuery->current; - + if (QUERY_IS_ASC_QUERY(pQuery)) { if (ekey < pDataBlockInfo->window.ekey) { num = getForwardStepsInBlock(pDataBlockInfo->rows, searchFn, ekey, startPos, order, pPrimaryColumn); @@ -703,12 +729,12 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo return num; } -static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, +static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsBuf, int32_t numOfTotal) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { int32_t functionId = pQuery->pSelectExpr[k].base.functionId; @@ -732,12 +758,11 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStat } } -static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SWindowStatus *pStatus, STimeWindow *pWin, - int32_t offset) { +static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; - if (IS_MASTER_SCAN(pRuntimeEnv) || pStatus->closed) { + if (IS_MASTER_SCAN(pRuntimeEnv) || closed) { for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].nStartQueryTimestamp = pWin->skey; @@ -822,14 +847,14 @@ static FORCE_INLINE TSKEY reviseWindowEkey(SQuery *pQuery, STimeWindow *pWindow) //todo binary search static void* getDataBlockImpl(SArray* pDataBlock, int32_t colId) { int32_t numOfCols = (int32_t)taosArrayGetSize(pDataBlock); - + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData *p = taosArrayGet(pDataBlock, i); if (colId == p->info.colId) { return p->pData; } } - + return NULL; } @@ -879,19 +904,19 @@ static char *getDataBlock(SQueryRuntimeEnv *pRuntimeEnv, SArithmeticSupport *sas } assert(dataBlock != NULL); - sas->data[i] = dataBlock/* + pQuery->colList[i].bytes*/; // start from the offset + sas->data[i] = dataBlock; // start from the offset } } else { // other type of query function SColIndex *pCol = &pQuery->pSelectExpr[col].base.colInfo; - if (TSDB_COL_IS_TAG(pCol->flag)) { - dataBlock = NULL; - } else { + if (TSDB_COL_IS_NORMAL_COL(pCol->flag)) { SColIndex* pColIndex = &pQuery->pSelectExpr[col].base.colInfo; SColumnInfoData *p = taosArrayGet(pDataBlock, pColIndex->colIndex); assert(p->info.colId == pColIndex->colId); dataBlock = p->pData; + } else { + dataBlock = NULL; } } @@ -958,7 +983,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &win); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); + bool pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } @@ -986,8 +1011,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * TSKEY ekey = reviseWindowEkey(pQuery, &nextWin); forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true); - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows); } pWindowResInfo->curIndex = index; @@ -1040,8 +1065,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat return -1; } - pWindowRes->window.skey = v; - pWindowRes->window.ekey = v; + pWindowRes->skey = v; + assert(pRuntimeEnv->windowResInfo.interval == 0); if (pWindowRes->pos.pageId == -1) { int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); @@ -1103,7 +1128,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first - if (pCtx[0].tag.i64Key != elem.tag) { + if (tVariantCompare(&pCtx[0].tag, &elem.tag) != 0) { return TS_JOIN_TAG_NOT_EQUALS; } @@ -1140,7 +1165,7 @@ static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx if (functionId == TSDB_FUNC_TS) { return true; } - + if (pResInfo->complete || functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TS_DUMMY) { return false; } @@ -1247,8 +1272,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS continue; } - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &win, offset); + bool closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset); STimeWindow nextWin = win; int32_t index = pWindowResInfo->curIndex; @@ -1271,8 +1296,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS } if (hasTimeWindow) { - pStatus = getTimeWindowResStatus(pWindowResInfo, curTimeWindow(pWindowResInfo)); - doRowwiseApplyFunctions(pRuntimeEnv, pStatus, &nextWin, offset); + closed = getTimeWindowResStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo)); + doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset); } } @@ -1327,10 +1352,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pDataBlockInfo, SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) { SQuery *pQuery = pRuntimeEnv->pQuery; - + STableQueryInfo* pTableQInfo = pQuery->current; SWindowResInfo* pWindowResInfo = &pRuntimeEnv->windowResInfo; - + if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTSBuf != NULL || pRuntimeEnv->groupbyNormalCol) { rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock); } else { @@ -1373,10 +1398,10 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, SDataStatis *pStatis, void *param, int32_t colIndex) { - + int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId; int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId; - + SDataStatis *tpField = NULL; pCtx->hasNull = hasNullValue(&pQuery->pSelectExpr[colIndex].base.colInfo, pStatis, &tpField); pCtx->aInputElemBuf = inputData; @@ -1432,7 +1457,7 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pInterpInfo->type = (int8_t)pQuery->fillType; pInterpInfo->ts = pQuery->window.skey; pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); - + if (pQuery->fillVal != NULL) { if (isNull((const char*) &pQuery->fillVal[colIndex], pCtx->inputType)) { pCtx->param[1].nType = TSDB_DATA_TYPE_NULL; @@ -1459,19 +1484,22 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY } // set the output buffer for the selectivity + tag query -static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { +static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx) { SQuery* pQuery = pRuntimeEnv->pQuery; if (isSelectivityWithTagsQuery(pQuery)) { int32_t num = 0; int16_t tagLen = 0; - + SQLFunctionCtx *p = NULL; SQLFunctionCtx **pTagCtx = calloc(pQuery->numOfOutput, POINTER_BYTES); + if (pTagCtx == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SSqlFuncMsg *pSqlFuncMsg = &pQuery->pSelectExpr[i].base; - + if (pSqlFuncMsg->functionId == TSDB_FUNC_TAG_DUMMY || pSqlFuncMsg->functionId == TSDB_FUNC_TS_DUMMY) { tagLen += pCtx[i].outputBytes; pTagCtx[num++] = &pCtx[i]; @@ -1490,9 +1518,11 @@ static void setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *p p->tagInfo.numOfTagCols = num; p->tagInfo.tagsLen = tagLen; } else { - taosTFree(pTagCtx); + taosTFree(pTagCtx); } } + + return TSDB_CODE_SUCCESS; } static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { @@ -1536,12 +1566,14 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputBytes = pQuery->tagColList[index].bytes; pCtx->inputType = pQuery->tagColList[index].type; } - + } else if (TSDB_COL_IS_UD_COL(pIndex->flag)) { + pCtx->inputBytes = pSqlFuncMsg->arg[0].argBytes; + pCtx->inputType = pSqlFuncMsg->arg[0].argType; } else { pCtx->inputBytes = pQuery->colList[index].bytes; pCtx->inputType = pQuery->colList[index].type; } - + assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1556,7 +1588,7 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order int16_t type = pSqlFuncMsg->arg[j].argType; int16_t bytes = pSqlFuncMsg->arg[j].argBytes; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg->argValue.pz, bytes, type); + tVariantCreateFromBinary(&pCtx->param[j], pSqlFuncMsg->arg[j].argValue.pz, bytes, type); } else { tVariantCreateFromBinary(&pCtx->param[j], (char *)&pSqlFuncMsg->arg[j].argValue.i64, bytes, type); } @@ -1592,7 +1624,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order resetCtxOutputBuf(pRuntimeEnv); } - setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx); + if (setCtxTagColumnInfo(pRuntimeEnv, pRuntimeEnv->pCtx) != TSDB_CODE_SUCCESS) { + goto _clean; + } qDebug("QInfo:%p init runtime completed", GET_QINFO_ADDR(pRuntimeEnv)); return TSDB_CODE_SUCCESS; @@ -1736,13 +1770,20 @@ static bool needReverseScan(SQuery *pQuery) { return false; } +/** + * The following 4 kinds of query are treated as the tags query + * tagprj, tid_tag query, count(tbname), 'abc' (user defined constant value column) query + */ static bool onlyQueryTags(SQuery* pQuery) { for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; int32_t functionId = pExprInfo->base.functionId; - if (functionId != TSDB_FUNC_TAGPRJ && functionId != TSDB_FUNC_TID_TAG && - (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX))) { + + if (functionId != TSDB_FUNC_TAGPRJ && + functionId != TSDB_FUNC_TID_TAG && + (!(functionId == TSDB_FUNC_COUNT && pExprInfo->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX)) && + (!(functionId == TSDB_FUNC_PRJ && pExprInfo->base.colInfo.flag == TSDB_COL_UDC))) { return false; } } @@ -1803,7 +1844,7 @@ bool colIdCheck(SQuery *pQuery) { return false; } } - + return true; } @@ -1859,16 +1900,12 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, bool stableQuery) { // descending order query for last_row query if (isFirstLastRowQuery(pQuery)) { qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), - pQuery->order.order, TSDB_ORDER_DESC); - - pQuery->order.order = TSDB_ORDER_DESC; - - int64_t skey = MIN(pQuery->window.skey, pQuery->window.ekey); - int64_t ekey = MAX(pQuery->window.skey, pQuery->window.ekey); - - pQuery->window.skey = ekey; - pQuery->window.ekey = skey; + pQuery->order.order, TSDB_ORDER_ASC); + pQuery->order.order = TSDB_ORDER_ASC; + if (pQuery->window.skey > pQuery->window.ekey) { + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + } return; } @@ -1963,7 +2000,7 @@ static void getIntermediateBufInfo(SQueryRuntimeEnv* pRuntimeEnv, int32_t* ps, i } pRuntimeEnv->numOfRowsPerPage = ((*ps) - sizeof(tFilePage)) / (*rowsize); - + assert(pRuntimeEnv->numOfRowsPerPage <= MAX_ROWS_PER_RESBUF_PAGE); } #define IS_PREFILTER_TYPE(_t) ((_t) != TSDB_DATA_TYPE_BINARY && (_t) != TSDB_DATA_TYPE_NCHAR) @@ -2117,21 +2154,21 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { // return DISK_DATA_LOAD_FAILED; } - + pRuntimeEnv->summary.loadBlockStatis += 1; - + if (*pStatis == NULL) { // data block statistics does not exist, load data block *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; } } else { assert(status == BLK_DATA_ALL_NEEDED); - + // load the data block statistics to perform further filter pRuntimeEnv->summary.loadBlockStatis += 1; if (tsdbRetrieveDataBlockStatisInfo(pQueryHandle, pStatis) != TSDB_CODE_SUCCESS) { } - + if (!needToLoadDataBlock(pRuntimeEnv, *pStatis, pRuntimeEnv->pCtx, pBlockInfo->rows)) { // current block has been discard due to filter applied pRuntimeEnv->summary.discardBlocks += 1; @@ -2139,7 +2176,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, void* pQueryHandle, pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); return BLK_DATA_DISCARD; } - + pRuntimeEnv->summary.totalCheckedRows += pBlockInfo->rows; pRuntimeEnv->summary.loadBlocks += 1; *pDataBlock = tsdbRetrieveDataBlock(pQueryHandle, NULL); @@ -2224,7 +2261,7 @@ static void ensureOutputBufferSimple(SQueryRuntimeEnv* pRuntimeEnv, int32_t capa char *tmp = realloc(pQuery->sdata[i], bytes * capacity + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { pQuery->sdata[i] = (tFilePage *)tmp; } @@ -2244,18 +2281,18 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB SQuery* pQuery = pRuntimeEnv->pQuery; if (!QUERY_IS_INTERVAL_QUERY(pQuery) && !pRuntimeEnv->groupbyNormalCol && !isFixedOutputQuery(pRuntimeEnv) && !isTSCompQuery(pQuery)) { SResultRec *pRec = &pQuery->rec; - + if (pQuery->rec.capacity - pQuery->rec.rows < pBlockInfo->rows) { int32_t remain = (int32_t)(pRec->capacity - pRec->rows); int32_t newSize = (int32_t)(pRec->capacity + (pBlockInfo->rows - remain)); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pQuery->pSelectExpr[i].bytes; assert(bytes > 0 && newSize > 0); char *tmp = realloc(pQuery->sdata[i], bytes * newSize + sizeof(tFilePage)); if (tmp == NULL) { // todo handle the oom - assert(0); + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } else { memset(tmp + sizeof(tFilePage) + bytes * pRec->rows, 0, (size_t)((newSize - pRec->rows) * bytes)); pQuery->sdata[i] = (tFilePage *)tmp; @@ -2272,7 +2309,7 @@ static void ensureOutputBuffer(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pB qDebug("QInfo:%p realloc output buffer, new size: %d rows, old:%" PRId64 ", remain:%" PRId64, GET_QINFO_ADDR(pRuntimeEnv), newSize, pRec->capacity, newSize - pRec->rows); - + pRec->capacity = newSize; } } @@ -2382,7 +2419,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa if (tagColId == TSDB_TBNAME_COLUMN_INDEX) { char* val = tsdbGetTableName(pTable); assert(val != NULL); - + tVariantCreateFromBinary(tag, varDataVal(val), varDataLen(val), TSDB_DATA_TYPE_BINARY); } else { char* val = tsdbGetTableTagVal(pTable, tagColId, type, bytes); @@ -2390,7 +2427,7 @@ static void doSetTagValueInParam(void *tsdb, void* pTable, int32_t tagColId, tVa tag->nType = TSDB_DATA_TYPE_NULL; return; } - + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { if (isNull(val, type)) { tag->nType = TSDB_DATA_TYPE_NULL; @@ -2437,7 +2474,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { // set tag value, by which the results are aggregated. for (int32_t idx = 0; idx < pQuery->numOfOutput; ++idx) { SExprInfo* pLocalExprInfo = &pQuery->pSelectExpr[idx]; - + // ts_comp column required the tag value for join filter if (!TSDB_COL_IS_TAG(pLocalExprInfo->base.colInfo.flag)) { continue; @@ -2487,14 +2524,14 @@ static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowRes // in case of tag column, the tag information should be extracted from input buffer if (functionId == TSDB_FUNC_TAG_DUMMY || functionId == TSDB_FUNC_TAG) { tVariantDestroy(&pCtx[i].tag); - + int32_t type = pCtx[i].outputType; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { tVariantCreateFromBinary(&pCtx[i].tag, varDataVal(pCtx[i].aInputElemBuf), varDataLen(pCtx[i].aInputElemBuf), type); } else { tVariantCreateFromBinary(&pCtx[i].tag, pCtx[i].aInputElemBuf, pCtx[i].inputBytes, pCtx[i].inputType); } - + } } @@ -2584,7 +2621,7 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim for (int32_t j = 0; j < numOfRows; ++j) { for (int32_t i = 0; i < numOfCols; ++i) { - + switch (pQuery->pSelectExpr[i].type) { case TSDB_DATA_TYPE_BINARY: { int32_t type = pQuery->pSelectExpr[i].type; @@ -2659,7 +2696,7 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } int32_t mergeIntoGroupResult(SQInfo *pQInfo) { - int64_t st = taosGetTimestampMs(); + int64_t st = taosGetTimestampUs(); int32_t ret = TSDB_CODE_SUCCESS; int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); @@ -2678,23 +2715,30 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) { break; } - assert(pQInfo->numOfGroupResultPages == 0); + assert(pQInfo->groupResInfo.numOfDataPages == 0); qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1); } - if (pQInfo->groupIndex == numOfGroups && pQInfo->offset == pQInfo->numOfGroupResultPages) { + SGroupResInfo* info = &pQInfo->groupResInfo; + if (pQInfo->groupIndex == numOfGroups && info->pos.pageId == info->numOfDataPages) { SET_STABLE_QUERY_OVER(pQInfo); } - qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "ms", pQInfo, - pQInfo->groupIndex - 1, numOfGroups, taosGetTimestampMs() - st); + int64_t elapsedTime = taosGetTimestampUs() - st; + qDebug("QInfo:%p merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", pQInfo, + pQInfo->groupIndex - 1, numOfGroups, elapsedTime); + pQInfo->runtimeEnv.summary.firstStageMergeTime += elapsedTime; return TSDB_CODE_SUCCESS; } void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { - if (pQInfo->offset == pQInfo->numOfGroupResultPages) { - pQInfo->numOfGroupResultPages = 0; + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + // all results have been return to client, try next group + if (pGroupResInfo->pos.pageId == pGroupResInfo->numOfDataPages) { + pGroupResInfo->numOfDataPages = 0; + pGroupResInfo->pos.rowId = 0; // current results of group has been sent to client, try next group if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) { @@ -2703,7 +2747,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { // check if all results has been sent to client int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - if (pQInfo->numOfGroupResultPages == 0 && pQInfo->groupIndex == numOfGroup) { + if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) { SET_STABLE_QUERY_OVER(pQInfo); return; } @@ -2712,29 +2756,50 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { SQueryRuntimeEnv * pRuntimeEnv = &pQInfo->runtimeEnv; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - int32_t id = getGroupResultId(pQInfo->groupIndex - 1); - SIDList list = getDataBufPagesIdList(pResultBuf, pQInfo->offset + id); - - int32_t size = (int32_t)(taosArrayGetSize(list)); + int32_t id = pQInfo->groupResInfo.groupId; + SIDList list = getDataBufPagesIdList(pResultBuf, id); int32_t offset = 0; - for (int32_t j = 0; j < size; ++j) { + int32_t numOfCopiedRows = 0; + + size_t size = taosArrayGetSize(list); + assert(size == pGroupResInfo->numOfDataPages); + + bool done = false; + for (int32_t j = pGroupResInfo->pos.pageId; j < size; ++j) { SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j); - tFilePage *pData = getResBufPage(pResultBuf, pi->pageId); + tFilePage* pData = getResBufPage(pResultBuf, pi->pageId); + + assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->pos.rowId < pData->num); + int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->pos.rowId); + + if (numOfRes > pQuery->rec.capacity - offset) { + numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset); + pGroupResInfo->pos.rowId += numOfCopiedRows; + done = true; + } else { + numOfCopiedRows = (int32_t)pData->num; + + pGroupResInfo->pos.pageId += 1; + pGroupResInfo->pos.rowId = 0; + } for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; char * pDest = pQuery->sdata[i]->data; - memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pData->num, (size_t)(bytes * pData->num)); + + memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage, + (size_t)bytes * numOfCopiedRows); } - offset += (int32_t)pData->num; + offset += numOfCopiedRows; + if (done) { + break; + } } assert(pQuery->rec.rows == 0); - pQuery->rec.rows += offset; - pQInfo->offset += 1; } int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { @@ -2767,7 +2832,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { size_t size = taosArrayGetSize(pGroup); tFilePage **buffer = pQuery->sdata; - int32_t* posList = calloc(size, sizeof(int32_t)); + int32_t *posList = calloc(size, sizeof(int32_t)); STableQueryInfo **pTableList = malloc(POINTER_BYTES * size); if (pTableList == NULL || posList == NULL) { @@ -2780,23 +2845,38 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { // todo opt for the case of one table per group int32_t numOfTables = 0; + SIDList pageList = NULL; + int32_t tid = -1; + for (int32_t i = 0; i < size; ++i) { STableQueryInfo *item = taosArrayGetP(pGroup, i); SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid); + pageList = list; + tid = TSDB_TABLEID(item->pTable)->tid; + if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) { - pTableList[numOfTables] = item; - numOfTables += 1; + pTableList[numOfTables++] = item; } } + // there is no data in current group if (numOfTables == 0) { taosTFree(posList); taosTFree(pTableList); - - assert(pQInfo->numOfGroupResultPages == 0); return 0; } else if (numOfTables == 1) { // no need to merge results since only one table in each group + taosTFree(posList); + taosTFree(pTableList); + + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + + pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList); + pGroupResInfo->groupId = tid; + pGroupResInfo->pos.pageId = 0; + pGroupResInfo->pos.rowId = 0; + + return pGroupResInfo->numOfDataPages; } SCompSupporter cs = {pTableList, posList, pQInfo}; @@ -2810,9 +2890,15 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } char* buf = calloc(1, pRuntimeEnv->interBufSize); + if (buf == NULL) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); + // todo add windowRes iterator int64_t lastTimestamp = -1; int64_t startt = taosGetTimestampMs(); @@ -2820,6 +2906,13 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { while (1) { if (IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p it is already killed, abort", pQInfo); + + taosTFree(pTableList); + taosTFree(posList); + taosTFree(pTree); + taosTFree(pResultInfo); + taosTFree(buf); + longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -2832,7 +2925,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); - assert(ts == pWindowRes->window.skey); + assert(ts == pWindowRes->skey); int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -2850,7 +2943,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { doMerge(pRuntimeEnv, ts, pWindowRes, true); } else { // copy data to disk buffer if (buffer[0]->num == pQuery->rec.capacity) { - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { return -1; } @@ -2887,7 +2980,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } if (buffer[0]->num != 0) { // there are data in buffer - if (flushFromResultBuf(pQInfo) != TSDB_CODE_SUCCESS) { + if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) { qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo); taosTFree(pTree); @@ -2911,16 +3004,14 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(posList); taosTFree(pTree); - pQInfo->offset = 0; - taosTFree(pResultInfo); taosTFree(buf); - return pQInfo->numOfGroupResultPages; + + return pQInfo->groupResInfo.numOfDataPages; } -int32_t flushFromResultBuf(SQInfo *pQInfo) { - SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - SQuery * pQuery = pRuntimeEnv->pQuery; +int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) { + SQuery *pQuery = pRuntimeEnv->pQuery; SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; @@ -2928,32 +3019,32 @@ int32_t flushFromResultBuf(SQInfo *pQInfo) { int32_t pageId = -1; int32_t capacity = pResultBuf->numOfRowsPerPage; - int32_t remain = (int32_t)pQuery->sdata[0]->num; + int32_t remain = (int32_t) pQuery->sdata[0]->num; int32_t offset = 0; while (remain > 0) { - int32_t r = remain; - if (r > capacity) { - r = capacity; - } + int32_t rows = (remain > capacity)? capacity:remain; + assert(rows > 0); - int32_t id = getGroupResultId(pQInfo->groupIndex) + pQInfo->numOfGroupResultPages; - tFilePage *buf = getNewDataBuf(pResultBuf, id, &pageId); + // get the output buffer page + tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId); + buf->num = rows; - // pagewise copy to dest buffer + // pagewisely copy to dest buffer for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; - buf->num = r; - memcpy(buf->data + pRuntimeEnv->offset[i] * buf->num, ((char *)pQuery->sdata[i]->data) + offset * bytes, - (size_t)(buf->num * bytes)); + char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage; + char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes; + memcpy(output, src, (size_t)(buf->num * bytes)); } - offset += r; - remain -= r; + offset += rows; + remain -= rows; + + pGroupResInfo->numOfDataPages += 1; } - pQInfo->numOfGroupResultPages += 1; return TSDB_CODE_SUCCESS; } @@ -2972,10 +3063,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * if (pTableQueryInfo == NULL) { return; } - + // order has changed already int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); - + // TODO validate the assertion // if (!QUERY_IS_ASC_QUERY(pQuery)) { // assert(pTableQueryInfo->win.ekey >= pTableQueryInfo->lastKey + step); @@ -2991,7 +3082,7 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * SWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - + SWITCH_ORDER(pTableQueryInfo->cur.order); pTableQueryInfo->cur.vgroupIndex = -1; @@ -3001,10 +3092,10 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { SQuery* pQuery = pQInfo->runtimeEnv.pQuery; - + for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = getTimeWindowResStatus(pWindowResInfo, i); - if (!pStatus->closed) { + bool closed = getTimeWindowResStatus(pWindowResInfo, i); + if (!closed) { continue; } @@ -3065,7 +3156,8 @@ static void setupQueryRangeForReverseScan(SQInfo* pQInfo) { STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); updateTableQueryInfoForReverseScan(pQuery, pCheckInfo); - // update the last key in tableKeyInfo list + // update the last key in tableKeyInfo list, the tableKeyInfo is used to build the tsdbQueryHandle and decide + // the start check timestamp of tsdbQueryHandle STableKeyInfo *pTableKeyInfo = taosArrayGet(tableKeyGroup, j); pTableKeyInfo->lastKey = pCheckInfo->lastKey; @@ -3178,7 +3270,7 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { if (pQuery->rec.rows <= pQuery->limit.offset) { qDebug("QInfo:%p skip rows:%" PRId64 ", new offset:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), pQuery->rec.rows, pQuery->limit.offset - pQuery->rec.rows); - + pQuery->limit.offset -= pQuery->rec.rows; pQuery->rec.rows = 0; @@ -3190,10 +3282,10 @@ void skipResults(SQueryRuntimeEnv *pRuntimeEnv) { int64_t numOfSkip = pQuery->limit.offset; pQuery->rec.rows -= numOfSkip; pQuery->limit.offset = 0; - + qDebug("QInfo:%p skip row:%"PRId64", new offset:%d, numOfRows remain:%" PRIu64, GET_QINFO_ADDR(pRuntimeEnv), numOfSkip, 0, pQuery->rec.rows); - + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { int32_t functionId = pQuery->pSelectExpr[i].base.functionId; int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes; @@ -3230,7 +3322,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { for (int32_t i = 0; i < pWindowResInfo->size; ++i) { SWindowResult *pResult = getWindowResult(pWindowResInfo, i); - if (!pResult->status.closed) { + if (!pResult->closed) { continue; } @@ -3268,10 +3360,10 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { static SQueryStatusInfo getQueryStatusInfo(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + assert((start <= pTableQueryInfo->lastKey && QUERY_IS_ASC_QUERY(pQuery)) || (start >= pTableQueryInfo->lastKey && !QUERY_IS_ASC_QUERY(pQuery))); - + SQueryStatusInfo info = { .status = pQuery->status, .windowIndex = pRuntimeEnv->windowResInfo.curIndex, @@ -3293,7 +3385,8 @@ static void setEnvBeforeReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatusI pStatus->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); // save the cursor if (pRuntimeEnv->pTSBuf) { SWITCH_ORDER(pRuntimeEnv->pTSBuf->cur.order); - tsBufNextPos(pRuntimeEnv->pTSBuf); + bool ret = tsBufNextPos(pRuntimeEnv->pTSBuf); + assert(ret); } // reverse order time range @@ -3351,7 +3444,7 @@ static void clearEnvAfterReverseScan(SQueryRuntimeEnv *pRuntimeEnv, SQueryStatus // update the pQuery->window.skey and pQuery->window.ekey to limit the scan scope of sliding query during reverse scan pTableQueryInfo->lastKey = pStatus->lastKey; pQuery->status = pStatus->status; - + pTableQueryInfo->win = pStatus->w; pQuery->window = pTableQueryInfo->win; } @@ -3367,7 +3460,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { SQInfo *pQInfo = (SQInfo *) GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + setQueryStatus(pQuery, QUERY_NOT_COMPLETED); // store the start query position @@ -3420,7 +3513,7 @@ void scanOneTableDataBlocks(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) { pRuntimeEnv->windowResInfo.curIndex = qstatus.windowIndex; setQueryStatus(pQuery, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; - + qDebug("QInfo:%p start to repeat scan data blocks due to query func required, qrange:%"PRId64"-%"PRId64, pQInfo, cond.twindow.skey, cond.twindow.ekey); @@ -3525,12 +3618,6 @@ void destroyTableQueryInfo(STableQueryInfo *pTableQueryInfo) { cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); } -#define CHECK_QUERY_TIME_RANGE(_q, _tableInfo) \ - do { \ - assert((((_tableInfo)->lastKey >= (_tableInfo)->win.skey) && QUERY_IS_ASC_QUERY(_q)) || \ - (((_tableInfo)->lastKey <= (_tableInfo)->win.skey) && !QUERY_IS_ASC_QUERY(_q))); \ - } while (0) - /** * set output buffer for different group * @param pRuntimeEnv @@ -3644,9 +3731,8 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ // both the master and supplement scan needs to set the correct ts comp start position if (pRuntimeEnv->pTSBuf != NULL) { if (pTableQueryInfo->cur.vgroupIndex == -1) { - pTableQueryInfo->tag = pRuntimeEnv->pCtx[0].tag.i64Key; - - tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, pTableQueryInfo->tag); + tVariantAssign(&pTableQueryInfo->tag, &pRuntimeEnv->pCtx[0].tag); + tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pTableQueryInfo->tag); // keep the cursor info of current meter pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; @@ -3671,7 +3757,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo *pTableQueryInfo = pQuery->current; - + if (pTableQueryInfo->queryRangeSet) { pTableQueryInfo->lastKey = key; } else { @@ -3702,7 +3788,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) { if (!QUERY_IS_ASC_QUERY(pQuery)) { assert(win.ekey == pQuery->window.ekey); } - + pWindowResInfo->prevSKey = w.skey; } @@ -3728,7 +3814,7 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) { */ STimeWindow *w = &pDataBlockInfo->window; STableQueryInfo* pTableQueryInfo = pQuery->current; - + bool loadPrimaryTS = (pTableQueryInfo->lastKey >= w->skey && pTableQueryInfo->lastKey <= w->ekey) || (pQuery->window.ekey >= w->skey && pQuery->window.ekey <= w->ekey) || requireTimestamp(pQuery); @@ -3755,27 +3841,27 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ step = -1; } + SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; + for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { if (result[i].numOfRows == 0) { - pQInfo->offset = 0; pQInfo->groupIndex += 1; + pGroupResInfo->pos.rowId = 0; continue; } - assert(pQInfo->offset <= 1); - - int32_t numOfRowsToCopy = result[i].numOfRows - pQInfo->offset; - int32_t oldOffset = pQInfo->offset; + int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->pos.rowId; + int32_t oldOffset = pGroupResInfo->pos.rowId; /* - * current output space is not enough to keep all the result data of this group, only copy partial results - * to SQuery object's result buffer + * current output space is not enough to accommodate all data of this page, only partial results + * will be copied to SQuery object's result buffer */ if (numOfRowsToCopy > pQuery->rec.capacity - numOfResult) { - numOfRowsToCopy = (int32_t)pQuery->rec.capacity - numOfResult; - pQInfo->offset += numOfRowsToCopy; + numOfRowsToCopy = (int32_t) pQuery->rec.capacity - numOfResult; + pGroupResInfo->pos.rowId += numOfRowsToCopy; } else { - pQInfo->offset = 0; + pGroupResInfo->pos.rowId = 0; pQInfo->groupIndex += 1; } @@ -3840,7 +3926,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } } @@ -3849,7 +3935,7 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc SArray *pDataBlock, __block_search_fn_t searchFn) { SQuery * pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + SWindowResInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo; pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1; @@ -3941,10 +4027,10 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); SQuery *pQuery = pRuntimeEnv->pQuery; SFillInfo* pFillInfo = pRuntimeEnv->pFillInfo; - + while (1) { int32_t ret = (int32_t)taosGenerateDataBlock(pFillInfo, (tFilePage**)pQuery->sdata, (int32_t)pQuery->rec.capacity); - + // todo apply limit output function /* reached the start position of according to offset value, return immediately */ if (pQuery->limit.offset == 0) { @@ -3955,7 +4041,7 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int if (pQuery->limit.offset < ret) { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, remain:%" PRId64 ", new offset:%d", pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, ret - pQuery->limit.offset, 0); - + ret -= (int32_t)pQuery->limit.offset; // todo !!!!there exactly number of interpo is not valid. // todo refactor move to the beginning of buffer @@ -3963,14 +4049,14 @@ int32_t doFillGapsInResults(SQueryRuntimeEnv* pRuntimeEnv, tFilePage **pDst, int memmove(pDst[i]->data, pDst[i]->data + pQuery->pSelectExpr[i].bytes * pQuery->limit.offset, ret * pQuery->pSelectExpr[i].bytes); } - + pQuery->limit.offset = 0; return ret; } else { qDebug("QInfo:%p initial numOfRows:%d, generate filled result:%d rows, offset:%" PRId64 ". Discard due to offset, " "remain:%d, new offset:%" PRId64, pQInfo, pFillInfo->numOfRows, ret, pQuery->limit.offset, 0, pQuery->limit.offset - ret); - + pQuery->limit.offset -= ret; pQuery->rec.rows = 0; ret = 0; @@ -3986,18 +4072,22 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; - qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, total blocks:%d, load block statis:%d," - " load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, - pQInfo, pSummary->elapsedTime, pSummary->totalBlocks, pSummary->loadBlockStatis, + // add the merge time + pSummary->elapsedTime += pSummary->firstStageMergeTime; + + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " + "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, + pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64, pQInfo, pSummary->internalSupSize); + qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, + pSummary->numOfTimeWindows); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { SQuery *pQuery = pRuntimeEnv->pQuery; STableQueryInfo* pTableQueryInfo = pQuery->current; - + int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); if (pQuery->limit.offset == pBlockInfo->rows) { // current block will ignore completed @@ -4087,7 +4177,7 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { assert(pRuntimeEnv->windowResInfo.prevSKey == TSKEY_INITIAL_VAL); STimeWindow w = TSWINDOW_INITIALIZER; - + SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; STableQueryInfo *pTableQueryInfo = pQuery->current; @@ -4136,21 +4226,21 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { // set the abort info pQuery->pos = startPos; - + // reset the query start timestamp pTableQueryInfo->win.skey = ((TSKEY *)pColInfoData->pData)[startPos]; pQuery->window.skey = pTableQueryInfo->win.skey; *start = pTableQueryInfo->win.skey; - + pWindowResInfo->prevSKey = tw.skey; int32_t index = pRuntimeEnv->windowResInfo.curIndex; - + int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, NULL, binarySearchForKey, pDataBlock); pRuntimeEnv->windowResInfo.curIndex = index; // restore the window index - + qDebug("QInfo:%p check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%"PRId64, GET_QINFO_ADDR(pRuntimeEnv), blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, numOfRes, pQuery->current->lastKey); - + return true; } else { // do nothing *start = tw.skey; @@ -4218,7 +4308,7 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) if (!isSTableQuery && (pQInfo->tableqinfoGroupInfo.numOfTables == 1) - && (cond.order == TSDB_ORDER_ASC) + && (cond.order == TSDB_ORDER_ASC) && (!QUERY_IS_INTERVAL_QUERY(pQuery)) && (!isGroupbyNormalCol(pQuery->pGroupbyExpr)) && (!isFixedOutputQuery(pRuntimeEnv)) @@ -4231,6 +4321,22 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + + // update the query time window + pQuery->window = cond.twindow; + + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + for(int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); + + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); + + pCheckInfo->win = pQuery->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; + } + } } else if (isPointInterpoQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); } else { @@ -4243,21 +4349,25 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) static SFillColInfo* taosCreateFillColInfo(SQuery* pQuery) { int32_t numOfCols = pQuery->numOfOutput; int32_t offset = 0; - + SFillColInfo* pFillCol = calloc(numOfCols, sizeof(SFillColInfo)); + if (pFillCol == NULL) { + return NULL; + } + for(int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; - + pFillCol[i].col.bytes = pExprInfo->bytes; pFillCol[i].col.type = (int8_t)pExprInfo->type; pFillCol[i].col.offset = offset; pFillCol[i].flag = TSDB_COL_NORMAL; // always be ta normal column for table query pFillCol[i].functionId = pExprInfo->base.functionId; pFillCol[i].fillVal.i = pQuery->fillVal[i]; - + offset += pExprInfo->bytes; } - + return pFillCol; } @@ -4272,13 +4382,16 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo pRuntimeEnv->hasTagResults = hasTagValOutput(pQuery); setScanLimitationByResultBuffer(pQuery); + + // NOTE: pTableCheckInfo need to update the query time range and the lastKey info + // TODO fixme changeExecuteScanOrder(pQInfo, false); code = setupQueryHandle(tsdb, pQInfo, isSTableQuery); if (code != TSDB_CODE_SUCCESS) { return code; } - + pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -4423,7 +4536,17 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { } pQuery->current = *pTableQueryInfo; - CHECK_QUERY_TIME_RANGE(pQuery, *pTableQueryInfo); + if (QUERY_IS_ASC_QUERY(pQuery)) { + assert( + ((*pTableQueryInfo)->win.skey <= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey >= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey >= pQuery->window.skey && (*pTableQueryInfo)->win.ekey <= pQuery->window.ekey)); + } else { + assert( + ((*pTableQueryInfo)->win.skey >= (*pTableQueryInfo)->win.ekey) && + ((*pTableQueryInfo)->lastKey <= (*pTableQueryInfo)->win.skey) && + ((*pTableQueryInfo)->win.skey <= pQuery->window.skey && (*pTableQueryInfo)->win.ekey >= pQuery->window.ekey)); + } if (!pRuntimeEnv->groupbyNormalCol) { setEnvForEachBlock(pQInfo, *pTableQueryInfo, &blockInfo); @@ -4439,8 +4562,10 @@ static int64_t scanMultiTableDataBlocks(SQInfo *pQInfo) { summary->totalRows += blockInfo.rows; stableApplyFunctionsOnBlock(pRuntimeEnv, &blockInfo, pStatis, pDataBlock, binarySearchForKey); - qDebug("QInfo:%p check data block, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, lastKey:%" PRId64, - pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, pQuery->current->lastKey); + qDebug("QInfo:%p check data block completed, uid:%"PRId64", tid:%d, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, " + "lastKey:%" PRId64, + pQInfo, blockInfo.uid, blockInfo.tid, blockInfo.window.skey, blockInfo.window.ekey, blockInfo.rows, + pQuery->current->lastKey); } if (terrno != TSDB_CODE_SUCCESS) { @@ -4501,8 +4626,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vgroupIndex == -1) { - int64_t tag = pRuntimeEnv->pCtx[0].tag.i64Key; - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pRuntimeEnv->pCtx[0].tag); // failed to find data with the specified tag value if (elem.vnode < 0) { @@ -4552,7 +4676,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SArray *g1 = taosArrayInit(1, POINTER_BYTES); SArray *tx = taosArrayClone(group); taosArrayPush(g1, &tx); - + STableGroupInfo gp = {.numOfTables = taosArrayGetSize(tx), .pGroupList = g1}; // include only current table @@ -4560,9 +4684,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); pRuntimeEnv->pQueryHandle = NULL; } - + if (isFirstLastRowQuery(pQuery)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(pQInfo->tsdb, &cond, &gp, pQInfo); + assert(0); // last_row query switch to other routine to handle } else { pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); } @@ -4574,10 +4698,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } initCtxOutputBuf(pRuntimeEnv); - + SArray* s = tsdbGetQueriedTableList(pRuntimeEnv->pQueryHandle); assert(taosArrayGetSize(s) >= 1); - + setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); if (isFirstLastRowQuery(pQuery)) { assert(taosArrayGetSize(s) == 1); @@ -4590,13 +4714,13 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pQuery->current = taosArrayGetP(first, 0); scanOneTableDataBlocks(pRuntimeEnv, pQuery->current->lastKey); - + int64_t numOfRes = getNumOfResult(pRuntimeEnv); if (numOfRes > 0) { pQuery->rec.rows += numOfRes; forwardCtxOutputBuf(pRuntimeEnv, numOfRes); } - + skipResults(pRuntimeEnv); pQInfo->groupIndex += 1; @@ -4655,18 +4779,17 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; // no results generated for current group, continue to try the next group - taosArrayDestroy(s); + taosArrayDestroy(s); if (pWindowResInfo->size <= 0) { continue; } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowStatus *pStatus = &pWindowResInfo->pResult[i].status; - pStatus->closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns SWindowResult *pResult = &pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } } @@ -4816,11 +4939,11 @@ static void doSaveContext(SQInfo *pQInfo) { SET_REVERSE_SCAN_FLAG(pRuntimeEnv); SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); SWITCH_ORDER(pQuery->order.order); - + if (pRuntimeEnv->pTSBuf != NULL) { pRuntimeEnv->pTSBuf->cur.order = pQuery->order.order; } - + STsdbQueryCond cond = { .order = pQuery->order.order, .colList = pQuery->colList, @@ -4924,7 +5047,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { el = scanMultiTableDataBlocks(pQInfo); qDebug("QInfo:%p reversed scan completed, elapsed time: %" PRId64 "ms", pQInfo, el); -// doCloseAllTimeWindowAfterScan(pQInfo); doRestoreContext(pQInfo); } else { qDebug("QInfo:%p no need to do reversed scan, query completed", pQInfo); @@ -4962,14 +5084,14 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { */ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; if (!pRuntimeEnv->topBotQuery && pQuery->limit.offset > 0) { // no need to execute, since the output will be ignore. return; } - + pQuery->current = pTableInfo; // set current query table info - + scanOneTableDataBlocks(pRuntimeEnv, pTableInfo->lastKey); finalizeQueryResult(pRuntimeEnv); @@ -4987,10 +5109,10 @@ static void tableFixedOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; - + SQuery *pQuery = pRuntimeEnv->pQuery; pQuery->current = pTableInfo; - + // for ts_comp query, re-initialized is not allowed if (!isTSCompQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); @@ -5081,7 +5203,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { int32_t numOfFilled = 0; TSKEY newStartKey = TSKEY_INITIAL_VAL; - + // skip blocks without load the actual data block from file if no filter condition present skipTimeInterval(pRuntimeEnv, &newStartKey); if (pQuery->limit.offset > 0 && pQuery->numOfFilterCols == 0 && pRuntimeEnv->pFillInfo == NULL) { @@ -5108,7 +5230,7 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) { taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey); taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (tFilePage**) pQuery->sdata); numOfFilled = 0; - + pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled); if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) { limitResults(pRuntimeEnv); @@ -5174,11 +5296,11 @@ static void tableQueryImpl(SQInfo *pQInfo) { // number of points returned during this query pQuery->rec.rows = 0; int64_t st = taosGetTimestampUs(); - + assert(pQInfo->tableqinfoGroupInfo.numOfTables == 1); SArray* g = GET_TABLEGROUP(pQInfo, 0); STableQueryInfo* item = taosArrayGetP(g, 0); - + // group by normal column, sliding window query, interval query are handled by interval query processor if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { // interval (down sampling operation) tableIntervalProcess(pQInfo, item); @@ -5202,15 +5324,13 @@ static void stableQueryImpl(SQInfo *pQInfo) { int64_t st = taosGetTimestampUs(); if (QUERY_IS_INTERVAL_QUERY(pQuery) || - (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && !pRuntimeEnv->groupbyNormalCol && - !isFirstLastRowQuery(pQuery))) { + (isFixedOutputQuery(pRuntimeEnv) && (!isPointInterpoQuery(pQuery)) && (!pRuntimeEnv->groupbyNormalCol))) { multiTableQueryProcess(pQInfo); } else { assert((pQuery->checkBuffer == 1 && pQuery->intervalTime == 0) || isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery) || pRuntimeEnv->groupbyNormalCol); sequentialTableProcess(pQInfo); - } // record the total elapsed time @@ -5222,7 +5342,7 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE if (TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { if (pExprMsg->colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { - return -1; + return TSDB_TBNAME_COLUMN_INDEX; } while(j < pQueryMsg->numOfTags) { @@ -5233,6 +5353,8 @@ static int32_t getColumnIndexInSource(SQueryTableMsg *pQueryMsg, SSqlFuncMsg *pE j += 1; } + } else if (pExprMsg->colInfo.flag == TSDB_COL_UDC) { // user specified column data + return TSDB_UD_COLUMN_INDEX; } else { while (j < pQueryMsg->numOfCols) { if (pExprMsg->colInfo.colId == pQueryMsg->colList[j].colId) { @@ -5371,11 +5493,15 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, int32_t numOfFilters = pColInfo->numOfFilters; if (numOfFilters > 0) { pColInfo->filters = calloc(numOfFilters, sizeof(SColumnFilterInfo)); + if (pColInfo->filters == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } } for (int32_t f = 0; f < numOfFilters; ++f) { SColumnFilterInfo *pFilterMsg = (SColumnFilterInfo *)pMsg; - + SColumnFilterInfo *pColFilter = &pColInfo->filters[f]; pColFilter->filterstr = htons(pFilterMsg->filterstr); @@ -5385,6 +5511,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, pColFilter->len = htobe64(pFilterMsg->len); pColFilter->pz = (int64_t)calloc(1, (size_t)(pColFilter->len + 1 * TSDB_NCHAR_SIZE)); // note: null-terminator + if (pColFilter->pz == 0) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + memcpy((void *)pColFilter->pz, pMsg, (size_t)pColFilter->len); pMsg += (pColFilter->len + 1); } else { @@ -5398,6 +5529,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, } *pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); + if (*pExpr == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + SSqlFuncMsg *pExprMsg = (SSqlFuncMsg *)pMsg; for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { @@ -5484,6 +5620,11 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, if (pQueryMsg->numOfTags > 0) { (*tagCols) = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); + if (*tagCols == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + } + for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { SColumnInfo* pTagCol = (SColumnInfo*) pMsg; @@ -5500,6 +5641,12 @@ static int32_t convertQueryMsg(SQueryTableMsg *pQueryMsg, SArray **pTableIdList, // the tag query condition expression string is located at the end of query msg if (pQueryMsg->tagCondLen > 0) { *tagCond = calloc(1, pQueryMsg->tagCondLen); + + if (*tagCond == NULL) { + code = TSDB_CODE_QRY_OUT_OF_MEMORY; + goto _cleanup; + + } memcpy(*tagCond, pMsg, pQueryMsg->tagCondLen); pMsg += pQueryMsg->tagCondLen; } @@ -5591,9 +5738,19 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * bytes = tDataTypeDesc[type].nSize; } else if (pExprs[i].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX && pExprs[i].base.functionId == TSDB_FUNC_TAGPRJ) { // parse the normal column SSchema s = tGetTableNameColumnSchema(); - type = s.type; + type = s.type; bytes = s.bytes; - } else{ + } else if (pExprs[i].base.colInfo.colId <= TSDB_UD_COLUMN_INDEX) { + // it is a user-defined constant value column + assert(pExprs[i].base.functionId == TSDB_FUNC_PRJ); + + type = pExprs[i].base.arg[1].argType; + bytes = pExprs[i].base.arg[1].argBytes; + + if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { + bytes += VARSTR_HEADER_SIZE; + } + } else { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); assert(j < pQueryMsg->numOfCols || j < pQueryMsg->numOfTags); @@ -5629,18 +5786,19 @@ static int32_t createQFunctionExprFromMsg(SQueryTableMsg *pQueryMsg, SExprInfo * if (functId == TSDB_FUNC_TOP || functId == TSDB_FUNC_BOTTOM) { int32_t j = getColumnIndexInSource(pQueryMsg, &pExprs[i].base, pTagCols); - assert(j < pQueryMsg->numOfCols); - - SColumnInfo *pCol = &pQueryMsg->colList[j]; - - int32_t ret = - getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, - &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); - assert(ret == TSDB_CODE_SUCCESS); + if (j < 0 || j >= pQueryMsg->numOfCols) { + assert(0); + } else { + SColumnInfo *pCol = &pQueryMsg->colList[j]; + int32_t ret = + getResultDataInfo(pCol->type, pCol->bytes, functId, (int32_t)pExprs[i].base.arg[0].argValue.i64, + &pExprs[i].type, &pExprs[i].bytes, &pExprs[i].interBytes, tagLen, isSuperTable); + assert(ret == TSDB_CODE_SUCCESS); + } } } - *pExprInfo = pExprs; + *pExprInfo = pExprs; return TSDB_CODE_SUCCESS; } @@ -5680,6 +5838,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { } pQuery->pFilterInfo = calloc(1, sizeof(SSingleColumnFilterInfo) * pQuery->numOfFilterCols); + if (pQuery->pFilterInfo == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t i = 0, j = 0; i < pQuery->numOfCols; ++i) { if (pQuery->colList[i].numOfFilters > 0) { @@ -5690,6 +5851,9 @@ static int32_t createFilterInfo(void *pQInfo, SQuery *pQuery) { pFilterInfo->numOfFilters = pQuery->colList[i].numOfFilters; pFilterInfo->pFilters = calloc(pFilterInfo->numOfFilters, sizeof(SColumnFilterElem)); + if (pFilterInfo->pFilters == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } for (int32_t f = 0; f < pFilterInfo->numOfFilters; ++f) { SColumnFilterElem *pSingleColFilter = &pFilterInfo->pFilters[f]; @@ -5766,7 +5930,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { // todo opt performance SColIndex *pColIndex = &pSqlExprMsg->colInfo; - if (!TSDB_COL_IS_TAG(pColIndex->flag)) { + if (TSDB_COL_IS_NORMAL_COL(pColIndex->flag)) { int32_t f = 0; for (f = 0; f < pQuery->numOfCols; ++f) { if (pColIndex->colId == pQuery->colList[f].colId) { @@ -5774,8 +5938,10 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - - assert (f < pQuery->numOfCols); + + assert(f < pQuery->numOfCols); + } else if (pColIndex->colId <= TSDB_UD_COLUMN_INDEX) { + // do nothing for user-defined constant value result columns } else { int32_t f = 0; for (f = 0; f < pQuery->numOfTags; ++f) { @@ -5784,7 +5950,7 @@ static void doUpdateExprColumnIndex(SQuery *pQuery) { break; } } - + assert(f < pQuery->numOfTags || pColIndex->colId == TSDB_TBNAME_COLUMN_INDEX); } } @@ -5837,6 +6003,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (pQuery == NULL) { goto _cleanup_query; } + pQInfo->runtimeEnv.pQuery = pQuery; pQuery->numOfCols = numOfCols; @@ -5922,6 +6089,10 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); + if (pQInfo->pBuf == NULL) { + goto _cleanup; + } + int32_t index = 0; for(int32_t i = 0; i < numOfGroups; ++i) { @@ -5932,20 +6103,18 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SArray* pTableIdList, if (p1 == NULL) { goto _cleanup; } + taosArrayPush(pQInfo->tableqinfoGroupInfo.pGroupList, &p1); for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); + STableId* id = TSDB_TABLEID(info->pTable); - STableIdInfo* pTableId = taosArraySearch(pTableIdList, id, compareTableIdInfo); - if (pTableId != NULL ) { - window.skey = pTableId->key; - } else { - window.skey = pQueryMsg->window.skey; - } + window.skey = (pTableId != NULL)? pTableId->key:pQueryMsg->window.skey; void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); + STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); if (item == NULL) { goto _cleanup; @@ -6040,7 +6209,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ if ((code = doInitQInfo(pQInfo, pTSBuf, tsdb, vgId, isSTable)) != TSDB_CODE_SUCCESS) { goto _error; } - + return code; _error: @@ -6066,35 +6235,59 @@ static void freeQInfo(SQInfo *pQInfo) { return; } - SQuery *pQuery = pQInfo->runtimeEnv.pQuery; qDebug("QInfo:%p start to free QInfo", pQInfo); - for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - taosTFree(pQuery->sdata[col]); - } teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); - for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { - SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; - if (pColFilter->numOfFilters > 0) { - taosTFree(pColFilter->pFilters); + SQuery *pQuery = pQInfo->runtimeEnv.pQuery; + if (pQuery != NULL) { + if (pQuery->sdata != NULL) { + for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { + taosTFree(pQuery->sdata[col]); + } + taosTFree(pQuery->sdata); } - } - if (pQuery->pSelectExpr != NULL) { - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SExprInfo* pExprInfo = &pQuery->pSelectExpr[i]; + if (pQuery->fillVal != NULL) { + taosTFree(pQuery->fillVal); + } - if (pExprInfo->pExpr != NULL) { - tExprTreeDestroy(&pExprInfo->pExpr, NULL); + for (int32_t i = 0; i < pQuery->numOfFilterCols; ++i) { + SSingleColumnFilterInfo *pColFilter = &pQuery->pFilterInfo[i]; + if (pColFilter->numOfFilters > 0) { + taosTFree(pColFilter->pFilters); } } - taosTFree(pQuery->pSelectExpr); - } + if (pQuery->pSelectExpr != NULL) { + for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { + SExprInfo *pExprInfo = &pQuery->pSelectExpr[i]; - if (pQuery->fillVal != NULL) { - taosTFree(pQuery->fillVal); + if (pExprInfo->pExpr != NULL) { + tExprTreeDestroy(&pExprInfo->pExpr, NULL); + } + } + + taosTFree(pQuery->pSelectExpr); + } + + if (pQuery->pGroupbyExpr != NULL) { + taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); + taosTFree(pQuery->pGroupbyExpr); + } + + taosTFree(pQuery->tagColList); + taosTFree(pQuery->pFilterInfo); + + if (pQuery->colList != NULL) { + for (int32_t i = 0; i < pQuery->numOfCols; i++) { + SColumnInfo *column = pQuery->colList + i; + freeColumnFilterInfo(column->filters, column->numOfFilters); + } + taosTFree(pQuery->colList); + } + + taosTFree(pQuery); } // todo refactor, extract method to destroytableDataInfo @@ -6118,25 +6311,8 @@ static void freeQInfo(SQInfo *pQInfo) { taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); - - if (pQuery->pGroupbyExpr != NULL) { - taosArrayDestroy(pQuery->pGroupbyExpr->columnInfo); - taosTFree(pQuery->pGroupbyExpr); - } - taosTFree(pQuery->tagColList); - taosTFree(pQuery->pFilterInfo); - if (pQuery->colList != NULL) { - for (int32_t i = 0; i < pQuery->numOfCols; i++) { - SColumnInfo* column = pQuery->colList + i; - freeColumnFilterInfo(column->filters, column->numOfFilters); - } - taosTFree(pQuery->colList); - } - - taosTFree(pQuery->sdata); - taosTFree(pQuery); pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); @@ -6176,14 +6352,16 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { // make sure file exist if (FD_VALID(fd)) { - int32_t s = lseek(fd, 0, SEEK_END); - UNUSED(s); - qDebug("QInfo:%p ts comp data return, file:%s, size:%d", pQInfo, pQuery->sdata[0]->data, s); + uint64_t s = lseek(fd, 0, SEEK_END); + + qDebug("QInfo:%p ts comp data return, file:%s, size:%"PRId64, pQInfo, pQuery->sdata[0]->data, s); if (lseek(fd, 0, SEEK_SET) >= 0) { - size_t sz = read(fd, data, s); - UNUSED(sz); + size_t sz = read(fd, data, (uint32_t)s); + if(sz < s) { // todo handle error + assert(0); + } } else { - // todo handle error + UNUSED(s); } close(fd); @@ -6193,7 +6371,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qError("QInfo:%p failed to open tmp file to send ts-comp data to client, path:%s, reason:%s", pQInfo, pQuery->sdata[0]->data, strerror(errno)); if (fd != -1) { - close(fd); + close(fd); } } @@ -6212,7 +6390,7 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) { qDebug("QInfo:%p results limitation reached, limitation:%"PRId64, pQInfo, pQuery->limit.limit); setQueryStatus(pQuery, QUERY_OVER); } - + return TSDB_CODE_SUCCESS; } @@ -6314,7 +6492,7 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi pExprs = NULL; pGroupbyExpr = NULL; pTagColumnInfo = NULL; - + if ((*pQInfo) == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _over; @@ -6329,7 +6507,7 @@ _over: if (pGroupbyExpr != NULL) { taosArrayDestroy(pGroupbyExpr->columnInfo); free(pGroupbyExpr); - } + } free(pTagColumnInfo); free(pExprs); free(pExprMsg); @@ -6496,7 +6674,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co (*pRsp)->offset = 0; (*pRsp)->useconds = htobe64(pRuntimeEnv->summary.elapsedTime); } - + (*pRsp)->precision = htons(pQuery->precision); if (pQuery->rec.rows > 0 && pQInfo->code == TSDB_CODE_SUCCESS) { doDumpQueryResult(pQInfo, (*pRsp)->data); @@ -6573,7 +6751,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { if (numOfGroup == 0) { return; } - + SArray* pa = GET_TABLEGROUP(pQInfo, 0); size_t num = taosArrayGetSize(pa); @@ -6664,6 +6842,10 @@ static void buildTagQueryResult(SQInfo* pQInfo) { char *data = NULL, *dst = NULL; int16_t type = 0, bytes = 0; for(int32_t j = 0; j < pQuery->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (pExprInfo[j].base.colInfo.flag == TSDB_COL_UDC) { + continue; + } if (pExprInfo[j].base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) { bytes = tbnameSchema.bytes; @@ -6674,7 +6856,7 @@ static void buildTagQueryResult(SQInfo* pQInfo) { } else { type = pExprInfo[j].type; bytes = pExprInfo[j].bytes; - + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.colInfo.colId, type, bytes); dst = pQuery->sdata[j]->data + count * pExprInfo[j].bytes; @@ -6710,12 +6892,16 @@ void freeqinfoFn(void *qhandle) { } void* qOpenQueryMgmt(int32_t vgId) { - const int32_t REFRESH_HANDLE_INTERVAL = 60; // every 30 seconds, refresh handle pool + const int32_t REFRESH_HANDLE_INTERVAL = 30; // every 30 seconds, refresh handle pool char cacheName[128] = {0}; sprintf(cacheName, "qhandle_%d", vgId); SQueryMgmt* pQueryMgmt = calloc(1, sizeof(SQueryMgmt)); + if (pQueryMgmt == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } pQueryMgmt->qinfoPool = taosCacheInit(TSDB_DATA_TYPE_BIGINT, REFRESH_HANDLE_INTERVAL, true, freeqinfoFn, cacheName); pQueryMgmt->closed = false; diff --git a/src/query/src/qParserImpl.c b/src/query/src/qParserImpl.c index d8f12da642..9629f24cc2 100644 --- a/src/query/src/qParserImpl.c +++ b/src/query/src/qParserImpl.c @@ -33,7 +33,7 @@ SSqlInfo qSQLParse(const char *pStr) { int32_t i = 0; while (1) { - SSQLToken t0 = {0}; + SStrToken t0 = {0}; if (pStr[i] == 0) { Parse(pParser, 0, t0, &sqlInfo); @@ -73,12 +73,12 @@ abort_parse: return sqlInfo; } -tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SSQLToken *pToken) { +tSQLExprList *tSQLExprListAppend(tSQLExprList *pList, tSQLExpr *pNode, SStrToken *pToken) { if (pList == NULL) { pList = calloc(1, sizeof(tSQLExprList)); } - if (pList->nAlloc <= pList->nExpr) { // + if (pList->nAlloc <= pList->nExpr) { pList->nAlloc = (pList->nAlloc << 1) + 4; pList->a = realloc(pList->a, pList->nAlloc * sizeof(pList->a[0])); if (pList->a == 0) { @@ -117,41 +117,45 @@ void tSQLExprListDestroy(tSQLExprList *pList) { free(pList); } -tSQLExpr *tSQLExprIdValueCreate(SSQLToken *pAliasToken, int32_t optrType) { - tSQLExpr *nodePtr = calloc(1, sizeof(tSQLExpr)); +tSQLExpr *tSQLExprIdValueCreate(SStrToken *pToken, int32_t optrType) { + tSQLExpr *pSQLExpr = calloc(1, sizeof(tSQLExpr)); + + if (pToken != NULL) { + pSQLExpr->token = *pToken; + } if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { - toTSDBType(pAliasToken->type); + toTSDBType(pToken->type); - tVariantCreate(&nodePtr->val, pAliasToken); - nodePtr->nSQLOptr = optrType; + tVariantCreate(&pSQLExpr->val, pToken); + pSQLExpr->nSQLOptr = optrType; } else if (optrType == TK_NOW) { // default use microsecond - nodePtr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSQLExpr->val.i64Key = taosGetTimestamp(TSDB_TIME_PRECISION_MICRO); + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond } else if (optrType == TK_VARIABLE) { - int32_t ret = getTimestampInUsFromStr(pAliasToken->z, pAliasToken->n, &nodePtr->val.i64Key); + int32_t ret = getTimestampInUsFromStr(pToken->z, pToken->n, &pSQLExpr->val.i64Key); UNUSED(ret); - nodePtr->val.nType = TSDB_DATA_TYPE_BIGINT; - nodePtr->nSQLOptr = TK_TIMESTAMP; + pSQLExpr->val.nType = TSDB_DATA_TYPE_BIGINT; + pSQLExpr->nSQLOptr = TK_TIMESTAMP; } else { // it must be the column name (tk_id) if it is not the number assert(optrType == TK_ID || optrType == TK_ALL); - if (pAliasToken != NULL) { - nodePtr->colInfo = *pAliasToken; + if (pToken != NULL) { + pSQLExpr->colInfo = *pToken; } - nodePtr->nSQLOptr = optrType; + pSQLExpr->nSQLOptr = optrType; } - return nodePtr; + return pSQLExpr; } /* * pList is the parameters for function with id(optType) * function name is denoted by pFunctionToken */ -tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQLToken *endToken, int32_t optType) { +tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SStrToken *pFuncToken, SStrToken *endToken, int32_t optType) { if (pFuncToken == NULL) return NULL; tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); @@ -163,6 +167,8 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQ pExpr->operand.n = len; // raw field name pExpr->operand.type = pFuncToken->type; + + pExpr->token = pExpr->operand; return pExpr; } @@ -173,8 +179,14 @@ tSQLExpr *tSQLExprCreateFunction(tSQLExprList *pList, SSQLToken *pFuncToken, SSQ tSQLExpr *tSQLExprCreate(tSQLExpr *pLeft, tSQLExpr *pRight, int32_t optrType) { tSQLExpr *pExpr = calloc(1, sizeof(tSQLExpr)); - if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || - optrType == TK_REM) { + if (pRight != NULL && pLeft != NULL) { + char* endPos = pRight->token.z + pRight->token.n; + pExpr->token.z = pLeft->token.z; + pExpr->token.n = (uint32_t)(endPos - pExpr->token.z); + pExpr->token.type = pLeft->token.type; + } + + if (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM) { /* * if a token is noted as the TK_TIMESTAMP, the time precision is microsecond * Otherwise, the time precision is adaptive, determined by the time precision from databases. @@ -373,7 +385,7 @@ void tVariantListDestroy(tVariantList *pList) { free(pList); } -tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToken, uint8_t sortOrder) { +tVariantList *tVariantListAppendToken(tVariantList *pList, SStrToken *pToken, uint8_t sortOrder) { if (pList == NULL) { pList = calloc(1, sizeof(tVariantList)); } @@ -382,9 +394,9 @@ tVariantList *tVariantListAppendToken(tVariantList *pList, SSQLToken *pAliasToke return pList; } - if (pAliasToken) { + if (pToken) { tVariant t = {0}; - tVariantCreate(&t, pAliasToken); + tVariantCreate(&t, pToken); tVariantListItem *pItem = &pList->a[pList->nExpr++]; memcpy(pItem, &t, sizeof(tVariant)); @@ -420,7 +432,7 @@ void tFieldListDestroy(tFieldList *pList) { free(pList); } -void setDBName(SSQLToken *pCpxName, SSQLToken *pDB) { +void setDBName(SStrToken *pCpxName, SStrToken *pDB) { pCpxName->type = pDB->type; pCpxName->z = pDB->z; pCpxName->n = pDB->n; @@ -464,7 +476,7 @@ int32_t getTimestampInUsFromStrImpl(int64_t val, char unit, int64_t *result) { return 0; } -void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) { +void tSQLSetColumnInfo(TAOS_FIELD *pField, SStrToken *pName, TAOS_FIELD *pType) { int32_t maxLen = sizeof(pField->name) / sizeof(pField->name[0]); // truncate the column name @@ -479,7 +491,7 @@ void tSQLSetColumnInfo(TAOS_FIELD *pField, SSQLToken *pName, TAOS_FIELD *pType) pField->bytes = pType->bytes; } -void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { +void tSQLSetColumnType(TAOS_FIELD *pField, SStrToken *type) { pField->type = -1; int32_t LENGTH_SIZE_OF_STR = 2; // in case of nchar and binary, there two bytes to keep the length of binary|nchar. @@ -517,9 +529,9 @@ void tSQLSetColumnType(TAOS_FIELD *pField, SSQLToken *type) { /* * extract the select info out of sql string */ -SQuerySQL *tSetQuerySQLElems(SSQLToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, - tVariantList *pGroupby, tVariantList *pSortOrder, SSQLToken *pInterval, - SSQLToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { +SQuerySQL *tSetQuerySQLElems(SStrToken *pSelectToken, tSQLExprList *pSelection, tVariantList *pFrom, tSQLExpr *pWhere, + tVariantList *pGroupby, tVariantList *pSortOrder, SStrToken *pInterval, + SStrToken *pSliding, tVariantList *pFill, SLimitVal *pLimit, SLimitVal *pGLimit) { assert(pSelection != NULL); SQuerySQL *pQuery = calloc(1, sizeof(SQuerySQL)); @@ -611,7 +623,7 @@ void destroyAllSelectClause(SSubclauseInfo *pClause) { taosTFree(pClause->pClause); } -SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLToken *pStableName, +SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SStrToken *pStableName, tVariantList *pTagVals, SQuerySQL *pSelect, int32_t type) { SCreateTableSQL *pCreate = calloc(1, sizeof(SCreateTableSQL)); @@ -644,7 +656,7 @@ SCreateTableSQL *tSetCreateSQLElems(tFieldList *pCols, tFieldList *pTags, SSQLTo return pCreate; } -SAlterTableSQL *tAlterTableSQLElems(SSQLToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { +SAlterTableSQL *tAlterTableSQLElems(SStrToken *pMeterName, tFieldList *pCols, tVariantList *pVals, int32_t type) { SAlterTableSQL *pAlterTable = calloc(1, sizeof(SAlterTableSQL)); pAlterTable->name = *pMeterName; @@ -716,7 +728,7 @@ SSubclauseInfo* setSubclause(SSubclauseInfo* pSubclause, void *pSqlExprInfo) { return pSubclause; } -SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SSQLToken *pMeterName, int32_t type) { +SSqlInfo* setSQLInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SStrToken *pMeterName, int32_t type) { pInfo->type = type; if (type == TSDB_SQL_SELECT) { @@ -745,7 +757,7 @@ SSubclauseInfo* appendSelectClause(SSubclauseInfo *pQueryInfo, void *pSubclause) return pQueryInfo; } -void setCreatedTableName(SSqlInfo *pInfo, SSQLToken *pMeterName, SSQLToken *pIfNotExists) { +void setCreatedTableName(SSqlInfo *pInfo, SStrToken *pMeterName, SStrToken *pIfNotExists) { pInfo->pCreateTableInfo->name = *pMeterName; pInfo->pCreateTableInfo->existCheck = (pIfNotExists->n != 0); } @@ -760,7 +772,7 @@ void tTokenListBuyMoreSpace(tDCLSQL *pTokenList) { } } -tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SSQLToken *pToken) { +tDCLSQL *tTokenListAppend(tDCLSQL *pTokenList, SStrToken *pToken) { if (pToken == NULL) return NULL; if (pTokenList == NULL) pTokenList = calloc(1, sizeof(tDCLSQL)); @@ -781,19 +793,19 @@ void setDCLSQLElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) { va_start(va, nParam); while (nParam-- > 0) { - SSQLToken *pToken = va_arg(va, SSQLToken *); + SStrToken *pToken = va_arg(va, SStrToken *); pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); } va_end(va); } -void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SSQLToken* pToken, SSQLToken* existsCheck) { +void setDropDBTableInfo(SSqlInfo *pInfo, int32_t type, SStrToken* pToken, SStrToken* existsCheck) { pInfo->type = type; pInfo->pDCLInfo = tTokenListAppend(pInfo->pDCLInfo, pToken); pInfo->pDCLInfo->existsCheck = (existsCheck->n == 1); } -void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* pPatterns) { +void setShowOptions(SSqlInfo *pInfo, int32_t type, SStrToken* prefix, SStrToken* pPatterns) { if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); } @@ -816,7 +828,7 @@ void setShowOptions(SSqlInfo *pInfo, int32_t type, SSQLToken* prefix, SSQLToken* } } -void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBInfo *pDB, SSQLToken *pIgExists) { +void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pToken, SCreateDBInfo *pDB, SStrToken *pIgExists) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -827,7 +839,7 @@ void setCreateDBSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pToken, SCreateDBI pInfo->pDCLInfo->dbOpt.ignoreExists = pIgExists->n; // sql.y has: ifnotexists(X) ::= IF NOT EXISTS. {X.n = 1;} } -void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken *pPwd, SCreateAcctSQL *pAcctInfo) { +void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SStrToken *pName, SStrToken *pPwd, SCreateAcctSQL *pAcctInfo) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -843,7 +855,7 @@ void setCreateAcctSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *pName, SSQLToken } } -void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { +void setCreateUserSQL(SSqlInfo *pInfo, SStrToken *pName, SStrToken *pPasswd) { pInfo->type = TSDB_SQL_CREATE_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -855,7 +867,7 @@ void setCreateUserSQL(SSqlInfo *pInfo, SSQLToken *pName, SSQLToken *pPasswd) { pInfo->pDCLInfo->user.passwd = *pPasswd; } -void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* pPwd, SSQLToken *pPrivilege) { +void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SStrToken *pName, SStrToken* pPwd, SStrToken *pPrivilege) { pInfo->type = TSDB_SQL_ALTER_USER; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -880,7 +892,7 @@ void setAlterUserSQL(SSqlInfo *pInfo, int16_t type, SSQLToken *pName, SSQLToken* } } -void setKillSQL(SSqlInfo *pInfo, int32_t type, SSQLToken *ip) { +void setKillSQL(SSqlInfo *pInfo, int32_t type, SStrToken *ip) { pInfo->type = type; if (pInfo->pDCLInfo == NULL) { pInfo->pDCLInfo = calloc(1, sizeof(tDCLSQL)); @@ -909,5 +921,5 @@ void setDefaultCreateDbOption(SCreateDBInfo *pDBInfo) { pDBInfo->quorum = -1; pDBInfo->keep = NULL; - memset(&pDBInfo->precision, 0, sizeof(SSQLToken)); + memset(&pDBInfo->precision, 0, sizeof(SStrToken)); } diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 18d2520542..b3e97459d3 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -137,8 +137,15 @@ static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { } //3. write to disk. - fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - fwrite(t, size, 1, pResultBuf->file); + int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); + if (ret != 0) { // todo handle the error case + + } + + ret = (int32_t)fwrite(t, size, 1, pResultBuf->file); + if (ret != size) { // todo handle the error case + + } if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { pResultBuf->fileSize = pg->info.offset + pg->info.length; diff --git a/src/query/src/qTokenizer.c b/src/query/src/qTokenizer.c index 227aded5f1..5fdabca166 100644 --- a/src/query/src/qTokenizer.c +++ b/src/query/src/qTokenizer.c @@ -85,7 +85,7 @@ static SKeyword keywordTable[] = { {"QUERIES", TK_QUERIES}, {"CONNECTIONS", TK_CONNECTIONS}, {"STREAMS", TK_STREAMS}, - {"CONFIGS", TK_CONFIGS}, + {"VARIABLES", TK_VARIABLES}, {"SCORES", TK_SCORES}, {"GRANTS", TK_GRANTS}, {"DOT", TK_DOT}, @@ -580,8 +580,8 @@ uint32_t tSQLGetToken(char* z, uint32_t* tokenType) { return 0; } -SSQLToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { - SSQLToken t0 = {0}; +SStrToken tStrGetToken(char* str, int32_t* i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t* ignoreTokenTypes) { + SStrToken t0 = {0}; // here we reach the end of sql string, null-terminated string if (str[*i] == 0) { diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 25eb33ff7d..b264f6cdc9 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -8,7 +8,6 @@ static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInf static STSBuf* allocResForTSBuf(STSBuf* pTSBuf); static int32_t STSBufUpdateHeader(STSBuf* pTSBuf, STSBufFileHeader* pHeader); - /** * todo error handling * support auto closeable tmp file @@ -76,7 +75,7 @@ STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete) { return NULL; } - if ((int32_t)header.numOfVnode > pTSBuf->numOfAlloc) { + if (header.numOfVnode > pTSBuf->numOfAlloc) { pTSBuf->numOfAlloc = header.numOfVnode; STSVnodeBlockInfoEx* tmp = realloc(pTSBuf->pData, sizeof(STSVnodeBlockInfoEx) * pTSBuf->numOfAlloc); if (tmp == NULL) { @@ -225,14 +224,15 @@ static void writeDataToDisk(STSBuf* pTSBuf) { } STSBlock* pBlock = &pTSBuf->block; - - pBlock->numOfElem = pTSBuf->tsData.len / TSDB_KEYSIZE; + STSList* pTsData = &pTSBuf->tsData; + + pBlock->numOfElem = pTsData->len / TSDB_KEYSIZE; pBlock->compLen = - tsCompressTimestamp(pTSBuf->tsData.rawBuf, pTSBuf->tsData.len, pTSBuf->tsData.len / TSDB_KEYSIZE, pBlock->payload, - pTSBuf->tsData.allocSize, TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); + tsCompressTimestamp(pTsData->rawBuf, pTsData->len, pTsData->len/TSDB_KEYSIZE, pBlock->payload, pTsData->allocSize, + TWO_STAGE_COMP, pTSBuf->assistBuf, pTSBuf->bufSize); int64_t r = fseek(pTSBuf->f, pTSBuf->fileSize, SEEK_SET); - UNUSED(r); + assert(r == 0); /* * format for output data: @@ -241,16 +241,22 @@ static void writeDataToDisk(STSBuf* pTSBuf) { * * both side has the compressed length is used to support load data forwards/backwords. */ - fwrite(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); + int32_t metaLen = 0; + metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); + } + fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - int32_t blockSize = sizeof(pBlock->tag) + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; + int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; pTSBuf->tsData.len = 0; @@ -298,9 +304,24 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { ret = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(ret); } - - size_t sz = fread(&pBlock->tag, sizeof(pBlock->tag), 1, pTSBuf->f); - UNUSED(sz); + + fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); + fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + + // NOTE: mix types tags are not supported + size_t sz = 0; + if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + char* tp = realloc(pBlock->tag.pz, pBlock->tag.nLen + 1); + assert(tp != NULL); + + memset(tp, 0, pBlock->tag.nLen + 1); + pBlock->tag.pz = tp; + + sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); + } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { + sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + } + sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); UNUSED(sz); sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); @@ -361,7 +382,7 @@ static int32_t setCheckTSOrder(STSBuf* pTSBuf, const char* pData, int32_t len) { return TSDB_CODE_SUCCESS; } -void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData, int32_t len) { +void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len) { STSVnodeBlockInfoEx* pBlockInfo = NULL; STSList* ptsData = &pTSBuf->tsData; @@ -375,15 +396,15 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag, const char* pData } assert(pBlockInfo->info.vnode == vnodeId); - - if (pTSBuf->block.tag != tag && ptsData->len > 0) { + + if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first writeDataToDisk(pTSBuf); } else { expandBuffer(ptsData, len); } - pTSBuf->block.tag = tag; + tVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); // todo check return value @@ -465,7 +486,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int return 0; } -static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int64_t tag) { +static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, tVariant* tag) { bool decomp = false; int64_t offset = 0; @@ -484,7 +505,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo return -1; } - if (pTSBuf->block.tag == tag) { + if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { return i; } } @@ -669,8 +690,8 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - elem1.tag = pBlock->tag; - + tVariantAssign(&elem1.tag, &pBlock->tag); + return elem1; } @@ -800,7 +821,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ return pTSBuf; } -STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, int64_t tag) { +STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { STSElem elem = {.vnode = -1}; if (pTSBuf == NULL) { @@ -881,7 +902,9 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag, elem.ts); + if (elem.tag.nType == TSDB_DATA_TYPE_BIGINT) { + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + } } pTSBuf->cur.order = old; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index a01eb33ec7..509362863c 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -45,8 +45,8 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; + + SQueryCostInfo* pSummary = &pRuntimeEnv->summary; // use the pointer arraylist pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); @@ -54,8 +54,11 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pRuntimeEnv->summary.internalSupSize += sizeof(SWindowResult) * threshold; - pRuntimeEnv->summary.internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pWindowResInfo->interval = pRuntimeEnv->pQuery->intervalTime; + + pSummary->internalSupSize += sizeof(SWindowResult) * threshold; + pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pSummary->numOfTimeWindows = threshold; for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); @@ -126,8 +129,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { for (int32_t i = 0; i < num; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; - if (pResult->status.closed) { // remove the window slot from hash table - taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->window.skey, pWindowResInfo->type); + if (pResult->closed) { // remove the window slot from hash table + taosHashRemove(pWindowResInfo->hashList, (const char *)&pResult->skey, pWindowResInfo->type); } else { break; } @@ -149,12 +152,12 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { SWindowResult *pResult = &pWindowResInfo->pResult[k]; - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->window.skey, + int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)&pResult->window.skey, tDataTypeDesc[pWindowResInfo->type].nSize, + taosHashPut(pWindowResInfo->hashList, (char *)&pResult->skey, tDataTypeDesc[pWindowResInfo->type].nSize, (char *)&v, sizeof(int32_t)); } @@ -173,7 +176,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].status.closed) { + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) { ++i; } @@ -184,11 +187,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i].status.closed) { + if (pWindowResInfo->pResult[i].closed) { continue; } - pWindowResInfo->pResult[i].status.closed = true; + pWindowResInfo->pResult[i].closed = true; } } @@ -204,7 +207,7 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].window.skey < pWindowResInfo->pResult[1].window.skey)? 1:-1; + int32_t resultOrder = (pWindowResInfo->pResult[0].skey < pWindowResInfo->pResult[1].skey)? 1:-1; if (order != resultOrder) { return; @@ -212,11 +215,12 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.ekey < lastKey)) { + TSKEY ekey = pWindowResInfo->pResult[i].skey + pWindowResInfo->interval; + while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].window.skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].skey > lastKey)) { ++i; } } @@ -227,11 +231,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { - return (getWindowResult(pWindowResInfo, slot)->status.closed == true); + return (getWindowResult(pWindowResInfo, slot)->closed == true); } void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { - getWindowResult(pWindowResInfo, slot)->status.closed = true; + getWindowResult(pWindowResInfo, slot)->closed = true; } void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { @@ -253,8 +257,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow pWindowRes->numOfRows = 0; pWindowRes->pos = (SPosInfo){-1, -1}; - pWindowRes->status.closed = false; - pWindowRes->window = TSWINDOW_INITIALIZER; + pWindowRes->closed = false; + pWindowRes->skey = TSKEY_INITIAL_VAL; } /** @@ -264,8 +268,8 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow */ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { dst->numOfRows = src->numOfRows; - dst->window = src->window; - dst->status = src->status; + dst->skey = src->skey; + dst->closed = src->closed; int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; diff --git a/src/query/src/sql.c b/src/query/src/sql.c index 75ef2f3218..f3f55791b0 100644 --- a/src/query/src/sql.c +++ b/src/query/src/sql.c @@ -99,7 +99,7 @@ #define YYCODETYPE unsigned short int #define YYNOCODE 274 #define YYACTIONTYPE unsigned short int -#define ParseTOKENTYPE SSQLToken +#define ParseTOKENTYPE SStrToken typedef union { int yyinit; ParseTOKENTYPE yy0; @@ -488,7 +488,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* QUERIES => nothing */ 0, /* CONNECTIONS => nothing */ 0, /* STREAMS => nothing */ - 0, /* CONFIGS => nothing */ + 0, /* VARIABLES => nothing */ 0, /* SCORES => nothing */ 0, /* GRANTS => nothing */ 0, /* VNODES => nothing */ @@ -781,7 +781,7 @@ static const char *const yyTokenName[] = { /* 51 */ "QUERIES", /* 52 */ "CONNECTIONS", /* 53 */ "STREAMS", - /* 54 */ "CONFIGS", + /* 54 */ "VARIABLES", /* 55 */ "SCORES", /* 56 */ "GRANTS", /* 57 */ "VNODES", @@ -1017,7 +1017,7 @@ static const char *const yyRuleName[] = { /* 7 */ "cmd ::= SHOW QUERIES", /* 8 */ "cmd ::= SHOW CONNECTIONS", /* 9 */ "cmd ::= SHOW STREAMS", - /* 10 */ "cmd ::= SHOW CONFIGS", + /* 10 */ "cmd ::= SHOW VARIABLES", /* 11 */ "cmd ::= SHOW SCORES", /* 12 */ "cmd ::= SHOW GRANTS", /* 13 */ "cmd ::= SHOW VNODES", @@ -1702,7 +1702,7 @@ static const struct { { 209, -2 }, /* (7) cmd ::= SHOW QUERIES */ { 209, -2 }, /* (8) cmd ::= SHOW CONNECTIONS */ { 209, -2 }, /* (9) cmd ::= SHOW STREAMS */ - { 209, -2 }, /* (10) cmd ::= SHOW CONFIGS */ + { 209, -2 }, /* (10) cmd ::= SHOW VARIABLES */ { 209, -2 }, /* (11) cmd ::= SHOW SCORES */ { 209, -2 }, /* (12) cmd ::= SHOW GRANTS */ { 209, -2 }, /* (13) cmd ::= SHOW VNODES */ @@ -2029,8 +2029,8 @@ static void yy_reduce( case 9: /* cmd ::= SHOW STREAMS */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_STREAMS, 0, 0); } break; - case 10: /* cmd ::= SHOW CONFIGS */ -{ setShowOptions(pInfo, TSDB_MGMT_TABLE_CONFIGS, 0, 0); } + case 10: /* cmd ::= SHOW VARIABLES */ +{ setShowOptions(pInfo, TSDB_MGMT_TABLE_VARIABLES, 0, 0); } break; case 11: /* cmd ::= SHOW SCORES */ { setShowOptions(pInfo, TSDB_MGMT_TABLE_SCORES, 0, 0); } @@ -2074,21 +2074,21 @@ static void yy_reduce( break; case 22: /* cmd ::= SHOW dbPrefix STABLES LIKE ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-3].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_METRIC, &token, &yymsp[0].minor.yy0); } break; case 23: /* cmd ::= SHOW dbPrefix VGROUPS */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-1].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, 0); } break; case 24: /* cmd ::= SHOW dbPrefix VGROUPS ids */ { - SSQLToken token; + SStrToken token; setDBName(&token, &yymsp[-2].minor.yy0); setShowOptions(pInfo, TSDB_MGMT_TABLE_VGROUP, &token, &yymsp[0].minor.yy0); } @@ -2139,7 +2139,7 @@ static void yy_reduce( { setDCLSQLElems(pInfo, TSDB_SQL_CFG_LOCAL, 2, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; case 38: /* cmd ::= ALTER DATABASE ids alter_db_optr */ -{ SSQLToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} +{ SStrToken t = {0}; setCreateDBSQL(pInfo, TSDB_SQL_ALTER_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy268, &t);} break; case 39: /* cmd ::= ALTER ACCOUNT ids acct_optr */ { setCreateAcctSQL(pInfo, TSDB_SQL_ALTER_ACCT, &yymsp[-1].minor.yy0, NULL, &yymsp[0].minor.yy149);} @@ -2277,6 +2277,7 @@ static void yy_reduce( yymsp[-1].minor.yy268 = yylhsminor.yy268; break; case 95: /* db_optr ::= db_optr fsync */ + case 107: /* alter_db_optr ::= alter_db_optr fsync */ yytestcase(yyruleno==107); { yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } yymsp[-1].minor.yy268 = yylhsminor.yy268; break; @@ -2297,10 +2298,6 @@ static void yy_reduce( case 99: /* alter_db_optr ::= */ { setDefaultCreateDbOption(&yymsp[1].minor.yy268);} break; - case 107: /* alter_db_optr ::= alter_db_optr fsync */ -{ yylhsminor.yy268 = yymsp[-1].minor.yy268; yylhsminor.yy268.fsyncPeriod = strtol(yymsp[0].minor.yy0.z, NULL, 10); } - yymsp[-1].minor.yy268 = yylhsminor.yy268; - break; case 108: /* typename ::= ids */ { yymsp[0].minor.yy0.type = 0; diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f8738eec9c..b78c5314f2 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -32,14 +32,16 @@ void simpleTest() { // write 10 ts points int32_t num = 10; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(10, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, sizeof(int64_t) * num); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -57,14 +59,16 @@ void largeTSTest() { // write 10 ts points int32_t num = 1000000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = 1; int64_t* list = createTsList(num, 10000000, 30); - tsBufAppend(pTSBuf, 0, tag, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); // the data has been flush to disk, no data in cache EXPECT_EQ(pTSBuf->tsData.len, 0); - EXPECT_EQ(pTSBuf->block.tag, tag); + EXPECT_EQ(tVariantCompare(&pTSBuf->block.tag, &t), 0); EXPECT_EQ(pTSBuf->numOfVnodes, 1); EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); @@ -80,14 +84,18 @@ void multiTagsTest() { STSBuf* pTSBuf = tsBufCreate(true, TSDB_ORDER_ASC); int32_t num = 10000; - int64_t tag = 1; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int64_t start = 10000000; int32_t numOfTags = 50; int32_t step = 30; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -96,7 +104,7 @@ void multiTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->numOfVnodes, 1); tsBufFlush(pTSBuf); @@ -118,9 +126,14 @@ void multiVnodeTagsTest() { for (int32_t j = 0; j < 20; ++j) { // vnodeId:0 start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -131,11 +144,11 @@ void multiVnodeTagsTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -157,9 +170,14 @@ void loadDataTest() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %" PRIu64 "\n", i, list[0]); free(list); @@ -172,11 +190,11 @@ void loadDataTest() { EXPECT_EQ(pTSBuf->tsOrder, TSDB_ORDER_ASC); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); EXPECT_EQ(pTSBuf->tsData.len, num * sizeof(int64_t)); - EXPECT_EQ(pTSBuf->block.tag, numOfTags - 1); + EXPECT_EQ(pTSBuf->block.tag.i64Key, numOfTags - 1); tsBufFlush(pTSBuf); EXPECT_EQ(pTSBuf->tsData.len, 0); @@ -230,16 +248,21 @@ void TSTraverse() { for (int32_t j = 0; j < numOfVnode; ++j) { // vnodeId:0 int64_t start = 10000000; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); start += step * num; list = createTsList(num, start, step); - tsBufAppend(pTSBuf, j, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf, j, &t, (const char*)list, num * sizeof(int64_t)); printf("%d - %d - %" PRIu64 ", %" PRIu64 "\n", j, i, list[0], list[num - 1]); free(list); @@ -272,12 +295,16 @@ void TSTraverse() { int32_t startVnode = 1; int32_t startTag = 2; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + t.i64Key = startTag; + + tsBufGetElemStartPos(pTSBuf, startVnode, &t); int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -286,7 +313,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag == 0) { startVnode -= 1; @@ -316,13 +345,14 @@ void TSTraverse() { startVnode = 1; startTag = 2; + t.i64Key = startTag; - tsBufGetElemStartPos(pTSBuf, startVnode, startTag); + tsBufGetElemStartPos(pTSBuf, startVnode, &t); totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -331,7 +361,9 @@ void TSTraverse() { if (--totalOutput <= 0) { totalOutput = 10; - tsBufGetElemStartPos(pTSBuf, startVnode, --startTag); + startTag -= 1; + t.i64Key = startTag; + tsBufGetElemStartPos(pTSBuf, startVnode, &t); if (startTag < 0) { startVnode -= 1; @@ -375,12 +407,17 @@ void mergeDiffVnodeBufferTest() { int32_t num = 1000; int32_t numOfTags = 10; + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + // vnodeId:0 int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf1, 0, i, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 0, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + + tsBufAppend(pTSBuf1, 0, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 0, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -403,6 +440,9 @@ void mergeIdenticalVnodeBufferTest() { STSBuf* pTSBuf1 = tsBufCreate(true, TSDB_ORDER_ASC); STSBuf* pTSBuf2 = tsBufCreate(true, TSDB_ORDER_ASC); + tVariant t = {0}; + t.nType = TSDB_DATA_TYPE_BIGINT; + int32_t step = 30; int32_t num = 1000; int32_t numOfTags = 10; @@ -411,17 +451,21 @@ void mergeIdenticalVnodeBufferTest() { int64_t start = 10000000; for (int32_t i = 0; i < numOfTags; ++i) { int64_t* list = createTsList(num, start, step); + t.i64Key = i; - tsBufAppend(pTSBuf1, 12, i, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf1, 12, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; } + + for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { int64_t* list = createTsList(num, start, step); - tsBufAppend(pTSBuf2, 77, i, (const char*)list, num * sizeof(int64_t)); + t.i64Key = i; + tsBufAppend(pTSBuf2, 77, &t, (const char*)list, num * sizeof(int64_t)); free(list); start += step * num; @@ -438,7 +482,7 @@ void mergeIdenticalVnodeBufferTest() { STSElem elem = tsBufGetElem(pTSBuf1); EXPECT_EQ(elem.vnode, 12); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); } tsBufDestroy(pTSBuf1); @@ -446,6 +490,8 @@ void mergeIdenticalVnodeBufferTest() { } } // namespace + +//TODO add binary tag value test case TEST(testCase, tsBufTest) { simpleTest(); largeTSTest(); @@ -453,7 +499,7 @@ TEST(testCase, tsBufTest) { multiVnodeTagsTest(); loadDataTest(); invalidFileTest(); - // randomIncTsTest(); +// randomIncTsTest(); TSTraverse(); mergeDiffVnodeBufferTest(); mergeIdenticalVnodeBufferTest(); diff --git a/src/query/tests/unitTest.cpp b/src/query/tests/unitTest.cpp index b59e0783a2..b6e1170d0e 100644 --- a/src/query/tests/unitTest.cpp +++ b/src/query/tests/unitTest.cpp @@ -16,7 +16,7 @@ namespace { int32_t testValidateName(char* name) { - SSQLToken token = {0}; + SStrToken token = {0}; token.z = name; token.n = strlen(name); token.type = 0; @@ -720,8 +720,8 @@ TEST(testCase, tGetToken_Test) { EXPECT_FALSE(type == TK_HEX); } -static SSQLToken createStrToken(char* s) { - SSQLToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; +static SStrToken createStrToken(char* s) { + SStrToken t = {0};//.type = TK_STRING, .z = s, .n = strlen(s)}; t.type = TK_STRING; t.z = s; t.n = strlen(s); @@ -730,7 +730,7 @@ static SSQLToken createStrToken(char* s) { } TEST(testCase, isValidNumber_test) { - SSQLToken t1 = createStrToken("123abc"); + SStrToken t1 = createStrToken("123abc"); EXPECT_EQ(isValidNumber(&t1), TK_ILLEGAL); diff --git a/src/sync/src/tarbitrator.c b/src/sync/src/tarbitrator.c index 96e3392cc5..b704b1ecae 100644 --- a/src/sync/src/tarbitrator.c +++ b/src/sync/src/tarbitrator.c @@ -99,9 +99,7 @@ int main(int argc, char *argv[]) { sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsArbitratorPort); - for (int res = tsem_wait(&tsArbSem); res != 0; res = tsem_wait(&tsArbSem)) { - if (res != EINTR) break; - } + tsem_wait(&tsArbSem); taosCloseTcpThreadPool(tsArbTcpPool); sInfo("TAOS arbitrator is shut down\n"); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index ac3a6dac07..f8ff25ddab 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -28,6 +28,14 @@ #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) #define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ + ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ + .numOfCols = (_block)->numOfCols, \ + .rows = (_block)->numOfRows, \ + .tid = (_checkInfo)->tableId.tid, \ + .uid = (_checkInfo)->tableId.uid}) + + enum { TSDB_QUERY_TYPE_ALL = 1, TSDB_QUERY_TYPE_LAST = 2, @@ -119,7 +127,14 @@ typedef struct STsdbQueryHandle { SIOCostSummary cost; } STsdbQueryHandle; -static void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle); +typedef struct STableGroupSupporter { + int32_t numOfCols; + SColIndex* pCols; + STSchema* pTagSchema; +} STableGroupSupporter; + +static STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList); + static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); @@ -242,10 +257,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab for (int32_t j = 0; j < gsize; ++j) { STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); - STableCheckInfo info = { - .lastKey = pKeyInfo->lastKey, - .pTableObj = pKeyInfo->pTable, - }; + STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; info.tableId = ((STable*)(pKeyInfo->pTable))->tableId; assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || @@ -283,12 +295,9 @@ out_of_memory: } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { + pCond->order = TSDB_ORDER_ASC; + pCond->twindow = changeTableGroupByLastrow(groupList); STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); - if (pQueryHandle != NULL) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; - pQueryHandle->order = TSDB_ORDER_DESC; - changeQueryHandleForLastrowQuery(pQueryHandle); - } return pQueryHandle; } @@ -393,7 +402,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SDataRow row = *(SDataRow *)SL_GET_NODE_DATA(node); TSKEY key = dataRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", %p", pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, pCheckInfo->lastKey, pHandle->qinfo); @@ -416,7 +425,7 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) { tSkipListDestroyIter(pCheckInfo->iiter); } -SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { +static SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { SDataRow rmem = NULL, rimem = NULL; if (pCheckInfo->iter) { SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter); @@ -432,47 +441,46 @@ SDataRow getSDataRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order) { } } - if (rmem != NULL && rimem != NULL) { - TSKEY r1 = dataRowKey(rmem); - TSKEY r2 = dataRowKey(rimem); - - if (r1 == r2) { // data ts are duplicated, ignore the data in mem - tSkipListIterNext(pCheckInfo->iter); - pCheckInfo->chosen = 1; - return rimem; - } else { - if (ASCENDING_TRAVERSE(order)) { - if (r1 < r2) { - pCheckInfo->chosen = 0; - return rmem; - } else { - pCheckInfo->chosen = 1; - return rimem; - } - } else { - if (r1 < r2) { - pCheckInfo->chosen = 1; - return rimem; - } else { - pCheckInfo->chosen = 0; - return rmem; - } - } - } + if (rmem == NULL && rimem == NULL) { + return NULL; } - // at least one (rmem or rimem) is absent here - if (rmem != NULL) { + if (rmem != NULL && rimem == NULL) { pCheckInfo->chosen = 0; return rmem; } - if (rimem != NULL) { + if (rmem == NULL && rimem != NULL) { pCheckInfo->chosen = 1; return rimem; } - return NULL; + TSKEY r1 = dataRowKey(rmem); + TSKEY r2 = dataRowKey(rimem); + + if (r1 == r2) { // data ts are duplicated, ignore the data in mem + tSkipListIterNext(pCheckInfo->iter); + pCheckInfo->chosen = 1; + return rimem; + } else { + if (ASCENDING_TRAVERSE(order)) { + if (r1 < r2) { + pCheckInfo->chosen = 0; + return rmem; + } else { + pCheckInfo->chosen = 1; + return rimem; + } + } else { + if (r1 < r2) { + pCheckInfo->chosen = 1; + return rimem; + } else { + pCheckInfo->chosen = 0; + return rmem; + } + } + } } static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { @@ -672,14 +680,6 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo return code; } -#define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ - ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ - .numOfCols = (_block)->numOfCols, \ - .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) - - static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { STsdbRepo *pRepo = pQueryHandle->pTsdb; int64_t st = taosGetTimestampUs(); @@ -736,6 +736,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); TSKEY key = (row != NULL)? dataRowKey(row):TSKEY_INITIAL_VAL; + tsdbDebug("%p key in mem:%"PRId64", %p", pQueryHandle, key, pQueryHandle->qinfo); + cur->pos = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:(binfo.rows-1); if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || @@ -747,7 +749,8 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc // do not load file block into buffer int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; - cur->rows = tsdbReadRowsFromCache(pCheckInfo, binfo.window.skey - step, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); + TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); pQueryHandle->realNumOfRows = cur->rows; // update the last key value @@ -1072,8 +1075,8 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, - int32_t numOfExisted, int32_t *start, int32_t *end) { +static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, + int32_t* start, int32_t* end) { *start = -1; if (ASCENDING_TRAVERSE(pQueryHandle->order)) { @@ -1162,6 +1165,11 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->mixBlock = true; } + tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + "end:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pQueryHandle->qinfo); + // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; int32_t pos = cur->pos; @@ -1191,6 +1199,10 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* moveDataToFront(pQueryHandle, numOfRows, numOfCols); updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); + + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, + cur->win.ekey, cur->rows, pQueryHandle->qinfo); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; @@ -1233,6 +1245,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t end = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, key, order); + assert(end != -1); + if (tsArray[end] == key) { // the value of key in cache equals to the end timestamp value, ignore it moveToNextRowInMem(pCheckInfo); } @@ -1269,6 +1283,7 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; + cur->mixBlock = true; } } } @@ -1285,7 +1300,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); doCheckGeneratedBlockRange(pQueryHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->win.skey, + tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, %p", + pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pQueryHandle->qinfo); } @@ -1658,6 +1674,128 @@ static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { return false; } +static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { + // filter the queried time stamp in the first place + STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + pQueryHandle->order = TSDB_ORDER_DESC; + + assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); + + // starts from the buffer in case of descending timestamp order check data blocks + // todo consider the query time window, current last_row does not apply the query time window + size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + + int32_t i = 0; + while(i < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && + pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { + break; + } + + i++; + } + + // there are no data in all the tables + if (i == numOfTables) { + return; + } + + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); + taosArrayClear(pQueryHandle->pTableCheckInfo); + + info.lastKey = pQueryHandle->window.skey; + taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + + // update the query time window according to the chosen last timestamp + pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; +} + +static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, + STsdbQueryHandle* pQueryHandle) { + int numOfRows = 0; + int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + win->skey = TSKEY_INITIAL_VAL; + + int64_t st = taosGetTimestampUs(); + STable* pTable = pCheckInfo->pTableObj; + + do { + SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); + if (row == NULL) { + break; + } + + TSKEY key = dataRowKey(row); + if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, + pQueryHandle->window.ekey); + + break; + } + + if (win->skey == INT64_MIN) { + win->skey = key; + } + + win->ekey = key; + copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); + + if (++numOfRows >= maxRowsToRead) { + moveToNextRowInMem(pCheckInfo); + break; + } + + } while(moveToNextRowInMem(pCheckInfo)); + + assert(numOfRows <= maxRowsToRead); + + // if the buffer is not full in case of descending order query, move the data in the front of the buffer + if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + int32_t emptySize = maxRowsToRead - numOfRows; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); + } + } + + int64_t elapsedTime = taosGetTimestampUs() - st; + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, + elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); + + return numOfRows; +} + +static int32_t getAllTableList(STable* pSuperTable, SArray* list) { + SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + while (tSkipListIterNext(iter)) { + SSkipListNode* pNode = tSkipListIterGet(iter); + + STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + + STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; + taosArrayPush(list, &info); + } + + tSkipListDestroyIter(iter); + return TSDB_CODE_SUCCESS; +} + +static void destroyHelper(void* param) { + if (param == NULL) { + return; + } + + + tQueryInfo* pInfo = (tQueryInfo*)param; + if (pInfo->optr != TSDB_RELATION_IN) { + taosTFree(pInfo->q); + } + + free(param); +} + // handle data in cache situation bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; @@ -1698,6 +1836,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return true; } else { STsdbQueryHandle* pSecQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); + if (pSecQueryHandle == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return false; + } + pSecQueryHandle->order = TSDB_ORDER_ASC; pSecQueryHandle->window = (STimeWindow) {pQueryHandle->window.skey, INT64_MAX}; pSecQueryHandle->pTsdb = pQueryHandle->pTsdb; @@ -1709,6 +1852,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->outputCapacity = ((STsdbRepo*)pSecQueryHandle->pTsdb)->config.maxRowsPerFileBlock; if (tsdbInitReadHelper(&pSecQueryHandle->rhelper, (STsdbRepo*) pSecQueryHandle->pTsdb) != 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; free(pSecQueryHandle); return false; } @@ -1720,6 +1864,11 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { pSecQueryHandle->statis = calloc(numOfCols, sizeof(SDataStatis)); pSecQueryHandle->pColumns = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + if (pSecQueryHandle->statis == NULL || pSecQueryHandle->pColumns == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {{0}, 0}; @@ -1727,6 +1876,12 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { colInfo.info = pCol->info; colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCol->info.bytes); + if (colInfo.pData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbCleanupQueryHandle(pSecQueryHandle); + return false; + } + taosArrayPush(pSecQueryHandle->pColumns, &colInfo); } @@ -1794,6 +1949,7 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { } if (pQueryHandle->checkFiles) { + // check if the query range overlaps with the file data block bool exists = true; int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); @@ -1822,150 +1978,48 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { return ret; } -void changeQueryHandleForLastrowQuery(TsdbQueryHandleT pqHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pqHandle; - assert(!ASCENDING_TRAVERSE(pQueryHandle->order)); - - // starts from the buffer in case of descending timestamp order check data blocks +STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { + STimeWindow window = {INT64_MAX, INT64_MIN}; + // NOTE: starts from the buffer in case of descending timestamp order check data blocks // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); + for(int32_t j = 0; j < numOfGroups; ++j) { + SArray* pGroup = taosArrayGetP(groupList->pGroupList, j); + TSKEY key = TSKEY_INITIAL_VAL; - TSKEY key = TSKEY_INITIAL_VAL; - int32_t index = -1; + STableKeyInfo keyInfo = {0}; - for(int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pCheckInfo->pTableObj->lastKey > key) { - key = pCheckInfo->pTableObj->lastKey; - index = i; + size_t numOfTables = taosArrayGetSize(pGroup); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); + TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + + if (key < lastKey) { + key = lastKey; + + keyInfo.pTable = pKeyInfo->pTable; + keyInfo.lastKey = key; + pKeyInfo->lastKey = key; + + if (key < window.skey) { + window.skey = key; + } + + if (key > window.ekey) { + window.ekey = key; + } + } + } + + // more than one table in each group, only one table left for each group + if (numOfTables > 1) { + taosArrayClear(pGroup); + taosArrayPush(pGroup, &keyInfo); } } - if (index == -1) { - // todo add failure test cases - return; - } - - // erase all other elements in array list - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - if (i == index) { - continue; - } - - STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - tSkipListDestroyIter(pTableCheckInfo->iter); - - if (pTableCheckInfo->pDataCols != NULL) { - taosTFree(pTableCheckInfo->pDataCols->buf); - } - - taosTFree(pTableCheckInfo->pDataCols); - taosTFree(pTableCheckInfo->pCompInfo); - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, index); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = key; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {key, key}; -} - -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { - // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - pQueryHandle->order = TSDB_ORDER_DESC; - - assert(pQueryHandle->window.skey == pQueryHandle->window.ekey); - - // starts from the buffer in case of descending timestamp order check data blocks - // todo consider the query time window, current last_row does not apply the query time window - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - - int32_t i = 0; - while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - if (pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey && - pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL) { - break; - } - - i++; - } - - // there are no data in all the tables - if (i == numOfTables) { - return; - } - - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); - - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); - - // update the query time window according to the chosen last timestamp - pQueryHandle->window = (STimeWindow) {info.lastKey, TSKEY_INITIAL_VAL}; -} - -static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { - int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - win->skey = TSKEY_INITIAL_VAL; - - int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; - - do { - SDataRow row = getSDataRowInTableMem(pCheckInfo, pQueryHandle->order); - if (row == NULL) { - break; - } - - TSKEY key = dataRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); - - break; - } - - if (win->skey == INT64_MIN) { - win->skey = key; - } - - win->ekey = key; - copyOneRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, numOfCols, pTable); - - if (++numOfRows >= maxRowsToRead) { - moveToNextRowInMem(pCheckInfo); - break; - } - - } while(moveToNextRowInMem(pCheckInfo)); - - assert(numOfRows <= maxRowsToRead); - - // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { - int32_t emptySize = maxRowsToRead - numOfRows; - - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); - } - } - - int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, %p", pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qinfo); - - return numOfRows; + return window; } void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { @@ -2102,36 +2156,6 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } -static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); - while (tSkipListIterNext(iter)) { - SSkipListNode* pNode = tSkipListIterGet(iter); - - STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); - - STableKeyInfo info = {.pTable = *pTable, .lastKey = TSKEY_INITIAL_VAL}; - taosArrayPush(list, &info); - } - - tSkipListDestroyIter(iter); - return TSDB_CODE_SUCCESS; -} - -static void destroyHelper(void* param) { - if (param == NULL) { - return; - } - - - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - taosTFree(pInfo->q); - } - -// tVariantDestroy(&(pInfo->q)); - free(param); -} - void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -2158,13 +2182,7 @@ void filterPrepare(void* expr, void* param) { } } -typedef struct STableGroupSupporter { - int32_t numOfCols; - SColIndex* pCols; - STSchema* pTagSchema; -} STableGroupSupporter; - -int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { +static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { STableGroupSupporter* pTableGroupSupp = (STableGroupSupporter*) param; STable* pTable1 = ((STableKeyInfo*) p1)->pTable; STable* pTable2 = ((STableKeyInfo*) p2)->pTable; @@ -2217,8 +2235,19 @@ int32_t tableGroupComparFn(const void *p1, const void *p2, const void *param) { return 0; } -void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, STableGroupSupporter* pSupp, - __ext_compar_fn_t compareFn) { +static int tsdbCheckInfoCompar(const void* key1, const void* key2) { + if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + return -1; + } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + return 1; + } else { + ASSERT(false); + return 0; + } +} + +void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTables, TSKEY skey, + STableGroupSupporter* pSupp, __ext_compar_fn_t compareFn) { STable* pTable = taosArrayGetP(pTableList, 0); SArray* g = taosArrayInit(16, sizeof(STableKeyInfo)); @@ -2264,6 +2293,10 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC if (numOfOrderCols == 0 || size == 1) { // no group by tags clause or only one table SArray* sa = taosArrayInit(size, sizeof(STableKeyInfo)); + if (sa == NULL) { + taosArrayDestroy(pTableGroup); + return NULL; + } for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); @@ -2278,20 +2311,19 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC taosArrayPush(pTableGroup, &sa); tsdbDebug("all %" PRIzu " tables belong to one group", size); } else { - STableGroupSupporter *pSupp = (STableGroupSupporter *) calloc(1, sizeof(STableGroupSupporter)); - pSupp->numOfCols = numOfOrderCols; - pSupp->pTagSchema = pTagSchema; - pSupp->pCols = pCols; + STableGroupSupporter sup = {0}; + sup.numOfCols = numOfOrderCols; + sup.pTagSchema = pTagSchema; + sup.pCols = pCols; - taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), pSupp, tableGroupComparFn); - createTableGroupImpl(pTableGroup, pTableList, size, skey, pSupp, tableGroupComparFn); - taosTFree(pSupp); + taosqsort(pTableList->pData, size, sizeof(STableKeyInfo), &sup, tableGroupComparFn); + createTableGroupImpl(pTableGroup, pTableList, size, skey, &sup, tableGroupComparFn); } return pTableGroup; } -bool indexedNodeFilterFp(const void* pNode, void* param) { +static bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); @@ -2562,7 +2594,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { tsdbDestroyHelper(&pQueryHandle->rhelper); SIOCostSummary* pCost = &pQueryHandle->cost; - tsdbDebug("%p :io-cost summary: statis-info:%"PRId64"us, datablock:%" PRId64"us, check data:%"PRId64"us, %p", + tsdbDebug("%p :io-cost summary: statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, %p", pQueryHandle, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qinfo); taosTFree(pQueryHandle); @@ -2589,14 +2621,3 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { taosArrayDestroy(pGroupList->pGroupList); } - -static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { - return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { - return 1; - } else { - ASSERT(false); - return 0; - } -} diff --git a/src/util/inc/tcache.h b/src/util/inc/tcache.h index 11121fcf3b..3d6f9705ee 100644 --- a/src/util/inc/tcache.h +++ b/src/util/inc/tcache.h @@ -103,7 +103,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext * @param keepTime survival time in second * @return cached element */ -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int keepTimeInSeconds); +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS); /** * get data from cache @@ -113,16 +113,6 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v */ void *taosCacheAcquireByKey(SCacheObj *pCacheObj, const void *key, size_t keyLen); -/** - * update the expire time of data in cache - * @param pCacheObj cache object - * @param key key - * @param keyLen keyLen - * @param expireTime new expire time of data - * @return - */ -//void* taosCacheUpdateExpireTimeByName(SCacheObj *pCacheObj, void *key, size_t keyLen, uint64_t expireTime); - /** * Add one reference count for the exist data, and assign this data for a new owner. * The new owner needs to invoke the taosCacheRelease when it does not need this data anymore. diff --git a/src/util/inc/tlockfree.h b/src/util/inc/tlockfree.h index a81f597832..e960b601ca 100644 --- a/src/util/inc/tlockfree.h +++ b/src/util/inc/tlockfree.h @@ -36,6 +36,14 @@ typedef void (*_ref_fn_t)(const void* pObj); _ref_fn_t end; \ } _ref_func = {.begin = (s), .end = (e)}; +// set the initial reference count value +#define T_REF_INIT_VAL(x, _v) \ + do { \ + assert(_v >= 0); \ + atomic_store_32(&((x)->_ref.val), (_v)); \ + } while (0) + +// increase the reference count by 1 #define T_REF_INC(x) (atomic_add_fetch_32(&((x)->_ref.val), 1)) #define T_REF_INC_WITH_CB(x, p) \ diff --git a/src/util/inc/tstoken.h b/src/util/inc/tstoken.h index c1c6f2de7a..7aeb2da6b6 100644 --- a/src/util/inc/tstoken.h +++ b/src/util/inc/tstoken.h @@ -28,11 +28,11 @@ extern "C" { #define TSQL_TBNAME_L "tbname" // used to denote the minimum unite in sql parsing -typedef struct SSQLToken { +typedef struct SStrToken { uint32_t n; uint32_t type; char * z; -} SSQLToken; +} SStrToken; /** * tokenizer for sql string @@ -52,7 +52,7 @@ uint32_t tSQLGetToken(char *z, uint32_t *tokenType); * @param ignoreTokenTypes * @return */ -SSQLToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); +SStrToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr, uint32_t numOfIgnoreToken, uint32_t *ignoreTokenTypes); /** * check if it is a keyword or not @@ -76,7 +76,7 @@ bool isKeyWord(const char *z, int32_t len); * @param pToken * @return token type, if it is not a number, TK_ILLEGAL will return */ -static FORCE_INLINE int32_t isValidNumber(const SSQLToken* pToken) { +static FORCE_INLINE int32_t isValidNumber(const SStrToken* pToken) { const char* z = pToken->z; int32_t type = TK_ILLEGAL; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 714f36f1cb..7d10545ce7 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -657,7 +657,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void*) * newSize); + void *pNewEntryList = realloc(pHashObj->hashList, sizeof(void *) * newSize); if (pNewEntryList == NULL) { // todo handle error // uDebug("cache resize failed due to out of memory, capacity remain:%d", pHashObj->capacity); return; diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index bd903c8c23..dfa982b848 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -189,12 +189,12 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext return pCacheObj; } -void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int duration) { +void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const void *pData, size_t dataSize, int durationMS) { if (pCacheObj == NULL || pCacheObj->pHashTable == NULL || pCacheObj->deleting == 1) { return NULL; } - SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, duration); + SCacheDataNode *pNode1 = taosCreateCacheNode(key, keyLen, pData, dataSize, durationMS); if (pNode1 == NULL) { uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key); return NULL; diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 9564588254..ba711ced8f 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -220,8 +220,14 @@ static int32_t compareStrPatternComp(const void* pLeft, const void* pRight) { char pattern[128] = {0}; memcpy(pattern, varDataVal(pRight), varDataLen(pRight)); assert(varDataLen(pRight) < 128); - - int32_t ret = patternMatch(pattern, varDataVal(pLeft), varDataLen(pLeft), &pInfo); + + size_t sz = varDataLen(pLeft); + char *buf = malloc(sz + 1); + memcpy(buf, varDataVal(pLeft), sz); + buf[sz] = 0; + + int32_t ret = patternMatch(pattern, buf, sz, &pInfo); + free(buf); return (ret == TSDB_PATTERN_MATCH) ? 0 : 1; } diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index cf7f5c10d4..f014dd0fab 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -123,11 +123,6 @@ void *taosProcessSchedQueue(void *param) { while (1) { if (tsem_wait(&pSched->fullSem) != 0) { - if (errno == EINTR) { - /* sem_wait is interrupted by interrupt, ignore and continue */ - uDebug("wait %s fullSem was interrupted", pSched->label); - continue; - } uError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); } if (pSched->stop) { @@ -163,12 +158,8 @@ int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { return 0; } - while (tsem_wait(&pSched->emptySem) != 0) { - if (errno != EINTR) { - uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); - break; - } - uDebug("wait %s emptySem was interrupted", pSched->label); + if (tsem_wait(&pSched->emptySem) != 0) { + uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); } if (pthread_mutex_lock(&pSched->queueMutex) != 0) diff --git a/src/util/src/version.c.in b/src/util/src/version.c.in index c7aea2afb1..21c78a0eb4 100644 --- a/src/util/src/version.c.in +++ b/src/util/src/version.c.in @@ -1,7 +1,7 @@ -char version[12] = "${TD_VER_1}.${TD_VER_2}.${TD_VER_3}.${TD_VER_4}"; +char version[12] = "${TD_VER_NUMBER}"; char compatible_version[12] = "${TD_VER_COMPATIBLE}"; char gitinfo[48] = "${TD_VER_GIT}"; char gitinfoOfInternal[48] = "${TD_VER_GIT_INTERNAL}"; char buildinfo[64] = "Built at ${TD_VER_DATE}"; -void libtaos_${TD_VER_1}_${TD_VER_2}_${TD_VER_3}_${TD_VER_4}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; +void libtaos_${TD_LIB_VER_NUMBER}_${TD_VER_OSTYPE}_${TD_VER_CPUTYPE}_${TD_VER_VERTYPE}() {}; diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index 176a451168..c294e86839 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -297,6 +297,11 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) { } pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId); + if (pVnode->qMgmt == NULL) { + vnodeCleanUp(pVnode); + return terrno; + } + pVnode->events = NULL; pVnode->status = TAOS_VN_STATUS_READY; vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode); diff --git a/tests/script/general/parser/bug.sim b/tests/script/general/parser/bug.sim deleted file mode 100644 index 2a46ad1fd6..0000000000 --- a/tests/script/general/parser/bug.sim +++ /dev/null @@ -1,43 +0,0 @@ -system sh/stop_dnodes.sh - -system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c walLevel -v 0 - -system sh/cfg.sh -n dnode1 -c dDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c mDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c sdbDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c cDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c monitorDebugflag -v 135 -system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 -system sh/cfg.sh -n dnode1 -c uDebugFlag -v 135 - -system sh/cfg.sh -n dnode1 -c httpCacheSessions -v 10 -system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 10 -system sh/cfg.sh -n dnode1 -c httpEnableCompress -v 0 - -system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 -system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 - -system sh/exec.sh -n dnode1 -s start -sql connect - -sql create database db; -sql use db; -sql create table t (ts timestamp, i int); -sql create table st1 (ts timestamp, f1 int) tags(t1 int); -sql create table st2 (ts timestamp, f2 int) tags(t2 int); - -sql create table t1 using st1 tags(1); -sql create table t2 using st2 tags(1); - -sql insert into t1 values(1575880055000, 1); -sql insert into t2 values(1575880055000, 2); - -sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts - -system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql - -system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/constCol.sim b/tests/script/general/parser/constCol.sim new file mode 100644 index 0000000000..a196ba2b50 --- /dev/null +++ b/tests/script/general/parser/constCol.sim @@ -0,0 +1,354 @@ +system sh/stop_dnodes.sh + +system sh/deploy.sh -n dnode1 -i 1 +system sh/cfg.sh -n dnode1 -c walLevel -v 0 + +system sh/cfg.sh -n dnode1 -c dDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c mDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c sdbDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c rpcDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c cDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c monitorDebugflag -v 135 +system sh/cfg.sh -n dnode1 -c httpDebugFlag -v 135 +system sh/cfg.sh -n dnode1 -c uDebugFlag -v 135 + +system sh/cfg.sh -n dnode1 -c httpCacheSessions -v 10 +system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 10 +system sh/cfg.sh -n dnode1 -c httpEnableCompress -v 0 + +system sh/cfg.sh -n dnode1 -c maxVnodeConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxMgmtConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxMeterConnections -v 30000 +system sh/cfg.sh -n dnode1 -c maxShellConns -v 30000 + +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database db; +sql use db; +sql create table t (ts timestamp, i int); +sql create table st1 (ts timestamp, f1 int) tags(t1 int); +sql create table st2 (ts timestamp, f2 int) tags(t2 int); + +sql create table t1 using st1 tags(1); +sql create table t2 using st2 tags(1); + +sql insert into t1 values(1575880055000, 1); +sql insert into t1 values(1575880059000, 1); +sql insert into t1 values(1575880069000, 1); + +sql insert into t2 values(1575880055000, 2); + +sql select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts + +system_content curl -H 'Authorization: Taosd /KfeAzX/f9na8qdtNZmtONryp201ma04bEl8LcvLUd7a8qdtNZmtONryp201ma04' -d 'select st1.ts, st1.f1, st2.f2 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts' 127.0.0.1:7111/restful/sql + +print ==============select with user-defined columns +sql select 'abc' as f, ts,f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +sql select 'abc', ts, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data01 != @19-12-09 16:27:35.000@ then + return -1 +endi + +if $data02 != 1 then + return -1 +endi + +if $data10 != @abc@ then + return -1 +endi + +sql select 'abc' from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 'abc' as f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != @abc@ then + return -1 +endi + +sql select 1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data20 != 1 then + return -1 +endi + +sql select 1 as f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +sql select 1 as f, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +sql select 1.123 as f, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.123000000 then + print expect 1.123000000 , actual:$data00 + return -1 +endi + +if $data10 != 1.123000000 then + print expect 1.123000000 , actual:$data10 + return -1 +endi + +sql select 1, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +sql select 1.2391, f1 from t1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + +if $data10 != 1.239100000 then + print expect 1.239100000 actual: $data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +print ===================== user-defined columns with agg functions +sql select 1 as t, count(*) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select 1, sum(f1) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 3 then + return -1 +endi + +sql select 1,2,3, sum(f1)*99, 4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != 2 then + return -1 +endi + +if $data02 != 3 then + return -1 +endi + +if $data03 != 297.000000000 then + print expect 297.000000000, actual:$data03 + return -1 +endi + +sql select sum(f1)*avg(f1)+12, 1,2,3,4,5,6,7,8,9,10 from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 15.000000000 then + print expect 15.000000000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +sql select 1.2987, f1, 'k' from t1 where f1=1 +if $rows != 3 then + return -1 +endi + +if $data00 != 1.298700000 then + print expect 1.298700000 actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != @k@ then + return -1 +endi + +print ====================user-defined columns with union +sql select f1, 'f1' from t1 union all select f1, 'f1' from t1; +if $rows != 6 then + return -1 +endi + +if $data00 != 1 then + return -1 +endi + +if $data01 != @f1@ then + return -1 +endi + +if $data10 != 1 then + return -1 +endi + +if $data11 != @f1@ then + return -1 +endi + +print =====================udc with join +sql select st1.ts, st1.f1, st2.f2, 'abc', 1.9827 from db.st1, db.st2 where st1.t1=st2.t2 and st1.ts=st2.ts +if $rows != 1 then + return -1 +endi + +if $data00 != @19-12-09 16:27:35.000@ then + print expect @19-12-09 16:27:35.000@ actual:$data00 + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data02 != 2 then + return -1 +endi + +if $data03 != @abc@ then + return -1 +endi + +if $data04 != 1.982700000 then + print expect 1.982700000 actual:$data04 + return -1 +endi + +print ======================udc with interval +sql select count(*), 'uuu' from t1 interval(1s) order by ts desc; +if $rows != 3 then + return -1 +endi + +print ======================udc with tags +sql select t1,'abc',tbname from st1 +if $rows != 1 then + return -1 +endi + +if $data01 != @abc@ then + return -1 +endi + +if $data02 != @t1@ then + return -1 +endi + +print ======================udc with arithmetic +sql select 1+1 from t1 +if $rows != 3 then + return -1 +endi + +sql_error select from t1 +sql_error select abc from t1 +sql_error select abc as tu from t1 + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/fill_stb.sim b/tests/script/general/parser/fill_stb.sim index 402b79d6b5..f0cd058352 100644 --- a/tests/script/general/parser/fill_stb.sim +++ b/tests/script/general/parser/fill_stb.sim @@ -133,12 +133,18 @@ endi if $data74 != -4.000000000 then return -1 endi + ## fill(value) + group by sql select max(c1), max(c2), max(c3), max(c4), max(c5) from $stb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, -1, -2, -3, -4, -5, -6, -7, -8) group by t1 $val = $rowNum * 2 +print $rowNum, $val + $val = $val - 1 $val = $val * $tbNum -if $rows != $val then +print ==================== $val + +if $rows != 190 then + print expect 190, actual:$rows return -1 endi if $data06 != 0 then diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 882f561ae1..1bb0ff5448 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -330,7 +330,10 @@ sql_error select join_tb1.* from $tb1 , $tb2 where join_tb1.ts != join_tb0.ts an sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts and join_tb1.ts >= 100000; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by ts; sql_error select join_tb1.* from $tb1 , $tb1 where join_tb1.ts = join_tb1.ts order by join_tb1.c7; - +sql_error select * from join_tb0, join_tb1 +sql_error select last_row(*) from join_tb0, join_tb1 +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts < now +sql_error select last_row(*) from $tb1, $tb2 where join_tb1.ts = join_tb2.ts print ==================================super table join ============================== # select duplicate columns diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 98eb5a8d6d..7954a8d228 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -128,3 +128,27 @@ if $rows != 86399 then return -1 endi +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 30 +if $rows != 48 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts DESC limit 2 +if $rows != 4 then + return -1 +endi + +sql select t1,t1,count(*),tbname,t1,t1,tbname from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1s) fill(NULL) group by tbname, t1 order by ts desc slimit 1 soffset 1 limit 250000 offset 1 +if $rows != 86399 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 1 +if $rows != 2 then + return -1 +endi + +sql select t1,t1,count(*),t1,t1 from lr_stb0 where ts>'2018-09-24 00:00:00.000' and ts<'2018-09-25 00:00:00.000' interval(1h) fill(NULL) group by t1 order by ts desc limit 25 offset 1 +if $rows != 46 then + return -1 +endi diff --git a/tests/script/general/parser/nchar.sim b/tests/script/general/parser/nchar.sim index 8d4e468f0e..bdac5ace55 100644 --- a/tests/script/general/parser/nchar.sim +++ b/tests/script/general/parser/nchar.sim @@ -223,6 +223,7 @@ if $rows != 2 then endi print data00 = $data00 if $data00 != 5 then + print expect 5, actual: $data00 return -1 endi if $data10 != 5 then diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index aafba2d328..4e26d14cfd 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -97,7 +97,7 @@ run general/parser/topbot.sim sleep 2000 run general/parser/union.sim sleep 2000 -run general/parser/bug.sim +run general/parser/constCol.sim sleep 2000 run general/parser/sliding.sim diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 9baa42a8d5..40598332d9 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -147,7 +147,6 @@ cd ../../../debug; make ./test.sh -f general/parser/join.sim ./test.sh -f general/parser/join_multivnode.sim ./test.sh -f general/parser/binary_escapeCharacter.sim -./test.sh -f general/parser/bug.sim ./test.sh -f general/parser/repeatAlter.sim ./test.sh -f general/parser/union.sim ./test.sh -f general/parser/topbot.sim diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 1574b5013e..6cdd63b42d 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -6,8 +6,8 @@ echo Start TDengine Testing Case ... set "SCRIPT_DIR=%~dp0" echo SCRIPT_DIR: %SCRIPT_DIR% -set "BUILD_DIR=%~dp0..\..\debug\build\bin" -set "TSIM=%~dp0..\..\debug\build\bin\tsim" +set "BUILD_DIR=%~dp0..\..\debug\32\build\bin" +set "TSIM=%~dp0..\..\debug\32\build\bin\tsim" echo BUILD_DIR: %BUILD_DIR% set "SIM_DIR=%~dp0..\..\sim" @@ -47,7 +47,7 @@ echo qdebugFlag 143 >> %TAOS_CFG% echo udebugFlag 143 >> %TAOS_CFG% set "FILE_NAME=windows\testSuite.sim" -set "FIRSTEP=localhost" +set "FIRSTEP=192.168.1.182" if "%1" == "-f" set "FILE_NAME=%2" if "%1" == "-h" set "FIRSTEP=%2" if "%3" == "-f" set "FILE_NAME=%4"