xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/tq

This commit is contained in:
Liu Jicong 2021-10-27 17:56:40 +08:00
parent 35869f2ed3 d12503e6dc
commit 0d14fb9900
71 changed files with 17935 additions and 858 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

528
Jenkinsfile vendored
View File

@ -41,72 +41,72 @@ def pre_test(){
killall -9 taosd ||echo "no taosd running"
killall -9 gdb || echo "no gdb running"
killall -9 python3.8 || echo "no python program running"
cd ${WORKSPACE}
cd ${WKC}
git reset --hard HEAD~10 >/dev/null
'''
script {
if (env.CHANGE_TARGET == 'master') {
sh '''
cd ${WORKSPACE}
cd ${WKC}
git checkout master
'''
}
else if(env.CHANGE_TARGET == '2.0'){
sh '''
cd ${WORKSPACE}
cd ${WKC}
git checkout 2.0
'''
}
else if(env.CHANGE_TARGET == '3.0'){
sh '''
cd ${WORKSPACE}
cd ${WKC}
git checkout 3.0
'''
}
else{
sh '''
cd ${WORKSPACE}
cd ${WKC}
git checkout develop
'''
}
}
sh'''
cd ${WORKSPACE}
cd ${WKC}
git pull >/dev/null
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
git clean -dfx
'''
// script {
// if (env.CHANGE_TARGET == 'master') {
// sh '''
// cd ${WK}
// git checkout master
// '''
// }
// else if(env.CHANGE_TARGET == '2.0'){
// sh '''
// cd ${WK}
// git checkout 2.0
// '''
// }
// else if(env.CHANGE_TARGET == '3.0'){
// sh '''
// cd ${WK}
// git checkout 3.0
// '''
// }
// else{
// sh '''
// cd ${WK}
// git checkout develop
// '''
// }
// }
// sh '''
// cd ${WK}
// git pull >/dev/null
script {
if (env.CHANGE_TARGET == 'master') {
sh '''
cd ${WK}
git checkout master
'''
}
else if(env.CHANGE_TARGET == '2.0'){
sh '''
cd ${WK}
git checkout 2.0
'''
}
else if(env.CHANGE_TARGET == '3.0'){
sh '''
cd ${WK}
git checkout 3.0
'''
}
else{
sh '''
cd ${WK}
git checkout develop
'''
}
}
sh '''
cd ${WK}
git pull >/dev/null
git clean -dfx
export TZ=Asia/Harbin
date
mkdir debug
@ -138,270 +138,228 @@ pipeline {
abort_previous()
abortPreviousBuilds()
}
sh'''
rm -rf ${WORKSPACE}.tes
cp -r ${WORKSPACE} ${WORKSPACE}.tes
cd ${WORKSPACE}.tes
git fetch
'''
script {
if (env.CHANGE_TARGET == 'master') {
sh '''
git checkout master
'''
}
else if(env.CHANGE_TARGET == '2.0'){
sh '''
git checkout 2.0
'''
}
else if(env.CHANGE_TARGET == '3.0'){
sh '''
git checkout 3.0
'''
}
else{
sh '''
git checkout develop
'''
}
}
sh'''
git fetch origin +refs/pull/${CHANGE_ID}/merge
git checkout -qf FETCH_HEAD
'''
script{
skipbuild='2'
skipbuild=sh(script: "git log -2 --pretty=%B | fgrep -ie '[skip ci]' -e '[ci skip]' && echo 1 || echo 2", returnStdout:true)
println skipbuild
}
sh'''
rm -rf ${WORKSPACE}.tes
'''
pre_test()
}
}
stage('Parallel test stage') {
//only build pr
options { skipDefaultCheckout() }
when {
allOf{
changeRequest()
expression{
return skipbuild.trim() == '2'
}
}
}
parallel {
stage('python_1_s1') {
agent{label " slave1 || slave11 "}
steps {
// stage('Parallel test stage') {
// //only build pr
// options { skipDefaultCheckout() }
// when {
// allOf{
// changeRequest()
// }
// }
// parallel {
// stage('python_1_s1') {
// agent{label " slave1 || slave11 "}
// steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh p1
// date'''
// }
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p1
// // date'''
// // }
}
}
stage('python_2_s5') {
agent{label " slave5 || slave15 "}
steps {
// }
// }
// stage('python_2_s5') {
// agent{label " slave5 || slave15 "}
// steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh p2
// date'''
// }
}
}
stage('python_3_s6') {
agent{label " slave6 || slave16 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p2
// // date'''
// // }
// }
// }
// stage('python_3_s6') {
// agent{label " slave6 || slave16 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh p3
// date'''
// }
}
}
stage('test_b1_s2') {
agent{label " slave2 || slave12 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh p3
// // date'''
// // }
// }
// }
// stage('test_b1_s2') {
// agent{label " slave2 || slave12 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// rm -rf /var/lib/taos/*
// rm -rf /var/log/taos/*
// nohup taosd >/dev/null &
// sleep 10
// '''
// sh '''
// cd ${WKC}/tests/examples/nodejs
// npm install td2.0-connector > /dev/null 2>&1
// node nodejsChecker.js host=localhost
// node test1970.js
// cd ${WKC}/tests/connectorTest/nodejsTest/nanosupport
// npm install td2.0-connector > /dev/null 2>&1
// node nanosecondTest.js
// // sh '''
// // rm -rf /var/lib/taos/*
// // rm -rf /var/log/taos/*
// // nohup taosd >/dev/null &
// // sleep 10
// // '''
// // sh '''
// // cd ${WKC}/tests/examples/nodejs
// // npm install td2.0-connector > /dev/null 2>&1
// // node nodejsChecker.js host=localhost
// // node test1970.js
// // cd ${WKC}/tests/connectorTest/nodejsTest/nanosupport
// // npm install td2.0-connector > /dev/null 2>&1
// // node nanosecondTest.js
// '''
// sh '''
// cd ${WKC}/tests/examples/C#/taosdemo
// mcs -out:taosdemo *.cs > /dev/null 2>&1
// echo '' |./taosdemo -c /etc/taos
// cd ${WKC}/tests/connectorTest/C#Test/nanosupport
// mcs -out:nano *.cs > /dev/null 2>&1
// echo '' |./nano
// '''
// sh '''
// cd ${WKC}/tests/gotest
// bash batchtest.sh
// '''
// sh '''
// cd ${WKC}/tests
// ./test-all.sh b1fq
// date'''
// }
}
}
stage('test_crash_gen_s3') {
agent{label " slave3 || slave13 "}
// // '''
// // sh '''
// // cd ${WKC}/tests/examples/C#/taosdemo
// // mcs -out:taosdemo *.cs > /dev/null 2>&1
// // echo '' |./taosdemo -c /etc/taos
// // cd ${WKC}/tests/connectorTest/C#Test/nanosupport
// // mcs -out:nano *.cs > /dev/null 2>&1
// // echo '' |./nano
// // '''
// // sh '''
// // cd ${WKC}/tests/gotest
// // bash batchtest.sh
// // '''
// // sh '''
// // cd ${WKC}/tests
// // ./test-all.sh b1fq
// // date'''
// // }
// }
// }
// stage('test_crash_gen_s3') {
// agent{label " slave3 || slave13 "}
steps {
pre_test()
// timeout(time: 60, unit: 'MINUTES'){
// sh '''
// cd ${WKC}/tests/pytest
// ./crash_gen.sh -a -p -t 4 -s 2000
// '''
// }
// timeout(time: 60, unit: 'MINUTES'){
// // sh '''
// // cd ${WKC}/tests/pytest
// // rm -rf /var/lib/taos/*
// // rm -rf /var/log/taos/*
// // ./handle_crash_gen_val_log.sh
// // '''
// sh '''
// cd ${WKC}/tests/pytest
// rm -rf /var/lib/taos/*
// rm -rf /var/log/taos/*
// ./handle_taosd_val_log.sh
// '''
// }
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b2fq
// date
// '''
// }
}
}
stage('test_valgrind_s4') {
agent{label " slave4 || slave14 "}
// steps {
// pre_test()
// // timeout(time: 60, unit: 'MINUTES'){
// // sh '''
// // cd ${WKC}/tests/pytest
// // ./crash_gen.sh -a -p -t 4 -s 2000
// // '''
// // }
// // timeout(time: 60, unit: 'MINUTES'){
// // // sh '''
// // // cd ${WKC}/tests/pytest
// // // rm -rf /var/lib/taos/*
// // // rm -rf /var/log/taos/*
// // // ./handle_crash_gen_val_log.sh
// // // '''
// // sh '''
// // cd ${WKC}/tests/pytest
// // rm -rf /var/lib/taos/*
// // rm -rf /var/log/taos/*
// // ./handle_taosd_val_log.sh
// // '''
// // }
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b2fq
// // date
// // '''
// // }
// }
// }
// stage('test_valgrind_s4') {
// agent{label " slave4 || slave14 "}
steps {
pre_test()
// catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// sh '''
// cd ${WKC}/tests/pytest
// ./valgrind-test.sh 2>&1 > mem-error-out.log
// ./handle_val_log.sh
// '''
// }
// timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b3fq
// date'''
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh full example
// date'''
// }
}
}
stage('test_b4_s7') {
agent{label " slave7 || slave17 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// steps {
// pre_test()
// // catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
// // sh '''
// // cd ${WKC}/tests/pytest
// // ./valgrind-test.sh 2>&1 > mem-error-out.log
// // ./handle_val_log.sh
// // '''
// // }
// // timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b3fq
// // date'''
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh full example
// // date'''
// // }
// }
// }
// stage('test_b4_s7') {
// agent{label " slave7 || slave17 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b4fq
// cd ${WKC}/tests
// ./test-all.sh p4
// cd ${WKC}/tests
// ./test-all.sh full jdbc
// cd ${WKC}/tests
// ./test-all.sh full unit
// date'''
// }
}
}
stage('test_b5_s8') {
agent{label " slave8 || slave18 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b4fq
// // cd ${WKC}/tests
// // ./test-all.sh p4
// // cd ${WKC}/tests
// // ./test-all.sh full jdbc
// // cd ${WKC}/tests
// // ./test-all.sh full unit
// // date'''
// // }
// }
// }
// stage('test_b5_s8') {
// agent{label " slave8 || slave18 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b5fq
// date'''
// }
}
}
stage('test_b6_s9') {
agent{label " slave9 || slave19 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b5fq
// // date'''
// // }
// }
// }
// stage('test_b6_s9') {
// agent{label " slave9 || slave19 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b6fq
// date'''
// }
}
}
stage('test_b7_s10') {
agent{label " slave10 || slave20 "}
steps {
pre_test()
// timeout(time: 55, unit: 'MINUTES'){
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b6fq
// // date'''
// // }
// }
// }
// stage('test_b7_s10') {
// agent{label " slave10 || slave20 "}
// steps {
// pre_test()
// // timeout(time: 55, unit: 'MINUTES'){
// sh '''
// date
// cd ${WKC}/tests
// ./test-all.sh b7fq
// date'''
// }
}
}
}
}
// // sh '''
// // date
// // cd ${WKC}/tests
// // ./test-all.sh b7fq
// // date'''
// // }
// }
// }
// }
// }
}
post {
success {

2
cmake/lucene_CMakeLists.txt.in
View File

@ -2,7 +2,7 @@
# lucene
ExternalProject_Add(lucene
GIT_REPOSITORY https://github.com/taosdata-contrib/LucenePlusPlus.git
GIT_TAG rel_3.0.8
GIT_TAG rel_3.0.8_td
SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lucene"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE

4
deps/test/CMakeLists.txt vendored
View File

@ -2,3 +2,7 @@
if(${BUILD_WITH_ROCKSDB})
add_subdirectory(rocksdb)
endif(${BUILD_WITH_ROCKSDB})
if(${BUILD_WITH_LUCENE})
add_subdirectory(lucene)
endif(${BUILD_WITH_LUCENE})

6
deps/test/lucene/CMakeLists.txt vendored Normal file
View File

@ -0,0 +1,6 @@
add_executable(luceneTest "")
target_sources(luceneTest
PRIVATE
"${CMAKE_CURRENT_SOURCE_DIR}/main.cpp"
)
target_link_libraries(luceneTest lucene++)

6
deps/test/lucene/main.cpp vendored Normal file
View File

@ -0,0 +1,6 @@
#include <iostream>
int main(int argc, char const *argv[]) {
std::cout << "Hello, this is lucene test" << std::endl;
return 0;
}

236
docs/scratch.md Normal file
View File

@ -0,0 +1,236 @@
```plantuml
@startuml create_table
skinparam sequenceMessageAlign center
skinparam responseMessageBelowArrow true
participant APP as app
box "dnode1"
participant RPC as rpc
participant VNODE as vnode
participant SYNC as sync
end box
box "dnode2"
participant SYNC as sync2
participant VNODE as vnode2
end box
box "dnode3"
participant SYNC as sync3
participant VNODE as vnode3
end box
' APP send request to dnode and RPC in dnode recv the request
app ->rpc: create table req
' RPC call vnodeProcessReq() function to process the request
rpc -> vnode: vnodeProcessReq
note right
callback function
run in RPC module
threads. The function
only puts the request
to a vnode queue.
end note
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
vnode -> vnode: vnodeProcessReqs()
note right
integrate reqs and
process as a whole
end note
' sync the request to other nodes
vnode -> sync: syncProcessReqs()
' make request persistent
' sync -->vnode: walWrite()\n(callback function)
' replicate requests to other DNODES
sync -> sync2: replication req
sync -> sync3: replication req
sync2 -> vnode2: walWrite()\n(callback function)
sync2 --> sync: replication rsp\n(confirm)
sync3 -> vnode3: walWrite()\n(callback function)
sync3 --> sync: replication rsp\n(confirm)
' send apply request
sync -> sync2: apply req
sync -> sync3: apply req
' vnode apply
sync2 -> vnode2: vnodeApplyReqs()
sync3 -> vnode3: vnodeApplyReqs()
' call apply request
sync --> vnode: vnodeApplyReqs()\n(callback function)
' send response
vnode --> rpc: rpcSendRsp()
' dnode send response to APP
rpc --> app: create table rsp
@enduml
```
## Leader处理强一致写入请求
```plantuml
@startuml leader_process_stict_consistency
box "dnode1"
participant CRPC as crpc
participant VNODE as vnode
participant SYNC as sync
end box
-> crpc: create table/submit req
' In CRPC threads
group #pink "In CRPC threads"
crpc -> vnode:vnodeProcessReq()
note right
A callback function
run by CRPC thread
to put the request
to a vnode queue
end note
end
' In VNODE worker threads
group #lightblue "In VNODE worker threads"
vnode -> vnode: vnodeProcessReqs()
note right
VNODE process requests
accumulated in a
vnode write queue and
process the batch reqs
as a whole
end note
vnode -> sync: syncProcessReqs()
sync -> : replication req1
sync -> : replication req2
end
group #red "SYNC threads"
sync <- : replication rsp1
sync <- : replication rsp2
sync -> vnode: notify apply
sync -> : apply rsp1
sync -> : apply rsp2
end
group #lightblue "In VNODE worker threads"
vnode -> vnode: vnodeApplyReqs()
vnode -> crpc:
end
<- crpc: create table/submit rsp
@enduml
```
## Follower处理强一致写入请求
```plantuml
@startuml follower_process_strict_consistency
participant SYNC as sync
participant VNODE as vnode
group #pink "SYNC threads"
-> sync: replication req
sync -> sync: syncProcessReqs()
note right
In the replication
only data is
persisted and response
is sent back
end note
<- sync: replication rsp
-> sync: apply req
sync -> vnode: notify apply
end
group #lightblue "VNODE worker threads"
vnode -> vnode: vnodeApplyReqs()
end
@enduml
```
## Leader处理最终一致写入请求
```plantuml
@startuml leader_process_eventual_consistency
box "dnode1"
participant CRPC as crpc
participant VNODE as vnode
participant SYNC as sync
end box
-> crpc: create table/submit req
' In CRPC threads
group #pink "In CRPC threads"
crpc -> vnode:vnodeProcessReq()
note right
A callback function
run by CRPC thread
to put the request
to a vnode queue
end note
end
' In VNODE worker threads
group #lightblue "In VNODE worker threads"
vnode -> vnode: vnodeProcessReqs()
note right
VNODE process requests
accumulated in a
vnode write queue and
process the batch reqs
as a whole
end note
vnode -> sync: syncProcessReqs()
sync -> : replication req1
sync -> : replication req2
sync -> vnode: notify apply
end
group #lightblue "In VNODE worker threads"
vnode -> vnode: vnodeApplyReqs()
vnode -> crpc:
end
<- crpc: create table/submit rsp
@enduml
```
## Follower处理最终一致写入请求
```plantuml
@startuml follower_process_eventual_consistency
participant SYNC as sync
participant VNODE as vnode
group #pink "SYNC threads"
-> sync: replication rsp
sync -> sync: syncProcessReqs()
sync -> vnode: notify VNODE \nthread to process\n the reqs
end
group #lightblue "VNODE worker threads"
vnode -> vnode: vnodeApplyReqs()
end
@enduml
```

311
docs/vnode_write.md Normal file
View File

@ -0,0 +1,311 @@
<center><h1>VNODE Write Processes</h1></center>
## META Operations
META data write operations including:
1. create table
2. drop table
3. alter table
We take create table as an example to figure out the whole process.
```plantuml
@startuml create_table
skinparam sequenceMessageAlign center
skinparam responseMessageBelowArrow true
participant APP as app
box "dnode1"
participant RPC as rpc
participant VNODE as vnode
participant SYNC as sync
end box
box "dnode2"
participant SYNC as sync2
participant VNODE as vnode2
end box
box "dnode3"
participant SYNC as sync3
participant VNODE as vnode3
end box
' APP send request to dnode and RPC in dnode recv the request
app ->rpc: create table req
' RPC call vnodeProcessReq() function to process the request
rpc -> vnode: vnodeProcessReq
note right
callback function
run in RPC module
threads. The function
only puts the request
to a vnode queue.
end note
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
vnode -> vnode: vnodeProcessReqs()
note right
integrate reqs and
process as a whole
end note
' sync the request to other nodes
vnode -> sync: syncProcessReqs()
' make request persistent
' sync -->vnode: walWrite()\n(callback function)
' replicate requests to other DNODES
sync -> sync2: replication req
sync -> sync3: replication req
sync2 -> vnode2: walWrite()\n(callback function)
sync2 --> sync: replication rsp\n(confirm)
sync3 -> vnode3: walWrite()\n(callback function)
sync3 --> sync: replication rsp\n(confirm)
' send apply request
sync -> sync2: apply req
sync -> sync3: apply req
' vnode apply
sync2 -> vnode2: vnodeApplyReqs()
sync3 -> vnode3: vnodeApplyReqs()
' call apply request
sync --> vnode: vnodeApplyReqs()\n(callback function)
' send response
vnode --> rpc: rpcSendRsp()
' dnode send response to APP
rpc --> app: create table rsp
@enduml
```
## Time-series data Operations
There are only one operations for time-series data: data insert. We will figure out the whole process.
```plantuml
@startuml create_table
skinparam sequenceMessageAlign center
skinparam responseMessageBelowArrow true
participant APP as app
box "dnode1"
participant RPC as rpc
participant VNODE as vnode
participant SYNC as sync
end box
box "dnode2"
participant SYNC as sync2
participant VNODE as vnode2
end box
box "dnode3"
participant SYNC as sync3
participant VNODE as vnode3
end box
' APP send request to dnode and RPC in dnode recv the request
app ->rpc: insert data req
' RPC call vnodeProcessReq() function to process the request
rpc -> vnode: vnodeProcessReq
note right
callback function
run in RPC module
threads. The function
only puts the request
to a vnode queue.
end note
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
vnode -> vnode: vnodeProcessReqs()
note right
integrate reqs and
process as a whole
end note
' sync the request to other nodes
vnode -> sync: syncProcessReqs()
' ' make request persistent
' ' sync -->vnode: walWrite()\n(callback function)
' ' replicate requests to other DNODES
sync -> sync2: replication req
sync -> sync3: replication req
' vnode apply
sync2 -> vnode2: vnodeApplyReqs()
sync3 -> vnode3: vnodeApplyReqs()
' call apply request
sync --> vnode: vnodeApplyReqs()\n(callback function)
' send response
vnode --> rpc: rpcSendRsp()
' dnode send response to APP
rpc --> app: insert data rsp
@enduml
```
## vnodeProcessReqs()
```plantuml
@startuml vnodeProcessReqs()
participant VNODE as v
participant SYNC as s
group vnodeProcessReqs()
' Group requests and get a request batch to process as a whole
v -> v: vnodeGetReqsFromQueue()
note right
integrate all write
requests as a batch
to process as a whole
end note
' VNODE call syncProcessReqs() function to process the batch request
v -> s: syncProcessReqs()
group syncProcessReqs()
' Check if current node is leader
alt not leader
return NOT_LEADER
end
s -> s: syncAppendReqsToLogStore()
group syncAppendReqsToLogStore()
s -> v: walWrite()
note right
There must be a
callback function
provided by VNODE
to persist the
requests in WAL
end note
alt (no unapplied reqs) AND (only one node OR no meta requests)
s -> v: vnodeApplyReqs()
note right
just use the woker
thread to apply
the requests. This
is a callback function
provided by VNODE
end note
else other cases need to wait response
s -> s:
note right
save the requests in log store
and wait for comfirmation or
other cases
end note
s ->]: send replication requests
s ->]: send replication requests
end
end
end
end
@enduml
```
<!-- ## syncProcessReplicationReq()
```plantuml
@startuml syncProcessReplicationReq
participant SYNC as s
participant VNODE as v
-> s: replication request
s -> s:
note right
process the request
to get the request
batch
end note
s -> s: syncAppendReqToLogStore()
s -> v: walWrite()
alt has meta req
<- s: comfirmation
else
s -> v: vnodeApplyReqs()
end
@enduml -->
<!-- ``` -->
## vnodeApplyReqs()
The function *vnodeApplyReqs()* is the actual function running by a vnode to process the requests.
```plantuml
@startuml vnodeApplyReqs()
skinparam sequenceMessageAlign left
skinparam responseMessageBelowArrow true
participant VNODE as vnode
participant TQ as tq
participant TSDB as tsdb
participant META as meta
group vnodeApplyReqs()
autonumber
loop nReqs
' Copy request message to vnode buffer pool
vnode -> vnode: vnodeCopyReq()
note right
copy request to
vnode buffer pool
end note
vnode -> tq: tqPush()
note right
push the request
to TQ so consumers
can consume
end note
alt META_REQ
autonumber 3
vnode -> meta: metaApplyReq()
else TS_REQ
autonumber 3
vnode -> tsdb: tsdbApplyReq()
end
end
' Check if need to commit
alt vnode buffer pool is full
group vnodeCommit()
autonumber 4.1
vnode -> tq: tqCommit()
note right
tqCommit may renew wal
end note
vnode -> meta: metaCommit();
note right
commit meta data
end note
vnode -> tsdb: tsdbCommit();
note right
commit time-series data
end note
end
end
end
@enduml
```
<!-- meta操作：建表，删表，改表（队队列/同步）
数据写入
快照文件与sync的结合
vnodeOpen（）
vnodeClose（）
sync.h -->

8
include/client/taos.h
View File

@ -46,6 +46,11 @@ typedef void **TAOS_ROW;
#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes
#define TSDB_DATA_TYPE_UINT 13 // 4 bytes
#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes
#define TSDB_DATA_TYPE_VARCHAR 15 // string
#define TSDB_DATA_TYPE_JSON 16 // json
#define TSDB_DATA_TYPE_DECIMAL 17 // decimal
#define TSDB_DATA_TYPE_BLOB 18 // binary string
#define TSDB_DATA_TYPE_LONGBLOB 19 // long binary string
typedef enum {
TSDB_OPTION_LOCALE,
@ -68,7 +73,7 @@ typedef struct taosField {
#define DLL_EXPORT
#endif
DLL_EXPORT int taos_init();
DLL_EXPORT int taos_init();
DLL_EXPORT void taos_cleanup(void);
DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...);
DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port);
@ -157,7 +162,6 @@ DLL_EXPORT int taos_errno(TAOS_RES *tres);
DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, int code), void *param);
DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, int numOfRows), void *param);
//DLL_EXPORT void taos_fetch_row_a(TAOS_RES *res, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), void *param);
typedef void (*TAOS_SUBSCRIBE_CALLBACK)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code);
DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, TAOS_SUBSCRIBE_CALLBACK fp, void *param, int interval);

30
include/common/common.h
View File

@ -17,6 +17,8 @@
#define TDENGINE_COMMON_H
#include "taosdef.h"
#include "taosmsg.h"
#include "tarray.h"
//typedef struct STimeWindow {
// TSKEY skey;
@ -36,4 +38,32 @@
// int16_t bytes;
//} SSchema;
typedef struct SColumnDataAgg {
int16_t colId;
int64_t sum;
int64_t max;
int64_t min;
int16_t maxIndex;
int16_t minIndex;
int16_t numOfNull;
} SColumnDataAgg;
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rows;
int32_t numOfCols;
int64_t uid;
} SDataBlockInfo;
typedef struct SSDataBlock {
SColumnDataAgg *pBlockAgg;
SArray *pDataBlock; // SArray<SColumnInfoData>
SDataBlockInfo info;
} SSDataBlock;
typedef struct SColumnInfoData {
SColumnInfo info; // TODO filter info needs to be removed
char *pData; // the corresponding block data in memory
} SColumnInfoData;
#endif // TDENGINE_COMMON_H

1
include/common/taosmsg.h
View File

@ -456,7 +456,6 @@ typedef struct SColumnInfo {
typedef struct STableIdInfo {
uint64_t uid;
int32_t tid;
TSKEY key; // last accessed ts, for subscription
} STableIdInfo;

2
include/common/tvariant.h
View File

@ -36,6 +36,8 @@ typedef struct SVariant {
};
} SVariant;
int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* issigned);
bool taosVariantIsValid(SVariant *pVar);
void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type);

3
include/libs/catalog/catalog.h
View File

@ -33,7 +33,6 @@ struct SCatalog;
typedef struct SMetaReq {
char clusterId[TSDB_CLUSTER_ID_LEN];
SArray *pTableName; // table full name
SArray *pVgroup; // vgroup id
SArray *pUdf; // udf name
bool qNodeEpset; // valid qnode
} SMetaReq;
@ -60,7 +59,6 @@ typedef struct STableComInfo {
typedef struct SCTableMeta {
int32_t vgId:24;
int8_t tableType;
uint32_t tid;
uint64_t uid;
uint64_t suid;
} SCTableMeta;
@ -71,7 +69,6 @@ typedef struct SCTableMeta {
typedef struct STableMeta {
int32_t vgId:24;
int8_t tableType;
uint32_t tid;
uint64_t uid;
uint64_t suid;
// if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info

206
include/libs/function/function.h Normal file
View File

@ -0,0 +1,206 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TFUNCTION_H
#define TDENGINE_TFUNCTION_H
#ifdef __cplusplus
extern "C" {
#endif
#include "common.h"
#include "tvariant.h"
#include "tbuffer.h"
#define FUNCTION_SCALAR 1
#define FUNCTION_AGG 2
#define TOP_BOTTOM_QUERY_LIMIT 100
#define FUNCTIONS_NAME_MAX_LENGTH 16
#define FUNCTION_INVALID_ID -1
#define FUNCTION_COUNT 0
#define FUNCTION_SUM 1
#define FUNCTION_AVG 2
#define FUNCTION_MIN 3
#define FUNCTION_MAX 4
#define FUNCTION_STDDEV 5
#define FUNCTION_PERCT 6
#define FUNCTION_APERCT 7
#define FUNCTION_FIRST 8
#define FUNCTION_LAST 9
#define FUNCTION_LAST_ROW 10
#define FUNCTION_TOP 11
#define FUNCTION_BOTTOM 12
#define FUNCTION_SPREAD 13
#define FUNCTION_TWA 14
#define FUNCTION_LEASTSQR 15
#define FUNCTION_TS 16
#define FUNCTION_TS_DUMMY 17
#define FUNCTION_TAG_DUMMY 18
#define FUNCTION_TS_COMP 19
#define FUNCTION_TAG 20
#define FUNCTION_PRJ 21
#define FUNCTION_TAGPRJ 22
#define FUNCTION_ARITHM 23
#define FUNCTION_DIFF 24
#define FUNCTION_FIRST_DST 25
#define FUNCTION_LAST_DST 26
#define FUNCTION_STDDEV_DST 27
#define FUNCTION_INTERP 28
#define FUNCTION_RATE 29
#define FUNCTION_IRATE 30
#define FUNCTION_TID_TAG 31
#define FUNCTION_DERIVATIVE 32
#define FUNCTION_BLKINFO 33
#define FUNCTION_HISTOGRAM 34
#define FUNCTION_HLL 35
#define FUNCTION_MODE 36
#define FUNCTION_SAMPLE 37
typedef struct SPoint1 {
int64_t key;
union{double val; char* ptr;};
} SPoint1;
struct SQLFunctionCtx;
struct SResultRowCellInfo;
//for selectivity query, the corresponding tag value is assigned if the data is qualified
typedef struct SExtTagsInfo {
int16_t tagsLen; // keep the tags data for top/bottom query result
int16_t numOfTagCols;
struct SQLFunctionCtx **pTagCtxList;
} SExtTagsInfo;
// sql function runtime context
typedef struct SQLFunctionCtx {
int32_t size; // number of rows
void * pInput; // input data buffer
uint32_t order; // asc|desc
int16_t inputType;
int16_t inputBytes;
int16_t outputType;
int16_t outputBytes; // size of results, determined by function and input column data type
int32_t interBufBytes; // internal buffer size
bool hasNull; // null value exist in current block
bool requireNull; // require null in some function
bool stableQuery;
int16_t functionId; // function id
char * pOutput; // final result output buffer, point to sdata->data
uint8_t currentStage; // record current running step, default: 0
int64_t startTs; // timestamp range of current query when function is executed on a specific data block
int32_t numOfParams;
SVariant param[4]; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param
int64_t *ptsList; // corresponding timestamp array list
void *ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/
SVariant tag;
bool isSmaSet;
SColumnDataAgg sma;
struct SResultRowCellInfo *resultInfo;
SExtTagsInfo tagInfo;
SPoint1 start;
SPoint1 end;
} SQLFunctionCtx;
enum {
TEXPR_NODE_DUMMY = 0x0,
TEXPR_BINARYEXPR_NODE= 0x1,
TEXPR_UNARYEXPR_NODE = 0x2,
TEXPR_COL_NODE = 0x4,
TEXPR_VALUE_NODE = 0x8,
};
typedef struct tExprNode {
uint8_t nodeType;
union {
struct {
union {
int32_t optr; // binary operator
int32_t functionId;// unary operator
};
void *info; // support filter operation on this expression only available for leaf node
struct tExprNode *pLeft; // left child pointer
struct tExprNode *pRight; // right child pointer
} _node;
SSchema *pSchema;// column node
struct SVariant *pVal; // value node
};
} tExprNode;
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
typedef struct SAggFunctionInfo {
char name[FUNCTIONS_NAME_MAX_LENGTH];
int8_t type; // Scalar function or aggregation function
uint8_t functionId; // Function Id
int8_t sFunctionId; // Transfer function for super table query
uint16_t status;
bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo); // setup the execute environment
void (*exec)(SQLFunctionCtx *pCtx);
// finalizer must be called after all exec has been executed to generated final result.
void (*xFinalize)(SQLFunctionCtx *pCtx);
void (*mergeFunc)(SQLFunctionCtx *pCtx);
int32_t (*dataReqFunc)(SQLFunctionCtx *pCtx, STimeWindow* w, int32_t colId);
} SAggFunctionInfo;
typedef struct SScalarFunctionInfo {
char name[FUNCTIONS_NAME_MAX_LENGTH];
int8_t type; // scalar function or aggregation function
uint8_t functionId; // index of scalar function
bool (*init)(SQLFunctionCtx *pCtx, struct SResultRowCellInfo* pResultCellInfo); // setup the execute environment
void (*exec)(SQLFunctionCtx *pCtx);
} SScalarFunctionInfo;
typedef struct SResultDataInfo {
int16_t type;
int16_t bytes;
int32_t intermediateBytes;
} SResultDataInfo;
int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, SResultDataInfo* pInfo, int16_t extLength,
bool isSuperTable);
/**
* If the given name is a valid built-in sql function, the value of true will be returned.
* @param name
* @param len
* @return
*/
int32_t qIsBuiltinFunction(const char* name, int32_t len);
bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId);
const char* qGetFunctionName(int32_t functionId);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TFUNCTION_H

69
include/libs/parser/parser.h
View File

@ -23,6 +23,29 @@ extern "C" {
#include "catalog.h"
#include "common.h"
#include "tname.h"
#include "tvariant.h"
typedef struct SColumn {
uint64_t tableUid;
int32_t columnIndex;
SColumnInfo info;
} SColumn;
// the structure for sql function in select clause
typedef struct SSqlExpr {
char token[TSDB_COL_NAME_LEN]; // original token
SSchema resSchema;
SColIndex colInfo;
uint64_t uid; // table uid, todo refactor use the pointer
int32_t interBytes; // inter result buffer size
int16_t numOfParams; // argument value of each function
SVariant param[3]; // parameters are not more than 3
} SSqlExpr;
typedef struct SExprInfo {
SSqlExpr base;
struct tExprNode *pExpr;
} SExprInfo;
//typedef struct SInterval {
// int32_t tz; // query client timezone
@ -95,10 +118,7 @@ typedef struct STagCond {
typedef struct STableMetaInfo {
STableMeta *pTableMeta; // table meta, cached in client side and acquired by name
uint32_t tableMetaSize;
size_t tableMetaCapacity;
SVgroupsInfo *vgroupList;
SArray *pVgroupTables; // SArray<SVgroupTableInfo>
/*
* 1. keep the vgroup index during the multi-vnode super table projection query
@ -110,6 +130,20 @@ typedef struct STableMetaInfo {
SArray *tagColList; // SArray<SColumn*>, involved tag columns
} STableMetaInfo;
typedef struct SQueryAttrInfo {
bool stableQuery;
bool groupbyColumn;
bool simpleAgg;
bool arithmeticOnAgg;
bool projectionQuery;
bool hasFilter;
bool onlyTagQuery;
bool orderProjectQuery;
bool stateWindow;
bool globalMerge;
bool multigroupResult;
} SQueryAttrInfo;
typedef struct SQueryStmtInfo {
int16_t command; // the command may be different for each subclause, so keep it seperately.
uint32_t type; // query/insert type
@ -152,19 +186,15 @@ typedef struct SQueryStmtInfo {
SArray *pUpstream; // SArray<struct SQueryStmtInfo>
struct SQueryStmtInfo *pDownstream;
int32_t havingFieldNum;
bool stableQuery;
bool groupbyColumn;
bool simpleAgg;
bool arithmeticOnAgg;
bool projectionQuery;
bool hasFilter;
bool onlyTagQuery;
bool orderProjectQuery;
bool stateWindow;
bool globalMerge;
bool multigroupResult;
SQueryAttrInfo info;
} SQueryStmtInfo;
typedef struct SColumnIndex {
int16_t tableIndex;
int16_t columnIndex;
int16_t type; // normal column/tag/ user input constant column
} SColumnIndex;
struct SInsertStmtInfo;
/**
@ -206,6 +236,17 @@ int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo*
*/
int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql);
void assignExprInfo(SExprInfo* dst, const SExprInfo* src);
void columnListCopy(SArray* dst, const SArray* src, uint64_t uid);
void columnListDestroy(SArray* pColumnList);
void dropAllExprInfo(SArray* pExprInfo);
SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, struct tExprNode* pParamExpr, SSchema* pResSchema, int16_t interSize);
int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy);
STableMetaInfo* getMetaInfo(SQueryStmtInfo* pQueryInfo, int32_t tableIndex);
int32_t getNewResColId();
#ifdef __cplusplus
}
#endif

24
include/libs/planner/planner.h
View File

@ -24,8 +24,8 @@ extern "C" {
#define QUERY_TYPE_PARTIAL 2
struct SEpSet;
struct SQueryNode;
struct SQueryPhyNode;
struct SQueryPlanNode;
struct SQueryDistPlanNode;
struct SQueryStmtInfo;
typedef struct SSubquery {
@ -33,7 +33,7 @@ typedef struct SSubquery {
int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL
int32_t level; // the execution level of current subquery, starting from 0.
SArray *pUpstream; // the upstream,from which to fetch the result
struct SQueryPhyNode *pNode; // physical plan of current subquery
struct SQueryDistPlanNode *pNode; // physical plan of current subquery
} SSubquery;
typedef struct SQueryJob {
@ -48,7 +48,7 @@ typedef struct SQueryJob {
* @param pQueryNode
* @return
*/
int32_t qOptimizeQueryPlan(struct SQueryNode* pQueryNode);
int32_t qOptimizeQueryPlan(struct SQueryPlanNode* pQueryNode);
/**
* Create the query plan according to the bound AST, which is in the form of pQueryInfo
@ -56,14 +56,14 @@ int32_t qOptimizeQueryPlan(struct SQueryNode* pQueryNode);
* @param pQueryNode
* @return
*/
int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryNode* pQueryNode);
int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode* pQueryNode);
/**
* Convert the query plan to string, in order to display it in the shell.
* @param pQueryNode
* @return
*/
int32_t qQueryPlanToString(struct SQueryNode* pQueryNode, char** str);
int32_t qQueryPlanToString(struct SQueryPlanNode* pQueryNode, char** str);
/**
* Restore the SQL statement according to the logic query plan.
@ -71,7 +71,7 @@ int32_t qQueryPlanToString(struct SQueryNode* pQueryNode, char** str);
* @param sql
* @return
*/
int32_t qQueryPlanToSql(struct SQueryNode* pQueryNode, char** sql);
int32_t qQueryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql);
/**
* Create the physical plan for the query, according to the logic plan.
@ -79,7 +79,7 @@ int32_t qQueryPlanToSql(struct SQueryNode* pQueryNode, char** sql);
* @param pPhyNode
* @return
*/
int32_t qCreatePhysicalPlan(struct SQueryNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyNode *pPhyNode);
int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDistPlanNode *pPhyNode);
/**
* Convert to physical plan to string to enable to print it out in the shell.
@ -87,20 +87,20 @@ int32_t qCreatePhysicalPlan(struct SQueryNode* pQueryNode, struct SEpSet* pQnode
* @param str
* @return
*/
int32_t qPhyPlanToString(struct SQueryPhyNode *pPhyNode, char** str);
int32_t qPhyPlanToString(struct SQueryDistPlanNode *pPhyNode, char** str);
/**
* Destroy the query plan object.
* @return
*/
void* qDestroyQueryPlan(struct SQueryNode* pQueryNode);
void* qDestroyQueryPlan(struct SQueryPlanNode* pQueryNode);
/**
* Destroy the physical plan.
* @param pQueryPhyNode
* @return
*/
void* qDestroyQueryPhyPlan(struct SQueryPhyNode* pQueryPhyNode);
void* qDestroyQueryPhyPlan(struct SQueryDistPlanNode* pQueryPhyNode);
/**
* Create the query job from the physical execution plan
@ -108,7 +108,7 @@ void* qDestroyQueryPhyPlan(struct SQueryPhyNode* pQueryPhyNode);
* @param pJob
* @return
*/
int32_t qCreateQueryJob(const struct SQueryPhyNode* pPhyNode, struct SQueryJob** pJob);
int32_t qCreateQueryJob(const struct SQueryDistPlanNode* pPhyNode, struct SQueryJob** pJob);
#ifdef __cplusplus
}

16
include/libs/sync/sync.h
View File

@ -62,24 +62,24 @@ typedef struct SSyncFSM {
void* pData;
// apply committed log, bufs will be free by raft module
int (*applyLog)(struct SSyncFSM *fsm, SyncIndex index, const SSyncBuffer *buf, void *pData);
int (*applyLog)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf, void* pData);
// cluster commit callback
int (*onClusterChanged)(struct SSyncFSM *fsm, const SSyncCluster* cluster, void *pData);
// cluster commit callback
int (*onClusterChanged)(struct SSyncFSM* fsm, const SSyncCluster* cluster, void* pData);
// fsm return snapshot in ppBuf, bufs will be free by raft module
// TODO: getSnapshot SHOULD be async?
int (*getSnapshot)(struct SSyncFSM *fsm, SSyncBuffer **ppBuf, int* objId, bool *isLast);
int (*getSnapshot)(struct SSyncFSM* fsm, SSyncBuffer** ppBuf, int* objId, bool* isLast);
// fsm apply snapshot with pBuf data
int (*applySnapshot)(struct SSyncFSM *fsm, SSyncBuffer *pBuf, int objId, bool isLast);
int (*applySnapshot)(struct SSyncFSM* fsm, SSyncBuffer* pBuf, int objId, bool isLast);
// call when restore snapshot and log done
int (*onRestoreDone)(struct SSyncFSM *fsm);
int (*onRestoreDone)(struct SSyncFSM* fsm);
void (*onRollback)(struct SSyncFSM *fsm, SyncIndex index, const SSyncBuffer *buf);
void (*onRollback)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf);
void (*onRoleChanged)(struct SSyncFSM *fsm, const SNodesRole* pRole);
void (*onRoleChanged)(struct SSyncFSM* fsm, const SNodesRole* pRole);
} SSyncFSM;

1
include/os/os.h
View File

@ -44,6 +44,7 @@ extern "C" {
#include <errno.h>
#include <float.h>
#include <math.h>
#include <sys/stat.h>
#include "osAtomic.h"
#include "osDef.h"

76
include/util/tdef.h
View File

@ -24,26 +24,20 @@ extern "C" {
#define TSDB__packed
#ifdef TSKEY32
#define TSKEY int32_t;
#else
#define TSKEY int64_t
#endif
#define TSKEY_INITIAL_VAL INT64_MIN
// Bytes for each type.
extern const int32_t TYPE_BYTES[15];
// TODO: replace and remove code below
#define CHAR_BYTES sizeof(char)
#define SHORT_BYTES sizeof(int16_t)
#define INT_BYTES sizeof(int32_t)
#define LONG_BYTES sizeof(int64_t)
#define FLOAT_BYTES sizeof(float)
#define DOUBLE_BYTES sizeof(double)
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
#define CHAR_BYTES sizeof(char)
#define SHORT_BYTES sizeof(int16_t)
#define INT_BYTES sizeof(int32_t)
#define LONG_BYTES sizeof(int64_t)
#define FLOAT_BYTES sizeof(float)
#define DOUBLE_BYTES sizeof(double)
#define POINTER_BYTES sizeof(void *) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*)
#define TSDB_KEYSIZE sizeof(TSKEY)
#define TSDB_NCHAR_SIZE sizeof(int32_t)
@ -88,10 +82,11 @@ extern const int32_t TYPE_BYTES[15];
#define TSDB_ERR -1
#define TS_PATH_DELIMITER "."
#define TS_ESCAPE_CHAR '`'
#define TSDB_TIME_PRECISION_MILLI 0
#define TSDB_TIME_PRECISION_MICRO 1
#define TSDB_TIME_PRECISION_NANO 2
#define TSDB_TIME_PRECISION_MILLI 0
#define TSDB_TIME_PRECISION_MICRO 1
#define TSDB_TIME_PRECISION_NANO 2
#define TSDB_TIME_PRECISION_MILLI_STR "ms"
#define TSDB_TIME_PRECISION_MICRO_STR "us"
@ -132,11 +127,12 @@ do { \
#define TSDB_RELATION_MATCH 14
#define TSDB_RELATION_NMATCH 15
#define TSDB_BINARY_OP_ADD 30
#define TSDB_BINARY_OP_SUBTRACT 31
#define TSDB_BINARY_OP_MULTIPLY 32
#define TSDB_BINARY_OP_DIVIDE 33
#define TSDB_BINARY_OP_REMAINDER 34
#define TSDB_BINARY_OP_ADD 4000
#define TSDB_BINARY_OP_SUBTRACT 4001
#define TSDB_BINARY_OP_MULTIPLY 4002
#define TSDB_BINARY_OP_DIVIDE 4003
#define TSDB_BINARY_OP_REMAINDER 4004
#define TSDB_BINARY_OP_CONCAT 4005
#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
@ -386,44 +382,6 @@ do { \
#define TSDB_DATA_TYPE_UINT 13 // 4 bytes
#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes
// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
//typedef int32_t VarDataOffsetT;
//typedef int16_t VarDataLenT; // maxVarDataLen: 32767
//typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535
//typedef uint32_t TDRowTLenT; // total length, including overhead
//
//typedef struct tstr {
// VarDataLenT len;
// char data[];
//} tstr;
//
//#pragma pack(push, 1)
//typedef struct {
// VarDataLenT len;
// uint8_t data;
//} SBinaryNullT;
//
//typedef struct {
// VarDataLenT len;
// uint32_t data;
//} SNCharNullT;
//#pragma pack(pop)
//
//#define VARSTR_HEADER_SIZE sizeof(VarDataLenT)
//
//#define varDataLen(v) ((VarDataLenT *)(v))[0]
//#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v))
//#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
//#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v))
//#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE))
//#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len))
//#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR))
//
//#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0]))
//#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v))
#ifdef __cplusplus
}
#endif

169
include/util/tpagedfile.h Normal file
View File

@ -0,0 +1,169 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TPAGEDFILE_H
#define TDENGINE_TPAGEDFILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tlist.h"
#include "thash.h"
#include "os.h"
#include "tlockfree.h"
typedef struct SArray* SIDList;
typedef struct SPageDiskInfo {
int32_t offset;
int32_t length;
} SPageDiskInfo;
typedef struct SPageInfo {
SListNode* pn; // point to list node
int32_t pageId;
SPageDiskInfo info;
void* pData;
bool used; // set current page is in used
} SPageInfo;
typedef struct SFreeListItem {
int32_t offset;
int32_t len;
} SFreeListItem;
typedef struct SResultBufStatis {
int32_t flushBytes;
int32_t loadBytes;
int32_t getPages;
int32_t releasePages;
int32_t flushPages;
} SResultBufStatis;
typedef struct SDiskbasedResultBuf {
int32_t numOfPages;
int64_t totalBufSize;
int64_t fileSize; // disk file size
FILE* file;
int32_t allocateId; // allocated page id
char* path; // file path
int32_t pageSize; // current used page size
int32_t inMemPages; // numOfPages that are allocated in memory
SHashObj* groupSet; // id hash table
SHashObj* all;
SList* lruList;
void* emptyDummyIdList; // dummy id list
void* assistBuf; // assistant buffer for compress/decompress data
SArray* pFree; // free area in file
bool comp; // compressed before flushed to disk
int32_t nextPos; // next page flush position
uint64_t qId; // for debug purpose
SResultBufStatis statis;
} SDiskbasedResultBuf;
#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes
#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1}
#define DEFAULT_PAGE_SIZE (16384L)
typedef struct SFilePage {
int64_t num;
char data[];
} SFilePage;
/**
* create disk-based result buffer
* @param pResultBuf
* @param rowSize
* @param pagesize
* @param inMemPages
* @param handle
* @return
*/
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir);
/**
*
* @param pResultBuf
* @param groupId
* @param pageId
* @return
*/
SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId);
/**
*
* @param pResultBuf
* @param groupId
* @return
*/
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId);
/**
* get the specified buffer page by id
* @param pResultBuf
* @param id
* @return
*/
SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id);
/**
* release the referenced buf pages
* @param pResultBuf
* @param page
*/
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page);
/**
*
* @param pResultBuf
* @param pi
*/
void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi);
/**
* get the total buffer size in the format of disk file
* @param pResultBuf
* @return
*/
size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf);
/**
* get the number of groups in the result buffer
* @param pResultBuf
* @return
*/
size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf);
/**
* destroy result buffer
* @param pResultBuf
*/
void destroyResultBuf(SDiskbasedResultBuf* pResultBuf);
/**
*
* @param pList
* @return
*/
SPageInfo* getLastPageInfo(SIDList pList);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TPAGEDFILE_H

2
include/util/tskiplist.h
View File

@ -21,7 +21,7 @@ extern "C" {
#endif
#include "os.h"
#include "tdef.h"
//#include "tdef.h"
#include "tarray.h"
#include "tfunctional.h"

4
include/util/tutil.h
View File

@ -26,6 +26,7 @@ extern "C" {
#include "tdef.h"
int32_t strdequote(char *src);
int32_t strndequote(char *dst, const char* z, int32_t len);
int32_t strRmquote(char *z, int32_t len);
size_t strtrim(char *src);
char * strnchr(char *haystack, char needle, int32_t len, bool skipquote);
@ -40,9 +41,6 @@ char * paGetToken(char *src, char **token, int32_t *tokenLen);
int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]);
int32_t taosHexStrToByteArray(char hexstr[], char bytes[]);
//bool taosGetVersionNumber(char *versionStr, int *versionNubmer);
//int taosCheckVersion(char *input_client_version, char *input_server_version, int compared_segments);
char * taosIpStr(uint32_t ipInt);
uint32_t ip2uint(const char *const ip_addr);

4
source/common/CMakeLists.txt
View File

@ -10,4 +10,6 @@ target_link_libraries(
PUBLIC os
PUBLIC util
INTERFACE api
)
)
ADD_SUBDIRECTORY(test)

0
source/common/src/sqlcmdstr.c → source/common/src/tmsgtype.c
View File

23
source/common/src/tname.c
View File

@ -120,29 +120,6 @@ int64_t taosGetIntervalStartTimestamp(int64_t startTime, int64_t slidingTime, in
#endif
/*
* tablePrefix.columnName
* extract table name and save it in pTable, with only column name in pToken
*/
//void extractTableNameFromToken(SStrToken* pToken, SStrToken* pTable) {
// const char sep = TS_PATH_DELIMITER[0];
//
// if (pToken == pTable || pToken == NULL || pTable == NULL) {
// return;
// }
//
// char* r = strnchr(pToken->z, sep, pToken->n, false);
//
// if (r != NULL) { // record the table name token
// pTable->n = (uint32_t)(r - pToken->z);
// pTable->z = pToken->z;
//
// r += 1;
// pToken->n -= (uint32_t)(r - pToken->z);
// pToken->z = r;
// }
//}
static struct SSchema _s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,

8
source/common/src/ttime.c
View File

@ -416,12 +416,14 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time
return -1;
}
}
/* get the value in microsecond */
return 0;
}
/*
* n - months
* y - Years
* is not allowed, since the duration of month or year are both variable.
*
* b - nanoseconds;
* u - microseconds;
* a - Millionseconds
@ -430,8 +432,6 @@ static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t time
* h - Hours
* d - Days (24 hours)
* w - Weeks (7 days)
* n - Months (30 days)
* y - Years (365 days)
*/
int32_t parseAbsoluteDuration(char* token, int32_t tokenlen, int64_t* duration, char* unit, int32_t timePrecision) {
errno = 0;

99
source/common/src/tvariant.c
View File

@ -15,20 +15,65 @@
#include "os.h"
#include "taos.h"
#include "thash.h"
#include "taosdef.h"
#include "thash.h"
#include "ttime.h"
#include "ttokendef.h"
#include "ttypes.h"
#include "tutil.h"
#include "tvariant.h"
#define SET_EXT_INFO(converted, res, minv, maxv, exti) do { \
if (converted == NULL || exti == NULL || *converted == false) { break; } \
if ((res) < (minv)) { *exti = -1; break; } \
if ((res) > (maxv)) { *exti = 1; break; } \
assert(0); \
} while (0)
#define SET_EXT_INFO(converted, res, minv, maxv, exti) \
do { \
if (converted == NULL || exti == NULL || *converted == false) { \
break; \
} \
if ((res) < (minv)) { \
*exti = -1; \
break; \
} \
if ((res) > (maxv)) { \
*exti = 1; \
break; \
} \
assert(0); \
} while (0)
int32_t toInteger(const char* z, int32_t n, int32_t base, int64_t* value, bool* isSigned) {
errno = 0;
char* endPtr = NULL;
int32_t index = 0;
bool specifiedSign = (z[0] == '+' || z[0] == '-');
if (specifiedSign) {
*isSigned = true;
index = 1;
}
uint64_t val = strtoull(&z[index], &endPtr, base);
if (errno == ERANGE || errno == EINVAL) {
errno = 0;
return -1;
}
if (specifiedSign && val > INT64_MAX) {
return -1;
}
if (endPtr - &z[index] != n - index) {
return -1;
}
*isSigned = specifiedSign || (val <= INT64_MAX);
if (*isSigned) {
*value = (z[0] == '-')? -val:val;
} else {
*(uint64_t*) value = val;
}
return 0;
}
void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
int32_t ret = 0;
@ -43,7 +88,6 @@ void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
} else {
return;
}
break;
}
@ -51,38 +95,38 @@ void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
case TSDB_DATA_TYPE_SMALLINT:
case TSDB_DATA_TYPE_BIGINT:
case TSDB_DATA_TYPE_INT:{
// ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, true);
// if (ret != 0) {
// SToken t = {0};
// tGetToken(token->z, &t.type);
// if (t.type == TK_MINUS) { // it is a signed number which is greater than INT64_MAX or less than INT64_MIN
// pVar->nType = -1; // -1 means error type
// return;
// }
//
// // data overflow, try unsigned parse the input number
// ret = tStrToInteger(token->z, token->type, token->n, &pVar->i64, false);
// if (ret != 0) {
// pVar->nType = -1; // -1 means error type
// return;
// }
// }
bool sign = true;
int32_t base = 10;
if (type == TK_HEX) {
base = 16;
} else if (type == TK_OCT) {
base = 8;
} else if (type == TK_BIN) {
base = 2;
}
ret = toInteger(z, n, base, &pVar->i64, &sign);
if (ret != 0) {
pVar->nType = -1; // -1 means error type
return;
}
pVar->nType = (sign)? TSDB_DATA_TYPE_BIGINT:TSDB_DATA_TYPE_UBIGINT;
break;
}
case TSDB_DATA_TYPE_DOUBLE:
case TSDB_DATA_TYPE_FLOAT: {
pVar->d = strtod(z, NULL);
break;
}
case TSDB_DATA_TYPE_BINARY: {
pVar->pz = strndup(z, n);
pVar->nLen = strRmquote(pVar->pz, n);
break;
}
case TSDB_DATA_TYPE_TIMESTAMP: {
assert(0);
pVar->i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
break;
}
@ -95,7 +139,6 @@ void taosVariantCreate(SVariant *pVar, char* z, int32_t n, int32_t type) {
pVar->nType = type;
}
/**
* create SVariant from binary string, not ascii data
* @param pVar
@ -901,7 +944,7 @@ int32_t taosVariantDump(SVariant *pVariant, char *payload, int16_t type, bool in
*
* It is actually the bigint/binary/bool/nchar type transfer
*/
int32_t tVariantTypeSetType(SVariant *pVariant, char type) {
int32_t taosVariantTypeSetType(SVariant *pVariant, char type) {
if (pVariant == NULL || pVariant->nType == 0) { // value is not set
return 0;
}

63
source/common/src/versionUtil.c Normal file
View File

@ -0,0 +1,63 @@
#include "os.h"
#include "tdef.h"
#include "ulog.h"
#include "taoserror.h"
bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
if (versionStr == NULL || versionNubmer == NULL) {
return false;
}
int versionNumberPos[5] = {0};
int len = (int)strlen(versionStr);
int dot = 0;
for (int pos = 0; pos < len && dot < 4; ++pos) {
if (versionStr[pos] == '.') {
versionStr[pos] = 0;
versionNumberPos[++dot] = pos + 1;
}
}
if (dot != 3) {
return false;
}
for (int pos = 0; pos < 4; ++pos) {
versionNubmer[pos] = atoi(versionStr + versionNumberPos[pos]);
}
versionStr[versionNumberPos[1] - 1] = '.';
versionStr[versionNumberPos[2] - 1] = '.';
versionStr[versionNumberPos[3] - 1] = '.';
return true;
}
int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) {
char client_version[TSDB_VERSION_LEN] = {0};
char server_version[TSDB_VERSION_LEN] = {0};
int clientVersionNumber[4] = {0};
int serverVersionNumber[4] = {0};
tstrncpy(client_version, input_client_version, sizeof(client_version));
tstrncpy(server_version, input_server_version, sizeof(server_version));
if (!taosGetVersionNumber(client_version, clientVersionNumber)) {
uError("invalid client version:%s", client_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
if (!taosGetVersionNumber(server_version, serverVersionNumber)) {
uError("invalid server version:%s", server_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
for(int32_t i = 0; i < comparedSegments; ++i) {
if (clientVersionNumber[i] != serverVersionNumber[i]) {
uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version,
client_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
}
return 0;
}

18
source/common/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
MESSAGE(STATUS "build parser unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(commonTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
commonTest
PUBLIC os util common gtest
)
TARGET_INCLUDE_DIRECTORIES(
commonTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/common/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/common/inc"
)

96
source/common/test/commonTests.cpp
View File

@ -0,0 +1,96 @@
#include <gtest/gtest.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "taos.h"
#include "tvariant.h"
#include "tdef.h"
namespace {
//
} // namespace
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(testCase, toInteger_test) {
char* s = "123";
uint32_t type = 0;
int64_t val = 0;
bool sign = true;
int32_t ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 123);
ASSERT_EQ(sign, true);
s = "9223372036854775807";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 9223372036854775807);
ASSERT_EQ(sign, true);
s = "9323372036854775807";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 9323372036854775807u);
ASSERT_EQ(sign, false);
s = "-9323372036854775807";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, -1);
s = "-1";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, -1);
ASSERT_EQ(sign, true);
s = "-9223372036854775807";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, -9223372036854775807);
ASSERT_EQ(sign, true);
s = "1000u";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, -1);
s = "0x10";
ret = toInteger(s, strlen(s), 16, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 16);
ASSERT_EQ(sign, true);
s = "110";
ret = toInteger(s, strlen(s), 2, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 6);
ASSERT_EQ(sign, true);
s = "110";
ret = toInteger(s, strlen(s), 8, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 72);
ASSERT_EQ(sign, true);
//18446744073709551615 UINT64_MAX
s = "18446744073709551615";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, 0);
ASSERT_EQ(val, 18446744073709551615u);
ASSERT_EQ(sign, false);
s = "18446744073709551616";
ret = toInteger(s, strlen(s), 10, &val, &sign);
ASSERT_EQ(ret, -1);
}

3
source/libs/CMakeLists.txt
View File

@ -8,4 +8,5 @@ add_subdirectory(scheduler)
add_subdirectory(lru)
add_subdirectory(catalog)
add_subdirectory(executor)
add_subdirectory(planner)
add_subdirectory(planner)
add_subdirectory(function)

8
source/libs/catalog/src/catalog.c
View File

@ -14,3 +14,11 @@
*/
#include "catalogInt.h"
struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps) {
return NULL;
}
int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData) {
return 0;
}

12
source/libs/function/CMakeLists.txt Normal file
View File

@ -0,0 +1,12 @@
aux_source_directory(src FUNCTION_SRC)
add_library(function ${FUNCTION_SRC})
target_include_directories(
function
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/function"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
)
target_link_libraries(
function
PRIVATE os util common
)

139
source/libs/function/inc/taggfunction.h Normal file
View File

@ -0,0 +1,139 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TAGGFUNCTION_H
#define TDENGINE_TAGGFUNCTION_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "tname.h"
#include "taosdef.h"
#include "tvariant.h"
#include "function.h"
#include "tudf.h"
extern SAggFunctionInfo aggFunc[34];
typedef struct SResultRowCellInfo {
int8_t hasResult; // result generated, not NULL value
bool initialized; // output buffer has been initialized
bool complete; // query has completed
uint32_t numOfRes; // num of output result in current buffer
} SResultRowCellInfo;
#define FUNCSTATE_SO 0x0u
#define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
#define FUNCSTATE_STREAM 0x2u // function avail for stream
#define FUNCSTATE_STABLE 0x4u // function avail for super table
#define FUNCSTATE_NEED_TS 0x8u // timestamp is required during query processing
#define FUNCSTATE_SELECTIVITY 0x10u // selectivity functions, can exists along with tag columns
#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE
#define AVG_FUNCTION_INTER_BUFFER_SIZE 50
#define DATA_SET_FLAG ',' // to denote the output area has data, not null value
#define DATA_SET_FLAG_SIZE sizeof(DATA_SET_FLAG)
#define QUERY_ASC_FORWARD_STEP 1
#define QUERY_DESC_FORWARD_STEP -1
#define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP)
#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results
#define TOP_BOTTOM_QUERY_LIMIT 100
enum {
MASTER_SCAN = 0x0u,
REVERSE_SCAN = 0x1u,
REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan
MERGE_STAGE = 0x20u,
};
#define QUERY_IS_STABLE_QUERY(type) (((type)&TSDB_QUERY_TYPE_STABLE_QUERY) != 0)
#define QUERY_IS_JOIN_QUERY(type) (TSDB_QUERY_HAS_TYPE(type, TSDB_QUERY_TYPE_JOIN_QUERY))
#define QUERY_IS_PROJECTION_QUERY(type) (((type)&TSDB_QUERY_TYPE_PROJECTION_QUERY) != 0)
#define QUERY_IS_FREE_RESOURCE(type) (((type)&TSDB_QUERY_TYPE_FREE_RESOURCE) != 0)
typedef struct SArithmeticSupport {
struct SExprInfo *pExprInfo;
int32_t numOfCols;
SColumnInfo *colList;
void *exprList; // client side used
int32_t offset;
char** data;
} SArithmeticSupport;
typedef struct SInterpInfoDetail {
TSKEY ts; // interp specified timestamp
int8_t type;
int8_t primaryCol;
} SInterpInfoDetail;
#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo)))
#define GET_RES_INFO(ctx) ((ctx)->resultInfo)
#define IS_STREAM_QUERY_VALID(x) (((x)&TSDB_FUNCSTATE_STREAM) != 0)
#define IS_MULTIOUTPUT(x) (((x)&TSDB_FUNCSTATE_MO) != 0)
// determine the real data need to calculated the result
enum {
BLK_DATA_NO_NEEDED = 0x0,
BLK_DATA_STATIS_NEEDED = 0x1,
BLK_DATA_ALL_NEEDED = 0x3,
BLK_DATA_DISCARD = 0x4, // discard current data block since it is not qualified for filter
};
typedef struct STwaInfo {
int8_t hasResult; // flag to denote has value
double dOutput;
SPoint1 p;
STimeWindow win;
} STwaInfo;
extern int32_t functionCompatList[]; // compatible check array list
bool topbot_datablock_filter(SQLFunctionCtx *pCtx, const char *minval, const char *maxval);
/**
* the numOfRes should be kept, since it may be used later
* and allow the ResultInfo to be re initialized
*/
#define RESET_RESULT_INFO(_r) \
do { \
(_r)->initialized = false; \
} while (0)
static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, int32_t bufLen) {
pResInfo->initialized = true; // the this struct has been initialized flag
pResInfo->complete = false;
pResInfo->hasResult = false;
pResInfo->numOfRes = 0;
memset(GET_ROWCELL_INTERBUF(pResInfo), 0, bufLen);
}
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TAGGFUNCTION_H

30
source/libs/executor/inc/texpr.h → source/libs/function/inc/texpr.h
View File

@ -25,7 +25,7 @@ extern "C" {
#include "taosmsg.h"
#include "taosdef.h"
#include "tskiplist.h"
#include "tbuffer.h"
#include "function.h"
struct tExprNode;
struct SSchema;
@ -43,13 +43,6 @@ struct SSchema;
typedef bool (*__result_filter_fn_t)(const void *, void *);
typedef void (*__do_filter_suppl_fn_t)(void *, void *);
enum {
TSQL_NODE_DUMMY = 0x0,
TSQL_NODE_EXPR = 0x1,
TSQL_NODE_COL = 0x2,
TSQL_NODE_VALUE = 0x4,
};
/**
* this structure is used to filter data in tags, so the offset of filtered tag column in tagdata string is required
*/
@ -61,37 +54,16 @@ typedef struct tQueryInfo {
bool indexed; // indexed columns
} tQueryInfo;
typedef struct tExprNode {
uint8_t nodeType;
union {
struct {
uint8_t optr; // filter operator
uint8_t hasPK; // 0: do not contain primary filter, 1: contain
void *info; // support filter operation on this expression only available for leaf node
struct tExprNode *pLeft; // left child pointer
struct tExprNode *pRight; // right child pointer
} _node;
SSchema *pSchema;
struct SVariant *pVal;
};
} tExprNode;
typedef struct SExprTraverseSupp {
__result_filter_fn_t nodeFilterFn;
__do_filter_suppl_fn_t setupInfoFn;
void *pExtInfo;
} SExprTraverseSupp;
void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *));
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
tExprNode* exprTreeFromBinary(const void* data, size_t size);
tExprNode* exprTreeFromTableName(const char* tbnameCond);
tExprNode* exprdup(tExprNode* pTree);
void exprTreeToBinary(SBufferWriter* bw, tExprNode* pExprTree);
bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp *param);
void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput, void *param, int32_t order,

96
source/libs/function/inc/tfill.h Normal file
View File

@ -0,0 +1,96 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TFILL_H
#define TDENGINE_TFILL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "taosdef.h"
struct SSDataBlock;
typedef struct {
STColumn col; // column info
int16_t functionId; // sql function id
int16_t flag; // column flag: TAG COLUMN|NORMAL COLUMN
int16_t tagIndex; // index of current tag in SFillTagColInfo array list
union {int64_t i; double d;} fillVal;
} SFillColInfo;
typedef struct {
SSchema col;
char* tagVal;
} SFillTagColInfo;
typedef struct SFillInfo {
TSKEY start; // start timestamp
TSKEY end; // endKey for fill
TSKEY currentKey; // current active timestamp, the value may be changed during the fill procedure.
int32_t order; // order [TSDB_ORDER_ASC|TSDB_ORDER_DESC]
int32_t type; // fill type
int32_t numOfRows; // number of rows in the input data block
int32_t index; // active row index
int32_t numOfTotal; // number of filled rows in one round
int32_t numOfCurrent; // number of filled rows in current results
int32_t numOfTags; // number of tags
int32_t numOfCols; // number of columns, including the tags columns
int32_t rowSize; // size of each row
SInterval interval;
char * prevValues; // previous row of data, to generate the interpolation results
char * nextValues; // next row of data
char** pData; // original result data block involved in filling data
int32_t alloc; // data buffer size in rows
int8_t precision; // time resoluation
SFillColInfo* pFillCol; // column info for fill operations
SFillTagColInfo* pTags; // tags value for filling gap
void* handle; // for debug purpose
} SFillInfo;
typedef struct SPoint {
int64_t key;
void * val;
} SPoint;
SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pFillCol, void* handle);
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp);
void* taosDestroyFillInfo(SFillInfo *pFillInfo);
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey);
void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const struct SSDataBlock* pInput);
bool taosFillHasMoreResults(SFillInfo* pFillInfo);
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, int64_t ekey, int32_t maxNumOfRows);
int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType);
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TFILL_H

79
source/libs/function/inc/thistogram.h Normal file
View File

@ -0,0 +1,79 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_HISTOGRAM_H
#define TDENGINE_HISTOGRAM_H
#ifdef __cplusplus
extern "C" {
#endif
#define USE_ARRAYLIST
#define MAX_HISTOGRAM_BIN 500
typedef struct SHistBin {
double val;
int64_t num;
#if !defined(USE_ARRAYLIST)
double delta;
int32_t index; // index in min-heap list
#endif
} SHistBin;
typedef struct SHeapEntry {
void* pData;
double val;
} SHeapEntry;
typedef struct SHistogramInfo {
int64_t numOfElems;
int32_t numOfEntries;
int32_t maxEntries;
double min;
double max;
#if defined(USE_ARRAYLIST)
SHistBin* elems;
#else
tSkipList* pList;
SLoserTreeInfo* pLoserTree;
int32_t maxIndex;
bool ordered;
#endif
} SHistogramInfo;
SHistogramInfo* tHistogramCreate(int32_t numOfBins);
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins);
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val);
int64_t tHistogramSum(SHistogramInfo* pHisto, double v);
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num);
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries);
void tHistogramDestroy(SHistogramInfo** pHisto);
void tHistogramPrint(SHistogramInfo* pHisto);
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
SHeapEntry* tHeapCreate(int32_t numOfEntries);
void tHeapSort(SHeapEntry* pEntry, int32_t len);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_HISTOGRAM_H

82
source/libs/function/inc/tpercentile.h Normal file
View File

@ -0,0 +1,82 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TPERCENTILE_H
#define TDENGINE_TPERCENTILE_H
#ifdef __cplusplus
extern "C" {
#endif
#include "tpagedfile.h"
#include "ttszip.h"
typedef struct MinMaxEntry {
union {
double dMinVal;
int64_t i64MinVal;
uint64_t u64MinVal;
};
union {
double dMaxVal;
int64_t i64MaxVal;
int64_t u64MaxVal;
};
} MinMaxEntry;
typedef struct {
int32_t size;
int32_t pageId;
SFilePage *data;
} SSlotInfo;
typedef struct tMemBucketSlot {
SSlotInfo info;
MinMaxEntry range;
} tMemBucketSlot;
struct tMemBucket;
typedef int32_t (*__perc_hash_func_t)(struct tMemBucket *pBucket, const void *value);
typedef struct tMemBucket {
int16_t numOfSlots;
int16_t type;
int16_t bytes;
int32_t total;
int32_t elemPerPage; // number of elements for each object
int32_t maxCapacity; // maximum allowed number of elements that can be sort directly to get the result
int32_t bufPageSize; // disk page size
MinMaxEntry range; // value range
int32_t times; // count that has been checked for deciding the correct data value buckets.
__compar_fn_t comparFn;
tMemBucketSlot * pSlots;
SDiskbasedResultBuf *pBuffer;
__perc_hash_func_t hashFunc;
} tMemBucket;
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval);
void tMemBucketDestroy(tMemBucket *pBucket);
int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size);
double getPercentile(tMemBucket *pMemBucket, double percent);
#endif // TDENGINE_TPERCENTILE_H
#ifdef __cplusplus
}
#endif

36
source/libs/function/inc/tscalarfunction.h Normal file
View File

@ -0,0 +1,36 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TSCALARFUNCTION_H
#define TDENGINE_TSCALARFUNCTION_H
#ifdef __cplusplus
extern "C" {
#endif
#include "function.h"
extern struct SScalarFunctionInfo scalarFunc[1];
#define FUNCTION_CEIL 38
#define FUNCTION_FLOOR 39
#define FUNCTION_ROUND 40
#define FUNCTION_MAVG 41
#define FUNCTION_CSUM 42
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TSCALARFUNCTION_H

147
source/libs/function/inc/ttszip.h Normal file
View File

@ -0,0 +1,147 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TTSZIP_H
#define TDENGINE_TTSZIP_H
#ifdef __cplusplus
extern "C" {
#endif
#include "os.h"
#include "taosdef.h"
#include "tvariant.h"
#define MEM_BUF_SIZE (1 << 20)
#define TS_COMP_FILE_MAGIC 0x87F5EC4C
#define TS_COMP_FILE_GROUP_MAX 512
typedef struct STSList {
char* rawBuf;
int32_t allocSize;
int32_t threshold;
int32_t len;
} STSList;
typedef struct STSElem {
TSKEY ts;
SVariant* tag;
int32_t id;
} STSElem;
typedef struct STSCursor {
int32_t vgroupIndex;
int32_t blockIndex;
int32_t tsIndex;
uint32_t order;
} STSCursor;
typedef struct STSBlock {
SVariant tag; // tag value
int32_t numOfElem; // number of elements
int32_t compLen; // size after compressed
int32_t padding; // 0xFFFFFFFF by default, after the payload
char* payload; // actual data that is compressed
} STSBlock;
/*
* The size of buffer file should not be greater than 2G,
* and the offset of int32_t type is enough
*/
typedef struct STSGroupBlockInfo {
int32_t id; // group id
int32_t offset; // offset set value in file
int32_t numOfBlocks; // number of total blocks
int32_t compLen; // compressed size
} STSGroupBlockInfo;
typedef struct STSGroupBlockInfoEx {
STSGroupBlockInfo info;
int32_t len; // length before compress
} STSGroupBlockInfoEx;
typedef struct STSBuf {
FILE* f;
char path[PATH_MAX];
uint32_t fileSize;
// todo use array
STSGroupBlockInfoEx* pData;
uint32_t numOfAlloc;
uint32_t numOfGroups;
char* assistBuf;
int32_t bufSize;
STSBlock block;
STSList tsData; // uncompressed raw ts data
uint64_t numOfTotal;
bool autoDelete;
bool remainOpen;
int32_t tsOrder; // order of timestamp in ts comp buffer
STSCursor cur;
} STSBuf;
typedef struct STSBufFileHeader {
uint32_t magic; // file magic number
uint32_t numOfGroup; // number of group stored in current file
int32_t tsOrder; // timestamp order in current file
} STSBufFileHeader;
STSBuf* tsBufCreate(bool autoDelete, int32_t order);
STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete);
STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t id);
void* tsBufDestroy(STSBuf* pTSBuf);
void tsBufAppend(STSBuf* pTSBuf, int32_t id, SVariant* tag, const char* pData, int32_t len);
int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf);
STSBuf* tsBufClone(STSBuf* pTSBuf);
STSGroupBlockInfo* tsBufGetGroupBlockInfo(STSBuf* pTSBuf, int32_t id);
void tsBufFlush(STSBuf* pTSBuf);
void tsBufResetPos(STSBuf* pTSBuf);
bool tsBufNextPos(STSBuf* pTSBuf);
STSElem tsBufGetElem(STSBuf* pTSBuf);
STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t id, SVariant* tag);
STSCursor tsBufGetCursor(STSBuf* pTSBuf);
void tsBufSetTraverseOrder(STSBuf* pTSBuf, int32_t order);
void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur);
/**
* display all data in comp block file, for debug purpose only
* @param pTSBuf
*/
void tsBufDisplay(STSBuf* pTSBuf);
int32_t tsBufGetNumOfGroup(STSBuf* pTSBuf);
void tsBufGetGroupIdList(STSBuf* pTSBuf, int32_t* num, int32_t** id);
int32_t dumpFileBlockByGroupId(STSBuf* pTSBuf, int32_t id, void* buf, int32_t* len, int32_t* numOfBlocks);
STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, SVariant* pTag);
bool tsBufIsValidElem(STSElem* pElem);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_TTSZIP_H

79
source/libs/function/inc/tudf.h Normal file
View File

@ -0,0 +1,79 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TUDF_H
#define TDENGINE_TUDF_H
enum {
TSDB_UDF_FUNC_NORMAL = 0,
TSDB_UDF_FUNC_INIT,
TSDB_UDF_FUNC_FINALIZE,
TSDB_UDF_FUNC_MERGE,
TSDB_UDF_FUNC_DESTROY,
TSDB_UDF_FUNC_MAX_NUM
};
typedef struct SUdfInit {
int32_t maybe_null; /* 1 if function can return NULL */
uint32_t decimals; /* for real functions */
uint64_t length; /* For string functions */
char* ptr; /* free pointer for function data */
int32_t const_item; /* 0 if result is independent of arguments */
// script like lua/javascript
void* script_ctx;
void (*destroyCtxFunc)(void* script_ctx);
} SUdfInit;
typedef struct SUdfInfo {
int32_t functionId; // system assigned function id
int32_t funcType; // scalar function or aggregate function
int8_t resType; // result type
int16_t resBytes; // result byte
int32_t contLen; // content length
int32_t bufSize; // interbuf size
char* name; // function name
void* handle; // handle loaded in mem
void* funcs[TSDB_UDF_FUNC_MAX_NUM]; // function ptr
// for script like lua/javascript only
int isScript;
void* pScriptCtx;
SUdfInit init;
char* content;
char* path;
} SUdfInfo;
// script
typedef int32_t (*scriptInitFunc)(void* pCtx);
typedef void (*scriptNormalFunc)(void* pCtx, char* data, int16_t iType, int16_t iBytes, int32_t numOfRows,
int64_t* ptList, int64_t key, char* dataOutput, char* tsOutput, int32_t* numOfOutput,
int16_t oType, int16_t oBytes);
typedef void (*scriptFinalizeFunc)(void* pCtx, int64_t key, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptMergeFunc)(void* pCtx, char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput);
typedef void (*scriptDestroyFunc)(void* pCtx);
// dynamic lib
typedef void (*udfNormalFunc)(char* data, int16_t itype, int16_t iBytes, int32_t numOfRows, int64_t* ts,
char* dataOutput, char* interBuf, char* tsOutput, int32_t* numOfOutput, int16_t oType,
int16_t oBytes, SUdfInit* buf);
typedef int32_t (*udfInitFunc)(SUdfInit* data);
typedef void (*udfFinalizeFunc)(char* dataOutput, char* interBuf, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfMergeFunc)(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf);
typedef void (*udfDestroyFunc)(SUdfInit* buf);
#endif // TDENGINE_TUDF_H

4777
source/libs/function/src/taggfunction.c Normal file
View File

File diff suppressed because it is too large Load Diff

107
source/libs/executor/src/texpr.c → source/libs/function/src/texpr.c
View File

@ -15,7 +15,6 @@
#include "os.h"
#include "texpr.h"
#include "exception.h"
#include "taosdef.h"
#include "taosmsg.h"
@ -26,21 +25,21 @@
#include "thash.h"
#include "tskiplist.h"
#include "texpr.h"
#include "tarithoperator.h"
//#include "tarithoperator.h"
#include "tvariant.h"
static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
if (pLeft->nodeType == TSQL_NODE_COL) {
// if left node is the primary column,return true
return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0;
} else {
// if any children have query on primary key, their parents are also keep this value
return ((pLeft->nodeType == TSQL_NODE_EXPR && pLeft->_node.hasPK == 1) ||
(pRight->nodeType == TSQL_NODE_EXPR && pRight->_node.hasPK == 1)) == true
? 1
: 0;
}
}
//static uint8_t UNUSED_FUNC isQueryOnPrimaryKey(const char *primaryColumnName, const tExprNode *pLeft, const tExprNode *pRight) {
// if (pLeft->nodeType == TEXPR_COL_NODE) {
// // if left node is the primary column,return true
// return (strcmp(primaryColumnName, pLeft->pSchema->name) == 0) ? 1 : 0;
// } else {
// // if any children have query on primary key, their parents are also keep this value
// return ((pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pLeft->_node.hasPK == 1) ||
// (pRight->nodeType == TEXPR_BINARYEXPR_NODE && pRight->_node.hasPK == 1)) == true
// ? 1
// : 0;
// }
//}
static void reverseCopy(char* dest, const char* src, int16_t type, int32_t numOfRows) {
switch(type) {
@ -114,11 +113,11 @@ void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)) {
return;
}
if (pNode->nodeType == TSQL_NODE_EXPR) {
if (pNode->nodeType == TEXPR_BINARYEXPR_NODE || pNode->nodeType == TEXPR_UNARYEXPR_NODE) {
doExprTreeDestroy(&pNode, fp);
} else if (pNode->nodeType == TSQL_NODE_VALUE) {
} else if (pNode->nodeType == TEXPR_VALUE_NODE) {
taosVariantDestroy(pNode->pVal);
} else if (pNode->nodeType == TSQL_NODE_COL) {
} else if (pNode->nodeType == TEXPR_COL_NODE) {
tfree(pNode->pSchema);
}
@ -130,17 +129,17 @@ static void doExprTreeDestroy(tExprNode **pExpr, void (*fp)(void *)) {
return;
}
if ((*pExpr)->nodeType == TSQL_NODE_EXPR) {
if ((*pExpr)->nodeType == TEXPR_BINARYEXPR_NODE) {
doExprTreeDestroy(&(*pExpr)->_node.pLeft, fp);
doExprTreeDestroy(&(*pExpr)->_node.pRight, fp);
if (fp != NULL) {
fp((*pExpr)->_node.info);
}
} else if ((*pExpr)->nodeType == TSQL_NODE_VALUE) {
} else if ((*pExpr)->nodeType == TEXPR_VALUE_NODE) {
taosVariantDestroy((*pExpr)->pVal);
free((*pExpr)->pVal);
} else if ((*pExpr)->nodeType == TSQL_NODE_COL) {
} else if ((*pExpr)->nodeType == TEXPR_COL_NODE) {
free((*pExpr)->pSchema);
}
@ -153,7 +152,7 @@ bool exprTreeApplyFilter(tExprNode *pExpr, const void *pItem, SExprTraverseSupp
tExprNode *pRight = pExpr->_node.pRight;
//non-leaf nodes, recursively traverse the expression tree in the post-root order
if (pLeft->nodeType == TSQL_NODE_EXPR && pRight->nodeType == TSQL_NODE_EXPR) {
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE && pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
if (pExpr->_node.optr == TSDB_RELATION_OR) { // or
if (exprTreeApplyFilter(pLeft, pItem, param)) {
return true;
@ -180,26 +179,26 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
if (pExprs == NULL) {
return;
}
#if 0
tExprNode *pLeft = pExprs->_node.pLeft;
tExprNode *pRight = pExprs->_node.pRight;
/* the left output has result from the left child syntax tree */
char *pLeftOutput = (char*)malloc(sizeof(int64_t) * numOfRows);
if (pLeft->nodeType == TSQL_NODE_EXPR) {
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE) {
arithmeticTreeTraverse(pLeft, numOfRows, pLeftOutput, param, order, getSourceDataBlock);
}
/* the right output has result from the right child syntax tree */
// the right output has result from the right child syntax tree
char *pRightOutput = malloc(sizeof(int64_t) * numOfRows);
char *pdata = malloc(sizeof(int64_t) * numOfRows);
if (pRight->nodeType == TSQL_NODE_EXPR) {
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
arithmeticTreeTraverse(pRight, numOfRows, pRightOutput, param, order, getSourceDataBlock);
}
if (pLeft->nodeType == TSQL_NODE_EXPR) {
if (pRight->nodeType == TSQL_NODE_EXPR) {
if (pLeft->nodeType == TEXPR_BINARYEXPR_NODE) {
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) {
/*
* exprLeft + exprRight
* the type of returned value of one expression is always double float precious
@ -207,7 +206,7 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC);
} else if (pRight->nodeType == TSQL_NODE_COL) { // exprLeft + columnRight
} else if (pRight->nodeType == TEXPR_COL_NODE) { // exprLeft + columnRight
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
// set input buffer
@ -219,14 +218,14 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pInputData, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // exprLeft + 12
} else if (pRight->nodeType == TEXPR_VALUE_NODE) { // exprLeft + 12
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
OperatorFn(pLeftOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, &pRight->pVal->i64, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC);
}
} else if (pLeft->nodeType == TSQL_NODE_COL) {
} else if (pLeft->nodeType == TEXPR_COL_NODE) {
// column data specified on left-hand-side
char *pLeftInputData = getSourceDataBlock(param, pLeft->pSchema->name, pLeft->pSchema->colId);
if (pRight->nodeType == TSQL_NODE_EXPR) { // columnLeft + expr2
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { // columnLeft + expr2
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
if (order == TSDB_ORDER_DESC) {
@ -236,14 +235,14 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_COL) { // columnLeft + columnRight
} else if (pRight->nodeType == TEXPR_COL_NODE) { // columnLeft + columnRight
// column data specified on right-hand-side
char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
// both columns are descending order, do not reverse the source data
OperatorFn(pLeftInputData, numOfRows, pLeft->pSchema->type, pRightInputData, numOfRows, pRight->pSchema->type, pOutput, order);
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // columnLeft + 12
} else if (pRight->nodeType == TEXPR_VALUE_NODE) { // columnLeft + 12
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
if (order == TSDB_ORDER_DESC) {
@ -255,11 +254,11 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
}
} else {
// column data specified on left-hand-side
if (pRight->nodeType == TSQL_NODE_EXPR) { // 12 + expr2
if (pRight->nodeType == TEXPR_BINARYEXPR_NODE) { // 12 + expr2
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, pRightOutput, numOfRows, TSDB_DATA_TYPE_DOUBLE, pOutput, TSDB_ORDER_ASC);
} else if (pRight->nodeType == TSQL_NODE_COL) { // 12 + columnRight
} else if (pRight->nodeType == TEXPR_COL_NODE) { // 12 + columnRight
// column data specified on right-hand-side
char *pRightInputData = getSourceDataBlock(param, pRight->pSchema->name, pRight->pSchema->colId);
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
@ -271,7 +270,7 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, pRightInputData, numOfRows, pRight->pSchema->type, pOutput, TSDB_ORDER_ASC);
}
} else if (pRight->nodeType == TSQL_NODE_VALUE) { // 12 + 12
} else if (pRight->nodeType == TEXPR_VALUE_NODE) { // 12 + 12
_arithmetic_operator_fn_t OperatorFn = getArithmeticOperatorFn(pExprs->_node.optr);
OperatorFn(&pLeft->pVal->i64, 1, pLeft->pVal->nType, &pRight->pVal->i64, 1, pRight->pVal->nType, pOutput, TSDB_ORDER_ASC);
}
@ -280,12 +279,14 @@ void arithmeticTreeTraverse(tExprNode *pExprs, int32_t numOfRows, char *pOutput,
tfree(pdata);
tfree(pLeftOutput);
tfree(pRightOutput);
#endif
}
static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
tbufWriteUint8(bw, expr->nodeType);
if (expr->nodeType == TSQL_NODE_VALUE) {
if (expr->nodeType == TEXPR_VALUE_NODE) {
SVariant* pVal = expr->pVal;
tbufWriteUint32(bw, pVal->nType);
@ -296,16 +297,15 @@ static void exprTreeToBinaryImpl(SBufferWriter* bw, tExprNode* expr) {
tbufWriteInt64(bw, pVal->i64);
}
} else if (expr->nodeType == TSQL_NODE_COL) {
} else if (expr->nodeType == TEXPR_COL_NODE) {
SSchema* pSchema = expr->pSchema;
tbufWriteInt16(bw, pSchema->colId);
tbufWriteInt16(bw, pSchema->bytes);
tbufWriteUint8(bw, pSchema->type);
tbufWriteString(bw, pSchema->name);
} else if (expr->nodeType == TSQL_NODE_EXPR) {
} else if (expr->nodeType == TEXPR_BINARYEXPR_NODE) {
tbufWriteUint8(bw, expr->_node.optr);
tbufWriteUint8(bw, expr->_node.hasPK);
exprTreeToBinaryImpl(bw, expr->_node.pLeft);
exprTreeToBinaryImpl(bw, expr->_node.pRight);
}
@ -353,7 +353,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, pExpr, NULL);
pExpr->nodeType = tbufReadUint8(br);
if (pExpr->nodeType == TSQL_NODE_VALUE) {
if (pExpr->nodeType == TEXPR_VALUE_NODE) {
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
pExpr->pVal = pVal;
@ -366,7 +366,7 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
pVal->i64 = tbufReadInt64(br);
}
} else if (pExpr->nodeType == TSQL_NODE_COL) {
} else if (pExpr->nodeType == TEXPR_COL_NODE) {
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
pExpr->pSchema = pSchema;
@ -375,9 +375,8 @@ static tExprNode* exprTreeFromBinaryImpl(SBufferReader* br) {
pSchema->type = tbufReadUint8(br);
tbufReadToString(br, pSchema->name, TSDB_COL_NAME_LEN);
} else if (pExpr->nodeType == TSQL_NODE_EXPR) {
} else if (pExpr->nodeType == TEXPR_BINARYEXPR_NODE) {
pExpr->_node.optr = tbufReadUint8(br);
pExpr->_node.hasPK = tbufReadUint8(br);
pExpr->_node.pLeft = exprTreeFromBinaryImpl(br);
pExpr->_node.pRight = exprTreeFromBinaryImpl(br);
assert(pExpr->_node.pLeft != NULL && pExpr->_node.pRight != NULL);
@ -406,12 +405,12 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
tExprNode* expr = exception_calloc(1, sizeof(tExprNode));
CLEANUP_PUSH_VOID_PTR_PTR(true, tExprTreeDestroy, expr, NULL);
expr->nodeType = TSQL_NODE_EXPR;
expr->nodeType = TEXPR_BINARYEXPR_NODE;
tExprNode* left = exception_calloc(1, sizeof(tExprNode));
expr->_node.pLeft = left;
left->nodeType = TSQL_NODE_COL;
left->nodeType = TEXPR_COL_NODE;
SSchema* pSchema = exception_calloc(1, sizeof(SSchema));
left->pSchema = pSchema;
@ -421,7 +420,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
expr->_node.pRight = right;
if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_LIKE, QUERY_COND_REL_PREFIX_LIKE_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_LIKE;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
@ -432,7 +431,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_MATCH, QUERY_COND_REL_PREFIX_MATCH_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_MATCH;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
@ -442,7 +441,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_NMATCH, QUERY_COND_REL_PREFIX_NMATCH_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_NMATCH;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
@ -452,7 +451,7 @@ tExprNode* exprTreeFromTableName(const char* tbnameCond) {
pVal->nType = TSDB_DATA_TYPE_BINARY;
pVal->nLen = (int32_t)len;
} else if (strncmp(tbnameCond, QUERY_COND_REL_PREFIX_IN, QUERY_COND_REL_PREFIX_IN_LEN) == 0) {
right->nodeType = TSQL_NODE_VALUE;
right->nodeType = TEXPR_VALUE_NODE;
expr->_node.optr = TSDB_RELATION_IN;
SVariant* pVal = exception_calloc(1, sizeof(SVariant));
right->pVal = pVal;
@ -699,25 +698,23 @@ err_ret:
tfree(tmp);
}
tExprNode* exprdup(tExprNode* pNode) {
if (pNode == NULL) {
return NULL;
}
tExprNode* pCloned = calloc(1, sizeof(tExprNode));
if (pNode->nodeType == TSQL_NODE_EXPR) {
if (pNode->nodeType == TEXPR_BINARYEXPR_NODE) {
tExprNode* pLeft = exprdup(pNode->_node.pLeft);
tExprNode* pRight = exprdup(pNode->_node.pRight);
pCloned->_node.pLeft = pLeft;
pCloned->_node.pRight = pRight;
pCloned->_node.optr = pNode->_node.optr;
pCloned->_node.hasPK = pNode->_node.hasPK;
} else if (pNode->nodeType == TSQL_NODE_VALUE) {
} else if (pNode->nodeType == TEXPR_VALUE_NODE) {
pCloned->pVal = calloc(1, sizeof(SVariant));
taosVariantAssign(pCloned->pVal, pNode->pVal);
} else if (pNode->nodeType == TSQL_NODE_COL) {
} else if (pNode->nodeType == TEXPR_COL_NODE) {
pCloned->pSchema = calloc(1, sizeof(SSchema));
*pCloned->pSchema = *pNode->pSchema;
}

524
source/libs/function/src/tfill.c Normal file
View File

@ -0,0 +1,524 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "ttypes.h"
#include "tfill.h"
#include "thash.h"
#include "function.h"
#include "common.h"
#include "ttime.h"
#define FILL_IS_ASC_FILL(_f) ((_f)->order == TSDB_ORDER_ASC)
#define DO_INTERPOLATION(_v1, _v2, _k1, _k2, _k) ((_v1) + ((_v2) - (_v1)) * (((double)(_k)) - ((double)(_k1))) / (((double)(_k2)) - ((double)(_k1))))
#define GET_FORWARD_DIRECTION_FACTOR(_ord) (((_ord) == TSDB_ORDER_ASC)? 1:-1)
static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) {
for(int32_t j = 0; j < pFillInfo->numOfCols; ++j) {
SFillColInfo* pCol = &pFillInfo->pFillCol[j];
if (TSDB_COL_IS_NORMAL_COL(pCol->flag) || TSDB_COL_IS_UD_COL(pCol->flag)) {
continue;
}
char* val1 = elePtrAt(data[j], pCol->col.bytes, genRows);
assert(pCol->tagIndex >= 0 && pCol->tagIndex < pFillInfo->numOfTags);
SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
assert (pTag->col.colId == pCol->col.colId);
assignVal(val1, pTag->tagVal, pCol->col.bytes, pCol->col.type);
}
}
static void setNullValueForRow(SFillInfo* pFillInfo, void** data, int32_t numOfCol, int32_t rowIndex) {
// the first are always the timestamp column, so start from the second column.
for (int32_t i = 1; i < numOfCol; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
char* output = elePtrAt(data[i], pCol->col.bytes, rowIndex);
setNull(output, pCol->col.type, pCol->col.bytes);
}
}
static void doFillOneRowResult(SFillInfo* pFillInfo, void** data, char** srcData, int64_t ts, bool outOfBound) {
char* prev = pFillInfo->prevValues;
char* next = pFillInfo->nextValues;
SPoint point1, point2, point;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
// set the primary timestamp column value
int32_t index = pFillInfo->numOfCurrent;
char* val = elePtrAt(data[0], TSDB_KEYSIZE, index);
*(TSKEY*) val = pFillInfo->currentKey;
// set the other values
if (pFillInfo->type == TSDB_FILL_PREV) {
char* p = FILL_IS_ASC_FILL(pFillInfo) ? prev : next;
if (p != NULL) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
} else { // no prev value yet, set the value for NULL
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else if (pFillInfo->type == TSDB_FILL_NEXT) {
char* p = FILL_IS_ASC_FILL(pFillInfo)? next : prev;
if (p != NULL) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(output, p + pCol->col.offset, pCol->col.bytes, pCol->col.type);
}
} else { // no prev value yet, set the value for NULL
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else if (pFillInfo->type == TSDB_FILL_LINEAR) {
// TODO : linear interpolation supports NULL value
if (prev != NULL && !outOfBound) {
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)) {
continue;
}
int16_t type = pCol->col.type;
int16_t bytes = pCol->col.bytes;
char *val1 = elePtrAt(data[i], pCol->col.bytes, index);
if (type == TSDB_DATA_TYPE_BINARY|| type == TSDB_DATA_TYPE_NCHAR || type == TSDB_DATA_TYPE_BOOL) {
setNull(val1, pCol->col.type, bytes);
continue;
}
point1 = (SPoint){.key = *(TSKEY*)(prev), .val = prev + pCol->col.offset};
point2 = (SPoint){.key = ts, .val = srcData[i] + pFillInfo->index * bytes};
point = (SPoint){.key = pFillInfo->currentKey, .val = val1};
taosGetLinearInterpolationVal(&point, type, &point1, &point2, type);
}
} else {
setNullValueForRow(pFillInfo, data, pFillInfo->numOfCols, index);
}
} else { // fill the default value */
for (int32_t i = 1; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) {
continue;
}
char* val1 = elePtrAt(data[i], pCol->col.bytes, index);
assignVal(val1, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
setTagsValue(pFillInfo, data, index);
pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step, pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->numOfCurrent++;
}
static void initBeforeAfterDataBuf(SFillInfo* pFillInfo, char** next) {
if (*next != NULL) {
return;
}
*next = calloc(1, pFillInfo->rowSize);
for (int i = 1; i < pFillInfo->numOfCols; i++) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
setNull(*next + pCol->col.offset, pCol->col.type, pCol->col.bytes);
}
}
static void copyCurrentRowIntoBuf(SFillInfo* pFillInfo, char** srcData, char* buf) {
int32_t rowIndex = pFillInfo->index;
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
memcpy(buf + pCol->col.offset, srcData[i] + rowIndex * pCol->col.bytes, pCol->col.bytes);
}
}
static int32_t fillResultImpl(SFillInfo* pFillInfo, void** data, int32_t outputRows) {
pFillInfo->numOfCurrent = 0;
char** srcData = pFillInfo->pData;
char** prev = &pFillInfo->prevValues;
char** next = &pFillInfo->nextValues;
int32_t step = GET_FORWARD_DIRECTION_FACTOR(pFillInfo->order);
if (FILL_IS_ASC_FILL(pFillInfo)) {
assert(pFillInfo->currentKey >= pFillInfo->start);
} else {
assert(pFillInfo->currentKey <= pFillInfo->start);
}
while (pFillInfo->numOfCurrent < outputRows) {
int64_t ts = ((int64_t*)pFillInfo->pData[0])[pFillInfo->index];
// set the next value for interpolation
if ((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) {
initBeforeAfterDataBuf(pFillInfo, next);
copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
}
if (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) || (pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
pFillInfo->numOfCurrent < outputRows) {
// fill the gap between two actual input rows
while (((pFillInfo->currentKey < ts && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->currentKey > ts && !FILL_IS_ASC_FILL(pFillInfo))) &&
pFillInfo->numOfCurrent < outputRows) {
doFillOneRowResult(pFillInfo, data, srcData, ts, false);
}
// output buffer is full, abort
if (pFillInfo->numOfCurrent == outputRows) {
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return outputRows;
}
} else {
assert(pFillInfo->currentKey == ts);
initBeforeAfterDataBuf(pFillInfo, prev);
if (pFillInfo->type == TSDB_FILL_NEXT && (pFillInfo->index + 1) < pFillInfo->numOfRows) {
initBeforeAfterDataBuf(pFillInfo, next);
++pFillInfo->index;
copyCurrentRowIntoBuf(pFillInfo, srcData, *next);
--pFillInfo->index;
}
// assign rows to dst buffer
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
if (TSDB_COL_IS_TAG(pCol->flag)/* || IS_VAR_DATA_TYPE(pCol->col.type)*/) {
continue;
}
char* output = elePtrAt(data[i], pCol->col.bytes, pFillInfo->numOfCurrent);
char* src = elePtrAt(srcData[i], pCol->col.bytes, pFillInfo->index);
if (i == 0 || (pCol->functionId != FUNCTION_COUNT && !isNull(src, pCol->col.type)) ||
(pCol->functionId == FUNCTION_COUNT && GET_INT64_VAL(src) != 0)) {
assignVal(output, src, pCol->col.bytes, pCol->col.type);
memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
} else { // i > 0 and data is null , do interpolation
if (pFillInfo->type == TSDB_FILL_PREV) {
assignVal(output, *prev + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else if (pFillInfo->type == TSDB_FILL_LINEAR) {
assignVal(output, src, pCol->col.bytes, pCol->col.type);
memcpy(*prev + pCol->col.offset, src, pCol->col.bytes);
} else if (pFillInfo->type == TSDB_FILL_NEXT) {
if (*next) {
assignVal(output, *next + pCol->col.offset, pCol->col.bytes, pCol->col.type);
} else {
setNull(output, pCol->col.type, pCol->col.bytes);
}
} else {
assignVal(output, (char*)&pCol->fillVal.i, pCol->col.bytes, pCol->col.type);
}
}
}
// set the tag value for final result
setTagsValue(pFillInfo, data, pFillInfo->numOfCurrent);
pFillInfo->currentKey = taosTimeAdd(pFillInfo->currentKey, pFillInfo->interval.sliding * step,
pFillInfo->interval.slidingUnit, pFillInfo->precision);
pFillInfo->index += 1;
pFillInfo->numOfCurrent += 1;
}
if (pFillInfo->index >= pFillInfo->numOfRows || pFillInfo->numOfCurrent >= outputRows) {
/* the raw data block is exhausted, next value does not exists */
if (pFillInfo->index >= pFillInfo->numOfRows) {
tfree(*next);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
return pFillInfo->numOfCurrent;
}
}
return pFillInfo->numOfCurrent;
}
static int64_t appendFilledResult(SFillInfo* pFillInfo, void** output, int64_t resultCapacity) {
/*
* These data are generated according to fill strategy, since the current timestamp is out of the time window of
* real result set. Note that we need to keep the direct previous result rows, to generated the filled data.
*/
pFillInfo->numOfCurrent = 0;
while (pFillInfo->numOfCurrent < resultCapacity) {
doFillOneRowResult(pFillInfo, output, pFillInfo->pData, pFillInfo->start, true);
}
pFillInfo->numOfTotal += pFillInfo->numOfCurrent;
assert(pFillInfo->numOfCurrent == resultCapacity);
return resultCapacity;
}
// there are no duplicated tags in the SFillTagColInfo list
static int32_t setTagColumnInfo(SFillInfo* pFillInfo, int32_t numOfCols, int32_t capacity) {
int32_t rowsize = 0;
int32_t numOfTags = 0;
int32_t k = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
SFillColInfo* pColInfo = &pFillInfo->pFillCol[i];
pFillInfo->pData[i] = NULL;
if (TSDB_COL_IS_TAG(pColInfo->flag) || pColInfo->col.type == TSDB_DATA_TYPE_BINARY) {
numOfTags += 1;
bool exists = false;
int32_t index = -1;
for (int32_t j = 0; j < k; ++j) {
if (pFillInfo->pTags[j].col.colId == pColInfo->col.colId) {
exists = true;
index = j;
break;
}
}
if (!exists) {
SSchema* pSchema = &pFillInfo->pTags[k].col;
pSchema->colId = pColInfo->col.colId;
pSchema->type = pColInfo->col.type;
pSchema->bytes = pColInfo->col.bytes;
pFillInfo->pTags[k].tagVal = calloc(1, pColInfo->col.bytes);
pColInfo->tagIndex = k;
k += 1;
} else {
pColInfo->tagIndex = index;
}
}
rowsize += pColInfo->col.bytes;
}
pFillInfo->numOfTags = numOfTags;
assert(k <= pFillInfo->numOfTags);
return rowsize;
}
static int32_t taosNumOfRemainRows(SFillInfo* pFillInfo) {
if (pFillInfo->numOfRows == 0 || (pFillInfo->numOfRows > 0 && pFillInfo->index >= pFillInfo->numOfRows)) {
return 0;
}
return pFillInfo->numOfRows - pFillInfo->index;
}
SFillInfo* taosCreateFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_t capacity, int32_t numOfCols,
int64_t slidingTime, int8_t slidingUnit, int8_t precision, int32_t fillType,
SFillColInfo* pCol, void* handle) {
if (fillType == TSDB_FILL_NONE) {
return NULL;
}
SFillInfo* pFillInfo = calloc(1, sizeof(SFillInfo));
taosResetFillInfo(pFillInfo, skey);
pFillInfo->order = order;
pFillInfo->type = fillType;
pFillInfo->pFillCol = pCol;
pFillInfo->numOfTags = numOfTags;
pFillInfo->numOfCols = numOfCols;
pFillInfo->precision = precision;
pFillInfo->alloc = capacity;
pFillInfo->handle = handle;
pFillInfo->interval.interval = slidingTime;
pFillInfo->interval.intervalUnit = slidingUnit;
pFillInfo->interval.sliding = slidingTime;
pFillInfo->interval.slidingUnit = slidingUnit;
pFillInfo->pData = malloc(POINTER_BYTES * numOfCols);
// if (numOfTags > 0) {
pFillInfo->pTags = calloc(numOfCols, sizeof(SFillTagColInfo));
for (int32_t i = 0; i < numOfCols; ++i) {
pFillInfo->pTags[i].col.colId = -2; // TODO
}
// }
pFillInfo->rowSize = setTagColumnInfo(pFillInfo, pFillInfo->numOfCols, pFillInfo->alloc);
assert(pFillInfo->rowSize > 0);
return pFillInfo;
}
void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) {
pFillInfo->start = startTimestamp;
pFillInfo->currentKey = startTimestamp;
pFillInfo->end = startTimestamp;
pFillInfo->index = -1;
pFillInfo->numOfRows = 0;
pFillInfo->numOfCurrent = 0;
pFillInfo->numOfTotal = 0;
}
void* taosDestroyFillInfo(SFillInfo* pFillInfo) {
if (pFillInfo == NULL) {
return NULL;
}
tfree(pFillInfo->prevValues);
tfree(pFillInfo->nextValues);
for(int32_t i = 0; i < pFillInfo->numOfTags; ++i) {
tfree(pFillInfo->pTags[i].tagVal);
}
tfree(pFillInfo->pTags);
tfree(pFillInfo->pData);
tfree(pFillInfo->pFillCol);
tfree(pFillInfo);
return NULL;
}
void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey) {
if (pFillInfo->type == TSDB_FILL_NONE) {
return;
}
pFillInfo->end = endKey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(endKey, &pFillInfo->interval, pFillInfo->precision);
}
pFillInfo->index = 0;
pFillInfo->numOfRows = numOfRows;
}
void taosFillSetInputDataBlock(SFillInfo* pFillInfo, const SSDataBlock* pInput) {
for (int32_t i = 0; i < pFillInfo->numOfCols; ++i) {
SFillColInfo* pCol = &pFillInfo->pFillCol[i];
SColumnInfoData* pColData = taosArrayGet(pInput->pDataBlock, i);
pFillInfo->pData[i] = pColData->pData;
if (TSDB_COL_IS_TAG(pCol->flag)) { // copy the tag value to tag value buffer
SFillTagColInfo* pTag = &pFillInfo->pTags[pCol->tagIndex];
assert (pTag->col.colId == pCol->col.colId);
memcpy(pTag->tagVal, pColData->pData, pCol->col.bytes); // TODO not memcpy??
}
}
}
bool taosFillHasMoreResults(SFillInfo* pFillInfo) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
if (remain > 0) {
return true;
}
if (pFillInfo->numOfTotal > 0 && (((pFillInfo->end > pFillInfo->start) && FILL_IS_ASC_FILL(pFillInfo)) ||
(pFillInfo->end < pFillInfo->start && !FILL_IS_ASC_FILL(pFillInfo)))) {
return getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, 4096) > 0;
}
return false;
}
int64_t getNumOfResultsAfterFillGap(SFillInfo* pFillInfo, TSKEY ekey, int32_t maxNumOfRows) {
int64_t* tsList = (int64_t*) pFillInfo->pData[0];
int32_t numOfRows = taosNumOfRemainRows(pFillInfo);
TSKEY ekey1 = ekey;
if (!FILL_IS_ASC_FILL(pFillInfo)) {
pFillInfo->end = taosTimeTruncate(ekey, &pFillInfo->interval, pFillInfo->precision);
}
int64_t numOfRes = -1;
if (numOfRows > 0) { // still fill gap within current data block, not generating data after the result set.
TSKEY lastKey = tsList[pFillInfo->numOfRows - 1];
numOfRes = taosTimeCountInterval(
lastKey,
pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
assert(numOfRes >= numOfRows);
} else { // reach the end of data
if ((ekey1 < pFillInfo->currentKey && FILL_IS_ASC_FILL(pFillInfo)) ||
(ekey1 > pFillInfo->currentKey && !FILL_IS_ASC_FILL(pFillInfo))) {
return 0;
}
numOfRes = taosTimeCountInterval(
ekey1,
pFillInfo->currentKey,
pFillInfo->interval.sliding,
pFillInfo->interval.slidingUnit,
pFillInfo->precision);
numOfRes += 1;
}
return (numOfRes > maxNumOfRows) ? maxNumOfRows : numOfRes;
}
int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType) {
double v1 = -1, v2 = -1;
GET_TYPED_DATA(v1, double, inputType, point1->val);
GET_TYPED_DATA(v2, double, inputType, point2->val);
double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key);
SET_TYPED_DATA(point->val, outputType, r);
return TSDB_CODE_SUCCESS;
}
int64_t taosFillResultDataBlock(SFillInfo* pFillInfo, void** output, int32_t capacity) {
int32_t remain = taosNumOfRemainRows(pFillInfo);
int64_t numOfRes = getNumOfResultsAfterFillGap(pFillInfo, pFillInfo->end, capacity);
assert(numOfRes <= capacity);
// no data existed for fill operation now, append result according to the fill strategy
if (remain == 0) {
appendFilledResult(pFillInfo, output, numOfRes);
} else {
fillResultImpl(pFillInfo, output, (int32_t) numOfRes);
assert(numOfRes == pFillInfo->numOfCurrent);
}
// qDebug("fill:%p, generated fill result, src block:%d, index:%d, brange:%"PRId64"-%"PRId64", currentKey:%"PRId64", current:%d, total:%d, %p",
// pFillInfo, pFillInfo->numOfRows, pFillInfo->index, pFillInfo->start, pFillInfo->end, pFillInfo->currentKey, pFillInfo->numOfCurrent,
// pFillInfo->numOfTotal, pFillInfo->handle);
return numOfRes;
}

421
source/libs/function/src/tfunction.c Normal file
View File

@ -0,0 +1,421 @@
#include "os.h"
#include "tarray.h"
#include "function.h"
#include "thash.h"
#include "taggfunction.h"
#include "tscalarfunction.h"
static SHashObj* functionHashTable = NULL;
static void doInitFunctionHashTable() {
int numOfEntries = tListLen(aggFunc);
functionHashTable = taosHashInit(numOfEntries, MurmurHash3_32, false, false);
for (int32_t i = 0; i < numOfEntries; i++) {
int32_t len = (uint32_t)strlen(aggFunc[i].name);
SAggFunctionInfo* ptr = &aggFunc[i];
taosHashPut(functionHashTable, aggFunc[i].name, len, (void*)&ptr, POINTER_BYTES);
}
numOfEntries = tListLen(scalarFunc);
for(int32_t i = 0; i < numOfEntries; ++i) {
int32_t len = (int32_t) strlen(scalarFunc[i].name);
SScalarFunctionInfo* ptr = &scalarFunc[i];
taosHashPut(functionHashTable, scalarFunc[i].name, len, (void*)&ptr, POINTER_BYTES);
}
}
static pthread_once_t functionHashTableInit = PTHREAD_ONCE_INIT;
int32_t qIsBuiltinFunction(const char* name, int32_t len) {
pthread_once(&functionHashTableInit, doInitFunctionHashTable);
SAggFunctionInfo** pInfo = taosHashGet(functionHashTable, name, len);
if (pInfo != NULL) {
return (*pInfo)->functionId;
} else {
return -1;
}
}
bool qIsValidUdf(SArray* pUdfInfo, const char* name, int32_t len, int32_t* functionId) {
return true;
}
const char* qGetFunctionName(int32_t functionId) {
}
bool isTagsQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int16_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
// "select count(tbname)" query
// if (functId == FUNCTION_COUNT && pExpr->base.colInfo.colId == TSDB_TBNAME_COLUMN_INDEX) {
// continue;
// }
if (f != FUNCTION_TAGPRJ && f != FUNCTION_TID_TAG) {
return false;
}
}
return true;
}
//bool tscMultiRoundQuery(SArray* pFunctionIdList, int32_t index) {
// if (!UTIL_TABLE_IS_SUPER_TABLE(pQueryInfo->pTableMetaInfo[index])) {
// return false;
// }
//
// size_t numOfExprs = (int32_t) getNumOfExprs(pQueryInfo);
// for(int32_t i = 0; i < numOfExprs; ++i) {
// SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
// if (pExpr->base.functionId == FUNCTION_STDDEV_DST) {
// return true;
// }
// }
//
// return false;
//}
bool isBlockInfoQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_BLKINFO) {
return true;
}
}
return false;
}
bool isProjectionQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TS_DUMMY) {
continue;
}
if (f != FUNCTION_PRJ && f != FUNCTION_TAGPRJ && f != FUNCTION_TAG &&
f != FUNCTION_TS && f != FUNCTION_ARITHM && f != FUNCTION_DIFF &&
f != FUNCTION_DERIVATIVE) {
return false;
}
}
return true;
}
bool isDiffDerivQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TS_DUMMY) {
continue;
}
if (f == FUNCTION_DIFF || f == FUNCTION_DERIVATIVE) {
return true;
}
}
return false;
}
bool isPointInterpQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TAG || f == FUNCTION_TS) {
continue;
}
if (f != FUNCTION_INTERP) {
return false;
}
}
return true;
}
bool isArithmeticQueryOnAggResult(SArray* pFunctionIdList) {
if (isProjectionQuery(pFunctionIdList)) {
return false;
}
assert(0);
// size_t numOfOutput = getNumOfFields(pQueryInfo);
// for(int32_t i = 0; i < numOfOutput; ++i) {
// SExprInfo* pExprInfo = tscFieldInfoGetInternalField(&pQueryInfo->fieldsInfo, i)->pExpr;
// if (pExprInfo->pExpr != NULL) {
// return true;
// }
// }
return false;
}
bool isGroupbyColumn(SArray* pFunctionIdList) {
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0);
// int32_t numOfCols = getNumOfColumns(pTableMetaInfo->pTableMeta);
//
// SGroupbyExpr* pGroupbyExpr = &pQueryInfo->groupbyExpr;
// for (int32_t k = 0; k < pGroupbyExpr->numOfGroupCols; ++k) {
// SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, k);
// if (!TSDB_COL_IS_TAG(pIndex->flag) && pIndex->colIndex < numOfCols) { // group by normal columns
// return true;
// }
// }
return false;
}
bool isTopBotQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TS) {
continue;
}
if (f == FUNCTION_TOP || f == FUNCTION_BOTTOM) {
return true;
}
}
return false;
}
bool isTsCompQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
if (num != 1) {
return false;
}
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, 0);
return f == FUNCTION_TS_COMP;
}
bool isTWAQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TWA) {
return true;
}
}
return false;
}
bool isIrateQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_IRATE) {
return true;
}
}
return false;
}
bool isStabledev(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_STDDEV_DST) {
return true;
}
}
return false;
}
bool needReverseScan(SArray* pFunctionIdList) {
assert(0);
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TS || f == FUNCTION_TS_DUMMY || f == FUNCTION_TAG) {
continue;
}
// if ((f == FUNCTION_FIRST || f == FUNCTION_FIRST_DST) && pQueryInfo->order.order == TSDB_ORDER_DESC) {
// return true;
// }
if (f == FUNCTION_LAST || f == FUNCTION_LAST_DST) {
// the scan order to acquire the last result of the specified column
// int32_t order = (int32_t)pExpr->base.param[0].i64;
// if (order != pQueryInfo->order.order) {
// return true;
// }
}
}
return false;
}
bool isSimpleAggregateRv(SArray* pFunctionIdList) {
assert(0);
// if (pQueryInfo->interval.interval > 0 || pQueryInfo->sessionWindow.gap > 0) {
// return false;
// }
//
// if (tscIsDiffDerivQuery(pQueryInfo)) {
// return false;
// }
//
// size_t numOfExprs = getNumOfExprs(pQueryInfo);
// for (int32_t i = 0; i < numOfExprs; ++i) {
// SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
// if (pExpr == NULL) {
// continue;
// }
//
// int32_t functionId = pExpr->base.functionId;
// if (functionId < 0) {
// SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
// if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
// return true;
// }
//
// continue;
// }
//
// if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY) {
// continue;
// }
//
// if ((!IS_MULTIOUTPUT(aAggs[functionId].status)) ||
// (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_TS_COMP)) {
// return true;
// }
// }
return false;
}
bool isBlockDistQuery(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, 0);
return (num == 1 && f == FUNCTION_BLKINFO);
}
bool isTwoStageSTableQuery(SArray* pFunctionIdList, int32_t tableIndex) {
// if (pQueryInfo == NULL) {
// return false;
// }
//
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
// if (pTableMetaInfo == NULL) {
// return false;
// }
//
// if ((pQueryInfo->type & TSDB_QUERY_TYPE_FREE_RESOURCE) == TSDB_QUERY_TYPE_FREE_RESOURCE) {
// return false;
// }
//
// // for ordered projection query, iterate all qualified vnodes sequentially
// if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, tableIndex)) {
// return false;
// }
//
// if (!TSDB_QUERY_HAS_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_STABLE_SUBQUERY) && pQueryInfo->command == TSDB_SQL_SELECT) {
// return UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo);
// }
return false;
}
bool isProjectionQueryOnSTable(SArray* pFunctionIdList, int32_t tableIndex) {
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, tableIndex);
//
// /*
// * In following cases, return false for non ordered project query on super table
// * 1. failed to get tableMeta from server; 2. not a super table; 3. limitation is 0;
// * 4. show queries, instead of a select query
// */
// size_t numOfExprs = getNumOfExprs(pQueryInfo);
// if (pTableMetaInfo == NULL || !UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo) ||
// pQueryInfo->command == TSDB_SQL_RETRIEVE_EMPTY_RESULT || numOfExprs == 0) {
// return false;
// }
//
// for (int32_t i = 0; i < numOfExprs; ++i) {
// int32_t functionId = getExprInfo(pQueryInfo, i)->base.functionId;
//
// if (functionId < 0) {
// SUdfInfo* pUdfInfo = taosArrayGet(pQueryInfo->pUdfInfo, -1 * functionId - 1);
// if (pUdfInfo->funcType == TSDB_UDF_TYPE_AGGREGATE) {
// return false;
// }
//
// continue;
// }
//
// if (functionId != FUNCTION_PRJ &&
// functionId != FUNCTION_TAGPRJ &&
// functionId != FUNCTION_TAG &&
// functionId != FUNCTION_TS &&
// functionId != FUNCTION_ARITHM &&
// functionId != FUNCTION_TS_COMP &&
// functionId != FUNCTION_DIFF &&
// functionId != FUNCTION_DERIVATIVE &&
// functionId != FUNCTION_TS_DUMMY &&
// functionId != FUNCTION_TID_TAG) {
// return false;
// }
// }
return true;
}
bool hasTagValOutput(SArray* pFunctionIdList) {
// size_t numOfExprs = getNumOfExprs(pQueryInfo);
// SExprInfo* pExpr1 = getExprInfo(pQueryInfo, 0);
//
// if (numOfExprs == 1 && pExpr1->base.functionId == FUNCTION_TS_COMP) {
// return true;
// }
//
// for (int32_t i = 0; i < numOfExprs; ++i) {
// SExprInfo* pExpr = getExprInfo(pQueryInfo, i);
// if (pExpr == NULL) {
// continue;
// }
//
// // ts_comp column required the tag value for join filter
// if (TSDB_COL_IS_TAG(pExpr->base.colInfo.flag)) {
// return true;
// }
// }
return false;
}
bool timeWindowInterpoRequired(SArray* pFunctionIdList) {
int32_t num = (int32_t) taosArrayGetSize(pFunctionIdList);
for (int32_t i = 0; i < num; ++i) {
int32_t f = *(int16_t*) taosArrayGet(pFunctionIdList, i);
if (f == FUNCTION_TWA || f == FUNCTION_INTERP) {
return true;
}
}
return false;
}
//SQueryAttrInfo setQueryType(SArray* pFunctionIdList) {
// assert(pFunctionIdList != NULL);
//
//
//}

578
source/libs/function/src/thistogram.c Normal file
View File

@ -0,0 +1,578 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "thistogram.h"
#include "taosdef.h"
#include "taosmsg.h"
#include "tlosertree.h"
/**
*
* implement the histogram and percentile_approx based on the paper:
* Yael Ben-Haim, Elad Tom-Tov. A Streaming Parallel Decision Tree Algorithm,
* The Journal of Machine Learning Research.Volume 11, 3/1/2010 pp.849-872
* https://dl.acm.org/citation.cfm?id=1756034
*
* @data 2018-12-14
* @version 0.1
*
*/
static int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val);
SHistogramInfo* tHistogramCreate(int32_t numOfEntries) {
/* need one redundant slot */
SHistogramInfo* pHisto = malloc(sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfEntries + 1));
#if !defined(USE_ARRAYLIST)
pHisto->pList = SSkipListCreate(MAX_SKIP_LIST_LEVEL, TSDB_DATA_TYPE_DOUBLE, sizeof(double));
SInsertSupporter* pss = malloc(sizeof(SInsertSupporter));
pss->numOfEntries = pHisto->maxEntries;
pss->pSkipList = pHisto->pList;
int32_t ret = tLoserTreeCreate1(&pHisto->pLoserTree, numOfEntries, pss, compare);
pss->pTree = pHisto->pLoserTree;
#endif
return tHistogramCreateFrom(pHisto, numOfEntries);
}
SHistogramInfo* tHistogramCreateFrom(void* pBuf, int32_t numOfBins) {
memset(pBuf, 0, sizeof(SHistogramInfo) + sizeof(SHistBin) * (numOfBins + 1));
SHistogramInfo* pHisto = (SHistogramInfo*)pBuf;
pHisto->elems = (SHistBin*)((char*)pBuf + sizeof(SHistogramInfo));
for(int32_t i = 0; i < numOfBins; ++i) {
pHisto->elems[i].val = -DBL_MAX;
}
pHisto->maxEntries = numOfBins;
pHisto->min = DBL_MAX;
pHisto->max = -DBL_MAX;
return pBuf;
}
int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
if (*pHisto == NULL) {
*pHisto = tHistogramCreate(MAX_HISTOGRAM_BIN);
}
#if defined(USE_ARRAYLIST)
int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL);
if ((*pHisto)->elems[idx].val == val && idx >= 0) {
(*pHisto)->elems[idx].num += 1;
if ((*pHisto)->numOfEntries == 0) {
(*pHisto)->numOfEntries += 1;
}
} else { /* insert a new slot */
if ((*pHisto)->numOfElems >= 1 && idx < (*pHisto)->numOfEntries) {
if (idx > 0) {
assert((*pHisto)->elems[idx - 1].val <= val);
} else {
assert((*pHisto)->elems[idx].val > val);
}
} else if ((*pHisto)->numOfElems > 0) {
assert((*pHisto)->elems[(*pHisto)->numOfEntries].val <= val);
}
histogramCreateBin(*pHisto, idx, val);
}
#else
tSkipListKey key = tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &val, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize);
SHistBin* entry = calloc(1, sizeof(SHistBin));
entry->val = val;
tSkipListNode* pResNode = SSkipListPut((*pHisto)->pList, entry, &key, 0);
SHistBin* pEntry1 = (SHistBin*)pResNode->pData;
pEntry1->index = -1;
tSkipListNode* pLast = NULL;
if (pEntry1->num == 0) { /* it is a new node */
(*pHisto)->numOfEntries += 1;
pEntry1->num += 1;
/* number of entries reaches the upper limitation */
if (pResNode->pForward[0] != NULL) {
/* we need to update the last updated slot in loser tree*/
pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val;
if ((*pHisto)->ordered) {
int32_t lastIndex = (*pHisto)->maxIndex;
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
(*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].pData = pResNode;
pEntry1->index = (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].index;
// update the loser tree
if ((*pHisto)->ordered) {
tLoserTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries);
}
tSkipListKey kx =
tSkipListCreateKey(TSDB_DATA_TYPE_DOUBLE, &(*pHisto)->max, tDataTypes[TSDB_DATA_TYPE_DOUBLE].nSize);
pLast = tSkipListGetOne((*pHisto)->pList, &kx);
}
} else {
/* this node located at the last position of the skiplist, we do not
* update the loser-tree */
pEntry1->delta = DBL_MAX;
pLast = pResNode;
}
if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) {
SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData;
pPrevEntry->delta = val - pPrevEntry->val;
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
if ((*pHisto)->ordered) {
tLoserTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries);
tLoserTreeDisplay(pTree);
}
}
if ((*pHisto)->numOfEntries >= (*pHisto)->maxEntries + 1) {
// set the right value for loser-tree
assert((*pHisto)->pLoserTree != NULL);
if (!(*pHisto)->ordered) {
SSkipListPrint((*pHisto)->pList, 1);
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
tSkipListNode* pHead = (*pHisto)->pList->pHead.pForward[0];
tSkipListNode* p1 = pHead;
printf("\n");
while (p1 != NULL) {
printf("%f\t", ((SHistBin*)(p1->pData))->delta);
p1 = p1->pForward[0];
}
printf("\n");
/* last one in skiplist is ignored */
for (int32_t i = pTree->numOfEntries; i < pTree->totalEntries; ++i) {
pTree->pNode[i].pData = pHead;
pTree->pNode[i].index = i - pTree->numOfEntries;
SHistBin* pBin = (SHistBin*)pHead->pData;
pBin->index = pTree->pNode[i].index;
pHead = pHead->pForward[0];
}
pLast = pHead;
for (int32_t i = 0; i < pTree->numOfEntries; ++i) {
pTree->pNode[i].index = -1;
}
tLoserTreeDisplay(pTree);
for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) {
tLoserTreeAdjust(pTree, i);
}
tLoserTreeDisplay(pTree);
(*pHisto)->ordered = true;
}
printf("delta is:%lf\n", pEntry1->delta);
SSkipListPrint((*pHisto)->pList, 1);
/* the chosen node */
tSkipListNode* pNode = (*pHisto)->pLoserTree->pNode[0].pData;
SHistBin* pEntry = (SHistBin*)pNode->pData;
tSkipListNode* pNext = pNode->pForward[0];
SHistBin* pNextEntry = (SHistBin*)pNext->pData;
assert(pNextEntry->val - pEntry->val == pEntry->delta);
double newVal = (pEntry->val * pEntry->num + pNextEntry->val * pNextEntry->num) / (pEntry->num + pNextEntry->num);
pEntry->val = newVal;
pNode->key.dKey = newVal;
pEntry->num = pEntry->num + pNextEntry->num;
// update delta value in current node
pEntry->delta = (pNextEntry->delta + pNextEntry->val) - pEntry->val;
// reset delta value in the previous node
SHistBin* pPrevEntry = (SHistBin*)pNode->pBackward[0]->pData;
if (pPrevEntry) {
pPrevEntry->delta = pEntry->val - pPrevEntry->val;
}
SLoserTreeInfo* pTree = (*pHisto)->pLoserTree;
if (pNextEntry->index != -1) {
(*pHisto)->maxIndex = pNextEntry->index;
// set the last element in skiplist, of which delta is FLT_MAX;
pTree->pNode[pNextEntry->index + pTree->numOfEntries].pData = pLast;
((SHistBin*)pLast->pData)->index = pNextEntry->index;
int32_t f = pTree->pNode[pNextEntry->index + pTree->numOfEntries].index;
printf("disappear index is:%d\n", f);
}
tLoserTreeAdjust(pTree, pEntry->index + pTree->numOfEntries);
// remove the next node in skiplist
tSkipListRemoveNode((*pHisto)->pList, pNext);
SSkipListPrint((*pHisto)->pList, 1);
tLoserTreeDisplay((*pHisto)->pLoserTree);
} else { // add to heap
if (pResNode->pForward[0] != NULL) {
pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val;
} else {
pEntry1->delta = DBL_MAX;
}
if (pResNode->pBackward[0] != &(*pHisto)->pList->pHead) {
SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData;
pEntry1->delta = val - pPrevEntry->val;
}
printf("delta is:%9lf\n", pEntry1->delta);
}
} else {
SHistBin* pEntry = (SHistBin*)pResNode->pData;
assert(pEntry->val == val);
pEntry->num += 1;
}
#endif
if (val > (*pHisto)->max) {
(*pHisto)->max = val;
}
if (val < (*pHisto)->min) {
(*pHisto)->min = val;
}
(*pHisto)->numOfElems += 1;
return 0;
}
int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) {
int32_t end = len - 1;
int32_t start = 0;
while (start <= end) {
int32_t mid = (end - start) / 2 + start;
if (pEntry[mid].val == val) {
return mid;
}
if (pEntry[mid].val < val) {
start = mid + 1;
} else {
end = mid - 1;
}
}
int32_t ret = start > end ? start : end;
if (ret < 0) {
return 0;
} else {
return ret;
}
}
static void histogramMergeImpl(SHistBin* pHistBin, int32_t* size) {
#if defined(USE_ARRAYLIST)
int32_t oldSize = *size;
double delta = DBL_MAX;
int32_t index = -1;
for (int32_t i = 1; i < oldSize; ++i) {
double d = pHistBin[i].val - pHistBin[i - 1].val;
if (d < delta) {
delta = d;
index = i - 1;
}
}
SHistBin* s1 = &pHistBin[index];
SHistBin* s2 = &pHistBin[index + 1];
double newVal = (s1->val * s1->num + s2->val * s2->num) / (s1->num + s2->num);
s1->val = newVal;
s1->num = s1->num + s2->num;
memmove(&pHistBin[index + 1], &pHistBin[index + 2], (oldSize - index - 2) * sizeof(SHistBin));
(*size) -= 1;
#endif
}
/* optimize this procedure */
int32_t histogramCreateBin(SHistogramInfo* pHisto, int32_t index, double val) {
#if defined(USE_ARRAYLIST)
int32_t remain = pHisto->numOfEntries - index;
if (remain > 0) {
memmove(&pHisto->elems[index + 1], &pHisto->elems[index], sizeof(SHistBin) * remain);
}
assert(index >= 0 && index <= pHisto->maxEntries);
pHisto->elems[index].num = 1;
pHisto->elems[index].val = val;
pHisto->numOfEntries += 1;
/* we need to merge the slot */
if (pHisto->numOfEntries == pHisto->maxEntries + 1) {
histogramMergeImpl(pHisto->elems, &pHisto->numOfEntries);
pHisto->elems[pHisto->maxEntries].val = 0;
pHisto->elems[pHisto->maxEntries].num = 0;
}
#endif
assert(pHisto->numOfEntries <= pHisto->maxEntries);
return 0;
}
void tHistogramDestroy(SHistogramInfo** pHisto) {
if (*pHisto == NULL) {
return;
}
free(*pHisto);
*pHisto = NULL;
}
void tHistogramPrint(SHistogramInfo* pHisto) {
printf("total entries: %d, elements: %"PRId64 "\n", pHisto->numOfEntries, pHisto->numOfElems);
#if defined(USE_ARRAYLIST)
for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
printf("%d: (%f, %" PRId64 ")\n", i + 1, pHisto->elems[i].val, pHisto->elems[i].num);
}
#else
tSkipListNode* pNode = pHisto->pList->pHead.pForward[0];
for (int32_t i = 0; i < pHisto->numOfEntries; ++i) {
SHistBin* pEntry = (SHistBin*)pNode->pData;
printf("%d: (%f, %" PRId64 ")\n", i + 1, pEntry->val, pEntry->num);
pNode = pNode->pForward[0];
}
#endif
}
/**
* Estimated number of points in the interval (−inf,b].
* @param pHisto
* @param v
*/
int64_t tHistogramSum(SHistogramInfo* pHisto, double v) {
#if defined(USE_ARRAYLIST)
int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v);
if (pHisto->elems[slotIdx].val != v) {
slotIdx -= 1;
if (slotIdx < 0) {
slotIdx = 0;
assert(v <= pHisto->elems[slotIdx].val);
} else {
assert(v >= pHisto->elems[slotIdx].val);
if (slotIdx + 1 < pHisto->numOfEntries) {
assert(v < pHisto->elems[slotIdx + 1].val);
}
}
}
double m1 = (double)pHisto->elems[slotIdx].num;
double v1 = pHisto->elems[slotIdx].val;
double m2 = (double)pHisto->elems[slotIdx + 1].num;
double v2 = pHisto->elems[slotIdx + 1].val;
double estNum = m1 + (m2 - m1) * (v - v1) / (v2 - v1);
double s1 = (m1 + estNum) * (v - v1) / (2 * (v2 - v1));
for (int32_t i = 0; i < slotIdx; ++i) {
s1 += pHisto->elems[i].num;
}
s1 = s1 + m1 / 2;
return (int64_t)s1;
#endif
}
double* tHistogramUniform(SHistogramInfo* pHisto, double* ratio, int32_t num) {
#if defined(USE_ARRAYLIST)
double* pVal = malloc(num * sizeof(double));
for (int32_t i = 0; i < num; ++i) {
double numOfElem = (ratio[i] / 100) * pHisto->numOfElems;
if (numOfElem == 0) {
pVal[i] = pHisto->min;
continue;
} else if (numOfElem <= pHisto->elems[0].num) {
pVal[i] = pHisto->elems[0].val;
continue;
} else if (numOfElem == pHisto->numOfElems) {
pVal[i] = pHisto->max;
continue;
}
int32_t j = 0;
int64_t total = 0;
while (j < pHisto->numOfEntries) {
total += pHisto->elems[j].num;
if (total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem) {
break;
}
j += 1;
}
assert(total <= numOfElem && total + pHisto->elems[j + 1].num > numOfElem);
double delta = numOfElem - total;
if (fabs(delta) < FLT_EPSILON) {
pVal[i] = pHisto->elems[j].val;
}
double start = (double)pHisto->elems[j].num;
double range = pHisto->elems[j + 1].num - start;
if (range == 0) {
pVal[i] = (pHisto->elems[j + 1].val - pHisto->elems[j].val) * delta / start + pHisto->elems[j].val;
} else {
double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
pVal[i] = pHisto->elems[j].val + (pHisto->elems[j + 1].val - pHisto->elems[j].val) * factor;
}
}
#else
double* pVal = malloc(num * sizeof(double));
for (int32_t i = 0; i < num; ++i) {
double numOfElem = ratio[i] * pHisto->numOfElems;
tSkipListNode* pFirst = pHisto->pList->pHead.pForward[0];
SHistBin* pEntry = (SHistBin*)pFirst->pData;
if (numOfElem == 0) {
pVal[i] = pHisto->min;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue;
} else if (numOfElem <= pEntry->num) {
pVal[i] = pEntry->val;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue;
} else if (numOfElem == pHisto->numOfElems) {
pVal[i] = pHisto->max;
printf("i/numofSlot: %f, v:%f, %f\n", ratio[i], numOfElem, pVal[i]);
continue;
}
int32_t j = 0;
int64_t total = 0;
SHistBin* pPrev = pEntry;
while (j < pHisto->numOfEntries) {
if (total <= numOfElem && total + pEntry->num > numOfElem) {
break;
}
total += pEntry->num;
pPrev = pEntry;
pFirst = pFirst->pForward[0];
pEntry = (SHistBin*)pFirst->pData;
j += 1;
}
assert(total <= numOfElem && total + pEntry->num > numOfElem);
double delta = numOfElem - total;
if (fabs(delta) < FLT_EPSILON) {
// printf("i/numofSlot: %f, v:%f, %f\n",
// (double)i/numOfSlots, numOfElem, pHisto->elems[j].val);
pVal[i] = pPrev->val;
}
double start = pPrev->num;
double range = pEntry->num - start;
if (range == 0) {
pVal[i] = (pEntry->val - pPrev->val) * delta / start + pPrev->val;
} else {
double factor = (-2 * start + sqrt(4 * start * start - 4 * range * (-2 * delta))) / (2 * range);
pVal[i] = pPrev->val + (pEntry->val - pPrev->val) * factor;
}
// printf("i/numofSlot: %f, v:%f, %f\n", (double)i/numOfSlots,
// numOfElem, val);
}
#endif
return pVal;
}
SHistogramInfo* tHistogramMerge(SHistogramInfo* pHisto1, SHistogramInfo* pHisto2, int32_t numOfEntries) {
SHistogramInfo* pResHistogram = tHistogramCreate(numOfEntries);
// error in histogram info
if (pHisto1->numOfEntries > MAX_HISTOGRAM_BIN || pHisto2->numOfEntries > MAX_HISTOGRAM_BIN) {
return pResHistogram;
}
SHistBin* pHistoBins = calloc(1, sizeof(SHistBin) * (pHisto1->numOfEntries + pHisto2->numOfEntries));
int32_t i = 0, j = 0, k = 0;
while (i < pHisto1->numOfEntries && j < pHisto2->numOfEntries) {
if (pHisto1->elems[i].val < pHisto2->elems[j].val) {
pHistoBins[k++] = pHisto1->elems[i++];
} else if (pHisto1->elems[i].val > pHisto2->elems[j].val) {
pHistoBins[k++] = pHisto2->elems[j++];
} else {
pHistoBins[k] = pHisto1->elems[i++];
pHistoBins[k++].num += pHisto2->elems[j++].num;
}
}
if (i < pHisto1->numOfEntries) {
int32_t remain = pHisto1->numOfEntries - i;
memcpy(&pHistoBins[k], &pHisto1->elems[i], sizeof(SHistBin) * remain);
k += remain;
}
if (j < pHisto2->numOfEntries) {
int32_t remain = pHisto2->numOfEntries - j;
memcpy(&pHistoBins[k], &pHisto2->elems[j], sizeof(SHistBin) * remain);
k += remain;
}
/* update other information */
pResHistogram->numOfElems = pHisto1->numOfElems + pHisto2->numOfElems;
pResHistogram->min = (pHisto1->min < pHisto2->min) ? pHisto1->min : pHisto2->min;
pResHistogram->max = (pHisto1->max > pHisto2->max) ? pHisto1->max : pHisto2->max;
while (k > numOfEntries) {
histogramMergeImpl(pHistoBins, &k);
}
pResHistogram->numOfEntries = k;
memcpy(pResHistogram->elems, pHistoBins, sizeof(SHistBin) * k);
free(pHistoBins);
return pResHistogram;
}

535
source/libs/function/src/tpercentile.c Normal file
View File

@ -0,0 +1,535 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include <tglobal.h>
#include "os.h"
#include "tpercentile.h"
#include "tpagedfile.h"
#include "taosdef.h"
#include "tcompare.h"
#include "ttypes.h"
#define DEFAULT_NUM_OF_SLOT 1024
int32_t getGroupId(int32_t numOfSlots, int32_t slotIndex, int32_t times) {
return (times * numOfSlots) + slotIndex;
}
static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) {
SFilePage *buffer = (SFilePage *)calloc(1, pMemBucket->bytes * pMemBucket->pSlots[slotIdx].info.size + sizeof(SFilePage));
int32_t groupId = getGroupId(pMemBucket->numOfSlots, slotIdx, pMemBucket->times);
SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
int32_t offset = 0;
for(int32_t i = 0; i < list->size; ++i) {
SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i);
SFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes));
offset += (int32_t)(pg->num * pMemBucket->bytes);
}
qsort(buffer->data, pMemBucket->pSlots[slotIdx].info.size, pMemBucket->bytes, pMemBucket->comparFn);
return buffer;
}
static void resetBoundingBox(MinMaxEntry* range, int32_t type) {
if (IS_SIGNED_NUMERIC_TYPE(type)) {
range->i64MaxVal = INT64_MIN;
range->i64MinVal = INT64_MAX;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
range->u64MaxVal = 0;
range->u64MinVal = UINT64_MAX;
} else {
range->dMaxVal = -DBL_MAX;
range->dMinVal = DBL_MAX;
}
}
static int32_t setBoundingBox(MinMaxEntry* range, int16_t type, double minval, double maxval) {
if (minval > maxval) {
return -1;
}
if (IS_SIGNED_NUMERIC_TYPE(type)) {
range->i64MinVal = (int64_t) minval;
range->i64MaxVal = (int64_t) maxval;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)){
range->u64MinVal = (uint64_t) minval;
range->u64MaxVal = (uint64_t) maxval;
} else {
range->dMinVal = minval;
range->dMaxVal = maxval;
}
return 0;
}
static void resetPosInfo(SSlotInfo* pInfo) {
pInfo->size = 0;
pInfo->pageId = -1;
pInfo->data = NULL;
}
double findOnlyResult(tMemBucket *pMemBucket) {
assert(pMemBucket->total == 1);
for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) {
tMemBucketSlot *pSlot = &pMemBucket->pSlots[i];
if (pSlot->info.size == 0) {
continue;
}
int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times);
SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
assert(list->size == 1);
SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0);
SFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
assert(pPage->num == 1);
double v = 0;
GET_TYPED_DATA(v, double, pMemBucket->type, pPage->data);
return v;
}
return 0;
}
int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) {
int64_t v = 0;
GET_TYPED_DATA(v, int64_t, pBucket->type, value);
int32_t index = -1;
if (v > pBucket->range.i64MaxVal || v < pBucket->range.i64MinVal) {
return index;
}
// divide the value range into 1024 buckets
uint64_t span = pBucket->range.i64MaxVal - pBucket->range.i64MinVal;
if (span < pBucket->numOfSlots) {
int64_t delta = v - pBucket->range.i64MinVal;
index = (delta % pBucket->numOfSlots);
} else {
double slotSpan = (double)span / pBucket->numOfSlots;
index = (int32_t)((v - pBucket->range.i64MinVal) / slotSpan);
if (v == pBucket->range.i64MaxVal) {
index -= 1;
}
}
assert(index >= 0 && index < pBucket->numOfSlots);
return index;
}
int32_t tBucketUintHash(tMemBucket *pBucket, const void *value) {
int64_t v = 0;
GET_TYPED_DATA(v, uint64_t, pBucket->type, value);
int32_t index = -1;
if (v > pBucket->range.u64MaxVal || v < pBucket->range.u64MinVal) {
return index;
}
// divide the value range into 1024 buckets
uint64_t span = pBucket->range.u64MaxVal - pBucket->range.u64MinVal;
if (span < pBucket->numOfSlots) {
int64_t delta = v - pBucket->range.u64MinVal;
index = (int32_t) (delta % pBucket->numOfSlots);
} else {
double slotSpan = (double)span / pBucket->numOfSlots;
index = (int32_t)((v - pBucket->range.u64MinVal) / slotSpan);
if (v == pBucket->range.u64MaxVal) {
index -= 1;
}
}
assert(index >= 0 && index < pBucket->numOfSlots);
return index;
}
int32_t tBucketDoubleHash(tMemBucket *pBucket, const void *value) {
double v = 0;
if (pBucket->type == TSDB_DATA_TYPE_FLOAT) {
v = GET_FLOAT_VAL(value);
} else {
v = GET_DOUBLE_VAL(value);
}
int32_t index = -1;
if (v > pBucket->range.dMaxVal || v < pBucket->range.dMinVal) {
return index;
}
// divide a range of [dMinVal, dMaxVal] into 1024 buckets
double span = pBucket->range.dMaxVal - pBucket->range.dMinVal;
if (span < pBucket->numOfSlots) {
int32_t delta = (int32_t)(v - pBucket->range.dMinVal);
index = (delta % pBucket->numOfSlots);
} else {
double slotSpan = span / pBucket->numOfSlots;
index = (int32_t)((v - pBucket->range.dMinVal) / slotSpan);
if (v == pBucket->range.dMaxVal) {
index -= 1;
}
}
assert(index >= 0 && index < pBucket->numOfSlots);
return index;
}
static __perc_hash_func_t getHashFunc(int32_t type) {
if (IS_SIGNED_NUMERIC_TYPE(type)) {
return tBucketIntHash;
} else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
return tBucketUintHash;
} else {
return tBucketDoubleHash;
}
}
static void resetSlotInfo(tMemBucket* pBucket) {
for (int32_t i = 0; i < pBucket->numOfSlots; ++i) {
tMemBucketSlot* pSlot = &pBucket->pSlots[i];
resetBoundingBox(&pSlot->range, pBucket->type);
resetPosInfo(&pSlot->info);
}
}
tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, double maxval) {
tMemBucket *pBucket = (tMemBucket *)calloc(1, sizeof(tMemBucket));
if (pBucket == NULL) {
return NULL;
}
pBucket->numOfSlots = DEFAULT_NUM_OF_SLOT;
pBucket->bufPageSize = DEFAULT_PAGE_SIZE * 4; // 4k per page
pBucket->type = dataType;
pBucket->bytes = nElemSize;
pBucket->total = 0;
pBucket->times = 1;
pBucket->maxCapacity = 200000;
if (setBoundingBox(&pBucket->range, pBucket->type, minval, maxval) != 0) {
// qError("MemBucket:%p, invalid value range: %f-%f", pBucket, minval, maxval);
free(pBucket);
return NULL;
}
pBucket->elemPerPage = (pBucket->bufPageSize - sizeof(SFilePage))/pBucket->bytes;
pBucket->comparFn = getKeyComparFunc(pBucket->type, TSDB_ORDER_ASC);
pBucket->hashFunc = getHashFunc(pBucket->type);
if (pBucket->hashFunc == NULL) {
// qError("MemBucket:%p, not support data type %d, failed", pBucket, pBucket->type);
free(pBucket);
return NULL;
}
pBucket->pSlots = (tMemBucketSlot *)calloc(pBucket->numOfSlots, sizeof(tMemBucketSlot));
if (pBucket->pSlots == NULL) {
free(pBucket);
return NULL;
}
resetSlotInfo(pBucket);
int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir);
if (ret != 0) {
tMemBucketDestroy(pBucket);
return NULL;
}
// qDebug("MemBucket:%p, elem size:%d", pBucket, pBucket->bytes);
return pBucket;
}
void tMemBucketDestroy(tMemBucket *pBucket) {
if (pBucket == NULL) {
return;
}
destroyResultBuf(pBucket->pBuffer);
tfree(pBucket->pSlots);
tfree(pBucket);
}
void tMemBucketUpdateBoundingBox(MinMaxEntry *r, const char *data, int32_t dataType) {
if (IS_SIGNED_NUMERIC_TYPE(dataType)) {
int64_t v = 0;
GET_TYPED_DATA(v, int64_t, dataType, data);
if (r->i64MinVal > v) {
r->i64MinVal = v;
}
if (r->i64MaxVal < v) {
r->i64MaxVal = v;
}
} else if (IS_UNSIGNED_NUMERIC_TYPE(dataType)) {
uint64_t v = 0;
GET_TYPED_DATA(v, uint64_t, dataType, data);
if (r->i64MinVal > v) {
r->i64MinVal = v;
}
if (r->i64MaxVal < v) {
r->i64MaxVal = v;
}
} else if (IS_FLOAT_TYPE(dataType)) {
double v = 0;
GET_TYPED_DATA(v, double, dataType, data);
if (r->dMinVal > v) {
r->dMinVal = v;
}
if (r->dMaxVal < v) {
r->dMaxVal = v;
}
} else {
assert(0);
}
}
/*
* in memory bucket, we only accept data array list
*/
int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) {
assert(pBucket != NULL && data != NULL && size > 0);
int32_t count = 0;
int32_t bytes = pBucket->bytes;
for (int32_t i = 0; i < size; ++i) {
char *d = (char *) data + i * bytes;
int32_t index = (pBucket->hashFunc)(pBucket, d);
if (index < 0) {
continue;
}
count += 1;
tMemBucketSlot *pSlot = &pBucket->pSlots[index];
tMemBucketUpdateBoundingBox(&pSlot->range, d, pBucket->type);
// ensure available memory pages to allocate
int32_t groupId = getGroupId(pBucket->numOfSlots, index, pBucket->times);
int32_t pageId = -1;
if (pSlot->info.data == NULL || pSlot->info.data->num >= pBucket->elemPerPage) {
if (pSlot->info.data != NULL) {
assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0);
// keep the pointer in memory
releaseResBufPage(pBucket->pBuffer, pSlot->info.data);
pSlot->info.data = NULL;
}
pSlot->info.data = getNewDataBuf(pBucket->pBuffer, groupId, &pageId);
pSlot->info.pageId = pageId;
}
memcpy(pSlot->info.data->data + pSlot->info.data->num * pBucket->bytes, d, pBucket->bytes);
pSlot->info.data->num += 1;
pSlot->info.size += 1;
}
pBucket->total += count;
return 0;
}
////////////////////////////////////////////////////////////////////////////////////////////
/*
*
* now, we need to find the minimum value of the next slot for
* interpolating the percentile value
* j is the last slot of current segment, we need to get the first
* slot of the next segment.
*/
static MinMaxEntry getMinMaxEntryOfNextSlotWithData(tMemBucket *pMemBucket, int32_t slotIdx) {
int32_t j = slotIdx + 1;
while (j < pMemBucket->numOfSlots && (pMemBucket->pSlots[j].info.size == 0)) {
++j;
}
assert(j < pMemBucket->numOfSlots);
return pMemBucket->pSlots[j].range;
}
static bool isIdenticalData(tMemBucket *pMemBucket, int32_t index);
static double getIdenticalDataVal(tMemBucket* pMemBucket, int32_t slotIndex) {
assert(isIdenticalData(pMemBucket, slotIndex));
tMemBucketSlot *pSlot = &pMemBucket->pSlots[slotIndex];
double finalResult = 0.0;
if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
finalResult = (double) pSlot->range.i64MinVal;
} else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
finalResult = (double) pSlot->range.u64MinVal;
} else {
finalResult = (double) pSlot->range.dMinVal;
}
return finalResult;
}
double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) {
int32_t num = 0;
for (int32_t i = 0; i < pMemBucket->numOfSlots; ++i) {
tMemBucketSlot *pSlot = &pMemBucket->pSlots[i];
if (pSlot->info.size == 0) {
continue;
}
// required value in current slot
if (num < (count + 1) && num + pSlot->info.size >= (count + 1)) {
if (pSlot->info.size + num == (count + 1)) {
/*
* now, we need to find the minimum value of the next slot for interpolating the percentile value
* j is the last slot of current segment, we need to get the first slot of the next segment.
*/
MinMaxEntry next = getMinMaxEntryOfNextSlotWithData(pMemBucket, i);
double maxOfThisSlot = 0;
double minOfNextSlot = 0;
if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
maxOfThisSlot = (double) pSlot->range.i64MaxVal;
minOfNextSlot = (double) next.i64MinVal;
} else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
maxOfThisSlot = (double) pSlot->range.u64MaxVal;
minOfNextSlot = (double) next.u64MinVal;
} else {
maxOfThisSlot = (double) pSlot->range.dMaxVal;
minOfNextSlot = (double) next.dMinVal;
}
assert(minOfNextSlot > maxOfThisSlot);
double val = (1 - fraction) * maxOfThisSlot + fraction * minOfNextSlot;
return val;
}
if (pSlot->info.size <= pMemBucket->maxCapacity) {
// data in buffer and file are merged together to be processed.
SFilePage *buffer = loadDataFromFilePage(pMemBucket, i);
int32_t currentIdx = count - num;
char *thisVal = buffer->data + pMemBucket->bytes * currentIdx;
char *nextVal = thisVal + pMemBucket->bytes;
double td = 1.0, nd = 1.0;
GET_TYPED_DATA(td, double, pMemBucket->type, thisVal);
GET_TYPED_DATA(nd, double, pMemBucket->type, nextVal);
double val = (1 - fraction) * td + fraction * nd;
tfree(buffer);
return val;
} else { // incur a second round bucket split
if (isIdenticalData(pMemBucket, i)) {
return getIdenticalDataVal(pMemBucket, i);
}
// try next round
pMemBucket->times += 1;
// qDebug("MemBucket:%p, start next round data bucketing, time:%d", pMemBucket, pMemBucket->times);
pMemBucket->range = pSlot->range;
pMemBucket->total = 0;
resetSlotInfo(pMemBucket);
int32_t groupId = getGroupId(pMemBucket->numOfSlots, i, pMemBucket->times - 1);
SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId);
assert(list->size > 0);
for (int32_t f = 0; f < list->size; ++f) {
SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f);
SFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId);
tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num);
releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo);
}
return getPercentileImpl(pMemBucket, count - num, fraction);
}
} else {
num += pSlot->info.size;
}
}
return 0;
}
double getPercentile(tMemBucket *pMemBucket, double percent) {
if (pMemBucket->total == 0) {
return 0.0;
}
// if only one elements exists, return it
if (pMemBucket->total == 1) {
return findOnlyResult(pMemBucket);
}
percent = fabs(percent);
// find the min/max value, no need to scan all data in bucket
if (fabs(percent - 100.0) < DBL_EPSILON || (percent < DBL_EPSILON)) {
MinMaxEntry* pRange = &pMemBucket->range;
if (IS_SIGNED_NUMERIC_TYPE(pMemBucket->type)) {
double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->i64MaxVal : pRange->i64MinVal);
return v;
} else if (IS_UNSIGNED_NUMERIC_TYPE(pMemBucket->type)) {
double v = (double)(fabs(percent - 100) < DBL_EPSILON ? pRange->u64MaxVal : pRange->u64MinVal);
return v;
} else {
return fabs(percent - 100) < DBL_EPSILON? pRange->dMaxVal:pRange->dMinVal;
}
}
double percentVal = (percent * (pMemBucket->total - 1)) / ((double)100.0);
// do put data by using buckets
int32_t orderIdx = (int32_t)percentVal;
return getPercentileImpl(pMemBucket, orderIdx, percentVal - orderIdx);
}
/*
* check if data in one slot are all identical only need to compare with the bounding box
*/
bool isIdenticalData(tMemBucket *pMemBucket, int32_t index) {
tMemBucketSlot *pSeg = &pMemBucket->pSlots[index];
if (IS_FLOAT_TYPE(pMemBucket->type)) {
return fabs(pSeg->range.dMaxVal - pSeg->range.dMinVal) < DBL_EPSILON;
} else {
return pSeg->range.i64MinVal == pSeg->range.i64MaxVal;
}
}

10
source/libs/function/src/tscalarfunction.c Normal file
View File

@ -0,0 +1,10 @@
#include "tscalarfunction.h"
SScalarFunctionInfo scalarFunc[1] = {
{
},
};

1108
source/libs/function/src/ttszip.c Normal file
View File

File diff suppressed because it is too large Load Diff

6
source/libs/parser/CMakeLists.txt
View File

@ -8,5 +8,7 @@ target_include_directories(
target_link_libraries(
parser
PRIVATE os util common catalog transport
)
PRIVATE os util common catalog function transport
)
ADD_SUBDIRECTORY(test)

14
source/libs/parser/inc/astGenerator.h
View File

@ -42,12 +42,6 @@ enum SQL_NODE_FROM_TYPE {
SQL_NODE_FROM_TABLELIST = 2,
};
//enum SQL_EXPR_FLAG {
// EXPR_FLAG_TS_ERROR = 1,
// EXPR_FLAG_NS_TIMESTAMP = 2,
// EXPR_FLAG_TIMESTAMP_VAR = 3,
//};
extern char tTokenTypeSwitcher[13];
#define toTSDBType(x) \
@ -239,7 +233,7 @@ typedef struct tSqlExpr {
// The complete string of the function(col, param), and the function name is kept in exprToken
struct {
SToken operand;
struct SArray *paramList; // function parameters list
struct SArray *paramList; // function parameters list
} Expr;
SToken columnName; // table column info
@ -252,6 +246,7 @@ typedef struct tSqlExpr {
// used in select clause. select <SArray> from xxx
typedef struct tSqlExprItem {
tSqlExpr *pNode; // The list of expressions
int32_t functionId;
char *aliasName; // alias name, null-terminated string
bool distinct;
} tSqlExprItem;
@ -267,7 +262,7 @@ SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *p
// sql expr leaf node
tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType);
tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SToken *pFuncToken, SToken *endToken, int32_t optType);
SArray * tAppendFuncName(SArray *pList, SToken *pToken);
SArray * tRecordFuncName(SArray *pList, SToken *pToken);
tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType);
tSqlExpr *tSqlExprClone(tSqlExpr *pSrc);
@ -277,6 +272,7 @@ bool tSqlExprIsParentOfLeaf(tSqlExpr *pExpr);
void tSqlExprDestroy(tSqlExpr *pExpr);
SArray * tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SToken *pDistinct, SToken *pToken);
void tSqlExprListDestroy(SArray *pList);
void tSqlExprEvaluate(tSqlExpr* pExpr);
SSqlNode *tSetQuerySqlNode(SToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere,
SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps,
@ -299,6 +295,8 @@ SArray *appendSelectClause(SArray *pList, void *pSubclause);
void setCreatedTableName(SSqlInfo *pInfo, SToken *pTableNameToken, SToken *pIfNotExists);
void* destroyCreateTableSql(SCreateTableSql* pCreate);
void setDropFuncInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken);
void setCreateFuncInfo(SSqlInfo *pInfo, int32_t type, SToken *pName, SToken *pPath, SField *output, SToken* bufSize, int32_t funcType);
void SqlInfoDestroy(SSqlInfo *pInfo);

43
source/libs/parser/inc/parserInt.h
View File

@ -35,6 +35,21 @@ typedef struct SInsertStmtInfo {
char *sql; // current sql statement position
} SInsertStmtInfo;
typedef struct SInternalField {
TAOS_FIELD field;
bool visible;
SExprInfo *pExpr;
} SInternalField;
typedef struct SMsgBuf {
int32_t len;
char *buf;
} SMsgBuf;
void clearTableMetaInfo(STableMetaInfo* pTableMetaInfo);
void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t id);
/**
* Validate the sql info, according to the corresponding metadata info from catalog.
* @param pCatalog
@ -44,15 +59,33 @@ typedef struct SInsertStmtInfo {
* @param msg
* @return
*/
int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg);
int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg, int32_t msgLen);
/**
*
* @param pSqlNode
* @param pMetaInfo
* Evaluate the numeric and timestamp arithmetic expression in the WHERE clause.
* @param pNode
* @param tsPrecision
* @param msg
* @param msgBufLen
* @return
*/
int32_t qParserExtractRequestedMetaInfo(const struct SSqlNode* pSqlNode, SMetaReq* pMetaInfo);
int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf);
int32_t validateSqlNode(SSqlNode* pSqlNode, SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf);
void initQueryInfo(SQueryStmtInfo* pQueryInfo);
int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf);
/**
* Extract request meta info from the sql statement
* @param pSqlInfo
* @param pMetaInfo
* @param msg
* @param msgBufLen
* @return
*/
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen);
#ifdef __cplusplus
}

41
source/libs/parser/inc/parserUtil.h
View File

@ -20,6 +20,47 @@
extern "C" {
#endif
#include "os.h"
#include "ttoken.h"
#include "parserInt.h"
#define UTIL_TABLE_IS_SUPER_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_SUPER_TABLE))
#define UTIL_TABLE_IS_CHILD_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_CHILD_TABLE))
#define UTIL_TABLE_IS_NORMAL_TABLE(metaInfo) \
(!(UTIL_TABLE_IS_SUPER_TABLE(metaInfo) || UTIL_TABLE_IS_CHILD_TABLE(metaInfo)))
#define UTIL_TABLE_IS_TMP_TABLE(metaInfo) \
(((metaInfo)->pTableMeta != NULL) && ((metaInfo)->pTableMeta->tableType == TSDB_TEMP_TABLE))
TAOS_FIELD createField(const SSchema* pSchema);
SSchema createSchema(uint8_t type, int16_t bytes, int16_t colId, const char* name);
SInternalField* insertFieldInfo(SFieldInfo* pFieldInfo, int32_t index, SSchema* field);
int32_t getNumOfFields(SFieldInfo* pFieldInfo);
SInternalField* getInternalField(SFieldInfo* pFieldInfo, int32_t index);
int32_t parserValidateIdToken(SToken* pToken);
int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
int32_t buildSyntaxErrMsg(char* dst, int32_t dstBufLen, const char* additionalInfo, const char* sourceStr);
int32_t createProjectionExpr(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SExprInfo*** pExpr, int32_t* num);
STableMetaInfo* addEmptyMetaInfo(SQueryStmtInfo* pQueryInfo);
void columnListCopyAll(SArray* dst, const SArray* src);
void columnListDestroy(SArray* pColumnList);
SColumn* columnListInsert(SArray* pColumnList, int32_t columnIndex, uint64_t uid, SSchema* pSchema);
SColumn* insertPrimaryTsColumn(SArray* pColumnList, uint64_t tableUid);
void cleanupTagCond(STagCond* pTagCond);
void cleanupColumnCond(SArray** pCond);
uint32_t convertRelationalOperator(SToken *pToken);
#ifdef __cplusplus
}

53
source/libs/parser/inc/queryInfoUtil.h Normal file
View File

@ -0,0 +1,53 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_QUERYINFOUTIL_H
#define TDENGINE_QUERYINFOUTIL_H
#ifdef __cplusplus
extern "C" {
#endif
#include "parserInt.h"
SSchema* getTbnameColumnSchema();
int32_t getNumOfColumns(const STableMeta* pTableMeta);
int32_t getNumOfTags(const STableMeta* pTableMeta);
SSchema *getTableColumnSchema(const STableMeta *pTableMeta);
SSchema *getTableTagSchema(const STableMeta* pTableMeta);
SSchema *getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex);
size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo);
//SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, struct tExprNode* pParamExpr, SSchema* pResSchema, int16_t interSize);
SExprInfo* createBinaryExprInfo(struct tExprNode* pNode, SSchema* pResSchema);
void destroyExprInfoList();
void addExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index, SExprInfo* pExprInfo);
void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize);
SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index);
int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy);
void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes);
void cleanupFieldInfo(SFieldInfo* pFieldInfo);
STableComInfo getTableInfo(const STableMeta* pTableMeta);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_QUERYINFOUTIL_H

12
source/libs/parser/inc/sql.y
View File

@ -325,7 +325,7 @@ alter_db_optr(Y) ::= alter_db_optr(Z) cachelast(X). { Y = Z; Y.cachelast = str
alter_topic_optr(Y) ::= alter_db_optr(Z). { Y = Z; Y.dbType = TSDB_DB_TYPE_TOPIC; }
alter_topic_optr(Y) ::= alter_topic_optr(Z) partitions(X). { Y = Z; Y.partitions = strtol(X.z, NULL, 10); }
%type typename {TAOS_FIELD}
%type typename {SField}
typename(A) ::= ids(X). {
X.type = 0;
tSetColumnType (&A, &X);
@ -425,11 +425,11 @@ create_table_args(A) ::= ifnotexists(U) ids(V) cpxName(Z) AS select(S). {
setCreatedTableName(pInfo, &V, &U);
}
%type column{TAOS_FIELD}
%type column{SField}
%type columnlist{SArray*}
%destructor columnlist {taosArrayDestroy($$);}
columnlist(A) ::= columnlist(X) COMMA column(Y). {taosArrayPush(X, &Y); A = X; }
columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(A, &X);}
columnlist(A) ::= column(X). {A = taosArrayInit(4, sizeof(SField)); taosArrayPush(A, &X);}
// The information used for a column is the name and type of column:
// tinyint smallint int bigint float double bool timestamp binary(x) nchar(x)
@ -601,7 +601,7 @@ fill_opt(N) ::= FILL LP ID(Y) COMMA tagitemlist(X) RP. {
toTSDBType(Y.type);
taosVariantCreate(&A, Y.z, Y.n, Y.type);
tVariantListInsert(X, &A, -1, 0);
tListItemInsert(X, &A, -1, 0);
N = X;
}
@ -719,10 +719,10 @@ expr(A) ::= BOOL(X). { A = tSqlExprCreateIdValue(&X, TK_BOOL);}
expr(A) ::= NULL(X). { A = tSqlExprCreateIdValue(&X, TK_NULL);}
// ordinary functions: min(x), max(x), top(k, 20)
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tAppendFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
expr(A) ::= ID(X) LP exprlist(Y) RP(E). { tRecordFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(Y, &X, &E, X.type); }
// for parsing sql functions with wildcard for parameters. e.g., count(*)/first(*)/last(*) operation
expr(A) ::= ID(X) LP STAR RP(Y). { tAppendFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
expr(A) ::= ID(X) LP STAR RP(Y). { tRecordFuncName(pInfo->funcs, &X); A = tSqlExprCreateFunction(NULL, &X, &Y, X.type); }
// is (not) null expression
expr(A) ::= expr(X) IS NULL. {A = tSqlExprCreate(X, NULL, TK_ISNULL);}

302
source/libs/parser/src/astGenerator.c
View File

@ -18,52 +18,6 @@
#include "astGenerator.h"
#include "tmsgtype.h"
int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) {
errno = 0;
int32_t ret = 0;
char* endPtr = NULL;
if (type == TK_FLOAT) {
double v = strtod(z, &endPtr);
if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) {
ret = -1;
} else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) {
ret = -1;
} else {
*value = (int64_t) round(v);
}
errno = 0;
return ret;
}
int32_t radix = 10;
if (type == TK_HEX) {
radix = 16;
} else if (type == TK_BIN) {
radix = 2;
}
// the string may be overflow according to errno
if (!issigned) {
const char *p = z;
while(*p != 0 && *p == ' ') p++;
if (*p != 0 && *p == '-') { return -1;}
*value = strtoull(z, &endPtr, radix);
} else {
*value = strtoll(z, &endPtr, radix);
}
// not a valid integer number, return error
if (endPtr - z != n || errno == ERANGE) {
ret = -1;
}
errno = 0;
return ret;
}
SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder) {
if (pList == NULL) {
pList = taosArrayInit(4, sizeof(SListItem));
@ -173,7 +127,6 @@ SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *p
}
// sql expr leaf node
// todo Evalute the value during the validation process of AST.
tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType) {
tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr));
@ -189,34 +142,10 @@ tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType) {
pSqlExpr->tokenId = optrType;
pSqlExpr->type = SQL_NODE_VALUE;
} else if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) {
// if (pToken) {
// toTSDBType(pToken->type);
// tVariantCreate(&pSqlExpr->value, pToken);
// }
pSqlExpr->tokenId = optrType;
pSqlExpr->type = SQL_NODE_VALUE;
} else if (optrType == TK_NOW) {
// use nanosecond by default TODO set value after getting database precision
// pSqlExpr->value.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO);
// pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT;
pSqlExpr->tokenId = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond
pSqlExpr->type = SQL_NODE_VALUE;
// pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP;
} else if (optrType == TK_VARIABLE) {
// use nanosecond by default
// TODO set value after getting database precision
// if (pToken) {
// char unit = 0;
// int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, &unit, TSDB_TIME_PRECISION_NANO);
// if (ret != TSDB_CODE_SUCCESS) {
// terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
// }
// }
// pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP;
// pSqlExpr->flags |= 1 << EXPR_FLAG_TIMESTAMP_VAR;
// pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT;
pSqlExpr->tokenId = TK_TIMESTAMP;
} else if (optrType == TK_NOW || optrType == TK_VARIABLE) {
pSqlExpr->tokenId = optrType; // TK_TIMESTAMP used to denote this is a timestamp value
pSqlExpr->type = SQL_NODE_VALUE;
} else {
// Here it must be the column name (tk_id) if it is not a number or string.
@ -252,7 +181,7 @@ tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SToken *pFuncToken, SToken *end
return pExpr;
}
SArray *tAppendFuncName(SArray *pList, SToken *pToken) {
SArray *tRecordFuncName(SArray *pList, SToken *pToken) {
assert(pList != NULL && pToken != NULL);
taosArrayPush(pList, pToken);
return pList;
@ -269,88 +198,7 @@ tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) {
pExpr->exprToken.type = pLeft->exprToken.type;
}
if ((pLeft != NULL && pRight != NULL) &&
(optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) {
/*
* if a exprToken is noted as the TK_TIMESTAMP, the time precision is microsecond
* Otherwise, the time precision is adaptive, determined by the time precision from databases.
*/
if ((pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_INTEGER) ||
(pLeft->tokenId == TK_TIMESTAMP && pRight->tokenId == TK_TIMESTAMP)) {
pExpr->value.nType = TSDB_DATA_TYPE_BIGINT;
pExpr->tokenId = pLeft->tokenId;
pExpr->type = SQL_NODE_VALUE;
switch (optrType) {
case TK_PLUS: {
pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64;
break;
}
case TK_MINUS: {
pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64;
break;
}
case TK_STAR: {
pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64;
break;
}
case TK_DIVIDE: {
pExpr->tokenId = TK_FLOAT;
pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE;
pExpr->value.d = (double)pLeft->value.i64 / pRight->value.i64;
break;
}
case TK_REM: {
pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64;
break;
}
}
tSqlExprDestroy(pLeft);
tSqlExprDestroy(pRight);
} else if ((pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_INTEGER) ||
(pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_FLOAT) ||
(pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_FLOAT)) {
pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE;
pExpr->tokenId = TK_FLOAT;
pExpr->type = SQL_NODE_VALUE;
double left = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.d : pLeft->value.i64;
double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.d : pRight->value.i64;
switch (optrType) {
case TK_PLUS: {
pExpr->value.d = left + right;
break;
}
case TK_MINUS: {
pExpr->value.d = left - right;
break;
}
case TK_STAR: {
pExpr->value.d = left * right;
break;
}
case TK_DIVIDE: {
pExpr->value.d = left / right;
break;
}
case TK_REM: {
pExpr->value.d = left - ((int64_t)(left / right)) * right;
break;
}
}
tSqlExprDestroy(pLeft);
tSqlExprDestroy(pRight);
} else {
pExpr->tokenId = optrType;
pExpr->pLeft = pLeft;
pExpr->pRight = pRight;
}
} else if (optrType == TK_IN) {
if (optrType == TK_IN) {
pExpr->tokenId = optrType;
pExpr->pLeft = pLeft;
@ -502,6 +350,105 @@ void tSqlExprListDestroy(SArray *pList) {
taosArrayDestroyEx(pList, freeExprElem);
}
void tSqlExprEvaluate(tSqlExpr* pExpr) {
tSqlExpr *pLeft = pExpr->pLeft;
tSqlExpr *pRight = pExpr->pRight;
if (pLeft == NULL || pRight == NULL) {
return;
}
int32_t optrType = pExpr->tokenId;
if ((optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE ||
optrType == TK_REM)) {
/*
* if a exprToken is noted as the TK_TIMESTAMP, the time precision is microsecond
* Otherwise, the time precision is adaptive, determined by the time precision from databases.
*/
int32_t ltoken = pLeft->tokenId;
int32_t rtoken = pRight->tokenId;
if ((ltoken == TK_INTEGER && rtoken == TK_INTEGER) || (ltoken == TK_TIMESTAMP && rtoken == TK_TIMESTAMP)) {
pExpr->value.nType = TSDB_DATA_TYPE_BIGINT;
pExpr->tokenId = ltoken;
pExpr->type = SQL_NODE_VALUE;
switch (optrType) {
case TK_PLUS: {
pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64;
break;
}
case TK_MINUS: {
pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64;
break;
}
case TK_STAR: {
pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64;
break;
}
case TK_DIVIDE: {
pExpr->tokenId = TK_FLOAT;
pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE;
pExpr->value.d = (double)pLeft->value.i64 / pRight->value.i64;
break;
}
case TK_REM: {
pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64;
break;
}
default:
assert(0);
}
tSqlExprDestroy(pLeft);
tSqlExprDestroy(pRight);
pExpr->pLeft = NULL;
pExpr->pRight = NULL;
} else if ((ltoken == TK_FLOAT && rtoken == TK_INTEGER) || (ltoken == TK_INTEGER && rtoken == TK_FLOAT) ||
(ltoken == TK_FLOAT && rtoken == TK_FLOAT)) {
pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE;
pExpr->tokenId = TK_FLOAT;
pExpr->type = SQL_NODE_VALUE;
double left = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.d : pLeft->value.i64;
double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.d : pRight->value.i64;
switch (optrType) {
case TK_PLUS: {
pExpr->value.d = left + right;
break;
}
case TK_MINUS: {
pExpr->value.d = left - right;
break;
}
case TK_STAR: {
pExpr->value.d = left * right;
break;
}
case TK_DIVIDE: {
pExpr->value.d = left / right;
break;
}
case TK_REM: {
pExpr->value.d = left - ((int64_t)(left / right)) * right;
break;
}
default:
assert(0);
}
tSqlExprDestroy(pLeft);
tSqlExprDestroy(pRight);
pExpr->pLeft = NULL;
pExpr->pRight = NULL;
}
}
}
SSqlNode *tSetQuerySqlNode(SToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere,
SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval,
SSessionWindowVal *pSession, SWindowStateVal *pWindowStateVal, SToken *pSliding, SArray *pFill, SLimit *pLimit,
@ -681,6 +628,7 @@ SAlterTableInfo *tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray *p
return pAlterTable;
}
SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, SArray *pTagVals, SToken *pToken, SToken* igExists) {
SCreatedTableInfo info;
memset(&info, 0, sizeof(SCreatedTableInfo));
@ -762,6 +710,7 @@ SSqlInfo* setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, in
return pInfo;
}
SArray* setSubclause(SArray* pList, void *pSqlNode) {
if (pList == NULL) {
pList = taosArrayInit(1, POINTER_BYTES);
@ -770,6 +719,7 @@ SArray* setSubclause(SArray* pList, void *pSqlNode) {
taosArrayPush(pList, &pSqlNode);
return pList;
}
SArray* appendSelectClause(SArray *pList, void *pSubclause) {
taosArrayPush(pList, &pSubclause);
return pList;
@ -792,6 +742,34 @@ void* destroyCreateTableSql(SCreateTableSql* pCreate) {
return NULL;
}
void setDropFuncInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = (SMiscInfo *)calloc(1, sizeof(SMiscInfo));
pInfo->pMiscInfo->a = taosArrayInit(4, sizeof(SToken));
}
taosArrayPush(pInfo->pMiscInfo->a, pToken);
}
void setCreateFuncInfo(SSqlInfo *pInfo, int32_t type, SToken *pName, SToken *pPath, SField *output, SToken* bufSize, int32_t funcType) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo));
}
pInfo->pMiscInfo->funcOpt.name = *pName;
pInfo->pMiscInfo->funcOpt.path = *pPath;
pInfo->pMiscInfo->funcOpt.output = *output;
pInfo->pMiscInfo->funcOpt.type = funcType;
if (bufSize->n > 0) {
pInfo->pMiscInfo->funcOpt.bufSize = strtol(bufSize->z, NULL, 10);
} else {
pInfo->pMiscInfo->funcOpt.bufSize = 0;
}
}
void SqlInfoDestroy(SSqlInfo *pInfo) {
if (pInfo == NULL) return;;
taosArrayDestroy(pInfo->funcs);
@ -840,6 +818,7 @@ void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParam, ...) {
va_end(va);
}
void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken, SToken* existsCheck, int16_t dbType, int16_t tableType) {
pInfo->type = type;
@ -854,6 +833,7 @@ void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken, SToken* e
pInfo->pMiscInfo->dbType = dbType;
pInfo->pMiscInfo->tableType = tableType;
}
void setShowOptions(SSqlInfo *pInfo, int32_t type, SToken* prefix, SToken* pPatterns) {
if (pInfo->pMiscInfo == NULL) {
pInfo->pMiscInfo = calloc(1, sizeof(SMiscInfo));
@ -903,6 +883,7 @@ void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SToken *pName, SToken *pPwd
pInfo->pMiscInfo->user.passwd = *pPwd;
}
}
void setCreateUserSql(SSqlInfo *pInfo, SToken *pName, SToken *pPasswd) {
pInfo->type = TSDB_SQL_CREATE_USER;
if (pInfo->pMiscInfo == NULL) {
@ -914,6 +895,7 @@ void setCreateUserSql(SSqlInfo *pInfo, SToken *pName, SToken *pPasswd) {
pInfo->pMiscInfo->user.user = *pName;
pInfo->pMiscInfo->user.passwd = *pPasswd;
}
void setKillSql(SSqlInfo *pInfo, int32_t type, SToken *id) {
pInfo->type = type;
if (pInfo->pMiscInfo == NULL) {
@ -980,7 +962,13 @@ void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo) {
memset(&pDBInfo->precision, 0, sizeof(SToken));
}
void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo);
void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo) {
setDefaultCreateDbOption(pDBInfo);
pDBInfo->dbType = TSDB_DB_TYPE_TOPIC;
pDBInfo->partitions = TSDB_DEFAULT_DB_PARTITON_OPTION;
}
// prefix show db.tables;
void tSetDbName(SToken *pCpxName, SToken *pDb) {

2747
source/libs/parser/src/astValidate.c
View File

File diff suppressed because it is too large Load Diff

161
source/libs/parser/src/parser.c
View File

@ -13,33 +13,37 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "parserInt.h"
#include "ttoken.h"
#include "astGenerator.h"
#include "parserInt.h"
#include "parserUtil.h"
#include "ttoken.h"
#include "function.h"
bool qIsInsertSql(const char* pStr, size_t length) {
return false;
int32_t index = 0;
do {
SToken t0 = tStrGetToken((char*) pStr, &index, false);
if (t0.type != TK_LP) {
return t0.type == TK_INSERT || t0.type == TK_IMPORT;
}
} while (1);
}
int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg, int32_t msgLen) {
*pQueryInfo = calloc(1, sizeof(SQueryStmtInfo));
if (*pQueryInfo == NULL) {
return -1; // set correct error code.
return TSDB_CODE_TSC_OUT_OF_MEMORY; // set correct error code.
}
SSqlInfo info = doGenerateAST(pStr);
if (!info.valid) {
strcpy(msg, info.msg);
return -1; // set correct error code.
strncpy(msg, info.msg, msgLen);
return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
}
struct SCatalog* pCatalog = getCatalogHandle(NULL);
int32_t code = qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg);
if (code != 0) {
return code;
}
return 0;
return qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg, msgLen);
}
int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg, int32_t msgLen) {
@ -50,6 +54,135 @@ int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) {
return 0;
}
int32_t qParserExtractRequestedMetaInfo(const struct SSqlNode* pSqlNode, SMetaReq* pMetaInfo) {
return 0;
static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SMsgBuf* pMsgBuf);
static int32_t tnameComparFn(const void* p1, const void* p2) {
SName* pn1 = (SName*)p1;
SName* pn2 = (SName*)p2;
int32_t ret = strncmp(pn1->acctId, pn2->acctId, tListLen(pn1->acctId));
if (ret != 0) {
return ret > 0? 1:-1;
} else {
ret = strncmp(pn1->dbname, pn2->dbname, tListLen(pn1->dbname));
if (ret != 0) {
return ret > 0? 1:-1;
} else {
ret = strncmp(pn1->tname, pn2->tname, tListLen(pn1->tname));
if (ret != 0) {
return ret > 0? 1:-1;
} else {
return 0;
}
}
}
}
static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SMsgBuf* pMsgBuf) {
int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list);
for (int32_t j = 0; j < numOfSub; ++j) {
SRelElementPair* sub = taosArrayGet(pSqlNode->from->list, j);
int32_t num = (int32_t)taosArrayGetSize(sub->pSubquery);
for (int32_t i = 0; i < num; ++i) {
SSqlNode* p = taosArrayGetP(sub->pSubquery, i);
if (p->from->type == SQL_NODE_FROM_TABLELIST) {
int32_t code = getTableNameFromSqlNode(p, tableNameList, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} else {
getTableNameFromSubquery(p, tableNameList, pMsgBuf);
}
}
}
return TSDB_CODE_SUCCESS;
}
int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SMsgBuf* pMsgBuf) {
const char* msg1 = "invalid table name";
int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list);
assert(pSqlNode->from->type == SQL_NODE_FROM_TABLELIST);
for(int32_t j = 0; j < numOfTables; ++j) {
SRelElementPair* item = taosArrayGet(pSqlNode->from->list, j);
SToken* t = &item->tableName;
if (t->type == TK_INTEGER || t->type == TK_FLOAT || t->type == TK_STRING) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
if (parserValidateIdToken(t) != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg1);
}
SName name = {0};
strndequote(name.tname, t->z, t->n);
taosArrayPush(tableNameList, &name);
}
return TSDB_CODE_SUCCESS;
}
static void freePtrElem(void* p) {
tfree(*(char**)p);
}
int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SMetaReq* pMetaInfo, char* msg, int32_t msgBufLen) {
int32_t code = TSDB_CODE_SUCCESS;
SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen};
pMetaInfo->pTableName = taosArrayInit(4, sizeof(SName));
pMetaInfo->pUdf = taosArrayInit(4, POINTER_BYTES);
size_t size = taosArrayGetSize(pSqlInfo->list);
for (int32_t i = 0; i < size; ++i) {
SSqlNode* pSqlNode = taosArrayGetP(pSqlInfo->list, i);
if (pSqlNode->from == NULL) {
return buildInvalidOperationMsg(&msgBuf, "invalid from clause");
}
// load the table meta in the FROM clause
if (pSqlNode->from->type == SQL_NODE_FROM_TABLELIST) {
code = getTableNameFromSqlNode(pSqlNode, pMetaInfo->pTableName, &msgBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
} else {
code = getTableNameFromSubquery(pSqlNode, pMetaInfo->pTableName, &msgBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
}
}
taosArraySort(pMetaInfo->pTableName, tnameComparFn);
taosArrayRemoveDuplicate(pMetaInfo->pTableName, tnameComparFn, NULL);
size_t funcSize = 0;
if (pSqlInfo->funcs) {
funcSize = taosArrayGetSize(pSqlInfo->funcs);
}
if (funcSize > 0) {
for (size_t i = 0; i < funcSize; ++i) {
SToken* t = taosArrayGet(pSqlInfo->funcs, i);
assert(t != NULL);
if (t->n >= TSDB_FUNC_NAME_LEN) {
return buildSyntaxErrMsg(msg, msgBufLen, "too long function name", t->z);
}
// Let's assume that it is an UDF/UDAF, if it is not a built-in function.
if (!qIsBuiltinFunction(t->z, t->n)) {
char* fname = strndup(t->z, t->n);
taosArrayPush(pMetaInfo->pUdf, &fname);
}
}
}
return code;
}

1962
source/libs/parser/src/parserUtil.c Normal file
View File

File diff suppressed because it is too large Load Diff

375
source/libs/parser/src/queryInfoUtil.c Normal file
View File

@ -0,0 +1,375 @@
#include "queryInfoUtil.h"
#include "astGenerator.h"
#include "function.h"
#include "os.h"
#include "parser.h"
#include "parserInt.h"
#include "parserUtil.h"
static struct SSchema _s = {
.colId = TSDB_TBNAME_COLUMN_INDEX,
.type = TSDB_DATA_TYPE_BINARY,
.bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE,
.name = "tbname",
};
SSchema* getTbnameColumnSchema() {
return &_s;
}
size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo) {
return taosArrayGetSize(pQueryInfo->exprList);
}
SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
assert(pTableMeta != NULL && pTableMeta->schema != NULL && colIndex >= 0 && colIndex < getNumOfColumns(pTableMeta));
SSchema* pSchema = (SSchema*) pTableMeta->schema;
return &pSchema[colIndex];
}
STableComInfo getTableInfo(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
return pTableMeta->tableInfo;
}
int32_t getNumOfColumns(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
// table created according to super table, use data from super table
return getTableInfo(pTableMeta).numOfColumns;
}
int32_t getNumOfTags(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL);
return getTableInfo(pTableMeta).numOfTags;
}
SSchema *getTableColumnSchema(const STableMeta *pTableMeta) {
assert(pTableMeta != NULL);
return (SSchema*) pTableMeta->schema;
}
SSchema* getTableTagSchema(const STableMeta* pTableMeta) {
assert(pTableMeta != NULL && (pTableMeta->tableType == TSDB_SUPER_TABLE || pTableMeta->tableType == TSDB_CHILD_TABLE));
return getOneColumnSchema(pTableMeta, getTableInfo(pTableMeta).numOfColumns);
}
static tExprNode* createUnaryFunctionExprNode(int32_t functionId, SSchema* pSchema, tExprNode* pColumnNode) {
if (pColumnNode == NULL) {
pColumnNode = calloc(1, sizeof(tExprNode));
pColumnNode->nodeType = TEXPR_COL_NODE;
pColumnNode->pSchema = calloc(1, sizeof(SSchema));
memcpy(pColumnNode->pSchema, pSchema, sizeof(SSchema));
} else {
assert(pSchema == NULL);
}
tExprNode* pNode = calloc(1, sizeof(tExprNode));
pNode->nodeType = TEXPR_UNARYEXPR_NODE;
pNode->_node.functionId = functionId;
pNode->_node.pLeft = pColumnNode;
return pNode;
}
SExprInfo* createBinaryExprInfo(tExprNode* pNode, SSchema* pResSchema) {
assert(pNode != NULL && pResSchema != NULL);
SExprInfo* pExpr = calloc(1, sizeof(SExprInfo));
if (pExpr == NULL) {
return NULL;
}
pExpr->pExpr = pNode;
memcpy(&pExpr->base.resSchema, pResSchema, sizeof(SSchema));
return pExpr;
}
SExprInfo* createExprInfo(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, tExprNode* pParamExpr, SSchema* pResSchema, int16_t interSize) {
SExprInfo* pExpr = calloc(1, sizeof(SExprInfo));
if (pExpr == NULL) {
return NULL;
}
SSqlExpr* p = &pExpr->base;
if (pParamExpr != NULL) {
pExpr->pExpr = createUnaryFunctionExprNode(functionId, NULL, pParamExpr);
} else if (pColIndex->columnIndex == TSDB_TBNAME_COLUMN_INDEX) {
assert(pParamExpr == NULL);
SSchema* s = getTbnameColumnSchema();
p->colInfo.colId = TSDB_TBNAME_COLUMN_INDEX;
pExpr->pExpr = createUnaryFunctionExprNode(functionId, s, pParamExpr);
} else if (pColIndex->columnIndex <= TSDB_UD_COLUMN_INDEX || functionId == FUNCTION_BLKINFO) {
assert(pParamExpr == NULL);
p->colInfo.colId = pColIndex->columnIndex;
SSchema s = createSchema(pResSchema->type, pResSchema->bytes, pColIndex->columnIndex, pResSchema->name);
pExpr->pExpr = createUnaryFunctionExprNode(functionId, &s, pParamExpr);
} else {
int32_t len = tListLen(p->colInfo.name);
if (TSDB_COL_IS_TAG(pColIndex->type)) {
SSchema* pSchema = getTableTagSchema(pTableMetaInfo->pTableMeta);
p->colInfo.colId = pSchema[pColIndex->columnIndex].colId;
pExpr->pExpr = createUnaryFunctionExprNode(functionId, &pSchema[pColIndex->columnIndex], pParamExpr);
snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema[pColIndex->columnIndex].name);
} else if (pTableMetaInfo->pTableMeta != NULL) {
// in handling select database/version/server_status(), the pTableMeta is NULL
SSchema* pSchema = getOneColumnSchema(pTableMetaInfo->pTableMeta, pColIndex->columnIndex);
p->colInfo.colId = pSchema->colId;
snprintf(p->colInfo.name, len, "%s.%s", pTableMetaInfo->aliasName, pSchema->name);
pExpr->pExpr = createUnaryFunctionExprNode(functionId, pSchema, pParamExpr);
}
}
p->colInfo.flag = pColIndex->type;
p->colInfo.colIndex = pColIndex->columnIndex;
p->interBytes = interSize;
memcpy(&p->resSchema, pResSchema, sizeof(SSchema));
if (pTableMetaInfo->pTableMeta) {
p->uid = pTableMetaInfo->pTableMeta->uid;
}
return pExpr;
}
void addExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index, SExprInfo* pExprInfo) {
assert(pQueryInfo != NULL && pQueryInfo->exprList != NULL);
int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList);
if (index == num) {
taosArrayPush(pQueryInfo->exprList, &pExprInfo);
} else {
taosArrayInsert(pQueryInfo->exprList, index, &pExprInfo);
}
}
void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) {
assert(pExprInfo != NULL);
SSqlExpr* pse = &pExprInfo->base;
pExprInfo->pExpr->_node.functionId = functionId;
pse->colInfo.colIndex = srcColumnIndex;
pse->colInfo.colId = colId;
pse->resSchema.type = resType;
pse->resSchema.bytes = resSize;
}
SExprInfo* getExprInfo(SQueryStmtInfo* pQueryInfo, int32_t index) {
assert(pQueryInfo != NULL && pQueryInfo->exprList && index >= 0);
return taosArrayGetP(pQueryInfo->exprList, index);
}
void destroyExprInfo(SExprInfo* pExprInfo) {
tExprTreeDestroy(pExprInfo->pExpr, NULL);
tfree(pExprInfo);
}
void dropAllExprInfo(SArray* pExprInfo) {
size_t size = taosArrayGetSize(pExprInfo);
for(int32_t i = 0; i < size; ++i) {
SExprInfo* pExpr = taosArrayGetP(pExprInfo, i);
destroyExprInfo(pExpr);
}
taosArrayDestroy(pExprInfo);
}
void addExprInfoParam(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes) {
assert (pExpr != NULL || argument != NULL || bytes != 0);
// set parameter value
// transfer to tVariant from byte data/no ascii data
taosVariantCreateFromBinary(&pExpr->param[pExpr->numOfParams], argument, bytes, type);
pExpr->numOfParams += 1;
assert(pExpr->numOfParams <= 3);
}
void assignExprInfo(SExprInfo* dst, const SExprInfo* src) {
assert(dst != NULL && src != NULL);
*dst = *src;
#if 0
if (src->base.flist.numOfFilters > 0) {
dst->base.flist.filterInfo = calloc(src->base.flist.numOfFilters, sizeof(SColumnFilterInfo));
memcpy(dst->base.flist.filterInfo, src->base.flist.filterInfo, sizeof(SColumnFilterInfo) * src->base.flist.numOfFilters);
}
#endif
// dst->pExpr = exprdup(src->pExpr);
memset(dst->base.param, 0, sizeof(SVariant) * tListLen(dst->base.param));
for (int32_t j = 0; j < src->base.numOfParams; ++j) {
taosVariantAssign(&dst->base.param[j], &src->base.param[j]);
}
}
int32_t copyExprInfoList(SArray* dst, const SArray* src, uint64_t uid, bool deepcopy) {
assert(src != NULL && dst != NULL);
size_t size = taosArrayGetSize(src);
for (int32_t i = 0; i < size; ++i) {
SExprInfo* pExpr = taosArrayGetP(src, i);
if (pExpr->base.uid == uid) {
if (deepcopy) {
SExprInfo* p1 = calloc(1, sizeof(SExprInfo));
assignExprInfo(p1, pExpr);
taosArrayPush(dst, &p1);
} else {
taosArrayPush(dst, &pExpr);
}
}
}
return 0;
}
int32_t copyAllExprInfo(SArray* dst, const SArray* src, bool deepcopy) {
assert(src != NULL && dst != NULL);
size_t size = taosArrayGetSize(src);
for (int32_t i = 0; i < size; ++i) {
SExprInfo* pExpr = taosArrayGetP(src, i);
SExprInfo* p1 = calloc(1, sizeof(SExprInfo));
assignExprInfo(p1, pExpr);
taosArrayPush(dst, &p1);
}
return 0;
}
//void* tSqlExprDestroy(SExprInfo* pExpr) {
// if (pExpr == NULL) {
// return NULL;
// }
//
// SSqlExpr* p = &pExpr->base;
// for(int32_t i = 0; i < tListLen(p->param); ++i) {
// taosVariantDestroy(&p->param[i]);
// }
//
// if (p->flist.numOfFilters > 0) {
// tfree(p->flist.filterInfo);
// }
//
// if (pExpr->pExpr != NULL) {
// tExprTreeDestroy(pExpr->pExpr, NULL);
// }
//
// tfree(pExpr);
// return NULL;
//}
int32_t getResRowLength(SArray* pExprList) {
size_t num = taosArrayGetSize(pExprList);
if (num == 0) {
return 0;
}
int32_t size = 0;
for(int32_t i = 0; i < num; ++i) {
SExprInfo* pExpr = taosArrayGetP(pExprList, i);
size += pExpr->base.resSchema.bytes;
}
return size;
}
static void freeQueryInfoImpl(SQueryStmtInfo* pQueryInfo) {
cleanupTagCond(&pQueryInfo->tagCond);
cleanupColumnCond(&pQueryInfo->colCond);
cleanupFieldInfo(&pQueryInfo->fieldsInfo);
dropAllExprInfo(pQueryInfo->exprList);
pQueryInfo->exprList = NULL;
if (pQueryInfo->exprList1 != NULL) {
dropAllExprInfo(pQueryInfo->exprList1);
pQueryInfo->exprList1 = NULL;
}
columnListDestroy(pQueryInfo->colList);
pQueryInfo->colList = NULL;
if (pQueryInfo->groupbyExpr.columnInfo != NULL) {
taosArrayDestroy(pQueryInfo->groupbyExpr.columnInfo);
pQueryInfo->groupbyExpr.columnInfo = NULL;
}
pQueryInfo->fillType = 0;
tfree(pQueryInfo->fillVal);
tfree(pQueryInfo->buf);
taosArrayDestroy(pQueryInfo->pUpstream);
pQueryInfo->pUpstream = NULL;
pQueryInfo->bufLen = 0;
}
void freeQueryInfo(SQueryStmtInfo* pQueryInfo, bool removeCachedMeta, uint64_t id) {
while(pQueryInfo != NULL) {
SQueryStmtInfo* p = pQueryInfo->sibling;
size_t numOfUpstream = taosArrayGetSize(pQueryInfo->pUpstream);
for(int32_t i = 0; i < numOfUpstream; ++i) {
SQueryStmtInfo* pUpQueryInfo = taosArrayGetP(pQueryInfo->pUpstream, i);
freeQueryInfoImpl(pUpQueryInfo);
clearAllTableMetaInfo(pUpQueryInfo, removeCachedMeta, id);
tfree(pUpQueryInfo);
}
freeQueryInfoImpl(pQueryInfo);
clearAllTableMetaInfo(pQueryInfo, removeCachedMeta, id);
tfree(pQueryInfo);
pQueryInfo = p;
}
}
SArray* extractFunctionIdList(SArray* pExprInfoList) {
assert(pExprInfoList != NULL);
size_t len = taosArrayGetSize(pExprInfoList);
SArray* p = taosArrayInit(len, sizeof(int16_t));
for(int32_t i = 0; i < len; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pExprInfoList, i);
taosArrayPush(p, &pExprInfo->pExpr->_node.functionId);
}
return p;
}
bool tscHasColumnFilter(SQueryStmtInfo* pQueryInfo) {
// filter on primary timestamp column
if (pQueryInfo->window.skey != INT64_MIN || pQueryInfo->window.ekey != INT64_MAX) {
return true;
}
size_t size = taosArrayGetSize(pQueryInfo->colList);
for (int32_t i = 0; i < size; ++i) {
SColumn* pCol = taosArrayGetP(pQueryInfo->colList, i);
if (pCol->info.flist.numOfFilters > 0) {
return true;
}
}
return false;
}
//void tscClearInterpInfo(SQueryStmtInfo* pQueryInfo) {
// if (!tscIsPointInterpQuery(pQueryInfo)) {
// return;
// }
//
// pQueryInfo->fillType = TSDB_FILL_NONE;
// tfree(pQueryInfo->fillVal);
//}

34
source/libs/parser/src/sql.c
View File

@ -108,13 +108,13 @@ typedef union {
int yy130;
SArray* yy135;
SIntervalVal yy160;
TAOS_FIELD yy181;
SVariant yy191;
SLimit yy247;
SCreateDbInfo yy256;
SWindowStateVal yy258;
int32_t yy262;
SCreateAcctInfo yy277;
SField yy304;
SRelationInfo* yy460;
SSqlNode* yy488;
SSessionWindowVal yy511;
@ -2431,10 +2431,10 @@ static void yy_reduce(
{ setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy256, &yymsp[-2].minor.yy0);}
break;
case 62: /* cmd ::= CREATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */
{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy181, &yymsp[0].minor.yy0, 1);}
{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy304, &yymsp[0].minor.yy0, 1);}
break;
case 63: /* cmd ::= CREATE AGGREGATE FUNCTION ids AS ids OUTPUTTYPE typename bufsize */
{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy181, &yymsp[0].minor.yy0, 2);}
{ setCreateFuncInfo(pInfo, TSDB_SQL_CREATE_FUNCTION, &yymsp[-5].minor.yy0, &yymsp[-3].minor.yy0, &yymsp[-1].minor.yy304, &yymsp[0].minor.yy0, 2);}
break;
case 64: /* cmd ::= CREATE USER ids PASS ids */
{ setCreateUserSql(pInfo, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);}
@ -2601,29 +2601,29 @@ static void yy_reduce(
case 133: /* typename ::= ids */
{
yymsp[0].minor.yy0.type = 0;
tSetColumnType (&yylhsminor.yy181, &yymsp[0].minor.yy0);
tSetColumnType (&yylhsminor.yy304, &yymsp[0].minor.yy0);
}
yymsp[0].minor.yy181 = yylhsminor.yy181;
yymsp[0].minor.yy304 = yylhsminor.yy304;
break;
case 134: /* typename ::= ids LP signed RP */
{
if (yymsp[-1].minor.yy531 <= 0) {
yymsp[-3].minor.yy0.type = 0;
tSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
tSetColumnType(&yylhsminor.yy304, &yymsp[-3].minor.yy0);
} else {
yymsp[-3].minor.yy0.type = -yymsp[-1].minor.yy531; // negative value of name length
tSetColumnType(&yylhsminor.yy181, &yymsp[-3].minor.yy0);
tSetColumnType(&yylhsminor.yy304, &yymsp[-3].minor.yy0);
}
}
yymsp[-3].minor.yy181 = yylhsminor.yy181;
yymsp[-3].minor.yy304 = yylhsminor.yy304;
break;
case 135: /* typename ::= ids UNSIGNED */
{
yymsp[-1].minor.yy0.type = 0;
yymsp[-1].minor.yy0.n = ((yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z);
tSetColumnType (&yylhsminor.yy181, &yymsp[-1].minor.yy0);
tSetColumnType (&yylhsminor.yy304, &yymsp[-1].minor.yy0);
}
yymsp[-1].minor.yy181 = yylhsminor.yy181;
yymsp[-1].minor.yy304 = yylhsminor.yy304;
break;
case 136: /* signed ::= INTEGER */
{ yylhsminor.yy531 = strtol(yymsp[0].minor.yy0.z, NULL, 10); }
@ -2711,18 +2711,18 @@ static void yy_reduce(
yymsp[-4].minor.yy110 = yylhsminor.yy110;
break;
case 152: /* columnlist ::= columnlist COMMA column */
{taosArrayPush(yymsp[-2].minor.yy135, &yymsp[0].minor.yy181); yylhsminor.yy135 = yymsp[-2].minor.yy135; }
{taosArrayPush(yymsp[-2].minor.yy135, &yymsp[0].minor.yy304); yylhsminor.yy135 = yymsp[-2].minor.yy135; }
yymsp[-2].minor.yy135 = yylhsminor.yy135;
break;
case 153: /* columnlist ::= column */
{yylhsminor.yy135 = taosArrayInit(4, sizeof(TAOS_FIELD)); taosArrayPush(yylhsminor.yy135, &yymsp[0].minor.yy181);}
{yylhsminor.yy135 = taosArrayInit(4, sizeof(SField)); taosArrayPush(yylhsminor.yy135, &yymsp[0].minor.yy304);}
yymsp[0].minor.yy135 = yylhsminor.yy135;
break;
case 154: /* column ::= ids typename */
{
tSetColumnInfo(&yylhsminor.yy181, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy181);
tSetColumnInfo(&yylhsminor.yy304, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy304);
}
yymsp[-1].minor.yy181 = yylhsminor.yy181;
yymsp[-1].minor.yy304 = yylhsminor.yy304;
break;
case 161: /* tagitem ::= NULL */
{ yymsp[0].minor.yy0.type = 0; taosVariantCreate(&yylhsminor.yy191, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.type); }
@ -2893,7 +2893,7 @@ static void yy_reduce(
toTSDBType(yymsp[-3].minor.yy0.type);
taosVariantCreate(&A, yymsp[-3].minor.yy0.z, yymsp[-3].minor.yy0.n, yymsp[-3].minor.yy0.type);
tVariantListInsert(yymsp[-1].minor.yy135, &A, -1, 0);
tListItemInsert(yymsp[-1].minor.yy135, &A, -1, 0);
yymsp[-5].minor.yy135 = yymsp[-1].minor.yy135;
}
break;
@ -3049,11 +3049,11 @@ static void yy_reduce(
yymsp[0].minor.yy526 = yylhsminor.yy526;
break;
case 247: /* expr ::= ID LP exprlist RP */
{ tAppendFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy526 = tSqlExprCreateFunction(yymsp[-1].minor.yy135, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
{ tRecordFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy526 = tSqlExprCreateFunction(yymsp[-1].minor.yy135, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
yymsp[-3].minor.yy526 = yylhsminor.yy526;
break;
case 248: /* expr ::= ID LP STAR RP */
{ tAppendFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy526 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
{ tRecordFuncName(pInfo->funcs, &yymsp[-3].minor.yy0); yylhsminor.yy526 = tSqlExprCreateFunction(NULL, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, yymsp[-3].minor.yy0.type); }
yymsp[-3].minor.yy526 = yylhsminor.yy526;
break;
case 249: /* expr ::= expr IS NULL */

3
source/libs/parser/src/ttokenizer.c
View File

@ -411,6 +411,7 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) {
*tokenId = TK_QUESTION;
return 1;
}
case '`':
case '\'':
case '"': {
int delim = z[0];
@ -434,7 +435,7 @@ uint32_t tGetToken(char* z, uint32_t* tokenId) {
if (z[i]) i++;
if (strEnd) {
*tokenId = TK_STRING;
*tokenId = (delim == '`')? TK_ID:TK_STRING;
return i;
}

18
source/libs/parser/test/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
MESSAGE(STATUS "build parser unit test")
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(astTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
astTest
PUBLIC os util common parser catalog transport gtest
)
TARGET_INCLUDE_DIRECTORIES(
astTest
PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser/"
PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/parser/inc"
)

366
source/libs/parser/test/parserTests.cpp
View File

@ -11,4 +11,368 @@
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
*/
#include <gtest/gtest.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "astGenerator.h"
#include "parserInt.h"
#include "taos.h"
#include "tdef.h"
#include "tvariant.h"
namespace {
void setSchema(SSchema* p, int32_t type, int32_t bytes, const char* name, int32_t colId) {
p->colId = colId;
p->bytes = bytes;
p->type = type;
strcpy(p->name, name);
}
void setTableMetaInfo(SQueryStmtInfo* pQueryInfo, SMetaReq *req) {
pQueryInfo->numOfTables = 1;
pQueryInfo->pTableMetaInfo = (STableMetaInfo**)calloc(1, POINTER_BYTES);
STableMetaInfo* pTableMetaInfo = (STableMetaInfo*)calloc(1, sizeof(STableMetaInfo));
pQueryInfo->pTableMetaInfo[0] = pTableMetaInfo;
SName* name = (SName*)taosArrayGet(req->pTableName, 0);
memcpy(&pTableMetaInfo->name, taosArrayGet(req->pTableName, 0), sizeof(SName));
pTableMetaInfo->pTableMeta = (STableMeta*)calloc(1, sizeof(STableMeta) + 4 * sizeof(SSchema));
strcpy(pTableMetaInfo->aliasName, name->tname);
STableMeta* pTableMeta = pTableMetaInfo->pTableMeta;
pTableMeta->tableType = TSDB_NORMAL_TABLE;
pTableMeta->tableInfo.numOfColumns = 4;
pTableMeta->tableInfo.rowSize = 28;
pTableMeta->uid = 110;
pTableMetaInfo->tagColList = (SArray*) taosArrayInit(4, POINTER_BYTES);
SSchema* pSchema = pTableMetaInfo->pTableMeta->schema;
setSchema(&pSchema[0], TSDB_DATA_TYPE_TIMESTAMP, 8, "ts", 0);
setSchema(&pSchema[1], TSDB_DATA_TYPE_INT, 4, "a", 1);
setSchema(&pSchema[2], TSDB_DATA_TYPE_DOUBLE, 8, "b", 2);
setSchema(&pSchema[3], TSDB_DATA_TYPE_DOUBLE, 8, "col", 3);
}
}
//TEST(testCase, validateAST_test) {
// SSqlInfo info1 = doGenerateAST("select a a1111, a+b + 22, tbname from `t.1abc` where ts<now+2h and `col` < 20 + 99");
// ASSERT_EQ(info1.valid, true);
//
// char msg[128] = {0};
// SMsgBuf buf;
// buf.len = 128;
// buf.buf = msg;
//
// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
// ASSERT_EQ(code, 0);
//
// SMetaReq req = {0};
// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
// ASSERT_EQ(ret, 0);
// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
//
// SQueryStmtInfo* pQueryInfo = (SQueryStmtInfo*)calloc(1, sizeof(SQueryStmtInfo));
// initQueryInfo(pQueryInfo);
// setTableMetaInfo(pQueryInfo, &req);
//
// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0);
// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
//
// SArray* pExprList = pQueryInfo->exprList;
// ASSERT_EQ(taosArrayGetSize(pExprList), 3);
//
// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0);
// ASSERT_EQ(p1->base.uid, 110);
// ASSERT_EQ(p1->base.numOfParams, 0);
// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_INT);
// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1111");
// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.a");
// ASSERT_EQ(p1->base.colInfo.colId, 1);
// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL);
// ASSERT_STRCASEEQ(p1->base.token, "a");
//
// ASSERT_EQ(taosArrayGetSize(pExprList), 3);
//
// SExprInfo* p2 = (SExprInfo*) taosArrayGetP(pExprList, 1);
// ASSERT_EQ(p2->base.uid, 0);
// ASSERT_EQ(p2->base.numOfParams, 1); // it is the serialized binary string of expression.
// ASSERT_EQ(p2->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE);
// ASSERT_STRCASEEQ(p2->base.resSchema.name, "a+b + 22");
//
//// ASSERT_STRCASEEQ(p2->base.colInfo.name, "t.1abc.a");
//// ASSERT_EQ(p1->base.colInfo.colId, 1);
//// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL);
// ASSERT_STRCASEEQ(p2->base.token, "a+b + 22");
//
// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3);
// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 3);
//}
//
//TEST(testCase, function_Test) {
// SSqlInfo info1 = doGenerateAST("select count(a) from `t.1abc`");
// ASSERT_EQ(info1.valid, true);
//
// char msg[128] = {0};
// SMsgBuf buf;
// buf.len = 128;
// buf.buf = msg;
//
// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
// ASSERT_EQ(code, 0);
//
// SMetaReq req = {0};
// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
// ASSERT_EQ(ret, 0);
// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
//
// SQueryStmtInfo* pQueryInfo = (SQueryStmtInfo*)calloc(1, sizeof(SQueryStmtInfo));
// initQueryInfo(pQueryInfo);
// setTableMetaInfo(pQueryInfo, &req);
//
// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0);
// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
//
// SArray* pExprList = pQueryInfo->exprList;
// ASSERT_EQ(taosArrayGetSize(pExprList), 1);
//
// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0);
// ASSERT_EQ(p1->base.uid, 110);
// ASSERT_EQ(p1->base.numOfParams, 0);
// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT);
// ASSERT_STRCASEEQ(p1->base.resSchema.name, "count(a)");
// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.a");
// ASSERT_EQ(p1->base.colInfo.colId, 1);
// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL);
// ASSERT_STRCASEEQ(p1->base.token, "count(a)");
// ASSERT_EQ(p1->base.interBytes, 8);
//
// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2);
// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1);
//}
//
//TEST(testCase, function_Test2) {
// SSqlInfo info1 = doGenerateAST("select count(a) abc from `t.1abc`");
// ASSERT_EQ(info1.valid, true);
//
// char msg[128] = {0};
// SMsgBuf buf;
// buf.len = 128;
// buf.buf = msg;
//
// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
// ASSERT_EQ(code, 0);
//
// SMetaReq req = {0};
// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
// ASSERT_EQ(ret, 0);
// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
//
// SQueryStmtInfo* pQueryInfo = (SQueryStmtInfo*)calloc(1, sizeof(SQueryStmtInfo));
// initQueryInfo(pQueryInfo);
// setTableMetaInfo(pQueryInfo, &req);
//
// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0);
// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
//
// SArray* pExprList = pQueryInfo->exprList;
// ASSERT_EQ(taosArrayGetSize(pExprList), 1);
//
// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0);
// ASSERT_EQ(p1->base.uid, 110);
// ASSERT_EQ(p1->base.numOfParams, 0);
// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BIGINT);
// ASSERT_STRCASEEQ(p1->base.resSchema.name, "abc");
// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.a");
// ASSERT_EQ(p1->base.colInfo.colId, 1);
// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL);
// ASSERT_STRCASEEQ(p1->base.token, "count(a)");
// ASSERT_EQ(p1->base.interBytes, 8);
//
// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 2);
// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1);
//}
//
//TEST(testCase, function_Test3) {
// SSqlInfo info1 = doGenerateAST("select first(*) from `t.1abc`");
// ASSERT_EQ(info1.valid, true);
//
// char msg[128] = {0};
// SMsgBuf buf;
// buf.len = 128;
// buf.buf = msg;
//
// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
// ASSERT_EQ(code, 0);
//
// SMetaReq req = {0};
// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
// ASSERT_EQ(ret, 0);
// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
//
// SQueryStmtInfo* pQueryInfo = (SQueryStmtInfo*)calloc(1, sizeof(SQueryStmtInfo));
// initQueryInfo(pQueryInfo);
// setTableMetaInfo(pQueryInfo, &req);
//
// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0);
// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
//
// SArray* pExprList = pQueryInfo->exprList;
// ASSERT_EQ(taosArrayGetSize(pExprList), 4);
//
// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0);
// ASSERT_EQ(p1->base.uid, 110);
// ASSERT_EQ(p1->base.numOfParams, 0);
// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_TIMESTAMP);
// ASSERT_STRCASEEQ(p1->base.resSchema.name, "first(ts)");
// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts");
// ASSERT_EQ(p1->base.colInfo.colId, 0);
// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL);
// ASSERT_STRCASEEQ(p1->base.token, "first(ts)");
// ASSERT_EQ(p1->base.interBytes, 24);
//
// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 4);
//}
//
//TEST(testCase, function_Test4) {
// SSqlInfo info1 = doGenerateAST("select _block_dist() as a1 from `t.1abc`");
// ASSERT_EQ(info1.valid, true);
//
// char msg[128] = {0};
// SMsgBuf buf;
// buf.len = 128;
// buf.buf = msg;
//
// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
// ASSERT_EQ(code, 0);
//
// SMetaReq req = {0};
// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
// ASSERT_EQ(ret, 0);
// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
//
// SQueryStmtInfo* pQueryInfo = (SQueryStmtInfo*)calloc(1, sizeof(SQueryStmtInfo));
// initQueryInfo(pQueryInfo);
// setTableMetaInfo(pQueryInfo, &req);
//
// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0);
// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
//
// SArray* pExprList = pQueryInfo->exprList;
// ASSERT_EQ(taosArrayGetSize(pExprList), 1);
//
// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0);
// ASSERT_EQ(p1->base.uid, 110);
// ASSERT_EQ(p1->base.numOfParams, 1);
// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_BINARY);
// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1");
//// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts");
//// ASSERT_EQ(p1->base.colInfo.colId, 0);
// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL);
// ASSERT_STRCASEEQ(p1->base.token, "_block_dist()");
// ASSERT_EQ(p1->base.interBytes, 0);
//
// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 1);
// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1);
//}
//
//TEST(testCase, function_Test5) {
// SSqlInfo info1 = doGenerateAST("select sum(a) + avg(b) as a1 from `t.1abc`");
// ASSERT_EQ(info1.valid, true);
//
// char msg[128] = {0};
// SMsgBuf buf;
// buf.len = 128;
// buf.buf = msg;
//
// SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
// int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
// ASSERT_EQ(code, 0);
//
// SMetaReq req = {0};
// int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
// ASSERT_EQ(ret, 0);
// ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
//
// SQueryStmtInfo* pQueryInfo = (SQueryStmtInfo*)calloc(1, sizeof(SQueryStmtInfo));
// initQueryInfo(pQueryInfo);
// setTableMetaInfo(pQueryInfo, &req);
//
// SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0);
// ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
// ASSERT_EQ(ret, 0);
//
// SArray* pExprList = pQueryInfo->exprList;
// ASSERT_EQ(taosArrayGetSize(pExprList), 3);
//
// SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0);
// ASSERT_EQ(p1->base.uid, 0);
// ASSERT_EQ(p1->base.numOfParams, 1);
// ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE);
// ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1");
//// ASSERT_STRCASEEQ(p1->base.colInfo.name, "t.1abc.ts");
//// ASSERT_EQ(p1->base.colInfo.colId, 0);
// ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL);
// ASSERT_STRCASEEQ(p1->base.token, "sum(a) + avg(b)");
// ASSERT_EQ(p1->base.interBytes, 0);
//
// ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3);
// ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 1);
//}
TEST(testCase, function_Test6) {
SSqlInfo info1 = doGenerateAST("select sum(a+b) as a1, first(b*a) from `t.1abc`");
ASSERT_EQ(info1.valid, true);
char msg[128] = {0};
SMsgBuf buf;
buf.len = 128;
buf.buf = msg;
SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &buf);
ASSERT_EQ(code, 0);
SMetaReq req = {0};
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
ASSERT_EQ(ret, 0);
ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
SQueryStmtInfo* pQueryInfo = (SQueryStmtInfo*)calloc(1, sizeof(SQueryStmtInfo));
initQueryInfo(pQueryInfo);
setTableMetaInfo(pQueryInfo, &req);
SSqlNode* pSqlNode = (SSqlNode*)taosArrayGetP(info1.list, 0);
ret = validateSqlNode(pSqlNode, pQueryInfo, &buf);
ASSERT_EQ(ret, 0);
SArray* pExprList = pQueryInfo->exprList;
ASSERT_EQ(taosArrayGetSize(pExprList), 2);
SExprInfo* p1 = (SExprInfo*) taosArrayGetP(pExprList, 0);
ASSERT_EQ(p1->base.uid, 110);
ASSERT_EQ(p1->base.numOfParams, 0);
ASSERT_EQ(p1->base.resSchema.type, TSDB_DATA_TYPE_DOUBLE);
ASSERT_STRCASEEQ(p1->base.resSchema.name, "a1");
ASSERT_EQ(p1->base.colInfo.flag, TSDB_COL_NORMAL);
ASSERT_STRCASEEQ(p1->base.token, "sum(a+b)");
ASSERT_EQ(p1->base.interBytes, 16);
ASSERT_EQ(taosArrayGetSize(pQueryInfo->colList), 3);
ASSERT_EQ(pQueryInfo->fieldsInfo.numOfOutput, 2);
}

717
source/libs/parser/test/tokenizerTest.cpp Normal file
View File

@ -0,0 +1,717 @@
#include <gtest/gtest.h>
#include <iostream>
#pragma GCC diagnostic ignored "-Wwrite-strings"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wsign-compare"
#include "os.h"
#include "taos.h"
#include "tvariant.h"
#include "tdef.h"
#include "ttoken.h"
#include "astGenerator.h"
#include "parserUtil.h"
#include "parserInt.h"
namespace {
int32_t testValidateName(char* name) {
SToken token = {0};
token.z = name;
token.n = strlen(name);
token.type = 0;
tGetToken(name, &token.type);
return parserValidateIdToken(&token);
}
SToken createToken(char* s) {
SToken t = {0};
t.type = TK_STRING;
t.z = s;
t.n = strlen(s);
return t;
}
} // namespace
static void _init_tvariant_bool(SVariant* t) {
t->i64 = TSDB_FALSE;
t->nType = TSDB_DATA_TYPE_BOOL;
}
static void _init_tvariant_tinyint(SVariant* t) {
t->i64 = -27;
t->nType = TSDB_DATA_TYPE_TINYINT;
}
static void _init_tvariant_int(SVariant* t) {
t->i64 = -23997659;
t->nType = TSDB_DATA_TYPE_INT;
}
static void _init_tvariant_bigint(SVariant* t) {
t->i64 = -3333333333333;
t->nType = TSDB_DATA_TYPE_BIGINT;
}
static void _init_tvariant_float(SVariant* t) {
t->d = -8991212199.8987878776;
t->nType = TSDB_DATA_TYPE_FLOAT;
}
static void _init_tvariant_binary(SVariant* t) {
taosVariantDestroy(t);
t->pz = (char*)calloc(1, 20); //"2e3");
t->nType = TSDB_DATA_TYPE_BINARY;
strcpy(t->pz, "2e5");
t->nLen = strlen(t->pz);
}
static void _init_tvariant_nchar(SVariant* t) {
taosVariantDestroy(t);
t->wpz = (wchar_t*)calloc(1, 20 * TSDB_NCHAR_SIZE);
t->nType = TSDB_DATA_TYPE_NCHAR;
wcscpy(t->wpz, L"-2000000.8765");
t->nLen = twcslen(t->wpz);
}
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
TEST(testCase, validateToken_test) {
char t01[] = "abc";
EXPECT_EQ(testValidateName(t01), TSDB_CODE_SUCCESS);
char t110[] = "`1233abc.911`";
EXPECT_EQ(testValidateName(t110), TSDB_CODE_SUCCESS);
char t02[] = "'abc'";
EXPECT_EQ(testValidateName(t02), TSDB_CODE_TSC_INVALID_OPERATION);
char t1[] = "abc.def";
EXPECT_EQ(testValidateName(t1), TSDB_CODE_SUCCESS);
printf("%s\n", t1);
char t98[] = "abc.DeF";
EXPECT_EQ(testValidateName(t98), TSDB_CODE_SUCCESS);
EXPECT_STREQ(t98, "abc.def");
printf("%s\n", t98);
char t97[] = "257.abc";
EXPECT_EQ(testValidateName(t97), TSDB_CODE_TSC_INVALID_OPERATION);
printf("%s\n", t97);
char t96[] = "_257.aBc";
EXPECT_EQ(testValidateName(t96), TSDB_CODE_SUCCESS);
EXPECT_STREQ(t96, "_257.abc");
printf("%s\n", t96);
char t99[] = "abc . def";
EXPECT_EQ(testValidateName(t99), TSDB_CODE_TSC_INVALID_OPERATION);
printf("%s\n", t99);
char t2[] = "'abc.def'";
EXPECT_EQ(testValidateName(t2), TSDB_CODE_TSC_INVALID_OPERATION);
printf("%s\n", t2);
char t3[] = "'abc'.def";
EXPECT_EQ(testValidateName(t3), TSDB_CODE_TSC_INVALID_OPERATION);
printf("%s\n", t3);
char t4[] = "'abc'.'def'";
EXPECT_EQ(testValidateName(t4), TSDB_CODE_TSC_INVALID_OPERATION);
char t5[] = "table.'def'";
EXPECT_EQ(testValidateName(t5), TSDB_CODE_TSC_INVALID_OPERATION);
char t6[] = "'table'.'def'";
EXPECT_EQ(testValidateName(t6), TSDB_CODE_TSC_INVALID_OPERATION);
char t7[] = "'_ab1234'.'def'";
EXPECT_EQ(testValidateName(t7), TSDB_CODE_TSC_INVALID_OPERATION);
printf("%s\n", t7);
char t8[] = "'_ab&^%1234'.'def'";
EXPECT_EQ(testValidateName(t8), TSDB_CODE_TSC_INVALID_OPERATION);
char t9[] = "'_123'.'gtest中文'";
EXPECT_EQ(testValidateName(t9), TSDB_CODE_TSC_INVALID_OPERATION);
char t10[] = "abc.'gtest中文'";
EXPECT_EQ(testValidateName(t10), TSDB_CODE_TSC_INVALID_OPERATION);
char t10_1[] = "abc.'中文gtest'";
EXPECT_EQ(testValidateName(t10_1), TSDB_CODE_TSC_INVALID_OPERATION);
char t11[] = "'192.168.0.1'.abc";
EXPECT_EQ(testValidateName(t11), TSDB_CODE_TSC_INVALID_OPERATION);
char t12[] = "192.168.0.1.abc";
EXPECT_EQ(testValidateName(t12), TSDB_CODE_TSC_INVALID_OPERATION);
char t13[] = "abc.";
EXPECT_EQ(testValidateName(t13), TSDB_CODE_TSC_INVALID_OPERATION);
char t14[] = ".abc";
EXPECT_EQ(testValidateName(t14), TSDB_CODE_TSC_INVALID_OPERATION);
char t15[] = ".'abc'";
EXPECT_EQ(testValidateName(t15), TSDB_CODE_TSC_INVALID_OPERATION);
char t16[] = ".abc'";
EXPECT_EQ(testValidateName(t16), TSDB_CODE_TSC_INVALID_OPERATION);
char t17[] = "123a.\"abc\"";
EXPECT_EQ(testValidateName(t17), TSDB_CODE_TSC_INVALID_OPERATION);
printf("%s\n", t17);
char t18[] = "a.\"abc\"";
EXPECT_EQ(testValidateName(t18), TSDB_CODE_TSC_INVALID_OPERATION);
printf("%s\n", t18);
char t19[] = "'_ab1234'.'def'.'ab123'";
EXPECT_EQ(testValidateName(t19), TSDB_CODE_TSC_INVALID_OPERATION);
char t20[] = "'_ab1234*&^'";
EXPECT_EQ(testValidateName(t20), TSDB_CODE_TSC_INVALID_OPERATION);
char t21[] = "'1234_abc'";
EXPECT_EQ(testValidateName(t21), TSDB_CODE_TSC_INVALID_OPERATION);
// =======Containing capital letters=================
char t30[] = "ABC";
EXPECT_EQ(testValidateName(t30), TSDB_CODE_SUCCESS);
char t31[] = "'ABC'";
EXPECT_EQ(testValidateName(t31), TSDB_CODE_TSC_INVALID_OPERATION);
char t32[] = "ABC.def";
EXPECT_EQ(testValidateName(t32), TSDB_CODE_SUCCESS);
char t33[] = "'ABC.def";
EXPECT_EQ(testValidateName(t33), TSDB_CODE_TSC_INVALID_OPERATION);
char t33_0[] = "abc.DEF'";
EXPECT_EQ(testValidateName(t33_0), TSDB_CODE_TSC_INVALID_OPERATION);
char t34[] = "'ABC.def'";
// int32_t tmp0 = testValidateName(t34);
EXPECT_EQ(testValidateName(t34), TSDB_CODE_TSC_INVALID_OPERATION);
char t35[] = "'ABC'.def";
EXPECT_EQ(testValidateName(t35), TSDB_CODE_TSC_INVALID_OPERATION);
char t36[] = "ABC.DEF";
EXPECT_EQ(testValidateName(t36), TSDB_CODE_SUCCESS);
char t37[] = "abc.DEF";
EXPECT_EQ(testValidateName(t37), TSDB_CODE_SUCCESS);
char t37_1[] = "abc._123DEF";
EXPECT_EQ(testValidateName(t37_1), TSDB_CODE_SUCCESS);
char t38[] = "'abc'.\"DEF\"";
EXPECT_EQ(testValidateName(t38), TSDB_CODE_TSC_INVALID_OPERATION);
// do not use key words
char t39[] = "table.'DEF'";
EXPECT_EQ(testValidateName(t39), TSDB_CODE_TSC_INVALID_OPERATION);
char t40[] = "'table'.'DEF'";
EXPECT_EQ(testValidateName(t40), TSDB_CODE_TSC_INVALID_OPERATION);
char t41[] = "'_abXYZ1234'.'deFF'";
EXPECT_EQ(testValidateName(t41), TSDB_CODE_TSC_INVALID_OPERATION);
char t42[] = "'_abDEF&^%1234'.'DIef'";
EXPECT_EQ(testValidateName(t42), TSDB_CODE_TSC_INVALID_OPERATION);
char t43[] = "'_123'.'Gtest中文'";
EXPECT_EQ(testValidateName(t43), TSDB_CODE_TSC_INVALID_OPERATION);
char t44[] = "'aABC'.'Gtest中文'";
EXPECT_EQ(testValidateName(t44), TSDB_CODE_TSC_INVALID_OPERATION);
char t45[] = "'ABC'.";
EXPECT_EQ(testValidateName(t45), TSDB_CODE_TSC_INVALID_OPERATION);
char t46[] = ".'ABC'";
EXPECT_EQ(testValidateName(t46), TSDB_CODE_TSC_INVALID_OPERATION);
char t47[] = "a.\"aTWc\"";
EXPECT_EQ(testValidateName(t47), TSDB_CODE_TSC_INVALID_OPERATION);
// ================has space =================
char t60[] = " ABC ";
EXPECT_EQ(testValidateName(t60), TSDB_CODE_TSC_INVALID_OPERATION);
char t60_1[] = " ABC ";
EXPECT_EQ(testValidateName(t60_1), TSDB_CODE_TSC_INVALID_OPERATION);
char t61[] = "' ABC '";
EXPECT_EQ(testValidateName(t61), TSDB_CODE_TSC_INVALID_OPERATION);
char t61_1[] = "' ABC '";
EXPECT_EQ(testValidateName(t61_1), TSDB_CODE_TSC_INVALID_OPERATION);
char t62[] = " ABC . def ";
EXPECT_EQ(testValidateName(t62), TSDB_CODE_TSC_INVALID_OPERATION);
char t63[] = "' ABC . def ";
EXPECT_EQ(testValidateName(t63), TSDB_CODE_TSC_INVALID_OPERATION);
char t63_0[] = " abc . DEF ' ";
EXPECT_EQ(testValidateName(t63_0), TSDB_CODE_TSC_INVALID_OPERATION);
char t64[] = " ' ABC . def ' ";
// int32_t tmp1 = testValidateName(t64);
EXPECT_EQ(testValidateName(t64), TSDB_CODE_TSC_INVALID_OPERATION);
char t65[] = " ' ABC '. def ";
EXPECT_EQ(testValidateName(t65), TSDB_CODE_TSC_INVALID_OPERATION);
char t66[] = "' ABC '.' DEF '";
EXPECT_EQ(testValidateName(t66), TSDB_CODE_TSC_INVALID_OPERATION);
char t67[] = "abc . ' DEF '";
EXPECT_EQ(testValidateName(t67), TSDB_CODE_TSC_INVALID_OPERATION);
char t68[] = "' abc '.' DEF '";
EXPECT_EQ(testValidateName(t68), TSDB_CODE_TSC_INVALID_OPERATION);
// do not use key words
char t69[] = "table.'DEF'";
EXPECT_EQ(testValidateName(t69), TSDB_CODE_TSC_INVALID_OPERATION);
char t70[] = "'table'.'DEF'";
EXPECT_EQ(testValidateName(t70), TSDB_CODE_TSC_INVALID_OPERATION);
char t71[] = "'_abXYZ1234 '.' deFF '";
EXPECT_EQ(testValidateName(t71), TSDB_CODE_TSC_INVALID_OPERATION);
char t72[] = "'_abDEF&^%1234'.' DIef'";
EXPECT_EQ(testValidateName(t72), TSDB_CODE_TSC_INVALID_OPERATION);
char t73[] = "'_123'.' Gtest中文'";
EXPECT_EQ(testValidateName(t73), TSDB_CODE_TSC_INVALID_OPERATION);
char t74[] = "' aABC'.'Gtest中文'";
EXPECT_EQ(testValidateName(t74), TSDB_CODE_TSC_INVALID_OPERATION);
char t75[] = "' ABC '.";
EXPECT_EQ(testValidateName(t75), TSDB_CODE_TSC_INVALID_OPERATION);
char t76[] = ".' ABC'";
EXPECT_EQ(testValidateName(t76), TSDB_CODE_TSC_INVALID_OPERATION);
char t77[] = " a . \"aTWc\" ";
EXPECT_EQ(testValidateName(t77), TSDB_CODE_TSC_INVALID_OPERATION);
char t78[] = " a.\"aTWc \"";
EXPECT_EQ(testValidateName(t78), TSDB_CODE_TSC_INVALID_OPERATION);
// ===============muti string by space ===================
// There's no such case.
// char t160[] = "A BC";
// EXPECT_EQ(testValidateName(t160), TSDB_CODE_TSC_INVALID_OPERATION);
// printf("end:%s\n", t160);
// There's no such case.
// char t161[] = "' A BC '";
// EXPECT_EQ(testValidateName(t161), TSDB_CODE_TSC_INVALID_OPERATION);
char t162[] = " AB C . de f ";
EXPECT_EQ(testValidateName(t162), TSDB_CODE_TSC_INVALID_OPERATION);
char t163[] = "' AB C . de f ";
EXPECT_EQ(testValidateName(t163), TSDB_CODE_TSC_INVALID_OPERATION);
char t163_0[] = " ab c . DE F ' ";
EXPECT_EQ(testValidateName(t163_0), TSDB_CODE_TSC_INVALID_OPERATION);
char t164[] = " ' AB C . de f ' ";
// int32_t tmp2 = testValidateName(t164);
EXPECT_EQ(testValidateName(t164), TSDB_CODE_TSC_INVALID_OPERATION);
char t165[] = " ' A BC '. de f ";
EXPECT_EQ(testValidateName(t165), TSDB_CODE_TSC_INVALID_OPERATION);
char t166[] = "' AB C '.' DE F '";
EXPECT_EQ(testValidateName(t166), TSDB_CODE_TSC_INVALID_OPERATION);
char t167[] = "ab c . ' D EF '";
EXPECT_EQ(testValidateName(t167), TSDB_CODE_TSC_INVALID_OPERATION);
char t168[] = "' a bc '.' DE F '";
EXPECT_EQ(testValidateName(t168), TSDB_CODE_TSC_INVALID_OPERATION);
}
#if 0
TEST(testCase, tvariant_convert) {
// 1. bool data to all other data types
SVariant t = {0};
_init_tvariant_bool(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.d, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.d, 0);
_init_tvariant_bool(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "FALSE");
taosVariantDestroy(&t);
_init_tvariant_bool(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"FALSE");
taosVariantDestroy(&t);
// 2. tinyint to other data types
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
EXPECT_EQ(t.i64, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
EXPECT_EQ(t.i64, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
EXPECT_EQ(t.i64, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.d, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.d, -27);
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-27");
taosVariantDestroy(&t);
_init_tvariant_tinyint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-27");
taosVariantDestroy(&t);
// 3. int to other data
// types//////////////////////////////////////////////////////////////////
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
EXPECT_EQ(t.i64, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.d, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.d, -23997659);
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-23997659");
taosVariantDestroy(&t);
_init_tvariant_int(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-23997659");
taosVariantDestroy(&t);
// 4. bigint to other data
// type//////////////////////////////////////////////////////////////////////////////
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_TINYINT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_SMALLINT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_INT), 0);
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_EQ(t.d, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_EQ(t.d, -3333333333333);
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-3333333333333");
taosVariantDestroy(&t);
_init_tvariant_bigint(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-3333333333333");
taosVariantDestroy(&t);
// 5. float to other data
// types////////////////////////////////////////////////////////////////////////
_init_tvariant_float(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_float(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -8991212199);
_init_tvariant_float(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.d, -8991212199.8987885);
_init_tvariant_float(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.d, -8991212199.8987885);
_init_tvariant_float(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-8991212199.898788");
taosVariantDestroy(&t);
_init_tvariant_float(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-8991212199.898788");
taosVariantDestroy(&t);
// 6. binary to other data types
// //////////////////////////////////////////////////////////////////
t.pz = "true";
t.nLen = strlen(t.pz);
t.nType = TSDB_DATA_TYPE_BINARY;
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 1);
_init_tvariant_binary(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), -1);
_init_tvariant_binary(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.d, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.d, 200000);
_init_tvariant_binary(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "2e5");
taosVariantDestroy(&t);
_init_tvariant_binary(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"2e5");
taosVariantDestroy(&t);
// 7. nchar to other data types
// //////////////////////////////////////////////////////////////////
t.wpz = L"FALSE";
t.nLen = wcslen(t.wpz);
t.nType = TSDB_DATA_TYPE_NCHAR;
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
EXPECT_EQ(t.i64, 0);
_init_tvariant_nchar(&t);
EXPECT_LE(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BOOL), 0);
_init_tvariant_nchar(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BIGINT), 0);
EXPECT_EQ(t.i64, -2000000);
_init_tvariant_nchar(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_FLOAT), 0);
EXPECT_DOUBLE_EQ(t.d, -2000000.8765);
_init_tvariant_nchar(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_DOUBLE), 0);
EXPECT_DOUBLE_EQ(t.d, -2000000.8765);
_init_tvariant_nchar(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_BINARY), 0);
EXPECT_STREQ(t.pz, "-2000000.8765");
taosVariantDestroy(&t);
_init_tvariant_nchar(&t);
EXPECT_EQ(taosVariantTypeSetType(&t, TSDB_DATA_TYPE_NCHAR), 0);
EXPECT_STREQ(t.wpz, L"-2000000.8765");
taosVariantDestroy(&t);
}
#endif
TEST(testCase, tGetToken_Test) {
char* s = ".123 ";
uint32_t type = 0;
int32_t len = tGetToken(s, &type);
EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s) - 1);
char s1[] = "1.123e10 ";
len = tGetToken(s1, &type);
EXPECT_EQ(type, TK_FLOAT);
EXPECT_EQ(len, strlen(s1) - 1);
char s4[] = "0xff ";
len = tGetToken(s4, &type);
EXPECT_EQ(type, TK_HEX);
EXPECT_EQ(len, strlen(s4) - 1);
// invalid data type
char s2[] = "e10 ";
len = tGetToken(s2, &type);
EXPECT_FALSE(type == TK_FLOAT);
char s3[] = "1.1.1.1";
len = tGetToken(s3, &type);
EXPECT_EQ(type, TK_IPTOKEN);
EXPECT_EQ(len, strlen(s3));
char s5[] = "0x ";
len = tGetToken(s5, &type);
EXPECT_FALSE(type == TK_HEX);
}
TEST(testCase, isValidNumber_test) {
SToken t1 = createToken("123abc");
EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL);
t1 = createToken("0xabc");
EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX);
t1 = createToken("0b11101");
EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN);
t1 = createToken(".134abc");
EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL);
t1 = createToken("1e1 ");
EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL);
t1 = createToken("1+2");
EXPECT_EQ(tGetNumericStringType(&t1), TK_ILLEGAL);
t1 = createToken("-0x123");
EXPECT_EQ(tGetNumericStringType(&t1), TK_HEX);
t1 = createToken("-1");
EXPECT_EQ(tGetNumericStringType(&t1), TK_INTEGER);
t1 = createToken("-0b1110");
EXPECT_EQ(tGetNumericStringType(&t1), TK_BIN);
t1 = createToken("-.234");
EXPECT_EQ(tGetNumericStringType(&t1), TK_FLOAT);
}
TEST(testCase, generateAST_test) {
SSqlInfo info = doGenerateAST("select * from t1 where ts < now");
ASSERT_EQ(info.valid, true);
SSqlInfo info1 = doGenerateAST("select * from `t.1abc` where ts<now+2h and col < 20+99");
ASSERT_EQ(info1.valid, true);
char msg[128] = {0};
SMsgBuf msgBuf = {0};
msgBuf.buf = msg;
msgBuf.len = 128;
SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &msgBuf);
ASSERT_EQ(code, 0);
SSqlInfo info2 = doGenerateAST("select * from abc where ts<now+2");
SSqlNode* pNode2 = (SSqlNode*) taosArrayGetP(((SArray*)info2.list), 0);
code = evaluateSqlNode(pNode2, TSDB_TIME_PRECISION_MILLI, &msgBuf);
ASSERT_NE(code, 0);
}
TEST(testCase, evaluateAST_test) {
SSqlInfo info1 = doGenerateAST("select a, b+22 from `t.1abc` where ts<now+2h and `col` < 20 + 99");
ASSERT_EQ(info1.valid, true);
char msg[128] = {0};
SMsgBuf msgBuf = {0};
msgBuf.buf = msg;
msgBuf.len = 128;
SSqlNode* pNode = (SSqlNode*) taosArrayGetP(((SArray*)info1.list), 0);
int32_t code = evaluateSqlNode(pNode, TSDB_TIME_PRECISION_NANO, &msgBuf);
ASSERT_EQ(code, 0);
}
TEST(testCase, extractMeta_test) {
SSqlInfo info1 = doGenerateAST("select a, b+22 from `t.1abc` where ts<now+2h and `col` < 20 + 99");
ASSERT_EQ(info1.valid, true);
char msg[128] = {0};
SMetaReq req = {0};
int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, msg, 128);
ASSERT_EQ(ret, 0);
ASSERT_EQ(taosArrayGetSize(req.pTableName), 1);
}

2
source/libs/planner/CMakeLists.txt
View File

@ -8,5 +8,5 @@ target_include_directories(
target_link_libraries(
planner
PRIVATE os util common catalog parser transport
PRIVATE os util common catalog parser transport function
)

13
source/libs/planner/inc/plannerInt.h
View File

@ -33,26 +33,25 @@ typedef struct SQueryNodeBasicInfo {
typedef struct SQueryTableInfo {
char *tableName;
uint64_t uid;
int32_t tid;
} SQueryTableInfo;
typedef struct SQueryNode {
typedef struct SQueryPlanNode {
SQueryNodeBasicInfo info;
SQueryTableInfo tableInfo;
SSchema *pSchema; // the schema of the input SSDatablock
int32_t numOfCols; // number of input columns
struct SExprInfo *pExpr; // the query functions or sql aggregations
SArray *pExpr; // the query functions or sql aggregations
int32_t numOfOutput; // number of result columns, which is also the number of pExprs
void *pExtInfo; // additional information
// previous operator to generated result for current node to process
// in case of join, multiple prev nodes exist.
SArray *pPrevNodes; // upstream nodes
struct SQueryNode *nextNode;
} SQueryNode;
struct SQueryPlanNode *nextNode;
} SQueryPlanNode;
typedef struct SQueryPhyNode {
typedef struct SQueryDistPlanNode {
} SQueryPhyNode;
} SQueryDistPlanNode;
#ifdef __cplusplus
}

553
source/libs/planner/src/planner.c
View File

@ -16,39 +16,568 @@
#include "os.h"
#include "plannerInt.h"
#include "parser.h"
#include "function.h"
int32_t qOptimizeQueryPlan(struct SQueryNode* pQueryNode) {
#define QNODE_TAGSCAN 1
#define QNODE_TABLESCAN 2
#define QNODE_PROJECT 3
#define QNODE_AGGREGATE 4
#define QNODE_GROUPBY 5
#define QNODE_LIMIT 6
#define QNODE_JOIN 7
#define QNODE_DISTINCT 8
#define QNODE_SORT 9
#define QNODE_UNIONALL 10
#define QNODE_TIMEWINDOW 11
#define QNODE_SESSIONWINDOW 12
#define QNODE_FILL 13
typedef struct SFillEssInfo {
int32_t fillType; // fill type
int64_t *val; // fill value
} SFillEssInfo;
typedef struct SJoinCond {
bool tagExists; // denote if tag condition exists or not
SColumn *tagCond[2];
SColumn *colCond[2];
} SJoinCond;
static SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo);
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode);
int32_t qOptimizeQueryPlan(struct SQueryPlanNode* pQueryNode) {
return 0;
}
int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryNode* pQueryNode) {
int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryPlanNode* pQueryNode) {
SArray* upstream = createQueryPlanImpl((struct SQueryStmtInfo*) pQueryInfo);
assert(taosArrayGetSize(upstream) == 1);
/*SQueryPlanNode* p = */taosArrayGetP(upstream, 0);
taosArrayDestroy(upstream);
return TSDB_CODE_SUCCESS;
}
int32_t qQueryPlanToString(struct SQueryPlanNode* pQueryNode, char** str) {
return 0;
}
int32_t qQueryPlanToString(struct SQueryNode* pQueryNode, char** str) {
int32_t qQueryPlanToSql(struct SQueryPlanNode* pQueryNode, char** sql) {
return 0;
}
int32_t qQueryPlanToSql(struct SQueryNode* pQueryNode, char** sql) {
int32_t qCreatePhysicalPlan(struct SQueryPlanNode* pQueryNode, struct SEpSet* pQnode, struct SQueryDistPlanNode *pPhyNode) {
return 0;
}
int32_t qCreatePhysicalPlan(struct SQueryNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyNode *pPhyNode) {
int32_t qPhyPlanToString(struct SQueryDistPlanNode *pPhyNode, char** str) {
return 0;
}
int32_t qPhyPlanToString(struct SQueryPhyNode *pPhyNode, char** str) {
return 0;
}
void* qDestroyQueryPlan(SQueryPlanNode* pQueryNode) {
if (pQueryNode == NULL) {
return NULL;
}
void* qDestroyQueryPlan(struct SQueryNode* pQueryNode) {
doDestroyQueryNode(pQueryNode);
return NULL;
}
void* qDestroyQueryPhyPlan(struct SQueryPhyNode* pQueryPhyNode) {
void* qDestroyQueryPhyPlan(struct SQueryDistPlanNode* pQueryPhyNode) {
return NULL;
}
int32_t qCreateQueryJob(const struct SQueryPhyNode* pPhyNode, struct SQueryJob** pJob) {
int32_t qCreateQueryJob(const struct SQueryDistPlanNode* pPhyNode, struct SQueryJob** pJob) {
return 0;
}
}
//======================================================================================================================
static SQueryPlanNode* createQueryNode(int32_t type, const char* name, SQueryPlanNode** prev, int32_t numOfPrev,
SExprInfo** pExpr, int32_t numOfOutput, SQueryTableInfo* pTableInfo,
void* pExtInfo) {
SQueryPlanNode* pNode = calloc(1, sizeof(SQueryPlanNode));
pNode->info.type = type;
pNode->info.name = strdup(name);
if (pTableInfo->uid != 0 && pTableInfo->tableName) { // it is a true table
pNode->tableInfo.uid = pTableInfo->uid;
pNode->tableInfo.tableName = strdup(pTableInfo->tableName);
}
pNode->numOfOutput = numOfOutput;
pNode->pExpr = calloc(numOfOutput, sizeof(SExprInfo));
for(int32_t i = 0; i < numOfOutput; ++i) {
SExprInfo* pExprInfo = taosArrayGet(pNode->pExpr, i);
assignExprInfo(pExprInfo, pExpr[i]);
}
pNode->pPrevNodes = taosArrayInit(4, POINTER_BYTES);
for(int32_t i = 0; i < numOfPrev; ++i) {
taosArrayPush(pNode->pPrevNodes, &prev[i]);
}
switch(type) {
case QNODE_TABLESCAN: {
STimeWindow* window = calloc(1, sizeof(STimeWindow));
memcpy(window, pExtInfo, sizeof(STimeWindow));
pNode->pExtInfo = window;
break;
}
case QNODE_TIMEWINDOW: {
SInterval* pInterval = calloc(1, sizeof(SInterval));
pNode->pExtInfo = pInterval;
memcpy(pInterval, pExtInfo, sizeof(SInterval));
break;
}
case QNODE_GROUPBY: {
SGroupbyExpr* p = (SGroupbyExpr*) pExtInfo;
SGroupbyExpr* pGroupbyExpr = calloc(1, sizeof(SGroupbyExpr));
pGroupbyExpr->tableIndex = p->tableIndex;
pGroupbyExpr->orderType = p->orderType;
pGroupbyExpr->orderIndex = p->orderIndex;
pGroupbyExpr->columnInfo = taosArrayDup(p->columnInfo);
pNode->pExtInfo = pGroupbyExpr;
break;
}
case QNODE_FILL: { // todo !!
pNode->pExtInfo = pExtInfo;
break;
}
case QNODE_LIMIT: {
pNode->pExtInfo = calloc(1, sizeof(SLimit));
memcpy(pNode->pExtInfo, pExtInfo, sizeof(SLimit));
break;
}
}
return pNode;
}
static SQueryPlanNode* doAddTableColumnNode(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SQueryTableInfo* info,
SArray* pExprs, SArray* tableCols) {
if (pQueryInfo->info.onlyTagQuery) {
int32_t num = (int32_t) taosArrayGetSize(pExprs);
SQueryPlanNode* pNode = createQueryNode(QNODE_TAGSCAN, "TableTagScan", NULL, 0, pExprs->pData, num, info, NULL);
if (pQueryInfo->distinct) {
pNode = createQueryNode(QNODE_DISTINCT, "Distinct", &pNode, 1, pExprs->pData, num, info, NULL);
}
return pNode;
}
STimeWindow* window = &pQueryInfo->window;
SQueryPlanNode* pNode = createQueryNode(QNODE_TABLESCAN, "TableScan", NULL, 0, NULL, 0, info, window);
if (pQueryInfo->info.projectionQuery) {
int32_t numOfOutput = (int32_t) taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExprs->pData, numOfOutput, info, NULL);
} else {
// table source column projection, generate the projection expr
int32_t numOfCols = (int32_t) taosArrayGetSize(tableCols);
SExprInfo** pExpr = calloc(numOfCols, POINTER_BYTES);
SSchema* pSchema = pTableMetaInfo->pTableMeta->schema;
STableMetaInfo* pTableMetaInfo1 = getMetaInfo(pQueryInfo, 0);
SSchema resultSchema = *pSchema;
resultSchema.colId = getNewResColId();
for (int32_t i = 0; i < numOfCols; ++i) {
SColumn* pCol = taosArrayGetP(tableCols, i);
SColumnIndex index = {.tableIndex = 0, .columnIndex = pCol->columnIndex};
SExprInfo* p = createExprInfo(pTableMetaInfo1, FUNCTION_PRJ, &index, NULL, &resultSchema, 0);
pExpr[i] = p;
}
pNode = createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pExpr, numOfCols, info, NULL);
// dropAllExprInfo(pExpr);
tfree(pExpr);
}
return pNode;
}
static SQueryPlanNode* doCreateQueryPlanForOneTableImpl(SQueryStmtInfo* pQueryInfo, SQueryPlanNode* pNode, SQueryTableInfo* info,
SArray* pExprs) {
// check for aggregation
size_t numOfGroupCols = taosArrayGetSize(pQueryInfo->groupbyExpr.columnInfo);
if (pQueryInfo->interval.interval > 0) {
int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_TIMEWINDOW, "TimeWindowAgg", &pNode, 1, pExprs->pData, numOfOutput, info, &pQueryInfo->interval);
if (numOfGroupCols != 0) {
pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info, &pQueryInfo->groupbyExpr);
}
} else if (numOfGroupCols > 0) {
int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_GROUPBY, "Groupby", &pNode, 1, pExprs->pData, numOfOutput, info,
&pQueryInfo->groupbyExpr);
} else if (pQueryInfo->sessionWindow.gap > 0) {
pNode = createQueryNode(QNODE_SESSIONWINDOW, "SessionWindowAgg", &pNode, 1, NULL, 0, info, NULL);
} else if (pQueryInfo->info.simpleAgg) {
int32_t numOfOutput = (int32_t)taosArrayGetSize(pExprs);
pNode = createQueryNode(QNODE_AGGREGATE, "Aggregate", &pNode, 1, pExprs->pData, numOfOutput, info, NULL);
}
if (pQueryInfo->havingFieldNum > 0 || pQueryInfo->info.arithmeticOnAgg) {
int32_t numOfExpr = (int32_t)taosArrayGetSize(pQueryInfo->exprList1);
pNode =
createQueryNode(QNODE_PROJECT, "Projection", &pNode, 1, pQueryInfo->exprList1->pData, numOfExpr, info, NULL);
}
if (pQueryInfo->fillType != TSDB_FILL_NONE) {
SFillEssInfo* pInfo = calloc(1, sizeof(SFillEssInfo));
pInfo->fillType = pQueryInfo->fillType;
pInfo->val = calloc(pNode->numOfOutput, sizeof(int64_t));
memcpy(pInfo->val, pQueryInfo->fillVal, pNode->numOfOutput);
pNode = createQueryNode(QNODE_FILL, "Fill", &pNode, 1, NULL, 0, info, pInfo);
}
if (pQueryInfo->limit.limit != -1 || pQueryInfo->limit.offset != 0) {
pNode = createQueryNode(QNODE_LIMIT, "Limit", &pNode, 1, NULL, 0, info, &pQueryInfo->limit);
}
return pNode;
}
static SQueryPlanNode* doCreateQueryPlanForOneTable(SQueryStmtInfo* pQueryInfo, STableMetaInfo* pTableMetaInfo, SArray* pExprs,
SArray* tableCols) {
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pTableMetaInfo->name, name);
SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
// handle the only tag query
SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, pExprs, tableCols);
if (pQueryInfo->info.onlyTagQuery) {
tfree(info.tableName);
return pNode;
}
SQueryPlanNode* pNode1 = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pExprs);
tfree(info.tableName);
return pNode1;
}
SArray* createQueryPlanImpl(SQueryStmtInfo* pQueryInfo) {
SArray* upstream = NULL;
if (pQueryInfo->pUpstream != NULL && taosArrayGetSize(pQueryInfo->pUpstream) > 0) { // subquery in the from clause
upstream = taosArrayInit(4, POINTER_BYTES);
size_t size = taosArrayGetSize(pQueryInfo->pUpstream);
for(int32_t i = 0; i < size; ++i) {
SQueryStmtInfo* pq = taosArrayGet(pQueryInfo->pUpstream, i);
SArray* p = createQueryPlanImpl(pq);
taosArrayAddBatch(upstream, p->pData, (int32_t) taosArrayGetSize(p));
}
}
if (pQueryInfo->numOfTables > 1) { // it is a join query
// 1. separate the select clause according to table
taosArrayDestroy(upstream);
upstream = taosArrayInit(5, POINTER_BYTES);
for(int32_t i = 0; i < pQueryInfo->numOfTables; ++i) {
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[i];
uint64_t uid = pTableMetaInfo->pTableMeta->uid;
SArray* exprList = taosArrayInit(4, POINTER_BYTES);
if (copyExprInfoList(exprList, pQueryInfo->exprList, uid, true) != 0) {
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
dropAllExprInfo(exprList);
exit(-1);
}
// 2. create the query execution node
char name[TSDB_TABLE_FNAME_LEN] = {0};
tNameExtractFullName(&pTableMetaInfo->name, name);
SQueryTableInfo info = {.tableName = strdup(name), .uid = pTableMetaInfo->pTableMeta->uid,};
// 3. get the required table column list
SArray* tableColumnList = taosArrayInit(4, sizeof(SColumn));
columnListCopy(tableColumnList, pQueryInfo->colList, uid);
// 4. add the projection query node
SQueryPlanNode* pNode = doAddTableColumnNode(pQueryInfo, pTableMetaInfo, &info, exprList, tableColumnList);
columnListDestroy(tableColumnList);
dropAllExprInfo(exprList);
taosArrayPush(upstream, &pNode);
}
// 3. add the join node here
SQueryTableInfo info = {0};
int32_t num = (int32_t) taosArrayGetSize(pQueryInfo->exprList);
SQueryPlanNode* pNode = createQueryNode(QNODE_JOIN, "Join", upstream->pData, pQueryInfo->numOfTables,
pQueryInfo->exprList->pData, num, &info, NULL);
// 4. add the aggregation or projection execution node
pNode = doCreateQueryPlanForOneTableImpl(pQueryInfo, pNode, &info, pQueryInfo->exprList);
upstream = taosArrayInit(5, POINTER_BYTES);
taosArrayPush(upstream, &pNode);
} else { // only one table, normal query process
STableMetaInfo* pTableMetaInfo = pQueryInfo->pTableMetaInfo[0];
SQueryPlanNode* pNode = doCreateQueryPlanForOneTable(pQueryInfo, pTableMetaInfo, pQueryInfo->exprList, pQueryInfo->colList);
upstream = taosArrayInit(5, POINTER_BYTES);
taosArrayPush(upstream, &pNode);
}
return upstream;
}
static void doDestroyQueryNode(SQueryPlanNode* pQueryNode) {
tfree(pQueryNode->pExtInfo);
tfree(pQueryNode->pSchema);
tfree(pQueryNode->info.name);
tfree(pQueryNode->tableInfo.tableName);
dropAllExprInfo(pQueryNode->pExpr);
if (pQueryNode->pPrevNodes != NULL) {
int32_t size = (int32_t) taosArrayGetSize(pQueryNode->pPrevNodes);
for(int32_t i = 0; i < size; ++i) {
SQueryPlanNode* p = taosArrayGetP(pQueryNode->pPrevNodes, i);
doDestroyQueryNode(p);
}
taosArrayDestroy(pQueryNode->pPrevNodes);
}
tfree(pQueryNode);
}
bool hasAliasName(SExprInfo* pExpr) {
assert(pExpr != NULL);
return true;
// return strncmp(pExpr->base.token, pExpr->base., tListLen(pExpr->base.aliasName)) != 0;
}
static int32_t doPrintPlan(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
if (level > 0) {
sprintf(buf + totalLen, "%*c", level, ' ');
totalLen += level;
}
int32_t len1 = sprintf(buf + totalLen, "%s(", pQueryNode->info.name);
int32_t len = len1 + totalLen;
switch(pQueryNode->info.type) {
case QNODE_TABLESCAN: {
STimeWindow* win = (STimeWindow*)pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "%s #%" PRIu64 ") time_range: %" PRId64 " - %" PRId64 "\n",
pQueryNode->tableInfo.tableName, pQueryNode->tableInfo.uid, win->skey, win->ekey);
len += len1;
break;
}
case QNODE_PROJECT: {
len1 = sprintf(buf + len, "cols: ");
len += len1;
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
SSqlExpr* p = &pExprInfo->base;
len1 = sprintf(buf + len, "[%s #%d]", p->resSchema.name, p->resSchema.colId);
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len, ", ");
len += len1;
}
}
len1 = sprintf(buf + len, ")");
len += len1;
//todo print filter info
len1 = sprintf(buf + len, " filters:(nil)\n");
len += len1;
break;
}
case QNODE_AGGREGATE: {
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
SSqlExpr* pExpr = &pExprInfo->base;
// if (hasAliasName(&pQueryNode->pExpr[i])) {
len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->resSchema.name);
// } else {
// len1 = sprintf(buf + len,"[%s]", pExpr->token);
// }
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len, ", ");
len += len1;
}
}
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
case QNODE_TIMEWINDOW: {
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
SSqlExpr* pExpr = &pExprInfo->base;
if (hasAliasName(pExprInfo)) {
len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->resSchema.name);
} else {
len1 = sprintf(buf + len,"[%s]", pExpr->token);
}
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len,", ");
len += len1;
}
}
len1 = sprintf(buf + len,") ");
len += len1;
SInterval* pInterval = pQueryNode->pExtInfo;
len1 = sprintf(buf + len, "interval:%" PRId64 "(%s), sliding:%" PRId64 "(%s), offset:%" PRId64 "\n",
pInterval->interval, TSDB_TIME_PRECISION_MILLI_STR, pInterval->sliding, TSDB_TIME_PRECISION_MILLI_STR,
pInterval->offset);
len += len1;
break;
}
case QNODE_GROUPBY: { // todo hide the invisible column
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
SSqlExpr* pExpr = &pExprInfo->base;
if (hasAliasName(pExprInfo)) {
len1 = sprintf(buf + len,"[%s #%s]", pExpr->token, pExpr->resSchema.name);
} else {
len1 = sprintf(buf + len,"[%s]", pExpr->token);
}
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len,", ");
len += len1;
}
}
SGroupbyExpr* pGroupbyExpr = pQueryNode->pExtInfo;
SColIndex* pIndex = taosArrayGet(pGroupbyExpr->columnInfo, 0);
len1 = sprintf(buf + len,") groupby_col: [%s #%d]\n", pIndex->name, pIndex->colId);
len += len1;
break;
}
case QNODE_FILL: {
SFillEssInfo* pEssInfo = pQueryNode->pExtInfo;
len1 = sprintf(buf + len,"%d", pEssInfo->fillType);
len += len1;
if (pEssInfo->fillType == TSDB_FILL_SET_VALUE) {
len1 = sprintf(buf + len,", val:");
len += len1;
// todo get the correct fill data type
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
len1 = sprintf(buf + len,"%"PRId64, pEssInfo->val[i]);
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len,", ");
len += len1;
}
}
}
len1 = sprintf(buf + len,")\n");
len += len1;
break;
}
case QNODE_LIMIT: {
SLimit* pVal = pQueryNode->pExtInfo;
len1 = sprintf(buf + len,"limit: %"PRId64", offset: %"PRId64")\n", pVal->limit, pVal->offset);
len += len1;
break;
}
case QNODE_DISTINCT:
case QNODE_TAGSCAN: {
len1 = sprintf(buf + len,"cols: ");
len += len1;
for(int32_t i = 0; i < pQueryNode->numOfOutput; ++i) {
SExprInfo* pExprInfo = taosArrayGetP(pQueryNode->pExpr, i);
SSchema* resSchema = &pExprInfo->base.resSchema;
len1 = sprintf(buf + len,"[%s #%d]", resSchema->name, resSchema->colId);
len += len1;
if (i < pQueryNode->numOfOutput - 1) {
len1 = sprintf(buf + len,", ");
len += len1;
}
}
len1 = sprintf(buf + len,")\n");
len += len1;
break;
}
case QNODE_JOIN: {
// print join condition
len1 = sprintf(buf + len, ")\n");
len += len1;
break;
}
}
return len;
}
int32_t queryPlanToStringImpl(char* buf, SQueryPlanNode* pQueryNode, int32_t level, int32_t totalLen) {
int32_t len = doPrintPlan(buf, pQueryNode, level, totalLen);
for(int32_t i = 0; i < taosArrayGetSize(pQueryNode->pPrevNodes); ++i) {
SQueryPlanNode* p1 = taosArrayGetP(pQueryNode->pPrevNodes, i);
int32_t len1 = queryPlanToStringImpl(buf, p1, level + 1, len);
len = len1;
}
return len;
}
char* queryPlanToString(SQueryPlanNode* pQueryNode) {
assert(pQueryNode);
char* buf = calloc(1, 4096);
int32_t len = sprintf(buf, "===== logic plan =====\n");
queryPlanToStringImpl(buf, pQueryNode, 0, len);
return buf;
}
SQueryPlanNode* queryPlanFromString() {
return NULL;
}

451
source/util/src/tpagedfile.c Normal file
View File

@ -0,0 +1,451 @@
#include "tpagedfile.h"
#include "thash.h"
#include "stddef.h"
#include "taoserror.h"
#include "tcompression.h"
#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES)
#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages)
int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) {
*pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf));
SDiskbasedResultBuf* pResBuf = *pResultBuf;
if (pResBuf == NULL) {
return TSDB_CODE_COM_OUT_OF_MEMORY;
}
pResBuf->pageSize = pagesize;
pResBuf->numOfPages = 0; // all pages are in buffer in the first place
pResBuf->totalBufSize = 0;
pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit.
pResBuf->allocateId = -1;
pResBuf->comp = true;
pResBuf->file = NULL;
pResBuf->qId = qId;
pResBuf->fileSize = 0;
// at least more than 2 pages must be in memory
assert(inMemBufSize >= pagesize * 2);
pResBuf->lruList = tdListNew(POINTER_BYTES);
// init id hash table
pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES
pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false);
char path[PATH_MAX] = {0};
taosGetTmpfilePath(dir, "qbuf", path);
pResBuf->path = strdup(path);
pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t));
// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize,
// pResBuf->inMemPages, pResBuf->path);
return TSDB_CODE_SUCCESS;
}
static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) {
pResultBuf->file = fopen(pResultBuf->path, "wb+");
if (pResultBuf->file == NULL) {
// qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno));
return TAOS_SYSTEM_ERROR(errno);
}
return TSDB_CODE_SUCCESS;
}
static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
if (!pResultBuf->comp) {
*dst = srcSize;
return data;
}
*dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0);
memcpy(data, pResultBuf->assistBuf, *dst);
return data;
}
static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing
if (!pResultBuf->comp) {
*dst = srcSize;
return data;
}
*dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0);
if (*dst > 0) {
memcpy(data, pResultBuf->assistBuf, *dst);
}
return data;
}
static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) {
if (pResultBuf->pFree == NULL) {
return pResultBuf->nextPos;
} else {
int32_t offset = -1;
size_t num = taosArrayGetSize(pResultBuf->pFree);
for(int32_t i = 0; i < num; ++i) {
SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i);
if (pi->len >= size) {
offset = pi->offset;
pi->offset += (int32_t)size;
pi->len -= (int32_t)size;
return offset;
}
}
// no available recycle space, allocate new area in file
return pResultBuf->nextPos;
}
}
static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
assert(!pg->used && pg->pData != NULL);
int32_t size = -1;
char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf);
// this page is flushed to disk for the first time
if (pg->info.offset == -1) {
pg->info.offset = allocatePositionInFile(pResultBuf, size);
pResultBuf->nextPos += size;
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
assert(ret == 0);
ret = (int32_t) fwrite(t, 1, size, pResultBuf->file);
assert(ret == size);
if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
pResultBuf->fileSize = pg->info.offset + pg->info.length;
}
} else {
// length becomes greater, current space is not enough, allocate new place, otherwise, do nothing
if (pg->info.length < size) {
// 1. add current space to free list
taosArrayPush(pResultBuf->pFree, &pg->info);
// 2. allocate new position, and update the info
pg->info.offset = allocatePositionInFile(pResultBuf, size);
pResultBuf->nextPos += size;
}
//3. write to disk.
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
if (ret != 0) { // todo handle the error case
}
ret = (int32_t)fwrite(t, size, 1, pResultBuf->file);
if (ret != size) { // todo handle the error case
}
if (pResultBuf->fileSize < pg->info.offset + pg->info.length) {
pResultBuf->fileSize = pg->info.offset + pg->info.length;
}
}
char* ret = pg->pData;
memset(ret, 0, pResultBuf->pageSize);
pg->pData = NULL;
pg->info.length = size;
pResultBuf->statis.flushBytes += pg->info.length;
return ret;
}
static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
int32_t ret = TSDB_CODE_SUCCESS;
assert(((int64_t) pResultBuf->numOfPages * pResultBuf->pageSize) == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages);
if (pResultBuf->file == NULL) {
if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) {
terrno = ret;
return NULL;
}
}
return doFlushPageToDisk(pResultBuf, pg);
}
// load file block data in disk
static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) {
int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET);
ret = (int32_t)fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file);
if (ret != pg->info.length) {
terrno = errno;
return NULL;
}
pResultBuf->statis.loadBytes += pg->info.length;
int32_t fullSize = 0;
doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf);
return (char*)GET_DATA_PAYLOAD(pg);
}
static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL);
SArray* pa = taosArrayInit(1, POINTER_BYTES);
int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES);
assert(ret == 0);
return pa;
}
static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) {
SIDList list = NULL;
char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
if (p == NULL) { // it is a new group id
list = addNewGroup(pResultBuf, groupId);
} else {
list = (SIDList) (*p);
}
pResultBuf->numOfPages += 1;
SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL};
ppi->pageId = pageId;
ppi->pData = NULL;
ppi->info = PAGE_INFO_INITIALIZER;
ppi->used = true;
ppi->pn = NULL;
return *(SPageInfo**) taosArrayPush(list, &ppi);
}
static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) {
SListIter iter = {0};
tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD);
SListNode* pn = NULL;
while((pn = tdListNext(&iter)) != NULL) {
assert(pn != NULL);
SPageInfo* pageInfo = *(SPageInfo**) pn->data;
assert(pageInfo->pageId >= 0 && pageInfo->pn == pn);
if (!pageInfo->used) {
break;
}
}
return pn;
}
static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) {
char* bufPage = NULL;
SListNode* pn = getEldestUnrefedPage(pResultBuf);
// all pages are referenced by user, try to allocate new space
if (pn == NULL) {
int32_t prev = pResultBuf->inMemPages;
// increase by 50% of previous mem pages
pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f);
// qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev,
// pResultBuf->inMemPages, pResultBuf->pageSize);
} else {
pResultBuf->statis.flushPages += 1;
tdListPopNode(pResultBuf->lruList, pn);
SPageInfo* d = *(SPageInfo**) pn->data;
assert(d->pn == pn);
d->pn = NULL;
tfree(pn);
bufPage = flushPageToDisk(pResultBuf, d);
}
return bufPage;
}
static void lruListPushFront(SList *pList, SPageInfo* pi) {
tdListPrepend(pList, &pi);
SListNode* front = tdListGetHead(pList);
pi->pn = front;
}
static void lruListMoveToFront(SList *pList, SPageInfo* pi) {
tdListPopNode(pList, pi->pn);
tdListPrependNode(pList, pi->pn);
}
static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) {
return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage);
}
SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) {
pResultBuf->statis.getPages += 1;
char* availablePage = NULL;
if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
availablePage = evicOneDataPage(pResultBuf);
}
// register new id in this group
*pageId = (++pResultBuf->allocateId);
// register page id info
SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId);
// add to LRU list
assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0);
lruListPushFront(pResultBuf->lruList, pi);
// add to hash map
taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES);
// allocate buf
if (availablePage == NULL) {
pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); // add extract bytes in case of zipped buffer increased.
} else {
pi->pData = availablePage;
}
pResultBuf->totalBufSize += pResultBuf->pageSize;
((void**)pi->pData)[0] = pi;
pi->used = true;
return (void *)(GET_DATA_PAYLOAD(pi));
}
SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) {
assert(pResultBuf != NULL && id >= 0);
pResultBuf->statis.getPages += 1;
SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t));
assert(pi != NULL && *pi != NULL);
if ((*pi)->pData != NULL) { // it is in memory
// no need to update the LRU list if only one page exists
if (pResultBuf->numOfPages == 1) {
(*pi)->used = true;
return (void *)(GET_DATA_PAYLOAD(*pi));
}
SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data);
assert(*pInfo == *pi);
lruListMoveToFront(pResultBuf->lruList, (*pi));
(*pi)->used = true;
return (void *)(GET_DATA_PAYLOAD(*pi));
} else { // not in memory
assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0);
char* availablePage = NULL;
if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) {
availablePage = evicOneDataPage(pResultBuf);
}
if (availablePage == NULL) {
(*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize));
} else {
(*pi)->pData = availablePage;
}
((void**)((*pi)->pData))[0] = (*pi);
lruListPushFront(pResultBuf->lruList, *pi);
(*pi)->used = true;
loadPageFromDisk(pResultBuf, *pi);
return (void *)(GET_DATA_PAYLOAD(*pi));
}
}
void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) {
assert(pResultBuf != NULL && page != NULL);
char* p = (char*) page - POINTER_BYTES;
SPageInfo* ppi = ((SPageInfo**) p)[0];
releaseResBufPageInfo(pResultBuf, ppi);
}
void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) {
assert(pi->pData != NULL && pi->used);
pi->used = false;
pResultBuf->statis.releasePages += 1;
}
size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); }
size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; }
SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) {
assert(pResultBuf != NULL);
char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t));
if (p == NULL) { // it is a new group id
return pResultBuf->emptyDummyIdList;
} else {
return (SArray*) (*p);
}
}
void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) {
if (pResultBuf == NULL) {
return;
}
if (pResultBuf->file != NULL) {
// qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb",
// pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0,
// pResultBuf->fileSize/1024.0);
fclose(pResultBuf->file);
} else {
// qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId,
// pResultBuf->totalBufSize/1024.0);
}
remove(pResultBuf->path);
tfree(pResultBuf->path);
SArray** p = taosHashIterate(pResultBuf->groupSet, NULL);
while(p) {
size_t n = taosArrayGetSize(*p);
for(int32_t i = 0; i < n; ++i) {
SPageInfo* pi = taosArrayGetP(*p, i);
tfree(pi->pData);
tfree(pi);
}
taosArrayDestroy(*p);
p = taosHashIterate(pResultBuf->groupSet, p);
}
tdListFree(pResultBuf->lruList);
taosArrayDestroy(pResultBuf->emptyDummyIdList);
taosHashCleanup(pResultBuf->groupSet);
taosHashCleanup(pResultBuf->all);
tfree(pResultBuf->assistBuf);
tfree(pResultBuf);
}
SPageInfo* getLastPageInfo(SIDList pList) {
size_t size = taosArrayGetSize(pList);
return (SPageInfo*) taosArrayGetP(pList, size - 1);
}

96
source/util/src/tutil.c
View File

@ -14,11 +14,7 @@
*/
#include "os.h"
#include "tcrc32c.h"
#include "tdef.h"
#include "tutil.h"
#include "ulog.h"
#include "taoserror.h"
int32_t strdequote(char *z) {
if (z == NULL) {
@ -26,7 +22,7 @@ int32_t strdequote(char *z) {
}
int32_t quote = z[0];
if (quote != '\'' && quote != '"') {
if (quote != '\'' && quote != '"' && quote != '`') {
return (int32_t)strlen(z);
}
@ -51,7 +47,6 @@ int32_t strdequote(char *z) {
return j + 1; // only one quote, do nothing
}
int32_t strRmquote(char *z, int32_t len){
// delete escape character: \\, \', \"
char delim = z[0];
@ -83,6 +78,33 @@ int32_t strRmquote(char *z, int32_t len){
return len - 2 - cnt;
}
int32_t strndequote(char *dst, const char* z, int32_t len) {
assert(dst != NULL);
if (z == NULL || len == 0) {
return 0;
}
int32_t quote = z[0];
int32_t i = 1, j = 0;
while (z[i] != 0) {
if (z[i] == quote) {
if (z[i + 1] == quote) {
dst[j++] = (char) quote;
i++;
} else {
dst[j++] = 0;
return (j - 1);
}
} else {
dst[j++] = z[i];
}
i++;
}
return j + 1; // only one quote, do nothing
}
size_t strtrim(char *z) {
int32_t i = 0;
@ -164,8 +186,6 @@ char *strnchr(char *haystack, char needle, int32_t len, bool skipquote) {
return NULL;
}
char* strtolower(char *dst, const char *src) {
int esc = 0;
char quote = 0, *p = dst, c;
@ -380,66 +400,6 @@ int32_t taosHexStrToByteArray(char hexstr[], char bytes[]) {
return 0;
}
// TODO move to comm module
bool taosGetVersionNumber(char *versionStr, int *versionNubmer) {
if (versionStr == NULL || versionNubmer == NULL) {
return false;
}
int versionNumberPos[5] = {0};
int len = (int)strlen(versionStr);
int dot = 0;
for (int pos = 0; pos < len && dot < 4; ++pos) {
if (versionStr[pos] == '.') {
versionStr[pos] = 0;
versionNumberPos[++dot] = pos + 1;
}
}
if (dot != 3) {
return false;
}
for (int pos = 0; pos < 4; ++pos) {
versionNubmer[pos] = atoi(versionStr + versionNumberPos[pos]);
}
versionStr[versionNumberPos[1] - 1] = '.';
versionStr[versionNumberPos[2] - 1] = '.';
versionStr[versionNumberPos[3] - 1] = '.';
return true;
}
int taosCheckVersion(char *input_client_version, char *input_server_version, int comparedSegments) {
char client_version[TSDB_VERSION_LEN] = {0};
char server_version[TSDB_VERSION_LEN] = {0};
int clientVersionNumber[4] = {0};
int serverVersionNumber[4] = {0};
tstrncpy(client_version, input_client_version, sizeof(client_version));
tstrncpy(server_version, input_server_version, sizeof(server_version));
if (!taosGetVersionNumber(client_version, clientVersionNumber)) {
uError("invalid client version:%s", client_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
if (!taosGetVersionNumber(server_version, serverVersionNumber)) {
uError("invalid server version:%s", server_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
for(int32_t i = 0; i < comparedSegments; ++i) {
if (clientVersionNumber[i] != serverVersionNumber[i]) {
uError("the %d-th number of server version:%s not matched with client version:%s", i, server_version,
client_version);
return TSDB_CODE_TSC_INVALID_VERSION;
}
}
return 0;
}
char *taosIpStr(uint32_t ipInt) {
static char ipStrArray[3][30];
static int ipStrIndex = 0;

12
src/client/src/tscUtil.c
View File

@ -2667,19 +2667,7 @@ void tscColumnCopy(SColumn* pDest, const SColumn* pSrc) {
pDest->info.bytes = pSrc->info.bytes;
}
void tscColumnListCopy(SArray* dst, const SArray* src, uint64_t tableUid) {
assert(src != NULL && dst != NULL);
size_t num = taosArrayGetSize(src);
for (int32_t i = 0; i < num; ++i) {
SColumn* pCol = taosArrayGetP(src, i);
if (pCol->tableUid == tableUid) {
SColumn* p = tscColumnClone(pCol);
taosArrayPush(dst, &p);
}
}
}
void tscColumnListCopyAll(SArray* dst, const SArray* src) {
assert(src != NULL && dst != NULL);

26
src/query/tests/astTest.cpp
View File

@ -459,7 +459,7 @@ namespace {
// two level expression tree
tExprNode *createExpr1() {
auto *pLeft = (tExprNode*) calloc(1, sizeof(tExprNode));
pLeft->nodeType = TSQL_NODE_COL;
pLeft->nodeType = TEXPR_COL_NODE;
pLeft->pSchema = (SSchema*) calloc(1, sizeof(SSchema));
strcpy(pLeft->pSchema->name, "col_a");
@ -468,14 +468,14 @@ tExprNode *createExpr1() {
pLeft->pSchema->colId = 1;
auto *pRight = (tExprNode*) calloc(1, sizeof(tExprNode));
pRight->nodeType = TSQL_NODE_VALUE;
pRight->nodeType = TEXPR_VALUE_NODE;
pRight->pVal = (tVariant*) calloc(1, sizeof(tVariant));
pRight->pVal->nType = TSDB_DATA_TYPE_INT;
pRight->pVal->i64 = 12;
auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode));
pRoot->nodeType = TSQL_NODE_EXPR;
pRoot->nodeType = TEXPR_NODE_EXPR;
pRoot->_node.optr = TSDB_RELATION_EQUAL;
pRoot->_node.pLeft = pLeft;
@ -488,7 +488,7 @@ tExprNode *createExpr1() {
// thress level expression tree
tExprNode* createExpr2() {
auto *pLeft2 = (tExprNode*) calloc(1, sizeof(tExprNode));
pLeft2->nodeType = TSQL_NODE_COL;
pLeft2->nodeType = TEXPR_COL_NODE;
pLeft2->pSchema = (SSchema*) calloc(1, sizeof(SSchema));
strcpy(pLeft2->pSchema->name, "col_a");
@ -497,7 +497,7 @@ tExprNode* createExpr2() {
pLeft2->pSchema->colId = 1;
auto *pRight2 = (tExprNode*) calloc(1, sizeof(tExprNode));
pRight2->nodeType = TSQL_NODE_VALUE;
pRight2->nodeType = TEXPR_VALUE_NODE;
pRight2->pVal = (tVariant*) calloc(1, sizeof(tVariant));
pRight2->pVal->nType = TSDB_DATA_TYPE_BINARY;
@ -506,7 +506,7 @@ tExprNode* createExpr2() {
pRight2->pVal->nLen = strlen(v);
auto *p1 = (tExprNode*) calloc(1, sizeof(tExprNode));
p1->nodeType = TSQL_NODE_EXPR;
p1->nodeType = TEXPR_NODE_EXPR;
p1->_node.optr = TSDB_RELATION_LIKE;
p1->_node.pLeft = pLeft2;
@ -514,7 +514,7 @@ tExprNode* createExpr2() {
p1->_node.hasPK = false;
auto *pLeft1 = (tExprNode*) calloc(1, sizeof(tExprNode));
pLeft1->nodeType = TSQL_NODE_COL;
pLeft1->nodeType = TEXPR_COL_NODE;
pLeft1->pSchema = (SSchema*) calloc(1, sizeof(SSchema));
strcpy(pLeft1->pSchema->name, "col_b");
@ -523,14 +523,14 @@ tExprNode* createExpr2() {
pLeft1->pSchema->colId = 99;
auto *pRight1 = (tExprNode*) calloc(1, sizeof(tExprNode));
pRight1->nodeType = TSQL_NODE_VALUE;
pRight1->nodeType = TEXPR_VALUE_NODE;
pRight1->pVal = (tVariant*) calloc(1, sizeof(tVariant));
pRight1->pVal->nType = TSDB_DATA_TYPE_DOUBLE;
pRight1->pVal->dKey = 91.99;
auto *p2 = (tExprNode*) calloc(1, sizeof(tExprNode));
p2->nodeType = TSQL_NODE_EXPR;
p2->nodeType = TEXPR_NODE_EXPR;
p2->_node.optr = TSDB_RELATION_GREATER_EQUAL;
p2->_node.pLeft = pLeft1;
@ -538,7 +538,7 @@ tExprNode* createExpr2() {
p2->_node.hasPK = false;
auto *pRoot = (tExprNode*) calloc(1, sizeof(tExprNode));
pRoot->nodeType = TSQL_NODE_EXPR;
pRoot->nodeType = TEXPR_NODE_EXPR;
pRoot->_node.optr = TSDB_RELATION_OR;
pRoot->_node.pLeft = p1;
@ -605,11 +605,11 @@ void exprSerializeTest2() {
ASSERT_EQ(c1Left->nodeType, c2Left->nodeType);
ASSERT_EQ(c2Left->nodeType, TSQL_NODE_EXPR);
ASSERT_EQ(c2Left->nodeType, TEXPR_NODE_EXPR);
ASSERT_EQ(c2Left->_node.optr, TSDB_RELATION_LIKE);
ASSERT_STRCASEEQ(c2Left->_node.pLeft->pSchema->name, "col_a");
ASSERT_EQ(c2Left->_node.pRight->nodeType, TSQL_NODE_VALUE);
ASSERT_EQ(c2Left->_node.pRight->nodeType, TEXPR_VALUE_NODE);
ASSERT_STRCASEEQ(c2Left->_node.pRight->pVal->pz, "hello world!");
@ -617,7 +617,7 @@ void exprSerializeTest2() {
tExprNode* c2Right = p2->_node.pRight;
ASSERT_EQ(c1Right->nodeType, c2Right->nodeType);
ASSERT_EQ(c2Right->nodeType, TSQL_NODE_EXPR);
ASSERT_EQ(c2Right->nodeType, TEXPR_NODE_EXPR);
ASSERT_EQ(c2Right->_node.optr, TSDB_RELATION_GREATER_EQUAL);
ASSERT_EQ(c2Right->_node.pRight->pVal->dKey, 91.99);