Hongze Cheng
GitHub
commit
2a4637dd83
|
|
@ -2,7 +2,7 @@
|
|||
# lucene
|
||||
ExternalProject_Add(lucene
|
||||
GIT_REPOSITORY https://github.com/taosdata-contrib/LucenePlusPlus.git
|
||||
GIT_TAG rel_3.0.8
|
||||
GIT_TAG rel_3.0.8_td
|
||||
SOURCE_DIR "${CMAKE_SOURCE_DIR}/deps/lucene"
|
||||
BINARY_DIR ""
|
||||
#BUILD_IN_SOURCE TRUE
|
||||
|
|
|
|||
|
|
@ -2,3 +2,7 @@
|
|||
if(${BUILD_WITH_ROCKSDB})
|
||||
add_subdirectory(rocksdb)
|
||||
endif(${BUILD_WITH_ROCKSDB})
|
||||
|
||||
if(${BUILD_WITH_LUCENE})
|
||||
add_subdirectory(lucene)
|
||||
endif(${BUILD_WITH_LUCENE})
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
add_executable(luceneTest "")
|
||||
target_sources(luceneTest
|
||||
PRIVATE
|
||||
"${CMAKE_CURRENT_SOURCE_DIR}/main.cpp"
|
||||
)
|
||||
target_link_libraries(luceneTest lucene++)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#include <iostream>
|
||||
|
||||
int main(int argc, char const *argv[]) {
|
||||
std::cout << "Hello, this is lucene test" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -0,0 +1,236 @@
|
|||
```plantuml
|
||||
@startuml create_table
|
||||
skinparam sequenceMessageAlign center
|
||||
skinparam responseMessageBelowArrow true
|
||||
|
||||
participant APP as app
|
||||
box "dnode1"
|
||||
participant RPC as rpc
|
||||
participant VNODE as vnode
|
||||
participant SYNC as sync
|
||||
end box
|
||||
|
||||
box "dnode2"
|
||||
participant SYNC as sync2
|
||||
participant VNODE as vnode2
|
||||
end box
|
||||
|
||||
box "dnode3"
|
||||
participant SYNC as sync3
|
||||
participant VNODE as vnode3
|
||||
end box
|
||||
|
||||
' APP send request to dnode and RPC in dnode recv the request
|
||||
app ->rpc: create table req
|
||||
|
||||
' RPC call vnodeProcessReq() function to process the request
|
||||
rpc -> vnode: vnodeProcessReq
|
||||
note right
|
||||
callback function
|
||||
run in RPC module
|
||||
threads. The function
|
||||
only puts the request
|
||||
to a vnode queue.
|
||||
end note
|
||||
|
||||
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
|
||||
vnode -> vnode: vnodeProcessReqs()
|
||||
note right
|
||||
integrate reqs and
|
||||
process as a whole
|
||||
end note
|
||||
|
||||
|
||||
' sync the request to other nodes
|
||||
vnode -> sync: syncProcessReqs()
|
||||
|
||||
' make request persistent
|
||||
' sync -->vnode: walWrite()\n(callback function)
|
||||
|
||||
' replicate requests to other DNODES
|
||||
sync -> sync2: replication req
|
||||
sync -> sync3: replication req
|
||||
sync2 -> vnode2: walWrite()\n(callback function)
|
||||
sync2 --> sync: replication rsp\n(confirm)
|
||||
sync3 -> vnode3: walWrite()\n(callback function)
|
||||
|
||||
sync3 --> sync: replication rsp\n(confirm)
|
||||
|
||||
' send apply request
|
||||
sync -> sync2: apply req
|
||||
sync -> sync3: apply req
|
||||
|
||||
' vnode apply
|
||||
sync2 -> vnode2: vnodeApplyReqs()
|
||||
sync3 -> vnode3: vnodeApplyReqs()
|
||||
|
||||
' call apply request
|
||||
sync --> vnode: vnodeApplyReqs()\n(callback function)
|
||||
|
||||
' send response
|
||||
vnode --> rpc: rpcSendRsp()
|
||||
|
||||
' dnode send response to APP
|
||||
rpc --> app: create table rsp
|
||||
@enduml
|
||||
```
|
||||
|
||||
## Leader处理强一致写入请求
|
||||
```plantuml
|
||||
@startuml leader_process_stict_consistency
|
||||
box "dnode1"
|
||||
participant CRPC as crpc
|
||||
participant VNODE as vnode
|
||||
participant SYNC as sync
|
||||
end box
|
||||
|
||||
-> crpc: create table/submit req
|
||||
|
||||
' In CRPC threads
|
||||
group #pink "In CRPC threads"
|
||||
crpc -> vnode:vnodeProcessReq()
|
||||
note right
|
||||
A callback function
|
||||
run by CRPC thread
|
||||
to put the request
|
||||
to a vnode queue
|
||||
end note
|
||||
end
|
||||
|
||||
' In VNODE worker threads
|
||||
group #lightblue "In VNODE worker threads"
|
||||
vnode -> vnode: vnodeProcessReqs()
|
||||
note right
|
||||
VNODE process requests
|
||||
accumulated in a
|
||||
vnode write queue and
|
||||
process the batch reqs
|
||||
as a whole
|
||||
end note
|
||||
|
||||
vnode -> sync: syncProcessReqs()
|
||||
|
||||
sync -> : replication req1
|
||||
sync -> : replication req2
|
||||
end
|
||||
|
||||
group #red "SYNC threads"
|
||||
sync <- : replication rsp1
|
||||
sync <- : replication rsp2
|
||||
sync -> vnode: notify apply
|
||||
sync -> : apply rsp1
|
||||
sync -> : apply rsp2
|
||||
end
|
||||
|
||||
group #lightblue "In VNODE worker threads"
|
||||
vnode -> vnode: vnodeApplyReqs()
|
||||
vnode -> crpc:
|
||||
end
|
||||
|
||||
<- crpc: create table/submit rsp
|
||||
|
||||
@enduml
|
||||
```
|
||||
|
||||
## Follower处理强一致写入请求
|
||||
```plantuml
|
||||
@startuml follower_process_strict_consistency
|
||||
participant SYNC as sync
|
||||
participant VNODE as vnode
|
||||
|
||||
group #pink "SYNC threads"
|
||||
-> sync: replication req
|
||||
|
||||
sync -> sync: syncProcessReqs()
|
||||
note right
|
||||
In the replication
|
||||
only data is
|
||||
persisted and response
|
||||
is sent back
|
||||
end note
|
||||
|
||||
<- sync: replication rsp
|
||||
|
||||
-> sync: apply req
|
||||
|
||||
sync -> vnode: notify apply
|
||||
end
|
||||
|
||||
group #lightblue "VNODE worker threads"
|
||||
vnode -> vnode: vnodeApplyReqs()
|
||||
end
|
||||
|
||||
@enduml
|
||||
```
|
||||
|
||||
## Leader处理最终一致写入请求
|
||||
```plantuml
|
||||
@startuml leader_process_eventual_consistency
|
||||
box "dnode1"
|
||||
participant CRPC as crpc
|
||||
participant VNODE as vnode
|
||||
participant SYNC as sync
|
||||
end box
|
||||
|
||||
-> crpc: create table/submit req
|
||||
|
||||
' In CRPC threads
|
||||
group #pink "In CRPC threads"
|
||||
crpc -> vnode:vnodeProcessReq()
|
||||
note right
|
||||
A callback function
|
||||
run by CRPC thread
|
||||
to put the request
|
||||
to a vnode queue
|
||||
end note
|
||||
end
|
||||
|
||||
' In VNODE worker threads
|
||||
group #lightblue "In VNODE worker threads"
|
||||
vnode -> vnode: vnodeProcessReqs()
|
||||
note right
|
||||
VNODE process requests
|
||||
accumulated in a
|
||||
vnode write queue and
|
||||
process the batch reqs
|
||||
as a whole
|
||||
end note
|
||||
|
||||
vnode -> sync: syncProcessReqs()
|
||||
|
||||
sync -> : replication req1
|
||||
sync -> : replication req2
|
||||
|
||||
sync -> vnode: notify apply
|
||||
end
|
||||
|
||||
|
||||
group #lightblue "In VNODE worker threads"
|
||||
vnode -> vnode: vnodeApplyReqs()
|
||||
vnode -> crpc:
|
||||
end
|
||||
|
||||
<- crpc: create table/submit rsp
|
||||
|
||||
@enduml
|
||||
```
|
||||
|
||||
## Follower处理最终一致写入请求
|
||||
```plantuml
|
||||
@startuml follower_process_eventual_consistency
|
||||
participant SYNC as sync
|
||||
participant VNODE as vnode
|
||||
|
||||
group #pink "SYNC threads"
|
||||
-> sync: replication rsp
|
||||
|
||||
sync -> sync: syncProcessReqs()
|
||||
|
||||
sync -> vnode: notify VNODE \nthread to process\n the reqs
|
||||
end
|
||||
|
||||
group #lightblue "VNODE worker threads"
|
||||
vnode -> vnode: vnodeApplyReqs()
|
||||
end
|
||||
@enduml
|
||||
```
|
||||
|
|
@ -0,0 +1,311 @@
|
|||
<center><h1>VNODE Write Processes</h1></center>
|
||||
|
||||
## META Operations
|
||||
META data write operations including:
|
||||
|
||||
1. create table
|
||||
2. drop table
|
||||
3. alter table
|
||||
|
||||
We take create table as an example to figure out the whole process.
|
||||
```plantuml
|
||||
@startuml create_table
|
||||
skinparam sequenceMessageAlign center
|
||||
skinparam responseMessageBelowArrow true
|
||||
|
||||
participant APP as app
|
||||
box "dnode1"
|
||||
participant RPC as rpc
|
||||
participant VNODE as vnode
|
||||
participant SYNC as sync
|
||||
end box
|
||||
|
||||
box "dnode2"
|
||||
participant SYNC as sync2
|
||||
participant VNODE as vnode2
|
||||
end box
|
||||
|
||||
box "dnode3"
|
||||
participant SYNC as sync3
|
||||
participant VNODE as vnode3
|
||||
end box
|
||||
|
||||
' APP send request to dnode and RPC in dnode recv the request
|
||||
app ->rpc: create table req
|
||||
|
||||
' RPC call vnodeProcessReq() function to process the request
|
||||
rpc -> vnode: vnodeProcessReq
|
||||
note right
|
||||
callback function
|
||||
run in RPC module
|
||||
threads. The function
|
||||
only puts the request
|
||||
to a vnode queue.
|
||||
end note
|
||||
|
||||
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
|
||||
vnode -> vnode: vnodeProcessReqs()
|
||||
note right
|
||||
integrate reqs and
|
||||
process as a whole
|
||||
end note
|
||||
|
||||
|
||||
' sync the request to other nodes
|
||||
vnode -> sync: syncProcessReqs()
|
||||
|
||||
' make request persistent
|
||||
' sync -->vnode: walWrite()\n(callback function)
|
||||
|
||||
' replicate requests to other DNODES
|
||||
sync -> sync2: replication req
|
||||
sync -> sync3: replication req
|
||||
sync2 -> vnode2: walWrite()\n(callback function)
|
||||
sync2 --> sync: replication rsp\n(confirm)
|
||||
sync3 -> vnode3: walWrite()\n(callback function)
|
||||
|
||||
sync3 --> sync: replication rsp\n(confirm)
|
||||
|
||||
' send apply request
|
||||
sync -> sync2: apply req
|
||||
sync -> sync3: apply req
|
||||
|
||||
' vnode apply
|
||||
sync2 -> vnode2: vnodeApplyReqs()
|
||||
sync3 -> vnode3: vnodeApplyReqs()
|
||||
|
||||
' call apply request
|
||||
sync --> vnode: vnodeApplyReqs()\n(callback function)
|
||||
|
||||
' send response
|
||||
vnode --> rpc: rpcSendRsp()
|
||||
|
||||
' dnode send response to APP
|
||||
rpc --> app: create table rsp
|
||||
@enduml
|
||||
```
|
||||
|
||||
## Time-series data Operations
|
||||
There are only one operations for time-series data: data insert. We will figure out the whole process.
|
||||
|
||||
```plantuml
|
||||
@startuml create_table
|
||||
skinparam sequenceMessageAlign center
|
||||
skinparam responseMessageBelowArrow true
|
||||
|
||||
participant APP as app
|
||||
box "dnode1"
|
||||
participant RPC as rpc
|
||||
participant VNODE as vnode
|
||||
participant SYNC as sync
|
||||
end box
|
||||
|
||||
box "dnode2"
|
||||
participant SYNC as sync2
|
||||
participant VNODE as vnode2
|
||||
end box
|
||||
|
||||
box "dnode3"
|
||||
participant SYNC as sync3
|
||||
participant VNODE as vnode3
|
||||
end box
|
||||
|
||||
' APP send request to dnode and RPC in dnode recv the request
|
||||
app ->rpc: insert data req
|
||||
|
||||
' RPC call vnodeProcessReq() function to process the request
|
||||
rpc -> vnode: vnodeProcessReq
|
||||
note right
|
||||
callback function
|
||||
run in RPC module
|
||||
threads. The function
|
||||
only puts the request
|
||||
to a vnode queue.
|
||||
end note
|
||||
|
||||
' VNODE call vnodeProcessReqs() function to integrate requests and process as a whole
|
||||
vnode -> vnode: vnodeProcessReqs()
|
||||
note right
|
||||
integrate reqs and
|
||||
process as a whole
|
||||
end note
|
||||
|
||||
|
||||
' sync the request to other nodes
|
||||
vnode -> sync: syncProcessReqs()
|
||||
|
||||
' ' make request persistent
|
||||
' ' sync -->vnode: walWrite()\n(callback function)
|
||||
|
||||
' ' replicate requests to other DNODES
|
||||
sync -> sync2: replication req
|
||||
sync -> sync3: replication req
|
||||
|
||||
' vnode apply
|
||||
sync2 -> vnode2: vnodeApplyReqs()
|
||||
sync3 -> vnode3: vnodeApplyReqs()
|
||||
|
||||
' call apply request
|
||||
sync --> vnode: vnodeApplyReqs()\n(callback function)
|
||||
|
||||
' send response
|
||||
vnode --> rpc: rpcSendRsp()
|
||||
|
||||
' dnode send response to APP
|
||||
rpc --> app: insert data rsp
|
||||
@enduml
|
||||
```
|
||||
|
||||
## vnodeProcessReqs()
|
||||
```plantuml
|
||||
@startuml vnodeProcessReqs()
|
||||
participant VNODE as v
|
||||
participant SYNC as s
|
||||
|
||||
group vnodeProcessReqs()
|
||||
' Group requests and get a request batch to process as a whole
|
||||
v -> v: vnodeGetReqsFromQueue()
|
||||
note right
|
||||
integrate all write
|
||||
requests as a batch
|
||||
to process as a whole
|
||||
end note
|
||||
|
||||
' VNODE call syncProcessReqs() function to process the batch request
|
||||
v -> s: syncProcessReqs()
|
||||
|
||||
group syncProcessReqs()
|
||||
' Check if current node is leader
|
||||
alt not leader
|
||||
return NOT_LEADER
|
||||
end
|
||||
|
||||
s -> s: syncAppendReqsToLogStore()
|
||||
group syncAppendReqsToLogStore()
|
||||
s -> v: walWrite()
|
||||
note right
|
||||
There must be a
|
||||
callback function
|
||||
provided by VNODE
|
||||
to persist the
|
||||
requests in WAL
|
||||
end note
|
||||
|
||||
alt (no unapplied reqs) AND (only one node OR no meta requests)
|
||||
s -> v: vnodeApplyReqs()
|
||||
note right
|
||||
just use the woker
|
||||
thread to apply
|
||||
the requests. This
|
||||
is a callback function
|
||||
provided by VNODE
|
||||
end note
|
||||
else other cases need to wait response
|
||||
s -> s:
|
||||
note right
|
||||
save the requests in log store
|
||||
and wait for comfirmation or
|
||||
other cases
|
||||
end note
|
||||
|
||||
s ->]: send replication requests
|
||||
s ->]: send replication requests
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
@enduml
|
||||
```
|
||||
|
||||
<!-- ## syncProcessReplicationReq()
|
||||
```plantuml
|
||||
@startuml syncProcessReplicationReq
|
||||
participant SYNC as s
|
||||
participant VNODE as v
|
||||
|
||||
-> s: replication request
|
||||
s -> s:
|
||||
note right
|
||||
process the request
|
||||
to get the request
|
||||
batch
|
||||
end note
|
||||
|
||||
s -> s: syncAppendReqToLogStore()
|
||||
|
||||
s -> v: walWrite()
|
||||
|
||||
alt has meta req
|
||||
<- s: comfirmation
|
||||
else
|
||||
s -> v: vnodeApplyReqs()
|
||||
end
|
||||
|
||||
@enduml -->
|
||||
<!-- ``` -->
|
||||
|
||||
## vnodeApplyReqs()
|
||||
The function *vnodeApplyReqs()* is the actual function running by a vnode to process the requests.
|
||||
```plantuml
|
||||
@startuml vnodeApplyReqs()
|
||||
skinparam sequenceMessageAlign left
|
||||
skinparam responseMessageBelowArrow true
|
||||
|
||||
participant VNODE as vnode
|
||||
participant TQ as tq
|
||||
participant TSDB as tsdb
|
||||
participant META as meta
|
||||
|
||||
group vnodeApplyReqs()
|
||||
autonumber
|
||||
loop nReqs
|
||||
' Copy request message to vnode buffer pool
|
||||
vnode -> vnode: vnodeCopyReq()
|
||||
note right
|
||||
copy request to
|
||||
vnode buffer pool
|
||||
end note
|
||||
|
||||
vnode -> tq: tqPush()
|
||||
note right
|
||||
push the request
|
||||
to TQ so consumers
|
||||
can consume
|
||||
end note
|
||||
alt META_REQ
|
||||
autonumber 3
|
||||
vnode -> meta: metaApplyReq()
|
||||
else TS_REQ
|
||||
autonumber 3
|
||||
vnode -> tsdb: tsdbApplyReq()
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
' Check if need to commit
|
||||
alt vnode buffer pool is full
|
||||
group vnodeCommit()
|
||||
autonumber 4.1
|
||||
vnode -> tq: tqCommit()
|
||||
note right
|
||||
tqCommit may renew wal
|
||||
end note
|
||||
vnode -> meta: metaCommit();
|
||||
note right
|
||||
commit meta data
|
||||
end note
|
||||
vnode -> tsdb: tsdbCommit();
|
||||
note right
|
||||
commit time-series data
|
||||
end note
|
||||
end
|
||||
end
|
||||
end
|
||||
@enduml
|
||||
```
|
||||
<!-- meta操作：建表，删表，改表（队队列/同步）
|
||||
数据写入
|
||||
快照文件与sync的结合
|
||||
vnodeOpen（）
|
||||
vnodeClose（）
|
||||
sync.h -->
|
||||
|
|
@ -62,24 +62,24 @@ typedef struct SSyncFSM {
|
|||
void* pData;
|
||||
|
||||
// apply committed log, bufs will be free by raft module
|
||||
int (*applyLog)(struct SSyncFSM *fsm, SyncIndex index, const SSyncBuffer *buf, void *pData);
|
||||
int (*applyLog)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf, void* pData);
|
||||
|
||||
// cluster commit callback
|
||||
int (*onClusterChanged)(struct SSyncFSM *fsm, const SSyncCluster* cluster, void *pData);
|
||||
int (*onClusterChanged)(struct SSyncFSM* fsm, const SSyncCluster* cluster, void* pData);
|
||||
|
||||
// fsm return snapshot in ppBuf, bufs will be free by raft module
|
||||
// TODO: getSnapshot SHOULD be async?
|
||||
int (*getSnapshot)(struct SSyncFSM *fsm, SSyncBuffer **ppBuf, int* objId, bool *isLast);
|
||||
int (*getSnapshot)(struct SSyncFSM* fsm, SSyncBuffer** ppBuf, int* objId, bool* isLast);
|
||||
|
||||
// fsm apply snapshot with pBuf data
|
||||
int (*applySnapshot)(struct SSyncFSM *fsm, SSyncBuffer *pBuf, int objId, bool isLast);
|
||||
int (*applySnapshot)(struct SSyncFSM* fsm, SSyncBuffer* pBuf, int objId, bool isLast);
|
||||
|
||||
// call when restore snapshot and log done
|
||||
int (*onRestoreDone)(struct SSyncFSM *fsm);
|
||||
int (*onRestoreDone)(struct SSyncFSM* fsm);
|
||||
|
||||
void (*onRollback)(struct SSyncFSM *fsm, SyncIndex index, const SSyncBuffer *buf);
|
||||
void (*onRollback)(struct SSyncFSM* fsm, SyncIndex index, const SSyncBuffer* buf);
|
||||
|
||||
void (*onRoleChanged)(struct SSyncFSM *fsm, const SNodesRole* pRole);
|
||||
void (*onRoleChanged)(struct SSyncFSM* fsm, const SNodesRole* pRole);
|
||||
|
||||
} SSyncFSM;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue