xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into feature/mnode

This commit is contained in:
Shengliang 2022-01-24 01:56:19 -08:00
parent 9608965791 ba2898eda9
commit 0e2d0334e9
13 changed files with 580 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

79
packaging/release.sh Executable file
View File

@ -0,0 +1,79 @@
#!/bin/bash
#
# Generate the deb package for ubuntu, or rpm package for centos, or tar.gz package for other linux os
set -e
#set -x
# set parameters by default value
version="3.0.0.0"
curr_dir=$(pwd)
script_dir="$(dirname $(readlink -f $0))"
top_dir="$(readlink -f ${script_dir}/..)"
echo "=======================new version number: ${verNumber}======================================"
build_time=$(date +"%F %R")
echo "top_dir: ${top_dir}"
cd ${top_dir}
git pull || :
echo "curr_dir: ${curr_dir}"
# 2. cmake executable file
compile_dir="${top_dir}/debug"
if [ -d ${compile_dir} ]; then
rm -rf ${compile_dir}
fi
mkdir -p ${compile_dir}
cd ${compile_dir}
echo "compile_dir: ${compile_dir}"
cmake ..
make -j32
release_dir="${top_dir}/release"
if [ -d ${release_dir} ]; then
rm -rf ${release_dir}
fi
mkdir -p ${release_dir}
cd ${release_dir}
install_dir="${release_dir}/TDengine-server-${version}"
mkdir -p ${install_dir}
mkdir -p ${install_dir}/lib
bin_files="${compile_dir}/source/dnode/mgmt/daemon/taosd ${compile_dir}/tools/shell/taos ${compile_dir}/tests/test/c/create_table"
cp ${bin_files} ${install_dir}/ && chmod a+x ${install_dir}/* || :
cp ${compile_dir}/source/client/libtaos.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/mnode/impl/libmnode.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/qnode/libqnode.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/snode/libsnode.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/bnode/libbnode.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/wal/libwal.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/scheduler/libscheduler.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/planner/libplanner.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/parser/libparser.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/qcom/libqcom.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/transport/libtransport.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/function/libfunction.so ${install_dir}/lib/
cp ${compile_dir}/source/common/libcommon.so ${install_dir}/lib/
cp ${compile_dir}/source/os/libos.so ${install_dir}/lib/
cp ${compile_dir}/source/dnode/mnode/sdb/libsdb.so ${install_dir}/lib/
cp ${compile_dir}/source/libs/catalog/libcatalog.so ${install_dir}/lib/
pkg_name=${install_dir}-Linux-x64
tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || :

2
source/dnode/vnode/src/tsdb/tsdbFS.c
View File

@ -359,7 +359,7 @@ void tsdbStartFSTxn(STsdb *pRepo, int64_t pointsAdd, int64_t storageAdd) {
tsdbResetFSStatus(pfs->nstatus);
pfs->nstatus->meta = pfs->cstatus->meta;
// if (pfs->cstatus->pmf == NULL) {
pfs->nstatus->meta.version = 0;
pfs->nstatus->meta.version += 1;
// } else {
// pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1;
// }

2
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -217,7 +217,7 @@ static int32_t vnodeGetTableList(SVnode *pVnode, SRpcMsg *pMsg) {
STR_TO_VARSTR(p, n);
p += (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE);
free(n);
// free(n);
}
pFetchRsp->numOfRows = htonl(numOfTables);

19
source/libs/transport/src/rpcMain.c
View File

@ -42,6 +42,8 @@ int tsRpcMaxRetry;
int tsRpcHeadSize;
int tsRpcOverhead;
SHashObj *tsFqdnHash;
#ifndef USE_UV
typedef struct {
@ -215,6 +217,8 @@ static void rpcInitImp(void) {
tsRpcOverhead = sizeof(SRpcReqContext);
tsRpcRefId = taosOpenRef(200, rpcFree);
tsFqdnHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK);
}
int32_t rpcInit(void) {
@ -224,6 +228,9 @@ int32_t rpcInit(void) {
void rpcCleanup(void) {
taosCloseRef(tsRpcRefId);
taosHashClear(tsFqdnHash);
taosHashCleanup(tsFqdnHash);
tsFqdnHash = NULL;
tsRpcRefId = -1;
}
@ -571,7 +578,17 @@ static void rpcFreeMsg(void *msg) {
static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType) {
SRpcConn *pConn;
uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn);
uint32_t peerIp = 0;
uint32_t *pPeerIp = taosHashGet(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1);
if (pPeerIp != NULL) {
peerIp = *pPeerIp;
} else {
peerIp = taosGetIpv4FromFqdn(peerFqdn);
if (peerIp != 0xFFFFFFFF) {
taosHashPut(tsFqdnHash, peerFqdn, strlen(peerFqdn) + 1, &peerIp, sizeof(peerIp));
}
}
if (peerIp == 0xFFFFFFFF) {
tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn);
terrno = TSDB_CODE_RPC_FQDN_ERROR;

38
tests/script/sh/dockerbuilder/addDnodeToCluster.sh Executable file
View File

@ -0,0 +1,38 @@
#!/bin/bash
#
# deploy test cluster
set -e
#set -x
dnodeNumber=1
serverPort=6030
while getopts "hn:p:" arg
do
case $arg in
n)
dnodeNumber=$(echo $OPTARG)
;;
p)
serverPort=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -n [ dnode number] "
exit 0
;;
?) #unknow option
echo "unkonw argument"
exit 1
;;
esac
done
for ((i=2; i<=${dnodeNumber}; i++)); do
taos -s "create dnode node${i} port ${serverPort};" ||:
echo "create dnode node${i} port ${serverPort};"
done

46
tests/script/sh/dockerbuilder/addFqdnToHost.sh Executable file
View File

@ -0,0 +1,46 @@
#!/bin/bash
#
# deploy test cluster
set -e
#set -x
dnodeNumber=1
subnet="172.33.0.0/16"
while getopts "hn:s:" arg
do
case $arg in
n)
dnodeNumber=$(echo $OPTARG)
;;
s)
subnet=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -n [ dnode number] "
echo " -s [ subnet] "
exit 0
;;
?) #unknow option
echo "unkonw argument"
exit 1
;;
esac
done
addFqdnToHosts() {
index=$1
ipPrefix=$2
let ipIndex=index+1
echo "${ipPrefix}.${ipIndex} node${i}" >> /etc/hosts
}
ipPrefix=${subnet%.*}
for ((i=1; i<=${dnodeNumber}; i++)); do
addFqdnToHosts ${i} ${ipPrefix}
done

4
tests/script/sh/dockerbuilder/clientCfg/taos.cfg Normal file
View File

@ -0,0 +1,4 @@
debugFlag 131
firstEp node1:6030
asyncLog 0
telemetryReporting 0

136
tests/script/sh/dockerbuilder/createComposeYml.sh Executable file
View File

@ -0,0 +1,136 @@
#!/bin/bash
#
# create docker-compose.yml
set -e
#set -x
# set parameters by default value
composeYmlFile="./docker-compose.yml"
dnodeNumber=1
subnet="172.33.0.0/16"
while getopts "hn:f:s:" arg
do
case $arg in
n)
dnodeNumber=$(echo $OPTARG)
;;
f)
composeYmlFile=$(echo $OPTARG)
;;
s)
subnet=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -n [ dnode number] "
echo " -f [ yml file] "
exit 0
;;
?) #unknow option
echo "unkonw argument"
exit 1
;;
esac
done
echo "dnodeNumber=${dnodeNumber} composeYmlFile=${composeYmlFile}"
createFirstSection() {
ymlFile=$1
echo "version: '3.7'" > ${ymlFile}
echo "" >> ${ymlFile}
echo "x-node: &x-node" >> ${ymlFile}
echo " build:" >> ${ymlFile}
echo " context: ." >> ${ymlFile}
echo " dockerfile: ./tdserver/Dockerfile" >> ${ymlFile}
echo " args:" >> ${ymlFile}
echo " - PACKAGE=TDengine-server-3.0.0.0-Linux-x64.tar.gz" >> ${ymlFile}
echo " - EXTRACTDIR=TDengine-server-3.0.0.0" >> ${ymlFile}
echo " image: 'tdengine:3.0.0.0'" >> ${ymlFile}
echo " container_name: 'node1'" >> ${ymlFile}
echo " privileged: true" >> ${ymlFile}
echo " cap_add:" >> ${ymlFile}
echo " - ALL" >> ${ymlFile}
echo " stdin_open: true" >> ${ymlFile}
echo " tty: true" >> ${ymlFile}
echo " environment:" >> ${ymlFile}
echo ' TZ: "Asia/Shanghai"' >> ${ymlFile}
echo ' command: >' >> ${ymlFile}
echo ' sh -c "ln -snf /usr/share/zoneinfo/$TZ /etc/localtime &&' >> ${ymlFile}
echo ' echo $TZ > /etc/timezone &&' >> ${ymlFile}
echo ' exec sysctl -w kernel.core_pattern=/corefile/core-%e-%p"' >> ${ymlFile}
echo " restart: always" >> ${ymlFile}
echo " hostname: node1" >> ${ymlFile}
echo " command: taosd" >> ${ymlFile}
echo " deploy:" >> ${ymlFile}
echo " resources:" >> ${ymlFile}
echo " limits:" >> ${ymlFile}
echo ' cpus: "2.00"' >> ${ymlFile}
echo " memory: 4G" >> ${ymlFile}
echo " reservations:" >> ${ymlFile}
echo ' cpus: "1.00"' >> ${ymlFile}
echo " memory: 500M" >> ${ymlFile}
echo " volumes:" >> ${ymlFile}
echo " - /etc/localtime:/etc/localtime:ro" >> ${ymlFile}
echo ' - $PWD:/work' >> ${ymlFile}
echo ' - $PWD/storage/dnode1/data:/var/lib/taos' >> ${ymlFile}
echo ' - $PWD/storage/dnode1/log:/var/log/taos' >> ${ymlFile}
echo ' - $PWD/storage/dnode1/cfg:/etc/taos' >> ${ymlFile}
echo ' - $PWD/storage/dnode1/core:/corefile' >> ${ymlFile}
echo "" >> ${ymlFile}
echo "networks:" >> ${ymlFile}
echo " tdnet:" >> ${ymlFile}
echo " ipam:" >> ${ymlFile}
echo " driver: default" >> ${ymlFile}
echo " config:" >> ${ymlFile}
echo " - subnet: ${subnet}" >> ${ymlFile}
echo "" >> ${ymlFile}
echo "services:" >> ${ymlFile}
}
createSingleDnodesCfg() {
ymlFile=$1
index=$2
ipPrefix=${subnet%.*}
let ipIndex=index+1
echo " node${index}:" >> ${ymlFile}
echo " <<: *x-node" >> ${ymlFile}
echo " container_name: 'node${index}'" >> ${ymlFile}
echo " hostname: node${index}" >> ${ymlFile}
echo " networks:" >> ${ymlFile}
echo " tdnet:" >> ${ymlFile}
echo " ipv4_address: ${ipPrefix}.${ipIndex}" >> ${ymlFile}
echo " volumes:" >> ${ymlFile}
echo " - /etc/localtime:/etc/localtime:ro" >> ${ymlFile}
echo " - \$PWD:/work" >> ${ymlFile}
echo " - \$PWD/storage/dnode${index}/data:/var/lib/taos" >> ${ymlFile}
echo " - \$PWD/storage/dnode${index}/log:/var/log/taos" >> ${ymlFile}
echo " - \$PWD/storage/dnode${index}/cfg:/etc/taos" >> ${ymlFile}
echo " - \$PWD/storage/dnode${index}/core:/corefile" >> ${ymlFile}
echo "" >> ${ymlFile}
}
createDnodesOfDockerComposeYml() {
ymlFile=$1
dnodeNumber=$2
for ((i=1; i<=${dnodeNumber}; i++)); do
createSingleDnodesCfg ${ymlFile} ${i}
done
}
########################################################################################
############################### main process ##########################################
createFirstSection ${composeYmlFile}
createDnodesOfDockerComposeYml ${composeYmlFile} ${dnodeNumber}
echo "====create docker-compose.yml end===="
echo " "

88
tests/script/sh/dockerbuilder/createDnodeStorage.sh Executable file
View File

@ -0,0 +1,88 @@
#!/bin/bash
#
# setup test environment
set -e
#set -x
# set parameters by default value
dataRootDir="/data/dockerbuilder/storage"
dnodeNumber=1
firstEp="node1:6030"
while getopts "hn:r:f:" arg
do
case $arg in
n)
dnodeNumber=$(echo $OPTARG)
;;
f)
firstEp=$(echo $OPTARG)
;;
r)
dataRootDir=$(echo $OPTARG)
;;
h)
echo "Usage: `basename $0` -n [ dnode number] "
echo " -n [ dnode number] "
echo " -f [ first ep] "
echo " -r [ data root dir] "
exit 0
;;
?) #unknow option
echo "unkonw argument"
exit 1
;;
esac
done
echo "dnodeNumber=${dnodeNumber} dataRootDir=${dataRootDir} firstEp=${firstEp}"
createTaosCfg() {
cfgFile=$1/cfg/taos.cfg
firstEp=$2
fqdn=$3
echo "debugFlag 131" > ${cfgFile}
echo "firstEp ${firstEp}" >> ${cfgFile}
#echo "dataDir ${dataDir}" >> ${cfgFile}
#echo "logDir ${logDir}" >> ${cfgFile}
echo "fqdn ${fqdn}" >> ${cfgFile}
echo "supportVnodes 1024" >> ${cfgFile}
echo "asyncLog 0" >> ${cfgFile}
echo "telemetryReporting 0" >> ${cfgFile}
}
createDnodesDataDir() {
if [ -d ${dataRootDir} ]; then
rm -rf ${dataRootDir}/*
else
echo "${dataRootDir} not exist"
exit 1
fi
dnodeNumber=$1
firstEp=$2
serverPort=${startPort}
for ((i=1; i<=${dnodeNumber}; i++)); do
mkdir -p ${dataRootDir}/dnode${i}/cfg
mkdir -p ${dataRootDir}/dnode${i}/log
mkdir -p ${dataRootDir}/dnode${i}/data
mkdir -p ${dataRootDir}/dnode${i}/core
createTaosCfg ${dataRootDir}/dnode${i} ${firstEp} node${i}
done
}
########################################################################################
############################### main process ##########################################
## create director and taos.cfg for all dnode
createDnodesDataDir ${dnodeNumber} ${firstEp}
echo "====create end===="
echo " "

42
tests/script/sh/dockerbuilder/tdserver/Dockerfile Normal file
View File

@ -0,0 +1,42 @@
FROM centos:8 AS builder
ARG PACKAGE=TDengine-server-3.0.0.0-Linux-x64.tar.gz
ARG EXTRACTDIR=TDengine-server-3.0.0.0
WORKDIR /root
COPY ${PACKAGE} .
COPY tmux.conf .
COPY addDnodeToCluster.sh .
RUN tar -zxf ${PACKAGE}
RUN mv ${EXTRACTDIR}/* ./
FROM centos:8
WORKDIR /root
RUN yum install -y glibc-langpack-en dmidecode gdb
RUN yum install -y tmux net-tools
RUN yum install -y sysstat
RUN yum install -y vim
RUN echo 'alias ll="ls -l --color=auto"' >> ~/.bashrc
COPY --from=builder /root/taosd /usr/bin
COPY --from=builder /root/taos /usr/bin
COPY --from=builder /root/create_table /usr/bin
COPY --from=builder /root/tmux.conf /root/.tmux.conf
#COPY --from=builder /root/addDnodeToCluster.sh /root/addDnodeToCluster.sh
#COPY --from=builder /root/cfg/taos.cfg /etc/taos/
COPY --from=builder /root/lib/* /usr/lib/
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib"
ENV LC_CTYPE=en_US.UTF-8
ENV LANG=en_US.UTF-8
EXPOSE 6030-6042/tcp 6060/tcp 6030-6039/udp
# VOLUME [ "/var/lib/taos", "/var/log/taos", "/etc/taos" ]
CMD [ "bash" ]

115
tests/script/sh/dockerbuilder/tmux.conf Normal file
View File

@ -0,0 +1,115 @@
unbind C-b
# remap prefix to Control + z
set -g prefix C-a
# bind 'C-z C-z' to type 'C-z'
bind C-a send-prefix
# 0 is too far from ` ;)
set -g base-index 1
# Automatically set window title
set-window-option -g automatic-rename on
set-option -g set-titles on
#set -g default-terminal screen-256color
set -g status-keys vi
set -g history-limit 10000
setw -g mode-keys vi
#setw -g mode-mouse on
setw -g monitor-activity on
set -g mouse on
bind-key v split-window -h -c "#{pane_current_path}"
bind-key s split-window -v -c "#{pane_current_path}"
bind-key c new-window -c "#{pane_current_path}"
bind-key -n C-S-Left swap-window -t -1
bind-key -n C-S-Right swap-window -t +1
# bind-key J resize-pane -D 5
# bind-key K resize-pane -U 5
# bind-key H resize-pane -L 5
# bind-key L resize-pane -R 5
# bind-key M-j resize-pane -D
# bind-key M-k resize-pane -U
# bind-key M-h resize-pane -L
# bind-key M-l resize-pane -R
# Vim style pane selection
bind h select-pane -L
bind j select-pane -D
bind k select-pane -U
bind l select-pane -R
bind K kill-pane -a
# Use Alt-vim keys without prefix key to switch panes
bind -n M-h select-pane -L
bind -n M-j select-pane -D
bind -n M-k select-pane -U
bind -n M-l select-pane -R
# bind -n M-h select-pane -L \; resize-pane -Z
# bind -n M-j select-pane -D \; resize-pane -Z
# bind -n M-k select-pane -U \; resize-pane -Z
# bind -n M-l select-pane -R \; resize-pane -Z
# # Use Alt-arrow keys without prefix key to switch panes
bind -n M-Left select-pane -L
bind -n M-Right select-pane -R
bind -n M-Up select-pane -U
bind -n M-Down select-pane -D
# Shift arrow to switch windows
bind -n C-Left previous-window
bind -n C-Right next-window
bind -n M-n previous-window
bind -n M-p nejt-window
# Map Alt-z to zoom in and out pane
bind -n M-z resize-pane -Z
bind -n M-q resize-pane -Z
# # Use v to trigger selection
bind-key -T copy-mode-vi v send-keys -X begin-selection
#
# # Use y to yank current selection
bind-key -T copy-mode-vi y send-keys -X copy-selection-and-cancel
# No delay for escape key press
set -sg escape-time 0
# THEME
set -g status-style bg=black,fg=white
# set -g status-bg black
# set -g status-fg white
set -g window-status-current-style bg=green,fg=black,bold
set -g pane-active-border-style fg=green,bg=black
set -g status-interval 60
set -g status-left-length 30
set -g status-left '#[fg=green](#S) #(whoami)'
set -g status-right '#[fg=yellow]#(cut -d " " -f 1-3 /proc/loadavg)#[default] #[fg=white]%H:%M#[default]'
# synchronouse windows
# setw synchronize-panes
# pane movement
bind-key b command-prompt -p "join pane from:" "join-pane -s ':%%'"
bind-key a command-prompt -p "send pane to:" "join-pane -t ':%%'"
# # for vim
# # Smart pane switching with awareness of Vim splits.
# # See: https://github.com/christoomey/vim-tmux-navigator
# is_vim="ps -o state= -o comm= -t '#{pane_tty}' \
# | grep -iqE '^[^TXZ ]+ +(\\S+\\/)?g?(view|n?vim?x?)(diff)?$'"
# bind-key -n C-h if-shell "$is_vim" "send-keys C-h" "select-pane -L"
# bind-key -n C-j if-shell "$is_vim" "send-keys C-j" "select-pane -D"
# bind-key -n C-k if-shell "$is_vim" "send-keys C-k" "select-pane -U"
# bind-key -n C-l if-shell "$is_vim" "send-keys C-l" "select-pane -R"
# bind-key -n C-\ if-shell "$is_vim" "send-keys C-\\" "select-pane -l"
# bind-key -T copy-mode-vi C-h select-pane -L
# bind-key -T copy-mode-vi C-j select-pane -D
# bind-key -T copy-mode-vi C-k select-pane -U
# bind-key -T copy-mode-vi C-l select-pane -R
# bind-key -T copy-mode-vi C-\ select-pane -l

20
tests/script/sh/massiveTable/deployCluster.sh
View File

@ -5,10 +5,12 @@
set -e
#set -x
fqdn=`hostname`
masterDnode=slave
dataRootDir="/data"
firstEp="trd02:7000"
startPort=7000
firstEp="${fqdn}:6030"
startPort=6030
dnodeNumber=1
updateSrc=no
@ -52,23 +54,17 @@ done
curr_dir=$(readlink -f "$(dirname "$0")")
echo $curr_dir
${curr_dir}/cleanCluster.sh -r "/data"
#${curr_dir}/cleanCluster.sh -r "/data2"
${curr_dir}/cleanCluster.sh -r ${dataRootDir}
if [[ "${updateSrc}" == "yes" ]]; then
${curr_dir}/compileVersion.sh -r ${curr_dir}/../../../../ -v "3.0"
fi
${curr_dir}/setupDnodes.sh -r "/data" -n ${dnodeNumber} -f ${firstEp} -p 7000
#${curr_dir}/setupDnodes.sh -r "/data2" -n ${dnodeNumber} -f ${firstEp} -p 8000
${curr_dir}/setupDnodes.sh -r ${dataRootDir} -n ${dnodeNumber} -f ${firstEp} -p ${startPort}
if [[ "${masterDnode}" == "master" ]]; then
# create all dnode into cluster
#taos -s "create dnode trd02 port 8000;"
taos -s "create dnode trd03 port 7000;"
#taos -s "create dnode trd03 port 8000;"
taos -s "create dnode trd04 port 7000;"
#taos -s "create dnode trd04 port 8000;"
taos -s "create dnode trd03 port 6030;"
taos -s "create dnode trd04 port 6030;"
fi

6
tests/script/sh/massiveTable/setupDnodes.sh
View File

@ -13,10 +13,12 @@ set -e
# -r [ dnode root dir]
# set parameters by default value
fqdn=`hostname`
enviMode=new
dataRootDir="/data"
firstEp="localhost:7000"
startPort=7000
firstEp="${fqdn}:6030"
startPort=6030
dnodeNumber=1