fix: enroll should handle recognized nodes (#261)

Reviewed-on: https://git.infini.ltd/infini/console/pulls/261
This commit is contained in:
medcl 2023-12-16 14:48:51 +08:00
parent 0046d4d4ab
commit 0a6d0e5696
2 changed files with 94 additions and 57 deletions

View File

@ -398,7 +398,6 @@ func (h *APIHandler) autoEnrollESNode(w http.ResponseWriter, req *http.Request,
panic(errors.New("please select cluster to enroll")) panic(errors.New("please select cluster to enroll"))
} }
if autoEnrollRunning.Load() { if autoEnrollRunning.Load() {
return return
} }
@ -407,6 +406,7 @@ func (h *APIHandler) autoEnrollESNode(w http.ResponseWriter, req *http.Request,
go func(clusterInfo ClusterInfo) { go func(clusterInfo ClusterInfo) {
defer func() { defer func() {
autoEnrollRunning.Swap(false) autoEnrollRunning.Swap(false)
if !global.Env().IsDebug {
if r := recover(); r != nil { if r := recover(); r != nil {
var v string var v string
switch r.(type) { switch r.(type) {
@ -421,6 +421,7 @@ func (h *APIHandler) autoEnrollESNode(w http.ResponseWriter, req *http.Request,
log.Error(v) log.Error(v)
} }
} }
}
log.Debug("finish auto enroll") log.Debug("finish auto enroll")
}() }()
@ -454,11 +455,20 @@ func (h *APIHandler) autoEnrollESNode(w http.ResponseWriter, req *http.Request,
pids := h.bindInstanceToCluster(clusterInfo, nodes, instanceID, instanceEndpoint) pids := h.bindInstanceToCluster(clusterInfo, nodes, instanceID, instanceEndpoint)
log.Infof("instance:%v,%v, success enroll %v nodes", instanceID, instanceEndpoint, len(pids)) log.Infof("instance:%v,%v, success enroll %v nodes", instanceID, instanceEndpoint, len(pids))
} }
}
}
}
}
if len(nodes.Nodes)>0{
for k,v:=range nodes.Nodes{
log.Debug(k,v.Status,v.Enrolled)
if !v.Enrolled{
pids := h.bindInstanceToCluster(clusterInfo, nodes, instanceID, instanceEndpoint)
log.Infof("instance:%v,%v, success enroll %v nodes", instanceID, instanceEndpoint, len(pids))
}
}
}
}
}
}
}
}(clusterInfo) }(clusterInfo)
@ -528,7 +538,6 @@ func (h *APIHandler) bindInstanceToCluster(clusterInfo ClusterInfo, nodes *elast
for _, clusterID := range clusterInfo.ClusterIDs { for _, clusterID := range clusterInfo.ClusterIDs {
meta := elastic.GetMetadata(clusterID) meta := elastic.GetMetadata(clusterID)
if meta != nil { if meta != nil {
states, err := elastic.GetClient(clusterID).GetClusterState() states, err := elastic.GetClient(clusterID).GetClusterState()
if err != nil || states == nil { if err != nil || states == nil {
log.Error(err) log.Error(err)
@ -537,33 +546,67 @@ func (h *APIHandler) bindInstanceToCluster(clusterInfo ClusterInfo, nodes *elast
clusterUUID := states.ClusterUUID clusterUUID := states.ClusterUUID
if meta.Config.AgentCredentialID != "" { //no auth or agent auth configured
if meta.Config.AgentCredentialID != "" || meta.Config.CredentialID == "" {
auth, err := common.GetAgentBasicAuth(meta.Config) auth, err := common.GetAgentBasicAuth(meta.Config)
if err != nil { if err != nil {
panic(err) panic(err)
} }
if auth != nil {
for _,v:=range nodes.Nodes{
if !v.Enrolled{
if v.NodeInfo!=nil{
pid:=v.NodeInfo.Process.Id
nodeHost:=v.NodeInfo.GetHttpPublishHost()
nodeInfo:=h.internalProcessBind(clusterID,clusterUUID,instanceID,instanceEndpoint,pid,nodeHost,auth)
if nodeInfo!=nil{
discoveredPIDs[pid] = nodeInfo
}
}
}
}
//try connect //try connect
for _, node := range nodes.UnknownProcess { for _, node := range nodes.UnknownProcess {
pid:=node.PID
for _, v := range node.ListenAddresses { for _, v := range node.ListenAddresses {
ip := v.IP ip := v.IP
if util.ContainStr(v.IP, "::") { port:=v.Port
ip = fmt.Sprintf("[%s]", v.IP)
if util.ContainStr(ip, "::") {
ip = fmt.Sprintf("[%s]", ip)
} }
nodeHost := fmt.Sprintf("%s:%d", ip, v.Port) nodeHost := fmt.Sprintf("%s:%d", ip, port)
nodeInfo:=h.internalProcessBind(clusterID,clusterUUID,instanceID,instanceEndpoint,pid,nodeHost,auth)
if nodeInfo!=nil{
discoveredPIDs[pid] = nodeInfo
}
}
}
}
}
}
}
return discoveredPIDs
}
func (h *APIHandler) internalProcessBind(clusterID,clusterUUID,instanceID,instanceEndpoint string,pid int,nodeHost string,auth *model.BasicAuth) *elastic.LocalNodeInfo{
success, tryAgain, nodeInfo := h.getESNodeInfoViaProxy(nodeHost, "http", auth, instanceEndpoint) success, tryAgain, nodeInfo := h.getESNodeInfoViaProxy(nodeHost, "http", auth, instanceEndpoint)
if !success && tryAgain { if !success && tryAgain {
//try https again //try https again
success, tryAgain, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "https", auth, instanceEndpoint) success, tryAgain, nodeInfo = h.getESNodeInfoViaProxy(nodeHost, "https", auth, instanceEndpoint)
} }
if success { log.Debug(clusterUUID,nodeHost,instanceEndpoint,success, tryAgain, nodeInfo)
log.Debug("connect to es node success:", nodeHost, ", pid: ", node.PID)
discoveredPIDs[node.PID] = nodeInfo
if success {
log.Debug("connect to es node success:", nodeHost, ", pid: ", pid)
if nodeInfo.ClusterInfo.ClusterUUID != clusterUUID { if nodeInfo.ClusterInfo.ClusterUUID != clusterUUID {
log.Error("cluster uuid not match, cluster id: ", clusterID, ", cluster uuid: ", clusterUUID, ", node cluster uuid: ", nodeInfo.ClusterInfo.ClusterUUID) log.Info("cluster uuid not match, cluster id: ", clusterID, ", cluster uuid: ", clusterUUID, ", node cluster uuid: ", nodeInfo.ClusterInfo.ClusterUUID)
continue return nil
} }
//enroll this node //enroll this node
@ -574,7 +617,7 @@ func (h *APIHandler) bindInstanceToCluster(clusterInfo ClusterInfo, nodes *elast
} }
settings := NewNodeAgentSettings(instanceID, &item) settings := NewNodeAgentSettings(instanceID, &item)
err = orm.Update(&orm.Context{ err := orm.Update(&orm.Context{
Refresh: "wait_for", Refresh: "wait_for",
}, settings) }, settings)
@ -582,17 +625,11 @@ func (h *APIHandler) bindInstanceToCluster(clusterInfo ClusterInfo, nodes *elast
nodeInfo.ClusterID = clusterID nodeInfo.ClusterID = clusterID
nodeInfo.Enrolled = true nodeInfo.Enrolled = true
} }
break return nodeInfo
} }
return nil
} }
}
}
}
}
}
}
return discoveredPIDs
}
func (h *APIHandler) getESNodeInfoViaProxy(esHost string, esSchema string, auth *model.BasicAuth, endpoint string) (success, tryAgain bool, info *elastic.LocalNodeInfo) { func (h *APIHandler) getESNodeInfoViaProxy(esHost string, esSchema string, auth *model.BasicAuth, endpoint string) (success, tryAgain bool, info *elastic.LocalNodeInfo) {
esConfig := elastic.ElasticsearchConfig{Host: esHost, Schema: esSchema, BasicAuth: auth} esConfig := elastic.ElasticsearchConfig{Host: esHost, Schema: esSchema, BasicAuth: auth}

View File

@ -22,7 +22,7 @@ func GetAgentConfig() *model.AgentConfig {
} }
_, err := env.ParseConfig("agent", agentCfg ) _, err := env.ParseConfig("agent", agentCfg )
if err != nil { if err != nil {
log.Debug("agent config not found: %v", err) log.Errorf("agent config not found: %v", err)
} }
if agentCfg.Setup.CACertFile == "" && agentCfg.Setup.CAKeyFile == "" { if agentCfg.Setup.CACertFile == "" && agentCfg.Setup.CAKeyFile == "" {
agentCfg.Setup.CACertFile, agentCfg.Setup.CAKeyFile, err = common.GetOrInitDefaultCaCerts() agentCfg.Setup.CACertFile, agentCfg.Setup.CAKeyFile, err = common.GetOrInitDefaultCaCerts()