[notification] add /notification/read, add message_type (#47)

[notification] add /notification/read, add message_type

Co-authored-by: Kassian Sun <kassiansun@outlook.com>
This commit is contained in:
sunjiacheng 2023-04-02 15:02:30 +08:00 committed by medcl
parent 55ac534b02
commit 8fddcdaf09
5 changed files with 155 additions and 86 deletions

View File

@ -7,9 +7,16 @@ import (
type NotificationType string
const (
NotificationTypeProductNews NotificationType = "PRODUCT_NEWS"
NotificationTypeAlertTriggered NotificationType = "ALERT_TRIGGERED"
NotificationTypeDataMigration NotificationType = "DATA_MIGRATION"
NotificationTypeNotification NotificationType = "NOTIFICATION"
NotificationTypeTodo NotificationType = "TODO"
)
type MessageType string
const (
MessageTypeNews MessageType = "NEWS"
MessageTypeAlerting MessageType = "ALERTING"
MessageTypeMigration MessageType = "MIGRATION"
)
type NotificationStatus string
@ -24,6 +31,7 @@ type Notification struct {
UserId string `json:"user_id,omitempty" elastic_mapping:"user_id: { type: keyword }"`
NotificationType NotificationType `json:"notification_type,omitempty" elastic_mapping:"notification_type:{type:keyword,fields:{text: {type: text}}}"`
MessageType MessageType `json:"message_type,omitempty" elastic_mapping:"message_type:{type:keyword,fields:{text: {type: text}}}"`
Status NotificationStatus `json:"status,omitempty" elastic_mapping:"status: { type: keyword }"`
Title string `json:"title,omitempty" elastic_mapping:"title: { type: keyword }"`
Body string `json:"body,omitempty" elastic_mapping:"body: { type: keyword }"`

View File

@ -11,4 +11,5 @@ type NotificationAPI struct {
func InitAPI() {
notification := NotificationAPI{}
api.HandleAPIMethod(api.GET, "/notification", notification.RequireLogin(notification.listNotifications))
api.HandleAPIMethod(api.POST, "/notification/read", notification.RequireLogin(notification.setNotificationsRead))
}

View File

@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"strconv"
"time"
log "github.com/cihub/seelog"
"infini.sh/console/model"
@ -17,7 +18,7 @@ import (
func (h *NotificationAPI) listNotifications(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
user, err := rbac.FromUserContext(req.Context())
if err != nil {
log.Error(err)
log.Error("failed to get user from context, err: %v", err)
h.WriteError(w, err.Error(), http.StatusInternalServerError)
return
}
@ -35,7 +36,8 @@ func (h *NotificationAPI) listNotifications(w http.ResponseWriter, req *http.Req
],
"query": {
"bool": { "must": [
{ "term": {"user_id": { "value": "%s" } } }
{ "term": {"user_id": { "value": "%s" } } },
{ "term": {"status": { "value": "%s" } } }
] }
},
"size": %d, "from": %d
@ -53,7 +55,7 @@ func (h *NotificationAPI) listNotifications(w http.ResponseWriter, req *http.Req
}
q := orm.Query{}
queryDSL = fmt.Sprintf(queryDSL, user.UserId, size, from)
queryDSL = fmt.Sprintf(queryDSL, user.UserId, model.NotificationStatusNew, size, from)
q.RawQuery = util.UnsafeStringToBytes(queryDSL)
err, res := orm.Search(&model.Notification{}, &q)
@ -65,3 +67,64 @@ func (h *NotificationAPI) listNotifications(w http.ResponseWriter, req *http.Req
h.WriteJSONHeader(w)
h.Write(w, res.Raw)
}
type SetNotificationsReadRequest struct {
Ids []string `json:"ids"`
}
func (h *NotificationAPI) setNotificationsRead(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
user, err := rbac.FromUserContext(req.Context())
if err != nil {
log.Error("failed to get user from context, err: %v", err)
h.WriteError(w, err.Error(), http.StatusInternalServerError)
return
}
if user == nil {
log.Error(errors.New("no user info"))
h.WriteError(w, "no user info", http.StatusInternalServerError)
return
}
var reqData = SetNotificationsReadRequest{}
err = h.DecodeJSON(req, &reqData)
if err != nil {
log.Error("failed to parse request: ", err)
h.WriteError(w, err.Error(), http.StatusBadRequest)
return
}
now := time.Now().Format(time.RFC3339Nano)
queryDsl := util.MapStr{
"query": util.MapStr{
"bool": util.MapStr{
"must": []util.MapStr{
{
"terms": util.MapStr{
"_id": reqData.Ids,
},
},
{
"term": util.MapStr{
"status": util.MapStr{
"value": model.NotificationStatusNew,
},
},
},
},
},
},
"script": util.MapStr{
"source": fmt.Sprintf("ctx._source['status'] = '%s';ctx._source['updated'] = '%s'", model.NotificationStatusRead, now),
},
}
err = orm.UpdateBy(model.Notification{}, util.MustToJSONBytes(queryDsl))
if err != nil {
log.Errorf("failed to update notifications, err: %v", err)
h.WriteError(w, "update notifications failed", http.StatusInternalServerError)
return
}
h.WriteAckOKJSON(w)
}

View File

@ -7,6 +7,11 @@ package migration
import (
"context"
"fmt"
"net/http"
"strconv"
"strings"
"time"
log "github.com/cihub/seelog"
"infini.sh/console/model"
"infini.sh/framework/core/api"
@ -17,16 +22,10 @@ import (
"infini.sh/framework/core/orm"
task2 "infini.sh/framework/core/task"
"infini.sh/framework/core/util"
"net/http"
"strconv"
"strings"
"time"
)
func InitAPI() {
handler := APIHandler{
}
handler := APIHandler{}
api.HandleAPIMethod(api.GET, "/migration/data/_search", handler.RequireLogin(handler.searchDataMigrationTask))
api.HandleAPIMethod(api.POST, "/migration/data", handler.RequireLogin(handler.createDataMigrationTask))
api.HandleAPIMethod(api.POST, "/migration/data/_validate", handler.RequireLogin(handler.validateDataMigration))
@ -47,7 +46,7 @@ type APIHandler struct {
bulkResultIndexName string
}
func (h *APIHandler) createDataMigrationTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) createDataMigrationTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
clusterTaskConfig := &ElasticDataConfig{}
err := h.DecodeJSON(req, clusterTaskConfig)
if err != nil {
@ -194,7 +193,7 @@ func (h *APIHandler) searchDataMigrationTask(w http.ResponseWriter, req *http.Re
h.WriteJSON(w, searchRes, http.StatusOK)
}
func (h *APIHandler) getIndexPartitionInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) getIndexPartitionInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
var (
index = ps.MustGetParameter("index")
clusterID = ps.MustGetParameter("id")
@ -218,7 +217,7 @@ func (h *APIHandler) getIndexPartitionInfo(w http.ResponseWriter, req *http.Requ
h.WriteJSON(w, partitions, http.StatusOK)
}
func (h *APIHandler) startDataMigration(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) startDataMigration(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
taskID := ps.MustGetParameter("task_id")
obj := task2.Task{}
@ -348,7 +347,7 @@ func (h *APIHandler) stopDataMigrationTask(w http.ResponseWriter, req *http.Requ
}, 200)
}
func getTaskConfig(task *task2.Task, config interface{}) error{
func getTaskConfig(task *task2.Task, config interface{}) error {
configBytes, err := util.ToJSONBytes(task.Config)
if err != nil {
return err
@ -357,7 +356,7 @@ func getTaskConfig(task *task2.Task, config interface{}) error{
return util.FromJSONBytes(configBytes, config)
}
func getIndexRefreshInterval(indexNames []string, targetESClient elastic.API)(map[string]string, error){
func getIndexRefreshInterval(indexNames []string, targetESClient elastic.API) (map[string]string, error) {
const step = 50
var (
length = len(indexNames)
@ -366,7 +365,7 @@ func getIndexRefreshInterval(indexNames []string, targetESClient elastic.API)(ma
refreshIntervals := map[string]string{}
for i := 0; i < length; i += step {
end = i + step
if end > length - 1 {
if end > length-1 {
end = length
}
tempNames := indexNames[i:end]
@ -395,7 +394,7 @@ func getIndexRefreshInterval(indexNames []string, targetESClient elastic.API)(ma
}
func (h *APIHandler) getIndexRefreshIntervals(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) getIndexRefreshIntervals(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.MustGetParameter("task_id")
obj := task2.Task{}
@ -433,7 +432,7 @@ func (h *APIHandler) getIndexRefreshIntervals(w http.ResponseWriter, req *http.R
h.WriteJSON(w, vals, http.StatusOK)
}
func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.MustGetParameter("task_id")
obj := task2.Task{}
@ -480,7 +479,7 @@ func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.R
taskConfig.Indices[i].Percent = util.ToFixed(percent, 2)
taskConfig.Indices[i].ErrorPartitions = indexState[indexName].ErrorPartitions
if int64(count) == index.Source.Docs {
completedIndices ++
completedIndices++
}
}
cfg := global.MustLookup("cluster_migration_config")
@ -497,7 +496,7 @@ func (h *APIHandler) getDataMigrationTaskInfo(w http.ResponseWriter, req *http.R
obj.Metadata.Labels["completed_indices"] = completedIndices
h.WriteJSON(w, obj, http.StatusOK)
}
func getMajorTaskInfoByIndex(taskID string) (map[string]IndexStateInfo, error){
func getMajorTaskInfoByIndex(taskID string) (map[string]IndexStateInfo, error) {
query := util.MapStr{
"size": 0,
"aggs": util.MapStr{
@ -591,7 +590,7 @@ func getMajorTaskInfoByIndex(taskID string) (map[string]IndexStateInfo, error){
return resBody, nil
}
func getIndexTaskDocCount(index *IndexConfig, targetESClient elastic.API) (int64, error) {
func getIndexTaskDocCount(ctx context.Context, index *IndexConfig, targetESClient elastic.API) (int64, error) {
targetIndexName := index.Target.Name
if targetIndexName == "" {
if v, ok := index.IndexRename[index.Source.Name].(string); ok {
@ -622,8 +621,6 @@ func getIndexTaskDocCount(index *IndexConfig, targetESClient elastic.API) (int64
body = util.MustToJSONBytes(query)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
countRes, err := targetESClient.Count(ctx, targetIndexName, body)
if err != nil {
return 0, err
@ -634,7 +631,7 @@ func getIndexTaskDocCount(index *IndexConfig, targetESClient elastic.API) (int64
return countRes.Count, nil
}
func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.MustGetParameter("task_id")
uniqueIndexName := ps.MustGetParameter("index")
majorTask := task2.Task{}
@ -722,7 +719,7 @@ func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *htt
}
}
taskInfo["data_partition"] = len(subTasks)
var taskStats = map[string]struct{
var taskStats = map[string]struct {
ScrolledDocs float64
IndexDocs float64
}{}
@ -799,13 +796,13 @@ func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *htt
if stat, ok := taskStats[ptask.ID]; ok {
scrollDocs = stat.ScrolledDocs
indexDocs = stat.IndexDocs
}else{
} else {
if ptask.Status == task2.StatusComplete || ptask.Status == task2.StatusError {
if ptask.Metadata.Labels != nil {
if v, ok := ptask.Metadata.Labels["scrolled_docs"].(float64); ok{
if v, ok := ptask.Metadata.Labels["scrolled_docs"].(float64); ok {
scrollDocs = v
}
if v, ok := ptask.Metadata.Labels["index_docs"].(float64); ok{
if v, ok := ptask.Metadata.Labels["index_docs"].(float64); ok {
indexDocs = v
}
}
@ -840,7 +837,7 @@ func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *htt
if len(subTasks) == completedPartitions {
taskInfo["completed_time"] = completedTime
taskInfo["duration"] = completedTime - startTime
}else{
} else {
taskInfo["duration"] = time.Now().UnixMilli() - startTime
}
taskInfo["start_time"] = startTime
@ -849,7 +846,7 @@ func (h *APIHandler) getDataMigrationTaskOfIndex(w http.ResponseWriter, req *htt
h.WriteJSON(w, taskInfo, http.StatusOK)
}
func (h *APIHandler) getMigrationTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) getMigrationTask(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.MustGetParameter("task_id")
obj := task2.Task{}
@ -871,7 +868,7 @@ func (h *APIHandler) getMigrationTask(w http.ResponseWriter, req *http.Request,
}, 200)
}
func (h *APIHandler) countDocuments(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) countDocuments(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
var (
index = ps.MustGetParameter("index")
clusterID = ps.MustGetParameter("id")
@ -893,8 +890,8 @@ func (h *APIHandler) countDocuments(w http.ResponseWriter, req *http.Request, ps
})
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
ctx := req.Context()
countRes, err := client.Count(ctx, index, query)
if err != nil {
log.Error(err)
@ -904,7 +901,7 @@ func (h *APIHandler) countDocuments(w http.ResponseWriter, req *http.Request, ps
h.WriteJSON(w, countRes, http.StatusOK)
}
func (h *APIHandler) getMigrationTaskLog(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) getMigrationTaskLog(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.MustGetParameter("task_id")
query := util.MapStr{
"sort": []util.MapStr{
@ -924,7 +921,7 @@ func (h *APIHandler) getMigrationTaskLog(w http.ResponseWriter, req *http.Reques
"value": id,
},
},
},{
}, {
"term": util.MapStr{
"id": util.MapStr{
"value": id,
@ -946,7 +943,7 @@ func (h *APIHandler) getMigrationTaskLog(w http.ResponseWriter, req *http.Reques
}
}
func (h *APIHandler) updateDataMigrationTaskStatus(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) updateDataMigrationTaskStatus(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
id := ps.MustGetParameter("task_id")
obj := task2.Task{}
@ -982,7 +979,7 @@ func (h *APIHandler) updateDataMigrationTaskStatus(w http.ResponseWriter, req *h
}, 200)
}
func (h *APIHandler) validateDataMigration(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) validateDataMigration(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
typ := h.GetParameter(req, "type")
switch typ {
case "multi_type":
@ -992,9 +989,9 @@ func (h *APIHandler) validateDataMigration(w http.ResponseWriter, req *http.Requ
h.WriteError(w, "unknown parameter type", http.StatusOK)
}
func (h *APIHandler) validateMultiType(w http.ResponseWriter, req *http.Request, ps httprouter.Params){
func (h *APIHandler) validateMultiType(w http.ResponseWriter, req *http.Request, ps httprouter.Params) {
var reqBody = struct {
Cluster struct{
Cluster struct {
SourceID string `json:"source_id"`
TargetID string `json:"target_id"`
} `json:"cluster"`
@ -1018,7 +1015,7 @@ func (h *APIHandler) validateMultiType(w http.ResponseWriter, req *http.Request,
h.WriteJSON(w, util.MapStr{
"result": typeInfo,
} , http.StatusOK)
}, http.StatusOK)
}
func stopPipelineTasks(parentID string) error {
@ -1036,7 +1033,7 @@ func stopPipelineTasks(parentID string) error {
},
{
"terms": util.MapStr{
"metadata.labels.pipeline_id": []string{"es_scroll","bulk_indexing"},
"metadata.labels.pipeline_id": []string{"es_scroll", "bulk_indexing"},
},
},
},
@ -1125,7 +1122,6 @@ func getMajorTaskStatsFromInstances(majorTaskID string) (taskStats MajorTaskStat
},
},
},
},
},
},
@ -1184,7 +1180,7 @@ func getMajorTaskStatsFromInstances(majorTaskID string) (taskStats MajorTaskStat
return taskStats, nil
}
func getMajorTaskByIndexFromES(majorTaskID string)(map[string]IndexStateInfo, error){
func getMajorTaskByIndexFromES(majorTaskID string) (map[string]IndexStateInfo, error) {
taskQuery := util.MapStr{
"size": 500,
"query": util.MapStr{
@ -1252,7 +1248,7 @@ func getMajorTaskByIndexFromES(majorTaskID string)(map[string]IndexStateInfo, er
for pipelineID, status := range pipelines {
indexName := pipelineIndexNames[pipelineID]
if v, err := status.Context.GetValue("bulk_indexing.success.count"); err == nil {
if vv, ok := v.(float64); ok && indexName != ""{
if vv, ok := v.(float64); ok && indexName != "" {
st := state[indexName]
st.IndexDocs += vv
state[indexName] = st

View File

@ -1503,7 +1503,8 @@ func (p *DispatcherProcessor) sendMajorTaskNotification(taskItem *task2.Task) {
}
notification := &model.Notification{
UserId: util.ToString(creatorID),
NotificationType: model.NotificationTypeDataMigration,
NotificationType: model.NotificationTypeNotification,
MessageType: model.MessageTypeMigration,
Status: model.NotificationStatusNew,
Title: title,
Body: body,