refactor: survey 包 AllWithPath 函数更改为 Analyze，新增分析报告，及分析器，提供方便的统计功能

utils/log/survey/analyzer.go

@@ -0,0 +1,99 @@
+package survey
+
+import (
+	"strings"
+	"sync"
+)
+
+// Analyzer 分析器
+type Analyzer struct {
+	v      map[string]float64
+	repeat map[string]struct{}
+	subs   map[string]*Analyzer
+	m      sync.Mutex
+}
+
+// Sub 获取子分析器
+func (slf *Analyzer) Sub(key string) *Analyzer {
+	slf.m.Lock()
+	defer slf.m.Unlock()
+	if slf.subs == nil {
+		slf.subs = make(map[string]*Analyzer)
+	}
+	sub, e := slf.subs[key]
+	if !e {
+		sub = &Analyzer{}
+		slf.subs[key] = sub
+	}
+	return sub
+}
+
+// Increase 在指定 key 现有值的基础上增加 recordKey 的值
+func (slf *Analyzer) Increase(key string, record R, recordKey string) {
+	slf.m.Lock()
+	defer slf.m.Unlock()
+	if !record.Exist(recordKey) {
+		return
+	}
+	if slf.v == nil {
+		slf.v = make(map[string]float64)
+	}
+	v, e := slf.v[key]
+	if !e {
+		slf.v[key] = record.GetFloat64(recordKey)
+		return
+	}
+	slf.v[key] = v + record.GetFloat64(recordKey)
+}
+
+// IncreaseValue 在指定 key 现有值的基础上增加 value
+func (slf *Analyzer) IncreaseValue(key string, value float64) {
+	slf.m.Lock()
+	defer slf.m.Unlock()
+	if slf.v == nil {
+		slf.v = make(map[string]float64)
+	}
+	slf.v[key] += value
+}
+
+// IncreaseNonRepeat 在指定 key 现有值的基础上增加 recordKey 的值，但是当去重维度 dimension 相同时，不会增加
+func (slf *Analyzer) IncreaseNonRepeat(key string, record R, recordKey string, dimension ...string) {
+	slf.m.Lock()
+	if !record.Exist(recordKey) {
+		slf.m.Unlock()
+		return
+	}
+	if slf.repeat == nil {
+		slf.repeat = make(map[string]struct{})
+	}
+	dvs := make([]string, 0, len(dimension))
+	for _, v := range dimension {
+		dvs = append(dvs, record.GetString(v))
+	}
+	dk := strings.Join(dvs, "_")
+	if _, e := slf.repeat[dk]; e {
+		slf.m.Unlock()
+		return
+	}
+	slf.m.Unlock()
+	slf.Increase(key, record, recordKey)
+}
+
+// IncreaseValueNonRepeat 在指定 key 现有值的基础上增加 value，但是当去重维度 dimension 相同时，不会增加
+func (slf *Analyzer) IncreaseValueNonRepeat(key string, record R, value float64, dimension ...string) {
+	slf.m.Lock()
+	if slf.repeat == nil {
+		slf.repeat = make(map[string]struct{})
+	}
+	dvs := make([]string, 0, len(dimension))
+	for _, v := range dimension {
+		dvs = append(dvs, record.GetString(v))
+	}
+	dk := strings.Join(dvs, "_")
+	if _, e := slf.repeat[dk]; e {
+		slf.m.Unlock()
+		return
+	}
+	slf.m.Unlock()
+	slf.IncreaseValue(key, value)
+}

utils/log/survey/analyzer_test.go

@@ -0,0 +1,48 @@
+package survey_test
+
+import (
+	"fmt"
+	"github.com/kercylan98/minotaur/utils/log/survey"
+	"testing"
+)
+
+func TestClose(t *testing.T) {
+	path := `./test/day.2023-09-06.log`
+
+	report := survey.Analyze(path, func(analyzer *survey.Analyzer, record survey.R) {
+		switch record.GetString("type") {
+		case "open_conn":
+			analyzer.IncreaseValueNonRepeat("开播人数", record, 1, "live_id")
+		case "report_rank":
+			analyzer.IncreaseValue("开始游戏次数", 1)
+			analyzer.Increase("开播时长", record, "game_time")
+			analyzer.Sub(record.GetString("live_id")).IncreaseValue("开始游戏次数", 1)
+			analyzer.Sub(record.GetString("live_id")).Increase("开播时长", record, "game_time")
+		case "statistics":
+			analyzer.IncreaseValueNonRepeat("活跃人数", record, 1, "active_player")
+			analyzer.IncreaseValueNonRepeat("评论人数", record, 1, "comment_player")
+			analyzer.IncreaseValueNonRepeat("点赞人数", record, 1, "like_player")
+			analyzer.Sub(record.GetString("live_id")).IncreaseValueNonRepeat("活跃人数", record, 1, "active_player")
+			analyzer.Sub(record.GetString("live_id")).IncreaseValueNonRepeat("评论人数", record, 1, "comment_player")
+			analyzer.Sub(record.GetString("live_id")).IncreaseValueNonRepeat("点赞人数", record, 1, "like_player")
+
+			giftId := record.GetString("gift.gift_id")
+			if len(giftId) > 0 {
+				giftPrice := record.GetFloat64("gift.price")
+				giftCount := record.GetFloat64("gift.count")
+				giftSender := record.GetString("gift.gift_senders")
+
+				analyzer.IncreaseValue("礼物总价值", giftPrice*giftCount)
+				analyzer.IncreaseValueNonRepeat(fmt.Sprintf("送礼人数_%s", giftId), record, 1, giftSender)
+				analyzer.IncreaseValue(fmt.Sprintf("礼物总数_%s", giftId), giftCount)
+
+				analyzer.Sub(record.GetString("live_id")).IncreaseValue("礼物总价值", giftPrice*giftCount)
+				analyzer.Sub(record.GetString("live_id")).IncreaseValueNonRepeat(fmt.Sprintf("送礼人数_%s", giftId), record, 1, giftSender)
+				analyzer.Sub(record.GetString("live_id")).IncreaseValue(fmt.Sprintf("礼物总数_%s", giftId), giftCount)
+			}
+
+		}
+	})
+
+	fmt.Println(report.FilterSub("warzone0009"))
+}

utils/log/survey/record.go

@@ -18,7 +18,8 @@ func (slf R) Get(key string) Result {
 
 // Exist 判断指定 key 是否存在
 func (slf R) Exist(key string) bool {
-	return slf.Get(key).Exists()
+	v := slf.Get(key)
+	return v.Exists() && len(v.String()) > 0
 }
 
 // GetString 该函数为 Get(key).String() 的简写

utils/log/survey/report.go

@@ -0,0 +1,72 @@
+package survey
+
+import (
+	"github.com/kercylan98/minotaur/utils/super"
+)
+
+func newReport(analyzer *Analyzer) *Report {
+	report := &Report{
+		analyzer: analyzer,
+		Name:     "ROOT",
+		Values:   analyzer.v,
+		Subs:     make([]*Report, 0, len(analyzer.subs)),
+	}
+	for k, v := range analyzer.subs {
+		sub := newReport(v)
+		sub.Name = k
+		report.Subs = append(report.Subs, sub)
+	}
+	return report
+}
+
+// Report 分析报告
+type Report struct {
+	analyzer *Analyzer
+	Name     string             // 报告名称（默认为 ROOT）
+	Values   map[string]float64 `json:"Values,omitempty"`
+	Subs     []*Report          `json:"Reports,omitempty"`
+}
+
+// ReserveSub 仅保留特定名称子报告
+func (slf *Report) ReserveSub(names ...string) *Report {
+	report := newReport(slf.analyzer)
+	var newSub []*Report
+	for _, sub := range slf.Subs {
+		var exist bool
+		for _, name := range names {
+			if sub.Name == name {
+				exist = true
+				break
+			}
+		}
+		if exist {
+			newSub = append(newSub, sub)
+		}
+	}
+	report.Subs = newSub
+	return report
+}
+
+// FilterSub 过滤特定名称的子报告
+func (slf *Report) FilterSub(names ...string) *Report {
+	report := newReport(slf.analyzer)
+	var newSub []*Report
+	for _, sub := range slf.Subs {
+		var exist bool
+		for _, name := range names {
+			if sub.Name == name {
+				exist = true
+				break
+			}
+		}
+		if !exist {
+			newSub = append(newSub, sub)
+		}
+	}
+	report.Subs = newSub
+	return report
+}
+
+func (slf *Report) String() string {
+	return string(super.MarshalIndentJSON(slf, "", "  "))
+}

utils/log/survey/survey.go

@@ -113,14 +113,17 @@ func Close(names ...string) {
 	}
 }
 
-// AllWithPath 处理特定记录器特定日期的所有记录，当发生错误时，会发生 panic
+// Analyze 分析特定文件的记录，当发生错误时，会发生 panic
 //   - handle 为并行执行的，需要自行处理并发安全
 //   - 适用于外部进程对于日志文件的读取，但是需要注意的是，此时日志文件可能正在被写入，所以可能会读取到错误的数据
-func AllWithPath(filePath string, handle func(record R)) {
+func Analyze(filePath string, handle func(analyzer *Analyzer, record R)) *Report {
+	analyzer := new(Analyzer)
 	err := file.ReadLineWithParallel(filePath, 1*1024*1024*1024, func(s string) {
-		handle(R(s))
+		handle(analyzer, R(s))
 	})
 	if err != nil {
 		panic(err)
 	}
+
+	return newReport(analyzer)
 }

utils/log/survey/survey_test.go

@@ -1,36 +0,0 @@
-package survey_test
-
-import (
-	"fmt"
-	"github.com/kercylan98/minotaur/utils/log/survey"
-	"os"
-	"sync/atomic"
-	"testing"
-	"time"
-)
-
-func TestRecord(t *testing.T) {
-	_ = os.MkdirAll("./test", os.ModePerm)
-	survey.Reg("GLOBAL_DATA", "./test/global_data.log")
-	now := time.Now()
-	//for i := 0; i < 100000000; i++ {
-	//	survey.Record("GLOBAL_DATA", map[string]any{
-	//		"joinTime": time.Now().Unix(),
-	//		"action":   random.Int64(1, 999),
-	//	})
-	//	// 每500w flush一次
-	//	if i%5000000 == 0 {
-	//		survey.Flush()
-	//	}
-	//}
-	//survey.Flush()
-	//
-	var i atomic.Int64
-	survey.All("GLOBAL_DATA", time.Now(), func(record survey.R) bool {
-		i.Add(record.Get("action").Int())
-		return true
-	})
-	fmt.Println("write cost:", time.Since(now), i.Load())
-}
-
-// Line: 30000001, time: 1.45s

utils/log/survey/test/global_data.2023-08-22.log

@@ -1 +0,0 @@
-2023-08-22 19:34:15 - {"action":1,"joinTime":1692704055}