diff --git a/utils/log/survey/analyze.go b/utils/log/survey/analyze.go new file mode 100644 index 0000000..31302a4 --- /dev/null +++ b/utils/log/survey/analyze.go @@ -0,0 +1,68 @@ +package survey + +import ( + "bufio" + "github.com/kercylan98/minotaur/utils/super" + "io" + "os" + "time" +) + +// All 处理特定记录器特定日期的所有记录，当 handle 返回 false 时停止处理 +func All(name string, t time.Time, handle func(record map[string]any) bool) { + logger := survey[name] + if logger == nil { + return + } + fp := logger.filePath(t.Format(logger.layout)) + logger.wl.Lock() + defer logger.wl.Unlock() + + f, err := os.Open(fp) + if err != nil { + return + } + defer func() { + _ = f.Close() + }() + reader := bufio.NewReader(f) + var m = make(map[string]any) + for { + line, _, err := reader.ReadLine() + if err == io.EOF { + break + } + if err != nil { + panic(err) + } + if err = super.UnmarshalJSON(line[logger.dataLayoutLen:], &m); err != nil { + panic(err) + } + if !handle(m) { + break + } + } +} + +// Sum 处理特定记录器特定日期的所有记录，根据指定的字段进行汇总 +func Sum(name string, t time.Time, field string) float64 { + var res float64 + All(name, t, func(record map[string]any) bool { + v, exist := record[field] + if !exist { + return true + } + switch value := v.(type) { + case float64: + res += value + case int: + res += float64(value) + case int64: + res += float64(value) + case string: + res += super.StringToFloat64(value) + } + return true + }) + return res +} diff --git a/utils/log/survey/logger.go b/utils/log/survey/logger.go new file mode 100644 index 0000000..25a4a7c --- /dev/null +++ b/utils/log/survey/logger.go @@ -0,0 +1,86 @@ +package survey + +import ( + "bufio" + "fmt" + "github.com/kercylan98/minotaur/utils/log" + "os" + "path/filepath" + "sync" +) + +const ( + DATETIME_FORMAT = "2006-01-02 15:04:05" + DATE_FORMAT = "2006-01-02" + dateLen = len(DATE_FORMAT) + datetimeLen = len(DATETIME_FORMAT) +) + +// logger 用于埋点数据的运营日志记录器 +type logger struct { + bl sync.Mutex // writer lock + wl sync.Mutex // flush lock + dir string + fn string + fe string + bs []string + layout string + layoutLen int + dataLayout string + dataLayoutLen int +} + +// flush 将记录器缓冲区的数据写入到文件 +func (slf *logger) flush() { + slf.bl.Lock() + count := len(slf.bs) + if count == 0 { + slf.bl.Unlock() + return + } + ds := slf.bs[:] + slf.bs = slf.bs[count:] + slf.bl.Unlock() + + slf.wl.Lock() + defer slf.wl.Unlock() + + var ( + file *os.File + writer *bufio.Writer + err error + last string + ) + for _, data := range ds { + tick := data[0:slf.layoutLen] + if tick != last { + if file != nil { + _ = writer.Flush() + _ = file.Close() + } + fp := slf.filePath(tick) + file, err = os.OpenFile(fp, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + log.Fatal("Survey", log.String("Action", "DateSwitch"), log.String("FilePath", fp), log.Err(err)) + return + } + writer = bufio.NewWriterSize(file, 1024*10240) + last = tick + } + _, _ = writer.WriteString(data) + } + _ = writer.Flush() + _ = file.Close() +} + +// writer 写入数据到记录器缓冲区 +func (slf *logger) writer(d string) { + slf.bl.Lock() + slf.bs = append(slf.bs, d) + slf.bl.Unlock() +} + +// filePath 获取文件路径 +func (slf *logger) filePath(t string) string { + return filepath.Join(slf.dir, fmt.Sprintf("%s.%s%s", slf.fn, t, slf.fe)) +} diff --git a/utils/log/survey/options.go b/utils/log/survey/options.go new file mode 100644 index 0000000..b2e4b6c --- /dev/null +++ b/utils/log/survey/options.go @@ -0,0 +1,13 @@ +package survey + +// Option 选项 +type Option func(logger *logger) + +// WithLayout 设置日志文件名的时间戳格式 +// - 默认为 time.DateOnly +func WithLayout(layout string) Option { + return func(logger *logger) { + logger.layout = layout + logger.layoutLen = len(layout) + } +} diff --git a/utils/log/survey/survey.go b/utils/log/survey/survey.go new file mode 100644 index 0000000..c9d3a65 --- /dev/null +++ b/utils/log/survey/survey.go @@ -0,0 +1,56 @@ +package survey + +import ( + "fmt" + "github.com/kercylan98/minotaur/utils/log" + "github.com/kercylan98/minotaur/utils/super" + "path/filepath" + "strings" + "time" +) + +var ( + survey = make(map[string]*logger) +) + +// RegSurvey 注册一个运营日志记录器 +func RegSurvey(name, filePath string, options ...Option) { + fn := filepath.Base(filePath) + ext := filepath.Ext(fn) + fn = strings.TrimSuffix(fn, ext) + _, exist := survey[name] + if exist { + panic(fmt.Errorf("survey %s already exist", name)) + } + dir := filepath.Dir(filePath) + logger := &logger{ + dir: dir, + fn: fn, + fe: ext, + layout: time.DateOnly, + layoutLen: len(time.DateOnly), + dataLayout: time.DateTime, + dataLayoutLen: len(time.DateTime) + 3, + } + for _, option := range options { + option(logger) + } + survey[name] = logger + log.Info("Survey", log.String("Action", "RegSurvey"), log.String("Name", name), log.String("FilePath", dir+"/"+fn+".${DATE}"+ext)) +} + +// Record 记录一条运营日志 +func Record(name string, data map[string]any) { + logger, exist := survey[name] + if !exist { + panic(fmt.Errorf("survey %s not exist", name)) + } + logger.writer(fmt.Sprintf("%s - %s\n", time.Now().Format(time.DateTime), super.MarshalJSON(data))) +} + +// Flush 将所有运营日志记录器的缓冲区数据写入到文件 +func Flush() { + for _, logger := range survey { + logger.flush() + } +} diff --git a/utils/log/survey/survey_test.go b/utils/log/survey/survey_test.go new file mode 100644 index 0000000..2a568e7 --- /dev/null +++ b/utils/log/survey/survey_test.go @@ -0,0 +1,21 @@ +package survey_test + +import ( + "fmt" + "github.com/kercylan98/minotaur/utils/log/survey" + "os" + "testing" + "time" +) + +func TestRecord(t *testing.T) { + _ = os.MkdirAll("./test", os.ModePerm) + survey.RegSurvey("GLOBAL_DATA", "./test/global_data.log") + survey.Record("GLOBAL_DATA", map[string]any{ + "joinTime": time.Now().Unix(), + "action": 1, + }) + survey.Flush() + + fmt.Println(survey.Sum("GLOBAL_DATA", time.Now(), "action")) +} diff --git a/utils/log/survey/test/global_data.2023-08-22.log b/utils/log/survey/test/global_data.2023-08-22.log new file mode 100644 index 0000000..775a995 --- /dev/null +++ b/utils/log/survey/test/global_data.2023-08-22.log @@ -0,0 +1 @@ +2023-08-22 19:34:15 - {"action":1,"joinTime":1692704055}