From 95bcc92f1159e04060d50cfff2f3e93621ad96c8 Mon Sep 17 00:00:00 2001 From: lion Date: Mon, 16 Jun 2025 19:13:52 +0800 Subject: [PATCH] log-level flag and debug/info/warn/error value supports and apply --- maker/golang/main.go | 76 ++++++++++++++++++++++++++++++++------- maker/golang/xdb/maker.go | 32 ++++++++--------- 2 files changed, 80 insertions(+), 28 deletions(-) diff --git a/maker/golang/main.go b/maker/golang/main.go index 91b6d4e..6e3937f 100644 --- a/maker/golang/main.go +++ b/maker/golang/main.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/lionsoul2014/ip2region/maker/golang/xdb" "log" + "log/slog" "os" "regexp" "strings" @@ -50,9 +51,33 @@ func iterateFlags(cb func(key string, val string) error) error { return nil } +func applyLogLevel(logLevel string) error { + // check and apply the log level + var levelLog = slog.LevelInfo + switch strings.ToLower(logLevel) { + case "debug": + levelLog = slog.LevelDebug + case "info": + levelLog = slog.LevelInfo + case "warn": + levelLog = slog.LevelWarn + case "error": + levelLog = slog.LevelError + case "": + // ignore the empty value + // and default it to LevelInfo + default: + return fmt.Errorf("invalid log level %s", logLevel) + } + + slog.SetLogLoggerLevel(levelLog) + return nil +} + func genDb() { var err error var srcFile, dstFile = "", "" + var fieldList, logLevel = "", "" var indexPolicy = xdb.VectorIndexPolicy var fErr = iterateFlags(func(key string, val string) error { switch key { @@ -60,6 +85,10 @@ func genDb() { srcFile = val case "dst": dstFile = val + case "log-level": + logLevel = val + case "field-list": + fieldList = val case "index": indexPolicy, err = xdb.IndexPolicyFromString(val) if err != nil { @@ -79,11 +108,22 @@ func genDb() { if srcFile == "" || dstFile == "" { fmt.Printf("%s gen [command options]\n", os.Args[0]) fmt.Printf("options:\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --dst string destination binary xdb file path\n") + fmt.Printf(" --src string source ip text file path\n") + fmt.Printf(" --dst string destination binary xdb file path\n") + fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") + fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") return } + // check and apply the log level + err = applyLogLevel(logLevel) + if err != nil { + slog.Error("failed to apply log level", "error", err) + return + } + + slog.Info("field-list", "value", fieldList) + // make the binary file tStart := time.Now() maker, err := xdb.NewMaker(indexPolicy, srcFile, dstFile) @@ -109,7 +149,7 @@ func genDb() { fmt.Printf("failed End: %s\n", err) } - log.Printf("Done, elapsed: %s\n", time.Since(tStart)) + slog.Info("make done", "elapsed", time.Since(tStart)) } func testSearch() { @@ -154,7 +194,8 @@ quit : exit the test program`) fmt.Print("ip2region>> ") str, err := reader.ReadString('\n') if err != nil { - log.Fatalf("failed to read string: %s", err) + slog.Error("failed to read string", "error", err) + return } line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) @@ -166,7 +207,8 @@ quit : exit the test program`) if line == "loadIndex" { err = searcher.LoadVectorIndex() if err != nil { - log.Fatalf("failed to load vector index: %s", err) + slog.Error("failed to load vector index", "error", err) + return } fmt.Printf("vector index cached\n") continue @@ -196,7 +238,7 @@ quit : exit the test program`) func testBench() { var err error - var dbFile, srcFile = "", "" + var dbFile, srcFile, logLevel = "", "", "" var ignoreError = false var fErr = iterateFlags(func(key string, val string) error { switch key { @@ -204,6 +246,8 @@ func testBench() { dbFile = val case "src": srcFile = val + case "log-level": + logLevel = val case "ignore-error": if val == "true" || val == "1" { ignoreError = true @@ -227,10 +271,18 @@ func testBench() { fmt.Printf("options:\n") fmt.Printf(" --db string ip2region binary xdb file path\n") fmt.Printf(" --src string source ip text file path\n") + fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") fmt.Printf(" --ignore-error bool keep going if bench failed\n") return } + // check and apply the log level + err = applyLogLevel(logLevel) + if err != nil { + slog.Error("failed to apply log level", "error", err) + return + } + searcher, err := xdb.NewSearcher(dbFile) if err != nil { fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err) @@ -247,12 +299,13 @@ func testBench() { } var count, errCount, tStart = 0, 0, time.Now() + slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile) var iErr = xdb.IterateSegments(handle, nil, func(seg *xdb.Segment) error { var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) - fmt.Printf("try to bench segment: `%s`\n", l) + slog.Debug("try to bench", "segment", l) mip := xdb.MidIP(seg.StartIP, seg.EndIP) for _, ip := range []uint32{seg.StartIP, xdb.MidIP(seg.EndIP, mip), mip, xdb.MidIP(mip, seg.EndIP), seg.EndIP} { - fmt.Printf("|-try to bench ip '%s' ... ", xdb.Long2IP(ip)) + slog.Debug("|-try to bench", "ip", xdb.Long2IP(ip)) r, _, err := searcher.Search(ip) if err != nil { return fmt.Errorf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) @@ -262,12 +315,12 @@ func testBench() { count++ if r != seg.Region { errCount++ - fmt.Printf(" --[Failed] (%s != %s)\n", r, seg.Region) + slog.Error(" --[Failed] region not match", "src", r, "dst", seg.Region) if ignoreError == false { return fmt.Errorf("") } } else { - fmt.Printf(" --[Ok]\n") + slog.Debug(" --[Ok]") } } return nil @@ -277,7 +330,7 @@ func testBench() { return } - fmt.Printf("Bench finished, {count: %d, failed: %d, took: %s}\n", count, errCount, time.Since(tStart)) + slog.Info("Bench finished", "count", count, "failed", errCount, "elapsed", time.Since(tStart)) } func edit() { @@ -416,7 +469,6 @@ func main() { return } - // set the log flag log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) switch strings.ToLower(os.Args[1]) { case "gen": diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index 36ddaea..54d156f 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -52,7 +52,7 @@ package xdb import ( "encoding/binary" "fmt" - "log" + "log/slog" "os" "time" ) @@ -100,7 +100,7 @@ func NewMaker(policy IndexPolicy, srcFile string, dstFile string) (*Maker, error } func (m *Maker) initDbHeader() error { - log.Printf("try to init the db header ... ") + slog.Info("try to init the db header ... ") _, err := m.dstHandle.Seek(0, 0) if err != nil { @@ -134,12 +134,12 @@ func (m *Maker) initDbHeader() error { } func (m *Maker) loadSegments() error { - log.Printf("try to load the segments ... ") + slog.Info("try to load the segments ... ") var last *Segment = nil var tStart = time.Now() var iErr = IterateSegments(m.srcHandle, func(l string) { - log.Printf("load segment: `%s`", l) + slog.Debug("loaded", "segment", l) }, func(seg *Segment) error { // check the continuity of the data segment if err := seg.AfterCheck(last); err != nil { @@ -154,7 +154,7 @@ func (m *Maker) loadSegments() error { return fmt.Errorf("failed to load segments: %s", iErr) } - log.Printf("all segments loaded, length: %d, elapsed: %s", len(m.segments), time.Since(tStart)) + slog.Info("all segments loaded", "length", len(m.segments), "elapsed", time.Since(tStart)) return nil } @@ -201,12 +201,12 @@ func (m *Maker) Start() error { return fmt.Errorf("seek to data first ptr: %w", err) } - log.Printf("try to write the data block ... ") + slog.Info("try to write the data block ... ") for _, seg := range m.segments { - log.Printf("try to write region '%s' ... ", seg.Region) + slog.Debug("try to write", "region", seg.Region) ptr, has := m.regionPool[seg.Region] if has { - log.Printf(" --[Cached] with ptr=%d", ptr) + slog.Debug(" --[Cached]", "ptr=", ptr) continue } @@ -227,11 +227,11 @@ func (m *Maker) Start() error { } m.regionPool[seg.Region] = uint32(pos) - log.Printf(" --[Added] with ptr=%d", pos) + slog.Debug(" --[Added] with", "ptr", pos) } // 2, write the index block and cache the super index block - log.Printf("try to write the segment index block ... ") + slog.Info("try to write the segment index block ... ") var indexBuff = make([]byte, SegmentIndexSize) var counter, startIndexPtr, endIndexPtr = 0, int64(-1), int64(-1) for _, seg := range m.segments { @@ -249,7 +249,7 @@ func (m *Maker) Start() error { } var segList = seg.Split() - log.Printf("try to index segment(%d splits) %s ...", len(segList), seg.String()) + slog.Debug("try to index segment", "length", len(segList), "splits", seg.String()) for _, s := range segList { pos, err := m.dstHandle.Seek(0, 1) if err != nil { @@ -266,7 +266,7 @@ func (m *Maker) Start() error { return fmt.Errorf("write segment index for '%s': %w", s.String(), err) } - log.Printf("|-segment index: %d, ptr: %d, segment: %s\n", counter, pos, s.String()) + slog.Debug("|-segment index", "counter", counter, "ptr", pos, "segment", s.String()) m.setVectorIndex(s.StartIP, uint32(pos)) counter++ @@ -280,7 +280,7 @@ func (m *Maker) Start() error { } // synchronized the vector index block - log.Printf("try to write the vector index block ... ") + slog.Info("try to write the vector index block ... ") _, err = m.dstHandle.Seek(int64(HeaderInfoLength), 0) if err != nil { return fmt.Errorf("seek vector index first ptr: %w", err) @@ -291,7 +291,7 @@ func (m *Maker) Start() error { } // synchronized the segment index info - log.Printf("try to write the segment index ptr ... ") + slog.Info("try to write the segment index ptr ... ") binary.LittleEndian.PutUint32(indexBuff, uint32(startIndexPtr)) binary.LittleEndian.PutUint32(indexBuff[4:], uint32(endIndexPtr)) _, err = m.dstHandle.Seek(8, 0) @@ -304,8 +304,8 @@ func (m *Maker) Start() error { return fmt.Errorf("write segment index ptr: %w", err) } - log.Printf("write done, dataBlocks: %d, indexBlocks: (%d, %d), indexPtr: (%d, %d)", - len(m.regionPool), len(m.segments), counter, startIndexPtr, endIndexPtr) + slog.Info("write done", "dataBlocks", len(m.regionPool), "indexBlocks", len(m.segments), + "counter", counter, "startIndexPtr", startIndexPtr, "endIndexPtr", endIndexPtr) return nil }