diff --git a/maker/golang/cmd/process.go b/maker/golang/cmd/process.go index 23413ea..256ecf1 100644 --- a/maker/golang/cmd/process.go +++ b/maker/golang/cmd/process.go @@ -8,6 +8,7 @@ import ( "fmt" "log/slog" "os" + "strconv" "time" "github.com/lionsoul2014/ip2region/maker/golang/xdb" @@ -19,6 +20,8 @@ func Process() { var err error var srcFile, dstFile = "", "" var fieldList, logLevel = "", "" + var clearBasedIndex = -1 + var clearValueEqual, clearValueExcept = "", "" var fErr = iterateFlags(func(key string, val string) error { switch key { case "src": @@ -27,6 +30,17 @@ func Process() { dstFile = val case "field-list": fieldList = val + case "clear-based-index": + num, err := strconv.Atoi(val) + if err != nil { + return fmt.Errorf("invalid clear-based-index '%s=%s', integer expected", key, val) + } + + clearBasedIndex = num + case "clear-value-equal": + clearValueEqual = val + case "clear-value-except": + clearValueExcept = val case "log-level": logLevel = val default: @@ -42,13 +56,28 @@ func Process() { if srcFile == "" || dstFile == "" { fmt.Printf("%s process [command options]\n", os.Args[0]) fmt.Printf("options:\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --dst string target ip text file path\n") - fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") - fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") + fmt.Printf(" --src string source ip text file path\n") + fmt.Printf(" --dst string target ip text file path\n") + fmt.Printf(" --field-list string field index list imploded with ',' eg: 0,1,2,3-6,7\n") + fmt.Printf(" --clear-based-index integer clear based index eg: 3\n") + fmt.Printf(" --clear-value-equal string clear value equal to the specified one\n") + fmt.Printf(" --clear-value-except string clear value except the specified one\n") + fmt.Printf(" --log-level string set the log level, options: debug/info/warn/error\n") return } + if clearBasedIndex > -1 { + if len(clearValueEqual) > 0 && len(clearValueExcept) > 0 { + fmt.Print("Only one can be specified besides clear-value-equal and clear-value-except") + return + } + + if len(clearValueEqual) == 0 && len(clearValueExcept) == 0 { + fmt.Print("At least one must be specified for clear-value-equal and clear-value-except") + return + } + } + // check and apply the log level err = applyLogLevel(logLevel) if err != nil { @@ -64,7 +93,7 @@ func Process() { // make the binary file tStart := time.Now() - processor, err := xdb.NewProcessor(srcFile, dstFile, fields) + processor, err := xdb.NewProcessor(srcFile, dstFile, fields, clearBasedIndex, clearValueEqual, clearValueExcept) if err != nil { fmt.Printf("failed to create %s\n", err) return @@ -76,7 +105,8 @@ func Process() { return } - slog.Info("Processing", "src", srcFile, "dst", dstFile, "logLevel", logLevel) + slog.Info("Processing", "src", srcFile, "dst", dstFile, "fields", fields, "clearBasedIndex", + clearBasedIndex, "clearValueEqual", clearValueEqual, "clearValueExcept", clearValueExcept, "logLevel", logLevel) err = processor.Start() if err != nil { fmt.Printf("failed Start: %s\n", err) diff --git a/maker/golang/xdb/processor.go b/maker/golang/xdb/processor.go index dd703ee..43cfc09 100644 --- a/maker/golang/xdb/processor.go +++ b/maker/golang/xdb/processor.go @@ -11,6 +11,7 @@ import ( "log/slog" "os" "sort" + "strings" "time" ) @@ -18,11 +19,17 @@ type Processor struct { srcHandle *os.File dstHandle *os.File + // value clear + clearBasedIndex int + clearValueEqual string + clearValueExcept string + fields []int segments []*Segment } -func NewProcessor(srcFile string, dstFile string, fields []int) (*Processor, error) { +func NewProcessor(srcFile string, dstFile string, fields []int, + clearBasedIndex int, clearValueEqual string, clearValueExcept string) (*Processor, error) { // open the source file with READONLY mode srcHandle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) if err != nil { @@ -39,6 +46,11 @@ func NewProcessor(srcFile string, dstFile string, fields []int) (*Processor, err srcHandle: srcHandle, dstHandle: dstHandle, + // clear + clearBasedIndex: clearBasedIndex, + clearValueEqual: clearValueEqual, + clearValueExcept: clearValueExcept, + // filter fields index fields: fields, @@ -53,6 +65,34 @@ func (p *Processor) loadSegments() error { var iErr = IterateSegments(p.srcHandle, func(l string) { slog.Debug("loaded", "segment", l) }, func(region string) (string, error) { + if p.clearBasedIndex > -1 { + var ps = strings.Split(region, "|") + var pl = len(ps) + if p.clearBasedIndex >= pl { + return region, fmt.Errorf("clearBasedIndex(%d) >= fields length(%d)", p.clearBasedIndex, pl) + } + + clear := false + if len(p.clearValueEqual) > 0 { + if ps[p.clearBasedIndex] == p.clearValueEqual { + clear = true + } + } else if len(p.clearValueExcept) > 0 { + if ps[p.clearBasedIndex] != p.clearValueExcept { + clear = true + } + } + + if clear { + for i := 0; i < pl; i++ { + ps[i] = "" + } + + // reset the region + region = strings.Join(ps, "|") + } + } + return RegionFiltering(region, p.fields) }, func(seg *Segment) error { // check the continuity of the data segment