source data clear based supports

maker/golang/cmd/process.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"log/slog"
 	"os"
+	"strconv"
 	"time"
 
 	"github.com/lionsoul2014/ip2region/maker/golang/xdb"
@@ -19,6 +20,8 @@ func Process() {
 	var err error
 	var srcFile, dstFile = "", ""
 	var fieldList, logLevel = "", ""
+	var clearBasedIndex = -1
+	var clearValueEqual, clearValueExcept = "", ""
 	var fErr = iterateFlags(func(key string, val string) error {
 		switch key {
 		case "src":
@@ -27,6 +30,17 @@ func Process() {
 			dstFile = val
 		case "field-list":
 			fieldList = val
+		case "clear-based-index":
+			num, err := strconv.Atoi(val)
+			if err != nil {
+				return fmt.Errorf("invalid clear-based-index '%s=%s', integer expected", key, val)
+			}
+
+			clearBasedIndex = num
+		case "clear-value-equal":
+			clearValueEqual = val
+		case "clear-value-except":
+			clearValueExcept = val
 		case "log-level":
 			logLevel = val
 		default:
@@ -42,13 +56,28 @@ func Process() {
 	if srcFile == "" || dstFile == "" {
 		fmt.Printf("%s process [command options]\n", os.Args[0])
 		fmt.Printf("options:\n")
-		fmt.Printf(" --src string           source ip text file path\n")
+		fmt.Printf(" --src string                 source ip text file path\n")
-		fmt.Printf(" --dst string           target ip text file path\n")
+		fmt.Printf(" --dst string                 target ip text file path\n")
-		fmt.Printf(" --field-list string    field index list imploded with ',' eg: 0,1,2,3-6,7\n")
+		fmt.Printf(" --field-list string          field index list imploded with ',' eg: 0,1,2,3-6,7\n")
-		fmt.Printf(" --log-level string     set the log level, options: debug/info/warn/error\n")
+		fmt.Printf(" --clear-based-index integer  clear based index eg: 3\n")
+		fmt.Printf(" --clear-value-equal string   clear value equal to the specified one\n")
+		fmt.Printf(" --clear-value-except string  clear value except the specified one\n")
+		fmt.Printf(" --log-level string           set the log level, options: debug/info/warn/error\n")
 		return
 	}
 
+	if clearBasedIndex > -1 {
+		if len(clearValueEqual) > 0 && len(clearValueExcept) > 0 {
+			fmt.Print("Only one can be specified besides clear-value-equal and clear-value-except")
+			return
+		}
+
+		if len(clearValueEqual) == 0 && len(clearValueExcept) == 0 {
+			fmt.Print("At least one must be specified for clear-value-equal and clear-value-except")
+			return
+		}
+	}
+
 	// check and apply the log level
 	err = applyLogLevel(logLevel)
 	if err != nil {
@@ -64,7 +93,7 @@ func Process() {
 
 	// make the binary file
 	tStart := time.Now()
-	processor, err := xdb.NewProcessor(srcFile, dstFile, fields)
+	processor, err := xdb.NewProcessor(srcFile, dstFile, fields, clearBasedIndex, clearValueEqual, clearValueExcept)
 	if err != nil {
 		fmt.Printf("failed to create %s\n", err)
 		return
@@ -76,7 +105,8 @@ func Process() {
 		return
 	}
 
-	slog.Info("Processing", "src", srcFile, "dst", dstFile, "logLevel", logLevel)
+	slog.Info("Processing", "src", srcFile, "dst", dstFile, "fields", fields, "clearBasedIndex",
+		clearBasedIndex, "clearValueEqual", clearValueEqual, "clearValueExcept", clearValueExcept, "logLevel", logLevel)
 	err = processor.Start()
 	if err != nil {
 		fmt.Printf("failed Start: %s\n", err)

maker/golang/xdb/processor.go

@@ -11,6 +11,7 @@ import (
 	"log/slog"
 	"os"
 	"sort"
+	"strings"
 	"time"
 )
 
@@ -18,11 +19,17 @@ type Processor struct {
 	srcHandle *os.File
 	dstHandle *os.File
 
+	// value clear
+	clearBasedIndex  int
+	clearValueEqual  string
+	clearValueExcept string
+
 	fields   []int
 	segments []*Segment
 }
 
-func NewProcessor(srcFile string, dstFile string, fields []int) (*Processor, error) {
+func NewProcessor(srcFile string, dstFile string, fields []int,
+	clearBasedIndex int, clearValueEqual string, clearValueExcept string) (*Processor, error) {
 	// open the source file with READONLY mode
 	srcHandle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600)
 	if err != nil {
@@ -39,6 +46,11 @@ func NewProcessor(srcFile string, dstFile string, fields []int) (*Processor, err
 		srcHandle: srcHandle,
 		dstHandle: dstHandle,
 
+		// clear
+		clearBasedIndex:  clearBasedIndex,
+		clearValueEqual:  clearValueEqual,
+		clearValueExcept: clearValueExcept,
+
 		// filter fields index
 		fields: fields,
 
@@ -53,6 +65,34 @@ func (p *Processor) loadSegments() error {
 	var iErr = IterateSegments(p.srcHandle, func(l string) {
 		slog.Debug("loaded", "segment", l)
 	}, func(region string) (string, error) {
+		if p.clearBasedIndex > -1 {
+			var ps = strings.Split(region, "|")
+			var pl = len(ps)
+			if p.clearBasedIndex >= pl {
+				return region, fmt.Errorf("clearBasedIndex(%d) >= fields length(%d)", p.clearBasedIndex, pl)
+			}
+
+			clear := false
+			if len(p.clearValueEqual) > 0 {
+				if ps[p.clearBasedIndex] == p.clearValueEqual {
+					clear = true
+				}
+			} else if len(p.clearValueExcept) > 0 {
+				if ps[p.clearBasedIndex] != p.clearValueExcept {
+					clear = true
+				}
+			}
+
+			if clear {
+				for i := 0; i < pl; i++ {
+					ps[i] = ""
+				}
+
+				// reset the region
+				region = strings.Join(ps, "|")
+			}
+		}
+
 		return RegionFiltering(region, p.fields)
 	}, func(seg *Segment) error {
 		// check the continuity of the data segment