load segments while editor init

binding/golang/go.mod

@@ -3,3 +3,5 @@ module github.com/lionsoul2014/ip2region/binding/golang
 go 1.17
 
 require github.com/mitchellh/go-homedir v1.1.0
+
+require github.com/yookoala/realpath v1.0.0 // indirect

binding/golang/go.sum

@@ -1,2 +1,4 @@
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ=
+github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE=

maker/golang/main.go

@@ -246,55 +246,35 @@ func testBench() {
 	}
 
 	var count, errCount, tStart = 0, 0, time.Now()
-	var scanner = bufio.NewScanner(handle)
+	var iErr = xdb.IterateSegments(handle, nil, func(sip uint32, eip uint32, region *string) error {
-	scanner.Split(bufio.ScanLines)
+		var s = *region
-	for scanner.Scan() {
+		var l = fmt.Sprintf("%d|%d|%s", sip, eip, s)
-		var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n"))
-		var ps = strings.SplitN(l, "|", 3)
-		if len(ps) != 3 {
-			fmt.Printf("invalid ip segment line `%s`\n", l)
-			return
-		}
-
-		sip, err := xdb.CheckIP(ps[0])
-		if err != nil {
-			fmt.Printf("check start ip `%s`: %s\n", ps[0], err)
-			return
-		}
-
-		eip, err := xdb.CheckIP(ps[1])
-		if err != nil {
-			fmt.Printf("check end ip `%s`: %s\n", ps[1], err)
-			return
-		}
-
-		if sip > eip {
-			fmt.Printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1])
-			return
-		}
-
 		fmt.Printf("try to bench segment: `%s`\n", l)
 		mip := xdb.MidIP(sip, eip)
 		for _, ip := range []uint32{sip, xdb.MidIP(sip, mip), mip, xdb.MidIP(mip, eip), eip} {
 			fmt.Printf("|-try to bench ip '%s' ... ", xdb.Long2IP(ip))
-			region, _, err := searcher.Search(ip)
+			r, _, err := searcher.Search(ip)
 			if err != nil {
-				fmt.Printf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err)
+				return fmt.Errorf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err)
-				return
 			}
 
 			// check the region info
 			count++
-			if region != ps[2] {
+			if r != s {
 				errCount++
-				fmt.Printf(" --[Failed] (%s != %s)\n", region, ps[2])
+				fmt.Printf(" --[Failed] (%s != %s)\n", r, s)
 				if ignoreError == false {
-					return
+					return fmt.Errorf("")
 				}
 			} else {
 				fmt.Printf(" --[Ok]\n")
 			}
 		}
+		return nil
+	})
+	if iErr != nil {
+		fmt.Printf("%s", err)
+		return
 	}
 
 	fmt.Printf("Bench finished, {count: %d, failed: %d, took: %s}\n", count, errCount, time.Since(tStart))
@@ -381,6 +361,8 @@ func edit() {
 				continue
 			}
 			fmt.Printf("PutFile(%s): Ok\n", file)
+		} else if len(cmd) > 0 {
+			help()
 		}
 	}
 }

maker/golang/xdb/editor.go

@@ -6,31 +6,102 @@
 
 package xdb
 
-import "container/list"
+import (
+	"container/list"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+)
 
 type Editor struct {
 	// source ip file
-	srcFile string
+	srcHandle *os.File
-	dstFile string
+	dstHandle *os.File
 
 	// region info
-	region map[int]string
+	// @Note: 2^32 items at most
+	region map[uint32]string
+	rIndex uint32
 
 	// segments list
 	segments *list.List
 }
 
 func NewEditor(srcFile string, dstFile string) (*Editor, error) {
-	e := &Editor{
+	// check the src and dst file
-		srcFile: srcFile,
+	srcPath, err := filepath.Abs(srcFile)
-		dstFile: dstFile,
+	if err != nil {
+		return nil, err
+	}
 
-		region:   map[int]string{},
+	dstPath, err := filepath.Abs(dstFile)
+	if err != nil {
+		return nil, err
+	}
+
+	if srcPath == dstPath {
+		return nil, fmt.Errorf("src_path(%s) = dst_path(%s)", srcFile, dstFile)
+	}
+
+	srcHandle, err := os.OpenFile(srcPath, os.O_RDONLY, 0600)
+	if err != nil {
+		return nil, err
+	}
+
+	dstHandle, err := os.OpenFile(dstPath, os.O_CREATE|os.O_RDWR, 0644)
+	if err != nil {
+		return nil, err
+	}
+
+	e := &Editor{
+		srcHandle: srcHandle,
+		dstHandle: dstHandle,
+
+		region:   map[uint32]string{},
+		rIndex:   uint32(0),
 		segments: list.New(),
 	}
+
+	// load the segments
+	if err = e.loadSegments(); err != nil {
+		return nil, fmt.Errorf("failed load segments: %s", err)
+	}
+
 	return e, nil
 }
 
+// Load all the segments from the source file
+func (e *Editor) loadSegments() error {
+	var last *Segment = nil
+	var tStart = time.Now()
+
+	var err = IterateSegments(e.srcHandle, nil, func(sip uint32, eip uint32, region *string) error {
+		var str = *region
+		var seg = &Segment{
+			StartIP: sip,
+			EndIP:   eip,
+			Region:  str,
+		}
+
+		// check the continuity of the data segment
+		if last != nil {
+			if last.EndIP+1 != seg.StartIP {
+				return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, str)
+			}
+		}
+
+		last = seg
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to load segments: %s", err)
+	}
+
+	fmt.Printf("all segments loaded, length: %d, elapsed: %s\n", e.segments.Len(), time.Since(tStart))
+	return nil
+}
+
 func (e *Editor) Put(ip string) error {
 	return nil
 }

maker/golang/xdb/maker.go

@@ -50,12 +50,10 @@
 package xdb
 
 import (
-	"bufio"
 	"encoding/binary"
 	"fmt"
 	"log"
 	"os"
-	"strings"
 	"time"
 )
 
@@ -140,50 +138,29 @@ func (m *Maker) loadSegments() error {
 	var last *Segment = nil
 	var tStart = time.Now()
 
-	var scanner = bufio.NewScanner(m.srcHandle)
+	var err = IterateSegments(m.srcHandle, func(l string) {
-	scanner.Split(bufio.ScanLines)
-	for scanner.Scan() {
-		var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n"))
 		log.Printf("load segment: `%s`", l)
-
+	}, func(sip uint32, eip uint32, region *string) error {
-		var ps = strings.SplitN(l, "|", 3)
+		var str = *region
-		if len(ps) != 3 {
-			return fmt.Errorf("invalid ip segment line `%s`", l)
-		}
-
-		sip, err := CheckIP(ps[0])
-		if err != nil {
-			return fmt.Errorf("check start ip `%s`: %s", ps[0], err)
-		}
-
-		eip, err := CheckIP(ps[1])
-		if err != nil {
-			return fmt.Errorf("check end ip `%s`: %s", ps[1], err)
-		}
-
-		if sip > eip {
-			return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
-		}
-
-		if len(ps[2]) < 1 {
-			return fmt.Errorf("empty region info in segment line `%s`", l)
-		}
-
 		var seg = &Segment{
 			StartIP: sip,
 			EndIP:   eip,
-			Region:  ps[2],
+			Region:  str,
 		}
 
 		// check the continuity of the data segment
 		if last != nil {
 			if last.EndIP+1 != seg.StartIP {
-				return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, ps[0])
+				return fmt.Errorf("discontinuous data segment: last.eip+1(%d) != seg.sip(%d, %s)", sip, eip, str)
 			}
 		}
 
 		m.segments = append(m.segments, seg)
 		last = seg
+		return nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to load segments: %s", err)
 	}
 
 	log.Printf("all segments loaded, length: %d, elapsed: %s", len(m.segments), time.Since(tStart))

maker/golang/xdb/util.go

@@ -5,7 +5,9 @@
 package xdb
 
 import (
+	"bufio"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 )
@@ -45,6 +47,46 @@ func MidIP(sip uint32, eip uint32) uint32 {
 	return uint32((uint64(sip) + uint64(eip)) >> 1)
 }
 
+func IterateSegments(handle *os.File, before func(l string), cb func(sip uint32, eip uint32, region *string) error) error {
+	var scanner = bufio.NewScanner(handle)
+	scanner.Split(bufio.ScanLines)
+	for scanner.Scan() {
+		var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n"))
+		if before != nil {
+			before(l)
+		}
+
+		var ps = strings.SplitN(l, "|", 3)
+		if len(ps) != 3 {
+			return fmt.Errorf("invalid ip segment line `%s`", l)
+		}
+
+		sip, err := CheckIP(ps[0])
+		if err != nil {
+			return fmt.Errorf("check start ip `%s`: %s", ps[0], err)
+		}
+
+		eip, err := CheckIP(ps[1])
+		if err != nil {
+			return fmt.Errorf("check end ip `%s`: %s", ps[1], err)
+		}
+
+		if sip > eip {
+			return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
+		}
+
+		if len(ps[2]) < 1 {
+			return fmt.Errorf("empty region info in segment line `%s`", l)
+		}
+
+		if err = cb(sip, eip, &ps[2]); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 func CheckSegments(segList []*Segment) error {
 	var last *Segment
 	for _, seg := range segList {