diff --git a/.gitignore b/.gitignore index b563880..ae58d2e 100644 --- a/.gitignore +++ b/.gitignore @@ -29,10 +29,6 @@ META-INF/ .vscode # binding -/v1.0//binding/java/classes/ -/v1.0//binding/java/doc/ -/v1.0//binding/java/target/ -/v1.0//binding/java/*.jar /binding/java/classes/ /binding/java/doc/ /binding/java/target/ @@ -61,18 +57,12 @@ target # VS ignore cases /**/*.sln -/v1.0/binding/c#/**/.vs/ -/v1.0/binding/c#/**/packages -/v1.0/binding/c#/**/bin -/v1.0/binding/c#/**/obj /binding/c#/**/.vs/ /binding/c#/**/packages /binding/c#/**/bin /binding/c#/**/obj # Nodejs -/v1.0/binding/nodejs/tests/unitTests/__snapshots__ -/v1.0/binding/nodejs/coverage /binding/nodejs/tests/unitTests/__snapshots__ /binding/nodejs/coverage /binding/nodejs/node_modules diff --git a/maker/golang/cmd/bench.go b/maker/golang/cmd/bench.go index 1c17350..7f1fa51 100644 --- a/maker/golang/cmd/bench.go +++ b/maker/golang/cmd/bench.go @@ -90,7 +90,7 @@ func Bench() { var count, errCount, tStart = 0, 0, time.Now() slog.Info("Bench start", "xdbPath", dbFile, "srcPath", srcFile) - var iErr = xdb.IterateSegments(handle, nil, nil, func(seg *xdb.Segment) error { + _, _, iErr := xdb.IterateSegments(handle, false, nil, nil, func(seg *xdb.Segment) error { var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) slog.Debug("try to bench", "segment", l) // mip := xdb.IPMiddle(seg.StartIP, seg.EndIP) diff --git a/maker/golang/xdb/editor.go b/maker/golang/xdb/editor.go index 3f4ad13..2c4f259 100644 --- a/maker/golang/xdb/editor.go +++ b/maker/golang/xdb/editor.go @@ -60,7 +60,7 @@ func (e *Editor) loadSegments() error { var segments []*Segment var sorting = false - var iErr = IterateSegments(e.srcHandle, func(l string) { + _, _, iErr := IterateSegments(e.srcHandle, true, func(l string) { // do nothing here }, nil, func(seg *Segment) error { // version check @@ -302,7 +302,7 @@ func (e *Editor) PutFile(src string) (int, int, error) { } var oldRows, newRows = 0, 0 - iErr := IterateSegments(handle, func(l string) { + _, _, iErr := IterateSegments(handle, true, func(l string) { // do nothing here }, nil, func(seg *Segment) error { o, n, err := e.PutSegment(seg) diff --git a/maker/golang/xdb/maker.go b/maker/golang/xdb/maker.go index be48407..bd4f56c 100644 --- a/maker/golang/xdb/maker.go +++ b/maker/golang/xdb/maker.go @@ -168,7 +168,7 @@ func (m *Maker) loadSegments() error { var tStart = time.Now() var sorting = false - var iErr = IterateSegments(m.srcHandle, func(l string) { + _, mergeCount, iErr := IterateSegments(m.srcHandle, true, func(l string) { slog.Debug("loaded", "segment", l) }, func(region string) (string, error) { // apply the field filter @@ -217,7 +217,7 @@ func (m *Maker) loadSegments() error { } } - slog.Info("all segments loaded", "length", len(m.segments), "sorting", sorting, "elapsed", time.Since(tStart)) + slog.Info("all segments loaded", "length", len(m.segments), "merged", mergeCount, "sorting", sorting, "elapsed", time.Since(tStart)) return nil } diff --git a/maker/golang/xdb/processor.go b/maker/golang/xdb/processor.go index 43cfc09..e791d0f 100644 --- a/maker/golang/xdb/processor.go +++ b/maker/golang/xdb/processor.go @@ -62,7 +62,7 @@ func (p *Processor) loadSegments() error { slog.Info("try to load the segments ... ") var tStart = time.Now() - var iErr = IterateSegments(p.srcHandle, func(l string) { + _, mergeCount, iErr := IterateSegments(p.srcHandle, true, func(l string) { slog.Debug("loaded", "segment", l) }, func(region string) (string, error) { if p.clearBasedIndex > -1 { @@ -108,7 +108,7 @@ func (p *Processor) loadSegments() error { return fmt.Errorf("failed to load segments: %s", iErr) } - slog.Info("all segments loaded", "length", len(p.segments), "elapsed", time.Since(tStart)) + slog.Info("all segments loaded", "length", len(p.segments), "merged", mergeCount, "elapsed", time.Since(tStart)) return nil } diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index 2add0df..1235ebd 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -87,22 +87,62 @@ func IPSubOne(ip []byte) []byte { return r } -func IPMiddle(sip, eip []byte) []byte { - var result = make([]byte, len(sip)) - var carry uint16 = 0 +// IPSub Sub the spcecified two byte ip +func IPSub(sip, eip []byte) ([]byte, error) { + if len(sip) != len(eip) { + return []byte{}, fmt.Errorf("length of the two ips are not the same") + } + + var carry uint16 = 0 + var result = make([]byte, len(sip)+1) - // Add the two addresses with carry for i := len(sip) - 1; i >= 0; i-- { sum := uint16(sip[i]) + uint16(eip[i]) + carry - result[i] = byte(sum >> 0x01) // Divide by 2 - carry = (sum & 0x01) << 7 // Carry for next byte (shift to MSB) + result[i+1] = byte(sum) // Store standard 8-bit result + carry = sum >> 8 // Extract the 1-bit carry for the next byte + } + + // check and append the carry + if carry > 0 { + result[0] = byte(carry) + return result, nil + } else { + return result[1:], nil + } +} + +// IPHalf get the half value of an input byte ip +func IPHalf(ip []byte) []byte { + var length = len(ip) + var result = make([]byte, length) + // Tracks the bit falling off from the previous byte + var carry byte = 0 + + for i := 0; i < length; i++ { + // 1. Shift current byte right by 1 + // 2. Or (|) with the carry from the previous byte (shifted to the MSB position) + result[i] = (ip[i] >> 1) | (carry << 7) + + // 3. Capture the Least Significant Bit (LSB) to use as carry for the next byte + carry = ip[i] & 1 } return result } -func IterateSegments(handle *os.File, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) error { +// IPMiddle get the middle value of two input ip address +func IPMiddle(sip, eip []byte) ([]byte, error) { + buf, err := IPSub(sip, eip) + if err != nil { + return []byte{}, fmt.Errorf("IPSub(%s, %s): %w", IP2String(sip), IP2String(eip), err) + } + + return IPHalf(buf), nil +} + +func IterateSegments(handle *os.File, autoMerge bool, before func(l string), filter func(region string) (string, error), done func(seg *Segment) error) (int, int, error) { var last *Segment = nil + var totalCount, mergeCount = 0, 0 var scanner = bufio.NewScanner(handle) scanner.Split(bufio.ScanLines) for scanner.Scan() { @@ -115,31 +155,32 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region continue } + totalCount++ if before != nil { before(l) } var ps = strings.SplitN(l, "|", 3) if len(ps) != 3 { - return fmt.Errorf("invalid ip segment line `%s`", l) + return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l) } sip, err := ParseIP(ps[0]) if err != nil { - return fmt.Errorf("check start ip `%s`: %s", ps[0], err) + return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err) } eip, err := ParseIP(ps[1]) if err != nil { - return fmt.Errorf("check end ip `%s`: %s", ps[1], err) + return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err) } if len(sip) != len(eip) { - return fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l) + return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l) } if IPCompare(sip, eip) > 0 { - return fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) + return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) } // Allow empty region info since 2024/09/24 @@ -152,7 +193,7 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region if filter != nil { region, err = filter(ps[2]) if err != nil { - return fmt.Errorf("failed to filter region `%s`: %s", ps[2], err) + return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", ps[2], err) } } @@ -168,15 +209,16 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region if last == nil { last = seg continue - } else if last.Region == seg.Region { + } else if autoMerge && last.Region == seg.Region { if err = seg.RightBehind(last); err == nil { + mergeCount++ last.EndIP = seg.EndIP continue } } if err = done(last); err != nil { - return err + return totalCount, mergeCount, err } // reset the last @@ -185,10 +227,10 @@ func IterateSegments(handle *os.File, before func(l string), filter func(region // process the last segment if last != nil { - return done(last) + return totalCount, mergeCount, done(last) } - return nil + return totalCount, mergeCount, nil } func CheckSegments(segList []*Segment) error { diff --git a/maker/golang/xdb/util_test.go b/maker/golang/xdb/util_test.go index 85b97c7..f428974 100644 --- a/maker/golang/xdb/util_test.go +++ b/maker/golang/xdb/util_test.go @@ -5,6 +5,7 @@ package xdb import ( + "encoding/binary" "fmt" "os" "testing" @@ -99,6 +100,102 @@ func TestIPSubOne2(t *testing.T) { fmt.Printf("nip: %+v, ip:%+v", ip, nip) } +func TestIPSub(t *testing.T) { + var strToSub = "1.2.3.4" + bytesToSub, err := ParseIP(strToSub) + if err != nil { + t.Fatalf("failed to parse ip %s", strToSub) + } + var intToSub = int(binary.BigEndian.Uint32(bytesToSub)) + t.Logf("to sub ip: %d -> %s", intToSub, strToSub) + + counter := 0 + buf := make([]byte, 4) + for i := 0; i < 0x2FFFFFFF; i++ { + binary.BigEndian.PutUint32(buf, uint32(i)) + subVal, err := IPSub(buf, bytesToSub) + if err != nil { + t.Fatalf("failed to IPSub(%s,%s): %s", IP2String(buf), strToSub, err) + } + + // do it as two integers + byteSub := int(binary.BigEndian.Uint32(subVal)) + intSub := i + intToSub + if byteSub != intSub { + t.Fatal("byte and int sub value are not the same") + } + + counter++ + } + + t.Logf("test done with %d ips", counter) +} + +func TestIPHalf(t *testing.T) { + var buf = make([]byte, 4) + for i := 0; i < 0xFFFFFFFF; i++ { + binary.BigEndian.PutUint32(buf, uint32(i)) + half := IPHalf(buf) + + // do it as two integers + byteMiddle := binary.BigEndian.Uint32(half) + intMidle := i >> 1 + if byteMiddle != uint32(intMidle) { + t.Fatal("byte middle and int middle are not the same") + } + } +} + +func TestSubOverflow(t *testing.T) { + var ip1Str = "255.255.255.250" + ip1Bytes, err := ParseIP(ip1Str) + if err != nil { + t.Fatalf("failed to ParseIP(%s): %s", ip1Str, err) + } + + var buff = make([]byte, 4) + for i := 0; i < 10; i++ { + binary.BigEndian.PutUint32(buff, uint32(i)) + ipSub, err := IPSub(ip1Bytes, buff) + if err != nil { + t.Fatalf("failed to IPSub(%s, %s): %s", ip1Str, IP2String(buff), err) + } + + t.Logf("IPSub(%s, %s) = %+v", ip1Str, IP2String(buff), ipSub) + } +} + +func TestIPMiddle(t *testing.T) { + var sIPStr = "0.0.0.0" + sBytes, err := ParseIP(sIPStr) + if err != nil { + t.Fatalf("failed to parse ip %s", sIPStr) + } + var sInt = int(binary.BigEndian.Uint32(sBytes)) + t.Logf("start ip: %d -> %s", sInt, sIPStr) + + counter := 0 + buf := make([]byte, 4) + for i := 0; i < 0x0FFFFFFF; i++ { + binary.BigEndian.PutUint32(buf, uint32(i)) + midVal, err := IPMiddle(sBytes, buf) + if err != nil { + t.Fatalf("failed to IPMiddle(%s,%s): %s", sIPStr, IP2String(buf), err) + } + + // do it as two integers + byteMid := int(binary.BigEndian.Uint32(midVal)) + intMid := (sInt + i) >> 1 + if byteMid != intMid { + t.Fatal("byte and int middle value are not the same") + } + + counter++ + } + + t.Logf("test done with %d ips", counter) +} + func TestSplitSegmentV4(t *testing.T) { // var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信" // var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP" @@ -161,7 +258,7 @@ func TestIterateSegments(t *testing.T) { t.Fatalf("failed to open tests file: %s", err) } - _ = IterateSegments(handle, func(l string) { + _, _, _ = IterateSegments(handle, true, func(l string) { // fmt.Printf("load segment: `%s`\n", l) }, nil, func(seg *Segment) error { fmt.Printf("get segment: `%s`\n", seg)