diff --git a/data/sample/segments.tests.mixed b/data/sample/segments.tests.mixed index 632b8ef..96dabff 100644 --- a/data/sample/segments.tests.mixed +++ b/data/sample/segments.tests.mixed @@ -3,8 +3,16 @@ 192.168.2.31|192.168.2.60|0|0|内网IP|办公室B 192.168.2.61|192.168.2.91|0|0|内网IP|办公室B 223.255.236.0|223.255.239.255|中国|上海|上海市|电信 +# ipv4 cidr +14.32.46.0/24|3491 4766 9696|IGP +24.104.37.0/24|14537 174 7922 7016 401657 401657|IGP +45.15.32.0/22|14537 29119 206496 206496 206496 206496 206496 206496 206496 206496 206496|IGP 2c0f:fff1::|2c0f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|毛里求斯|威廉平原区|卡特勒博尔纳|专线用户 2e00::|2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|德国|黑森|美因河畔法兰克福|专线用户 3000::|fbff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户 fe00::|fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户 fe80::|febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户 +# ipv6 cidr +2c0f:fda8::/32|4635 6939 37662 37662 37035 37349|IGP +2c0f:fdb0::/32|3491 33765|IGP +2c0f:fe08:7::/48|2497 6939 36914|IGP diff --git a/maker/golang/xdb/segment.go b/maker/golang/xdb/segment.go index 831bdf9..8792129 100644 --- a/maker/golang/xdb/segment.go +++ b/maker/golang/xdb/segment.go @@ -15,30 +15,66 @@ type Segment struct { Region *Region } -func SegmentFrom(seg string, cRegion func(string) *Region) (*Segment, error) { - var ps = strings.SplitN(strings.TrimSpace(seg), "|", 3) - if len(ps) != 3 { - return nil, fmt.Errorf("invalid ip segment `%s`", seg) +func ParseSegment(seg string) ([]byte, []byte, string, error) { + var count = 0 + var ps = StringTokenizer(strings.TrimSpace(seg), "|", func(s string, start int) bool { + // CIDR format + if strings.Index(s, "/") > 0 { + return false + } + + count++ + return count < 2 + }) + + var err error + var sip, eip []byte + var rIdx = 0 + switch pl := len(ps); pl { + case 2: + // CIDR format + rIdx = 1 + sip, eip, err = CIDR2Range(ps[0]) + if err != nil { + return nil, nil, "", fmt.Errorf("parse cidr: %s", err) + } + case 3: + // triditional ip range + rIdx = 2 + sip, err = ParseIP(ps[0]) + if err != nil { + return nil, nil, "", fmt.Errorf("parse start ip `%s`: %s", ps[0], err) + } + + eip, err = ParseIP(ps[1]) + if err != nil { + return nil, nil, "", fmt.Errorf("check end ip `%s`: %s", ps[1], err) + } + default: + return nil, nil, "", fmt.Errorf("invalid ip segment `%s`", seg) } - sip, err := ParseIP(ps[0]) - if err != nil { - return nil, fmt.Errorf("check start ip `%s`: %s", ps[0], err) - } - - eip, err := ParseIP(ps[1]) - if err != nil { - return nil, fmt.Errorf("check end ip `%s`: %s", ps[1], err) + if len(sip) != len(eip) { + return nil, nil, "", fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", seg) } if IPCompare(sip, eip) > 0 { - return nil, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) + return nil, nil, "", fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) + } + + return sip, eip, ps[rIdx], nil +} + +func SegmentFrom(seg string, cRegion func(string) *Region) (*Segment, error) { + sip, eip, region, err := ParseSegment(seg) + if err != nil { + return nil, err } return &Segment{ StartIP: sip, EndIP: eip, - Region: cRegion(ps[2]), + Region: cRegion(region), }, nil } diff --git a/maker/golang/xdb/util.go b/maker/golang/xdb/util.go index 4d394ea..6566c91 100644 --- a/maker/golang/xdb/util.go +++ b/maker/golang/xdb/util.go @@ -10,6 +10,7 @@ import ( "fmt" "math/big" "net" + "net/netip" "os" "strings" ) @@ -35,6 +36,45 @@ func ParseIP(ip string) ([]byte, error) { return nil, fmt.Errorf("invalid ip address: %s", ip) } +var bitMaskList = []uint8{ + 0b1111_1111, // all zero + 0b0111_1111, + 0b0011_1111, + 0b0001_1111, + 0b0000_1111, + 0b0000_0111, + 0b0000_0011, + 0b0000_0001, +} + +func CIDR2Range(cidrStr string) ([]byte, []byte, error) { + prefix, err := netip.ParsePrefix(cidrStr) + if err != nil { + return nil, nil, err + } + + // Get the start IP (Network Address) + // Masked() zeros out the host bits, which gives the starting IP of the subnet. + sip := prefix.Masked().Addr().AsSlice() + ipl := len(sip) + eip := make([]byte, ipl) + copy(eip, sip) + + // Calculate the end IP (Broadcast Address) + bits := prefix.Bits() + + // border byte rest bit filled with 1 + byteIdx := bits / 8 + eip[byteIdx] |= bitMaskList[bits-(byteIdx*8)] + + // fill all the rest bits with 1 + for bi := byteIdx + 1; bi < ipl; bi++ { + eip[bi] |= 0b1111_1111 + } + + return sip, eip, nil +} + func IP2String(ip []byte) string { return net.IP(ip[:]).String() } @@ -160,38 +200,14 @@ func IterateSegments(handle *os.File, autoMerge bool, before func(l string), fil before(l) } - var ps = strings.SplitN(l, "|", 3) - if len(ps) != 3 { - return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l) - } - - sip, err := ParseIP(ps[0]) + sip, eip, region, err := ParseSegment(l) if err != nil { - return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err) + return totalCount, mergeCount, err } - eip, err := ParseIP(ps[1]) - if err != nil { - return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err) - } - - if len(sip) != len(eip) { - return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l) - } - - if IPCompare(sip, eip) > 0 { - return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1]) - } - - // Allow empty region info since 2024/09/24 - // if len(ps[2]) < 1 { - // return fmt.Errorf("empty region info in segment line `%s`", l) - // } - // check and do the region filter - var region = ps[2] if filter != nil { - region, err = filter(ps[2]) + region, err = filter(region) if err != nil { return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", region, err) } @@ -309,3 +325,41 @@ func RegionFiltering(region string, fields []int) (string, error) { return strings.Join(sb, "|"), nil } + +// do the string split step by step as caller needed +func StringTokenizer(str, substr string, cb func(s string, start int) bool) []string { + var tokens []string + var token string + var sIdx, oIdx, isEOF = 0, 0, false + for { + // do the token match + nIdx := strings.Index(str[sIdx:], substr) + if nIdx == -1 { + isEOF = true + token = str[sIdx:] + } else { + token = str[sIdx : sIdx+nIdx] + } + + oIdx = sIdx // backup the old index + sIdx = sIdx + nIdx + 1 // reset the next start index + tokens = append(tokens, token) // append the token + + // check and call the callback + if cb(token, oIdx) == false { + // keep the last token + if sIdx < len(str) { + tokens = append(tokens, str[sIdx:]) + } + + break + } + + // check the EOF + if isEOF { + break + } + } + + return tokens +} diff --git a/maker/golang/xdb/util_test.go b/maker/golang/xdb/util_test.go index 7f5b8cb..296c506 100644 --- a/maker/golang/xdb/util_test.go +++ b/maker/golang/xdb/util_test.go @@ -8,6 +8,7 @@ import ( "encoding/binary" "fmt" "os" + "strings" "testing" ) @@ -196,6 +197,39 @@ func TestIPMiddle(t *testing.T) { t.Logf("test done with %d ips", counter) } +func TestSegmentFromt(t *testing.T) { + var lines = []string{ + // ipv4 range + "2.10.222.0|2.10.223.255|France|Brittany|0|Orange S.A.|FR", + "8.35.35.0|8.35.159.255|United States|Colorado|0|ZSCALER, INC.|US", + "223.104.64.128|223.104.64.159|中国|广东省|深圳市|移动|CN", + + // ipv4 CIDR + "1.0.4.0/24|2497 6453 7545 2764 38803|IGP", + "2.18.209.0/24|14537 3356 1299 34164 34164|IGP", + "14.32.46.0/24|2497 4766 9696|IGP", + + // IPv6 range + "2001:200:17c::|2001:200:180:ffff:ffff:ffff:ffff:ffff|Japan|Tokyo|Tokyo|WIDE Project|JP", + "2001:506:100:226e::8|2001:506:100:226f::7|United States|Michigan|Detroit|Transact Ltd.|US", + "2a13:aac4:1000::|2a13:aac4:ffff:ffff:ffff:ffff:ffff:ffff|中国|广东省|深圳市|MLGT|CN", + + // IPv6 CIDR + "2c0f:fc89:8081::/48|32590 9002 3257 8452 36992 36992 36992 36992 36992 36992 36992 36992 36992|EGP", + "2c0f:fe08:20a::/48|2497 6939 36914|IGP", + "2c0f:ffc8::/32|14537 23764 37468 22355|IGP", + } + + for _, str := range lines { + seg, err := SegmentFrom(str, NewRegion) + if err != nil { + t.Fatalf("parse segment: %s", err) + } + + fmt.Printf("seg={%s, %s, %s}\n", IP2String(seg.StartIP), IP2String(seg.EndIP), seg.Region.Str) + } +} + func TestSplitSegmentV4(t *testing.T) { // var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信" // var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP" @@ -258,10 +292,75 @@ func TestIterateSegments(t *testing.T) { t.Fatalf("failed to open tests file: %s", err) } - _, _, _ = IterateSegments(handle, true, func(l string) { + total, merged, err := IterateSegments(handle, true, func(l string) { // fmt.Printf("load segment: `%s`\n", l) }, nil, NewRegion, func(seg *Segment) error { - fmt.Printf("get segment: `%s`\n", seg) + fmt.Printf("{%s, %s} -> `%s`\n", IP2String(seg.StartIP), IP2String(seg.EndIP), seg.Region) return nil }) + if err != nil { + t.Fatalf("iterate error: %s", err) + } + + fmt.Printf("done iterate -> total:%d, merged:%d\n", total, merged) +} + +func TestStringTokenizer(t *testing.T) { + var strList = []string{ + "24.231.126.0/24|14537 2914 29866|IGP", + "24.231.126.0|24.231.126.255|14537 2914 29866|IGP", + } + + var counter = 0 + for _, str := range strList { + tokens := StringTokenizer(str, "|", func(s string, start int) bool { + // fmt.Printf("%s[idx=%d, |]=%s\n", str, start, s) + if counter == 0 { + if strings.Index(s, "/") > 0 { + return false + } + } + + counter++ + return counter < 2 + }) + fmt.Printf("%d tokens: %s\n", len(tokens), strings.Join(tokens, ", ")) + } +} + +func TestCIDR2Range(t *testing.T) { + var strList = [][3]string{ + {"43.247.92.0/22", "43.247.92.0", "43.247.95.255"}, + {"64.252.86.39/29", "64.252.86.32", "64.252.86.39"}, + {"103.37.44.0/22", "103.37.44.0", "103.37.47.255"}, + {"111.223.12.0/22", "111.223.12.0", "111.223.15.255"}, + {"43.248.80.0/20", "43.248.80.0", "43.248.95.255"}, + {"192.168.100.0/22", "192.168.100.0", "192.168.103.255"}, + {"2403:3380::/32", "2403:3380::", "2403:3380:ffff:ffff:ffff:ffff:ffff:ffff"}, + {"2001:db8:85a3::/64", "2001:db8:85a3::", "2001:db8:85a3:0:ffff:ffff:ffff:ffff"}, + {"2001:db8:abcd::/48", "2001:db8:abcd::", "2001:db8:abcd:ffff:ffff:ffff:ffff:ffff"}, + } + + for _, item := range strList { + sip, eip, err := CIDR2Range(item[0]) + if err != nil { + t.Fatalf("CIDR2Range: %s", err) + } + + tSip, err := ParseIP(item[1]) + if err != nil { + t.Fatalf("parse start ip: %s", err) + } else if IPCompare(sip, tSip) != 0 { + t.Fatalf("start ip %s != %s", IP2String(sip), item[1]) + } + + tEip, err := ParseIP(item[2]) + if err != nil { + t.Fatalf("parse end ip: %s", err) + } else if IPCompare(eip, tEip) != 0 { + t.Fatalf("end ip %s != %s", IP2String(eip), item[2]) + } + + fmt.Printf("cidr=%s: {sip=%s, eip=%s}\n", item[0], item[1], item[2]) + } }