Merge pull request #484 from lionsoul2014/fr_golang_cidr_to_range

Fr golang CIDR supports

data/sample/segments.tests.mixed

@@ -3,8 +3,16 @@
 192.168.2.31|192.168.2.60|0|0|内网IP|办公室B
 192.168.2.61|192.168.2.91|0|0|内网IP|办公室B
 223.255.236.0|223.255.239.255|中国|上海|上海市|电信
+# ipv4 cidr
+14.32.46.0/24|3491 4766 9696|IGP
+24.104.37.0/24|14537 174 7922 7016 401657 401657|IGP
+45.15.32.0/22|14537 29119 206496 206496 206496 206496 206496 206496 206496 206496 206496|IGP
 2c0f:fff1::|2c0f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|毛里求斯|威廉平原区|卡特勒博尔纳|专线用户
 2e00::|2fff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|德国|黑森|美因河畔法兰克福|专线用户
 3000::|fbff:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户
 fe00::|fe7f:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户
 fe80::|febf:ffff:ffff:ffff:ffff:ffff:ffff:ffff|瑞士|弗里堡州||专线用户
+# ipv6 cidr
+2c0f:fda8::/32|4635 6939 37662 37662 37035 37349|IGP
+2c0f:fdb0::/32|3491 33765|IGP
+2c0f:fe08:7::/48|2497 6939 36914|IGP

maker/golang/xdb/segment.go

@@ -15,30 +15,66 @@ type Segment struct {
 	Region  *Region
 }
 
-func SegmentFrom(seg string, cRegion func(string) *Region) (*Segment, error) {
+func ParseSegment(seg string) ([]byte, []byte, string, error) {
-	var ps = strings.SplitN(strings.TrimSpace(seg), "|", 3)
+	var count = 0
-	if len(ps) != 3 {
+	var ps = StringTokenizer(strings.TrimSpace(seg), "|", func(s string, start int) bool {
-		return nil, fmt.Errorf("invalid ip segment `%s`", seg)
+		// CIDR format
+		if strings.Index(s, "/") > 0 {
+			return false
+		}
+
+		count++
+		return count < 2
+	})
+
+	var err error
+	var sip, eip []byte
+	var rIdx = 0
+	switch pl := len(ps); pl {
+	case 2:
+		// CIDR format
+		rIdx = 1
+		sip, eip, err = CIDR2Range(ps[0])
+		if err != nil {
+			return nil, nil, "", fmt.Errorf("parse cidr: %s", err)
+		}
+	case 3:
+		// triditional ip range
+		rIdx = 2
+		sip, err = ParseIP(ps[0])
+		if err != nil {
+			return nil, nil, "", fmt.Errorf("parse start ip `%s`: %s", ps[0], err)
+		}
+
+		eip, err = ParseIP(ps[1])
+		if err != nil {
+			return nil, nil, "", fmt.Errorf("check end ip `%s`: %s", ps[1], err)
+		}
+	default:
+		return nil, nil, "", fmt.Errorf("invalid ip segment `%s`", seg)
 	}
 
-	sip, err := ParseIP(ps[0])
+	if len(sip) != len(eip) {
-	if err != nil {
+		return nil, nil, "", fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", seg)
-		return nil, fmt.Errorf("check start ip `%s`: %s", ps[0], err)
-	}
-
-	eip, err := ParseIP(ps[1])
-	if err != nil {
-		return nil, fmt.Errorf("check end ip `%s`: %s", ps[1], err)
 	}
 
 	if IPCompare(sip, eip) > 0 {
-		return nil, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
+		return nil, nil, "", fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
+	}
+
+	return sip, eip, ps[rIdx], nil
+}
+
+func SegmentFrom(seg string, cRegion func(string) *Region) (*Segment, error) {
+	sip, eip, region, err := ParseSegment(seg)
+	if err != nil {
+		return nil, err
 	}
 
 	return &Segment{
 		StartIP: sip,
 		EndIP:   eip,
-		Region:  cRegion(ps[2]),
+		Region:  cRegion(region),
 	}, nil
 }
 

maker/golang/xdb/util.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"math/big"
 	"net"
+	"net/netip"
 	"os"
 	"strings"
 )
@@ -35,6 +36,45 @@ func ParseIP(ip string) ([]byte, error) {
 	return nil, fmt.Errorf("invalid ip address: %s", ip)
 }
 
+var bitMaskList = []uint8{
+	0b1111_1111, // all zero
+	0b0111_1111,
+	0b0011_1111,
+	0b0001_1111,
+	0b0000_1111,
+	0b0000_0111,
+	0b0000_0011,
+	0b0000_0001,
+}
+
+func CIDR2Range(cidrStr string) ([]byte, []byte, error) {
+	prefix, err := netip.ParsePrefix(cidrStr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Get the start IP (Network Address)
+	// Masked() zeros out the host bits, which gives the starting IP of the subnet.
+	sip := prefix.Masked().Addr().AsSlice()
+	ipl := len(sip)
+	eip := make([]byte, ipl)
+	copy(eip, sip)
+
+	// Calculate the end IP (Broadcast Address)
+	bits := prefix.Bits()
+
+	// border byte rest bit filled with 1
+	byteIdx := bits / 8
+	eip[byteIdx] |= bitMaskList[bits-(byteIdx*8)]
+
+	// fill all the rest bits with 1
+	for bi := byteIdx + 1; bi < ipl; bi++ {
+		eip[bi] |= 0b1111_1111
+	}
+
+	return sip, eip, nil
+}
+
 func IP2String(ip []byte) string {
 	return net.IP(ip[:]).String()
 }
@@ -160,38 +200,14 @@ func IterateSegments(handle *os.File, autoMerge bool, before func(l string), fil
 			before(l)
 		}
 
-		var ps = strings.SplitN(l, "|", 3)
+		sip, eip, region, err := ParseSegment(l)
-		if len(ps) != 3 {
-			return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`", l)
-		}
-
-		sip, err := ParseIP(ps[0])
 		if err != nil {
-			return totalCount, mergeCount, fmt.Errorf("check start ip `%s`: %s", ps[0], err)
+			return totalCount, mergeCount, err
 		}
 
-		eip, err := ParseIP(ps[1])
-		if err != nil {
-			return totalCount, mergeCount, fmt.Errorf("check end ip `%s`: %s", ps[1], err)
-		}
-
-		if len(sip) != len(eip) {
-			return totalCount, mergeCount, fmt.Errorf("invalid ip segment line `%s`, sip/eip version not match", l)
-		}
-
-		if IPCompare(sip, eip) > 0 {
-			return totalCount, mergeCount, fmt.Errorf("start ip(%s) should not be greater than end ip(%s)", ps[0], ps[1])
-		}
-
-		// Allow empty region info since 2024/09/24
-		// if len(ps[2]) < 1 {
-		// 	return fmt.Errorf("empty region info in segment line `%s`", l)
-		// }
-
 		// check and do the region filter
-		var region = ps[2]
 		if filter != nil {
-			region, err = filter(ps[2])
+			region, err = filter(region)
 			if err != nil {
 				return totalCount, mergeCount, fmt.Errorf("failed to filter region `%s`: %s", region, err)
 			}
@@ -309,3 +325,41 @@ func RegionFiltering(region string, fields []int) (string, error) {
 
 	return strings.Join(sb, "|"), nil
 }
+
+// do the string split step by step as caller needed
+func StringTokenizer(str, substr string, cb func(s string, start int) bool) []string {
+	var tokens []string
+	var token string
+	var sIdx, oIdx, isEOF = 0, 0, false
+	for {
+		// do the token match
+		nIdx := strings.Index(str[sIdx:], substr)
+		if nIdx == -1 {
+			isEOF = true
+			token = str[sIdx:]
+		} else {
+			token = str[sIdx : sIdx+nIdx]
+		}
+
+		oIdx = sIdx                    // backup the old index
+		sIdx = sIdx + nIdx + 1         // reset the next start index
+		tokens = append(tokens, token) // append the token
+
+		// check and call the callback
+		if cb(token, oIdx) == false {
+			// keep the last token
+			if sIdx < len(str) {
+				tokens = append(tokens, str[sIdx:])
+			}
+
+			break
+		}
+
+		// check the EOF
+		if isEOF {
+			break
+		}
+	}
+
+	return tokens
+}

maker/golang/xdb/util_test.go

@@ -8,6 +8,7 @@ import (
 	"encoding/binary"
 	"fmt"
 	"os"
+	"strings"
 	"testing"
 )
 
@@ -196,6 +197,39 @@ func TestIPMiddle(t *testing.T) {
 	t.Logf("test done with %d ips", counter)
 }
 
+func TestSegmentFromt(t *testing.T) {
+	var lines = []string{
+		// ipv4 range
+		"2.10.222.0|2.10.223.255|France|Brittany|0|Orange S.A.|FR",
+		"8.35.35.0|8.35.159.255|United States|Colorado|0|ZSCALER, INC.|US",
+		"223.104.64.128|223.104.64.159|中国|广东省|深圳市|移动|CN",
+
+		// ipv4 CIDR
+		"1.0.4.0/24|2497 6453 7545 2764 38803|IGP",
+		"2.18.209.0/24|14537 3356 1299 34164 34164|IGP",
+		"14.32.46.0/24|2497 4766 9696|IGP",
+
+		// IPv6 range
+		"2001:200:17c::|2001:200:180:ffff:ffff:ffff:ffff:ffff|Japan|Tokyo|Tokyo|WIDE Project|JP",
+		"2001:506:100:226e::8|2001:506:100:226f::7|United States|Michigan|Detroit|Transact Ltd.|US",
+		"2a13:aac4:1000::|2a13:aac4:ffff:ffff:ffff:ffff:ffff:ffff|中国|广东省|深圳市|MLGT|CN",
+
+		// IPv6 CIDR
+		"2c0f:fc89:8081::/48|32590 9002 3257 8452 36992 36992 36992 36992 36992 36992 36992 36992 36992|EGP",
+		"2c0f:fe08:20a::/48|2497 6939 36914|IGP",
+		"2c0f:ffc8::/32|14537 23764 37468 22355|IGP",
+	}
+
+	for _, str := range lines {
+		seg, err := SegmentFrom(str, NewRegion)
+		if err != nil {
+			t.Fatalf("parse segment: %s", err)
+		}
+
+		fmt.Printf("seg={%s, %s, %s}\n", IP2String(seg.StartIP), IP2String(seg.EndIP), seg.Region.Str)
+	}
+}
+
 func TestSplitSegmentV4(t *testing.T) {
 	// var str = "1.1.0.0|1.3.3.24|中国|广东|深圳|电信"
 	// var str = "0.0.0.0|1.255.225.254|0|0|0|内网IP|内网IP"
@@ -258,10 +292,75 @@ func TestIterateSegments(t *testing.T) {
 		t.Fatalf("failed to open tests file: %s", err)
 	}
 
-	_, _, _ = IterateSegments(handle, true, func(l string) {
+	total, merged, err := IterateSegments(handle, true, func(l string) {
 		// fmt.Printf("load segment: `%s`\n", l)
 	}, nil, NewRegion, func(seg *Segment) error {
-		fmt.Printf("get segment: `%s`\n", seg)
+		fmt.Printf("{%s, %s} -> `%s`\n", IP2String(seg.StartIP), IP2String(seg.EndIP), seg.Region)
 		return nil
 	})
+	if err != nil {
+		t.Fatalf("iterate error: %s", err)
+	}
+
+	fmt.Printf("done iterate -> total:%d, merged:%d\n", total, merged)
+}
+
+func TestStringTokenizer(t *testing.T) {
+	var strList = []string{
+		"24.231.126.0/24|14537 2914 29866|IGP",
+		"24.231.126.0|24.231.126.255|14537 2914 29866|IGP",
+	}
+
+	var counter = 0
+	for _, str := range strList {
+		tokens := StringTokenizer(str, "|", func(s string, start int) bool {
+			// fmt.Printf("%s[idx=%d, |]=%s\n", str, start, s)
+			if counter == 0 {
+				if strings.Index(s, "/") > 0 {
+					return false
+				}
+			}
+
+			counter++
+			return counter < 2
+		})
+		fmt.Printf("%d tokens: %s\n", len(tokens), strings.Join(tokens, ", "))
+	}
+}
+
+func TestCIDR2Range(t *testing.T) {
+	var strList = [][3]string{
+		{"43.247.92.0/22", "43.247.92.0", "43.247.95.255"},
+		{"64.252.86.39/29", "64.252.86.32", "64.252.86.39"},
+		{"103.37.44.0/22", "103.37.44.0", "103.37.47.255"},
+		{"111.223.12.0/22", "111.223.12.0", "111.223.15.255"},
+		{"43.248.80.0/20", "43.248.80.0", "43.248.95.255"},
+		{"192.168.100.0/22", "192.168.100.0", "192.168.103.255"},
+		{"2403:3380::/32", "2403:3380::", "2403:3380:ffff:ffff:ffff:ffff:ffff:ffff"},
+		{"2001:db8:85a3::/64", "2001:db8:85a3::", "2001:db8:85a3:0:ffff:ffff:ffff:ffff"},
+		{"2001:db8:abcd::/48", "2001:db8:abcd::", "2001:db8:abcd:ffff:ffff:ffff:ffff:ffff"},
+	}
+
+	for _, item := range strList {
+		sip, eip, err := CIDR2Range(item[0])
+		if err != nil {
+			t.Fatalf("CIDR2Range: %s", err)
+		}
+
+		tSip, err := ParseIP(item[1])
+		if err != nil {
+			t.Fatalf("parse start ip: %s", err)
+		} else if IPCompare(sip, tSip) != 0 {
+			t.Fatalf("start ip %s != %s", IP2String(sip), item[1])
+		}
+
+		tEip, err := ParseIP(item[2])
+		if err != nil {
+			t.Fatalf("parse end ip: %s", err)
+		} else if IPCompare(eip, tEip) != 0 {
+			t.Fatalf("end ip %s != %s", IP2String(eip), item[2])
+		}
+
+		fmt.Printf("cidr=%s: {sip=%s, eip=%s}\n", item[0], item[1], item[2])
+	}
 }