diff --git a/binding/golang/ReadMe.md b/binding/golang/ReadMe.md index 23c23c5..34852f4 100644 --- a/binding/golang/ReadMe.md +++ b/binding/golang/ReadMe.md @@ -97,20 +97,24 @@ make # 查询测试 通过 `xdb_searcher search` 命令来测试 ip2region.xdb 的查询： -``` -➜ golang git:(v2.0_xdb) ./xdb_searcher search -./xdb_searcher search [command options] -options: - --db string ip2region binary xdb file path - --cache-policy string cache policy: file/vectorIndex/content +```bash +$ ./xdb_searcher search +Usage: + xdb_searcher search [flags] + +Flags: + -c, --cache-policy string cache policy (file|vectorIndex|content) (default "vectorIndex") + -d, --db string ip2region binary xdb file path (required) + -h, --help help for search ``` 例如：使用默认的 data/ip2region.xdb 进行查询测试 ```bash -➜ golang git:(v2.0_xdb) ✗ ./xdb_searcher search --db=../../data/ip2region.xdb -ip2region xdb searcher test program, type `quit` to exit -ip2region>> 1.2.3.4 -{region:美国|0|华盛顿|0|谷歌, took:101.57µs} +$ ./xdb_searcher search --db=../../data/ip2region.xdb +ip2region xdb searcher test program, cachePolicy: vectorIndex + type 'quit' to exit + ip2region>> 1.2.3.4 +{region: 美国|0|华盛顿|0|谷歌, ioCount: 7, took: 776.292µs} ``` 输入 ip 地址进行查询即可，输入 quit 退出测试程序。可以设置 `cache-policy` 为 file/vectorIndex/content 来测试不同的查询缓存机制。 @@ -120,17 +124,20 @@ ip2region>> 1.2.3.4 通过 `xdb_searcher bench` 命令来进行自动 bench 测试，一方面确保程序和 `xdb` 文件都没有错误，另一方面通过大量的查询得到平均查询性能： ```bash -➜ golang git:(v2.0_xdb) ./xdb_searcher bench -./xdb_searcher bench [command options] -options: - --db string ip2region binary xdb file path - --src string source ip text file path - --cache-policy string cache policy: file/vectorIndex/content +$ ./xdb_searcher bench +Usage: + xdb_searcher bench [flags] + +Flags: + -c, --cache-policy string cache policy (file|vectorIndex|content) (default "vectorIndex") + -d, --db string ip2region binary xdb file path (required) + -h, --help help for bench + -s, --src string source ip text file path (required) ``` 例如：通过 data/ip2region.xdb 和 data/ip.merge.txt 进行 bench 测试： ```bash -➜ golang git:(v2.0_xdb) ✗ ./xdb_searcher bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt +$ ./xdb_searcher bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt Bench finished, {total: 3417955, took: 28.211578339s, cost: 8253 ns/op} ``` diff --git a/binding/golang/cmd/bench.go b/binding/golang/cmd/bench.go new file mode 100644 index 0000000..2f4cfb3 --- /dev/null +++ b/binding/golang/cmd/bench.go @@ -0,0 +1,120 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "bufio" + "fmt" + "github.com/lionsoul2014/ip2region/binding/golang/xdb" + "github.com/mitchellh/go-homedir" + "os" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var ( + cachePolicyBench string + dbFileBench string + srcFileBench string +) + +// benchCmd represents the bench command +var benchCmd = &cobra.Command{ + Use: "bench", + Short: "search bench test", + Run: func(cmd *cobra.Command, args []string) { + bench() + }, +} + +func init() { + rootCmd.AddCommand(benchCmd) + + benchCmd.Flags().StringVarP(&dbFileBench, "db", "d", "", "ip2region binary xdb file path (required)") + benchCmd.Flags().StringVarP(&srcFileBench, "src", "s", "", "source ip text file path (required)") + benchCmd.Flags().StringVarP(&cachePolicyBench, "cache-policy", "c", "vectorIndex", "cache policy (file|vectorIndex|content)") + + // Required flags + _ = benchCmd.MarkFlagRequired("db") + _ = benchCmd.MarkFlagRequired("src") +} + +func bench() { + dbPath, err := homedir.Expand(dbFileBench) + if err != nil { + fmt.Printf("invalid xdb file path `%s`: %s", dbFileBench, err) + return + } + + searcher, err := createSearcher(dbPath, cachePolicyBench) + if err != nil { + fmt.Printf("failed to create searcher: %s\n", err.Error()) + return + } + defer func() { + searcher.Close() + }() + + handle, err := os.OpenFile(srcFileBench, os.O_RDONLY, 0600) + if err != nil { + fmt.Printf("failed to open source text file: %s\n", err) + return + } + + var count, tStart, costs = int64(0), time.Now(), int64(0) + var scanner = bufio.NewScanner(handle) + scanner.Split(bufio.ScanLines) + for scanner.Scan() { + var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n")) + var ps = strings.SplitN(l, "|", 3) + if len(ps) != 3 { + fmt.Printf("invalid ip segment line `%s`\n", l) + return + } + + sip, err := xdb.CheckIP(ps[0]) + if err != nil { + fmt.Printf("check start ip `%s`: %s\n", ps[0], err) + return + } + + eip, err := xdb.CheckIP(ps[1]) + if err != nil { + fmt.Printf("check end ip `%s`: %s\n", ps[1], err) + return + } + + if sip > eip { + fmt.Printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]) + return + } + + mip := xdb.MidIP(sip, eip) + for _, ip := range []uint32{sip, xdb.MidIP(sip, mip), mip, xdb.MidIP(mip, eip), eip} { + sTime := time.Now() + region, err := searcher.Search(ip) + if err != nil { + fmt.Printf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) + return + } + + costs += time.Since(sTime).Nanoseconds() + + // check the region info + if region != ps[2] { + fmt.Printf("failed Search(%s) with (%s != %s)\n", xdb.Long2IP(ip), region, ps[2]) + return + } + + count++ + } + } + + cost := time.Since(tStart) + fmt.Printf("Bench finished, {cachePolicy: %s, total: %d, took: %s, cost: %d μs/op}\n", + cachePolicyBench, count, cost, costs/count/1000) +} diff --git a/binding/golang/cmd/root.go b/binding/golang/cmd/root.go new file mode 100644 index 0000000..82e057b --- /dev/null +++ b/binding/golang/cmd/root.go @@ -0,0 +1,53 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "github.com/lionsoul2014/ip2region/binding/golang/xdb" + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "xdb_searcher", + Short: "Go Implementation For Ip2region", + Long: `Ip2region (2.0 - xdb) is a offline IP address manager framework and locator, +support billions of data segments, ten microsecond searching performance.`, +} + +func createSearcher(dbPath string, cachePolicy string) (*xdb.Searcher, error) { + switch cachePolicy { + case "nil", "file": + return xdb.NewWithFileOnly(dbPath) + case "vectorIndex": + vIndex, err := xdb.LoadVectorIndexFromFile(dbPath) + if err != nil { + return nil, fmt.Errorf("failed to load vector index from `%s`: %w", dbPath, err) + } + + return xdb.NewWithVectorIndex(dbPath, vIndex) + case "content": + cBuff, err := xdb.LoadContentFromFile(dbPath) + if err != nil { + return nil, fmt.Errorf("failed to load content from '%s': %w", dbPath, err) + } + + return xdb.NewWithBuffer(cBuff) + default: + return nil, fmt.Errorf("invalid cache policy `%s`, options: file/vectorIndex/content", cachePolicy) + } +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} diff --git a/binding/golang/cmd/search.go b/binding/golang/cmd/search.go new file mode 100644 index 0000000..baea8ff --- /dev/null +++ b/binding/golang/cmd/search.go @@ -0,0 +1,86 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "bufio" + "fmt" + "github.com/mitchellh/go-homedir" + "github.com/spf13/cobra" + "log" + "os" + "strings" + "time" +) + +var ( + dbFileSearch string + cachePolicySearch string +) + +// searchCmd represents the search command +var searchCmd = &cobra.Command{ + Use: "search", + Short: "search input test", + Run: func(cmd *cobra.Command, args []string) { + searcher() + }, +} + +func init() { + rootCmd.AddCommand(searchCmd) + + searchCmd.Flags().StringVarP(&dbFileSearch, "db", "d", "", "ip2region binary xdb file path (required)") + searchCmd.Flags().StringVarP(&cachePolicySearch, "cache-policy", "c", "vectorIndex", "cache policy (file|vectorIndex|content)") + // Required flags + _ = searchCmd.MarkFlagRequired("db") + +} + +func searcher() { + dbPath, err := homedir.Expand(dbFileSearch) + if err != nil { + fmt.Printf("invalid xdb file path `%s`: %s", dbFileSearch, err) + return + } + // create the searcher with the cache policy setting + searcher, err := createSearcher(dbPath, "vectorIndex") + if err != nil { + fmt.Printf("failed to create searcher: %s\n", err.Error()) + return + } + defer func() { + searcher.Close() + fmt.Println("searcher test program exited, thanks for trying") + }() + + fmt.Printf("ip2region xdb searcher test program, cachePolicy: %s \n type 'quit' to exit \n ", cachePolicySearch) + reader := bufio.NewReader(os.Stdin) + for { + fmt.Print("ip2region>> ") + str, err := reader.ReadString('\n') + if err != nil { + log.Fatalf("failed to read string: %s", err) + } + + line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) + if len(line) == 0 { + continue + } + + if line == "quit" { + break + } + + tStart := time.Now() + region, err := searcher.SearchByStr(line) + if err != nil { + fmt.Printf("\x1b[0;31m{err: %s, ioCount: %d}\x1b[0m\n", err.Error(), searcher.GetIOCount()) + } else { + fmt.Printf("\x1b[0;32m{region: %s, ioCount: %d, took: %s}\x1b[0m\n", region, searcher.GetIOCount(), time.Since(tStart)) + } + } + +} diff --git a/binding/golang/go.mod b/binding/golang/go.mod index 2bb4dc1..ebf5f46 100644 --- a/binding/golang/go.mod +++ b/binding/golang/go.mod @@ -1,7 +1,13 @@ module github.com/lionsoul2014/ip2region/binding/golang -go 1.17 +go 1.19 -require github.com/mitchellh/go-homedir v1.1.0 +require ( + github.com/mitchellh/go-homedir v1.1.0 + github.com/spf13/cobra v1.7.0 +) -require github.com/yookoala/realpath v1.0.0 // indirect +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/binding/golang/go.sum b/binding/golang/go.sum index 837982a..b522ae3 100644 --- a/binding/golang/go.sum +++ b/binding/golang/go.sum @@ -1,4 +1,12 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/yookoala/realpath v1.0.0 h1:7OA9pj4FZd+oZDsyvXWQvjn5oBdcHRTV44PpdMSuImQ= -github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Ya9AIoYBpE= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/binding/golang/main.go b/binding/golang/main.go index ad58abb..ef2a0ca 100644 --- a/binding/golang/main.go +++ b/binding/golang/main.go @@ -1,270 +1,10 @@ // Copyright 2022 The Ip2Region Authors. All rights reserved. // Use of this source code is governed by a Apache2.0-style // license that can be found in the LICENSE file. - -// --- -// @Author Lion -// @Date 2022/06/16 - package main -import ( - "bufio" - "fmt" - "github.com/lionsoul2014/ip2region/binding/golang/xdb" - "github.com/mitchellh/go-homedir" - "log" - "os" - "strings" - "time" -) - -func printHelp() { - fmt.Printf("ip2region xdb searcher\n") - fmt.Printf("%s [command] [command options]\n", os.Args[0]) - fmt.Printf("Command: \n") - fmt.Printf(" search search input test\n") - fmt.Printf(" bench search bench test\n") -} - -func testSearch() { - var err error - var dbFile, cachePolicy = "", "vectorIndex" - for i := 2; i < len(os.Args); i++ { - r := os.Args[i] - if len(r) < 5 { - continue - } - - if strings.Index(r, "--") != 0 { - continue - } - - var sIdx = strings.Index(r, "=") - if sIdx < 0 { - fmt.Printf("missing = for args pair '%s'\n", r) - return - } - - switch r[2:sIdx] { - case "db": - dbFile = r[sIdx+1:] - case "cache-policy": - cachePolicy = r[sIdx+1:] - default: - fmt.Printf("undefined option `%s`\n", r) - return - } - } - - if dbFile == "" { - fmt.Printf("%s search [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --db string ip2region binary xdb file path\n") - fmt.Printf(" --cache-policy string cache policy: file/vectorIndex/content\n") - return - } - - dbPath, err := homedir.Expand(dbFile) - if err != nil { - fmt.Printf("invalid xdb file path `%s`: %s", dbFile, err) - return - } - - // create the searcher with the cache policy setting - searcher, err := createSearcher(dbPath, cachePolicy) - if err != nil { - fmt.Printf("failed to create searcher: %s\n", err.Error()) - return - } - defer func() { - searcher.Close() - fmt.Printf("searcher test program exited, thanks for trying\n") - }() - - fmt.Printf(`ip2region xdb searcher test program, cachePolicy: %s -type 'quit' to exit -`, cachePolicy) - reader := bufio.NewReader(os.Stdin) - for { - fmt.Print("ip2region>> ") - str, err := reader.ReadString('\n') - if err != nil { - log.Fatalf("failed to read string: %s", err) - } - - line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) - if len(line) == 0 { - continue - } - - if line == "quit" { - break - } - - tStart := time.Now() - region, err := searcher.SearchByStr(line) - if err != nil { - fmt.Printf("\x1b[0;31m{err: %s, ioCount: %d}\x1b[0m\n", err.Error(), searcher.GetIOCount()) - } else { - fmt.Printf("\x1b[0;32m{region: %s, ioCount: %d, took: %s}\x1b[0m\n", region, searcher.GetIOCount(), time.Since(tStart)) - } - } -} - -func testBench() { - var err error - var dbFile, srcFile, cachePolicy = "", "", "vectorIndex" - for i := 2; i < len(os.Args); i++ { - r := os.Args[i] - if len(r) < 5 { - continue - } - - if strings.Index(r, "--") != 0 { - continue - } - - var sIdx = strings.Index(r, "=") - if sIdx < 0 { - fmt.Printf("missing = for args pair '%s'\n", r) - return - } - - switch r[2:sIdx] { - case "db": - dbFile = r[sIdx+1:] - case "src": - srcFile = r[sIdx+1:] - case "cache-policy": - cachePolicy = r[sIdx+1:] - default: - fmt.Printf("undefined option `%s`\n", r) - return - } - } - - if dbFile == "" || srcFile == "" { - fmt.Printf("%s bench [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --db string ip2region binary xdb file path\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --cache-policy string cache policy: file/vectorIndex/content\n") - return - } - - dbPath, err := homedir.Expand(dbFile) - if err != nil { - fmt.Printf("invalid xdb file path `%s`: %s", dbFile, err) - return - } - - searcher, err := createSearcher(dbPath, cachePolicy) - if err != nil { - fmt.Printf("failed to create searcher: %s\n", err.Error()) - return - } - defer func() { - searcher.Close() - }() - - handle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) - if err != nil { - fmt.Printf("failed to open source text file: %s\n", err) - return - } - - var count, tStart, costs = int64(0), time.Now(), int64(0) - var scanner = bufio.NewScanner(handle) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - var l = strings.TrimSpace(strings.TrimSuffix(scanner.Text(), "\n")) - var ps = strings.SplitN(l, "|", 3) - if len(ps) != 3 { - fmt.Printf("invalid ip segment line `%s`\n", l) - return - } - - sip, err := xdb.CheckIP(ps[0]) - if err != nil { - fmt.Printf("check start ip `%s`: %s\n", ps[0], err) - return - } - - eip, err := xdb.CheckIP(ps[1]) - if err != nil { - fmt.Printf("check end ip `%s`: %s\n", ps[1], err) - return - } - - if sip > eip { - fmt.Printf("start ip(%s) should not be greater than end ip(%s)\n", ps[0], ps[1]) - return - } - - mip := xdb.MidIP(sip, eip) - for _, ip := range []uint32{sip, xdb.MidIP(sip, mip), mip, xdb.MidIP(mip, eip), eip} { - sTime := time.Now() - region, err := searcher.Search(ip) - if err != nil { - fmt.Printf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) - return - } - - costs += time.Since(sTime).Nanoseconds() - - // check the region info - if region != ps[2] { - fmt.Printf("failed Search(%s) with (%s != %s)\n", xdb.Long2IP(ip), region, ps[2]) - return - } - - count++ - } - } - - cost := time.Since(tStart) - fmt.Printf("Bench finished, {cachePolicy: %s, total: %d, took: %s, cost: %d μs/op}\n", - cachePolicy, count, cost, costs/count/1000) -} - -func createSearcher(dbPath string, cachePolicy string) (*xdb.Searcher, error) { - switch cachePolicy { - case "nil", "file": - return xdb.NewWithFileOnly(dbPath) - case "vectorIndex": - vIndex, err := xdb.LoadVectorIndexFromFile(dbPath) - if err != nil { - return nil, fmt.Errorf("failed to load vector index from `%s`: %w", dbPath, err) - } - - return xdb.NewWithVectorIndex(dbPath, vIndex) - case "content": - cBuff, err := xdb.LoadContentFromFile(dbPath) - if err != nil { - return nil, fmt.Errorf("failed to load content from '%s': %w", dbPath, err) - } - - return xdb.NewWithBuffer(cBuff) - default: - return nil, fmt.Errorf("invalid cache policy `%s`, options: file/vectorIndex/content", cachePolicy) - } -} +import "github.com/lionsoul2014/ip2region/binding/golang/cmd" func main() { - if len(os.Args) < 2 { - printHelp() - return - } - - // set the log flag - log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) - switch strings.ToLower(os.Args[1]) { - case "search": - testSearch() - case "bench": - testBench() - default: - printHelp() - } + cmd.Execute() } diff --git a/maker/golang/ReadMe.md b/maker/golang/ReadMe.md index a0b7bb9..cd22b9c 100644 --- a/maker/golang/ReadMe.md +++ b/maker/golang/ReadMe.md @@ -14,17 +14,21 @@ make # `xdb` 数据生成 通过 `xdb_maker gen` 命令生成 ip2region.xdb 二进制文件: -``` -➜ golang git:(v2.0_xdb) ✗ ./xdb_maker gen -./xdb_maker gen [command options] -options: - --src string source ip text file path - --dst string destination binary xdb file path +```bash +$ ./xdb_maker gen +Usage: + xdb_maker gen [flags] + +Flags: + -d, --dst string destination binary xdb file path (required) + -h, --help help for gen + -i, --index-policy string generate index policy (vector|btree) (default "vector") + -s, --src string source ip text file path (required) ``` 例如，使用默认的 data/ip.merge.txt 作为源数据，生成一个 ip2region.xdb 到当前目录： ```bash -➜ golang git:(v2.0_xdb) ✗ ./xdb_maker gen --src=../../data/ip.merge.txt --dst=./ip2region.xdb +$ ./xdb_maker gen --src=../../data/ip.merge.txt --dst=./ip2region.xdb # 会看到一堆输出，最终会看到类似如下输出表示运行结束 ... 2022/06/16 16:38:48 maker.go:317: write done, with 13804 data blocks and (683591, 720221) index blocks @@ -35,54 +39,58 @@ options: # `xdb` 数据查询 通过 `xdb_maker search` 命令来测试查询输入的 ip： -``` -➜ golang git:(v2.0_xdb) ✗ ./xdb_maker search -./xdb_maker search [command options] -options: - --db string ip2region binary xdb file path +```bash +$ ./xdb_maker search +Usage: + xdb_maker search [flags] + +Flags: + -d, --db string ip2region binary xdb file path (required) + -h, --help help for search ``` 例如，使用自带的 xdb 文件来运行查询测试： ```bash -➜ golang git:(v2.0_xdb) ✗ ./xdb_maker search --db=../../data/ip2region.xdb +$ ./xdb_maker search --db=../../data/ip2region.xdb ip2region xdb search test program, commands: loadIndex : load the vector index for search speedup. clearIndex: clear the vector index. -quit : exit the test program +quit : exit the test program. ip2region>> 103.192.227.215 -{region:中国|0|香港|0|0, iocount:8, took:87.151µs} -ip2region>> loadIndex -vector index cached +{region:中国|0|香港|0|0, iocount:8, took:1.390702ms} ip2region>> 39.114.2.16 -{region:韩国|0|0|0|SK宽带, iocount:2, took:50.005µs} +{region:韩国|0|0|0|SK宽带, iocount:3, took:1.337932ms} ip2region>> 120.24.130.96 -{region:中国|0|广东省|深圳市|阿里云, iocount:2, took:32.805µs} +{region:中国|0|广东省|深圳市|阿里云, iocount:3, took:1.387141ms} ip2region>> ``` # `xdb` 数据编辑 通过 `xdb_maker edit` 命令来编辑原始的　IP 数据： -``` -➜ golang git:(fr_editor) ✗ ./xdb_maker edit -./xdb_maker edit [command options] -options: - --src string source ip text file path +```bash +$ ./xdb_maker edit +Usage: + xdb_maker edit [flags] + +Flags: + -h, --help help for edit + -s, --src string source ip text file path (required) ``` 例如，使用编辑器打开 `./data/ip.merge.txt` 会看到如下的操作面板： ```bash -➜ golang git:(fr_editor) ✗ ./xdb_maker edit --src=../../data/ip.merge.txt +$ ./xdb_maker edit --src=../../data/ip.merge.txt init the editor from source @ `../../data/ip.merge.txt` ... -all segments loaded, length: 683591, elapsed: 479.73743ms -command list: +all segments loaded, length: 683843, elapsed: 503.631945ms +command list: put [segment] : put the specifield $segment put_file [file] : put all the segments from the specified $file list [offset] [size] : list the first $size segments start from $offset save : save all the changes to the destination source file quit : exit the program help : print this help menu -editor>> +editor>> ``` 通过 `put` 命令修改指定 IP 段的定位信息，例如： @@ -109,18 +117,22 @@ editor>> # bench 测试 如果你自主生成了 `xdb` 文件，请确保运行如下的 `xdb_maker bench` 命令来确保生成的的 `xdb` 文件的正确性： -``` -➜ golang git:(v2.0_xdb) ✗ ./xdb_maker bench -./xdb_maker bench [command options] -options: - --db string ip2region binary xdb file path - --src string source ip text file path - --ignore-error bool keep going if bench failed +```bash +$ ./xdb_maker bench +Usage: + xdb_maker bench [flags] + +Flags: + -d, --db string ip2region binary xdb file path (required) + -h, --help help for bench + -i, --ignore-error keep going if bench failed (default "false") + -s, --src string source ip text file path (required) + ``` 例如：使用 data/ip.merge.txt 源文件来 bench 测试 data/ip2region.xdb 这个 xdb 文件： ```bash -➜ golang git:(v2.0_xdb) ✗ ./xdb_maker bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt +$ ./xdb_maker bench --db=../../data/ip2region.xdb --src=../../data/ip.merge.txt # 会看到一堆输出，看到类似如下的数据表示 bench 测试通过了，否则就会报错 ... try to bench segment: `224.0.0.0|255.255.255.255|0|0|0|内网IP|内网IP` diff --git a/maker/golang/cmd/bench.go b/maker/golang/cmd/bench.go new file mode 100644 index 0000000..4e07f29 --- /dev/null +++ b/maker/golang/cmd/bench.go @@ -0,0 +1,91 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "github.com/lionsoul2014/ip2region/maker/golang/xdb" + "os" + "time" + + "github.com/spf13/cobra" +) + +var ( + ignoreErrorBench bool + dbFileBench string + srcFileBench string +) + +// benchCmd represents the bench command +var benchCmd = &cobra.Command{ + Use: "bench", + Short: "binary xdb bench test", + Run: func(cmd *cobra.Command, args []string) { + bench() + }, +} + +func init() { + rootCmd.AddCommand(benchCmd) + + benchCmd.Flags().StringVarP(&dbFileBench, "db", "d", "", "ip2region binary xdb file path (required)") + benchCmd.Flags().StringVarP(&srcFileBench, "src", "s", "", "source ip text file path (required)") + benchCmd.Flags().BoolVarP(&ignoreErrorBench, "ignore-error", "i", false, "keep going if bench failed (default \"false\")") + + // Required flags + _ = benchCmd.MarkFlagRequired("db") + _ = benchCmd.MarkFlagRequired("src") + +} + +func bench() { + searcher, err := xdb.NewSearcher(dbFileBench) + if err != nil { + fmt.Printf("failed to create searcher with `%s`: %s\n", dbFileBench, err) + return + } + defer func() { + searcher.Close() + }() + + handle, err := os.OpenFile(srcFileBench, os.O_RDONLY, 0600) + if err != nil { + fmt.Printf("failed to open source text file: %s\n", err) + return + } + + var count, errCount, tStart = 0, 0, time.Now() + var iErr = xdb.IterateSegments(handle, nil, func(seg *xdb.Segment) error { + var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) + fmt.Printf("try to bench segment: `%s`\n", l) + mip := xdb.MidIP(seg.StartIP, seg.EndIP) + for _, ip := range []uint32{seg.StartIP, xdb.MidIP(seg.EndIP, mip), mip, xdb.MidIP(mip, seg.EndIP), seg.EndIP} { + fmt.Printf("|-try to bench ip '%s' ... ", xdb.Long2IP(ip)) + r, _, err := searcher.Search(ip) + if err != nil { + return fmt.Errorf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) + } + // check the region info + count++ + if r != seg.Region { + errCount++ + fmt.Printf(" --[Failed] (%s != %s)\n", r, seg.Region) + if !ignoreErrorBench { + return fmt.Errorf("") + } + } else { + fmt.Println(" --[Ok]") + } + } + return nil + }) + if iErr != nil { + fmt.Printf("%s", err) + return + } + + fmt.Printf("Bench finished, {count: %d, failed: %d, took: %s}\n", count, errCount, time.Since(tStart)) +} diff --git a/maker/golang/cmd/edit.go b/maker/golang/cmd/edit.go new file mode 100644 index 0000000..677764a --- /dev/null +++ b/maker/golang/cmd/edit.go @@ -0,0 +1,143 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "bufio" + "fmt" + "github.com/lionsoul2014/ip2region/maker/golang/xdb" + "os" + "regexp" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var srcFileEdit string + +// editCmd represents the edit command +var editCmd = &cobra.Command{ + Use: "edit", + Short: "edit the source ip data", + Run: func(cmd *cobra.Command, args []string) { + edit() + }, +} + +func init() { + rootCmd.AddCommand(editCmd) + + editCmd.Flags().StringVarP(&srcFileEdit, "src", "s", "", "source ip text file path (required)") + // Required flags + _ = editCmd.MarkFlagRequired("src") +} + +func edit() { + rExp, err := regexp.Compile("\\s+") + if err != nil { + fmt.Printf("failed to compile regexp: %s\n", err) + return + } + + fmt.Printf("init the editor from source @ `%s` ... \n", srcFileEdit) + var tStart = time.Now() + editor, err := xdb.NewEditor(srcFileEdit) + if err != nil { + fmt.Printf("failed to init editor: %s", err) + return + } + + fmt.Printf("all segments loaded, length: %d, elapsed: %s\n", editor.SegLen(), time.Since(tStart)) + var help = func() { + fmt.Println("command list:") + fmt.Println(" put [segment] : put the specifield $segment") + fmt.Println(" put_file [file] : put all the segments from the specified $file") + fmt.Println(" list [offset] [size] : list the first $size segments start from $offset") + fmt.Println(" save : save all the changes to the destination source file") + fmt.Println(" quit : exit the program") + fmt.Println(" help : print this help menu") + } + + help() + var sTip = "" + var reader = bufio.NewReader(os.Stdin) + for { + if editor.NeedSave() { + sTip = "*" + } else { + sTip = "" + } + + fmt.Printf("%seditor>> ", sTip) + line, err := reader.ReadString('\n') + if err != nil { + fmt.Printf("failed to read line from cli: %s\n", err) + break + } + + cmd := strings.TrimSpace(line) + if cmd == "help" { + help() + } else if cmd == "quit" { + if editor.NeedSave() { + fmt.Printf("there are changes that need to save, type 'quit!' to force quit\n") + } else { + break + } + } else if cmd == "quit!" { + // quit directly + break + } else if cmd == "save" { + err = editor.Save() + if err != nil { + fmt.Printf("failed to save the changes: %s\n", err) + continue + } + fmt.Printf("all segments saved to %s\n", srcFileEdit) + } else if strings.HasPrefix(cmd, "list") { + var sErr error + off, size, l := 0, 10, len("list") + str := strings.TrimSpace(cmd) + if len(str) > l { + sets := rExp.Split(cmd, 3) + switch len(sets) { + case 2: + _, sErr = fmt.Sscanf(cmd, "%s %d", &str, &off) + case 3: + _, sErr = fmt.Sscanf(cmd, "%s %d %d", &str, &off, &size) + } + } + + if sErr != nil { + fmt.Printf("failed to parse the offset and size: %s\n", sErr) + continue + } + + fmt.Printf("+-slice(%d,%d): \n", off, size) + for _, s := range editor.Slice(off, size) { + fmt.Printf("%s\n", s) + } + } else if strings.HasPrefix(cmd, "put ") { + seg := strings.TrimSpace(cmd[len("put "):]) + o, n, err := editor.Put(seg) + if err != nil { + fmt.Printf("failed to Put(%s): %s\n", seg, err) + continue + } + fmt.Printf("Put(%s): Ok, with %d deletes and %d additions\n", seg, o, n) + } else if strings.HasPrefix(cmd, "put_file ") { + file := strings.TrimSpace(cmd[len("put_file "):]) + o, n, err := editor.PutFile(file) + if err != nil { + fmt.Printf("failed to PutFile(%s): %s\n", file, err) + continue + } + fmt.Printf("PutFile(%s): Ok, with %d deletes and %d additions\n", file, o, n) + } else if len(cmd) > 0 { + help() + } + } +} diff --git a/maker/golang/cmd/gen.go b/maker/golang/cmd/gen.go new file mode 100644 index 0000000..e8b3f4d --- /dev/null +++ b/maker/golang/cmd/gen.go @@ -0,0 +1,75 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "fmt" + "github.com/lionsoul2014/ip2region/maker/golang/xdb" + "log" + "time" + + "github.com/spf13/cobra" +) + +var ( + dstFileGen string + srcFileGen string + indexPolicyGen string +) + +// genCmd represents the gen command +var genCmd = &cobra.Command{ + Use: "gen", + Short: "generate the binary db file", + Run: func(cmd *cobra.Command, args []string) { + genDb() + }, +} + +func init() { + rootCmd.AddCommand(genCmd) + + genCmd.Flags().StringVarP(&dstFileGen, "dst", "d", "", "destination binary xdb file path (required)") + genCmd.Flags().StringVarP(&srcFileGen, "src", "s", "", "source ip text file path (required)") + genCmd.Flags().StringVarP(&indexPolicyGen, "index-policy", "i", "vector", "generate index policy (vector|btree)") + + // Required flags + _ = genCmd.MarkFlagRequired("dst") + _ = genCmd.MarkFlagRequired("src") + +} + +func genDb() { + indexPolicy, err := xdb.IndexPolicyFromString(indexPolicyGen) + if err != nil { + fmt.Println(err) + } + // make the binary file + tStart := time.Now() + maker, err := xdb.NewMaker(indexPolicy, srcFileGen, dstFileGen) + if err != nil { + fmt.Printf("failed to create %s\n", err) + return + } + + err = maker.Init() + if err != nil { + fmt.Printf("failed Init: %s\n", err) + return + } + + err = maker.Start() + if err != nil { + fmt.Printf("failed Start: %s\n", err) + return + } + + err = maker.End() + if err != nil { + fmt.Printf("failed End: %s\n", err) + } + + log.Printf("Done, elapsed: %s\n", time.Since(tStart)) +} diff --git a/maker/golang/cmd/root.go b/maker/golang/cmd/root.go new file mode 100644 index 0000000..16d56e9 --- /dev/null +++ b/maker/golang/cmd/root.go @@ -0,0 +1,28 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "os" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "xdb_maker", + Short: "Go Implementation For Ip2region", + Long: `Ip2region (2.0 - xdb) is a offline IP address manager framework and locator, +support billions of data segments, ten microsecond searching performance.`, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} diff --git a/maker/golang/cmd/search.go b/maker/golang/cmd/search.go new file mode 100644 index 0000000..8803891 --- /dev/null +++ b/maker/golang/cmd/search.go @@ -0,0 +1,96 @@ +// Copyright 2022 The Ip2Region Authors. All rights reserved. +// Use of this source code is governed by a Apache2.0-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "bufio" + "fmt" + "github.com/lionsoul2014/ip2region/maker/golang/xdb" + "log" + "os" + "strings" + "time" + + "github.com/spf13/cobra" +) + +var dbFileSearch string + +// searchCmd represents the search command +var searchCmd = &cobra.Command{ + Use: "search", + Short: "binary xdb search test", + Run: func(cmd *cobra.Command, args []string) { + search() + }, +} + +func init() { + rootCmd.AddCommand(searchCmd) + + searchCmd.Flags().StringVarP(&dbFileSearch, "db", "d", "", "ip2region binary xdb file path (required)") + // Required flags + _ = searchCmd.MarkFlagRequired("db") +} + +func search() { + searcher, err := xdb.NewSearcher(dbFileSearch) + if err != nil { + fmt.Printf("failed to create searcher with `%s`: %s\n", dbFileSearch, err.Error()) + return + } + defer func() { + searcher.Close() + fmt.Println("test program exited, thanks for trying") + }() + + fmt.Println("ip2region xdb search test program, commands:") + fmt.Println("loadIndex : load the vector index for search speedup.") + fmt.Println("clearIndex: clear the vector index.") + fmt.Println("quit : exit the test program.") + + reader := bufio.NewReader(os.Stdin) + for { + fmt.Print("ip2region>> ") + str, err := reader.ReadString('\n') + if err != nil { + log.Fatalf("failed to read string: %s", err) + } + + line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) + if len(line) == 0 { + continue + } + // command interception and execution + if line == "loadIndex" { + err = searcher.LoadVectorIndex() + if err != nil { + log.Fatalf("failed to load vector index: %s", err) + } + fmt.Printf("vector index cached\n") + continue + } else if line == "clearIndex" { + searcher.ClearVectorIndex() + fmt.Printf("vector index cleared\n") + continue + } else if line == "quit" { + break + } + + ip, err := xdb.CheckIP(line) + if err != nil { + fmt.Printf("invalid ip address `%s`\n", line) + continue + } + + tStart := time.Now() + region, ioCount, err := searcher.Search(ip) + if err != nil { + fmt.Printf("\x1b[0;31m{err:%s, iocount:%d}\x1b[0m\n", err.Error(), ioCount) + } else { + fmt.Printf("\x1b[0;32m{region:%s, iocount:%d, took:%s}\x1b[0m\n", region, ioCount, time.Since(tStart)) + } + } +} diff --git a/maker/golang/go.mod b/maker/golang/go.mod index 7c3242a..fc2913b 100644 --- a/maker/golang/go.mod +++ b/maker/golang/go.mod @@ -1,3 +1,10 @@ module github.com/lionsoul2014/ip2region/maker/golang -go 1.17 +go 1.19 + +require github.com/spf13/cobra v1.7.0 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect +) diff --git a/maker/golang/go.sum b/maker/golang/go.sum index e69de29..f3366a9 100644 --- a/maker/golang/go.sum +++ b/maker/golang/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/maker/golang/main.go b/maker/golang/main.go index 91b6d4e..9ed56ea 100644 --- a/maker/golang/main.go +++ b/maker/golang/main.go @@ -4,430 +4,8 @@ package main -import ( - "bufio" - "fmt" - "github.com/lionsoul2014/ip2region/maker/golang/xdb" - "log" - "os" - "regexp" - "strings" - "time" -) - -func printHelp() { - fmt.Printf("ip2region xdb maker\n") - fmt.Printf("%s [command] [command options]\n", os.Args[0]) - fmt.Printf("Command: \n") - fmt.Printf(" gen generate the binary db file\n") - fmt.Printf(" search binary xdb search test\n") - fmt.Printf(" bench binary xdb bench test\n") - fmt.Printf(" edit edit the source ip data\n") -} - -// Iterate the cli flags -func iterateFlags(cb func(key string, val string) error) error { - for i := 2; i < len(os.Args); i++ { - r := os.Args[i] - if len(r) < 5 { - continue - } - - if strings.Index(r, "--") != 0 { - continue - } - - var sIdx = strings.Index(r, "=") - if sIdx < 0 { - return fmt.Errorf("missing = for args pair '%s'", r) - } - - if err := cb(r[2:sIdx], r[sIdx+1:]); err != nil { - return err - } - } - - return nil -} - -func genDb() { - var err error - var srcFile, dstFile = "", "" - var indexPolicy = xdb.VectorIndexPolicy - var fErr = iterateFlags(func(key string, val string) error { - switch key { - case "src": - srcFile = val - case "dst": - dstFile = val - case "index": - indexPolicy, err = xdb.IndexPolicyFromString(val) - if err != nil { - return fmt.Errorf("parse policy: %w", err) - } - default: - return fmt.Errorf("undefine option `%s=%s`\n", key, val) - } - - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if srcFile == "" || dstFile == "" { - fmt.Printf("%s gen [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --dst string destination binary xdb file path\n") - return - } - - // make the binary file - tStart := time.Now() - maker, err := xdb.NewMaker(indexPolicy, srcFile, dstFile) - if err != nil { - fmt.Printf("failed to create %s\n", err) - return - } - - err = maker.Init() - if err != nil { - fmt.Printf("failed Init: %s\n", err) - return - } - - err = maker.Start() - if err != nil { - fmt.Printf("failed Start: %s\n", err) - return - } - - err = maker.End() - if err != nil { - fmt.Printf("failed End: %s\n", err) - } - - log.Printf("Done, elapsed: %s\n", time.Since(tStart)) -} - -func testSearch() { - var err error - var dbFile = "" - var fErr = iterateFlags(func(key string, val string) error { - if key == "db" { - dbFile = val - } else { - return fmt.Errorf("undefined option '%s=%s'\n", key, val) - } - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if dbFile == "" { - fmt.Printf("%s search [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --db string ip2region binary xdb file path\n") - return - } - - searcher, err := xdb.NewSearcher(dbFile) - if err != nil { - fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err.Error()) - return - } - defer func() { - searcher.Close() - fmt.Printf("test program exited, thanks for trying\n") - }() - - fmt.Println(`ip2region xdb search test program, commands: -loadIndex : load the vector index for search speedup. -clearIndex: clear the vector index. -quit : exit the test program`) - reader := bufio.NewReader(os.Stdin) - for { - fmt.Print("ip2region>> ") - str, err := reader.ReadString('\n') - if err != nil { - log.Fatalf("failed to read string: %s", err) - } - - line := strings.TrimSpace(strings.TrimSuffix(str, "\n")) - if len(line) == 0 { - continue - } - - // command interception and execution - if line == "loadIndex" { - err = searcher.LoadVectorIndex() - if err != nil { - log.Fatalf("failed to load vector index: %s", err) - } - fmt.Printf("vector index cached\n") - continue - } else if line == "clearIndex" { - searcher.ClearVectorIndex() - fmt.Printf("vector index cleared\n") - continue - } else if line == "quit" { - break - } - - ip, err := xdb.CheckIP(line) - if err != nil { - fmt.Printf("invalid ip address `%s`\n", line) - continue - } - - tStart := time.Now() - region, ioCount, err := searcher.Search(ip) - if err != nil { - fmt.Printf("\x1b[0;31m{err:%s, iocount:%d}\x1b[0m\n", err.Error(), ioCount) - } else { - fmt.Printf("\x1b[0;32m{region:%s, iocount:%d, took:%s}\x1b[0m\n", region, ioCount, time.Since(tStart)) - } - } -} - -func testBench() { - var err error - var dbFile, srcFile = "", "" - var ignoreError = false - var fErr = iterateFlags(func(key string, val string) error { - switch key { - case "db": - dbFile = val - case "src": - srcFile = val - case "ignore-error": - if val == "true" || val == "1" { - ignoreError = true - } else if val == "false" || val == "0" { - ignoreError = false - } else { - return fmt.Errorf("invalid value for ignore-error option, could be false/0 or true/1\n") - } - default: - return fmt.Errorf("undefined option '%s=%s'\n", key, val) - } - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if dbFile == "" || srcFile == "" { - fmt.Printf("%s bench [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --db string ip2region binary xdb file path\n") - fmt.Printf(" --src string source ip text file path\n") - fmt.Printf(" --ignore-error bool keep going if bench failed\n") - return - } - - searcher, err := xdb.NewSearcher(dbFile) - if err != nil { - fmt.Printf("failed to create searcher with `%s`: %s\n", dbFile, err) - return - } - defer func() { - searcher.Close() - }() - - handle, err := os.OpenFile(srcFile, os.O_RDONLY, 0600) - if err != nil { - fmt.Printf("failed to open source text file: %s\n", err) - return - } - - var count, errCount, tStart = 0, 0, time.Now() - var iErr = xdb.IterateSegments(handle, nil, func(seg *xdb.Segment) error { - var l = fmt.Sprintf("%d|%d|%s", seg.StartIP, seg.EndIP, seg.Region) - fmt.Printf("try to bench segment: `%s`\n", l) - mip := xdb.MidIP(seg.StartIP, seg.EndIP) - for _, ip := range []uint32{seg.StartIP, xdb.MidIP(seg.EndIP, mip), mip, xdb.MidIP(mip, seg.EndIP), seg.EndIP} { - fmt.Printf("|-try to bench ip '%s' ... ", xdb.Long2IP(ip)) - r, _, err := searcher.Search(ip) - if err != nil { - return fmt.Errorf("failed to search ip '%s': %s\n", xdb.Long2IP(ip), err) - } - - // check the region info - count++ - if r != seg.Region { - errCount++ - fmt.Printf(" --[Failed] (%s != %s)\n", r, seg.Region) - if ignoreError == false { - return fmt.Errorf("") - } - } else { - fmt.Printf(" --[Ok]\n") - } - } - return nil - }) - if iErr != nil { - fmt.Printf("%s", err) - return - } - - fmt.Printf("Bench finished, {count: %d, failed: %d, took: %s}\n", count, errCount, time.Since(tStart)) -} - -func edit() { - var err error - var srcFile = "" - var fErr = iterateFlags(func(key string, val string) error { - switch key { - case "src": - srcFile = val - default: - return fmt.Errorf("undefined option '%s=%s'\n", key, val) - } - return nil - }) - if fErr != nil { - fmt.Printf("failed to parse flags: %s", fErr) - return - } - - if srcFile == "" { - fmt.Printf("%s edit [command options]\n", os.Args[0]) - fmt.Printf("options:\n") - fmt.Printf(" --src string source ip text file path\n") - return - } - - rExp, err := regexp.Compile("\\s+") - if err != nil { - fmt.Printf("failed to compile regexp: %s\n", err) - return - } - - fmt.Printf("init the editor from source @ `%s` ... \n", srcFile) - var tStart = time.Now() - editor, err := xdb.NewEditor(srcFile) - if err != nil { - fmt.Printf("failed to init editor: %s", err) - return - } - - fmt.Printf("all segments loaded, length: %d, elapsed: %s\n", editor.SegLen(), time.Since(tStart)) - var help = func() { - fmt.Printf("command list: \n") - fmt.Printf(" put [segment] : put the specifield $segment\n") - fmt.Printf(" put_file [file] : put all the segments from the specified $file\n") - fmt.Printf(" list [offset] [size] : list the first $size segments start from $offset\n") - fmt.Printf(" save : save all the changes to the destination source file\n") - fmt.Printf(" quit : exit the program\n") - fmt.Printf(" help : print this help menu\n") - } - - help() - var sTip = "" - var reader = bufio.NewReader(os.Stdin) - for { - if editor.NeedSave() { - sTip = "*" - } else { - sTip = "" - } - - fmt.Printf("%seditor>> ", sTip) - line, err := reader.ReadString('\n') - if err != nil { - fmt.Printf("failed to read line from cli: %s\n", err) - break - } - - cmd := strings.TrimSpace(line) - if cmd == "help" { - help() - } else if cmd == "quit" { - if editor.NeedSave() { - fmt.Printf("there are changes that need to save, type 'quit!' to force quit\n") - } else { - break - } - } else if cmd == "quit!" { - // quit directly - break - } else if cmd == "save" { - err = editor.Save() - if err != nil { - fmt.Printf("failed to save the changes: %s\n", err) - continue - } - fmt.Printf("all segments saved to %s\n", srcFile) - } else if strings.HasPrefix(cmd, "list") { - var sErr error - off, size, l := 0, 10, len("list") - str := strings.TrimSpace(cmd) - if len(str) > l { - sets := rExp.Split(cmd, 3) - switch len(sets) { - case 2: - _, sErr = fmt.Sscanf(cmd, "%s %d", &str, &off) - case 3: - _, sErr = fmt.Sscanf(cmd, "%s %d %d", &str, &off, &size) - } - } - - if sErr != nil { - fmt.Printf("failed to parse the offset and size: %s\n", sErr) - continue - } - - fmt.Printf("+-slice(%d,%d): \n", off, size) - for _, s := range editor.Slice(off, size) { - fmt.Printf("%s\n", s) - } - } else if strings.HasPrefix(cmd, "put ") { - seg := strings.TrimSpace(cmd[len("put "):]) - o, n, err := editor.Put(seg) - if err != nil { - fmt.Printf("failed to Put(%s): %s\n", seg, err) - continue - } - fmt.Printf("Put(%s): Ok, with %d deletes and %d additions\n", seg, o, n) - } else if strings.HasPrefix(cmd, "put_file ") { - file := strings.TrimSpace(cmd[len("put_file "):]) - o, n, err := editor.PutFile(file) - if err != nil { - fmt.Printf("failed to PutFile(%s): %s\n", file, err) - continue - } - fmt.Printf("PutFile(%s): Ok, with %d deletes and %d additions\n", file, o, n) - } else if len(cmd) > 0 { - help() - } - } -} +import "github.com/lionsoul2014/ip2region/maker/golang/cmd" func main() { - if len(os.Args) < 2 { - printHelp() - return - } - - // set the log flag - log.SetFlags(log.Ldate | log.Ltime | log.Lshortfile) - switch strings.ToLower(os.Args[1]) { - case "gen": - genDb() - case "search": - testSearch() - case "bench": - testBench() - case "edit": - edit() - default: - printHelp() - } + cmd.Execute() }