v0.1.14 search add uniqByIP

Author: LubyRuffy

HISTORY.md

@@ -1,3 +1,7 @@
+## v0.1.14 search add uniqByIP
+
+-   search add uniqByIP argument, which can be used to filter data as group by ip. ```./fofa --fixUrl --size 1000 --fields host --uniqByIP 'host="edu.cn"'```
+
 ## v0.1.13 dump add inFile
 
 -   dump add inFile/json argument, which can be used to dump data from queries file. ```./fofa dump -inFile a.txt -outFile out.json -j```

README.md

@@ -20,11 +20,8 @@ The official library doesn't has unittests,  之前官方的库功能不全，
 
 ```
 ~ go install github.com/LubyRuffy/gofofa/cmd/fofa@latest
-~ $GOPATH/bin/fofa
-~ $GOPATH/bin/fofa search port=80
-WARN[0000] auth failed
-2022/08/07 14:38:29 auth failed: '[-700] Account Invalid', make sure key is valid
-~ FOFA_CLIENT_URL='https://fofa.info/[email protected]&key=xxx' $GOPATH/bin/fofa search port=80
+~ fofa
+~ FOFA_CLIENT_URL='https://fofa.info/[email protected]&key=xxx' fofa search port=80
 ```
 
 ### Search
@@ -102,6 +99,12 @@ redis://152.136.145.87:6379
 ./fofa -fields "host" -fixUrl 'app="Aspera-Faspex"' | nuclei -t http/cves/2022/CVE-2022-47986.yaml
 ```
 
+-   uniq by ip
+
+```shell
+./fofa --fixUrl --size 1000 --fields host --uniqByIP 'host="edu.cn"'
+```
+
 ### Stats
 
 -   stats subcommand

client_test.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"github.com/LubyRuffy/gofofa"
 	"github.com/LubyRuffy/gofofa/pkg/outformats"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -48,6 +49,24 @@ var randomCmd = &cli.Command{
 			Usage:       "ms",
 			Destination: &sleepMS,
 		},
+		&cli.BoolFlag{
+			Name:        "fixUrl",
+			Value:       false,
+			Usage:       "each host fix as url, like 1.1.1.1,80 will change to http://1.1.1.1",
+			Destination: &fixUrl,
+		},
+		&cli.StringFlag{
+			Name:        "urlPrefix",
+			Value:       "http://",
+			Usage:       "prefix of url, default is http://, can be redis:// and so on ",
+			Destination: &urlPrefix,
+		},
+		&cli.BoolFlag{
+			Name:        "full",
+			Value:       false,
+			Usage:       "search result for over a year",
+			Destination: &full,
+		},
 	},
 	Action: randomAction,
 }
@@ -104,7 +123,11 @@ func randomAction(ctx *cli.Context) error {
 			newQuery = newQuery + ` && before="` + ts + `"`
 		}
 
-		res, err := fofaCli.HostSearch(newQuery, 1, fields)
+		res, err := fofaCli.HostSearch(newQuery, 1, fields, gofofa.SearchOptions{
+			FixUrl:    fixUrl,
+			UrlPrefix: urlPrefix,
+			Full:      full,
+		})
 		if err != nil {
 			return err
 		}

cmd/fofa/cmd/random.go

@@ -24,6 +24,7 @@ var (
 	full        bool   // search result for over a year
 	batchSize   int    // amount of data contained in each batch, only for dump
 	json        bool   // out format as json for short
+	uniqByIP    bool   // group by ip
 )
 
 // search subcommand
@@ -82,6 +83,12 @@ var searchCmd = &cli.Command{
 			Usage:       "search result for over a year",
 			Destination: &full,
 		},
+		&cli.BoolFlag{
+			Name:        "uniqByIP",
+			Value:       false,
+			Usage:       "uniq by ip",
+			Destination: &uniqByIP,
+		},
 	},
 	Action: SearchAction,
 }
@@ -157,6 +164,7 @@ func SearchAction(ctx *cli.Context) error {
 		FixUrl:    fixUrl,
 		UrlPrefix: urlPrefix,
 		Full:      full,
+		UniqByIP:  uniqByIP,
 	})
 	if err != nil {
 		return err

cmd/fofa/cmd/search.go

@@ -9,7 +9,7 @@ import (
 )
 
 var (
-	version = "v0.1.13"
+	version = "v0.1.15"
 	commit  = "none"
 	date    = "unknown"
 	builtBy = "unknown" // goreleaser fill

cmd/fofa/main.go

@@ -43,6 +43,7 @@ type SearchOptions struct {
 	FixUrl    bool   // each host fix as url, like 1.1.1.1,80 will change to http://1.1.1.1, https://1.1.1.1:8443 will no change
 	UrlPrefix string // default is http://
 	Full      bool   // search result for over a year
+	UniqByIP  bool   // uniq by ip
 }
 
 // fixHostToUrl 替换host为url
@@ -70,8 +71,10 @@ func fixHostToUrl(res [][]string, fields []string, hostIndex int, urlPrefix stri
 // options for search
 func (c *Client) HostSearch(query string, size int, fields []string, options ...SearchOptions) (res [][]string, err error) {
 	var full bool
+	var uniqByIP bool
 	if len(options) > 0 {
 		full = options[0].Full
+		uniqByIP = options[0].UniqByIP
 	}
 
 	freeSize := c.freeSize()
@@ -113,6 +116,24 @@ func (c *Client) HostSearch(query string, size int, fields []string, options ...
 		fields = []string{"ip", "port"}
 	}
 
+	// 确认fields包含ip
+	ipIndex := -1
+	uniqIPMap := make(map[string]bool)
+	if uniqByIP {
+		ipExists := false
+		for index, f := range fields {
+			if f == "ip" {
+				ipIndex = index
+				ipExists = true
+				break
+			}
+		}
+		if !ipExists {
+			fields = append(fields, "ip")
+			ipIndex = len(fields) - 1
+		}
+	}
+
 	// 分页取数据
 	for {
 		if ctx := c.GetContext(); ctx != nil {
@@ -157,6 +178,12 @@ func (c *Client) HostSearch(query string, size int, fields []string, options ...
 					for _, vStr := range vStrSlice {
 						newSlice = append(newSlice, vStr.(string))
 					}
+					if uniqByIP {
+						if _, ok := uniqIPMap[newSlice[ipIndex]]; ok {
+							continue
+						}
+						uniqIPMap[newSlice[ipIndex]] = true
+					}
 					results = append(results, newSlice)
 				} else if vStr, ok := result.(string); ok {
 					results = append(results, []string{vStr})
@@ -178,7 +205,7 @@ func (c *Client) HostSearch(query string, size int, fields []string, options ...
 		}
 
 		// 数据已经没有了
-		if len(results) < perPage {
+		if len(hr.Results.([]interface{})) < perPage {
 			break
 		}
 

host.go

@@ -141,6 +141,19 @@ func TestClient_HostSearch(t *testing.T) {
 	})
 	assert.Nil(t, err)
 	assert.Equal(t, 10, len(res))
+
+	// 过滤UniqByIP
+	res, err = cli.HostSearch("port=80", 500, []string{"ip", "port"}, SearchOptions{
+		FixUrl: true,
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, 1000, len(res))
+	res, err = cli.HostSearch("port=80", 500, []string{"ip", "port"}, SearchOptions{
+		FixUrl:   true,
+		UniqByIP: true,
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, 986, len(res))
 }
 
 func TestClient_HostSize(t *testing.T) {