v0.1.15 update fixUrl

Author: LubyRuffy

HISTORY.md

@@ -1,3 +1,7 @@
+## v0.1.15 update fixUrl
+
+-   support redis/socks5/mysql ```./fofa --fixUrl --size 1000 --fields host 'protocol=socks5 || protocol=redis'```
+
 ## v0.1.14 search add uniqByIP
 
 -   search add uniqByIP argument, which can be used to filter data as group by ip. ```./fofa --fixUrl --size 1000 --fields host --uniqByIP 'host="edu.cn"'```

README.md

@@ -246,8 +246,10 @@ every 500ms generate one line, never stop
             -   ☑ fields/f
             -   ☑ size/s
             - group/g 根据字段聚合：group by ip 根据ip合并，比如查询一个app会有很多域名，其中多个域名对应一个ip，这时只测试其中一个就好了
-            -   ☑ fixUrl 构建完整的url，默认的字段如果是http的话前面没有http://前缀，导致命令行一些工具不能使用，通过这个参数进行修复
-              -   ☑ 可以配合urlPrefix使用，比如不用http://而用redis://
+            -   ☑ fixUrl build valid url，默认的字段如果是http的话前面没有http://前缀，导致命令行一些工具不能使用，通过这个参数进行修复
+              -   ☑ can use with urlPrefix, such as use `app://` instead of `http://`
+              -   ☑ support socks5
+              -   ☑ support redis
             -   ☑ full 匹配所有，而不只是一年内的
             -   ☑ format
                 -   ☑ csv

account.go

@@ -102,9 +102,7 @@ func (c *Client) freeSize() int {
 		if info.RemainApiQuery > 0 {
 			return info.RemainApiData
 		}
-	default:
-		// other level, ignore free limit check
-		return -1
 	}
-	return 0
+	// other level, ignore free limit check
+	return -1
 }

account_test.go

@@ -60,6 +60,16 @@ func TestClient_AccountInfo(t *testing.T) {
 
 	var cli *Client
 	var err error
+
+	// 请求失败
+	errTs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+
+	}))
+	defer ts.Close()
+	_, err = NewClient(WithURL(errTs.URL + "[email protected]&key=wrong"))
+	assert.EqualError(t, err, "unexpected end of JSON input")
+
+	// 账号无效
 	_, err = NewClient(WithURL(ts.URL + "[email protected]&key=wrong"))
 	assert.Contains(t, err.Error(), "[-700] Account Invalid")
 
@@ -108,6 +118,18 @@ func TestClient_AccountInfo(t *testing.T) {
 	assert.Equal(t, 10, cli.Account.FCoin)
 	assert.Equal(t, 100, cli.freeSize())
 
+	// 订阅商业版
+	account = validAccounts[7]
+	cli, err = NewClient(WithURL(ts.URL + "?email=" + account.Email + "&key=" + account.Key))
+	assert.Nil(t, err)
+	assert.True(t, cli.Account.IsVIP)
+	assert.Equal(t, VipLevelSubBuss, cli.Account.VIPLevel)
+	assert.Equal(t, 0, cli.Account.FCoin)
+	assert.Equal(t, 100000, cli.freeSize())
+	// 构造异常
+	cli.Server = errTs.URL
+	assert.Equal(t, 100000, cli.freeSize())
+
 	// 红队？
 	account = validAccounts[8]
 	cli, err = NewClient(WithURL(ts.URL + "?email=" + account.Email + "&key=" + account.Key))

client_test.go

@@ -73,7 +73,7 @@ var searchCmd = &cli.Command{
 		},
 		&cli.StringFlag{
 			Name:        "urlPrefix",
-			Value:       "http://",
+			Value:       "",
 			Usage:       "prefix of url, default is http://, can be redis:// and so on ",
 			Destination: &urlPrefix,
 		},
@@ -118,6 +118,12 @@ func hasBodyField(fields []string) bool {
 // SearchAction search action
 func SearchAction(ctx *cli.Context) error {
 	// valid same config
+	for _, arg := range ctx.Args().Slice() {
+		if arg[0] == '-' {
+			return errors.New(fmt.Sprintln("there is args after fofa query:", arg))
+		}
+	}
+
 	query := ctx.Args().First()
 	if len(query) == 0 {
 		return errors.New("fofa query cannot be empty")

cmd/fofa/cmd/search.go

@@ -9,6 +9,10 @@ import (
 	"strings"
 )
 
+const (
+	NoHostWithFixURL = "host field must included when fixUrl option set"
+)
+
 // HostResults /search/all api results
 type HostResults struct {
 	Mode    string      `json:"mode"`
@@ -47,14 +51,24 @@ type SearchOptions struct {
 }
 
 // fixHostToUrl 替换host为url
-func fixHostToUrl(res [][]string, fields []string, hostIndex int, urlPrefix string) [][]string {
+func fixHostToUrl(res [][]string, fields []string, hostIndex int, urlPrefix string, protocolIndex int) [][]string {
+
 	newRes := make([][]string, 0, len(res))
 	for _, row := range res {
 		newRow := make([]string, 0, len(fields))
 		for j, r := range row {
 			if j == hostIndex {
 				if !strings.Contains(r, "://") {
-					r = urlPrefix + r
+					if urlPrefix != "" {
+						r = urlPrefix + r
+					} else if protocolIndex != -1 &&
+						(row[protocolIndex] == "socks5" || row[protocolIndex] == "redis" ||
+							row[protocolIndex] == "http" || row[protocolIndex] == "https" ||
+							row[protocolIndex] == "mongodb" || row[protocolIndex] == "mysql") {
+						r = row[protocolIndex] + "://" + r
+					} else {
+						r = "http://" + r
+					}
 				}
 			}
 			newRow = append(newRow, r)
@@ -64,6 +78,71 @@ func fixHostToUrl(res [][]string, fields []string, hostIndex int, urlPrefix stri
 	return newRes
 }
 
+// fixUrlCheck 检查参数，构建新的field和记录相关字段的偏移
+// 返回hostIndex, protocolIndex, fields, rawFieldSize, err
+func (c *Client) fixUrlCheck(fields []string, options ...SearchOptions) (int, int, []string, int, error) {
+	noSetFields := false
+	if len(fields) == 0 {
+		noSetFields = true
+		fields = []string{"host", "ip", "port"}
+	}
+	rawFieldSize := len(fields)
+
+	// 确保urlfix开启后带上了protocol字段
+	protocolIndex := -1
+	hostIndex := -1
+	if len(options) > 0 && options[0].FixUrl {
+		if noSetFields {
+			fields = []string{"host", "ip", "port", "protocol"}
+			rawFieldSize = len(fields)
+			hostIndex = 0
+			protocolIndex = 3
+		} else {
+			// 检查host字段存在
+			for index, f := range fields {
+				switch f {
+				case "host":
+					hostIndex = index
+					break
+				}
+			}
+			if hostIndex == -1 {
+				err := errors.New(NoHostWithFixURL)
+				return hostIndex, protocolIndex, fields, rawFieldSize, err
+			}
+			for index, f := range fields {
+				switch f {
+				case "protocol":
+					protocolIndex = index
+					break
+				}
+			}
+			if protocolIndex == -1 {
+				fields = append(fields, "protocol")
+				protocolIndex = len(fields) - 1
+			}
+		}
+	}
+	return hostIndex, protocolIndex, fields, rawFieldSize, nil
+}
+
+func (c *Client) postProcess(res [][]string, fields []string,
+	hostIndex int, protocolIndex int, rawFieldSize int, options ...SearchOptions) [][]string {
+	if len(options) > 0 && options[0].FixUrl {
+		res = fixHostToUrl(res, fields, hostIndex, options[0].UrlPrefix, protocolIndex)
+	}
+
+	// 返回用户指定的字段
+	if rawFieldSize != len(fields) {
+		var newRes [][]string
+		for _, r := range res {
+			newRes = append(newRes, r[0:rawFieldSize])
+		}
+		return newRes
+	}
+	return res
+}
+
 // HostSearch search fofa host data
 // query fofa query string
 // size data size: -1 means all，0 means just data total info, >0 means actual size
@@ -112,23 +191,22 @@ func (c *Client) HostSearch(query string, size int, fields []string, options ...
 		perPage = 1000
 	}
 
-	if len(fields) == 0 {
-		fields = []string{"ip", "port"}
+	hostIndex, protocolIndex, fields, rawFieldSize, err := c.fixUrlCheck(fields, options...)
+	if err != nil {
+		return nil, err
 	}
 
+	uniqIPMap := make(map[string]bool)
 	// 确认fields包含ip
 	ipIndex := -1
-	uniqIPMap := make(map[string]bool)
 	if uniqByIP {
-		ipExists := false
 		for index, f := range fields {
 			if f == "ip" {
 				ipIndex = index
-				ipExists = true
 				break
 			}
 		}
-		if !ipExists {
+		if ipIndex == -1 {
 			fields = append(fields, "ip")
 			ipIndex = len(fields) - 1
 		}
@@ -213,18 +291,7 @@ func (c *Client) HostSearch(query string, size int, fields []string, options ...
 	}
 
 	// 后处理
-	if len(options) > 0 && options[0].FixUrl {
-		urlPrefix := options[0].UrlPrefix
-		if urlPrefix == "" {
-			urlPrefix = "http://"
-		}
-		for i, f := range fields {
-			if f == "host" {
-				res = fixHostToUrl(res, fields, i, urlPrefix)
-				break
-			}
-		}
-	}
+	res = c.postProcess(res, fields, hostIndex, protocolIndex, rawFieldSize, options...)
 
 	return
 }
@@ -272,8 +339,11 @@ func (c *Client) DumpSearch(query string, allSize int, batchSize int, fields []s
 	if perPage < 1 || perPage > 100000 {
 		return errors.New("batchSize must between 1 and 100000")
 	}
-	if len(fields) == 0 {
-		fields = []string{"host", "ip", "port"}
+
+	// 确保urlfix开启后带上了protocol字段
+	hostIndex, protocolIndex, fields, rawFieldSize, err := c.fixUrlCheck(fields, options...)
+	if err != nil {
+		return err
 	}
 
 	// 分页取数据
@@ -331,18 +401,7 @@ func (c *Client) DumpSearch(query string, allSize int, batchSize int, fields []s
 		}
 
 		// 后处理
-		if len(options) > 0 && options[0].FixUrl {
-			urlPrefix := options[0].UrlPrefix
-			if urlPrefix == "" {
-				urlPrefix = "http://"
-			}
-			for i, f := range fields {
-				if f == "host" {
-					results = fixHostToUrl(results, fields, i, urlPrefix)
-					break
-				}
-			}
-		}
+		results = c.postProcess(results, fields, hostIndex, protocolIndex, rawFieldSize, options...)
 
 		if c.onResults != nil {
 			c.onResults(results)