v0.1.11 add fixUrl/urlPrefix

Author: LubyRuffy

HISTORY.md

@@ -1,3 +1,8 @@
+## v0.1.11 add fixUrl/urlPrefix
+
+- add fixUrl/urlPrefix: ```./fofa --size 1 --fields "host" --urlPrefix "redis://" protocol=redis```
+- add accountDebug option, it doesn't print account information at console when error by default, can be opened by set ```./fofa -accountDebug account```
+
 ## v0.1.10 fix page
 
 - fix page issue

README.md

@@ -62,6 +62,23 @@ if size is larger than your account free limit, you can set `-deductMode` to dec
 ./fofa search -o a.txt 'port=6379'
 ```
 
+-   fix host to url:
+
+```shell
+./fofa --size 2 --fields "host" title=Gitblit
+47.92.145.232:8998
+https://114.55.35.145:8443
+./fofa --size 2 --fields "host" --fixUrl title=Gitblit
+http://47.92.145.232:8998
+https://114.55.35.145:8443
+./fofa --size 2 --fields "host" --fixUrl title=Gitblit
+```
+use another url prefix:
+```shell
+./fofa --size 1 --fields "host" --fixUrl --urlPrefix "redis://" protocol=redis
+redis://152.136.145.87:6379
+```
+
 -   verbose mode
 
 ```shell

client.go

@@ -35,7 +35,8 @@ type Client struct {
 	logger     *logrus.Logger
 	ctx        context.Context // use to cancel requests
 
-	onResults func(results [][]string) // when fetch results callback
+	onResults    func(results [][]string) // when fetch results callback
+	accountDebug bool                     // 调试账号明文信息
 }
 
 // Update merge config from config url
@@ -105,6 +106,14 @@ func WithOnResults(onResults func(results [][]string)) ClientOption {
 	}
 }
 
+// WithAccountDebug 是否错误里面显示账号密码原始信息
+func WithAccountDebug(v bool) ClientOption {
+	return func(c *Client) error {
+		c.accountDebug = v
+		return nil
+	}
+}
+
 // NewClient from fofa connection string to config
 // and with env config merge
 func NewClient(options ...ClientOption) (*Client, error) {

client_test.go

@@ -6,8 +6,10 @@ import (
 	"github.com/stretchr/testify/assert"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"strconv"
+	"strings"
 	"testing"
 )
 
@@ -228,4 +230,17 @@ func TestNewClient(t *testing.T) {
 	}})
 	cli, err = NewClient(WithURL(ts.URL+"/?email="+account.Email+"&key=1&version=v1"), WithLogger(logger))
 	assert.True(t, len(logs) > 0)
+
+	// 账号调试信息
+	account = validAccounts[2]
+	invalidUrl := "https://" + strings.Split(ts.URL, "://")[1] + "/?email=" + account.Email + "&key=" + account.Key + "&version=v1"
+	cli, err = NewClient(WithURL(invalidUrl), WithAccountDebug(true))
+	var u string
+	u, err = url.QueryUnescape(err.Error())
+	assert.Nil(t, err)
+	assert.Contains(t, u, account.Email)
+	cli, err = NewClient(WithURL(invalidUrl))
+	u, err = url.QueryUnescape(err.Error())
+	assert.Nil(t, err)
+	assert.NotContains(t, u, account.Email)
 }

cmd/fofa/cmd/cmd.go

@@ -70,6 +70,10 @@ var GlobalOptions = []cli.Flag{
 		Name:  "verbose",
 		Usage: "print more information",
 	},
+	&cli.BoolFlag{
+		Name:  "accountDebug",
+		Usage: "print account in error log",
+	},
 }
 
 //// isSubCmd 判断是否指定的子命令
@@ -104,13 +108,17 @@ func BeforAction(context *cli.Context) error {
 	if context.Bool("verbose") {
 		logrus.SetLevel(logrus.DebugLevel)
 	}
+	var accountDebug bool
+	if context.Bool("accountDebug") {
+		accountDebug = true
+	}
 
 	//// icon no need client
 	//if isSubCmd(os.Args[1:], "icon") {
 	//	return nil
 	//}
 
-	fofaCli, err = gofofa.NewClient(gofofa.WithURL(fofaURL))
+	fofaCli, err = gofofa.NewClient(gofofa.WithURL(fofaURL), gofofa.WithAccountDebug(accountDebug))
 	if err != nil {
 		return err
 	}

cmd/fofa/cmd/search.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"errors"
 	"fmt"
+	"github.com/LubyRuffy/gofofa"
 	"github.com/LubyRuffy/gofofa/pkg/outformats"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -17,6 +18,8 @@ var (
 	format      string // out format
 	outFile     string // out file
 	deductMode  string // deduct Mode
+	fixUrl      bool   // each host fix as url, like 1.1.1.1,80 will change to http://1.1.1.1
+	urlPrefix   string // each host fix as url, like 1.1.1.1,80 will change to http://1.1.1.1
 )
 
 // search subcommand
@@ -57,6 +60,18 @@ var searchCmd = &cli.Command{
 			Usage:       "DeductModeFree or DeductModeFCoin",
 			Destination: &deductMode,
 		},
+		&cli.BoolFlag{
+			Name:        "fixUrl",
+			Value:       false,
+			Usage:       "each host fix as url, like 1.1.1.1,80 will change to http://1.1.1.1",
+			Destination: &fixUrl,
+		},
+		&cli.StringFlag{
+			Name:        "urlPrefix",
+			Value:       "http://",
+			Usage:       "prefix of url, default is http://, can be redis:// and so on ",
+			Destination: &urlPrefix,
+		},
 	},
 	Action: SearchAction,
 }
@@ -128,7 +143,10 @@ func SearchAction(ctx *cli.Context) error {
 	}
 
 	// do search
-	res, err := fofaCli.HostSearch(query, size, fields)
+	res, err := fofaCli.HostSearch(query, size, fields, gofofa.SearchOptions{
+		FixUrl:    fixUrl,
+		UrlPrefix: urlPrefix,
+	})
 	if err != nil {
 		return err
 	}

host.go

@@ -20,7 +20,7 @@ type HostResults struct {
 	Results interface{} `json:"results"`
 }
 
-// HostStats /host api results
+// HostStatsData /host api results
 type HostStatsData struct {
 	Error       bool     `json:"error"`
 	Errmsg      string   `json:"errmsg"`
@@ -37,11 +37,35 @@ type HostStatsData struct {
 	UpdateTime  string   `json:"update_time"`
 }
 
+// SearchOptions options of search, for post processors
+type SearchOptions struct {
+	FixUrl    bool   // each host fix as url, like 1.1.1.1,80 will change to http://1.1.1.1, https://1.1.1.1:8443 will no change
+	UrlPrefix string // default is http://
+}
+
+// fixHostToUrl 替换host为url
+func fixHostToUrl(res [][]string, fields []string, hostIndex int, urlPrefix string) [][]string {
+	newRes := make([][]string, 0, len(res))
+	for _, row := range res {
+		newRow := make([]string, 0, len(fields))
+		for j, r := range row {
+			if j == hostIndex {
+				if !strings.Contains(r, "://") {
+					r = urlPrefix + r
+				}
+			}
+			newRow = append(newRow, r)
+		}
+		newRes = append(newRes, newRow)
+	}
+	return newRes
+}
+
 // HostSearch search fofa host data
 // query fofa query string
 // size data size: -1 means all，0 means just data total info, >0 means actual size
 // fields field of fofa host struct
-func (c *Client) HostSearch(query string, size int, fields []string) (res [][]string, err error) {
+func (c *Client) HostSearch(query string, size int, fields []string, options ...SearchOptions) (res [][]string, err error) {
 	// check level
 	if c.freeSize() == 0 {
 		// 不是会员
@@ -139,6 +163,20 @@ func (c *Client) HostSearch(query string, size int, fields []string) (res [][]st
 		page++ // 翻页
 	}
 
+	// 后处理
+	if len(options) > 0 && options[0].FixUrl {
+		urlPrefix := options[0].UrlPrefix
+		if urlPrefix == "" {
+			urlPrefix = "http://"
+		}
+		for i, f := range fields {
+			if f == "host" {
+				res = fixHostToUrl(res, fields, i, urlPrefix)
+				break
+			}
+		}
+	}
+
 	return
 }
 

host_test.go

@@ -87,6 +87,22 @@ func TestClient_HostSearch(t *testing.T) {
 	res, err = cli.HostSearch("aaa=bbb", 10, nil)
 	assert.Contains(t, err.Error(), "[820000] FOFA Query Syntax Incorrect")
 
+	// 带有fixurl
+	res, err = cli.HostSearch("port=80", 10, []string{"host"}, SearchOptions{
+		FixUrl:    true,
+		UrlPrefix: "",
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, 10, len(res))
+	assert.Contains(t, res[0][0], "http://")
+	res, err = cli.HostSearch("port=80", 10, []string{"host"}, SearchOptions{
+		FixUrl:    true,
+		UrlPrefix: "redis://",
+	})
+	assert.Nil(t, err)
+	assert.Equal(t, 10, len(res))
+	assert.Contains(t, res[0][0], "redis://")
+
 	// 请求失败
 	cli = &Client{
 		Server:     "http://fofa.info:66666",

request.go

@@ -52,6 +52,16 @@ func (c *Client) fetchBody(apiURI string, params map[string]string) (body []byte
 
 	resp, err = c.httpClient.Do(req)
 	if err != nil {
+		if !c.accountDebug {
+			// 替换账号明文信息
+			if e, ok := err.(*url.Error); ok {
+				newClient := c
+				newClient.Email = "<email>"
+				newClient.Key = "<key>"
+				e.URL = newClient.buildURL(apiURI, params)
+				err = e
+			}
+		}
 		return
 	}
 	defer resp.Body.Close()

scripts/README.md

@@ -0,0 +1,13 @@
+# 发布流程
+
+## 修改代码
+
+## 运行单元测试，确保100%的覆盖度
+
+## 修改 HISTORY.md 版本说明文件，确保功能每一项都有记录
+
+## 修改 README.md 更新功能说明，确保每一项都有运行的示例
+
+## 提交代码
+
+## 运行发布脚本 ./tag_release.sh