dump add inFile

Author: LubyRuffy

HISTORY.md

@@ -1,3 +1,7 @@
+## v0.1.13 dump add inFile
+
+-   dump add inFile/json argument, which can be used to dump data from queries file. ```./fofa dump -inFile a.txt -outFile out.json -j```
+
 ## v0.1.12 dump
 
 -   dump is used to perform large-scale data retrieval for the same search query, https://en.fofa.info/api/batches_pages

README.md

@@ -168,6 +168,25 @@ UpdateTime:      2022-05-30 17:00:00
 ./fofa dump --format json -fixUrl -outFile a.json -batchSize 10000 'title=phpinfo'
 ```
 
+-   dump large-scale data by queries file (line by line)
+
+```shell
+
+cat queries.txt
+port=13344
+port=23455
+
+# csv
+./fofa dump -outFile out.csv -inFile queries.txt
+
+# json
+./fofa dump -inFile queries.txt -outFile out.json -j
+2023/08/09 10:05:33 dump data of query: port=13344
+2023/08/09 10:05:35 size: 11/11, 100.00%
+2023/08/09 10:05:35 dump data of query: port=23455
+2023/08/09 10:05:37 size: 499/499, 100.00%
+```
+
 ### Utils
 
 -   random subcommand

account.go

@@ -41,8 +41,9 @@ const (
 )
 
 const (
-	VipLevelRed     VipLevel = 20 // 红队版
-	VipLevelStudent VipLevel = 22 // 教育账户
+	VipLevelRed     VipLevel = 20  // 红队版
+	VipLevelStudent VipLevel = 22  // 教育账户
+	VipLevelNever   VipLevel = 100 // 不可能的等级
 )
 
 // AccountInfo fofa account info

account_test.go

@@ -43,6 +43,15 @@ func TestAccountInfo_String(t *testing.T) {
 		RemainApiQuery: 0,
 		RemainApiData:  0,
 	}, ai)
+	assert.Equal(t, `{
+  "error": false,
+  "fcoin": 0,
+  "fofa_point": 0,
+  "isvip": true,
+  "vip_level": 3,
+  "remain_api_query": 0,
+  "remain_api_data": 0
+}`, ai.String())
 }
 
 func TestClient_AccountInfo(t *testing.T) {
@@ -98,4 +107,31 @@ func TestClient_AccountInfo(t *testing.T) {
 	assert.Equal(t, VipLevelSubPersonal, cli.Account.VIPLevel)
 	assert.Equal(t, 10, cli.Account.FCoin)
 	assert.Equal(t, 100, cli.freeSize())
+
+	// 红队？
+	account = validAccounts[8]
+	cli, err = NewClient(WithURL(ts.URL + "?email=" + account.Email + "&key=" + account.Key))
+	assert.Nil(t, err)
+	assert.True(t, cli.Account.IsVIP)
+	assert.Equal(t, VipLevelRed, cli.Account.VIPLevel)
+	assert.Equal(t, 0, cli.Account.FCoin)
+	assert.Equal(t, 10000, cli.freeSize())
+
+	// 学生
+	account = validAccounts[9]
+	cli, err = NewClient(WithURL(ts.URL + "?email=" + account.Email + "&key=" + account.Key))
+	assert.Nil(t, err)
+	assert.True(t, cli.Account.IsVIP)
+	assert.Equal(t, VipLevelStudent, cli.Account.VIPLevel)
+	assert.Equal(t, 0, cli.Account.FCoin)
+	assert.Equal(t, 10000, cli.freeSize())
+
+	// 不可能的等级
+	account = validAccounts[10]
+	cli, err = NewClient(WithURL(ts.URL + "?email=" + account.Email + "&key=" + account.Key))
+	assert.Nil(t, err)
+	assert.True(t, cli.Account.IsVIP)
+	assert.Equal(t, VipLevelNever, cli.Account.VIPLevel)
+	assert.Equal(t, 0, cli.Account.FCoin)
+	assert.Equal(t, -1, cli.freeSize())
 }

client_test.go

@@ -38,7 +38,9 @@ var (
 		{"[email protected]", "99999", true, VipLevelSubBuss, 0, 0, 0},         // 订阅商业
 
 		{"[email protected]", "10001", true, VipLevelRed, 0, 0, 0},     // 红队
-		{"[email protected]", "10001", true, VipLevelStudent, 0, 0, 0}, // 教育
+		{"[email protected]", "10002", true, VipLevelStudent, 0, 0, 0}, // 教育
+
+		{"[email protected]", "10003", true, VipLevelNever, 0, 0, 0}, // 不可能的等级
 	}
 
 	queryHander = func(w http.ResponseWriter, r *http.Request) {

cmd/fofa/cmd/dump.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"github.com/LubyRuffy/gofofa"
@@ -32,12 +33,24 @@ var dumpCmd = &cli.Command{
 			Usage:       "can be csv/json/xml",
 			Destination: &format,
 		},
+		&cli.BoolFlag{
+			Name:        "json",
+			Aliases:     []string{"j"},
+			Usage:       "output use json format",
+			Destination: &json,
+		},
 		&cli.StringFlag{
 			Name:        "outFile",
 			Aliases:     []string{"o"},
 			Usage:       "if not set, wirte to stdout",
 			Destination: &outFile,
 		},
+		&cli.StringFlag{
+			Name:        "inFile",
+			Aliases:     []string{"i"},
+			Usage:       "queries line by line",
+			Destination: &inFile,
+		},
 		&cli.IntFlag{
 			Name:        "size",
 			Aliases:     []string{"s"},
@@ -77,9 +90,35 @@ var dumpCmd = &cli.Command{
 // DumpAction search action
 func DumpAction(ctx *cli.Context) error {
 	// valid same config
+	var queries []string
 	query := ctx.Args().First()
-	if len(query) == 0 {
-		return errors.New("fofa query cannot be empty")
+	if len(query) > 0 {
+		queries = append(queries, query)
+	}
+	if len(inFile) > 0 {
+		// 打开文件
+		file, err := os.Open(inFile)
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer file.Close()
+
+		// 创建一个 Scanner 对象来逐行读取文件
+		scanner := bufio.NewScanner(file)
+
+		// 逐行读取并打印
+		for scanner.Scan() {
+			queries = append(queries, scanner.Text())
+		}
+
+		// 检查是否有读取错误
+		if err = scanner.Err(); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if len(queries) == 0 {
+		return errors.New("fofa query cannot be empty, use args or -inFile")
 	}
 	fields := strings.Split(fieldString, ",")
 	if len(fields) == 0 {
@@ -100,6 +139,9 @@ func DumpAction(ctx *cli.Context) error {
 		outTo = os.Stdout
 	}
 
+	if json {
+		format = "json"
+	}
 	// gen writer
 	var writer outformats.OutWriter
 	if hasBodyField(fields) && format == "csv" {
@@ -119,20 +161,25 @@ func DumpAction(ctx *cli.Context) error {
 	}
 
 	// do search
-	fetchedSize := 0
-	err := fofaCli.DumpSearch(query, size, batchSize, fields, func(res [][]string, allSize int) (err error) {
-		fetchedSize += len(res)
-		log.Printf("size: %d/%d, %.2f%%", fetchedSize, allSize, 100*float32(fetchedSize)/float32(allSize))
-		// output
-		err = writer.WriteAll(res)
-		return err
-	}, gofofa.SearchOptions{
-		FixUrl:    fixUrl,
-		UrlPrefix: urlPrefix,
-		Full:      full,
-	})
-	if err != nil {
-		return err
+	for _, query := range queries {
+		log.Println("dump data of query:", query)
+
+		fetchedSize := 0
+		err := fofaCli.DumpSearch(query, size, batchSize, fields, func(res [][]string, allSize int) (err error) {
+			fetchedSize += len(res)
+			log.Printf("size: %d/%d, %.2f%%", fetchedSize, allSize, 100*float32(fetchedSize)/float32(allSize))
+			// output
+			err = writer.WriteAll(res)
+			return err
+		}, gofofa.SearchOptions{
+			FixUrl:    fixUrl,
+			UrlPrefix: urlPrefix,
+			Full:      full,
+		})
+		if err != nil {
+			log.Println("fetch error:", err)
+			//return err
+		}
 	}
 
 	return nil

cmd/fofa/cmd/search.go

@@ -17,11 +17,13 @@ var (
 	size        int    // fetch size
 	format      string // out format
 	outFile     string // out file
+	inFile      string // in file
 	deductMode  string // deduct Mode
 	fixUrl      bool   // each host fix as url, like 1.1.1.1,80 will change to http://1.1.1.1
 	urlPrefix   string // each host fix as url, like 1.1.1.1,80 will change to http://1.1.1.1
 	full        bool   // search result for over a year
 	batchSize   int    // amount of data contained in each batch, only for dump
+	json        bool   // out format as json for short
 )
 
 // search subcommand

cmd/fofa/main.go

@@ -9,7 +9,7 @@ import (
 )
 
 var (
-	version = "v0.1.12"
+	version = "v0.1.13"
 	commit  = "none"
 	date    = "unknown"
 	builtBy = "unknown" // goreleaser fill