Skip to content

Commit e8dbc76

Browse files
kkweonkkweon
committed
refactor: extract config variables
1 parent 8f9c2fd commit e8dbc76

File tree

4 files changed

+109
-85
lines changed

4 files changed

+109
-85
lines changed

‎dbctl/cmd/genmapping.go

+17-38
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"sort"
77
"time"
88

9+
"github.com/codingpot/pr12er/dbctl/internal/config"
910
"github.com/codingpot/pr12er/dbctl/internal/transform"
1011
"github.com/codingpot/pr12er/server/pkg/pr12er"
1112
googlesearch "github.com/rocketlaunchr/google-search"
@@ -18,41 +19,19 @@ import (
1819
"google.golang.org/protobuf/encoding/prototext"
1920
)
2021

21-
const (
22-
envNameYouTubePlaylistID = "YOUTUBE_PLAYLIST_ID"
23-
envNameMinPrID = "MIN_PR_ID"
24-
envNameMaxPrID = "MAX_PR_ID"
25-
envNameCx = "CX"
26-
)
27-
2822
var genmappingCmd = &cobra.Command{
2923
Use: "gen-mapping",
3024
Short: "[Experimental] Generate Mapping Table from Playlist",
3125
RunE: func(cmd *cobra.Command, args []string) error {
32-
youtubeAPIKey := viper.GetString(envNameYouTubeAPIKey)
33-
youtubePlaylistID := viper.GetString(envNameYouTubePlaylistID)
34-
mappingFilepath := viper.GetString(envNameMappingFile)
35-
36-
minPrID := viper.GetInt32(envNameMinPrID)
37-
maxPrID := viper.GetInt32(envNameMaxPrID)
38-
39-
cx := viper.GetString(envNameCx)
40-
41-
log.WithFields(log.Fields{
42-
envNameYouTubeAPIKey: youtubeAPIKey,
43-
envNameYouTubePlaylistID: youtubePlaylistID,
44-
envNameMappingFile: mappingFilepath,
45-
envNameMinPrID: minPrID,
46-
envNameMaxPrID: maxPrID,
47-
envNameCx: cx,
48-
}).Info("binding variables")
26+
cfg := config.ProvideConfigFromViper()
27+
log.Info(cfg)
4928

5029
// Define RateLimit: 1/sec
51-
if cx == "" {
30+
if cfg.Cx == "" {
5231
googlesearch.RateLimit = rate.NewLimiter(rate.Every(time.Second), 3)
5332
}
5433

55-
svc, err := youtube.NewService(context.Background(), option.WithAPIKey(youtubeAPIKey))
34+
svc, err := youtube.NewService(context.Background(), option.WithAPIKey(cfg.YouTubeAPIKey))
5635
if err != nil {
5736
return err
5837
}
@@ -62,7 +41,7 @@ var genmappingCmd = &cobra.Command{
6241
err = svc.
6342
PlaylistItems.
6443
List([]string{"contentDetails", "snippet"}).
65-
PlaylistId(youtubePlaylistID).
44+
PlaylistId(cfg.YouTubePlaylistID).
6645
Pages(context.Background(), func(resp *youtube.PlaylistItemListResponse) error {
6746
// rateLimiter is used for CSE search.
6847
rateLimiter := rate.NewLimiter(rate.Every(time.Second), 1)
@@ -76,20 +55,20 @@ var genmappingCmd = &cobra.Command{
7655
return err
7756
}
7857

79-
if !(minPrID <= prID && prID <= maxPrID) {
58+
if !(cfg.MinPrID <= prID && prID <= cfg.MaxPrID) {
8059
continue
8160
}
8261

8362
var paperIDs []string
8463

85-
if cx == "" {
64+
if cfg.Cx == "" {
8665
paperIDs, err = transform.ExtractPaperIDs(item.Snippet.Title)
8766
} else {
8867
paperIDs, err = transform.
8968
ExtractPaperIDsViaProgrammableSearch(
9069
item.Snippet.Title,
91-
cx,
92-
youtubeAPIKey,
70+
cfg.Cx,
71+
cfg.YouTubeAPIKey,
9372
rateLimiter)
9473
}
9574
if err != nil {
@@ -122,23 +101,23 @@ var genmappingCmd = &cobra.Command{
122101
return err
123102
}
124103

125-
return os.WriteFile(mappingFilepath, marshal, 0o600)
104+
return os.WriteFile(cfg.MappingFile, marshal, 0o600)
126105
},
127106
}
128107

129108
func init() {
130109
rootCmd.AddCommand(genmappingCmd)
131110

132111
genmappingCmd.Flags().String("youtube-api-key", "", "YouTube API Key (required)")
133-
_ = viper.BindPFlag(envNameYouTubeAPIKey, genmappingCmd.Flag("youtube-api-key"))
112+
_ = viper.BindPFlag(config.EnvNameYouTubeAPIKey, genmappingCmd.Flag("youtube-api-key"))
134113
genmappingCmd.Flags().String("playlist-id", "", "YouTube Playlist ID (required)")
135-
_ = viper.BindPFlag(envNameYouTubePlaylistID, genmappingCmd.Flag("playlist-id"))
114+
_ = viper.BindPFlag(config.EnvNameYouTubePlaylistID, genmappingCmd.Flag("playlist-id"))
136115
genmappingCmd.Flags().String("mapping-file", "../server/internal/data/mapping_table.pbtxt", "Filepath to mapping_table.pbtxt")
137-
_ = viper.BindPFlag(envNameMappingFile, genmappingCmd.Flag("mapping-file"))
116+
_ = viper.BindPFlag(config.EnvNameMappingFile, genmappingCmd.Flag("mapping-file"))
138117
genmappingCmd.Flags().Int32("min-pr-id", 0, "Minimum PR ID to fetch from (inclusive). It will start fetching from this ID.")
139-
_ = viper.BindPFlag(envNameMinPrID, genmappingCmd.Flag("min-pr-id"))
118+
_ = viper.BindPFlag(config.EnvNameMinPrID, genmappingCmd.Flag("min-pr-id"))
140119
genmappingCmd.Flags().Int32("max-pr-id", 999, "Maximum PR ID to fetch from (inclusive). It will start fetching until this PR.")
141-
_ = viper.BindPFlag(envNameMaxPrID, genmappingCmd.Flag("max-pr-id"))
120+
_ = viper.BindPFlag(config.EnvNameMaxPrID, genmappingCmd.Flag("max-pr-id"))
142121
genmappingCmd.Flags().String("cx", "", "Search Engine ID from https://programmablesearchengine.google.com/ If this is not set, it will use googlesearch (free)")
143-
_ = viper.BindPFlag(envNameCx, genmappingCmd.Flag("cx"))
122+
_ = viper.BindPFlag(config.EnvNameCx, genmappingCmd.Flag("cx"))
144123
}

‎dbctl/cmd/genmetadata.go

+12-28
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
paperswithcode_go "github.com/codingpot/paperswithcode-go/v2"
7+
"github.com/codingpot/pr12er/dbctl/internal/config"
78
"github.com/codingpot/pr12er/dbctl/internal/fetcher"
89
"github.com/codingpot/pr12er/dbctl/internal/io"
910
"github.com/codingpot/pr12er/server/pkg/pr12er"
@@ -14,14 +15,6 @@ import (
1415
"google.golang.org/api/youtube/v3"
1516
)
1617

17-
// required environment variables.
18-
const (
19-
envNameYouTubeAPIKey = "YOUTUBE_API_KEY"
20-
envNameMappingFile = "MAPPING_FILE"
21-
envNameDatabaseOutFile = "DATABASE_OUT_FILE"
22-
envNameWorkers = "WORKERS"
23-
)
24-
2518
// genMetaCmd represents the gen-meta command.
2619
var genMetaCmd = &cobra.Command{
2720
Use: "gen-meta",
@@ -30,20 +23,11 @@ var genMetaCmd = &cobra.Command{
3023
}
3124

3225
func generateMetadata(cmd *cobra.Command, args []string) error {
33-
apiKey := viper.GetString(envNameYouTubeAPIKey)
34-
mappingFile := viper.GetString(envNameMappingFile)
35-
databaseOutFile := viper.GetString(envNameDatabaseOutFile)
36-
workers := viper.GetInt(envNameWorkers)
37-
38-
log.WithFields(log.Fields{
39-
envNameMappingFile: mappingFile,
40-
envNameYouTubeAPIKey: apiKey,
41-
envNameDatabaseOutFile: databaseOutFile,
42-
envNameWorkers: workers,
43-
}).WithField(envNameYouTubeAPIKey, apiKey).Info("bind variables")
26+
cfg := config.ProvideConfigFromViper()
27+
log.Info(cfg)
4428

4529
// read file and unmarshal mapping file
46-
mappingTable, err := io.MappingTableFromFilepath(mappingFile)
30+
mappingTable, err := io.MappingTableFromFilepath(cfg.MappingFile)
4731
if err != nil {
4832
return err
4933
}
@@ -53,16 +37,16 @@ func generateMetadata(cmd *cobra.Command, args []string) error {
5337
PrIdToVideo: make(map[int32]*pr12er.PrVideo),
5438
}
5539

56-
client, err := fetcherClient(apiKey)
40+
client, err := fetcherClient(cfg.YouTubeAPIKey)
5741
if err != nil {
5842
return err
5943
}
6044

6145
in := make(chan *pr12er.MappingTableRow, len(mappingTable.GetRows()))
6246
out := make(chan *pr12er.PrVideo, len(mappingTable.GetRows()))
6347

64-
startWorkers(workers, client, in, out)
65-
// required for multi youtube video fetching.
48+
startWorkers(cfg.Workers, client, in, out)
49+
// required for multi YouTube video fetching.
6650
videoIDToPrMap := beginTheJobAndPrepareVideoMap(mappingTable, in)
6751

6852
close(in)
@@ -73,7 +57,7 @@ func generateMetadata(cmd *cobra.Command, args []string) error {
7357

7458
updateDatabaseWithYouTubeMetadata(client, videoIDToPrMap, database)
7559

76-
return io.DumpDatabase(database, databaseOutFile)
60+
return io.DumpDatabase(database, cfg.DatabaseOutFile)
7761
}
7862

7963
func fetcherClient(apiKey string) (*fetcher.Fetcher, error) {
@@ -146,22 +130,22 @@ func init() {
146130
"f",
147131
"../server/internal/data/mapping_table.pbtxt",
148132
"A mapping file which generate database.pbtxt from. default name is 'mapping_table.pbtxt'")
149-
_ = viper.BindPFlag(envNameMappingFile, genMetaCmd.Flag("mapping-file"))
133+
_ = viper.BindPFlag(config.EnvNameMappingFile, genMetaCmd.Flag("mapping-file"))
150134

151135
genMetaCmd.
152136
Flags().
153137
String("youtube-api-key", "", "YouTube Data API (v3) key")
154-
_ = viper.BindPFlag(envNameYouTubeAPIKey, genMetaCmd.Flag("youtube-api-key"))
138+
_ = viper.BindPFlag(config.EnvNameYouTubeAPIKey, genMetaCmd.Flag("youtube-api-key"))
155139

156140
genMetaCmd.
157141
Flags().
158142
String("database-out-file",
159143
"../server/internal/data/database.pbtxt",
160144
"Filepath to write database.pbtxt")
161-
_ = viper.BindPFlag(envNameDatabaseOutFile, genMetaCmd.Flag("database-out-file"))
145+
_ = viper.BindPFlag(config.EnvNameDatabaseOutFile, genMetaCmd.Flag("database-out-file"))
162146

163147
genMetaCmd.
164148
Flags().
165149
Int("workers", 10, "The number of workers to use for fetching")
166-
_ = viper.BindPFlag(envNameWorkers, genMetaCmd.Flag("workers"))
150+
_ = viper.BindPFlag(config.EnvNameWorkers, genMetaCmd.Flag("workers"))
167151
}

‎dbctl/cmd/youtube.go

+11-19
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"time"
66

7+
"github.com/codingpot/pr12er/dbctl/internal/config"
78
"github.com/codingpot/pr12er/dbctl/internal/transform"
89
"github.com/codingpot/pr12er/server/pkg/pr12er"
910
log "github.com/sirupsen/logrus"
@@ -13,8 +14,6 @@ import (
1314
"google.golang.org/protobuf/encoding/prototext"
1415
)
1516

16-
const envNameYouTubeLink = "YOUTUBE_LINK"
17-
1817
// youtubeCmd represents the youtube command.
1918
var youtubeCmd = &cobra.Command{
2019
Use: "youtube",
@@ -25,24 +24,17 @@ For example,
2524
go run main.go youtube --youtube-link https://www.youtube.com/watch?v=rtuJqQDWmIA >> ../server/internal/data/mapping_table.pbtxt
2625
`,
2726
RunE: func(cmd *cobra.Command, args []string) error {
28-
apiKey := viper.GetString(envNameYouTubeAPIKey)
29-
youTubeLink := viper.GetString(envNameYouTubeLink)
30-
cx := viper.GetString(envNameCx)
31-
32-
log.WithFields(log.Fields{
33-
envNameYouTubeAPIKey: apiKey,
34-
envNameYouTubeLink: youTubeLink,
35-
envNameCx: cx,
36-
}).Info("binding variables")
27+
cfg := config.ProvideConfigFromViper()
28+
log.Info(cfg)
3729

3830
rateLimiter := rate.NewLimiter(rate.Every(time.Second), 1)
3931

40-
youtubeID, err := transform.ExtractYouTubeID(youTubeLink)
32+
youtubeID, err := transform.ExtractYouTubeID(cfg.YouTubeLink)
4133
if err != nil {
4234
return err
4335
}
4436

45-
client, err := fetcherClient(apiKey)
37+
client, err := fetcherClient(cfg.YouTubeAPIKey)
4638
if err != nil {
4739
return err
4840
}
@@ -68,14 +60,14 @@ For example,
6860
var paperIDs []string
6961

7062
// Get paper IDs.
71-
if cx == "" {
63+
if cfg.Cx == "" {
7264
paperIDs, err = transform.ExtractPaperIDs(vid.GetVideoTitle())
7365
} else {
7466
paperIDs, err = transform.
7567
ExtractPaperIDsViaProgrammableSearch(
7668
vid.GetVideoTitle(),
77-
cx,
78-
apiKey,
69+
cfg.Cx,
70+
cfg.YouTubeAPIKey,
7971
rateLimiter)
8072
}
8173
if err != nil {
@@ -111,9 +103,9 @@ For example,
111103
func init() {
112104
rootCmd.AddCommand(youtubeCmd)
113105
youtubeCmd.Flags().String("youtube-api-key", "", "YouTube API Key (required)")
114-
_ = viper.BindPFlag(envNameYouTubeAPIKey, youtubeCmd.Flag("youtube-api-key"))
106+
_ = viper.BindPFlag(config.EnvNameYouTubeAPIKey, youtubeCmd.Flag("youtube-api-key"))
115107
youtubeCmd.Flags().String("youtube-link", "", "YouTube Link to fetch")
116-
_ = viper.BindPFlag(envNameYouTubeLink, youtubeCmd.Flag("youtube-link"))
108+
_ = viper.BindPFlag(config.EnvNameYouTubeLink, youtubeCmd.Flag("youtube-link"))
117109
youtubeCmd.Flags().String("cx", "", "Search Engine ID from https://programmablesearchengine.google.com/ If this is not set, it will use googlesearch (free)")
118-
_ = viper.BindPFlag(envNameCx, youtubeCmd.Flag("cx"))
110+
_ = viper.BindPFlag(config.EnvNameCx, youtubeCmd.Flag("cx"))
119111
}

‎dbctl/internal/config/config.go

+69
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package config
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/spf13/viper"
7+
)
8+
9+
const (
10+
EnvNameCx = "CX"
11+
EnvNameDatabaseOutFile = "DATABASE_OUT_FILE"
12+
EnvNameMappingFile = "MAPPING_FILE"
13+
EnvNameMaxPrID = "MAX_PR_ID"
14+
EnvNameMinPrID = "MIN_PR_ID"
15+
EnvNameWorkers = "WORKERS"
16+
EnvNameYouTubeAPIKey = "YOUTUBE_API_KEY"
17+
EnvNameYouTubeLink = "YOUTUBE_LINK"
18+
EnvNameYouTubePlaylistID = "YOUTUBE_PLAYLIST_ID"
19+
)
20+
21+
type Config struct {
22+
Cx string
23+
DatabaseOutFile string
24+
MappingFile string
25+
MaxPrID int32
26+
MinPrID int32
27+
Workers int
28+
YouTubeAPIKey string
29+
YouTubeLink string
30+
YouTubePlaylistID string
31+
}
32+
33+
func (c *Config) String() string {
34+
return fmt.Sprintf(`Config{
35+
Cx: %q
36+
DatabaseOutFile: %q
37+
MappingFile: %q
38+
MaxPrID: %d
39+
MinPrID: %d
40+
Workers: %d
41+
YouTubeAPIKey: %q
42+
YouTubeLink: %q
43+
YouTubePlaylistID: %q
44+
}`,
45+
c.Cx,
46+
c.DatabaseOutFile,
47+
c.MappingFile,
48+
c.MaxPrID,
49+
c.MinPrID,
50+
c.Workers,
51+
c.YouTubeAPIKey,
52+
c.YouTubeLink,
53+
c.YouTubePlaylistID,
54+
)
55+
}
56+
57+
func ProvideConfigFromViper() *Config {
58+
return &Config{
59+
Cx: viper.GetString(EnvNameCx),
60+
DatabaseOutFile: viper.GetString(EnvNameDatabaseOutFile),
61+
MappingFile: viper.GetString(EnvNameMappingFile),
62+
MaxPrID: viper.GetInt32(EnvNameMaxPrID),
63+
MinPrID: viper.GetInt32(EnvNameMinPrID),
64+
Workers: viper.GetInt(EnvNameWorkers),
65+
YouTubeAPIKey: viper.GetString(EnvNameYouTubeAPIKey),
66+
YouTubeLink: viper.GetString(EnvNameYouTubeLink),
67+
YouTubePlaylistID: viper.GetString(EnvNameYouTubePlaylistID),
68+
}
69+
}

0 commit comments

Comments
 (0)