feat: add more videos - update mapping_table.pbtxt (#139)

* feat: add more videos
- update mapping_table.pbtxt

* chore: update database

* fix: only append unique arxivIDs

* chore: add new mapping_table.pbtxt

* chore: update database

Author: kkweon

.gitignore

@@ -4,3 +4,4 @@
 /client/test/**/*.mocks.dart
 .devcontainer/
 /.env
+*.pbtxt.bak

dbctl/internal/transform/transform.go

@@ -118,7 +118,9 @@ func ExtractPaperIDsViaProgrammableSearch(title, cx, apiKey string, limiter *rat
 		return nil, err
 	}
 
-	var paperIDs []string
+	var arxivIDs []string
+
+	seenArxivIDs := map[string]bool{}
 
 	for _, item := range do.Items {
 		arxivID, err := ExtractArxivIDFromURL(item.Link)
@@ -128,11 +130,14 @@ func ExtractPaperIDsViaProgrammableSearch(title, cx, apiKey string, limiter *rat
 		}
 
 		if arxivID != "" {
-			paperIDs = append(paperIDs, arxivID)
+			if !seenArxivIDs[arxivID] {
+				seenArxivIDs[arxivID] = true
+				arxivIDs = append(arxivIDs, arxivID)
+			}
 		}
 	}
 
-	return paperIDs, nil
+	return arxivIDs, nil
 }
 
 // ExtractArxivIDFromURL extracts ArxivID from the URL.

server/internal/data/data_test.go

@@ -2,13 +2,36 @@ package data
 
 import (
 	"regexp"
+	"sort"
 	"strconv"
 	"strings"
 	"testing"
 
+	"github.com/codingpot/pr12er/server/pkg/pr12er"
 	"github.com/stretchr/testify/assert"
+	"google.golang.org/protobuf/encoding/prototext"
 )
 
+func TestValidMappingTable(t *testing.T) {
+	bs, err := pbFiles.ReadFile("mapping_table.pbtxt")
+	assert.NoError(t, err)
+
+	// Unmarshalled correctly.
+	var table pr12er.MappingTable
+	err = prototext.Unmarshal(bs, &table)
+	assert.NoError(t, err)
+
+	// Sorted by PR ID.
+	sort.SliceIsSorted(table.GetRows(), func(i, j int) bool {
+		return table.GetRows()[i].GetPrId() < table.GetRows()[j].GetPrId()
+	})
+
+	// PR ID and Video ID must exist.
+	for _, row := range table.GetRows() {
+		assert.Greater(t, row.GetPrId(), int32(0))
+	}
+}
+
 // TestValidData validates data integrity of the database.
 func TestValidData(t *testing.T) {
 	r := regexp.MustCompile(`pr-?(\d+)`)