parse.go: Print process progress.

Author: yifan-gu

README.md

@@ -131,9 +131,11 @@ Remember to run `M-x org-roam-db-sync` to sync the org-roam database.
 - [ ] One-click export from Kindle app.
 - [ ] Change parser/exporter type from string to safe type.
 - [x] `My Clippings.txt` parser.
-- [ ] Diff the previous processed `My Clippings.txt`
+- [ ] Diff the previous processed `My Clippings.txt` so don't parse the whole file again.
 - [ ] Support multiple authors.
-- [ ] Add progress indicator
+- [x] Add progress indicator.
+- [ ] Difference between createdAt and when notes are added to the database.
+- [ ] Figure out the difference in length between json and loaded mongodb json.
 
 ### Server Backend
 - [x] Database storage.

pkg/parser/kindlemyclippings/parser.go

@@ -29,39 +29,71 @@ func (p *KindleMyClippingsParser) LoadConfigs(cmd *cobra.Command) {
 	cmd.Flags().Float64Var(&p.minSimilarity, "min-similarity", 0.8, "Minimum similarity percentage (0-1) to consider a highlight as duplicate")
 }
 
-// Parse processes the MyClippings.txt file and returns a list of Books with deduplicated Marks.
 func (p *KindleMyClippingsParser) Parse(inputPath string) ([]*model.Book, error) {
+	// Open the file
 	file, err := os.Open(inputPath)
 	defer file.Close()
 	if err != nil {
-		return nil, errors.Wrap(err, "")
+		return nil, errors.Wrap(err, "failed to open input file")
+	}
+
+	// Count total lines in the file to calculate parsing progress
+	totalLines, err := countLines(file)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to count lines in input file")
+	}
+	// Reset the file cursor to the beginning after counting lines
+	if _, err := file.Seek(0, 0); err != nil {
+		return nil, errors.Wrap(err, "failed to reset file cursor")
 	}
 
 	var books []*model.Book
 	markListMap := make(map[string][]*model.Mark) // Maps book title -> list of marks
 
 	scanner := bufio.NewScanner(file)
+	lineCount := 0  // Total number of lines processed
+	entryCount := 0 // Total number of marks (entries)
+
+	// Function to print parsing progress
+	printParsingProgress := func() {
+		if lineCount%500 == 0 || lineCount == totalLines {
+			progress := (float64(lineCount) / float64(totalLines)) * 100
+			fmt.Fprintf(os.Stderr, "\rParsing Progress: %.2f%% (%d/%d lines processed, %d entries parsed)", progress, lineCount, totalLines, entryCount)
+		}
+	}
+
+	// Parsing phase
 	for scanner.Scan() {
+		lineCount++
+		printParsingProgress()
+
 		// Extract book details
 		title, author, err := extractTitleAndAuthor(stripLeadingBOM(scanner.Text()))
 		if err != nil {
-			return nil, errors.Wrap(err, "")
+			return nil, errors.Wrap(err, fmt.Sprintf("error parsing title and author at line %d", lineCount))
 		}
 
 		// Parse metadata
 		if !scanner.Scan() {
-			return nil, fmt.Errorf("unexpected EOF, expecting metadata")
+			return nil, fmt.Errorf("unexpected EOF at line %d, expecting metadata", lineCount)
 		}
+		lineCount++
+		printParsingProgress()
 		meta := scanner.Text()
 		markType, location, createdAt, err := extractMeta(meta)
 		if err != nil {
-			return nil, errors.Wrap(err, "")
+			return nil, errors.Wrap(err, fmt.Sprintf("error parsing metadata at line %d", lineCount))
 		}
 
 		scanner.Scan() // Skip the empty line before the actual text.
+		lineCount++
+		printParsingProgress()
 
 		var text []string
 		for scanner.Scan() {
+			lineCount++
+			printParsingProgress()
+
 			line := scanner.Text()
 			if line == "==========" {
 				break
@@ -71,7 +103,7 @@ func (p *KindleMyClippingsParser) Parse(inputPath string) ([]*model.Book, error)
 
 		// Parse data or note
 		if text == nil { // Empty notes, skip
-			return nil, nil
+			continue
 		}
 
 		// Create the mark
@@ -99,42 +131,42 @@ func (p *KindleMyClippingsParser) Parse(inputPath string) ([]*model.Book, error)
 			markListMap[title] = []*model.Mark{}
 		}
 		markListMap[title] = append(markListMap[title], mark)
+
+		entryCount++
 	}
 
-	// Deduplicate marks for each book
+	// Deduplication phase
+	totalBooks := len(markListMap)
+	processedBooks := 0
+
 	for title, marks := range markListMap {
-		deduplicatedMarks := deduplicateMarks(marks, p.minSimilarity) // Use the configured threshold
+		processedBooks++
+		fmt.Fprintf(os.Stderr, "\nStarting deduplication for book: %s (%d marks)\n", title, len(marks))
+
+		deduplicatedMarks := deduplicateMarksWithProgress(marks, p.minSimilarity) // Deduplication with progress inside
 		book := &model.Book{
 			Title:  title,
 			Author: marks[0].Author,
 			Marks:  deduplicatedMarks,
 		}
 		books = append(books, book)
-	}
 
-	if err := scanner.Err(); err != nil {
-		return nil, errors.Wrap(err, "")
+		// Print progress across books
+		bookProgress := (float64(processedBooks) / float64(totalBooks)) * 100
+		fmt.Fprintf(os.Stderr, "\rTotal Deduplication Progress: %.2f%% (%d/%d books processed)", bookProgress, processedBooks, totalBooks)
 	}
 
-	model.SortBooksByTitle(books)
-
+	// Final summary
+	fmt.Fprintln(os.Stderr) // Add a newline after the progress output
+	fmt.Fprintf(os.Stderr, "Finished processing: %d lines parsed, %d books deduplicated, %d marks processed\n", lineCount, totalBooks, entryCount)
 	return books, nil
 }
 
-func stripLeadingBOM(s string) string {
-	runes := []rune(s)
-	start := 0
-	for start < len(runes) && runes[start] == '\uFEFF' {
-		start++
-	}
-	return string(runes[start:])
-}
-
-// Deduplicate marks by comparing similarity of content using a minimum common words threshold
-func deduplicateMarks(marks []*model.Mark, minSimilarity float64) []*model.Mark {
+func deduplicateMarksWithProgress(marks []*model.Mark, minSimilarity float64) []*model.Mark {
 	var deduplicated []*model.Mark
+	totalMarks := len(marks)
 
-	for _, mark := range marks {
+	for index, mark := range marks {
 		duplicateFound := false
 		for i, dedupMark := range deduplicated {
 			if hasMinCommonWords(dedupMark.Data, mark.Data, minSimilarity) {
@@ -149,11 +181,40 @@ func deduplicateMarks(marks []*model.Mark, minSimilarity float64) []*model.Mark
 		if !duplicateFound {
 			deduplicated = append(deduplicated, mark)
 		}
+
+		// Print deduplication progress for current book every 100 marks
+		if (index+1)%100 == 0 || index+1 == totalMarks {
+			progress := (float64(index+1) / float64(totalMarks)) * 100
+			fmt.Fprintf(os.Stderr, "\rDeduplication Progress for current book: %.2f%% (%d/%d marks processed)", progress, index+1, totalMarks)
+		}
 	}
 
+	// Print a new line after finishing the current book
+	fmt.Fprintln(os.Stderr)
 	return deduplicated
 }
 
+func countLines(file *os.File) (int, error) {
+	scanner := bufio.NewScanner(file)
+	lineCount := 0
+	for scanner.Scan() {
+		lineCount++
+	}
+	if err := scanner.Err(); err != nil {
+		return 0, err
+	}
+	return lineCount, nil
+}
+
+func stripLeadingBOM(s string) string {
+	runes := []rune(s)
+	start := 0
+	for start < len(runes) && runes[start] == '\uFEFF' {
+		start++
+	}
+	return string(runes[start:])
+}
+
 // Check if two strings share a longest common substring whose percentage of similarity meets the threshold
 func hasMinCommonWords(a, b string, minSimilarity float64) bool {
 	// Preprocess both strings to handle Chinese text

tests/my_clippings_output.json

@@ -55,6 +55,42 @@
       }
     ]
   },
+  {
+    "title": "红楼梦（人文社权威定本彩皮版；国务院文化组批准，红研所校注；豆瓣读书TOP250首位，评论上万条；出版四十年，三次修订）",
+    "author": "曹雪芹",
+    "marks": [
+      {
+        "type": "HIGHLIGHT",
+        "title": "红楼梦（人文社权威定本彩皮版；国务院文化组批准，红研所校注；豆瓣读书TOP250首位，评论上万条；出版四十年，三次修订）",
+        "author": "曹雪芹",
+        "location": {
+          "location": 15868
+        },
+        "data": "都是尤二姐素习所穿的，不禁又伤心哭了起来。自己用个包袱一齐包了，也不命小厮丫鬟来拿，便自己提着来烧。",
+        "createdAt": 1642265828
+      },
+      {
+        "type": "HIGHLIGHT",
+        "title": "红楼梦（人文社权威定本彩皮版；国务院文化组批准，红研所校注；豆瓣读书TOP250首位，评论上万条；出版四十年，三次修订）",
+        "author": "曹雪芹",
+        "location": {
+          "location": 15869
+        },
+        "data": "平儿又是伤心，又是好笑，忙将二百两一包的碎银子偷了出来，",
+        "createdAt": 1642265863
+      },
+      {
+        "type": "HIGHLIGHT",
+        "title": "红楼梦（人文社权威定本彩皮版；国务院文化组批准，红研所校注；豆瓣读书TOP250首位，评论上万条；出版四十年，三次修订）",
+        "author": "曹雪芹",
+        "location": {
+          "location": 15871
+        },
+        "data": "又将一条裙子递与平儿，说：“这是他家常穿的，你好生替我收着，作个念心儿。”",
+        "createdAt": 1642265884
+      }
+    ]
+  },
   {
     "title": "小行星掉在下午（首届宝珀理想国文学奖决选作者沈大成全新小说集，给沈大成6分钟，她给你一场颅内反乌托邦式冒险 理想国出品）",
     "author": "沈大成",
@@ -93,41 +129,5 @@
         "createdAt": 1642641563
       }
     ]
-  },
-  {
-    "title": "红楼梦（人文社权威定本彩皮版；国务院文化组批准，红研所校注；豆瓣读书TOP250首位，评论上万条；出版四十年，三次修订）",
-    "author": "曹雪芹",
-    "marks": [
-      {
-        "type": "HIGHLIGHT",
-        "title": "红楼梦（人文社权威定本彩皮版；国务院文化组批准，红研所校注；豆瓣读书TOP250首位，评论上万条；出版四十年，三次修订）",
-        "author": "曹雪芹",
-        "location": {
-          "location": 15868
-        },
-        "data": "都是尤二姐素习所穿的，不禁又伤心哭了起来。自己用个包袱一齐包了，也不命小厮丫鬟来拿，便自己提着来烧。",
-        "createdAt": 1642265828
-      },
-      {
-        "type": "HIGHLIGHT",
-        "title": "红楼梦（人文社权威定本彩皮版；国务院文化组批准，红研所校注；豆瓣读书TOP250首位，评论上万条；出版四十年，三次修订）",
-        "author": "曹雪芹",
-        "location": {
-          "location": 15869
-        },
-        "data": "平儿又是伤心，又是好笑，忙将二百两一包的碎银子偷了出来，",
-        "createdAt": 1642265863
-      },
-      {
-        "type": "HIGHLIGHT",
-        "title": "红楼梦（人文社权威定本彩皮版；国务院文化组批准，红研所校注；豆瓣读书TOP250首位，评论上万条；出版四十年，三次修订）",
-        "author": "曹雪芹",
-        "location": {
-          "location": 15871
-        },
-        "data": "又将一条裙子递与平儿，说：“这是他家常穿的，你好生替我收着，作个念心儿。”",
-        "createdAt": 1642265884
-      }
-    ]
   }
 ]