Skip to content

refactor(importer): remove dead code and consolidate archive processing #184

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
javi11 wants to merge 1 commit into from
Closed

refactor(importer): remove dead code and consolidate archive processing #184

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 1 addition & 8 deletions internal/importer/archive/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package archive

import "strconv"

// parseInt safely converts string to int
// ParseInt safely converts string to int
func ParseInt(s string) int {
num := 0
for _, r := range s {
Expand All @@ -14,8 +12,3 @@ func ParseInt(s string) int {
}
return num
}

// FormatInt converts an integer to a string
func FormatInt(n int) string {
return strconv.Itoa(n)
}
167 changes: 78 additions & 89 deletions internal/importer/processor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -416,8 +416,24 @@ func (proc *Processor) processMultiFile(
return targetBaseDir, nil
}

// processRarArchive handles RAR archive imports
func (proc *Processor) processRarArchive(
// archiveProcessFunc is the signature for archive-specific processing functions
type archiveProcessFunc func(
ctx context.Context,
nzbFolder string,
archiveFiles []parser.ParsedFile,
password string,
releaseDate int64,
nzbPath string,
archiveProgressTracker *progress.Tracker,
validationProgressTracker *progress.Tracker,
maxConnections int,
samplePercentage int,
allowedExtensions []string,
timeout time.Duration,
) error

// processArchive handles archive imports (RAR or 7zip) with a unified flow
func (proc *Processor) processArchive(
ctx context.Context,
virtualDir string,
regularFiles []parser.ParsedFile,
Expand All @@ -427,6 +443,7 @@ func (proc *Processor) processRarArchive(
maxConnections int,
allowedExtensions []string,
timeout time.Duration,
archiveFunc archiveProcessFunc,
) (string, error) {
// Create NZB folder
nzbFolder, err := filesystem.CreateNzbFolder(virtualDir, filepath.Base(parsed.Path), proc.metadataService)
Expand Down Expand Up @@ -457,18 +474,15 @@ func (proc *Processor) processRarArchive(
}
}

// Analyze and process RAR archive
// Process archive files if any
if len(archiveFiles) > 0 {
proc.updateProgress(queueID, 50)

// Create progress tracker for 50-80% range (archive analysis)
archiveProgressTracker := proc.broadcaster.CreateTracker(queueID, 50, 80)

// Get release date from first archive file
var releaseDate int64
if len(archiveFiles) > 0 {
releaseDate = archiveFiles[0].ReleaseDate.Unix()
}
releaseDate := archiveFiles[0].ReleaseDate.Unix()

// Create progress tracker for 80-95% range (validation only, metadata handled separately)
validationProgressTracker := proc.broadcaster.CreateTracker(queueID, 80, 95)
Expand All @@ -479,25 +493,21 @@ func (proc *Processor) processRarArchive(
samplePercentage = 0
}

// Process archive with unified aggregator
err := rar.ProcessArchive(
// Process archive with the provided archive-specific function
if err := archiveFunc(
ctx,
nzbFolder,
archiveFiles,
parsed.GetPassword(),
releaseDate,
parsed.Path,
proc.rarProcessor,
proc.metadataService,
proc.poolManager,
archiveProgressTracker,
validationProgressTracker,
maxConnections,
samplePercentage,
allowedExtensions,
timeout,
)
if err != nil {
); err != nil {
return "", err
}
// Archive analysis complete, validation and finalization will happen in aggregator (80-100%)
Expand All @@ -506,6 +516,40 @@ func (proc *Processor) processRarArchive(
return nzbFolder, nil
}

// processRarArchive handles RAR archive imports
func (proc *Processor) processRarArchive(
ctx context.Context,
virtualDir string,
regularFiles []parser.ParsedFile,
archiveFiles []parser.ParsedFile,
parsed *parser.ParsedNzb,
queueID int,
maxConnections int,
allowedExtensions []string,
timeout time.Duration,
) (string, error) {
return proc.processArchive(ctx, virtualDir, regularFiles, archiveFiles, parsed, queueID, maxConnections, allowedExtensions, timeout,
func(ctx context.Context, nzbFolder string, archiveFiles []parser.ParsedFile, password string, releaseDate int64, nzbPath string, archiveProgressTracker, validationProgressTracker *progress.Tracker, maxConnections, samplePercentage int, allowedExtensions []string, timeout time.Duration) error {
return rar.ProcessArchive(
ctx,
nzbFolder,
archiveFiles,
password,
releaseDate,
nzbPath,
proc.rarProcessor,
proc.metadataService,
proc.poolManager,
archiveProgressTracker,
validationProgressTracker,
maxConnections,
samplePercentage,
allowedExtensions,
timeout,
)
})
}

// processSevenZipArchive handles 7zip archive imports
func (proc *Processor) processSevenZipArchive(
ctx context.Context,
Expand All @@ -518,81 +562,26 @@ func (proc *Processor) processSevenZipArchive(
allowedExtensions []string,
timeout time.Duration,
) (string, error) {
// Create NZB folder
nzbFolder, err := filesystem.CreateNzbFolder(virtualDir, filepath.Base(parsed.Path), proc.metadataService)
if err != nil {
return "", err
}

// Process regular files first if any
if len(regularFiles) > 0 {
if err := filesystem.CreateDirectoriesForFiles(nzbFolder, regularFiles, proc.metadataService); err != nil {
return "", err
}

if err := multifile.ProcessRegularFiles(
ctx,
nzbFolder,
regularFiles,
nil, // No PAR2 files for archive imports
parsed.Path,
proc.metadataService,
proc.poolManager,
maxConnections,
proc.segmentSamplePercentage,
allowedExtensions,
proc.validationTimeout,
); err != nil { slog.DebugContext(ctx, "Failed to process regular files", "error", err)
}
}

// Analyze and process 7zip archive
if len(archiveFiles) > 0 {
proc.updateProgress(queueID, 50)

// Create progress tracker for 50-80% range (archive analysis)
archiveProgressTracker := proc.broadcaster.CreateTracker(queueID, 50, 80)

// Get release date from first archive file
var releaseDate int64
if len(archiveFiles) > 0 {
releaseDate = archiveFiles[0].ReleaseDate.Unix()
}

// Create progress tracker for 80-95% range (validation only, metadata handled separately)
validationProgressTracker := proc.broadcaster.CreateTracker(queueID, 80, 95)

// Determine sample percentage based on skipHealthCheck
samplePercentage := proc.segmentSamplePercentage
if proc.skipHealthCheck {
samplePercentage = 0
}

// Process archive with unified aggregator
err := sevenzip.ProcessArchive(
ctx,
nzbFolder,
archiveFiles,
parsed.GetPassword(),
releaseDate,
parsed.Path,
proc.sevenZipProcessor,
proc.metadataService,
proc.poolManager,
archiveProgressTracker,
validationProgressTracker,
maxConnections,
samplePercentage,
allowedExtensions,
timeout,
)
if err != nil {
return "", err
}
// Archive analysis complete, validation and finalization will happen in aggregator (80-100%)
}

return nzbFolder, nil
return proc.processArchive(ctx, virtualDir, regularFiles, archiveFiles, parsed, queueID, maxConnections, allowedExtensions, timeout,
func(ctx context.Context, nzbFolder string, archiveFiles []parser.ParsedFile, password string, releaseDate int64, nzbPath string, archiveProgressTracker, validationProgressTracker *progress.Tracker, maxConnections, samplePercentage int, allowedExtensions []string, timeout time.Duration) error {
return sevenzip.ProcessArchive(
ctx,
nzbFolder,
archiveFiles,
password,
releaseDate,
nzbPath,
proc.sevenZipProcessor,
proc.metadataService,
proc.poolManager,
archiveProgressTracker,
validationProgressTracker,
maxConnections,
samplePercentage,
allowedExtensions,
timeout,
)
})
}


Expand Down
114 changes: 0 additions & 114 deletions internal/importer/utils/file_extensions.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,10 @@
package utils

import (
"bufio"
"os"
"path/filepath"
"regexp"
"slices"
"strings"

"github.com/gabriel-vasile/mimetype"
)

// This file provides helpers translated from https://github.com/sabnzbd/sabnzbd/blob/develop/sabnzbd/utils/file_extension.py for detecting
Expand Down Expand Up @@ -75,37 +71,6 @@ var allExt = func() []string {
return out
}()

// AllExtensions returns the combined list of known extensions (dot-prefixed).
// If you need to add user-defined extensions, pass them to AllExtensionsWith.
func AllExtensions() []string { return allExt }

// AllExtensionsWith returns combined list plus user-defined dot- or non-dot-prefixed extensions.
func AllExtensionsWith(extra []string) []string {
if len(extra) == 0 {
return allExt
}
set := map[string]struct{}{}
for _, e := range allExt {
set[e] = struct{}{}
}
for _, e := range extra {
e = strings.ToLower(e)
if e == "" {
continue
}
if !strings.HasPrefix(e, ".") {
e = "." + e
}
set[e] = struct{}{}
}
out := make([]string, 0, len(set))
for k := range set {
out = append(out, k)
}
slices.Sort(out)
return out
}

// HasPopularExtension reports whether file_path has a popular extension (case-insensitive)
// or matches known RAR or 7zip patterns (e.g., .rar, .r00, .partXX.rar, .7z, .7z.001).
func HasPopularExtension(filePath string) bool {
Expand All @@ -123,82 +88,3 @@ func HasPopularExtension(filePath string) bool {
base := filepath.Base(filePath)
return rarPattern.MatchString(strings.ToLower(base)) || sevenZipPattern.MatchString(strings.ToLower(base))
}

// AllPossibleExtensions attempts to detect the file's extension(s).
// Unlike Python's puremagic (which may return multiple candidates), we
// typically have one strong match via signature-based detection.
// The returned extensions are dot-prefixed and lowercase.
func AllPossibleExtensions(filePath string) []string {
// Try MIME-based detection
if mt, err := mimetype.DetectFile(filePath); err == nil && mt != nil {
if ext := strings.ToLower(mt.Extension()); ext != "" {
return []string{ext}
}
}
return nil
}

// WhatIsMostLikelyExtension returns the most likely extension (dot-prefixed) for file_path.
// Logic mirrors the Python version:
// 1) If the start of the file is valid UTF-8 text, check for NZB clues, else return .txt
// 2) Otherwise, use signature detection and prefer a popular extension if it matches
// 3) Fallback to the first detected extension or empty string if none.
func WhatIsMostLikelyExtension(filePath string) string {
// 1) Quick text/NZB check on the first ~200 bytes
if ext, ok := sniffTextOrNZB(filePath, 200); ok {
return ext
}

// 2) signature detection
candidates := AllPossibleExtensions(filePath)
if len(candidates) == 0 {
return ""
}
// Prefer popular extension
all := allExt
for _, cand := range candidates {
if slices.Contains(all, strings.ToLower(cand)) {
return strings.ToLower(cand)
}
}
// 3) fallback to first
return strings.ToLower(candidates[0])
}

// sniffTextOrNZB reads up to n bytes and checks if it's valid UTF-8 text and
// whether it contains NZB markers. Returns (ext, true) when determined.
func sniffTextOrNZB(filePath string, n int) (string, bool) {
f, err := os.Open(filePath)
if err != nil {
return "", false
}
defer f.Close()

r := bufio.NewReader(f)
buf, _ := r.Peek(n)
// If valid UTF-8, treat as text
if !isLikelyUTF8(buf) {
return "", false
}
lower := strings.ToLower(string(buf))
if strings.Contains(lower, "!doctype nzb public") || strings.Contains(lower, "<nzb xmlns=") {
return ".nzb", true
}
return ".txt", true
}

// isLikelyUTF8 returns true if b looks like UTF-8 (simple heuristic)
func isLikelyUTF8(b []byte) bool {
// Use Go's decoder by converting to string and back
// If it contains NUL bytes or replacement characters after round-trip,
// consider it unlikely text.
s := string(b)
// If the conversion replaced invalid sequences, the resulting bytes differ
if !slices.Equal([]byte(s), b) {
return false
}
if strings.IndexByte(s, '\x00') >= 0 {
return false
}
return true
}
Loading