chore: improve storage size of symbol server data (postgres)

Author: blacktop

.cursorignore

@@ -0,0 +1,47 @@
+# ipsw files
+*.ipsw
+*.zip
+*.im4p
+*.img4
+*_Restore
+usr
+*.xz
+*.bin
+*.s
+.DS_Store
+*.dmg
+*.aea
+*.signature
+
+# Repo
+*.bu
+dist/
+test-caches/
+completions/
+manpages/
+*.txt
+*.swift
+*.prof
+*.download
+checksums.txt.sha1
+mg.*
+btm.go
+
+# Docs
+public
+site/
+www/docs/static/releases*.json
+
+# Other
+*.unstripped
+*.bndb
+*.lock
+internal/commands/ida/dscu/data/dscu.py
+venv/
+
+# Unicorn EMU
+unicorn2/
+
+.vscode/
+test-caches/
+dist/

internal/db/postgres.go

@@ -7,6 +7,7 @@ import (
 	"github.com/blacktop/ipsw/internal/model"
 	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
 )
 
 // Postgres is a database that stores data in a Postgres database.
@@ -58,14 +59,16 @@ func (p *Postgres) Connect() (err error) {
 		&model.Kernelcache{},
 		&model.DyldSharedCache{},
 		&model.Macho{},
+		&model.Path{},
 		&model.Symbol{},
+		&model.Name{},
 	)
 }
 
 // Create creates a new entry in the database.
 // It returns ErrAlreadyExists if the key already exists.
 func (p *Postgres) Create(value any) error {
-	if result := p.db.Create(value); result.Error != nil {
+	if result := p.db.FirstOrCreate(value); result.Error != nil {
 		return result.Error
 	}
 	return nil
@@ -171,15 +174,223 @@ func (p *Postgres) GetSymbols(uuid string) ([]*model.Symbol, error) {
 	return syms, nil
 }
 
-// Set sets the value for the given key.
+// Save sets the value for the given key.
 // It overwrites any previous value for that key.
 func (p *Postgres) Save(value any) error {
-	if result := p.db.Save(value); result.Error != nil {
-		return result.Error
+	if ipsw, ok := value.(*model.Ipsw); ok {
+		// Start transaction
+		return p.db.Transaction(func(tx *gorm.DB) error {
+			// Defer foreign key checks
+			// if err := tx.Exec("SET CONSTRAINTS ALL DEFERRED").Error; err != nil {
+			// 	return err
+			// }
+			// Process Paths
+			if err := p.processPaths(tx, ipsw); err != nil {
+				return err
+			}
+			// Process Names
+			if err := p.processNames(tx, ipsw); err != nil {
+				return err
+			}
+			// Save the main IPSW entry
+			if err := tx.Save(ipsw).Error; err != nil {
+				return fmt.Errorf("failed to save IPSW: %w", err)
+			}
+
+			return nil
+		})
+	}
+	return fmt.Errorf("invalid value type: %T", value)
+}
+
+func (p *Postgres) processPaths(tx *gorm.DB, ipsw *model.Ipsw) error {
+	uniquePaths := make(map[string]struct{})
+
+	// Collect unique paths
+	for _, kernel := range ipsw.Kernels {
+		for _, kext := range kernel.Kexts {
+			uniquePaths[kext.Path.Path] = struct{}{}
+		}
+	}
+	for _, dsc := range ipsw.DSCs {
+		for _, img := range dsc.Images {
+			uniquePaths[img.Path.Path] = struct{}{}
+		}
 	}
+	for _, fs := range ipsw.FileSystem {
+		uniquePaths[fs.Path.Path] = struct{}{}
+	}
+
+	// Process paths in batches
+	paths := make([]string, 0, len(uniquePaths))
+	for path := range uniquePaths {
+		paths = append(paths, path)
+	}
+
+	for i := 0; i < len(paths); i += p.BatchSize {
+		end := i + p.BatchSize
+		if end > len(paths) {
+			end = len(paths)
+		}
+		batch := paths[i:end]
+
+		// Bulk create or get Paths
+		if err := tx.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "path"}},
+			DoNothing: true,
+		}).Create(convertToPaths(batch)).Error; err != nil {
+			return fmt.Errorf("failed to create paths: %w", err)
+		}
+	}
+
+	// Fetch all created/existing Paths in batches
+	var allPaths []model.Path
+	for i := 0; i < len(paths); i += p.BatchSize {
+		end := i + p.BatchSize
+		if end > len(paths) {
+			end = len(paths)
+		}
+		batch := paths[i:end]
+
+		var batchPaths []model.Path
+		if err := tx.Where("path IN ?", batch).Find(&batchPaths).Error; err != nil {
+			return fmt.Errorf("failed to fetch paths: %w", err)
+		}
+		allPaths = append(allPaths, batchPaths...)
+	}
+
+	// Create a map for quick Path lookup
+	pathMap := make(map[string]*model.Path)
+	for i := range allPaths {
+		pathMap[allPaths[i].Path] = &allPaths[i]
+	}
+
+	// Update Path IDs
+	for _, kernel := range ipsw.Kernels {
+		for _, kext := range kernel.Kexts {
+			kext.Path.ID = pathMap[kext.GetPath()].ID
+		}
+	}
+	for _, dsc := range ipsw.DSCs {
+		for _, img := range dsc.Images {
+			img.Path.ID = pathMap[img.GetPath()].ID
+		}
+	}
+	for _, fs := range ipsw.FileSystem {
+		fs.Path.ID = pathMap[fs.GetPath()].ID
+	}
+
 	return nil
 }
 
+func (p *Postgres) processNames(tx *gorm.DB, ipsw *model.Ipsw) error {
+	uniqueNames := make(map[string]struct{})
+
+	// Collect unique names
+	for _, kernel := range ipsw.Kernels {
+		for _, kext := range kernel.Kexts {
+			for _, sym := range kext.Symbols {
+				uniqueNames[sym.GetName()] = struct{}{}
+			}
+		}
+	}
+	for _, dsc := range ipsw.DSCs {
+		for _, img := range dsc.Images {
+			for _, sym := range img.Symbols {
+				uniqueNames[sym.GetName()] = struct{}{}
+			}
+		}
+	}
+	for _, fs := range ipsw.FileSystem {
+		for _, sym := range fs.Symbols {
+			uniqueNames[sym.GetName()] = struct{}{}
+		}
+	}
+
+	// Process names in batches
+	names := make([]string, 0, len(uniqueNames))
+	for name := range uniqueNames {
+		names = append(names, name)
+	}
+
+	for i := 0; i < len(names); i += p.BatchSize {
+		end := i + p.BatchSize
+		if end > len(names) {
+			end = len(names)
+		}
+		batch := names[i:end]
+
+		// Bulk create or get Names
+		if err := tx.Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "name"}},
+			DoNothing: true,
+		}).Create(convertToNames(batch)).Error; err != nil {
+			return fmt.Errorf("failed to create names: %w", err)
+		}
+	}
+
+	// Fetch all created/existing Names in batches
+	var allNames []model.Name
+	for i := 0; i < len(names); i += p.BatchSize {
+		end := i + p.BatchSize
+		if end > len(names) {
+			end = len(names)
+		}
+		batch := names[i:end]
+
+		var batchNames []model.Name
+		if err := tx.Where("name IN ?", batch).Find(&batchNames).Error; err != nil {
+			return fmt.Errorf("failed to fetch names: %w", err)
+		}
+		allNames = append(allNames, batchNames...)
+	}
+
+	// Create a map for quick Name lookup
+	nameMap := make(map[string]*model.Name)
+	for i := range allNames {
+		nameMap[allNames[i].Name] = &allNames[i]
+	}
+
+	// Update Symbols with Name IDs
+	for _, kernel := range ipsw.Kernels {
+		for _, kext := range kernel.Kexts {
+			for _, sym := range kext.Symbols {
+				sym.Name.ID = nameMap[sym.GetName()].ID
+			}
+		}
+	}
+	for _, dsc := range ipsw.DSCs {
+		for _, img := range dsc.Images {
+			for _, sym := range img.Symbols {
+				sym.Name.ID = nameMap[sym.GetName()].ID
+			}
+		}
+	}
+	for _, fs := range ipsw.FileSystem {
+		for _, sym := range fs.Symbols {
+			sym.Name.ID = nameMap[sym.GetName()].ID
+		}
+	}
+
+	return nil
+}
+
+func convertToPaths(paths []string) []model.Path {
+	result := make([]model.Path, len(paths))
+	for i, path := range paths {
+		result[i] = model.Path{Path: path}
+	}
+	return result
+}
+
+func convertToNames(names []string) []model.Name {
+	result := make([]model.Name, len(names))
+	for i, name := range names {
+		result[i] = model.Name{Name: name}
+	}
+	return result
+}
+
 // Delete removes the given key.
 // It returns ErrNotFound if the key does not exist.
 func (p *Postgres) Delete(key string) error {

internal/model/model.go

@@ -22,7 +22,6 @@ type Ipsw struct {
 	Version    string             `json:"version,omitempty"`
 	BuildID    string             `json:"buildid,omitempty"`
 	Devices    []*Device          `gorm:"many2many:ipsw_devices;" json:"devices,omitempty"`
-	Date       time.Time          `json:"date,omitempty"`
 	Kernels    []*Kernelcache     `gorm:"many2many:ipsw_kernels;" json:"kernels,omitempty"`
 	DSCs       []*DyldSharedCache `gorm:"many2many:ipsw_dscs;" json:"dscs,omitempty"`
 	FileSystem []*Macho           `gorm:"many2many:ipsw_files;" json:"file_system,omitempty"`
@@ -34,10 +33,6 @@ type Ipsw struct {
 
 type Device struct {
 	Name string `gorm:"primaryKey" json:"name"`
-
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt gorm.DeletedAt `gorm:"index"`
 }
 
 // Kernelcache is the model for a kernelcache.
@@ -62,29 +57,45 @@ type DyldSharedCache struct {
 	Images            []*Macho `gorm:"many2many:dsc_images;" json:"images,omitempty"`
 }
 
+type Path struct {
+	ID   uint   `gorm:"primaryKey"`
+	Path string `gorm:"uniqueIndex" json:"name,omitempty"`
+}
+
 type Macho struct {
 	UUID      string `gorm:"primaryKey" json:"uuid"`
-	CreatedAt time.Time
-	UpdatedAt time.Time
-	DeletedAt gorm.DeletedAt `gorm:"index"`
-
-	Name      string    `json:"name,omitempty"`
+	PathID    uint
+	Path      Path      `gorm:"foreignKey:PathID"`
 	TextStart uint64    `gorm:"type:bigint" json:"text_start,omitempty"`
 	TextEnd   uint64    `gorm:"type:bigint" json:"text_end,omitempty"`
 	Symbols   []*Symbol `gorm:"many2many:macho_syms;"`
 }
 
+func (m Macho) GetPath() string {
+	return m.Path.Path
+}
+
+type Name struct {
+	ID   uint   `gorm:"primaryKey"`
+	Name string `gorm:"uniqueIndex" json:"name,omitempty"`
+}
+
 // swagger:model
 type Symbol struct {
 	// swagger:ignore
-	gorm.Model
-	Symbol string `json:"symbol"`
+	ID     uint `gorm:"primaryKey"`
+	NameID uint
+	Name   Name   `gorm:"foreignKey:NameID"`
 	Start  uint64 `gorm:"type:bigint" json:"start"`
 	End    uint64 `gorm:"type:bigint" json:"end"`
 }
 
+func (s Symbol) GetName() string {
+	return s.Name.Name
+}
+
 func (s Symbol) String() string {
-	return fmt.Sprintf("%#x: %s", s.Start, s.Symbol)
+	return fmt.Sprintf("%#x: %s", s.Start, s.Name.Name)
 }
 
 func (s Symbol) MarshalJSON() ([]byte, error) {
@@ -93,7 +104,7 @@ func (s Symbol) MarshalJSON() ([]byte, error) {
 		Start  uint64 `json:"start"`
 		End    uint64 `json:"end"`
 	}{
-		Symbol: s.Symbol,
+		Symbol: s.Name.Name,
 		Start:  s.Start,
 		End:    s.End,
 	})