feat: WIP firmware key scrapin code

Author: blacktop

Makefile

@@ -56,6 +56,10 @@ update_mod:
 update_devs:
 	CGO_ENABLED=1 CGO_CFLAGS=-I/usr/local/include CGO_LDFLAGS=-L/usr/local/lib CC=gcc go run ./cmd/ipsw/main.go device-list-gen pkg/info/data/device_traits.json
 
+.PHONY: update_keys
+update_keys:
+	CGO_ENABLED=0 go run ./cmd/ipsw/main.go key-list-gen pkg/info/data/firmware_keys.json
+
 .PHONY: dry_release
 dry_release:
 	goreleaser --skip-publish --rm-dist --skip-validate

cmd/ipsw/cmd/key_list_gen.go

@@ -0,0 +1,58 @@
+/*
+Copyright © 2020 blacktop
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package cmd
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/blacktop/ipsw/internal/download"
+	"github.com/spf13/cobra"
+)
+
+// keyListGenCmd represents the key-list-gen command
+var keyListGenCmd = &cobra.Command{
+	Use:    "key-list-gen",
+	Short:  "Generate iOS firmware key database",
+	Args:   cobra.MinimumNArgs(1),
+	Hidden: true,
+	RunE: func(cmd *cobra.Command, args []string) error {
+
+		keys, err := download.ScrapeKeys("")
+		if err != nil {
+			return err
+		}
+
+		keysJSON, err := json.Marshal(keys)
+		if err != nil {
+			return err
+		}
+		return ioutil.WriteFile(filepath.Clean(args[0]), keysJSON, 0644)
+
+		return nil
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(keyListGenCmd)
+}

cmd/ipsw/cmd/sep.go

@@ -40,6 +40,7 @@ import (
 // NOTE: https://www.blackhat.com/docs/us-16/materials/us-16-Mandt-Demystifying-The-Secure-Enclave-Processor.pdf
 // NOTE: http://mista.nu/research/sep-paper.pdf
 // NOTE: https://gist.github.com/xerub/0161aacd7258d31c6a27584f90fa2e8c
+// NOTE: https://github.com/matteyeux/sepsplit/blob/master/sepsplit.c
 
 const legionStr = "Built by legion2"
 const appListOffsetFromSEPOS32bit = 0xec8

hack/extras/NOTES.md

@@ -42,3 +42,8 @@ Ideas
 - https://blog.gopheracademy.com/advent-2018/llvm-ir-and-go/
 - https://blog.felixangell.com/an-introduction-to-llvm-in-go
 - https://ldhldh.myds.me:10081/docs/llvm342_docs/_mach_o_dump_8cpp.html
+
+### DWARF
+
+- https://github.com/volatilityfoundation/dwarf2json/blob/master/main.go
+- https://github.com/dutchcoders/disassembler/blob/master/disassembler.go

internal/download/scrape.go

@@ -4,6 +4,7 @@ import (
 	"regexp"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/gocolly/colly/v2"
 	"github.com/pkg/errors"
@@ -24,6 +25,10 @@ type BetaIPSW struct {
 	BuildID string   `json:"buildid,omitempty"`
 }
 
+type Keys map[string]string
+type BuildKeys map[string]Keys
+type DeviceKeys map[string]BuildKeys
+
 func trimQuotes(s string) string {
 	if len(s) > 0 && s[0] == '"' {
 		s = s[1:]
@@ -126,3 +131,69 @@ func ScrapeURLs(build string) (map[string]BetaIPSW, error) {
 
 	return ipsws, nil
 }
+
+// ScrapeKeys will scrape the iPhone Wiki for firmware keys
+func ScrapeKeys(version string) (map[string]map[string]map[string]string, error) {
+	keys := make(map[string]map[string]map[string]string, 1000)
+
+	c := colly.NewCollector(
+		colly.AllowedDomains("www.theiphonewiki.com"),
+		colly.URLFilters(
+			regexp.MustCompile("https://www.theiphonewiki.com/wiki/(.+)$"),
+		),
+		// colly.Async(true),
+		colly.MaxDepth(1),
+		colly.UserAgent("free0"),
+		colly.IgnoreRobotsTxt(),
+	)
+
+	// On every a element which has href attribute call callback
+	c.OnHTML("a[href]", func(e *colly.HTMLElement) {
+		if strings.Contains(e.Attr("href"), "/wiki/") && !strings.Contains(e.Attr("href"), "redlink=1") {
+			c.Visit(e.Request.AbsoluteURL(e.Attr("href")))
+		}
+	})
+
+	c.OnHTML("body", func(e *colly.HTMLElement) {
+		e.ForEach("code", func(_ int, code *colly.HTMLElement) {
+			if len(code.Attr("id")) > 0 {
+				if strings.Contains(code.Attr("id"), "-iv") || strings.Contains(code.Attr("id"), "-key") {
+					if code.Text != "Unknown" {
+						urlParts := strings.Split(code.Request.URL.Path, "_")
+						buildID := urlParts[1]
+						deviceID := strings.Trim(urlParts[2], "()")
+						if keys[deviceID] == nil {
+							keys[deviceID] = map[string]map[string]string{}
+						}
+						if keys[deviceID][buildID] == nil {
+							keys[deviceID][buildID] = map[string]string{}
+						}
+						keys[deviceID][buildID][strings.TrimPrefix(code.Attr("id"), "keypage-")] = code.Text
+						// fmt.Printf("%#v\n", keys[deviceID])
+					}
+
+				}
+			}
+		})
+	})
+
+	// Set error handler
+	// c.OnError(func(r *colly.Response, err error) {
+	// 	// fmt.Println("Request URL:", r.Request.URL, "failed with response:", r, "\nError:", err)
+	// 	fmt.Println("Error:", err)
+	// })
+
+	c.SetRequestTimeout(60 * time.Second)
+
+	// for _, v := range []string{"1.x", "2.x", "3.x", "4.x", "5.x", "6.x", "7.x", "8.x", "9.x", "10.x", "11.x", "12.x", "13.x", "14.x"} {
+	for _, v := range []string{"13.x", "14.x"} {
+		err := c.Visit("https://www.theiphonewiki.com/wiki/Firmware_Keys/" + v)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to scrape https://www.theiphonewiki.com/wiki/Firmware_Keys/"+v)
+		}
+	}
+
+	c.Wait()
+
+	return keys, nil
+}

internal/statik/statik.go

@@ -75,6 +75,31 @@ func getProcessor(cpuid string) processors {
 	return processors{}
 }
 
+func getFirmwareKeys(device, build string) map[string]string {
+	var keys map[string]map[string]map[string]string
+
+	statikFS, err := fs.New()
+	if err != nil {
+		log.Fatal(err)
+	}
+	keysJSON, err := statikFS.Open("/firmware_keys.json")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	data, err := ioutil.ReadAll(keysJSON)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = json.Unmarshal(data, &keys)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return keys[device][build]
+}
+
 func (i *Info) String() string {
 	var iStr string
 	iStr += fmt.Sprintf(