start of work to add singularity build from Dockerfile support

This is a prototype or proof of concept that we can build from Dockerfile
The work here currently includes support for FROM/ADD/COPY/ENV/LABEL/CMD/ENTRYPOINT
and is setup to also include support for multistage builds (although this needs to be finished
and I need to look into how singularity handles this with respect to copying files between stages.
There is no expectation of merging this, it was a lot of fun to work on / figure out and I learned
a lot!

Signed-off-by: vsoch <[email protected]>

Author: vsoch

CHANGELOG.md

@@ -25,6 +25,7 @@
 - `DOCKER_USERNAME` and `DOCKER_PASSWORD` supported without `SINGULARITY_` prefix.
 - The `--no-mount` flag now accepts the value `bind-paths` to disable mounting of
   all `bind path` entries in `singularity.conf.
+- ability to build directly from `Dockerfile`
 
 ### Bug Fixes
 

docs/content.go

@@ -47,14 +47,15 @@ Enterprise Performance Computing (EPC)`
 
   BUILD SPEC:
 
-  The build spec target is a definition (def) file, local image, or URI that can 
-  be used to create a Singularity container. Several different local target 
-  formats exist:
-
-      def file  : This is a recipe for building a container (examples below)
-      directory:  A directory structure containing a (ch)root file system
-      image:      A local image on your machine (will convert to sif if
-                  it is legacy format)
+  The build spec target is a definition (def) file, local image, a Dockerfile
+  (with more limited support) or URI that can be used to create a Singularity
+  container. Several different local target formats exist:
+
+      def file   : This is a recipe for building a container (examples below)
+      Dockerfile : with support for ADD/COPY/FROM/RUN/ENTRYPOINT/CMD/LABEL/ENV
+      directory: : A directory structure containing a (ch)root file system
+      image:     : A local image on your machine (will convert to sif if
+                   it is legacy format)
 
   Targets can also be remote and defined by a URI of the following formats:
 

go.mod

@@ -65,6 +65,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cilium/ebpf v0.7.0 // indirect
 	github.com/containerd/cgroups v1.0.3 // indirect
+	github.com/containerd/typeurl v1.0.2 // indirect
 	github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect
 	github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f // indirect
 	github.com/containers/storage v1.40.0 // indirect
@@ -105,8 +106,9 @@ require (
 	github.com/mattn/go-shellwords v1.0.12 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
+	github.com/moby/buildkit v0.10.3 // indirect
 	github.com/moby/locker v1.0.1 // indirect
-	github.com/moby/sys/mount v0.2.0 // indirect
+	github.com/moby/sys/mount v0.3.0 // indirect
 	github.com/moby/sys/mountinfo v0.6.1 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect

go.sum

@@ -318,6 +318,7 @@ github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Ev
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
 github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=
 github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
+github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
 github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
 github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw=
 github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y=
@@ -456,6 +457,7 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/felixge/httpsnoop v1.0.2 h1:+nS9g82KMXccJ/wp0zyRW9ZBHFETmMGtkk+2CTTrW4o=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
@@ -826,11 +828,15 @@ github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh
 github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
+github.com/moby/buildkit v0.10.3 h1:/dGykD8FW+H4p++q5+KqKEo6gAkYKyBQHdawdjVwVAU=
+github.com/moby/buildkit v0.10.3/go.mod h1:jxeOuly98l9gWHai0Ojrbnczrk/rf+o9/JqNhY+UCSo=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM=
 github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM=
+github.com/moby/sys/mount v0.3.0 h1:bXZYMmq7DBQPwHRxH/MG+u9+XF90ZOwoXpHTOznMGp0=
+github.com/moby/sys/mount v0.3.0/go.mod h1:U2Z3ur2rXPFrFmy4q6WMwWrBOAQGYtYTRVM8BIvzbwk=
 github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
 github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=

internal/pkg/build/build.go

@@ -534,7 +534,8 @@ func MakeAllDefs(spec string) ([]types.Definition, error) {
 	}
 	defer defFile.Close()
 
-	d, err := parser.All(defFile)
+	// The Dockerfile parser needs to read the raw file
+	d, err := parser.All(defFile, spec)
 	if err != nil {
 		return nil, fmt.Errorf("while parsing definition: %s: %v", spec, err)
 	}

pkg/build/types/parser/deffile.go

@@ -132,6 +132,23 @@ func getSectionName(line string) string {
 	return lineSplit[0]
 }
 
+// parseFileLin is a shared function to get a src, dest from a single line
+func parseFileLine(line string) (string, string) {
+	var src, dst string
+	// Split at space, but not within double quotes
+	lineSubs := fileSplitter.FindAllString(line, -1)
+	if len(lineSubs) < 2 {
+		src = strings.TrimSpace(lineSubs[0])
+		dst = ""
+	} else {
+		src = strings.TrimSpace(lineSubs[0])
+		dst = strings.TrimSpace(lineSubs[1])
+	}
+	src = strings.Trim(src, "\"")
+	dst = strings.Trim(dst, "\"")
+	return src, dst
+}
+
 // parseTokenSection into appropriate components to be placed into a types.Script struct
 func parseTokenSection(tok string, sections map[string]*types.Script, files *[]types.Files, appOrder *[]string) error {
 	split := strings.SplitN(tok, "\n", 2)
@@ -156,18 +173,7 @@ func parseTokenSection(tok string, sections map[string]*types.Script, files *[]t
 			if line = strings.TrimSpace(line); line == "" || strings.Index(line, "#") == 0 {
 				continue
 			}
-			var src, dst string
-			// Split at space, but not within double quotes
-			lineSubs := fileSplitter.FindAllString(line, -1)
-			if len(lineSubs) < 2 {
-				src = strings.TrimSpace(lineSubs[0])
-				dst = ""
-			} else {
-				src = strings.TrimSpace(lineSubs[0])
-				dst = strings.TrimSpace(lineSubs[1])
-			}
-			src = strings.Trim(src, "\"")
-			dst = strings.Trim(dst, "\"")
+			src, dst := parseFileLine(line)
 			f.Files = append(f.Files, types.FileTransport{Src: src, Dst: dst})
 		}
 
@@ -448,16 +454,33 @@ func ParseDefinitionFile(r io.Reader) (d types.Definition, err error) {
 // All receives a reader from a definition file
 // and parses it into a slice of Definition structs or returns error if
 // an error is encounter while parsing
-func All(r io.Reader) ([]types.Definition, error) {
-	var stages []types.Definition
-
+func All(r io.Reader, spec string) ([]types.Definition, error) {
 	raw, err := ioutil.ReadAll(r)
 	if err != nil {
 		return nil, fmt.Errorf("while attempting to read in definition: %v", err)
 	}
 
 	// copy raw data for parsing
 	buf := raw
+
+	// Determine if Singularity recipe (or Docker)
+	dockerRgx := regexp.MustCompile(`(?mi)^FROM `)
+	dfrom := dockerRgx.FindAllIndex(buf, -1)
+
+	// If we found a FROM with space, assume Dockerfile
+	if len(dfrom) > 0 {
+		return ParseDockerfile(spec, raw)
+	}
+	return ParseSingularityDefinition(raw)
+}
+
+// ParseSingularityDefinition breaks a buffer into sections (stages) and returns them parsed
+func ParseSingularityDefinition(raw []byte) ([]types.Definition, error) {
+	var stages []types.Definition
+
+	// copy raw data for parsing
+	buf := raw
+
 	rgx := regexp.MustCompile(`(?mi)^bootstrap:`)
 	i := rgx.FindAllIndex(buf, -1)
 
@@ -474,6 +497,7 @@ func All(r io.Reader) ([]types.Definition, error) {
 	// handles case of no header
 	splitBuf = append([][]byte{buf[:]}, splitBuf...)
 
+	// Second attempt to quit if everything empty!
 	if len(splitBuf) == 0 {
 		return nil, errEmptyDefinition
 	}
@@ -483,6 +507,8 @@ func All(r io.Reader) ([]types.Definition, error) {
 			continue
 		}
 
+		// Both regular defintiion parser and Docker return
+		// equivalent sections to be further processed
 		d, err := ParseDefinitionFile(bytes.NewReader(stage))
 		if err != nil {
 			if err == errEmptyDefinition {
@@ -493,10 +519,8 @@ func All(r io.Reader) ([]types.Definition, error) {
 
 		stages = append(stages, d)
 	}
-
 	// set raw of last stage to be entire specification
 	stages[len(stages)-1].Raw = raw
-
 	return stages, nil
 }