diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ad60b434e9..224a07889a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -165,20 +165,6 @@ jobs: distro: [fedora-current, fedora-prior, debian-sid] storage: [vfs, overlay] priv: [root, rootless] - exclude: - - storage: vfs - priv: rootless - - distro: debian-sid - priv: rootless - # Skip rootless+overlay: upstream Cirrus's PASSTHROUGH_ENV_RE drops - # STORAGE_DRIVER through SSH re-exec to rootlessuser, so their - # "Integration rootless ... w/ overlay" task silently tests vfs. - # When we propagate STORAGE_DRIVER properly, we expose a real - # rootless+overlay cleanup bug in buildah's storage code - # ("replacing mount point .../merged: directory not empty"). - # Skip these cells until that's fixed upstream. - - storage: overlay - priv: rootless include: - storage: overlay priv: root diff --git a/Makefile b/Makefile index c024f0e5b3d..413ade4e008 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ export GOLANGCI_LINT_VERSION := 2.12.2 # use source debugging tools like delve. all: binaries docs -binaries: bin/buildah bin/imgtype bin/copy bin/inet bin/tutorial bin/dumpspec bin/passwd bin/crash bin/wait bin/grpcnoop +binaries: bin/buildah bin/imgtype bin/copy bin/inet bin/tutorial bin/dumpspec bin/passwd bin/crash bin/wait bin/grpcnoop bin/pipeloop bin/buildah: $(SOURCES) internal/mkcw/embed/entrypoint_amd64.gz $(GO_BUILD) $(BUILDAH_LDFLAGS) $(GO_GCFLAGS) "$(GOGCFLAGS)" -o $@ $(BUILDFLAGS) ./cmd/buildah @@ -134,6 +134,9 @@ bin/passwd: tests/passwd/passwd.go bin/grpcnoop: tests/rpc/noop/noop.go $(GO_BUILD) $(BUILDAH_LDFLAGS) -o $@ $(BUILDFLAGS) ./tests/rpc/noop/noop.go +bin/pipeloop: tests/pipeloop/pipeloop.go + $(GO_BUILD) $(BUILDAH_LDFLAGS) -o $@ $(BUILDFLAGS) ./tests/pipeloop/pipeloop.go + .PHONY: clean clean: $(RM) -r bin tests/testreport/testreport tests/conformance/testdata/mount-targets/true internal/mkcw/embed/entrypoint_arm64 internal/mkcw/embed/entrypoint_ppc64le internal/mkcw/embed/entrypoint_s390x internal/mkcw/embed/entrypoint_arm64.gz internal/mkcw/embed/entrypoint_ppc64le.gz internal/mkcw/embed/entrypoint_s390x.gz internal/mkcw/embed/asm/*.o diff --git a/cmd/buildah/addcopy.go b/cmd/buildah/addcopy.go index 5c40a58db6b..782c8ffcd11 100644 --- a/cmd/buildah/addcopy.go +++ b/cmd/buildah/addcopy.go @@ -215,8 +215,17 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe }() } if err != nil { - return fmt.Errorf("reading build container %q: %w", iopts.from, err) + return fmt.Errorf("reading --from container %q: %w", iopts.from, err) } + + fromcdir, err := store.ContainerDirectory(from.ContainerID) + if err != nil { + return fmt.Errorf("finding container state directory for %q: %w", iopts.from, err) + } + if err := flagServingSFTP(fromcdir); err != nil { + return fmt.Errorf(`unable to use "buildah add" or "buildah copy" while container appears to be mounted: %v`, err) + } + fromMountPoint, err := from.Mount(from.MountLabel) if err != nil { return fmt.Errorf("mounting %q container %q: %w", iopts.from, from.Container, err) @@ -246,6 +255,14 @@ func addAndCopyCmd(c *cobra.Command, args []string, verb string, iopts addCopyRe return fmt.Errorf("reading build container %q: %w", name, err) } + cdir, err := store.ContainerDirectory(builder.ContainerID) + if err != nil { + return fmt.Errorf("finding container state directory for %q: %w", name, err) + } + if err := flagServingSFTP(cdir); err != nil { + return fmt.Errorf(`unable to use "buildah add" or "buildah copy" while container appears to be busy: %v`, err) + } + builder.ContentDigester.Restart() var timestamp *time.Time diff --git a/cmd/buildah/main.go b/cmd/buildah/main.go index 11a2d8136f6..bd0780fdd74 100644 --- a/cmd/buildah/main.go +++ b/cmd/buildah/main.go @@ -14,6 +14,7 @@ import ( rspecs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/pflag" "go.podman.io/buildah" "go.podman.io/buildah/define" "go.podman.io/buildah/pkg/cli" @@ -140,6 +141,16 @@ func mainInit() { rootCmd.AddGroup(commandGroups...) } +func getInheritedFlags(c *cobra.Command) []string { + var flags []string + c.InheritedFlags().VisitAll(func(f *pflag.Flag) { + if f.Changed { + flags = append(flags, fmt.Sprintf("--%s=%s", f.Name, f.Value.String())) + } + }) + return flags +} + func initConfig() { // TODO Cobra allows us to do extra stuff here at init // time if we ever want to take advantage. @@ -162,7 +173,7 @@ func before(cmd *cobra.Command) error { } switch cmd.Use { - case "", "help", "version", "mount": + case "", "help", "version", "mount", mountSshfs, umountSshfs: return nil } debugCapabilities() @@ -272,6 +283,7 @@ func main() { rmInit() rpcInit() runInit() + sftpInit() sourceInit() tagInit() umountInit() diff --git a/cmd/buildah/mount.go b/cmd/buildah/mount.go index c6ba41a81a5..7b8307f1dd1 100644 --- a/cmd/buildah/mount.go +++ b/cmd/buildah/mount.go @@ -89,6 +89,24 @@ func mountCmd(c *cobra.Command, args []string, opts mountOptions) error { lastError = fmt.Errorf("reading build container %q: %w", name, err) continue } + container, err := store.Container(builder.ContainerID) + if err == nil { + dd, err := store.ContainerDirectory(container.ID) + if err != nil { + if lastError != nil { + fmt.Fprintln(os.Stderr, lastError) + } + lastError = fmt.Errorf("determining data directory for %q container %q: %w", name, builder.Container, err) + continue + } + if err := flagServingSFTP(dd); err != nil { + if lastError != nil { + fmt.Fprintln(os.Stderr, lastError) + } + lastError = fmt.Errorf("mounting %q container %q: %w", name, builder.Container, err) + continue + } + } mountPoint, err := builder.Mount(builder.MountLabel) if err != nil { if lastError != nil { diff --git a/cmd/buildah/run.go b/cmd/buildah/run.go index ae5ecbc038d..cb488cbdc9a 100644 --- a/cmd/buildah/run.go +++ b/cmd/buildah/run.go @@ -133,6 +133,14 @@ func runCmd(c *cobra.Command, args []string, iopts runInputOptions) error { return fmt.Errorf("reading build container %q: %w", name, err) } + cdir, err := store.ContainerDirectory(builder.ContainerID) + if err != nil { + return fmt.Errorf("finding build container state directory for %q: %w", name, err) + } + if err := flagServingSFTP(cdir); err != nil { + return fmt.Errorf(`unable to use "buildah run" while container appears to be busy: %v`, err) + } + isolation, err := parse.IsolationOption(c.Flag("isolation").Value.String()) if err != nil { return err diff --git a/cmd/buildah/sftp.go b/cmd/buildah/sftp.go new file mode 100644 index 00000000000..8c19f81caa9 --- /dev/null +++ b/cmd/buildah/sftp.go @@ -0,0 +1,914 @@ +//go:build linux + +package main + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "os/exec" + "os/signal" + "path/filepath" + "slices" + "strings" + "sync" + "syscall" + "time" + + "github.com/pkg/sftp" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "go.podman.io/buildah" + buildahcli "go.podman.io/buildah/pkg/cli" + "go.podman.io/buildah/pkg/parse" + "go.podman.io/buildah/util" + "go.podman.io/storage" + "go.podman.io/storage/pkg/fileutils" + "go.podman.io/storage/pkg/reexec" + "golang.org/x/sys/unix" +) + +const ( + serveSftpInSubprocess = "buildah-serve-sftp-chroot" + serveSftp = "serve-sftp" + servingSftpFragment = "serving-sftp" + serveSftpCheckInterval = 100 * time.Millisecond + serveSftpCheckIterations = 100 + findDataDir = "find-data-dir" + mountSshfsMountFragment = "buildah-sshfs-mount" + mountSshfs = "mount-sshfs" + umountSshfs = "umount-sshfs" + listSshfsMountDirs = "list-sshfs-mount-dirs" + mountUmountCheckInterval = 100 * time.Millisecond + mountUmountCheckIterations = 300 +) + +var ( + serveSftpDescription = "\n Serves a container or image's rootfs using SFTP over stdio." + serveSftpCommand = &cobra.Command{ + Use: serveSftp, + Short: "Serve a container or image's rootfs using SFTP over stdio", + Long: serveSftpDescription, + RunE: serveSftpCmd, + Example: `buildah serve-sftp containerID|imageID [ro]`, + GroupID: groupSystem, + Args: cobra.RangeArgs(1, 2), + Hidden: true, + } + serveSftpReadOnly bool + + findDataDirDescription = "\n Find the data directory for a container or image." + findDataDirCommand = &cobra.Command{ + Use: findDataDir, + Short: "Find the container's or image's data directory", + Long: findDataDirDescription, + RunE: findDataDirCmd, + Example: `buildah find-data-dir containerID|imageID`, + GroupID: groupSystem, + Args: cobra.ExactArgs(1), + Hidden: true, + } + + mountSshfsDescription = "\n Mounts a container or image's rootfs using sshfs." + mountSshfsCommand = &cobra.Command{ + Use: mountSshfs, + Short: "Mount a container or image's rootfs using sshfs", + Long: mountSshfsDescription, + RunE: mountSshfsCmd, + Example: `buildah mount-sshfs [-o options,...] containerID|imageID [...]`, + GroupID: groupSystem, + Args: cobra.ArbitraryArgs, + Hidden: true, + } + mountSshfsOptions string + + umountSshfsDescription = "\n Unmounts a container or image mounted using sshfs." + umountSshfsCommand = &cobra.Command{ + Use: umountSshfs, + Aliases: []string{"unmount-sshfs"}, + Short: "Unmount a container or image mounted using sshfs", + Long: umountSshfsDescription, + RunE: umountSshfsCmd, + Example: `buildah unmount-sshfs /path/to/mounted/root-filesystem`, + GroupID: groupSystem, + Args: cobra.ArbitraryArgs, + Hidden: true, + } + umountSshfsAll bool + + listSshfsMountDirsDescription = "\n List containers and images mounted with sshfs." + listSshfsMountDirsCommand = &cobra.Command{ + Use: listSshfsMountDirs, + Short: "List containers and images mounted with sshfs", + Long: listSshfsMountDirsDescription, + RunE: listSshfsMountDirsCmd, + Example: `buildah list-sshfs-mount-dirs`, + GroupID: groupSystem, + Args: cobra.NoArgs, + Hidden: true, + } +) + +func init() { + reexec.Register(serveSftpInSubprocess, serveSftpSubprocessMain) +} + +func sftpInit() { + serveSftpCommand.SetUsageTemplate(UsageTemplate()) + flags := serveSftpCommand.Flags() + flags.SetInterspersed(false) + flags.BoolVar(&serveSftpReadOnly, "read-only", serveSftpReadOnly, "serve SFTP in read-only mode") + rootCmd.AddCommand(serveSftpCommand) + + findDataDirCommand.SetUsageTemplate(UsageTemplate()) + flags = findDataDirCommand.Flags() + flags.SetInterspersed(false) + rootCmd.AddCommand(findDataDirCommand) + + mountSshfsCommand.SetUsageTemplate(UsageTemplate()) + flags = mountSshfsCommand.Flags() + flags.SetInterspersed(false) + flags.StringVarP(&mountSshfsOptions, "options", "o", mountSshfsOptions, "comma-separated list of mount options") + rootCmd.AddCommand(mountSshfsCommand) + + umountSshfsCommand.SetUsageTemplate(UsageTemplate()) + flags = umountSshfsCommand.Flags() + flags.SetInterspersed(false) + flags.BoolVarP(&umountSshfsAll, "all", "a", false, "unmount all containers and images mounted with sshfs") + rootCmd.AddCommand(umountSshfsCommand) + + listSshfsMountDirsCommand.SetUsageTemplate(UsageTemplate()) + flags = listSshfsMountDirsCommand.Flags() + flags.SetInterspersed(false) + rootCmd.AddCommand(listSshfsMountDirsCommand) +} + +type pipeReadWriteCloser struct { + io.ReadCloser + io.WriteCloser +} + +func (p pipeReadWriteCloser) Close() error { + return errors.Join(p.ReadCloser.Close(), p.WriteCloser.Close()) +} + +// serveSftpCmd runs an SFTP service on stdio pointed at the container or +// image's rootfs. Because it mounts a filesystem, we need to be running as +// root or in the user's user namespace, hopefully with the right ID mappings +// for the container. +func serveSftpCmd(c *cobra.Command, args []string) error { + systemContext, err := parse.SystemContextFromOptions(c) + if err != nil { + return fmt.Errorf("initializing system context: %w", err) + } + store, err := getStore(c) + if err != nil { + return fmt.Errorf("opening storage: %w", err) + } + defer func() { + if _, err := store.Shutdown(false); err != nil { + logrus.Debugf("shutting down storage: %v", err) + } + }() + + toMount := args[0] + var mountPoint string + readOnly := serveSftpReadOnly + builder, err := buildah.OpenBuilder(store, toMount) + if err == nil { + cdir, err := store.ContainerDirectory(builder.ContainerID) + if err != nil { + return err + } + if err := lockServingSFTP(cdir); err != nil { + return fmt.Errorf("gaining exclusive access to mount build container %q: %w", toMount, err) + } + // mount a builder + mp, err := builder.Mount("") + if err != nil { + return fmt.Errorf("mounting builder %q: %w", toMount, err) + } + defer func() { + if err := builder.Unmount(); err != nil { + logrus.Debugf("unmounting builder %q: %v", toMount, err) + } + }() + mountPoint = mp + } else if errors.Is(err, os.ErrNotExist) || errors.Is(err, storage.ErrContainerUnknown) { + cdir, err := store.ContainerDirectory(toMount) + if err != nil { + return err + } + if err := lockServingSFTP(cdir); err != nil { + return fmt.Errorf("gaining exclusive access to mount container %q: %w", toMount, err) + } + // mount a random container + mp, err := store.Mount(toMount, "") + if err == nil { + defer func() { + if _, err := store.Unmount(toMount, false); err != nil { + logrus.Debugf("unmounting container %q: %v", toMount, err) + } + }() + mountPoint = mp + } else if errors.Is(err, os.ErrNotExist) || errors.Is(err, storage.ErrLayerUnknown) { + // mount an image + readOnly = true + imageSpec := toMount + _, img, err := util.FindImage(store, "", systemContext, imageSpec) + if err != nil { + return fmt.Errorf("mounting image %q: %w", toMount, err) + } + idir, err := store.ImageDirectory(img.ID) + if err != nil { + return err + } + if err := lockServingSFTP(idir); err != nil { + return fmt.Errorf("gaining exclusive access to mount image %q: %w", imageSpec, err) + } + mp, err := store.MountImage(img.ID, []string{"ro"}, "") + if err != nil { + return fmt.Errorf("mounting image %q: %w", toMount, err) + } + defer func() { + if _, err := store.UnmountImage(img.ID, false); err != nil { + logrus.Debugf("unmounting image %q: %v", toMount, err) + } + }() + mountPoint = mp + } else { + // was not a container, and failed to mount an image + return fmt.Errorf("mounting container %q: %w", toMount, err) + } + } else { + // failed to mount a builder + return fmt.Errorf("mounting builder %q: %w", toMount, err) + } + + logrus.Debugf("mounted %q, serving sftp", mountPoint) + sftpServerArgs := []string{serveSftpInSubprocess, mountPoint} + if readOnly { + sftpServerArgs = append(sftpServerArgs, "ro") + } + subprocess := reexec.Command(sftpServerArgs...) + subprocess.Stdin = os.Stdin + subprocess.Stdout = os.Stdout + subprocess.Stderr = os.Stderr + subprocess.Dir = "/" + err = subprocess.Start() + if err != nil { + return fmt.Errorf("starting sftp server: %w", err) + } + + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) + go func() { + for received := range signals { + if err := subprocess.Process.Signal(received); err != nil && !errors.Is(err, os.ErrProcessDone) { + logrus.Errorf("sending signal to sftp subprocess: %v", err) + } + } + }() + return subprocess.Wait() +} + +// serveSftpSubprocessMain chroots into a mountpoint (os.Args[1]) and runs an +// SFTP service on stdio. Subsequent arguments are nominally SFTP server +// options, but any value than "ro" is currently discarded. +func serveSftpSubprocessMain() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "expected directory name and optional flags") + os.Exit(1) + } + mountPoint := os.Args[1] + + var flags []string + if len(os.Args) > 2 { + flags = os.Args[2:] + } + var sftpServerOptions []sftp.ServerOption + for _, flag := range flags { + switch flag { + case "ro": + sftpServerOptions = append(sftpServerOptions, sftp.ReadOnly()) + default: + fmt.Fprintln(os.Stderr, "unrecognized flag", flag) + os.Exit(1) + } + } + + sftpPipe := pipeReadWriteCloser{ + ReadCloser: os.Stdin, + WriteCloser: os.Stdout, + } + sftpServer, err := sftp.NewServer(&sftpPipe, sftpServerOptions...) + if err != nil { + fmt.Fprintln(os.Stderr, "initializing sftp server:", err) + os.Exit(1) + } + + if err := unix.Chdir(mountPoint); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := unix.Chroot(mountPoint); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := unix.Chdir("/"); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + + signals := make(chan os.Signal, 10) + signal.Notify(signals, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) + var whichSig os.Signal + go func() { + whichSig = <-signals + switch whichSig { + case syscall.SIGHUP, syscall.SIGTERM: + os.Exit(0) + default: + fmt.Fprintln(os.Stderr, "sftp server:", whichSig) + os.Exit(1) + } + }() + err = sftpServer.Serve() + if err != nil && whichSig != syscall.SIGHUP && whichSig != syscall.SIGTERM { + fmt.Fprintln(os.Stderr, "sftp server:", err) + os.Exit(1) + } + os.Exit(0) +} + +// flagServingSFTP returns an error if it looks like an image or container is +// already mounted for use by an sftp server. That would create an overlay +// mount in its own mount namespace, and operations that would also try to +// mount the filesystem will either fail outright because the kernel won't +// allow its upper directory to be mounted more than once at a time, or will +// fail in maddening ways because the kernel allows it but overlay doesn't +// guarantee that anything will work right if you use the same upper directory +// for more than one mount at the same time. +func flagServingSFTP(datadir string) error { + lockfile := filepath.Join(datadir, servingSftpFragment) + fd, err := unix.Open(lockfile, unix.O_RDWR, 0o600) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return &os.PathError{Op: "open", Path: lockfile, Err: err} + } + defer func() { + if err := unix.Close(fd); err != nil { + logrus.Debugf("closing %q: %v", lockfile, err) + } + }() + for i := range serveSftpCheckIterations { + if err = unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB); err == nil { + break + } + if i < serveSftpCheckIterations-1 { + time.Sleep(serveSftpCheckInterval) + } + } + if err != nil { + return &os.PathError{Op: "flock", Path: lockfile, Err: err} + } + return nil +} + +// lockServingSFTP takes a lock on a file to signal to other processes that the +// filesystem is mounted for use by an sftp server. +func lockServingSFTP(datadir string) error { + lockfile := filepath.Join(datadir, servingSftpFragment) + fd, err := unix.Open(lockfile, unix.O_RDWR|unix.O_TRUNC|unix.O_CREAT, 0o600) + if err != nil { + return &os.PathError{Op: "open", Path: lockfile, Err: err} + } + if err := unix.Flock(fd, unix.LOCK_EX|unix.LOCK_NB); err != nil { + return &os.PathError{Op: "flock", Path: lockfile, Err: err} + } + return nil +} + +// findDataDirCmd locates the builder, container, or image named on its command +// line and prints the location of its data directory. We run this from a +// process which is not in the user's namespace, so that it can set up the +// user's namespace if it has to, so that it can examine the storage contents. +func findDataDirCmd(c *cobra.Command, args []string) error { + systemContext, err := parse.SystemContextFromOptions(c) + if err != nil { + return fmt.Errorf("initializing system context: %w", err) + } + store, err := getStore(c) + if err != nil { + return fmt.Errorf("opening storage: %w", err) + } + defer func() { + if _, err := store.Shutdown(false); err != nil { + logrus.Debugf("shutting down storage: %v", err) + } + }() + + var dataDir string + toFind := args[0] + builder, err := buildah.OpenBuilder(store, toFind) + if err == nil { + // it's a builder + logrus.Debugf("%q is a builder", toFind) + dd, err := store.ContainerDirectory(builder.ContainerID) + if err != nil { + return fmt.Errorf("finding userdata directory for builder %q: %w", toFind, err) + } + dataDir = dd + } else if errors.Is(err, os.ErrNotExist) || errors.Is(err, storage.ErrContainerUnknown) { + // it's a random container + container, err := store.Container(toFind) + if err == nil { + logrus.Debugf("%q is a non-builder container", toFind) + dd, err := store.ContainerDirectory(container.ID) + if err != nil { + return fmt.Errorf("finding userdata directory for builder or container %q: %w", toFind, err) + } + dataDir = dd + } else if errors.Is(err, os.ErrNotExist) || errors.Is(err, storage.ErrContainerUnknown) { + // it's an image + _, img, err := util.FindImage(store, "", systemContext, toFind) + if err != nil { + return fmt.Errorf("locating container or image %q: %w", toFind, err) + } + logrus.Debugf("%q is an image", toFind) + dd, err := store.ImageDirectory(img.ID) + if err != nil { + return fmt.Errorf("finding userdata directory for image %q: %w", toFind, err) + } + dataDir = dd + } else { + // was not a container, and failed to find an image + return fmt.Errorf("locating container %q: %w", toFind, err) + } + } else { + // failed to find a builder + return fmt.Errorf("locating builder %q: %w", toFind, err) + } + + fmt.Fprintln(os.Stdout, dataDir) + return nil +} + +// listSshfsMountDirsCmd prints pairs of lines: the ID of a container or image, +// and the name of where its rootfs would be mounted, if it were. +func listSshfsMountDirsCmd(c *cobra.Command, _ []string) error { + store, err := getStore(c) + if err != nil { + return fmt.Errorf("opening storage: %w", err) + } + defer func() { + if _, err := store.Shutdown(false); err != nil { + logrus.Debugf("shutting down storage: %v", err) + } + }() + whatAndWhere := make(map[string]string) + containers, err := store.Containers() + if err != nil { + return err + } + images, err := store.Images() + if err != nil { + return err + } + for _, container := range containers { + cdir, err := store.ContainerDirectory(container.ID) + if err != nil { + return err + } + whatAndWhere[container.ID] = filepath.Join(cdir, mountSshfsMountFragment) + logrus.Debugf("container %q would be mounted at %q", container.ID, whatAndWhere[container.ID]) + } + for _, image := range images { + idir, err := store.ImageDirectory(image.ID) + if err != nil { + return err + } + whatAndWhere[image.ID] = filepath.Join(idir, mountSshfsMountFragment) + logrus.Debugf("image %q would be mounted at %q", image.ID, whatAndWhere[image.ID]) + } + for k, v := range whatAndWhere { + fmt.Println(k) + fmt.Println(v) + } + return nil +} + +// listSshfsMounts returns a list of currently-used mountpoints, and a map from +// those to the IDs of the containers or images which we expect they hold. +func listSshfsMounts(inheritedFlags []string, selfExecutable string) ([]string, map[string]string, error) { + var listSshfsMountDirsStdout bytes.Buffer + listSshfsMountDirsArgs := append(slices.Clone(inheritedFlags), listSshfsMountDirs) + listSshfsMountDirs := exec.Command(selfExecutable, listSshfsMountDirsArgs...) + listSshfsMountDirs.Stdout = &listSshfsMountDirsStdout + listSshfsMountDirs.Stderr = os.Stderr + logrus.Debugf("running %v to get a list of possible sshfs mountpoints", listSshfsMountDirsArgs) + if err := listSshfsMountDirs.Run(); err != nil { + return nil, nil, fmt.Errorf("listing candidate locations where containers or images would be mounted: %w", err) + } + namesOfMountPoints := make(map[string]string) + var mountPoints []string + for { + whatName, err := listSshfsMountDirsStdout.ReadString('\n') + if err != nil { + break // possible EOF, discard + } + if whatName == "" { + break + } + whatName = strings.TrimSuffix(whatName, "\n") + mountPoint, err := listSshfsMountDirsStdout.ReadString('\n') + if mountPoint == "" { + break + } + mountPoint = strings.TrimSuffix(mountPoint, "\n") + var parentStat, mountPointStat unix.Stat_t + if err2 := unix.Stat(filepath.Dir(mountPoint), &parentStat); err2 != nil { + return nil, nil, fmt.Errorf("stat %q: %w", filepath.Dir(mountPoint), err2) + } + if err2 := unix.Stat(mountPoint, &mountPointStat); err2 != nil { + if errors.Is(err2, os.ErrNotExist) { + continue // doesn't look like it could be mounted + } + return nil, nil, fmt.Errorf("stat %q: %w", mountPoint, err2) + } + if err != nil { // possible EOF + break + } + if parentStat.Dev != mountPointStat.Dev { + logrus.Debugf("%q is a mountpoint", mountPoint) + mountPoints = append(mountPoints, mountPoint) + namesOfMountPoints[mountPoint] = whatName + } else { + logrus.Debugf("%q is not a mountpoint", mountPoint) + } + } + return mountPoints, namesOfMountPoints, nil +} + +// mountSshfsCmd mounts a container or image on a container- or image-specific +// location using sshfs. +func mountSshfsCmd(c *cobra.Command, args []string) error { + inheritedFlags := getInheritedFlags(c) + + selfExecutable, err := os.Executable() + if err != nil { + return fmt.Errorf("determining our executable: %w", err) + } + + if len(args) == 0 { + // Just list what's mounted right now. + mountPoints, namesOfMountPoints, err := listSshfsMounts(inheritedFlags, selfExecutable) + if err != nil { + return err + } + for _, mountPoint := range mountPoints { + fmt.Println(namesOfMountPoints[mountPoint], mountPoint) + } + return nil + } + + if _, err := exec.LookPath("sshfs"); err != nil { + return fmt.Errorf(`looking for "sshfs" in $PATH: %w`, err) + } + + // Work out where the mount points for the various items that we want + // to mount would be. + whatGoesWhere := make(map[string]string) + for _, toMount := range args { + var mountParentDirBuffer bytes.Buffer + findDataDirArgs := append(slices.Clone(inheritedFlags), findDataDir, toMount) + findDataDir := exec.Command(selfExecutable, findDataDirArgs...) + findDataDir.Stdin = nil + findDataDir.Stdout = &mountParentDirBuffer + findDataDir.Stderr = os.Stderr + findDataDir.Dir = "/" + if err := findDataDir.Run(); err != nil { + return fmt.Errorf("determining where %q would be mounted: %w", toMount, err) + } + + mountParentDir := strings.TrimSpace(mountParentDirBuffer.String()) + if mountParentDir == "" { + return fmt.Errorf("determining parent of mountpoint %q: %q", toMount, mountParentDir) + } + + if err := flagServingSFTP(mountParentDir); err != nil { + return err + } + + mountPoint := filepath.Join(mountParentDir, mountSshfsMountFragment) + var parentStat, mountPointStat unix.Stat_t + if err := unix.Mkdir(mountPoint, 0o700); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("mkdir(%q): %w", mountPoint, err) + } + if err := unix.Stat(mountPoint, &mountPointStat); err != nil { + return fmt.Errorf("stat %q: %w", mountPoint, err) + } + if err := unix.Stat(mountParentDir, &parentStat); err != nil { + return fmt.Errorf("stat %q: %w", mountParentDir, err) + } + if mountPointStat.Dev != parentStat.Dev { + return fmt.Errorf("filesystem (possibly for %q) already mounted on %s", toMount, mountPoint) + } + + whatGoesWhere[toMount] = mountPoint + } + + signals := make(chan os.Signal, 10) + signal.Notify(signals, syscall.SIGHUP, syscall.SIGTERM, syscall.SIGINT) + + // Launch each pair. + serveSftpStderr := make([]bytes.Buffer, len(args)) + mountSshfsStderr := make([]bytes.Buffer, len(args)) + for i, toMount := range args { + succeeded := false + + r1, w1, err := os.Pipe() + if err != nil { + return fmt.Errorf("pipe: %w", err) + } + defer func() { + if !succeeded { + r1.Close() + w1.Close() + } + }() + + r2, w2, err := os.Pipe() + if err != nil { + return fmt.Errorf("pipe: %w", err) + } + defer func() { + if !succeeded { + r2.Close() + w2.Close() + } + }() + + serveSftpArgs := append(slices.Clone(inheritedFlags), serveSftp, toMount) + serveSftp := exec.Command(selfExecutable, serveSftpArgs...) + serveSftp.Stdin = r1 + serveSftp.Stdout = w2 + serveSftp.Stderr = &serveSftpStderr[i] + serveSftp.Dir = "/" + if serveSftp.SysProcAttr == nil { + serveSftp.SysProcAttr = &syscall.SysProcAttr{} + } + if err := serveSftp.Start(); err != nil { + return fmt.Errorf("starting sftp server: %w", err) + } + r1.Close() + w2.Close() + defer func() { + if !succeeded { + if err := serveSftp.Process.Signal(syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) { + logrus.Errorf("terminating sftp server for %q after startup error: %v", err, toMount) + } + } + }() + + if err := os.Mkdir(whatGoesWhere[toMount], 0o700); err != nil && !errors.Is(err, os.ErrExist) { + return fmt.Errorf("creating mount point at %q: %v", whatGoesWhere[toMount], err) + } + + options := append(strings.Split(mountSshfsOptions, ","), "passive") + options = slices.DeleteFunc(options, func(s string) bool { return s == "" }) + mountSshfs := exec.Command("sshfs", "-f", "-o", strings.Join(options, ","), toMount+":/", whatGoesWhere[toMount]) + mountSshfs.Stdin = r2 + mountSshfs.Stdout = w1 + mountSshfs.Stderr = &mountSshfsStderr[i] + mountSshfs.Dir = "/" + if mountSshfs.SysProcAttr == nil { + mountSshfs.SysProcAttr = &syscall.SysProcAttr{} + } + if err := mountSshfs.Start(); err != nil { + return fmt.Errorf("mounting using sshfs: %w", err) + } + r2.Close() + w1.Close() + + var errs [2]error + succeeded = true + var wg sync.WaitGroup + go func() { + whichSig, ok := <-signals + if !ok { + whichSig = syscall.SIGTERM + } + if err := mountSshfs.Process.Signal(whichSig); err != nil && !errors.Is(err, os.ErrProcessDone) { + logrus.Errorf("sending signal to sshfs: %v", err) + } + if err := serveSftp.Process.Signal(whichSig); err != nil && !errors.Is(err, os.ErrProcessDone) { + logrus.Errorf("sending signal to sftp server: %v", err) + } + }() + + sftpDone := make(chan error, 1) + sshfsDone := make(chan error, 1) + wg.Go(func() { errs[0] = serveSftp.Wait(); signal.Stop(signals); sftpDone <- errs[0] }) + wg.Go(func() { errs[1] = mountSshfs.Wait(); signal.Stop(signals); sshfsDone <- errs[1] }) + wg.Go(func() { + <-sftpDone + if err := mountSshfs.Process.Signal(syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) { + logrus.Errorf("terminating sshfs after sftp server exited: %v", err) + } + }) + wg.Go(func() { + <-sshfsDone + if err := serveSftp.Process.Signal(syscall.SIGTERM); err != nil && !errors.Is(err, os.ErrProcessDone) { + logrus.Errorf("terminating sftp server after sshfs exited: %v", err) + } + }) + + go func() { + wg.Wait() + if errs[0] != nil { + errs[0] = fmt.Errorf("sftp server: %w", errs[0]) + } + if errs[1] != nil { + errs[1] = fmt.Errorf("sshfs: %w", errs[1]) + } + if err := errors.Join(errs[:]...); err != nil { + logrus.Errorln(err) + } + }() + + fmt.Println(whatGoesWhere[toMount]) + } + + // Wait for the mountpoints to have filesystems on them, so that when + // we return, the caller can access their contents immediately. + for _, where := range whatGoesWhere { + var parentStat, mountPointStat unix.Stat_t + mountParentDir := filepath.Dir(where) + if err := unix.Stat(mountParentDir, &parentStat); err != nil { + return fmt.Errorf("stat(%q): %w", filepath.Dir(where), err) + } + for range mountUmountCheckIterations { + if err := unix.Stat(where, &mountPointStat); err != nil { + return fmt.Errorf("stat(%q): %w", where, err) + } + if mountPointStat.Dev != parentStat.Dev { + break + } + logrus.Debugf("%q is not (yet?) a mountpoint, waiting %s", where, mountUmountCheckInterval) + time.Sleep(mountUmountCheckInterval) + } + } + + // Forward any error messages to the caller before exiting from main. + for i := range args { + prefix := "" + if len(args) > 0 { + prefix = args[i] + ": " + } + if serveSftpStderr[i].Len() > 0 { + fmt.Fprint(os.Stderr, prefix+serveSftpStderr[i].String()) + } + if mountSshfsStderr[i].Len() > 0 { + fmt.Fprint(os.Stderr, prefix+mountSshfsStderr[i].String()) + } + } + + return nil +} + +// umountSshfsCmd unmounts a container or image which was previously mounted by +// mountSshfsCmd(). +func umountSshfsCmd(c *cobra.Command, args []string) error { + selfExecutable, err := os.Executable() + if err != nil { + return fmt.Errorf("determining our executable: %w", err) + } + + inheritedFlags := getInheritedFlags(c) + + if len(args) == 0 && !umountSshfsAll { + return errors.New("at least one container or image ID must be specified") + } + if len(args) > 0 && umountSshfsAll { + return errors.New("when using the --all switch, you may not pass any container or image IDs") + } + if err := buildahcli.VerifyFlagsArgsOrder(args); err != nil { + return err + } + + var mountPoints []string + if umountSshfsAll { + if mountPoints, _, err = listSshfsMounts(inheritedFlags, selfExecutable); err != nil { + return err + } + } else { + mountPoints = make([]string, 0, len(args)) + for _, toUnmount := range args { + var mountParentDirBuffer bytes.Buffer + + if err := fileutils.Exists(toUnmount); err != nil && errors.Is(err, os.ErrNotExist) { + findDataDirArgs := append(slices.Clone(inheritedFlags), findDataDir, toUnmount) + findDataDir := exec.Command(selfExecutable, findDataDirArgs...) + findDataDir.Stdin = nil + findDataDir.Stdout = &mountParentDirBuffer + findDataDir.Stderr = os.Stderr + findDataDir.Dir = "/" + if err := findDataDir.Run(); err != nil { + return fmt.Errorf("determining where %q would be mounted: %w", toUnmount, err) + } + } else { + abs, err := filepath.Abs(toUnmount) + if err != nil { + return fmt.Errorf("converting %q to an absolute path: %w", toUnmount, err) + } + if filepath.Base(abs) != mountSshfsMountFragment { + return fmt.Errorf("%q doesn't look like an sshfs mountpoint", toUnmount) + } + mountParentDirBuffer.WriteString(filepath.Dir(abs)) + } + + mountParentDir := strings.TrimSpace(mountParentDirBuffer.String()) + if mountParentDir == "" { + return fmt.Errorf("determining parent of mountpoint %q: %q", toUnmount, mountParentDir) + } + mountPoint := filepath.Join(mountParentDir, mountSshfsMountFragment) + + var parentStat, mountPointStat unix.Stat_t + if err := unix.Stat(filepath.Dir(mountPoint), &parentStat); err != nil { + return fmt.Errorf("stat(%q): %w", filepath.Dir(mountPoint), err) + } + if err := unix.Stat(mountPoint, &mountPointStat); err != nil { + if errors.Is(err, os.ErrNotExist) { + // yeah, that'll do + return nil + } + return fmt.Errorf("stat(%q): %w", mountPoint, err) + } + if mountPointStat.Dev == parentStat.Dev { + continue + } + + mountPoints = append(mountPoints, mountPoint) + } + } + + var errs []error + for _, mountPoint := range mountPoints { + if mountError := unix.Unmount(mountPoint, unix.MNT_DETACH); mountError != nil { + errs = append(errs, fmt.Errorf("umount(%q): %w", mountPoint, mountError)) + logrus.Debugf("umount(%q): %v", mountPoint, mountError) + + umount := exec.Command("umount", "-l", mountPoint) + umount.Stdin = &bytes.Buffer{} + umount.Stdout = os.Stdout + umount.Stderr = os.Stderr + umount.Dir = "/" + umountError := umount.Run() + + if umountError != nil { + errs = append(errs, fmt.Errorf("umount %s: %w", mountPoint, umountError)) + logrus.Debugf("umount(%q): %v", mountPoint, umountError) + fumount := exec.Command("fuserumount3", "-u", mountPoint) + umount.Stdin = &bytes.Buffer{} + fumount.Stdout = os.Stdout + fumount.Stderr = os.Stderr + fumount.Dir = "/" + fumountError := fumount.Run() + if fumountError != nil { + logrus.Debugf("umount(%q): %v", mountPoint, fumountError) + errs = append(errs, fmt.Errorf("fusermount3 -u %s: %w", mountPoint, fumountError)) + } else { + errs = nil + } + } else { + errs = nil + } + } + } + + for _, mountPoint := range mountPoints { + var parentStat, mountPointStat unix.Stat_t + // Wait for the mountpoint to have a filesystem on it, so that when we + // return, the caller can access its contents immediately. + for range mountUmountCheckIterations { + if err := unix.Stat(filepath.Dir(mountPoint), &parentStat); err != nil { + return fmt.Errorf("stat(%q): %w", mountPoint, err) + } + if err := unix.Stat(mountPoint, &mountPointStat); err != nil { + return fmt.Errorf("stat(%q): %w", mountPoint, err) + } + if mountPointStat.Dev == parentStat.Dev { + break + } + logrus.Debugf("%q is still a mountpoint, waiting %s", mountPoint, mountUmountCheckInterval) + time.Sleep(mountUmountCheckInterval) + } + } + + return errors.Join(errs...) +} diff --git a/cmd/buildah/sftp_unsupported.go b/cmd/buildah/sftp_unsupported.go new file mode 100644 index 00000000000..87c32989ee0 --- /dev/null +++ b/cmd/buildah/sftp_unsupported.go @@ -0,0 +1,15 @@ +//go:build !linux + +package main + +const ( + mountSshfs = "mount-sshfs" + umountSshfs = "umount-sshfs" +) + +func sftpInit() { +} + +func flagServingSFTP(string) error { + return nil +} diff --git a/docs/buildah-mount.1.md b/docs/buildah-mount.1.md index 7d5031b5452..ac1f9dfaad8 100644 --- a/docs/buildah-mount.1.md +++ b/docs/buildah-mount.1.md @@ -63,4 +63,4 @@ $ buildah commit working-container newimage ``` ## SEE ALSO -buildah(1), buildah-unshare(1), buildah-umount(1) +buildah(1), buildah-umount(1), buildah-unshare(1) diff --git a/docs/buildah-umount.1.md b/docs/buildah-umount.1.md index 28a3ad216d0..82a1eac4a7d 100644 --- a/docs/buildah-umount.1.md +++ b/docs/buildah-umount.1.md @@ -24,4 +24,4 @@ buildah umount containerID1 containerID2 containerID3 buildah umount --all ## SEE ALSO -buildah(1), buildah-umount(1) +buildah(1), buildah-umount(1), buildah-unshare(1) diff --git a/go.mod b/go.mod index 04445912609..18ce5b25ae9 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/opencontainers/runtime-tools v0.9.1-0.20260316125833-8a4db579f5c8 github.com/opencontainers/selinux v1.15.1 github.com/openshift/imagebuilder v1.2.21 + github.com/pkg/sftp v1.13.10 github.com/seccomp/libseccomp-golang v0.11.1 github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 @@ -87,6 +88,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.19.0 // indirect github.com/klauspost/pgzip v1.2.6 // indirect + github.com/kr/fs v0.1.0 // indirect github.com/manifoldco/promptui v0.9.0 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect github.com/mattn/go-sqlite3 v1.14.47 // indirect diff --git a/go.sum b/go.sum index 311ab44b451..485078adf8e 100644 --- a/go.sum +++ b/go.sum @@ -129,6 +129,8 @@ github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2o github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -199,6 +201,8 @@ github.com/openshift/imagebuilder v1.2.21 h1:XX0tZVznWTxzYevvNVZ/0eeTzmgY6cfcT4/ github.com/openshift/imagebuilder v1.2.21/go.mod h1:+L09sXUQ0RPdCU1tmzKrfBhqMlYvZtaA3MHb7aTjVU8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= +github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= diff --git a/pkg/parse/parse.go b/pkg/parse/parse.go index 8d9c83dd943..5a68eb85269 100644 --- a/pkg/parse/parse.go +++ b/pkg/parse/parse.go @@ -866,7 +866,7 @@ func GetConfidentialWorkloadOptions(arg string) (define.ConfidentialWorkloadOpti case strings.HasPrefix(option, "memory="): options.Memory, err = strconv.Atoi(strings.TrimPrefix(option, "memory=")) if err != nil { - return options, fmt.Errorf("parsing memory= value %q: %w", strings.TrimPrefix(option, "memorys"), err) + return options, fmt.Errorf("parsing memory= value %q: %w", strings.TrimPrefix(option, "memory="), err) } case option == "ignore_attestation_errors", option == "ignore-attestation-errors": options.IgnoreAttestationErrors = true diff --git a/rpm/buildah.spec b/rpm/buildah.spec index 8f73e20c340..6905cf03600 100644 --- a/rpm/buildah.spec +++ b/rpm/buildah.spec @@ -157,6 +157,7 @@ export BUILDTAGS+=" containers_image_sequoia" %gobuild -o bin/crash ./tests/crash %gobuild -o bin/wait ./tests/wait %gobuild -o bin/grpcnoop ./tests/rpc/noop +%gobuild -o bin/pipeloop ./tests/pipeloop/pipeloop.go %{__make} docs %install @@ -173,6 +174,7 @@ cp bin/passwd %{buildroot}/%{_bindir}/%{name}-passwd cp bin/crash %{buildroot}/%{_bindir}/%{name}-crash cp bin/wait %{buildroot}/%{_bindir}/%{name}-wait cp bin/grpcnoop %{buildroot}/%{_bindir}/%{name}-grpcnoop +cp bin/pipeloop %{buildroot}/%{_bindir}/%{name}-pipeloop rm %{buildroot}%{_datadir}/%{name}/test/system/tools/build/* @@ -202,6 +204,7 @@ rm %{buildroot}%{_datadir}/%{name}/test/system/tools/build/* %{_bindir}/%{name}-crash %{_bindir}/%{name}-wait %{_bindir}/%{name}-grpcnoop +%{_bindir}/%{name}-pipeloop %{_datadir}/%{name}/test %changelog diff --git a/tests/add.bats b/tests/add.bats index b06fb0d9d9a..1224d1910ef 100644 --- a/tests/add.bats +++ b/tests/add.bats @@ -19,11 +19,12 @@ load helpers run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid + run_buildah_mount $cid root=$output mkdir $root/subdir $root/other-subdir # Copy a file to the working directory run_buildah config --workingdir=/ $cid + run_buildah_umount $cid run_buildah add --retry 4 --retry-delay 4s $cid ${TEST_SCRATCH_DIR}/randomfile # Copy a file to a specific subdirectory run_buildah add $cid ${TEST_SCRATCH_DIR}/randomfile /subdir @@ -36,13 +37,12 @@ load helpers # Copy a file to a different working directory run_buildah config --workingdir=/cwd $cid run_buildah add $cid ${TEST_SCRATCH_DIR}/randomfile - run_buildah unmount $cid run_buildah commit $WITH_POLICY_JSON $cid containers-storage:new-image run_buildah rm $cid run_buildah from $WITH_POLICY_JSON new-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -s $newroot/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $newroot/randomfile @@ -55,7 +55,6 @@ load helpers test -d $newroot/cwd test -s $newroot/cwd/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $newroot/cwd/randomfile - run_buildah rm $newcid } @test "add-local-archive" { @@ -91,7 +90,7 @@ load helpers run_buildah from $WITH_POLICY_JSON new-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -s $newroot/random1 cmp ${TEST_SCRATCH_DIR}/random1 $newroot/random1 @@ -245,11 +244,11 @@ stuff/mystuff" run_buildah add --contextdir=${mytest} --ignorefile ${mytest}/.ignore $cid ${mytest} /stuff - run_buildah mount $cid + run_buildah_mount $cid mnt=$output run find $mnt -printf "%P\n" filelist=$(LC_ALL=C sort <<<"$output") - run_buildah umount $cid + run_buildah_umount $cid expect_output --from="$filelist" "$expect" "container file list" } @@ -260,7 +259,7 @@ stuff/mystuff" cid=$output run_buildah add --quiet $cid ${TEST_SCRATCH_DIR}/randomfile /tmp/random expect_output "" - run_buildah mount $cid + run_buildah_mount $cid croot=$output cmp ${TEST_SCRATCH_DIR}/randomfile ${croot}/tmp/random } @@ -278,7 +277,7 @@ stuff/mystuff" expect_output "" run_buildah add --quiet $WITH_POLICY_JSON --from $from $cid tmp/random /tmp/random2 # relative path expect_output "" - run_buildah mount $cid + run_buildah_mount $cid croot=$output cmp ${TEST_SCRATCH_DIR}/randomfile ${croot}/tmp/random cmp ${TEST_SCRATCH_DIR}/randomfile ${croot}/tmp/random2 @@ -294,9 +293,9 @@ stuff/mystuff" expect_output "" run_buildah from --quiet $WITH_POLICY_JSON ubuntu ubuntu=$output - run_buildah mount $cid + run_buildah_mount $cid croot=$output - run_buildah mount $ubuntu + run_buildah_mount $ubuntu ubuntu=$output cmp $ubuntu/etc/passwd ${croot}/tmp/passwd cmp $ubuntu/etc/passwd ${croot}/tmp/passwd2 @@ -419,8 +418,10 @@ EOF run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid + run_buildah_mount $cid root=$output + mkdir $root/subdir + run_buildah_umount $cid run_buildah config --workingdir=/ $cid @@ -431,10 +432,8 @@ EOF run_buildah add --link $cid ${TEST_SCRATCH_DIR}/randomfile /renamed-file # Test 3: Multiple files to directory - mkdir $root/subdir run_buildah add --link $cid ${TEST_SCRATCH_DIR}/randomfile ${TEST_SCRATCH_DIR}/other-randomfile /subdir - run_buildah unmount $cid run_buildah commit $WITH_POLICY_JSON $cid add-link-image run_buildah inspect --type=image add-link-image @@ -448,7 +447,7 @@ EOF run_buildah from $WITH_POLICY_JSON add-link-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -s $newroot/randomfile @@ -489,7 +488,7 @@ EOF run_buildah from $WITH_POLICY_JSON add-link-archive-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -s $newroot/file1 @@ -519,7 +518,7 @@ EOF run_buildah from $WITH_POLICY_JSON add-link-dir-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -d $newroot/testdir @@ -540,13 +539,13 @@ EOF # Default (allow-wildcard=true): glob expands, adds both files run_buildah add $cid "${TEST_SCRATCH_DIR}/file-*" /dest/ - run_buildah mount $cid + run_buildah_mount $cid root=$output test -f $root/dest/file-a test -f $root/dest/file-b cmp ${TEST_SCRATCH_DIR}/file-a $root/dest/file-a cmp ${TEST_SCRATCH_DIR}/file-b $root/dest/file-b - run_buildah umount $cid + run_buildah_umount $cid # allow-wildcard=false: glob patterns are rejected run_buildah 125 add --allow-wildcard=false $cid "${TEST_SCRATCH_DIR}/file-*" /dest2/ @@ -566,17 +565,17 @@ EOF # Explicit true: non-matching glob should succeed run_buildah add --allow-empty-wildcard=true $cid "${TEST_SCRATCH_DIR}/nonexistent-*" /dest2/ - run_buildah mount $cid + run_buildah_mount $cid root=$output test ! -d $root/dest2 - run_buildah umount $cid + run_buildah_umount $cid # Existing files still add normally run_buildah add $cid ${TEST_SCRATCH_DIR}/file-a /dest3/ - run_buildah mount $cid + run_buildah_mount $cid root=$output test -f $root/dest3/file-a - run_buildah umount $cid + run_buildah_umount $cid # Literal non-existent file should still error even with allow-empty-wildcard=true run_buildah 125 add --allow-empty-wildcard=true $cid ${TEST_SCRATCH_DIR}/no-such-file /dest4/ @@ -592,7 +591,7 @@ EOF run_buildah add $cid ${TEST_SCRATCH_DIR}/symlink /dest - run_buildah mount $cid + run_buildah_mount $cid root=$output ls -lahR $root cmp ${TEST_SCRATCH_DIR}/file $root/dest @@ -610,7 +609,7 @@ EOF run_buildah add --no-follow-symlinks $cid ${TEST_SCRATCH_DIR}/file /file run_buildah add --no-follow-symlinks $cid ${TEST_SCRATCH_DIR}/symlink /dest - run_buildah mount $cid + run_buildah_mount $cid root=$output ls -lahR $root cmp ${TEST_SCRATCH_DIR}/file $root/dest @@ -628,7 +627,7 @@ EOF run_buildah add $cid ${TEST_SCRATCH_DIR}/src /dest - run_buildah mount $cid + run_buildah_mount $cid root=$output ls -lahR $root cmp ${TEST_SCRATCH_DIR}/src/file $root/dest/symlink @@ -645,9 +644,27 @@ EOF run_buildah add --no-follow-symlinks $cid ${TEST_SCRATCH_DIR}/src /dest - run_buildah mount $cid + run_buildah_mount $cid root=$output ls -lahR $root test -L $root/dest/symlink test "$(readlink $root/dest/symlink)" = "./file" } + +@test "add-from-image" { + _prefetch busybox + run_buildah from --quiet $WITH_POLICY_JSON busybox + cid=$output + run_buildah add $WITH_POLICY_JSON --quiet --from ubuntu $cid /etc/passwd /tmp/passwd # should pull the image, absolute path + expect_output "" + run_buildah add --quiet $WITH_POLICY_JSON --from ubuntu $cid etc/passwd /tmp/passwd2 # relative path + expect_output "" + run_buildah from --quiet $WITH_POLICY_JSON ubuntu + ubuntu=$output + run_buildah_mount $cid + croot=$output + run_buildah_mount $ubuntu + ubuntu=$output + cmp $ubuntu/etc/passwd ${croot}/tmp/passwd + cmp $ubuntu/etc/passwd ${croot}/tmp/passwd2 +} diff --git a/tests/basic.bats b/tests/basic.bats index 8ba77b94299..d83be0df9d6 100644 --- a/tests/basic.bats +++ b/tests/basic.bats @@ -38,22 +38,22 @@ load helpers @test "mount" { run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid + run_buildah_mount $cid root=$output - run_buildah unmount $cid - run_buildah mount $cid + run_buildah_umount $cid + run_buildah_mount $cid root=$output touch $root/foobar - run_buildah unmount $cid + run_buildah_umount $cid run_buildah rm $cid } @test "by-name" { run_buildah from $WITH_POLICY_JSON --name scratch-working-image-for-test scratch cid=$output - run_buildah mount scratch-working-image-for-test + run_buildah_mount scratch-working-image-for-test root=$output - run_buildah unmount scratch-working-image-for-test + run_buildah_umount scratch-working-image-for-test run_buildah rm scratch-working-image-for-test } @@ -63,10 +63,10 @@ load helpers run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid + run_buildah_mount $cid root=$output cp ${TEST_SCRATCH_DIR}/randomfile $root/randomfile - run_buildah unmount $cid + run_buildah_umount $cid run_buildah commit --iidfile ${TEST_SCRATCH_DIR}/output.iid $WITH_POLICY_JSON $cid containers-storage:new-image iid=$(< ${TEST_SCRATCH_DIR}/output.iid) assert "$iid" =~ "sha256:[0-9a-f]{64}" @@ -75,7 +75,7 @@ load helpers run_buildah rm $cid run_buildah from --quiet $WITH_POLICY_JSON new-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -s $newroot/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $newroot/randomfile @@ -86,43 +86,46 @@ load helpers run_buildah commit $WITH_POLICY_JSON $newcid containers-storage:another-new-image run_buildah commit $WITH_POLICY_JSON $newcid yet-another-new-image run_buildah commit $WITH_POLICY_JSON $newcid containers-storage:gratuitous-new-image - run_buildah unmount $newcid + run_buildah_umount $newcid run_buildah rm $newcid run_buildah from --quiet $WITH_POLICY_JSON other-new-image othernewcid=$output - run_buildah mount $othernewcid + run_buildah_mount $othernewcid othernewroot=$output test -s $othernewroot/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $othernewroot/randomfile test -s $othernewroot/other-randomfile cmp ${TEST_SCRATCH_DIR}/other-randomfile $othernewroot/other-randomfile + run_buildah_umount $othernewcid run_buildah rm $othernewcid run_buildah from --quiet $WITH_POLICY_JSON another-new-image anothernewcid=$output - run_buildah mount $anothernewcid + run_buildah_mount $anothernewcid anothernewroot=$output test -s $anothernewroot/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $anothernewroot/randomfile test -s $anothernewroot/other-randomfile cmp ${TEST_SCRATCH_DIR}/other-randomfile $anothernewroot/other-randomfile + run_buildah_umount $anothernewcid run_buildah rm $anothernewcid run_buildah from --quiet $WITH_POLICY_JSON yet-another-new-image yetanothernewcid=$output - run_buildah mount $yetanothernewcid + run_buildah_mount $yetanothernewcid yetanothernewroot=$output test -s $yetanothernewroot/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $yetanothernewroot/randomfile test -s $yetanothernewroot/other-randomfile cmp ${TEST_SCRATCH_DIR}/other-randomfile $yetanothernewroot/other-randomfile + run_buildah_umount $yetanothernewcid run_buildah delete $yetanothernewcid run_buildah from --quiet $WITH_POLICY_JSON new-image newcid=$output run_buildah commit --rm $WITH_POLICY_JSON $newcid containers-storage:remove-container-image - run_buildah 125 mount $newcid + run_buildah_mount 125 $newcid run_buildah rmi remove-container-image run_buildah rmi containers-storage:other-new-image diff --git a/tests/bud.bats b/tests/bud.bats index bbf13ccbf6e..23e2e19c90e 100644 --- a/tests/bud.bats +++ b/tests/bud.bats @@ -1055,11 +1055,11 @@ this is the output of test12" run_buildah from --pull=false testbud2 cid=$output - run_buildah mount $cid + run_buildah_mount $cid mnt=$output run find $mnt -printf "%P(%l)\n" filelist=$(LC_ALL=C sort <<<"$output") - run_buildah umount $cid + run_buildah_umount $cid # Format is: filename, and, in parentheses, symlink target (usually empty) # The list below has been painstakingly crafted; please be careful if @@ -2432,7 +2432,7 @@ _EOF run_buildah from --quiet $WITH_POLICY_JSON test3 ctr=$output - run_buildah mount ${ctr} + run_buildah_mount ${ctr} mnt=$output test -e $mnt/uuid test -e $mnt/date @@ -2528,7 +2528,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/multi-stage-builds/Dockerfile.extended $BUDFILES/multi-stage-builds run_buildah from $WITH_POLICY_JSON ${target} cid="$output" - run_buildah mount "$cid" + run_buildah_mount "$cid" root="$output" # cache should have used this one test -r "$root"/tmp/preCommit @@ -3053,11 +3053,12 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/from-multiple-files/Dockerfile1.scratch -f $BUDFILES/from-multiple-files/Dockerfile2.nofrom $BUDFILES/from-multiple-files run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile1 $BUDFILES/from-multiple-files/Dockerfile1.scratch cmp $root/Dockerfile2.nofrom $BUDFILES/from-multiple-files/Dockerfile2.nofrom test ! -s $root/etc/passwd + run_buildah_umount ${cid} run_buildah rm ${cid} run_buildah rmi -a @@ -3066,7 +3067,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile1.alpine -f Dockerfile2.nofrom $BUDFILES/from-multiple-files run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile1 $BUDFILES/from-multiple-files/Dockerfile1.alpine cmp $root/Dockerfile2.nofrom $BUDFILES/from-multiple-files/Dockerfile2.nofrom @@ -3079,11 +3080,12 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile1.scratch -f Dockerfile2.withfrom $BUDFILES/from-multiple-files run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output test ! -s $root/Dockerfile1 cmp $root/Dockerfile2.withfrom $BUDFILES/from-multiple-files/Dockerfile2.withfrom test -s $root/etc/passwd + run_buildah_umount ${cid} run_buildah rm ${cid} run_buildah rmi -a @@ -3092,7 +3094,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile1.alpine -f Dockerfile2.withfrom $BUDFILES/from-multiple-files run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output test ! -s $root/Dockerfile1 cmp $root/Dockerfile2.withfrom $BUDFILES/from-multiple-files/Dockerfile2.withfrom @@ -3364,10 +3366,11 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/multi-stage-builds/Dockerfile.index $BUDFILES/multi-stage-builds run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile.index $BUDFILES/multi-stage-builds/Dockerfile.index test -s $root/etc/passwd + run_buildah_umount ${cid} run_buildah rm ${cid} run_buildah rmi -a @@ -3376,10 +3379,11 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile.name $BUDFILES/multi-stage-builds run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile.name $BUDFILES/multi-stage-builds/Dockerfile.name test ! -s $root/etc/passwd + run_buildah_umount ${cid} run_buildah rm ${cid} run_buildah rmi -a @@ -3387,7 +3391,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/multi-stage-builds/Dockerfile.mixed $BUDFILES/multi-stage-builds run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile.name $BUDFILES/multi-stage-builds/Dockerfile.name cmp $root/Dockerfile.index $BUDFILES/multi-stage-builds/Dockerfile.index @@ -3400,10 +3404,11 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/multi-stage-builds-small-as/Dockerfile.index $BUDFILES/multi-stage-builds-small-as run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile.index $BUDFILES/multi-stage-builds-small-as/Dockerfile.index test -s $root/etc/passwd + run_buildah_umount ${cid} run_buildah rm ${cid} run_buildah rmi -a @@ -3412,10 +3417,11 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile.name $BUDFILES/multi-stage-builds-small-as run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile.name $BUDFILES/multi-stage-builds-small-as/Dockerfile.name test ! -s $root/etc/passwd + run_buildah_umount ${cid} run_buildah rm ${cid} run_buildah rmi -a @@ -3424,7 +3430,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/multi-stage-builds-small-as/Dockerfile.mixed $BUDFILES/multi-stage-builds-small-as run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile.name $BUDFILES/multi-stage-builds-small-as/Dockerfile.name cmp $root/Dockerfile.index $BUDFILES/multi-stage-builds-small-as/Dockerfile.index @@ -3442,7 +3448,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} ${layers} ${compat} $BUDFILES/preserve-volumes run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output # these files were created before VOLUME instructions froze the directories that contained them test -s $root/vol/subvol/subsubvol/subsubvolfile @@ -3459,6 +3465,7 @@ _EOF # and these were ADDed test -s $root/vol/Dockerfile test -s $root/vol/Dockerfile2 + run_buildah_umount ${cid} run_buildah rm ${cid} run_buildah rmi ${target} done @@ -3698,7 +3705,7 @@ function validate_instance_compression { run_buildah build $WITH_POLICY_JSON -t ${target} ${layers} ${compat} $BUDFILES/volume-perms run_buildah from --quiet $WITH_POLICY_JSON ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output if test "$compat" != "" ; then # true, /vol/subvol should not have contents, and its permissions should be the default 0755 @@ -3715,6 +3722,7 @@ function validate_instance_compression { assert "$status" -eq 0 "status code from stat $root/vol/subvol" expect_output "711" "stat($root/vol/subvol)" fi + run_buildah_umount ${cid} run_buildah rm ${cid} run_buildah rmi ${target} done @@ -3760,7 +3768,7 @@ function validate_instance_compression { run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile2.glob $BUDFILES/from-multiple-files run_buildah from --quiet $WITH_POLICY_JSON ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output cmp $root/Dockerfile1.alpine $BUDFILES/from-multiple-files/Dockerfile1.alpine cmp $root/Dockerfile2.withfrom $BUDFILES/from-multiple-files/Dockerfile2.withfrom @@ -3833,7 +3841,7 @@ function validate_instance_compression { run_buildah build $WITH_POLICY_JSON -t ${target} $BUDFILES/symlink run_buildah from --quiet $WITH_POLICY_JSON ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output run ls $root/data/log assert "$status" -eq 0 "status from ls $root/data/log" @@ -3852,7 +3860,7 @@ function validate_instance_compression { run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile.relative-symlink $BUDFILES/symlink run_buildah from --quiet $WITH_POLICY_JSON ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output run ls $root/log assert "$status" -eq 0 "status from ls $root/log" @@ -3870,7 +3878,7 @@ function validate_instance_compression { run_buildah build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/symlink/Dockerfile.multiple-symlinks $BUDFILES/symlink run_buildah from --quiet $WITH_POLICY_JSON ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output run ls $root/data/log assert "$status" -eq 0 "status from ls $root/data/log" @@ -3903,7 +3911,7 @@ function validate_instance_compression { run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile.absolute-symlink $BUDFILES/symlink run_buildah from --quiet $WITH_POLICY_JSON ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output run ls $root/bin assert "$status" -eq 0 "status from ls $root/bin" @@ -3920,7 +3928,7 @@ function validate_instance_compression { run_buildah build $WITH_POLICY_JSON -t ${target} -f Dockerfile.absolute-dir-symlink $BUDFILES/symlink run_buildah from --quiet $WITH_POLICY_JSON ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output run ls $root/data assert "$status" -eq 0 "status from ls $root/data" @@ -4077,42 +4085,44 @@ function validate_instance_compression { run_buildah from ${target} } -@test "bud-onbuild" { +@test "bud-onbuild-without-layers" { _prefetch alpine - target=onbuild - run_buildah build --format docker $WITH_POLICY_JSON -t ${target} $BUDFILES/onbuild - run_buildah inspect --format '{{printf "%q" .Docker.Config.OnBuild}}' ${target} + run_buildah build --format docker $WITH_POLICY_JSON --iidfile ${TEST_SCRATCH_DIR}/iid.txt $BUDFILES/onbuild + local iid=$(<${TEST_SCRATCH_DIR}/iid.txt) + run_buildah inspect --format '{{printf "%q" .Docker.Config.OnBuild}}' ${iid} expect_output '["RUN touch /onbuild1" "RUN touch /onbuild2"]' - run_buildah from --quiet ${target} - cid=${lines[0]} - run_buildah mount ${cid} - root=$output + run_buildah from --cidfile ${TEST_SCRATCH_DIR}/cid.txt ${iid} + local cid=$(<${TEST_SCRATCH_DIR}/cid.txt) + run_buildah_mount ${cid} + root=$output test -e ${root}/onbuild1 test -e ${root}/onbuild2 + run_buildah_umount ${cid} - run_buildah umount ${cid} - run_buildah rm ${cid} - - target=onbuild-image2 - run_buildah build --format docker $WITH_POLICY_JSON -t ${target} -f Dockerfile1 $BUDFILES/onbuild - run_buildah inspect --format '{{printf "%q" .Docker.Config.OnBuild}}' ${target} + run_buildah build --format docker $WITH_POLICY_JSON --iidfile ${TEST_SCRATCH_DIR}/iid.txt --from ${iid} -f Dockerfile1 $BUDFILES/onbuild + local iid=$(<${TEST_SCRATCH_DIR}/iid.txt) + run_buildah inspect --format '{{printf "%q" .Docker.Config.OnBuild}}' ${iid} expect_output '["RUN touch /onbuild3"]' - run_buildah from --quiet ${target} - cid=${lines[0]} - run_buildah mount ${cid} - root=$output + run_buildah from --cidfile ${TEST_SCRATCH_DIR}/cid.txt ${iid} + local cid=$(<${TEST_SCRATCH_DIR}/cid.txt) + run_buildah_mount ${cid} + root=$output test -e ${root}/onbuild1 test -e ${root}/onbuild2 test -e ${root}/onbuild3 - run_buildah umount ${cid} + run_buildah_umount ${cid} + cp -v ${TEST_SCRATCH_DIR}/root/${STORAGE_DRIVER:-vfs}-containers/${cid}/userdata/buildah.json \ + ${TEST_SCRATCH_DIR}/root/${STORAGE_DRIVER:-vfs}-containers/${cid}/userdata/buildah.json.bak run_buildah config --onbuild "RUN touch /onbuild4" ${cid} + run_buildah inspect --format '{{printf "%q" .Docker.Config.OnBuild}}' ${cid} + expect_output '["RUN touch /onbuild4"]' - target=onbuild-image3 - run_buildah commit $WITH_POLICY_JSON --format docker ${cid} ${target} - run_buildah inspect --format '{{printf "%q" .Docker.Config.OnBuild}}' ${target} + run_buildah commit $WITH_POLICY_JSON --iidfile ${TEST_SCRATCH_DIR}/iid.txt --format docker ${cid} ${target} + local iid=$(<${TEST_SCRATCH_DIR}/iid.txt) + run_buildah inspect --format '{{printf "%q" .Docker.Config.OnBuild}}' ${iid} expect_output '["RUN touch /onbuild4"]' } @@ -4183,7 +4193,7 @@ _EOF cat $BUDFILES/from-multiple-files/Dockerfile1.scratch | run_buildah build $WITH_POLICY_JSON -t ${target} -f - $BUDFILES/from-multiple-files run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output test -s $root/Dockerfile1 } @@ -4488,7 +4498,7 @@ _EOF run_buildah containers -a expect_output --substring "test1" - run_buildah mount $ctr + run_buildah_mount $ctr mnt=$output test -e $mnt/1 test ! -e $mnt/2 @@ -4583,11 +4593,11 @@ _EOF run_buildah from $WITH_POLICY_JSON ${imgname} ctr=$(echo "$output" | tail -1) - run_buildah mount ${ctr} + run_buildah_mount ${ctr} mnt=$output run cat ${mnt}${path} expect_output --substring "$expected" "case: \"$c\"" - run_buildah unmount ${ctr} + run_buildah_umount ${ctr} done } @@ -4633,7 +4643,7 @@ _EOF run_buildah from --quiet $WITH_POLICY_JSON ${target} ctr=$output - run_buildah mount ${ctr} + run_buildah_mount ${ctr} mnt=$output newfile="/home/busyboxpodman/copied-testimage-id" @@ -4660,12 +4670,12 @@ _EOF run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root_single_job=$output run_buildah from --quiet ${target_mt} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root_multi_job=$output # Check that both the version with --jobs 1 and --jobs=N have the same number of files @@ -4687,7 +4697,7 @@ _EOF expect_output --substring "\[2/2] STEP 1/3: FROM alpine:latest AS mytarget" run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output test -e ${root}/2 test ! -e ${root}/3 @@ -4939,7 +4949,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} $BUDFILES/copy-workdir run_buildah from ${target} cid="$output" - run_buildah mount "${cid}" + run_buildah_mount "${cid}" root="$output" test -s "${root}"/file1.txt test -d "${root}"/subdir @@ -4954,7 +4964,7 @@ _EOF run_buildah build $WITH_POLICY_JSON --layers -t ${target} -f Dockerfile.2 $BUDFILES/copy-workdir run_buildah from ${target} cid="$output" - run_buildah mount "${cid}" + run_buildah_mount "${cid}" root="$output" test -d "${root}"/subdir test -s "${root}"/subdir/file1.txt @@ -5030,6 +5040,8 @@ _EOF } @test "bud-copy-dockerignore-hardlinks" { + skip_if_rootless_environment "unprivileged tests use sshfs which doesn't represent hard links" + target=image local contextdir=${TEST_SCRATCH_DIR}/hardlinks mkdir -p $contextdir/subdir @@ -5042,7 +5054,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} $contextdir run_buildah from ${target} ctrid="$output" - run_buildah mount "$ctrid" + run_buildah_mount "$ctrid" root="$output" run stat -c "%d:%i" ${root}/subdir/test1.txt @@ -5240,7 +5252,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} - < $BUDFILES/context-from-stdin/Dockerfile run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output test -s $root/scratchfile @@ -5258,7 +5270,7 @@ _EOF run_buildah build $WITH_POLICY_JSON -t ${target} - < <(tar -c -C $BUDFILES/context-from-stdin .) run_buildah from --quiet ${target} cid=$output - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$output test -s $root/scratchfile @@ -7133,12 +7145,14 @@ _EOF } @test "bud-copy--parents-links" { + skip_if_rootless_environment "unprivileged tests use sshfs which doesn't represent hard links" + target=image run_buildah build $WITH_POLICY_JSON -t ${target} -f $BUDFILES/copy-parents/Containerfile-hardlinks $BUDFILES/copy-parents run_buildah from ${target} ctrid="$output" - run_buildah mount "$ctrid" + run_buildah_mount "$ctrid" root="$output" for dir in parents/x parents_dir_point ;do @@ -9706,7 +9720,7 @@ _EOF run_buildah from --quiet oci:${ocidir} # Extract just the container ID (last line), ignoring warning messages cid=$(echo "$output" | tail -1) - run_buildah mount ${cid} + run_buildah_mount ${cid} root=$(echo "$output" | tail -1) # Verify the Windows-specific directory structure was created @@ -9785,7 +9799,7 @@ _EOF done # Cleanup - run_buildah umount ${cid} + run_buildah_umount ${cid} run_buildah rm ${cid} } diff --git a/tests/commit.bats b/tests/commit.bats index 9c80a3fdc8a..f80ff7bcea9 100644 --- a/tests/commit.bats +++ b/tests/commit.bats @@ -34,10 +34,10 @@ load helpers run_buildah from scratch CONT=$output unset XDG_RUNTIME_DIR - run_buildah mount $CONT + run_buildah_mount $CONT MNT=$output run cp ${TEST_SCRATCH_DIR}/file $MNT/file - run_buildah umount $CONT + run_buildah_umount $CONT run_buildah commit $CONT foo run_buildah images foo expect_output --substring "10.5 MB" @@ -386,7 +386,7 @@ load helpers # first one should have the first file and not the second, and the shell should be there run_buildah from --quiet --pull=false $WITH_POLICY_JSON with-random-1 cid=$output - run_buildah mount $cid + run_buildah_mount $cid mountpoint=$output test -s $mountpoint/bin/sh || test -L $mountpoint/bin/sh cmp ${TEST_SCRATCH_DIR}/randomfile1 $mountpoint/randomfile1 @@ -401,7 +401,7 @@ load helpers # second one should have the second file and not the first, and the shell should be there run_buildah from --quiet --pull=false $WITH_POLICY_JSON with-random-2 cid=$output - run_buildah mount $cid + run_buildah_mount $cid mountpoint=$output test -s $mountpoint/bin/sh || test -L $mountpoint/bin/sh cmp ${TEST_SCRATCH_DIR}/randomfile2 $mountpoint/in-a-subdir/randomfile2 @@ -416,7 +416,7 @@ load helpers # third one should have both files, and the shell should be there run_buildah from --quiet --pull=false $WITH_POLICY_JSON with-random-both cid=$output - run_buildah mount $cid + run_buildah_mount $cid mountpoint=$output test -s $mountpoint/bin/sh || test -L $mountpoint/bin/sh cmp ${TEST_SCRATCH_DIR}/randomfile1 $mountpoint/randomfile1 diff --git a/tests/config.bats b/tests/config.bats index 455d4b431a1..1721bfe92bf 100644 --- a/tests/config.bats +++ b/tests/config.bats @@ -220,7 +220,7 @@ function check_matrix() { check_matrix 'Config.Labels.LABEL' '' } -@test "config" { +@test "config flags" { run_buildah from $WITH_POLICY_JSON scratch cid=$output run_buildah config \ @@ -370,7 +370,7 @@ EOF expect_output --substring "combined=bar/bar1" } -@test "user" { +@test "config user" { _prefetch alpine run_buildah from --quiet --pull $WITH_POLICY_JSON alpine cid=$output @@ -387,7 +387,7 @@ EOF expect_output "$bndoutput" } -@test "remove configs using '-' syntax" { +@test "config remove configs using '-' syntax" { run_buildah from $WITH_POLICY_JSON scratch cid=$output run_buildah config \ diff --git a/tests/copy.bats b/tests/copy.bats index a113d0992d2..72040270546 100644 --- a/tests/copy.bats +++ b/tests/copy.bats @@ -20,8 +20,6 @@ load helpers run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid - root=$output run_buildah config --workingdir / $cid # copy ${TEST_SCRATCH_DIR}/randomfile to a file of the same name in the container's working directory run_buildah copy --retry 4 --retry-delay 4s $cid ${TEST_SCRATCH_DIR}/randomfile @@ -36,8 +34,6 @@ load helpers _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid=$output - run_buildah mount $cid - root=$output run_buildah config --workingdir / $cid run_buildah copy $cid ${TEST_SCRATCH_DIR}/randomfile run_buildah copy $cid ${TEST_SCRATCH_DIR}/other-randomfile ${TEST_SCRATCH_DIR}/third-randomfile ${TEST_SCRATCH_DIR}/randomfile /etc @@ -45,10 +41,11 @@ load helpers run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid=$output - run_buildah mount $cid - root=$output run_buildah config --workingdir / $cid run_buildah copy $cid "${TEST_SCRATCH_DIR}/*randomfile" /etc + + run_buildah_mount $cid + root=$output (cd ${TEST_SCRATCH_DIR}; for i in *randomfile; do cmp $i ${root}/etc/$i; done) } @@ -59,18 +56,16 @@ load helpers run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid - root=$output run_buildah config --workingdir / $cid run_buildah copy $cid ${TEST_SCRATCH_DIR}/randomfile run_buildah copy $cid ${TEST_SCRATCH_DIR}/other-randomfile - run_buildah unmount $cid + run_buildah_umount $cid run_buildah commit $WITH_POLICY_JSON $cid containers-storage:new-image run_buildah rm $cid run_buildah from --quiet $WITH_POLICY_JSON new-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -s $newroot/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $newroot/randomfile @@ -87,13 +82,13 @@ load helpers cid=$output run_buildah config --workingdir /container-subdir $cid run_buildah copy $cid ${TEST_SCRATCH_DIR}/subdir - run_buildah mount $cid + run_buildah copy $cid ${TEST_SCRATCH_DIR}/subdir /other-subdir + run_buildah_mount $cid root=$output test -s $root/container-subdir/randomfile cmp ${TEST_SCRATCH_DIR}/subdir/randomfile $root/container-subdir/randomfile test -s $root/container-subdir/other-randomfile cmp ${TEST_SCRATCH_DIR}/subdir/other-randomfile $root/container-subdir/other-randomfile - run_buildah copy $cid ${TEST_SCRATCH_DIR}/subdir /other-subdir test -s $root/other-subdir/randomfile cmp ${TEST_SCRATCH_DIR}/subdir/randomfile $root/other-subdir/randomfile test -s $root/other-subdir/other-randomfile @@ -107,17 +102,18 @@ load helpers cid=$output run_buildah config --workingdir / $cid run_buildah copy $cid ${TEST_SCRATCH_DIR}/randomfile /randomfile - run_buildah mount $cid + run_buildah_mount $cid root=$output test -s $root/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $root/randomfile + run_buildah_umount $cid run_buildah rm $cid run_buildah from $WITH_POLICY_JSON scratch cid=$output run_buildah config --workingdir / $cid run_buildah copy $cid ${TEST_SCRATCH_DIR}/randomfile /randomsubdir/ - run_buildah mount $cid + run_buildah_mount $cid root=$output test -s $root/randomsubdir/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $root/randomsubdir/randomfile @@ -135,7 +131,7 @@ load helpers starthttpd ${TEST_SCRATCH_DIR} run_buildah copy $cid http://0.0.0.0:${HTTP_SERVER_PORT}/randomfile /urlfile stophttpd - run_buildah mount $cid + run_buildah_mount $cid root=$output test -s $root/urlfile cmp ${TEST_SCRATCH_DIR}/randomfile $root/urlfile @@ -259,17 +255,13 @@ load helpers run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid - root=$output run_buildah config --workingdir / $cid run_buildah copy $cid ${TEST_SCRATCH_DIR}/link-randomfile - run_buildah unmount $cid run_buildah commit $WITH_POLICY_JSON $cid containers-storage:new-image - run_buildah rm $cid run_buildah from --quiet $WITH_POLICY_JSON new-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -s $newroot/link-randomfile test -f $newroot/link-randomfile @@ -295,16 +287,16 @@ load helpers run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid + run_buildah_mount $cid root=$output run_buildah config --workingdir / $cid - run_buildah unmount $cid + run_buildah_umount $cid run_buildah commit $WITH_POLICY_JSON $cid containers-storage:new-image run_buildah rm $cid run_buildah from --quiet $WITH_POLICY_JSON new-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test \! -e $newroot/test.socket } @@ -315,17 +307,13 @@ load helpers run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid - root=$output run_buildah config --workingdir / $cid run_buildah copy $cid ${TEST_SCRATCH_DIR}/link-randomfile.tar.gz - run_buildah unmount $cid run_buildah commit $WITH_POLICY_JSON $cid containers-storage:new-image - run_buildah rm $cid run_buildah from --quiet $WITH_POLICY_JSON new-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output test -s $newroot/link-randomfile.tar.gz test -f $newroot/link-randomfile.tar.gz @@ -368,11 +356,11 @@ stuff/mystuff" run_buildah copy --contextdir=${mytest} --ignorefile ${mytest}/.ignore $cid ${mytest} /stuff - run_buildah mount $cid + run_buildah_mount $cid mnt=$output run find $mnt -printf "%P\n" filelist=$(LC_ALL=C sort <<<"$output") - run_buildah umount $cid + run_buildah_umount $cid expect_output --from="$filelist" "$expect" "container file list" } @@ -393,11 +381,11 @@ stuff/mystuff" run_buildah copy --exclude=**/*.go --exclude=.ignore --exclude=**/notmystuff $cid ${mytest} /stuff - run_buildah mount $cid + run_buildah_mount $cid mnt=$output run find $mnt -printf "%P\n" filelist=$(LC_ALL=C sort <<<"$output") - run_buildah umount $cid + run_buildah_umount $cid expect_output --from="$filelist" "$expect" "container file list" } @@ -455,11 +443,11 @@ parents/y/b.txt" run_buildah copy --parents $cid ${paths} /parents/ - run_buildah mount $cid + run_buildah_mount $cid mnt=$output run find $mnt -printf "%P\n" filelist=$(LC_ALL=C sort <<<"$output") - run_buildah umount $cid + run_buildah_umount $cid expect_output --from="$filelist" "$expect" "container file list" done < <(parse_table "$tests") } @@ -469,13 +457,11 @@ parents/y/b.txt" _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid=$output - run_buildah mount $cid - root=$output run_buildah copy --quiet $cid ${TEST_SCRATCH_DIR}/randomfile / expect_output "" + run_buildah_mount $cid + root=$output cmp ${TEST_SCRATCH_DIR}/randomfile $root/randomfile - run_buildah umount $cid - run_buildah rm $cid } @test "copy-from-container" { @@ -491,7 +477,7 @@ parents/y/b.txt" expect_output "" run_buildah copy --quiet $WITH_POLICY_JSON --from $from $cid tmp/random /tmp/random2 # relative path expect_output "" - run_buildah mount $cid + run_buildah_mount $cid croot=$output cmp ${TEST_SCRATCH_DIR}/randomfile ${croot}/tmp/random cmp ${TEST_SCRATCH_DIR}/randomfile ${croot}/tmp/random2 @@ -509,24 +495,24 @@ parents/y/b.txt" run_buildah copy --quiet $WITH_POLICY_JSON --from $from $cid / /tmp/ expect_output "" || \ expect_output --substring "copier: file disappeared while reading" - run_buildah mount $cid + run_buildah_mount $cid croot=$output cmp ${TEST_SCRATCH_DIR}/randomfile ${croot}/tmp/tmp/random } -@test "add-from-image" { +@test "copy-from-image" { _prefetch busybox run_buildah from --quiet $WITH_POLICY_JSON busybox cid=$output - run_buildah add $WITH_POLICY_JSON --quiet --from ubuntu $cid /etc/passwd /tmp/passwd # should pull the image, absolute path + run_buildah copy $WITH_POLICY_JSON --quiet --from ubuntu $cid /etc/passwd /tmp/passwd # should pull the image, absolute path expect_output "" - run_buildah add --quiet $WITH_POLICY_JSON --from ubuntu $cid etc/passwd /tmp/passwd2 # relative path + run_buildah copy --quiet $WITH_POLICY_JSON --from ubuntu $cid etc/passwd /tmp/passwd2 # relative path expect_output "" run_buildah from --quiet $WITH_POLICY_JSON ubuntu ubuntu=$output - run_buildah mount $cid + run_buildah_mount $cid croot=$output - run_buildah mount $ubuntu + run_buildah_mount $ubuntu ubuntu=$output cmp $ubuntu/etc/passwd ${croot}/tmp/passwd cmp $ubuntu/etc/passwd ${croot}/tmp/passwd2 @@ -688,15 +674,10 @@ parents/y/b.txt" run_buildah from $WITH_POLICY_JSON scratch cid=$output - run_buildah mount $cid - root=$output - run_buildah config --workingdir=/ $cid run_buildah copy --link $cid ${TEST_SCRATCH_DIR}/randomfile - - run_buildah copy --link $cid ${TEST_SCRATCH_DIR}/randomfile ${TEST_SCRATCH_DIR}/other-randomfile /subdir/ - run_buildah unmount $cid - run_buildah commit $WITH_POLICY_JSON $cid copy-link-image + run_buildah copy --link $cid ${TEST_SCRATCH_DIR}/randomfile ${TEST_SCRATCH_DIR}/other-randomfile /subdir/ + run_buildah commit --debug $WITH_POLICY_JSON $cid copy-link-image run_buildah inspect --type=image copy-link-image layers=$(echo "$output" | jq -r '.OCIv1.rootfs.diff_ids | length') @@ -704,9 +685,9 @@ parents/y/b.txt" run_buildah from $WITH_POLICY_JSON copy-link-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output - + test -s $newroot/randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $newroot/randomfile test -s $newroot/subdir/randomfile @@ -722,17 +703,17 @@ parents/y/b.txt" run_buildah from $WITH_POLICY_JSON scratch cid=$output - + run_buildah config --workingdir=/ $cid run_buildah copy --link $cid ${TEST_SCRATCH_DIR}/sourcedir /destdir - + run_buildah commit $WITH_POLICY_JSON $cid copy-link-dir-image run_buildah from $WITH_POLICY_JSON copy-link-dir-image newcid=$output - run_buildah mount $newcid + run_buildah_mount $newcid newroot=$output - + test -d $newroot/destdir test -s $newroot/destdir/file1 cmp ${TEST_SCRATCH_DIR}/sourcedir/file1 $newroot/destdir/file1 @@ -742,15 +723,15 @@ parents/y/b.txt" @test "copy-link-with-chown" { createrandom ${TEST_SCRATCH_DIR}/randomfile - + _prefetch busybox run_buildah from --quiet $WITH_POLICY_JSON busybox cid=$output - + run_buildah copy --link --chown bin:bin $cid ${TEST_SCRATCH_DIR}/randomfile /tmp/random - + run_buildah commit $WITH_POLICY_JSON $cid copy-link-chown-image - + run_buildah from $WITH_POLICY_JSON copy-link-chown-image newcid=$output run_buildah run $newcid ls -l /tmp/random @@ -759,15 +740,15 @@ parents/y/b.txt" @test "copy-link-with-chmod" { createrandom ${TEST_SCRATCH_DIR}/randomfile - + _prefetch busybox run_buildah from --quiet $WITH_POLICY_JSON busybox cid=$output - + run_buildah copy --link --chmod 777 $cid ${TEST_SCRATCH_DIR}/randomfile /tmp/random - + run_buildah commit $WITH_POLICY_JSON $cid copy-link-chmod-image - + run_buildah from $WITH_POLICY_JSON copy-link-chmod-image newcid=$output run_buildah run $newcid ls -l /tmp/random @@ -784,13 +765,13 @@ parents/y/b.txt" # Default (allow-wildcard=true): glob expands, copies both files run_buildah copy $cid "${TEST_SCRATCH_DIR}/file-*" /dest/ - run_buildah mount $cid + run_buildah_mount $cid root=$output test -f $root/dest/file-a test -f $root/dest/file-b cmp ${TEST_SCRATCH_DIR}/file-a $root/dest/file-a cmp ${TEST_SCRATCH_DIR}/file-b $root/dest/file-b - run_buildah umount $cid + run_buildah_umount $cid # allow-wildcard=false: glob patterns are rejected run_buildah 125 copy --allow-wildcard=false $cid "${TEST_SCRATCH_DIR}/file-*" /dest2/ @@ -810,17 +791,17 @@ parents/y/b.txt" # Explicit true: non-matching glob should succeed run_buildah copy --allow-empty-wildcard=true $cid "${TEST_SCRATCH_DIR}/nonexistent-*" /dest2/ - run_buildah mount $cid + run_buildah_mount $cid root=$output test ! -d $root/dest2 - run_buildah umount $cid + run_buildah_umount $cid # Existing files still copy normally run_buildah copy $cid ${TEST_SCRATCH_DIR}/file-a /dest3/ - run_buildah mount $cid + run_buildah_mount $cid root=$output test -f $root/dest3/file-a - run_buildah umount $cid + run_buildah_umount $cid # Literal non-existent file should still error even with allow-empty-wildcard=true run_buildah 125 copy --allow-empty-wildcard=true $cid ${TEST_SCRATCH_DIR}/no-such-file /dest4/ diff --git a/tests/formats.bats b/tests/formats.bats index afa9dbc904a..c7bc40cb77c 100644 --- a/tests/formats.bats +++ b/tests/formats.bats @@ -10,13 +10,13 @@ function check_imgtype() { image="$1" # Second argument: expected image type, 'oci' or 'docker' - imgtype_oci="application/vnd.oci.image.manifest.v1+json" - imgtype_dkr="application/vnd.docker.distribution.manifest.v2+json" + local imgtype_oci="application/vnd.oci.image.manifest.v1+json" + local imgtype_dkr="application/vnd.docker.distribution.manifest.v2+json" expect="" case "$2" in - oci) want=$imgtype_oci; reject=$imgtype_dkr;; - docker) want=$imgtype_dkr; reject=$imgtype_oci;; + oci) local want=$imgtype_oci; local reject=$imgtype_dkr;; + docker) local want=$imgtype_dkr; local reject=$imgtype_oci;; *) die "Internal error: unknown image type '$2'";; esac @@ -46,7 +46,6 @@ function check_imgtype() { @test "write-formats" { - skip_if_rootless_environment run_buildah from --pull=false $WITH_POLICY_JSON scratch cid=$output run_buildah commit $WITH_POLICY_JSON $cid scratch-image-default @@ -59,7 +58,6 @@ function check_imgtype() { } @test "bud-formats" { - skip_if_rootless_environment run_buildah build-using-dockerfile $WITH_POLICY_JSON -t scratch-image-default -f Containerfile $BUDFILES/from-scratch run_buildah build-using-dockerfile --format docker $WITH_POLICY_JSON -t scratch-image-docker -f Containerfile $BUDFILES/from-scratch run_buildah build-using-dockerfile --format oci $WITH_POLICY_JSON -t scratch-image-oci -f Containerfile $BUDFILES/from-scratch diff --git a/tests/helpers.bash b/tests/helpers.bash index ccf982164b6..7a64d8345b9 100644 --- a/tests/helpers.bash +++ b/tests/helpers.bash @@ -13,6 +13,7 @@ CRASH_BINARY=${CRASH_BINARY:-$TEST_SOURCES/../bin/crash} WAIT_BINARY=${WAIT_BINARY:-$TEST_SOURCES/../bin/wait} PASSWD_BINARY=${PASSWD_BINARY:-$TEST_SOURCES/../bin/passwd} GRPCNOOP_BINARY=${GRPCNOOP_BINARY:-$TEST_SOURCES/../bin/grpcnoop} +PIPELOOP_BINARY=${PIPELOOP_BINARY:-$TEST_SOURCES/../bin/pipeloop} STORAGE_DRIVER=${STORAGE_DRIVER:-vfs} PATH=$(dirname ${BASH_SOURCE})/../bin:${PATH} # Default timeout for a buildah command. @@ -384,14 +385,6 @@ function run_buildah() { # Remember command args, for possible use in later diagnostic messages MOST_RECENT_BUILDAH_COMMAND="buildah $*" - # If session is rootless and `buildah mount` is invoked, perform unshare, - # since normal user cannot mount a filesystem unless they're in a user namespace along with its own mount namespace. - if is_rootless; then - if [[ "$1" =~ mount ]]; then - set "unshare" "$BUILDAH_BINARY" ${BUILDAH_REGISTRY_OPTS} ${ROOTDIR_OPTS} "$@" - fi - fi - while [ $retry -gt 0 ]; do retry=$(( retry - 1 )) @@ -437,6 +430,41 @@ function run_buildah() { done } +####################### +# run_buildah_mount # Invoke buildah mount or buildah mount-sshfs, depending on UID +####################### +function run_buildah_mount() { + case "$1" in + [0-9]) expected_rc=$1; shift;; + [1-9][0-9]) expected_rc=$1; shift;; + [12][0-9][0-9]) expected_rc=$1; shift;; + '?') expected_rc='?' ; shift;; # ignore exit code + esac + if test $(id -u) -eq 0 ; then + run_buildah $expected_rc mount "$@" + else + skip_if_no_sshfs + run_buildah $expected_rc mount-sshfs -o no_contain_symlinks "$@" + fi +} + +######################## +# run_buildah_umount # Invoke buildah umount or buildah umount-sshfs, depending on UID +######################## +function run_buildah_umount() { + case "$1" in + [0-9]) expected_rc=$1; shift;; + [1-9][0-9]) expected_rc=$1; shift;; + [12][0-9][0-9]) expected_rc=$1; shift;; + '?') expected_rc='?' ; shift;; # ignore exit code + esac + if test $(id -u) -eq 0 ; then + run_buildah $expected_rc umount "$@" + else + run_buildah $expected_rc umount-sshfs "$@" + fi +} + ###################### # find_oci_runtime # Sets $OCI to the command for the OCI runtime, for invoking it directly ###################### @@ -687,6 +715,18 @@ function skip_if_no_podman() { fi } +###################### +# skip_if_no_sshfs # we need 'sshfs' to mount things as an unprivileged user +###################### +function skip_if_no_sshfs() { + if ! [ -c /dev/fuse ]; then + skip "/dev/fuse is not available for sshfs" + fi + run which ${SSHFS_BINARY:-sshfs} + if [[ $status -ne 0 ]]; then + skip "sshfs is not installed" + fi +} ########################## # skip_if_in_container # diff --git a/tests/mkcw.bats b/tests/mkcw.bats index d3a9c660782..ea95fdf5432 100644 --- a/tests/mkcw.bats +++ b/tests/mkcw.bats @@ -7,7 +7,7 @@ function mkcw_check_image() { # Mount the container and take a look at what it got from the image. run_buildah from "$imageID" local ctrID="$output" - run_buildah mount "$ctrID" + run_buildah_mount "$ctrID" local mountpoint="$output" # Should have a /disk.img file. test -s "$mountpoint"/disk.img @@ -50,7 +50,7 @@ function mkcw_check_image() { # shows nothing but `dmsetup info -c $uuid` shows the device is still # under use. Adding `--retry` in between somehow fixes this. run_with_log --retry cryptsetup luksClose "$uuid" - run_buildah umount "$ctrID" + run_buildah_umount "$ctrID" } @test "mkcw-convert" { diff --git a/tests/mount.bats b/tests/mount.bats index d267ce97ac2..2a8b59722a5 100644 --- a/tests/mount.bats +++ b/tests/mount.bats @@ -17,7 +17,7 @@ load helpers _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid=$output - run_buildah mount "$cid" + run_buildah_mount "$cid" } @test "mount bad container" { @@ -32,7 +32,7 @@ load helpers cid2=$output run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid3=$output - run_buildah mount "$cid1" "$cid2" "$cid3" + run_buildah_mount "$cid1" "$cid2" "$cid3" } @test "mount multi images one bad" { @@ -50,14 +50,14 @@ load helpers _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid1=$output - run_buildah mount "$cid1" + run_buildah_mount "$cid1" run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid2=$output - run_buildah mount "$cid2" + run_buildah_mount "$cid2" run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid3=$output - run_buildah mount "$cid3" - run_buildah mount + run_buildah_mount "$cid3" + run_buildah_mount expect_line_count 3 expect_output --from="${lines[0]}" --substring "/tmp" "mount line 1 of 3" expect_output --from="${lines[1]}" --substring "/tmp" "mount line 2 of 3" diff --git a/tests/namespaces.bats b/tests/namespaces.bats index 37998c5c745..18b072932d6 100644 --- a/tests/namespaces.bats +++ b/tests/namespaces.bats @@ -231,7 +231,7 @@ idmapping_check_permission() { expect_output "0:0 4700" "stat(someotherfile), in container test" # Check that the copied file has the right permissions on host. - run_buildah mount "$ctr" + run_buildah_mount "$ctr" mnt="$output" run stat -c '%u:%g %a' "$mnt"/somedir/someotherfile assert "$status" -eq 0 "status of stat $mnt/somedir/someotherfile" @@ -447,7 +447,7 @@ _EOF run_buildah commit --squash $WITH_POLICY_JSON --rm "$cid" squashed run_buildah from --quiet squashed cid=$output - run_buildah mount $cid + run_buildah_mount $cid mountpoint=$output run stat -c %u:%g $mountpoint/randomfile [ "$status" -eq 0 ] diff --git a/tests/overlay.bats b/tests/overlay.bats index ab2066bbe55..e64427e0191 100644 --- a/tests/overlay.bats +++ b/tests/overlay.bats @@ -33,9 +33,9 @@ load helpers } @test "overlay source permissions and owners" { - if test \! -e /usr/bin/fuse-overlayfs -a "$BUILDAH_ISOLATION" = "rootless"; then - skip "BUILDAH_ISOLATION = $BUILDAH_ISOLATION" and no /usr/bin/fuse-overlayfs present - elif test "$STORAGE_DRIVER" = "vfs"; then + skip_if_rootless_environment "unprivileged tests can't easily change ownership of test directories" + + if test "$STORAGE_DRIVER" = "vfs"; then skip "skipping overlay test because \$STORAGE_DRIVER = $STORAGE_DRIVER" fi image=alpine @@ -55,7 +55,7 @@ load helpers run_buildah run $cid touch /tmp/test/bar run_buildah run $cid rm /tmp/test/foo - # This should fail, second runs of containers go back to original + # This should fail, second runs of containers get a fresh overlay run_buildah 1 run $cid ls /tmp/test/bar # This should fail since /tmp/test was an overlay, not a bind mount diff --git a/tests/pipeloop/README.md b/tests/pipeloop/README.md new file mode 100644 index 00000000000..441699a3a07 --- /dev/null +++ b/tests/pipeloop/README.md @@ -0,0 +1,10 @@ +This is directly inspired by dpipe(1). + +dpipe's documentation describes running two processes with their respective +stdin and stdout descriptors tied together so that one process's output is fed +into the other and vice-versa. + +pipeloop attempts to generalize this to cases where there are two _or more_ +such processes. Each process has its stdout connected to the next process's +stdin, and the last has its stdout connected to the stdin of the first process, +forming a complete loop. When one of them exits, the others are killed. diff --git a/tests/pipeloop/pipeloop.go b/tests/pipeloop/pipeloop.go new file mode 100644 index 00000000000..284f23d4aa8 --- /dev/null +++ b/tests/pipeloop/pipeloop.go @@ -0,0 +1,90 @@ +package main + +import ( + "errors" + "fmt" + "os" + "os/exec" + "sync" + "syscall" + + "github.com/spf13/cobra" + "go.podman.io/buildah/pkg/cli" +) + +var pipeloopCmd = &cobra.Command{ + Use: "pipeloop 'CMD' [...]", + Long: "A pipeline that loops around to the start", + RunE: func(_ *cobra.Command, args []string) error { + n := len(args) + commands := make([]*exec.Cmd, 0, n) + pipes := make([][2]*os.File, n) + results := make([]error, n) + var err error + for i := range n { + pipes[i][0], pipes[i][1], err = os.Pipe() + if err != nil { + return err + } + } + for i := range n { + cmd := exec.Command("sh", "-c", args[i]) + cmd.Stdin = pipes[(i-1+n)%n][0] + cmd.Stdout = pipes[i][1] + cmd.Stderr = os.Stderr + commands = append(commands, cmd) + } + var cmds sync.WaitGroup + cleanup := make(chan int, len(commands)) + for i := range commands { + if err := commands[i].Start(); err != nil { + results[i] = fmt.Errorf("%s: %w", args[i], err) + } + } + for i := range pipes { + if err := pipes[i][0].Close(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + if err := pipes[i][1].Close(); err != nil { + fmt.Fprintln(os.Stderr, err) + } + } + cmds.Go(func() { + if firstToExit, ok := <-cleanup; ok { + for i := range commands { + if i != firstToExit { + if err := commands[i].Process.Signal(syscall.SIGTERM); err != nil { + fmt.Fprintln(os.Stderr, err) + } + } + } + } + }) + for i := range commands { + cmds.Go(func() { + if err := commands[i].Wait(); err != nil { + results[i] = fmt.Errorf("%s: %w", args[i], err) + } + cleanup <- i + }) + } + cmds.Wait() + return errors.Join(results...) + }, + Args: cobra.MinimumNArgs(1), + SilenceErrors: true, +} + +func main() { + var exitCode int + if err := pipeloopCmd.Execute(); err != nil { + exitCode = cli.ExecErrorCodeGeneric + var ee *exec.ExitError + if errors.As(err, &ee) { + if w, ok := ee.Sys().(syscall.WaitStatus); ok { + exitCode = w.ExitStatus() + } + } + } + os.Exit(exitCode) +} diff --git a/tests/rmi.bats b/tests/rmi.bats index df42b27980c..06f2955b879 100644 --- a/tests/rmi.bats +++ b/tests/rmi.bats @@ -85,19 +85,19 @@ load helpers run_buildah images -q expect_line_count 1 - run_buildah mount $cid + run_buildah_mount $cid root=$output cp ${TEST_SCRATCH_DIR}/randomfile $root/randomfile - run_buildah unmount $cid + run_buildah_umount $cid run_buildah commit $WITH_POLICY_JSON $cid containers-storage:new-image run_buildah images -q expect_line_count 2 - run_buildah mount $cid + run_buildah_mount $cid root=$output cp ${TEST_SCRATCH_DIR}/other-randomfile $root/other-randomfile - run_buildah unmount $cid + run_buildah_umount $cid run_buildah commit $WITH_POLICY_JSON $cid containers-storage:new-image run_buildah images -q @@ -123,10 +123,10 @@ load helpers run_buildah images -q -a expect_line_count 0 - run_buildah mount $cid + run_buildah_mount $cid root=$output cp ${TEST_SCRATCH_DIR}/randomfile $root/randomfile - run_buildah unmount $cid + run_buildah_umount $cid run_buildah commit --quiet $WITH_POLICY_JSON $cid image=$output run_buildah rm $cid @@ -136,10 +136,10 @@ load helpers run_buildah from --quiet $WITH_POLICY_JSON $image cid=$output - run_buildah mount $cid + run_buildah_mount $cid root=$output cp ${TEST_SCRATCH_DIR}/other-randomfile $root/other-randomfile - run_buildah unmount $cid + run_buildah_umount $cid run_buildah commit $WITH_POLICY_JSON $cid run_buildah rm $cid diff --git a/tests/run.bats b/tests/run.bats index 2775b7aafe8..2c9bcfc4887 100644 --- a/tests/run.bats +++ b/tests/run.bats @@ -2,7 +2,7 @@ load helpers -@test "run" { +@test "run-basic" { skip_if_no_runtime _prefetch alpine @@ -12,19 +12,20 @@ load helpers ${OCI} --version createrandom ${TEST_SCRATCH_DIR}/randomfile run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine - cid=$output - run_buildah mount $cid - root=$output + local cid=$output run_buildah config --workingdir /tmp $cid run_buildah run $cid pwd expect_output "/tmp" run_buildah config --workingdir /root $cid - run_buildah run $cid pwd + run_buildah run $cid pwd expect_output "/root" - cp ${TEST_SCRATCH_DIR}/randomfile $root/tmp/ - run_buildah run $cid cp /tmp/randomfile /tmp/other-randomfile + run_buildah add $cid ${TEST_SCRATCH_DIR}/randomfile $root/tmp/ + run_buildah run $cid cp /tmp/randomfile /tmp/other-randomfile + run_buildah_mount $cid + local root=$output test -s $root/tmp/other-randomfile cmp ${TEST_SCRATCH_DIR}/randomfile $root/tmp/other-randomfile + run_buildah_umount $cid seq 100000 | buildah run $cid -- sh -c 'while read i; do echo $i; done' } @@ -194,7 +195,7 @@ function configure_and_check_user() { _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid=$output - run_buildah mount $cid + run_buildah_mount $cid root=$output testuser=jimbo @@ -206,6 +207,7 @@ function configure_and_check_user() { testgroupid=$(random_unused_id $root/etc/group) echo "$testuser:x:$testuid:$testgid:Jimbo Jenkins:/home/$testuser:/bin/sh" >> $root/etc/passwd echo "$testgroup:x:$testgroupid:" >> $root/etc/group + run_buildah_umount $cid configure_and_check_user "" 0 0 configure_and_check_user "${testuser}" $testuid $testgid @@ -223,7 +225,8 @@ function configure_and_check_user() { run_buildah 125 run -- $cid id -g expect_output --substring "unknown user" "id -g (bogus user)" - ln -vsf /etc/passwd $root/etc/passwd + run_buildah config -u root:root $cid + run_buildah run -- $cid ln -vsf /etc/passwd /etc/passwd run_buildah config -u ${testuser}:${testgroup} $cid run_buildah 125 run -- $cid id -u echo "$output" @@ -594,20 +597,25 @@ function configure_and_check_user() { @test "run-builtin-volume-omitted" { # This image is known to include a volume, but not include the mountpoint # in the image. - _prefetch quay.io/libpod/registry:volume_omitted - run_buildah from --quiet --cidfile ${TEST_SCRATCH_DIR}/cid --pull=ifmissing $WITH_POLICY_JSON quay.io/libpod/registry:volume_omitted + image=quay.io/libpod/registry:volume_omitted + _prefetch $image + run_buildah from --cidfile ${TEST_SCRATCH_DIR}/cid --pull=ifmissing $WITH_POLICY_JSON $image cid="$(< ${TEST_SCRATCH_DIR}/cid)" - run_buildah mount $cid + + # Initially, the volume's mountpoint should not be there. + run_buildah_mount $cid mnt=$output - # By default, the mountpoint should not be there. run test -d "$mnt"/var/lib/registry echo "$output" [ "$status" -ne 0 ] - # We'll create the mountpoint for "run". + run_buildah_umount $cid + + # This will create the volume's mountpoint for a "run". run_buildah run $cid ls -1 /var/lib - expect_output --substring "registry" + expect_output --substring "registry" - # Double-check that the mountpoint is there. + # Double-check that the mountpoint is there now. + run_buildah_mount $cid test -d "$mnt"/var/lib/registry } @@ -824,7 +832,7 @@ $output" assert "$output" !~ "^nameserver 127." "Container contains local nameserver" assert "$nameservers" "Container nameservers match correct host nameservers" if ! is_rootless; then - run_buildah mount $cid + run_buildah_mount $cid assert "$output" != "" if test -e "$output/etc/resolv.conf" ; then assert "$(< $output/etc/resolv.conf)" = "" "resolv.conf is empty" @@ -841,7 +849,7 @@ $output" run_buildah run --isolation=chroot --network=host $cid grep nameserver /etc/resolv.conf assert "$nameservers" "Container nameservers match the host nameservers" if ! is_rootless; then - run_buildah mount $cid + run_buildah_mount $cid assert "$output" != "" if test -e "$output/etc/resolv.conf" ; then assert "$(< $output/etc/resolv.conf)" = "" "resolv.conf is empty" diff --git a/tests/sbom.bats b/tests/sbom.bats index d8d019563ad..4c20f5e5e49 100644 --- a/tests/sbom.bats +++ b/tests/sbom.bats @@ -26,7 +26,7 @@ load helpers # compare them to their equivalents in the image run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine-derived-image dcid=$output - run_buildah mount $dcid + run_buildah_mount $dcid mountpoint=$output cmp $mountpoint/root/purl.json ${TEST_SCRATCH_DIR}/localpurl.json cmp $mountpoint/root/sbom.json ${TEST_SCRATCH_DIR}/localsbom.json @@ -56,7 +56,7 @@ load helpers # compare them to their equivalents in the image run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine-derived-image dcid=$output - run_buildah mount $dcid + run_buildah_mount $dcid mountpoint=$output cmp $mountpoint/root/purl.json ${TEST_SCRATCH_DIR}/localpurl.json cmp $mountpoint/root/sbom.json ${TEST_SCRATCH_DIR}/localsbom.json diff --git a/tests/sftp.bats b/tests/sftp.bats new file mode 100644 index 00000000000..be8fac9e294 --- /dev/null +++ b/tests/sftp.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats + +load helpers + +@test "sftp-server" { + skip_if_no_sshfs + + _prefetch busybox + + local mountpoint=${TEST_SCRATCH_DIR}/mountpoint + mkdir -p ${mountpoint} + + run_buildah from --cidfile ${TEST_SCRATCH_DIR}/cid.txt $WITH_POLICY_JSON busybox + local cid=$(< ${TEST_SCRATCH_DIR}/cid.txt) + + run_buildah info | tee ${TEST_SCRATCH_DIR}/info.txt + + coproc "${PIPELOOP_BINARY}" "${BUILDAH_BINARY} ${BUILDAH_REGISTRY_OPTS} ${ROOTDIR_OPTS} serve-sftp ${cid}" "sshfs -f -o passive :/ ${mountpoint}" + waited=0 + while ! test -s ${mountpoint}/bin/id ; do + if test ${waited} -gt 30 ; then + kill $COPROC_PID + return + fi + sleep 1 + waited=$(( waited++ )) + done + + run ${mountpoint}/bin/id -u + echo "$output" + assert "$result" -eq 0 + assert "$output" -eq $UID + + umount -l ${mountpoint} +} diff --git a/tests/squash.bats b/tests/squash.bats index 35608666888..b99cbc42575 100644 --- a/tests/squash.bats +++ b/tests/squash.bats @@ -26,9 +26,10 @@ function check_lengths() { run_buildah copy "$cid" ${TEST_SCRATCH_DIR}/randomfile /layer${stage} image=stage${stage} if test $stage -eq ${remove[0]} ; then - run_buildah mount "$cid" + run_buildah_mount "$cid" mountpoint=$output rm -f ${mountpoint}/layer${remove[1]} + run_buildah_umount "$cid" fi run_buildah commit $WITH_POLICY_JSON --rm "$cid" ${image} check_lengths $image $stage @@ -41,7 +42,7 @@ function check_lengths() { run_buildah from --quiet squashed cid=$output - run_buildah mount $cid + run_buildah_mount $cid mountpoint=$output for stage in $(seq 10) ; do if test $stage -eq ${remove[1]} ; then @@ -80,7 +81,7 @@ function check_lengths() { run_buildah from --quiet squashed cid=$output - run_buildah mount $cid + run_buildah_mount $cid mountpoint=$output for stage in $(seq 10) ; do cmp $mountpoint/layer${stage} ${TEST_SCRATCH_DIR}/randomfile diff --git a/tests/test_runner.sh b/tests/test_runner.sh index d4ea467c92f..d81aa51659c 100755 --- a/tests/test_runner.sh +++ b/tests/test_runner.sh @@ -12,4 +12,4 @@ function execute() { } # Run the tests. -execute time bats -j $(nproc) --tap "${@:-.}" +execute time bats -j $(nproc) --tap --timing "${@:-.}" diff --git a/tests/tmt/system.fmf b/tests/tmt/system.fmf index dc84a01a7c0..91aee62ff85 100644 --- a/tests/tmt/system.fmf +++ b/tests/tmt/system.fmf @@ -13,6 +13,7 @@ environment: GRPCNOOP_BINARY: /usr/bin/buildah-grpcnoop WAIT_BINARY: /usr/bin/buildah-wait CRASH_BINARY: /usr/bin/buildah-crash + PIPELOOP_BINARY: /usr/bin/buildah-pipeloop TMPDIR: /var/tmp adjust: diff --git a/tests/umount.bats b/tests/umount.bats index d59928e7942..5da203cd9b2 100644 --- a/tests/umount.bats +++ b/tests/umount.bats @@ -3,13 +3,13 @@ load helpers @test "umount-flags-order-verification" { - run_buildah 125 umount cnt1 -a + run_buildah_umount 125 cnt1 -a check_options_flag_err "-a" - run_buildah 125 umount cnt1 --all cnt2 + run_buildah_umount 125 cnt1 --all cnt2 check_options_flag_err "--all" - run_buildah 125 umount cnt1 cnt2 --all + run_buildah_umount 125 cnt1 cnt2 --all check_options_flag_err "--all" } @@ -17,52 +17,52 @@ load helpers _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid=$output - run_buildah mount "$cid" - run_buildah umount "$cid" + run_buildah_mount "$cid" + run_buildah_umount "$cid" } @test "umount bad image" { - run_buildah 125 umount badcontainer + run_buildah_umount 125 badcontainer } @test "umount multi images" { _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid1=$output - run_buildah mount "$cid1" + run_buildah_mount "$cid1" run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid2=$output - run_buildah mount "$cid2" + run_buildah_mount "$cid2" run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid3=$output - run_buildah mount "$cid3" - run_buildah umount "$cid1" "$cid2" "$cid3" + run_buildah_mount "$cid3" + run_buildah_umount "$cid1" "$cid2" "$cid3" } @test "umount all images" { _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid1=$output - run_buildah mount "$cid1" + run_buildah_mount "$cid1" run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid2=$output - run_buildah mount "$cid2" + run_buildah_mount "$cid2" run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid3=$output - run_buildah mount "$cid3" - run_buildah umount --all + run_buildah_mount "$cid3" + run_buildah_umount --all } @test "umount multi images one bad" { _prefetch alpine run_buildah from --quiet --pull=false $WITH_POLICY_JSON alpine cid1=$output - run_buildah mount "$cid1" + run_buildah_mount "$cid1" run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid2=$output - run_buildah mount "$cid2" + run_buildah_mount "$cid2" run_buildah from --quiet --pull-never $WITH_POLICY_JSON alpine cid3=$output - run_buildah mount "$cid3" - run_buildah 125 umount "$cid1" badcontainer "$cid2" "$cid3" + run_buildah_mount "$cid3" + run_buildah_umount 125 "$cid1" badcontainer "$cid2" "$cid3" } diff --git a/vendor/github.com/kr/fs/LICENSE b/vendor/github.com/kr/fs/LICENSE new file mode 100644 index 00000000000..74487567632 --- /dev/null +++ b/vendor/github.com/kr/fs/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/kr/fs/Readme b/vendor/github.com/kr/fs/Readme new file mode 100644 index 00000000000..c95e13fc84f --- /dev/null +++ b/vendor/github.com/kr/fs/Readme @@ -0,0 +1,3 @@ +Filesystem Package + +http://godoc.org/github.com/kr/fs diff --git a/vendor/github.com/kr/fs/filesystem.go b/vendor/github.com/kr/fs/filesystem.go new file mode 100644 index 00000000000..f1c4805fbdb --- /dev/null +++ b/vendor/github.com/kr/fs/filesystem.go @@ -0,0 +1,36 @@ +package fs + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +// FileSystem defines the methods of an abstract filesystem. +type FileSystem interface { + + // ReadDir reads the directory named by dirname and returns a + // list of directory entries. + ReadDir(dirname string) ([]os.FileInfo, error) + + // Lstat returns a FileInfo describing the named file. If the file is a + // symbolic link, the returned FileInfo describes the symbolic link. Lstat + // makes no attempt to follow the link. + Lstat(name string) (os.FileInfo, error) + + // Join joins any number of path elements into a single path, adding a + // separator if necessary. The result is Cleaned; in particular, all + // empty strings are ignored. + // + // The separator is FileSystem specific. + Join(elem ...string) string +} + +// fs represents a FileSystem provided by the os package. +type fs struct{} + +func (f *fs) ReadDir(dirname string) ([]os.FileInfo, error) { return ioutil.ReadDir(dirname) } + +func (f *fs) Lstat(name string) (os.FileInfo, error) { return os.Lstat(name) } + +func (f *fs) Join(elem ...string) string { return filepath.Join(elem...) } diff --git a/vendor/github.com/kr/fs/walk.go b/vendor/github.com/kr/fs/walk.go new file mode 100644 index 00000000000..6ffa1e0b246 --- /dev/null +++ b/vendor/github.com/kr/fs/walk.go @@ -0,0 +1,95 @@ +// Package fs provides filesystem-related functions. +package fs + +import ( + "os" +) + +// Walker provides a convenient interface for iterating over the +// descendants of a filesystem path. +// Successive calls to the Step method will step through each +// file or directory in the tree, including the root. The files +// are walked in lexical order, which makes the output deterministic +// but means that for very large directories Walker can be inefficient. +// Walker does not follow symbolic links. +type Walker struct { + fs FileSystem + cur item + stack []item + descend bool +} + +type item struct { + path string + info os.FileInfo + err error +} + +// Walk returns a new Walker rooted at root. +func Walk(root string) *Walker { + return WalkFS(root, new(fs)) +} + +// WalkFS returns a new Walker rooted at root on the FileSystem fs. +func WalkFS(root string, fs FileSystem) *Walker { + info, err := fs.Lstat(root) + return &Walker{ + fs: fs, + stack: []item{{root, info, err}}, + } +} + +// Step advances the Walker to the next file or directory, +// which will then be available through the Path, Stat, +// and Err methods. +// It returns false when the walk stops at the end of the tree. +func (w *Walker) Step() bool { + if w.descend && w.cur.err == nil && w.cur.info.IsDir() { + list, err := w.fs.ReadDir(w.cur.path) + if err != nil { + w.cur.err = err + w.stack = append(w.stack, w.cur) + } else { + for i := len(list) - 1; i >= 0; i-- { + path := w.fs.Join(w.cur.path, list[i].Name()) + w.stack = append(w.stack, item{path, list[i], nil}) + } + } + } + + if len(w.stack) == 0 { + return false + } + i := len(w.stack) - 1 + w.cur = w.stack[i] + w.stack = w.stack[:i] + w.descend = true + return true +} + +// Path returns the path to the most recent file or directory +// visited by a call to Step. It contains the argument to Walk +// as a prefix; that is, if Walk is called with "dir", which is +// a directory containing the file "a", Path will return "dir/a". +func (w *Walker) Path() string { + return w.cur.path +} + +// Stat returns info for the most recent file or directory +// visited by a call to Step. +func (w *Walker) Stat() os.FileInfo { + return w.cur.info +} + +// Err returns the error, if any, for the most recent attempt +// by Step to visit a file or directory. If a directory has +// an error, w will not descend into that directory. +func (w *Walker) Err() error { + return w.cur.err +} + +// SkipDir causes the currently visited directory to be skipped. +// If w is not on a directory, SkipDir has no effect. +func (w *Walker) SkipDir() { + w.descend = false +} diff --git a/vendor/github.com/pkg/sftp/.gitignore b/vendor/github.com/pkg/sftp/.gitignore new file mode 100644 index 00000000000..caf2dca22e2 --- /dev/null +++ b/vendor/github.com/pkg/sftp/.gitignore @@ -0,0 +1,10 @@ +.*.swo +.*.swp + +server_standalone/server_standalone + +examples/*/id_rsa +examples/*/id_rsa.pub + +memprofile.out +memprofile.svg diff --git a/vendor/github.com/pkg/sftp/CONTRIBUTORS b/vendor/github.com/pkg/sftp/CONTRIBUTORS new file mode 100644 index 00000000000..5c7196ae6a7 --- /dev/null +++ b/vendor/github.com/pkg/sftp/CONTRIBUTORS @@ -0,0 +1,3 @@ +Dave Cheney +Saulius Gurklys +John Eikenberry diff --git a/vendor/github.com/pkg/sftp/LICENSE b/vendor/github.com/pkg/sftp/LICENSE new file mode 100644 index 00000000000..b7b53921e98 --- /dev/null +++ b/vendor/github.com/pkg/sftp/LICENSE @@ -0,0 +1,9 @@ +Copyright (c) 2013, Dave Cheney +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/pkg/sftp/Makefile b/vendor/github.com/pkg/sftp/Makefile new file mode 100644 index 00000000000..4d3a00799d7 --- /dev/null +++ b/vendor/github.com/pkg/sftp/Makefile @@ -0,0 +1,27 @@ +.PHONY: integration integration_w_race benchmark + +integration: + go test -integration -v ./... + go test -testserver -v ./... + go test -integration -testserver -v ./... + go test -integration -allocator -v ./... + go test -testserver -allocator -v ./... + go test -integration -testserver -allocator -v ./... + +integration_w_race: + go test -race -integration -v ./... + go test -race -testserver -v ./... + go test -race -integration -testserver -v ./... + go test -race -integration -allocator -v ./... + go test -race -testserver -allocator -v ./... + go test -race -integration -allocator -testserver -v ./... + +COUNT ?= 1 +BENCHMARK_PATTERN ?= "." + +benchmark: + go test -integration -run=NONE -bench=$(BENCHMARK_PATTERN) -benchmem -count=$(COUNT) + +benchmark_w_memprofile: + go test -integration -run=NONE -bench=$(BENCHMARK_PATTERN) -benchmem -count=$(COUNT) -memprofile memprofile.out + go tool pprof -svg -output=memprofile.svg memprofile.out diff --git a/vendor/github.com/pkg/sftp/README.md b/vendor/github.com/pkg/sftp/README.md new file mode 100644 index 00000000000..5e78cd3968f --- /dev/null +++ b/vendor/github.com/pkg/sftp/README.md @@ -0,0 +1,44 @@ +sftp +---- + +The `sftp` package provides support for file system operations on remote ssh +servers using the SFTP subsystem. It also implements an SFTP server for serving +files from the filesystem. + +![CI Status](https://github.com/pkg/sftp/workflows/CI/badge.svg?branch=master&event=push) [![Go Reference](https://pkg.go.dev/badge/github.com/pkg/sftp.svg)](https://pkg.go.dev/github.com/pkg/sftp) + +usage and examples +------------------ + +See [https://pkg.go.dev/github.com/pkg/sftp](https://pkg.go.dev/github.com/pkg/sftp) for +examples and usage. + +The basic operation of the package mirrors the facilities of the +[os](http://golang.org/pkg/os) package. + +The Walker interface for directory traversal is heavily inspired by Keith +Rarick's [fs](https://pkg.go.dev/github.com/kr/fs) package. + +roadmap +------- + +* There is way too much duplication in the Client methods. If there was an + unmarshal(interface{}) method this would reduce a heap of the duplication. + +contributing +------------ + +We welcome pull requests, bug fixes and issue reports. + +Before proposing a large change, first please discuss your change by raising an +issue. + +For API/code bugs, please include a small, self contained code example to +reproduce the issue. For pull requests, remember test coverage. + +We try to handle issues and pull requests with a 0 open philosophy. That means +we will try to address the submission as soon as possible and will work toward +a resolution. If progress can no longer be made (eg. unreproducible bug) or +stops (eg. unresponsive submitter), we will close the bug. + +Thanks. diff --git a/vendor/github.com/pkg/sftp/SECURITY.md b/vendor/github.com/pkg/sftp/SECURITY.md new file mode 100644 index 00000000000..9ebc2631ff5 --- /dev/null +++ b/vendor/github.com/pkg/sftp/SECURITY.md @@ -0,0 +1,13 @@ +# Security Policy + +## Supported Versions + +Security updates are provided for the latest released version of this package. +We also welcome vulnerability reports for the development version to help us ensure it is secure before the next release. + +## Reporting a Vulnerability + +If you believe you’ve found a security vulnerability in this project, we strongly encourage you to report it privately using GitHub’s [security advisory system](https://github.com/pkg/sftp/security/advisories/new). +This will allow us to review and address the issue before public disclosure. + +Thank you for helping us keep the project secure. diff --git a/vendor/github.com/pkg/sftp/allocator.go b/vendor/github.com/pkg/sftp/allocator.go new file mode 100644 index 00000000000..3e67e54335e --- /dev/null +++ b/vendor/github.com/pkg/sftp/allocator.go @@ -0,0 +1,96 @@ +package sftp + +import ( + "sync" +) + +type allocator struct { + sync.Mutex + available [][]byte + // map key is the request order + used map[uint32][][]byte +} + +func newAllocator() *allocator { + return &allocator{ + // micro optimization: initialize available pages with an initial capacity + available: make([][]byte, 0, SftpServerWorkerCount*2), + used: make(map[uint32][][]byte), + } +} + +// GetPage returns a previously allocated and unused []byte or create a new one. +// The slice have a fixed size = maxMsgLength, this value is suitable for both +// receiving new packets and reading the files to serve +func (a *allocator) GetPage(requestOrderID uint32) []byte { + a.Lock() + defer a.Unlock() + + var result []byte + + // get an available page and remove it from the available ones. + if len(a.available) > 0 { + truncLength := len(a.available) - 1 + result = a.available[truncLength] + + a.available[truncLength] = nil // clear out the internal pointer + a.available = a.available[:truncLength] // truncate the slice + } + + // no preallocated slice found, just allocate a new one + if result == nil { + result = make([]byte, maxMsgLength) + } + + // put result in used pages + a.used[requestOrderID] = append(a.used[requestOrderID], result) + + return result +} + +// ReleasePages marks unused all pages in use for the given requestID +func (a *allocator) ReleasePages(requestOrderID uint32) { + a.Lock() + defer a.Unlock() + + if used := a.used[requestOrderID]; len(used) > 0 { + a.available = append(a.available, used...) + } + delete(a.used, requestOrderID) +} + +// Free removes all the used and available pages. +// Call this method when the allocator is not needed anymore +func (a *allocator) Free() { + a.Lock() + defer a.Unlock() + + a.available = nil + a.used = make(map[uint32][][]byte) +} + +func (a *allocator) countUsedPages() int { + a.Lock() + defer a.Unlock() + + num := 0 + for _, p := range a.used { + num += len(p) + } + return num +} + +func (a *allocator) countAvailablePages() int { + a.Lock() + defer a.Unlock() + + return len(a.available) +} + +func (a *allocator) isRequestOrderIDUsed(requestOrderID uint32) bool { + a.Lock() + defer a.Unlock() + + _, ok := a.used[requestOrderID] + return ok +} diff --git a/vendor/github.com/pkg/sftp/attrs.go b/vendor/github.com/pkg/sftp/attrs.go new file mode 100644 index 00000000000..74ac03b7863 --- /dev/null +++ b/vendor/github.com/pkg/sftp/attrs.go @@ -0,0 +1,136 @@ +package sftp + +// ssh_FXP_ATTRS support +// see https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-5 + +import ( + "os" + "time" +) + +const ( + sshFileXferAttrSize = 0x00000001 + sshFileXferAttrUIDGID = 0x00000002 + sshFileXferAttrPermissions = 0x00000004 + sshFileXferAttrACmodTime = 0x00000008 + sshFileXferAttrExtended = 0x80000000 + + sshFileXferAttrAll = sshFileXferAttrSize | sshFileXferAttrUIDGID | sshFileXferAttrPermissions | + sshFileXferAttrACmodTime | sshFileXferAttrExtended +) + +// fileInfo is an artificial type designed to satisfy os.FileInfo. +type fileInfo struct { + name string + stat *FileStat +} + +// Name returns the base name of the file. +func (fi *fileInfo) Name() string { return fi.name } + +// Size returns the length in bytes for regular files; system-dependent for others. +func (fi *fileInfo) Size() int64 { return int64(fi.stat.Size) } + +// Mode returns file mode bits. +func (fi *fileInfo) Mode() os.FileMode { return fi.stat.FileMode() } + +// ModTime returns the last modification time of the file. +func (fi *fileInfo) ModTime() time.Time { return fi.stat.ModTime() } + +// IsDir returns true if the file is a directory. +func (fi *fileInfo) IsDir() bool { return fi.Mode().IsDir() } + +func (fi *fileInfo) Sys() interface{} { return fi.stat } + +// FileStat holds the original unmarshalled values from a call to READDIR or +// *STAT. It is exported for the purposes of accessing the raw values via +// os.FileInfo.Sys(). It is also used server side to store the unmarshalled +// values for SetStat. +type FileStat struct { + Size uint64 + Mode uint32 + Mtime uint32 + Atime uint32 + UID uint32 + GID uint32 + Extended []StatExtended +} + +// ModTime returns the Mtime SFTP file attribute converted to a time.Time +func (fs *FileStat) ModTime() time.Time { + return time.Unix(int64(fs.Mtime), 0) +} + +// AccessTime returns the Atime SFTP file attribute converted to a time.Time +func (fs *FileStat) AccessTime() time.Time { + return time.Unix(int64(fs.Atime), 0) +} + +// FileMode returns the Mode SFTP file attribute converted to an os.FileMode +func (fs *FileStat) FileMode() os.FileMode { + return toFileMode(fs.Mode) +} + +// StatExtended contains additional, extended information for a FileStat. +type StatExtended struct { + ExtType string + ExtData string +} + +func fileInfoFromStat(stat *FileStat, name string) os.FileInfo { + return &fileInfo{ + name: name, + stat: stat, + } +} + +// FileInfoUidGid extends os.FileInfo and adds callbacks for Uid and Gid retrieval, +// as an alternative to *syscall.Stat_t objects on unix systems. +type FileInfoUidGid interface { + os.FileInfo + Uid() uint32 + Gid() uint32 +} + +// FileInfoUidGid extends os.FileInfo and adds a callbacks for extended data retrieval. +type FileInfoExtendedData interface { + os.FileInfo + Extended() []StatExtended +} + +func fileStatFromInfo(fi os.FileInfo) (uint32, *FileStat) { + mtime := fi.ModTime().Unix() + atime := mtime + var flags uint32 = sshFileXferAttrSize | + sshFileXferAttrPermissions | + sshFileXferAttrACmodTime + + fileStat := &FileStat{ + Size: uint64(fi.Size()), + Mode: fromFileMode(fi.Mode()), + Mtime: uint32(mtime), + Atime: uint32(atime), + } + + // os specific file stat decoding + fileStatFromInfoOs(fi, &flags, fileStat) + + // The call above will include the sshFileXferAttrUIDGID in case + // the os.FileInfo can be casted to *syscall.Stat_t on unix. + // If fi implements FileInfoUidGid, retrieve Uid, Gid from it instead. + if fiExt, ok := fi.(FileInfoUidGid); ok { + flags |= sshFileXferAttrUIDGID + fileStat.UID = fiExt.Uid() + fileStat.GID = fiExt.Gid() + } + + // if fi implements FileInfoExtendedData, retrieve extended data from it + if fiExt, ok := fi.(FileInfoExtendedData); ok { + fileStat.Extended = fiExt.Extended() + if len(fileStat.Extended) > 0 { + flags |= sshFileXferAttrExtended + } + } + + return flags, fileStat +} diff --git a/vendor/github.com/pkg/sftp/attrs_stubs.go b/vendor/github.com/pkg/sftp/attrs_stubs.go new file mode 100644 index 00000000000..d20348c1153 --- /dev/null +++ b/vendor/github.com/pkg/sftp/attrs_stubs.go @@ -0,0 +1,12 @@ +//go:build plan9 || windows || android +// +build plan9 windows android + +package sftp + +import ( + "os" +) + +func fileStatFromInfoOs(fi os.FileInfo, flags *uint32, fileStat *FileStat) { + // todo +} diff --git a/vendor/github.com/pkg/sftp/attrs_unix.go b/vendor/github.com/pkg/sftp/attrs_unix.go new file mode 100644 index 00000000000..96ffc03df96 --- /dev/null +++ b/vendor/github.com/pkg/sftp/attrs_unix.go @@ -0,0 +1,17 @@ +//go:build darwin || dragonfly || freebsd || (!android && linux) || netbsd || openbsd || solaris || aix || js || zos +// +build darwin dragonfly freebsd !android,linux netbsd openbsd solaris aix js zos + +package sftp + +import ( + "os" + "syscall" +) + +func fileStatFromInfoOs(fi os.FileInfo, flags *uint32, fileStat *FileStat) { + if statt, ok := fi.Sys().(*syscall.Stat_t); ok { + *flags |= sshFileXferAttrUIDGID + fileStat.UID = statt.Uid + fileStat.GID = statt.Gid + } +} diff --git a/vendor/github.com/pkg/sftp/client.go b/vendor/github.com/pkg/sftp/client.go new file mode 100644 index 00000000000..307a35ea5b6 --- /dev/null +++ b/vendor/github.com/pkg/sftp/client.go @@ -0,0 +1,2298 @@ +package sftp + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "os" + "path" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/kr/fs" + "golang.org/x/crypto/ssh" + + "github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh" +) + +var ( + // ErrInternalInconsistency indicates the packets sent and the data queued to be + // written to the file don't match up. It is an unusual error and usually is + // caused by bad behavior server side or connection issues. The error is + // limited in scope to the call where it happened, the client object is still + // OK to use as long as the connection is still open. + ErrInternalInconsistency = errors.New("internal inconsistency") + // InternalInconsistency alias for ErrInternalInconsistency. + // + // Deprecated: please use ErrInternalInconsistency + InternalInconsistency = ErrInternalInconsistency +) + +// A ClientOption is a function which applies configuration to a Client. +type ClientOption func(*Client) error + +// MaxPacketChecked sets the maximum size of the payload, measured in bytes. +// This option only accepts sizes servers should support, ie. <= 32768 bytes. +// +// If you get the error "failed to send packet header: EOF" when copying a +// large file, try lowering this number. +// +// The default packet size is 32768 bytes. +func MaxPacketChecked(size int) ClientOption { + return func(c *Client) error { + if size < 1 { + return errors.New("size must be greater or equal to 1") + } + if size > 32768 { + return errors.New("sizes larger than 32KB might not work with all servers") + } + c.maxPacket = size + return nil + } +} + +// MaxPacketUnchecked sets the maximum size of the payload, measured in bytes. +// It accepts sizes larger than the 32768 bytes all servers should support. +// Only use a setting higher than 32768 if your application always connects to +// the same server or after sufficiently broad testing. +// +// If you get the error "failed to send packet header: EOF" when copying a +// large file, try lowering this number. +// +// The default packet size is 32768 bytes. +func MaxPacketUnchecked(size int) ClientOption { + return func(c *Client) error { + if size < 1 { + return errors.New("size must be greater or equal to 1") + } + c.maxPacket = size + return nil + } +} + +// MaxPacket sets the maximum size of the payload, measured in bytes. +// This option only accepts sizes servers should support, ie. <= 32768 bytes. +// This is a synonym for MaxPacketChecked that provides backward compatibility. +// +// If you get the error "failed to send packet header: EOF" when copying a +// large file, try lowering this number. +// +// The default packet size is 32768 bytes. +func MaxPacket(size int) ClientOption { + return MaxPacketChecked(size) +} + +// MaxConcurrentRequestsPerFile sets the maximum concurrent requests allowed for a single file. +// +// The default maximum concurrent requests is 64. +func MaxConcurrentRequestsPerFile(n int) ClientOption { + return func(c *Client) error { + if n < 1 { + return errors.New("n must be greater or equal to 1") + } + c.maxConcurrentRequests = n + return nil + } +} + +// UseConcurrentWrites allows the Client to perform concurrent Writes. +// +// Using concurrency while doing writes, requires special consideration. +// A write to a later offset in a file after an error, +// could end up with a file length longer than what was successfully written. +// +// When using this option, if you receive an error during `io.Copy` or `io.WriteTo`, +// you may need to `Truncate` the target Writer to avoid “holes” in the data written. +func UseConcurrentWrites(value bool) ClientOption { + return func(c *Client) error { + c.useConcurrentWrites = value + return nil + } +} + +// UseConcurrentReads allows the Client to perform concurrent Reads. +// +// Concurrent reads are generally safe to use and not using them will degrade +// performance, so this option is enabled by default. +// +// When enabled, WriteTo will use Stat/Fstat to get the file size and determines +// how many concurrent workers to use. +// Some "read once" servers will delete the file if they receive a stat call on an +// open file and then the download will fail. +// Disabling concurrent reads you will be able to download files from these servers. +// If concurrent reads are disabled, the UseFstat option is ignored. +func UseConcurrentReads(value bool) ClientOption { + return func(c *Client) error { + c.disableConcurrentReads = !value + return nil + } +} + +// UseFstat sets whether to use Fstat or Stat when File.WriteTo is called +// (usually when copying files). +// Some servers limit the amount of open files and calling Stat after opening +// the file will throw an error From the server. Setting this flag will call +// Fstat instead of Stat which is suppose to be called on an open file handle. +// +// It has been found that that with IBM Sterling SFTP servers which have +// "extractability" level set to 1 which means only 1 file can be opened at +// any given time. +// +// If the server you are working with still has an issue with both Stat and +// Fstat calls you can always open a file and read it until the end. +// +// Another reason to read the file until its end and Fstat doesn't work is +// that in some servers, reading a full file will automatically delete the +// file as some of these mainframes map the file to a message in a queue. +// Once the file has been read it will get deleted. +func UseFstat(value bool) ClientOption { + return func(c *Client) error { + c.useFstat = value + return nil + } +} + +// CopyStderrTo specifies a writer to which the standard error of the remote sftp-server command should be written. +// +// The writer passed in will not be automatically closed. +// It is the responsibility of the caller to coordinate closure of any writers. +func CopyStderrTo(wr io.Writer) ClientOption { + return func(c *Client) error { + c.stderrTo = wr + return nil + } +} + +// Client represents an SFTP session on a *ssh.ClientConn SSH connection. +// Multiple Clients can be active on a single SSH connection, and a Client +// may be called concurrently from multiple Goroutines. +// +// Client implements the github.com/kr/fs.FileSystem interface. +type Client struct { + clientConn + + stderrTo io.Writer + + ext map[string]string // Extensions (name -> data). + + maxPacket int // max packet size read or written. + maxConcurrentRequests int + nextid uint32 + + // write concurrency is… error prone. + // Default behavior should be to not use it. + useConcurrentWrites bool + useFstat bool + disableConcurrentReads bool +} + +// NewClient creates a new SFTP client on conn, using zero or more option +// functions. +func NewClient(conn *ssh.Client, opts ...ClientOption) (*Client, error) { + s, err := conn.NewSession() + if err != nil { + return nil, err + } + + pw, err := s.StdinPipe() + if err != nil { + return nil, err + } + pr, err := s.StdoutPipe() + if err != nil { + return nil, err + } + perr, err := s.StderrPipe() + if err != nil { + return nil, err + } + + if err := s.RequestSubsystem("sftp"); err != nil { + return nil, err + } + + return newClientPipe(pr, perr, pw, s.Wait, opts...) +} + +// NewClientPipe creates a new SFTP client given a Reader and a WriteCloser. +// This can be used for connecting to an SFTP server over TCP/TLS or by using +// the system's ssh client program (e.g. via exec.Command). +func NewClientPipe(rd io.Reader, wr io.WriteCloser, opts ...ClientOption) (*Client, error) { + return newClientPipe(rd, nil, wr, nil, opts...) +} + +func newClientPipe(rd, stderr io.Reader, wr io.WriteCloser, wait func() error, opts ...ClientOption) (*Client, error) { + c := &Client{ + clientConn: clientConn{ + conn: conn{ + Reader: rd, + WriteCloser: wr, + }, + inflight: make(map[uint32]chan<- result), + closed: make(chan struct{}), + wait: wait, + }, + + ext: make(map[string]string), + + maxPacket: 1 << 15, + maxConcurrentRequests: 64, + } + + for _, opt := range opts { + if err := opt(c); err != nil { + wr.Close() + return nil, err + } + } + + if stderr != nil { + wr := io.Discard + if c.stderrTo != nil { + wr = c.stderrTo + } + + go func() { + // DO NOT close the writer! + // Programs may pass in `os.Stderr` to write the remote stderr to, + // and the program may continue after disconnect by reconnecting. + // But if we've closed their stderr, then we just messed everything up. + + if _, err := io.Copy(wr, stderr); err != nil { + debug("error copying stderr: %v", err) + } + }() + } + + if err := c.sendInit(); err != nil { + wr.Close() + return nil, fmt.Errorf("error sending init packet to server: %w", err) + } + + if err := c.recvVersion(); err != nil { + wr.Close() + return nil, fmt.Errorf("error receiving version packet from server: %w", err) + } + + c.clientConn.wg.Add(1) + go func() { + defer c.clientConn.wg.Done() + + if err := c.clientConn.recv(); err != nil { + c.clientConn.broadcastErr(err) + } + }() + + return c, nil +} + +// Create creates the named file mode 0666 (before umask), truncating it if it +// already exists. If successful, methods on the returned File can be used for +// I/O; the associated file descriptor has mode O_RDWR. If you need more +// control over the flags/mode used to open the file see client.OpenFile. +// +// Note that some SFTP servers (eg. AWS Transfer) do not support opening files +// read/write at the same time. For those services you will need to use +// `client.OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC)`. +func (c *Client) Create(path string) (*File, error) { + return c.open(path, toPflags(os.O_RDWR|os.O_CREATE|os.O_TRUNC)) +} + +const sftpProtocolVersion = 3 // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt + +func (c *Client) sendInit() error { + return c.clientConn.conn.sendPacket(&sshFxInitPacket{ + Version: sftpProtocolVersion, // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt + }) +} + +// returns the next value of c.nextid +func (c *Client) nextID() uint32 { + return atomic.AddUint32(&c.nextid, 1) +} + +func (c *Client) recvVersion() error { + typ, data, err := c.recvPacket(0) + if err != nil { + if err == io.EOF { + return fmt.Errorf("server unexpectedly closed connection: %w", io.ErrUnexpectedEOF) + } + + return err + } + + if typ != sshFxpVersion { + return &unexpectedPacketErr{sshFxpVersion, typ} + } + + version, data, err := unmarshalUint32Safe(data) + if err != nil { + return err + } + + if version != sftpProtocolVersion { + return &unexpectedVersionErr{sftpProtocolVersion, version} + } + + for len(data) > 0 { + var ext extensionPair + ext, data, err = unmarshalExtensionPair(data) + if err != nil { + return err + } + c.ext[ext.Name] = ext.Data + } + + return nil +} + +// HasExtension checks whether the server supports a named extension. +// +// The first return value is the extension data reported by the server +// (typically a version number). +func (c *Client) HasExtension(name string) (string, bool) { + data, ok := c.ext[name] + return data, ok +} + +// Walk returns a new Walker rooted at root. +func (c *Client) Walk(root string) *fs.Walker { + return fs.WalkFS(root, c) +} + +// ReadDir reads the directory named by p +// and returns a list of directory entries. +func (c *Client) ReadDir(p string) ([]os.FileInfo, error) { + return c.ReadDirContext(context.Background(), p) +} + +// ReadDirContext reads the directory named by p +// and returns a list of directory entries. +// The passed context can be used to cancel the operation +// returning all entries listed up to the cancellation. +func (c *Client) ReadDirContext(ctx context.Context, p string) ([]os.FileInfo, error) { + handle, err := c.opendir(ctx, p) + if err != nil { + return nil, err + } + defer c.close(handle) // this has to defer earlier than the lock below + var entries []os.FileInfo + var done = false + for !done { + id := c.nextID() + typ, data, err1 := c.sendPacket(ctx, nil, &sshFxpReaddirPacket{ + ID: id, + Handle: handle, + }) + if err1 != nil { + err = err1 + done = true + break + } + switch typ { + case sshFxpName: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIDErr{id, sid} + } + count, data := unmarshalUint32(data) + for i := uint32(0); i < count; i++ { + var filename string + filename, data = unmarshalString(data) + _, data = unmarshalString(data) // discard longname + var attr *FileStat + attr, data, err = unmarshalAttrs(data) + if err != nil { + return nil, err + } + if filename == "." || filename == ".." { + continue + } + entries = append(entries, fileInfoFromStat(attr, path.Base(filename))) + } + case sshFxpStatus: + // TODO(dfc) scope warning! + err = normaliseError(unmarshalStatus(id, data)) + done = true + default: + return nil, unimplementedPacketErr(typ) + } + } + if err == io.EOF { + err = nil + } + return entries, err +} + +func (c *Client) opendir(ctx context.Context, path string) (string, error) { + id := c.nextID() + typ, data, err := c.sendPacket(ctx, nil, &sshFxpOpendirPacket{ + ID: id, + Path: path, + }) + if err != nil { + return "", err + } + switch typ { + case sshFxpHandle: + sid, data := unmarshalUint32(data) + if sid != id { + return "", &unexpectedIDErr{id, sid} + } + handle, _ := unmarshalString(data) + return handle, nil + case sshFxpStatus: + return "", normaliseError(unmarshalStatus(id, data)) + default: + return "", unimplementedPacketErr(typ) + } +} + +// Stat returns a FileInfo structure describing the file specified by path 'p'. +// If 'p' is a symbolic link, the returned FileInfo structure describes the referent file. +func (c *Client) Stat(p string) (os.FileInfo, error) { + fs, err := c.stat(p) + if err != nil { + return nil, err + } + return fileInfoFromStat(fs, path.Base(p)), nil +} + +// Lstat returns a FileInfo structure describing the file specified by path 'p'. +// If 'p' is a symbolic link, the returned FileInfo structure describes the symbolic link. +func (c *Client) Lstat(p string) (os.FileInfo, error) { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpLstatPacket{ + ID: id, + Path: p, + }) + if err != nil { + return nil, err + } + switch typ { + case sshFxpAttrs: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIDErr{id, sid} + } + attr, _, err := unmarshalAttrs(data) + if err != nil { + // avoid returning a valid value from fileInfoFromStats if err != nil. + return nil, err + } + return fileInfoFromStat(attr, path.Base(p)), nil + case sshFxpStatus: + return nil, normaliseError(unmarshalStatus(id, data)) + default: + return nil, unimplementedPacketErr(typ) + } +} + +// ReadLink reads the target of a symbolic link. +func (c *Client) ReadLink(p string) (string, error) { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpReadlinkPacket{ + ID: id, + Path: p, + }) + if err != nil { + return "", err + } + switch typ { + case sshFxpName: + sid, data := unmarshalUint32(data) + if sid != id { + return "", &unexpectedIDErr{id, sid} + } + count, data := unmarshalUint32(data) + if count != 1 { + return "", unexpectedCount(1, count) + } + filename, _ := unmarshalString(data) // ignore dummy attributes + return filename, nil + case sshFxpStatus: + return "", normaliseError(unmarshalStatus(id, data)) + default: + return "", unimplementedPacketErr(typ) + } +} + +// Link creates a hard link at 'newname', pointing at the same inode as 'oldname' +func (c *Client) Link(oldname, newname string) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpHardlinkPacket{ + ID: id, + Oldpath: oldname, + Newpath: newname, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// Symlink creates a symbolic link at 'newname', pointing at target 'oldname' +func (c *Client) Symlink(oldname, newname string) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpSymlinkPacket{ + ID: id, + Linkpath: newname, + Targetpath: oldname, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +func (c *Client) fsetstat(handle string, flags uint32, attrs interface{}) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpFsetstatPacket{ + ID: id, + Handle: handle, + Flags: flags, + Attrs: attrs, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// setstat is a convience wrapper to allow for changing of various parts of the file descriptor. +func (c *Client) setstat(path string, flags uint32, attrs interface{}) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpSetstatPacket{ + ID: id, + Path: path, + Flags: flags, + Attrs: attrs, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// Chtimes changes the access and modification times of the named file. +func (c *Client) Chtimes(path string, atime time.Time, mtime time.Time) error { + type times struct { + Atime uint32 + Mtime uint32 + } + attrs := times{uint32(atime.Unix()), uint32(mtime.Unix())} + return c.setstat(path, sshFileXferAttrACmodTime, attrs) +} + +// Chown changes the user and group owners of the named file. +func (c *Client) Chown(path string, uid, gid int) error { + type owner struct { + UID uint32 + GID uint32 + } + attrs := owner{uint32(uid), uint32(gid)} + return c.setstat(path, sshFileXferAttrUIDGID, attrs) +} + +// Chmod changes the permissions of the named file. +// +// Chmod does not apply a umask, because even retrieving the umask is not +// possible in a portable way without causing a race condition. Callers +// should mask off umask bits, if desired. +func (c *Client) Chmod(path string, mode os.FileMode) error { + return c.setstat(path, sshFileXferAttrPermissions, toChmodPerm(mode)) +} + +// Truncate sets the size of the named file. Although it may be safely assumed +// that if the size is less than its current size it will be truncated to fit, +// the SFTP protocol does not specify what behavior the server should do when setting +// size greater than the current size. +func (c *Client) Truncate(path string, size int64) error { + return c.setstat(path, sshFileXferAttrSize, uint64(size)) +} + +// SetExtendedData sets extended attributes of the named file. It uses the +// SSH_FILEXFER_ATTR_EXTENDED flag in the setstat request. +// +// This flag provides a general extension mechanism for vendor-specific extensions. +// Names of the attributes should be a string of the format "name@domain", where "domain" +// is a valid, registered domain name and "name" identifies the method. Server +// implementations SHOULD ignore extended data fields that they do not understand. +func (c *Client) SetExtendedData(path string, extended []StatExtended) error { + attrs := &FileStat{ + Extended: extended, + } + return c.setstat(path, sshFileXferAttrExtended, attrs) +} + +// Open opens the named file for reading. If successful, methods on the +// returned file can be used for reading; the associated file descriptor +// has mode O_RDONLY. +func (c *Client) Open(path string) (*File, error) { + return c.open(path, toPflags(os.O_RDONLY)) +} + +// OpenFile is the generalized open call; most users will use Open or +// Create instead. It opens the named file with specified flag (O_RDONLY +// etc.). If successful, methods on the returned File can be used for I/O. +func (c *Client) OpenFile(path string, f int) (*File, error) { + return c.open(path, toPflags(f)) +} + +func (c *Client) open(path string, pflags uint32) (*File, error) { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpOpenPacket{ + ID: id, + Path: path, + Pflags: pflags, + }) + if err != nil { + return nil, err + } + switch typ { + case sshFxpHandle: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIDErr{id, sid} + } + handle, _ := unmarshalString(data) + return &File{c: c, path: path, handle: handle}, nil + case sshFxpStatus: + return nil, normaliseError(unmarshalStatus(id, data)) + default: + return nil, unimplementedPacketErr(typ) + } +} + +// close closes a handle handle previously returned in the response +// to SSH_FXP_OPEN or SSH_FXP_OPENDIR. The handle becomes invalid +// immediately after this request has been sent. +func (c *Client) close(handle string) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpClosePacket{ + ID: id, + Handle: handle, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +func (c *Client) stat(path string) (*FileStat, error) { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpStatPacket{ + ID: id, + Path: path, + }) + if err != nil { + return nil, err + } + switch typ { + case sshFxpAttrs: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIDErr{id, sid} + } + attr, _, err := unmarshalAttrs(data) + return attr, err + case sshFxpStatus: + return nil, normaliseError(unmarshalStatus(id, data)) + default: + return nil, unimplementedPacketErr(typ) + } +} + +func (c *Client) fstat(handle string) (*FileStat, error) { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpFstatPacket{ + ID: id, + Handle: handle, + }) + if err != nil { + return nil, err + } + switch typ { + case sshFxpAttrs: + sid, data := unmarshalUint32(data) + if sid != id { + return nil, &unexpectedIDErr{id, sid} + } + attr, _, err := unmarshalAttrs(data) + return attr, err + case sshFxpStatus: + return nil, normaliseError(unmarshalStatus(id, data)) + default: + return nil, unimplementedPacketErr(typ) + } +} + +// StatVFS retrieves VFS statistics from a remote host. +// +// It implements the statvfs@openssh.com SSH_FXP_EXTENDED feature +// from http://www.opensource.apple.com/source/OpenSSH/OpenSSH-175/openssh/PROTOCOL?txt. +func (c *Client) StatVFS(path string) (*StatVFS, error) { + // send the StatVFS packet to the server + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpStatvfsPacket{ + ID: id, + Path: path, + }) + if err != nil { + return nil, err + } + + switch typ { + // server responded with valid data + case sshFxpExtendedReply: + var response StatVFS + err = binary.Read(bytes.NewReader(data), binary.BigEndian, &response) + if err != nil { + return nil, errors.New("can not parse reply") + } + + return &response, nil + + // the resquest failed + case sshFxpStatus: + return nil, normaliseError(unmarshalStatus(id, data)) + + default: + return nil, unimplementedPacketErr(typ) + } +} + +// Join joins any number of path elements into a single path, adding a +// separating slash if necessary. The result is Cleaned; in particular, all +// empty strings are ignored. +func (c *Client) Join(elem ...string) string { return path.Join(elem...) } + +// Remove removes the specified file or directory. An error will be returned if no +// file or directory with the specified path exists, or if the specified directory +// is not empty. +func (c *Client) Remove(path string) error { + errF := c.removeFile(path) + if errF == nil { + return nil + } + + errD := c.RemoveDirectory(path) + if errD == nil { + return nil + } + + // Both failed: figure out which error to return. + + if errF, ok := errF.(*os.PathError); ok { + // The only time it makes sense to compare errors, is when both are `*os.PathError`. + // We cannot test these directly with errF == errD, as that would be a pointer comparison. + + if errD, ok := errD.(*os.PathError); ok && errors.Is(errF.Err, errD.Err) { + // If they are both pointers to PathError, + // and the same underlying error, then return that. + return errF + } + } + + fi, err := c.Stat(path) + if err != nil { + return err + } + + if fi.IsDir() { + return errD + } + + return errF +} + +func (c *Client) removeFile(path string) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpRemovePacket{ + ID: id, + Filename: path, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + err = normaliseError(unmarshalStatus(id, data)) + if err == nil { + return nil + } + return &os.PathError{ + Op: "remove", + Path: path, + Err: err, + } + default: + return unimplementedPacketErr(typ) + } +} + +// RemoveDirectory removes a directory path. +func (c *Client) RemoveDirectory(path string) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpRmdirPacket{ + ID: id, + Path: path, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + err = normaliseError(unmarshalStatus(id, data)) + if err == nil { + return nil + } + return &os.PathError{ + Op: "remove", + Path: path, + Err: err, + } + default: + return unimplementedPacketErr(typ) + } +} + +// Rename renames a file. +func (c *Client) Rename(oldname, newname string) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpRenamePacket{ + ID: id, + Oldpath: oldname, + Newpath: newname, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// PosixRename renames a file using the posix-rename@openssh.com extension +// which will replace newname if it already exists. +func (c *Client) PosixRename(oldname, newname string) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpPosixRenamePacket{ + ID: id, + Oldpath: oldname, + Newpath: newname, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// RealPath can be used to have the server canonicalize any given path name to an absolute path. +// +// This is useful for converting path names containing ".." components, +// or relative pathnames without a leading slash into absolute paths. +func (c *Client) RealPath(path string) (string, error) { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpRealpathPacket{ + ID: id, + Path: path, + }) + if err != nil { + return "", err + } + switch typ { + case sshFxpName: + sid, data := unmarshalUint32(data) + if sid != id { + return "", &unexpectedIDErr{id, sid} + } + count, data := unmarshalUint32(data) + if count != 1 { + return "", unexpectedCount(1, count) + } + filename, _ := unmarshalString(data) // ignore attributes + return filename, nil + case sshFxpStatus: + return "", normaliseError(unmarshalStatus(id, data)) + default: + return "", unimplementedPacketErr(typ) + } +} + +// Getwd returns the current working directory of the server. Operations +// involving relative paths will be based at this location. +func (c *Client) Getwd() (string, error) { + return c.RealPath(".") +} + +// Mkdir creates the specified directory. An error will be returned if a file or +// directory with the specified path already exists, or if the directory's +// parent folder does not exist (the method cannot create complete paths). +func (c *Client) Mkdir(path string) error { + id := c.nextID() + typ, data, err := c.sendPacket(context.Background(), nil, &sshFxpMkdirPacket{ + ID: id, + Path: path, + }) + if err != nil { + return err + } + switch typ { + case sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return unimplementedPacketErr(typ) + } +} + +// MkdirAll creates a directory named path, along with any necessary parents, +// and returns nil, or else returns an error. +// If path is already a directory, MkdirAll does nothing and returns nil. +// If, while making any directory, that path is found to already be a regular file, an error is returned. +func (c *Client) MkdirAll(path string) error { + // Most of this code mimics https://golang.org/src/os/path.go?s=514:561#L13 + // Fast path: if we can tell whether path is a directory or file, stop with success or error. + dir, err := c.Stat(path) + if err == nil { + if dir.IsDir() { + return nil + } + return &os.PathError{Op: "mkdir", Path: path, Err: syscall.ENOTDIR} + } + + // Slow path: make sure parent exists and then call Mkdir for path. + i := len(path) + for i > 0 && path[i-1] == '/' { // Skip trailing path separator. + i-- + } + + j := i + for j > 0 && path[j-1] != '/' { // Scan backward over element. + j-- + } + + if j > 1 { + // Create parent + err = c.MkdirAll(path[0 : j-1]) + if err != nil { + return err + } + } + + // Parent now exists; invoke Mkdir and use its result. + err = c.Mkdir(path) + if err != nil { + // Handle arguments like "foo/." by + // double-checking that directory doesn't exist. + dir, err1 := c.Lstat(path) + if err1 == nil && dir.IsDir() { + return nil + } + return err + } + return nil +} + +// RemoveAll delete files recursively in the directory and Recursively delete subdirectories. +// An error will be returned if no file or directory with the specified path exists +func (c *Client) RemoveAll(path string) error { + + // Get the file/directory information + fi, err := c.Stat(path) + if err != nil { + return err + } + + if fi.IsDir() { + // Delete files recursively in the directory + files, err := c.ReadDir(path) + if err != nil { + return err + } + + for _, file := range files { + if file.IsDir() { + // Recursively delete subdirectories + err = c.RemoveAll(path + "/" + file.Name()) + if err != nil { + return err + } + } else { + // Delete individual files + err = c.Remove(path + "/" + file.Name()) + if err != nil { + return err + } + } + } + + } + + return c.Remove(path) + +} + +// File represents a remote file. +type File struct { + c *Client + path string + + mu sync.RWMutex + handle string + offset int64 // current offset within remote file +} + +// Close closes the File, rendering it unusable for I/O. It returns an +// error, if any. +func (f *File) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + + if f.handle == "" { + return os.ErrClosed + } + + // The design principle here is that when `openssh-portable/sftp-server.c` is doing `handle_close`, + // it will unconditionally mark the handle as unused, + // so we need to also unconditionally mark this handle as invalid. + // By invalidating our local copy of the handle, + // we ensure that there cannot be any erroneous use-after-close requests sent after Close. + + handle := f.handle + f.handle = "" + + return f.c.close(handle) +} + +// Name returns the name of the file as presented to Open or Create. +func (f *File) Name() string { + return f.path +} + +// Read reads up to len(b) bytes from the File. It returns the number of bytes +// read and an error, if any. Read follows io.Reader semantics, so when Read +// encounters an error or EOF condition after successfully reading n > 0 bytes, +// it returns the number of bytes read. +// +// To maximise throughput for transferring the entire file (especially +// over high latency links) it is recommended to use WriteTo rather +// than calling Read multiple times. io.Copy will do this +// automatically. +func (f *File) Read(b []byte) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + + n, err := f.readAt(b, f.offset) + f.offset += int64(n) + return n, err +} + +// readChunkAt attempts to read the whole entire length of the buffer from the file starting at the offset. +// It will continue progressively reading into the buffer until it fills the whole buffer, or an error occurs. +func (f *File) readChunkAt(ch chan result, b []byte, off int64) (n int, err error) { + for err == nil && n < len(b) { + id := f.c.nextID() + typ, data, err := f.c.sendPacket(context.Background(), ch, &sshFxpReadPacket{ + ID: id, + Handle: f.handle, + Offset: uint64(off) + uint64(n), + Len: uint32(len(b) - n), + }) + if err != nil { + return n, err + } + + switch typ { + case sshFxpStatus: + return n, normaliseError(unmarshalStatus(id, data)) + + case sshFxpData: + sid, data := unmarshalUint32(data) + if id != sid { + return n, &unexpectedIDErr{id, sid} + } + + l, data := unmarshalUint32(data) + n += copy(b[n:], data[:l]) + + default: + return n, unimplementedPacketErr(typ) + } + } + + return +} + +func (f *File) readAtSequential(b []byte, off int64) (read int, err error) { + for read < len(b) { + rb := b[read:] + if len(rb) > f.c.maxPacket { + rb = rb[:f.c.maxPacket] + } + n, err := f.readChunkAt(nil, rb, off+int64(read)) + if n < 0 { + panic("sftp.File: returned negative count from readChunkAt") + } + if n > 0 { + read += n + } + if err != nil { + return read, err + } + } + return read, nil +} + +// ReadAt reads up to len(b) byte from the File at a given offset `off`. It returns +// the number of bytes read and an error, if any. ReadAt follows io.ReaderAt semantics, +// so the file offset is not altered during the read. +func (f *File) ReadAt(b []byte, off int64) (int, error) { + f.mu.RLock() + defer f.mu.RUnlock() + + return f.readAt(b, off) +} + +// readAt must be called while holding either the Read or Write mutex in File. +// This code is concurrent safe with itself, but not with Close. +func (f *File) readAt(b []byte, off int64) (int, error) { + if f.handle == "" { + return 0, os.ErrClosed + } + + if len(b) <= f.c.maxPacket { + // This should be able to be serviced with 1/2 requests. + // So, just do it directly. + return f.readChunkAt(nil, b, off) + } + + if f.c.disableConcurrentReads { + return f.readAtSequential(b, off) + } + + // Split the read into multiple maxPacket-sized concurrent reads bounded by maxConcurrentRequests. + // This allows writes with a suitably large buffer to transfer data at a much faster rate + // by overlapping round trip times. + + cancel := make(chan struct{}) + + concurrency := len(b)/f.c.maxPacket + 1 + if concurrency > f.c.maxConcurrentRequests || concurrency < 1 { + concurrency = f.c.maxConcurrentRequests + } + + resPool := newResChanPool(concurrency) + + type work struct { + id uint32 + res chan result + + b []byte + off int64 + } + workCh := make(chan work) + + // Slice: cut up the Read into any number of buffers of length <= f.c.maxPacket, and at appropriate offsets. + go func() { + defer close(workCh) + + b := b + offset := off + chunkSize := f.c.maxPacket + + for len(b) > 0 { + rb := b + if len(rb) > chunkSize { + rb = rb[:chunkSize] + } + + id := f.c.nextID() + res := resPool.Get() + + f.c.dispatchRequest(res, &sshFxpReadPacket{ + ID: id, + Handle: f.handle, + Offset: uint64(offset), + Len: uint32(len(rb)), + }) + + select { + case workCh <- work{id, res, rb, offset}: + case <-cancel: + return + } + + offset += int64(len(rb)) + b = b[len(rb):] + } + }() + + type rErr struct { + off int64 + err error + } + errCh := make(chan rErr) + + var wg sync.WaitGroup + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + // Map_i: each worker gets work, and then performs the Read into its buffer from its respective offset. + go func() { + defer wg.Done() + + for packet := range workCh { + var n int + + s := <-packet.res + resPool.Put(packet.res) + + err := s.err + if err == nil { + switch s.typ { + case sshFxpStatus: + err = normaliseError(unmarshalStatus(packet.id, s.data)) + + case sshFxpData: + sid, data := unmarshalUint32(s.data) + if packet.id != sid { + err = &unexpectedIDErr{packet.id, sid} + + } else { + l, data := unmarshalUint32(data) + n = copy(packet.b, data[:l]) + + // For normal disk files, it is guaranteed that this will read + // the specified number of bytes, or up to end of file. + // This implies, if we have a short read, that means EOF. + if n < len(packet.b) { + err = io.EOF + } + } + + default: + err = unimplementedPacketErr(s.typ) + } + } + + if err != nil { + // return the offset as the start + how much we read before the error. + errCh <- rErr{packet.off + int64(n), err} + + // DO NOT return. + // We want to ensure that workCh is drained before wg.Wait returns. + } + } + }() + } + + // Wait for long tail, before closing results. + go func() { + wg.Wait() + close(errCh) + }() + + // Reduce: collect all the results into a relevant return: the earliest offset to return an error. + firstErr := rErr{math.MaxInt64, nil} + for rErr := range errCh { + if rErr.off <= firstErr.off { + firstErr = rErr + } + + select { + case <-cancel: + default: + // stop any more work from being distributed. (Just in case.) + close(cancel) + } + } + + if firstErr.err != nil { + // firstErr.err != nil if and only if firstErr.off > our starting offset. + return int(firstErr.off - off), firstErr.err + } + + // As per spec for io.ReaderAt, we return nil error if and only if we read everything. + return len(b), nil +} + +// writeToSequential implements WriteTo, but works sequentially with no parallelism. +func (f *File) writeToSequential(w io.Writer) (written int64, err error) { + b := make([]byte, f.c.maxPacket) + ch := make(chan result, 1) // reusable channel + + for { + n, err := f.readChunkAt(ch, b, f.offset) + if n < 0 { + panic("sftp.File: returned negative count from readChunkAt") + } + + if n > 0 { + f.offset += int64(n) + + m, err := w.Write(b[:n]) + written += int64(m) + + if err != nil { + return written, err + } + } + + if err != nil { + if err == io.EOF { + return written, nil // return nil explicitly. + } + + return written, err + } + } +} + +// WriteTo writes the file to the given Writer. +// The return value is the number of bytes written. +// Any error encountered during the write is also returned. +// +// This method is preferred over calling Read multiple times +// to maximise throughput for transferring the entire file, +// especially over high latency links. +func (f *File) WriteTo(w io.Writer) (written int64, err error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.handle == "" { + return 0, os.ErrClosed + } + + if f.c.disableConcurrentReads { + return f.writeToSequential(w) + } + + // For concurrency, we want to guess how many concurrent workers we should use. + var fileStat *FileStat + if f.c.useFstat { + fileStat, err = f.c.fstat(f.handle) + } else { + fileStat, err = f.c.stat(f.path) + } + if err != nil { + return 0, err + } + + fileSize := fileStat.Size + if fileSize <= uint64(f.c.maxPacket) || !isRegular(fileStat.Mode) { + // only regular files are guaranteed to return (full read) xor (partial read, next error) + return f.writeToSequential(w) + } + + concurrency64 := fileSize/uint64(f.c.maxPacket) + 1 // a bad guess, but better than no guess + if concurrency64 > uint64(f.c.maxConcurrentRequests) || concurrency64 < 1 { + concurrency64 = uint64(f.c.maxConcurrentRequests) + } + // Now that concurrency64 is saturated to an int value, we know this assignment cannot possibly overflow. + concurrency := int(concurrency64) + + chunkSize := f.c.maxPacket + pool := newBufPool(concurrency, chunkSize) + resPool := newResChanPool(concurrency) + + cancel := make(chan struct{}) + var wg sync.WaitGroup + defer func() { + // Once the writing Reduce phase has ended, all the feed work needs to unconditionally stop. + close(cancel) + + // We want to wait until all outstanding goroutines with an `f` or `f.c` reference have completed. + // Just to be sure we don’t orphan any goroutines any hanging references. + wg.Wait() + }() + + type writeWork struct { + b []byte + off int64 + err error + + next chan writeWork + } + writeCh := make(chan writeWork) + + type readWork struct { + id uint32 + res chan result + off int64 + + cur, next chan writeWork + } + readCh := make(chan readWork) + + // Slice: hand out chunks of work on demand, with a `cur` and `next` channel built-in for sequencing. + go func() { + defer close(readCh) + + off := f.offset + + cur := writeCh + for { + id := f.c.nextID() + res := resPool.Get() + + next := make(chan writeWork) + readWork := readWork{ + id: id, + res: res, + off: off, + + cur: cur, + next: next, + } + + f.c.dispatchRequest(res, &sshFxpReadPacket{ + ID: id, + Handle: f.handle, + Offset: uint64(off), + Len: uint32(chunkSize), + }) + + select { + case readCh <- readWork: + case <-cancel: + return + } + + off += int64(chunkSize) + cur = next + } + }() + + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + // Map_i: each worker gets readWork, and does the Read into a buffer at the given offset. + go func() { + defer wg.Done() + + for readWork := range readCh { + var b []byte + var n int + + s := <-readWork.res + resPool.Put(readWork.res) + + err := s.err + if err == nil { + switch s.typ { + case sshFxpStatus: + err = normaliseError(unmarshalStatus(readWork.id, s.data)) + + case sshFxpData: + sid, data := unmarshalUint32(s.data) + if readWork.id != sid { + err = &unexpectedIDErr{readWork.id, sid} + + } else { + l, data := unmarshalUint32(data) + b = pool.Get()[:l] + n = copy(b, data[:l]) + b = b[:n] + } + + default: + err = unimplementedPacketErr(s.typ) + } + } + + writeWork := writeWork{ + b: b, + off: readWork.off, + err: err, + + next: readWork.next, + } + + select { + case readWork.cur <- writeWork: + case <-cancel: + } + + // DO NOT return. + // We want to ensure that readCh is drained before wg.Wait returns. + } + }() + } + + // Reduce: serialize the results from the reads into sequential writes. + cur := writeCh + for { + packet, ok := <-cur + if !ok { + return written, errors.New("sftp.File.WriteTo: unexpectedly closed channel") + } + + // Because writes are serialized, this will always be the last successfully read byte. + f.offset = packet.off + int64(len(packet.b)) + + if len(packet.b) > 0 { + n, err := w.Write(packet.b) + written += int64(n) + if err != nil { + return written, err + } + } + + if packet.err != nil { + if packet.err == io.EOF { + return written, nil + } + + return written, packet.err + } + + pool.Put(packet.b) + cur = packet.next + } +} + +// Stat returns the FileInfo structure describing file. If there is an +// error. +func (f *File) Stat() (os.FileInfo, error) { + f.mu.RLock() + defer f.mu.RUnlock() + + if f.handle == "" { + return nil, os.ErrClosed + } + + return f.stat() +} + +func (f *File) stat() (os.FileInfo, error) { + fs, err := f.c.fstat(f.handle) + if err != nil { + return nil, err + } + return fileInfoFromStat(fs, path.Base(f.path)), nil +} + +// Write writes len(b) bytes to the File. It returns the number of bytes +// written and an error, if any. Write returns a non-nil error when n != +// len(b). +// +// To maximise throughput for transferring the entire file (especially +// over high latency links) it is recommended to use ReadFrom rather +// than calling Write multiple times. io.Copy will do this +// automatically. +func (f *File) Write(b []byte) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.handle == "" { + return 0, os.ErrClosed + } + + n, err := f.writeAt(b, f.offset) + f.offset += int64(n) + return n, err +} + +func (f *File) writeChunkAt(ch chan result, b []byte, off int64) (int, error) { + typ, data, err := f.c.sendPacket(context.Background(), ch, &sshFxpWritePacket{ + ID: f.c.nextID(), + Handle: f.handle, + Offset: uint64(off), + Length: uint32(len(b)), + Data: b, + }) + if err != nil { + return 0, err + } + + switch typ { + case sshFxpStatus: + id, _ := unmarshalUint32(data) + err := normaliseError(unmarshalStatus(id, data)) + if err != nil { + return 0, err + } + + default: + return 0, unimplementedPacketErr(typ) + } + + return len(b), nil +} + +// writeAtConcurrent implements WriterAt, but works concurrently rather than sequentially. +func (f *File) writeAtConcurrent(b []byte, off int64) (int, error) { + // Split the write into multiple maxPacket sized concurrent writes + // bounded by maxConcurrentRequests. This allows writes with a suitably + // large buffer to transfer data at a much faster rate due to + // overlapping round trip times. + + cancel := make(chan struct{}) + + type work struct { + id uint32 + res chan result + + off int64 + } + workCh := make(chan work) + + concurrency := len(b)/f.c.maxPacket + 1 + if concurrency > f.c.maxConcurrentRequests || concurrency < 1 { + concurrency = f.c.maxConcurrentRequests + } + + pool := newResChanPool(concurrency) + + // Slice: cut up the Read into any number of buffers of length <= f.c.maxPacket, and at appropriate offsets. + go func() { + defer close(workCh) + + var read int + chunkSize := f.c.maxPacket + + for read < len(b) { + wb := b[read:] + if len(wb) > chunkSize { + wb = wb[:chunkSize] + } + + id := f.c.nextID() + res := pool.Get() + off := off + int64(read) + + f.c.dispatchRequest(res, &sshFxpWritePacket{ + ID: id, + Handle: f.handle, + Offset: uint64(off), + Length: uint32(len(wb)), + Data: wb, + }) + + select { + case workCh <- work{id, res, off}: + case <-cancel: + return + } + + read += len(wb) + } + }() + + type wErr struct { + off int64 + err error + } + errCh := make(chan wErr) + + var wg sync.WaitGroup + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + // Map_i: each worker gets work, and does the Write from each buffer to its respective offset. + go func() { + defer wg.Done() + + for work := range workCh { + s := <-work.res + pool.Put(work.res) + + err := s.err + if err == nil { + switch s.typ { + case sshFxpStatus: + err = normaliseError(unmarshalStatus(work.id, s.data)) + default: + err = unimplementedPacketErr(s.typ) + } + } + + if err != nil { + errCh <- wErr{work.off, err} + } + } + }() + } + + // Wait for long tail, before closing results. + go func() { + wg.Wait() + close(errCh) + }() + + // Reduce: collect all the results into a relevant return: the earliest offset to return an error. + firstErr := wErr{math.MaxInt64, nil} + for wErr := range errCh { + if wErr.off <= firstErr.off { + firstErr = wErr + } + + select { + case <-cancel: + default: + // stop any more work from being distributed. (Just in case.) + close(cancel) + } + } + + if firstErr.err != nil { + // firstErr.err != nil if and only if firstErr.off >= our starting offset. + return int(firstErr.off - off), firstErr.err + } + + return len(b), nil +} + +// WriteAt writes up to len(b) byte to the File at a given offset `off`. It returns +// the number of bytes written and an error, if any. WriteAt follows io.WriterAt semantics, +// so the file offset is not altered during the write. +func (f *File) WriteAt(b []byte, off int64) (written int, err error) { + f.mu.RLock() + defer f.mu.RUnlock() + + if f.handle == "" { + return 0, os.ErrClosed + } + + return f.writeAt(b, off) +} + +// writeAt must be called while holding either the Read or Write mutex in File. +// This code is concurrent safe with itself, but not with Close. +func (f *File) writeAt(b []byte, off int64) (written int, err error) { + if len(b) <= f.c.maxPacket { + // We can do this in one write. + return f.writeChunkAt(nil, b, off) + } + + if f.c.useConcurrentWrites { + return f.writeAtConcurrent(b, off) + } + + ch := make(chan result, 1) // reusable channel + + chunkSize := f.c.maxPacket + + for written < len(b) { + wb := b[written:] + if len(wb) > chunkSize { + wb = wb[:chunkSize] + } + + n, err := f.writeChunkAt(ch, wb, off+int64(written)) + if n > 0 { + written += n + } + + if err != nil { + return written, err + } + } + + return len(b), nil +} + +// ReadFromWithConcurrency implements ReaderFrom, +// but uses the given concurrency to issue multiple requests at the same time. +// +// Giving a concurrency of less than one will default to the Client’s max concurrency. +// +// Otherwise, the given concurrency will be capped by the Client's max concurrency. +// +// When one needs to guarantee concurrent reads/writes, this method is preferred +// over ReadFrom. +func (f *File) ReadFromWithConcurrency(r io.Reader, concurrency int) (read int64, err error) { + f.mu.Lock() + defer f.mu.Unlock() + + return f.readFromWithConcurrency(r, concurrency) +} + +func (f *File) readFromWithConcurrency(r io.Reader, concurrency int) (read int64, err error) { + if f.handle == "" { + return 0, os.ErrClosed + } + + // Split the write into multiple maxPacket sized concurrent writes. + // This allows writes with a suitably large reader + // to transfer data at a much faster rate due to overlapping round trip times. + + cancel := make(chan struct{}) + + type work struct { + id uint32 + res chan result + + off int64 + } + workCh := make(chan work) + + type rwErr struct { + off int64 + err error + } + errCh := make(chan rwErr) + + if concurrency > f.c.maxConcurrentRequests || concurrency < 1 { + concurrency = f.c.maxConcurrentRequests + } + + pool := newResChanPool(concurrency) + + // Slice: cut up the Read into any number of buffers of length <= f.c.maxPacket, and at appropriate offsets. + go func() { + defer close(workCh) + + b := make([]byte, f.c.maxPacket) + off := f.offset + + for { + // Fill the entire buffer. + n, err := io.ReadFull(r, b) + + if n > 0 { + read += int64(n) + + id := f.c.nextID() + res := pool.Get() + + f.c.dispatchRequest(res, &sshFxpWritePacket{ + ID: id, + Handle: f.handle, + Offset: uint64(off), + Length: uint32(n), + Data: b[:n], + }) + + select { + case workCh <- work{id, res, off}: + case <-cancel: + return + } + + off += int64(n) + } + + if err != nil { + if !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) { + errCh <- rwErr{off, err} + } + return + } + } + }() + + var wg sync.WaitGroup + wg.Add(concurrency) + for i := 0; i < concurrency; i++ { + // Map_i: each worker gets work, and does the Write from each buffer to its respective offset. + go func() { + defer wg.Done() + + for work := range workCh { + s := <-work.res + pool.Put(work.res) + + err := s.err + if err == nil { + switch s.typ { + case sshFxpStatus: + err = normaliseError(unmarshalStatus(work.id, s.data)) + default: + err = unimplementedPacketErr(s.typ) + } + } + + if err != nil { + errCh <- rwErr{work.off, err} + + // DO NOT return. + // We want to ensure that workCh is drained before wg.Wait returns. + } + } + }() + } + + // Wait for long tail, before closing results. + go func() { + wg.Wait() + close(errCh) + }() + + // Reduce: Collect all the results into a relevant return: the earliest offset to return an error. + firstErr := rwErr{math.MaxInt64, nil} + for rwErr := range errCh { + if rwErr.off <= firstErr.off { + firstErr = rwErr + } + + select { + case <-cancel: + default: + // stop any more work from being distributed. + close(cancel) + } + } + + if firstErr.err != nil { + // firstErr.err != nil if and only if firstErr.off is a valid offset. + // + // firstErr.off will then be the lesser of: + // * the offset of the first error from writing, + // * the last successfully read offset. + // + // This could be less than the last successfully written offset, + // which is the whole reason for the UseConcurrentWrites() ClientOption. + // + // Callers are responsible for truncating any SFTP files to a safe length. + f.offset = firstErr.off + + // ReadFrom is defined to return the read bytes, regardless of any writer errors. + return read, firstErr.err + } + + f.offset += read + return read, nil +} + +// ReadFrom reads data from r until EOF and writes it to the file. The return +// value is the number of bytes read. Any error except io.EOF encountered +// during the read is also returned. +// +// This method is preferred over calling Write multiple times +// to maximise throughput for transferring the entire file, +// especially over high-latency links. +// +// To ensure concurrent writes, the given r needs to implement one of +// the following receiver methods: +// +// Len() int +// Size() int64 +// Stat() (os.FileInfo, error) +// +// or be an instance of [io.LimitedReader] to determine the number of possible +// concurrent requests. Otherwise, reads/writes are performed sequentially. +// ReadFromWithConcurrency can be used explicitly to guarantee concurrent +// processing of the reader. +func (f *File) ReadFrom(r io.Reader) (int64, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.handle == "" { + return 0, os.ErrClosed + } + + if f.c.useConcurrentWrites { + var remain int64 + switch r := r.(type) { + case interface{ Len() int }: + remain = int64(r.Len()) + + case interface{ Size() int64 }: + remain = r.Size() + + case *io.LimitedReader: + remain = r.N + + case interface{ Stat() (os.FileInfo, error) }: + info, err := r.Stat() + if err == nil { + remain = info.Size() + } + } + + if remain < 0 { + // We can strongly assert that we want default max concurrency here. + return f.readFromWithConcurrency(r, f.c.maxConcurrentRequests) + } + + if remain > int64(f.c.maxPacket) { + // Otherwise, only use concurrency, if it would be at least two packets. + + // This is the best reasonable guess we can make. + concurrency64 := remain/int64(f.c.maxPacket) + 1 + + // We need to cap this value to an `int` size value to avoid overflow on 32-bit machines. + // So, we may as well pre-cap it to `f.c.maxConcurrentRequests`. + if concurrency64 > int64(f.c.maxConcurrentRequests) { + concurrency64 = int64(f.c.maxConcurrentRequests) + } + + return f.readFromWithConcurrency(r, int(concurrency64)) + } + } + + ch := make(chan result, 1) // reusable channel + + b := make([]byte, f.c.maxPacket) + + var read int64 + for { + // Fill the entire buffer. + n, err := io.ReadFull(r, b) + if n < 0 { + panic("sftp.File: reader returned negative count from Read") + } + + if n > 0 { + read += int64(n) + + m, err2 := f.writeChunkAt(ch, b[:n], f.offset) + f.offset += int64(m) + + if err == nil { + err = err2 + } + } + + if err != nil { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return read, nil // return nil explicitly. + } + + return read, err + } + } +} + +// Seek implements io.Seeker by setting the client offset for the next Read or +// Write. It returns the next offset read. Seeking before or after the end of +// the file is undefined. Seeking relative to the end calls Stat. +func (f *File) Seek(offset int64, whence int) (int64, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.handle == "" { + return 0, os.ErrClosed + } + + switch whence { + case io.SeekStart: + case io.SeekCurrent: + offset += f.offset + case io.SeekEnd: + fi, err := f.stat() + if err != nil { + return f.offset, err + } + offset += fi.Size() + default: + return f.offset, unimplementedSeekWhence(whence) + } + + if offset < 0 { + return f.offset, os.ErrInvalid + } + + f.offset = offset + return f.offset, nil +} + +// Chown changes the uid/gid of the current file. +func (f *File) Chown(uid, gid int) error { + f.mu.RLock() + defer f.mu.RUnlock() + + if f.handle == "" { + return os.ErrClosed + } + + return f.c.fsetstat(f.handle, sshFileXferAttrUIDGID, &FileStat{ + UID: uint32(uid), + GID: uint32(gid), + }) +} + +// Chmod changes the permissions of the current file. +// +// See Client.Chmod for details. +func (f *File) Chmod(mode os.FileMode) error { + f.mu.RLock() + defer f.mu.RUnlock() + + if f.handle == "" { + return os.ErrClosed + } + + return f.c.fsetstat(f.handle, sshFileXferAttrPermissions, toChmodPerm(mode)) +} + +// SetExtendedData sets extended attributes of the current file. It uses the +// SSH_FILEXFER_ATTR_EXTENDED flag in the setstat request. +// +// This flag provides a general extension mechanism for vendor-specific extensions. +// Names of the attributes should be a string of the format "name@domain", where "domain" +// is a valid, registered domain name and "name" identifies the method. Server +// implementations SHOULD ignore extended data fields that they do not understand. +func (f *File) SetExtendedData(path string, extended []StatExtended) error { + f.mu.RLock() + defer f.mu.RUnlock() + + if f.handle == "" { + return os.ErrClosed + } + + attrs := &FileStat{ + Extended: extended, + } + + return f.c.fsetstat(f.handle, sshFileXferAttrExtended, attrs) +} + +// Truncate sets the size of the current file. Although it may be safely assumed +// that if the size is less than its current size it will be truncated to fit, +// the SFTP protocol does not specify what behavior the server should do when setting +// size greater than the current size. +// We send a SSH_FXP_FSETSTAT here since we have a file handle +func (f *File) Truncate(size int64) error { + f.mu.RLock() + defer f.mu.RUnlock() + + if f.handle == "" { + return os.ErrClosed + } + + return f.c.fsetstat(f.handle, sshFileXferAttrSize, uint64(size)) +} + +// Sync requests a flush of the contents of a File to stable storage. +// +// Sync requires the server to support the fsync@openssh.com extension. +func (f *File) Sync() error { + f.mu.Lock() + defer f.mu.Unlock() + + if f.handle == "" { + return os.ErrClosed + } + + if data, ok := f.c.HasExtension(openssh.ExtensionFSync().Name); !ok || data != "1" { + return &StatusError{ + Code: sshFxOPUnsupported, + msg: "fsync not supported", + } + } + + id := f.c.nextID() + typ, data, err := f.c.sendPacket(context.Background(), nil, &sshFxpFsyncPacket{ + ID: id, + Handle: f.handle, + }) + + switch { + case err != nil: + return err + case typ == sshFxpStatus: + return normaliseError(unmarshalStatus(id, data)) + default: + return &unexpectedPacketErr{want: sshFxpStatus, got: typ} + } +} + +// normaliseError normalises an error into a more standard form that can be +// checked against stdlib errors like io.EOF or os.ErrNotExist. +func normaliseError(err error) error { + switch err := err.(type) { + case *StatusError: + switch err.Code { + case sshFxEOF: + return io.EOF + case sshFxNoSuchFile: + return os.ErrNotExist + case sshFxPermissionDenied: + return os.ErrPermission + case sshFxOk: + return nil + default: + return err + } + default: + return err + } +} + +// flags converts the flags passed to OpenFile into ssh flags. +// Unsupported flags are ignored. +func toPflags(f int) uint32 { + var out uint32 + switch f & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR) { + case os.O_RDONLY: + out |= sshFxfRead + case os.O_WRONLY: + out |= sshFxfWrite + case os.O_RDWR: + out |= sshFxfRead | sshFxfWrite + } + if f&os.O_APPEND == os.O_APPEND { + out |= sshFxfAppend + } + if f&os.O_CREATE == os.O_CREATE { + out |= sshFxfCreat + } + if f&os.O_TRUNC == os.O_TRUNC { + out |= sshFxfTrunc + } + if f&os.O_EXCL == os.O_EXCL { + out |= sshFxfExcl + } + return out +} + +// toChmodPerm converts Go permission bits to POSIX permission bits. +// +// This differs from fromFileMode in that we preserve the POSIX versions of +// setuid, setgid and sticky in m, because we've historically supported those +// bits, and we mask off any non-permission bits. +func toChmodPerm(m os.FileMode) (perm uint32) { + const mask = os.ModePerm | os.FileMode(s_ISUID|s_ISGID|s_ISVTX) + perm = uint32(m & mask) + + if m&os.ModeSetuid != 0 { + perm |= s_ISUID + } + if m&os.ModeSetgid != 0 { + perm |= s_ISGID + } + if m&os.ModeSticky != 0 { + perm |= s_ISVTX + } + + return perm +} diff --git a/vendor/github.com/pkg/sftp/conn.go b/vendor/github.com/pkg/sftp/conn.go new file mode 100644 index 00000000000..e68a2bd06ff --- /dev/null +++ b/vendor/github.com/pkg/sftp/conn.go @@ -0,0 +1,212 @@ +package sftp + +import ( + "context" + "encoding" + "fmt" + "io" + "sync" +) + +// conn implements a bidirectional channel on which client and server +// connections are multiplexed. +type conn struct { + io.Reader + io.WriteCloser + // this is the same allocator used in packet manager + alloc *allocator + sync.Mutex // used to serialise writes to sendPacket +} + +// the orderID is used in server mode if the allocator is enabled. +// For the client mode just pass 0. +// It returns io.EOF if the connection is closed and +// there are no more packets to read. +func (c *conn) recvPacket(orderID uint32) (fxp, []byte, error) { + return recvPacket(c, c.alloc, orderID) +} + +func (c *conn) sendPacket(m encoding.BinaryMarshaler) error { + c.Lock() + defer c.Unlock() + + return sendPacket(c, m) +} + +func (c *conn) Close() error { + c.Lock() + defer c.Unlock() + return c.WriteCloser.Close() +} + +type clientConn struct { + conn + wg sync.WaitGroup + + wait func() error // if non-nil, call this during Wait() to get a possible remote status error. + + sync.Mutex // protects inflight + inflight map[uint32]chan<- result // outstanding requests + + closed chan struct{} + err error +} + +// Wait blocks until the conn has shut down, and return the error +// causing the shutdown. It can be called concurrently from multiple +// goroutines. +func (c *clientConn) Wait() error { + <-c.closed + + if c.wait == nil { + // Only return this error if c.wait won't return something more useful. + return c.err + } + + if err := c.wait(); err != nil { + + // TODO: when https://github.com/golang/go/issues/35025 is fixed, + // we can remove this if block entirely. + // Right now, it’s always going to return this, so it is not useful. + // But we have this code here so that as soon as the ssh library is updated, + // we can return a possibly more useful error. + if err.Error() == "ssh: session not started" { + return c.err + } + + return err + } + + // c.wait returned no error; so, let's return something maybe more useful. + return c.err +} + +// Close closes the SFTP session. +func (c *clientConn) Close() error { + defer c.wg.Wait() + return c.conn.Close() +} + +// recv continuously reads from the server and forwards responses to the +// appropriate channel. +func (c *clientConn) recv() error { + defer c.conn.Close() + + for { + typ, data, err := c.recvPacket(0) + if err != nil { + return err + } + sid, _, err := unmarshalUint32Safe(data) + if err != nil { + return err + } + + ch, ok := c.getChannel(sid) + if !ok { + // This is an unexpected occurrence. Send the error + // back to all listeners so that they terminate + // gracefully. + return fmt.Errorf("sid not found: %d", sid) + } + + ch <- result{typ: typ, data: data} + } +} + +func (c *clientConn) putChannel(ch chan<- result, sid uint32) bool { + c.Lock() + defer c.Unlock() + + select { + case <-c.closed: + // already closed with broadcastErr, return error on chan. + ch <- result{err: ErrSSHFxConnectionLost} + return false + default: + } + + c.inflight[sid] = ch + return true +} + +func (c *clientConn) getChannel(sid uint32) (chan<- result, bool) { + c.Lock() + defer c.Unlock() + + ch, ok := c.inflight[sid] + delete(c.inflight, sid) + + return ch, ok +} + +// result captures the result of receiving the a packet from the server +type result struct { + typ fxp + data []byte + err error +} + +type idmarshaler interface { + id() uint32 + encoding.BinaryMarshaler +} + +func (c *clientConn) sendPacket(ctx context.Context, ch chan result, p idmarshaler) (fxp, []byte, error) { + if cap(ch) < 1 { + ch = make(chan result, 1) + } + + c.dispatchRequest(ch, p) + + select { + case <-ctx.Done(): + return 0, nil, ctx.Err() + case s := <-ch: + return s.typ, s.data, s.err + } +} + +// dispatchRequest should ideally only be called by race-detection tests outside of this file, +// where you have to ensure two packets are in flight sequentially after each other. +func (c *clientConn) dispatchRequest(ch chan<- result, p idmarshaler) { + sid := p.id() + + if !c.putChannel(ch, sid) { + // already closed. + return + } + + if err := c.conn.sendPacket(p); err != nil { + if ch, ok := c.getChannel(sid); ok { + ch <- result{err: err} + } + } +} + +// broadcastErr sends an error to all goroutines waiting for a response. +func (c *clientConn) broadcastErr(err error) { + c.Lock() + defer c.Unlock() + + bcastRes := result{err: ErrSSHFxConnectionLost} + for sid, ch := range c.inflight { + ch <- bcastRes + + // Replace the chan in inflight, + // we have hijacked this chan, + // and this guarantees always-only-once sending. + c.inflight[sid] = make(chan<- result, 1) + } + + c.err = err + close(c.closed) +} + +type serverConn struct { + conn +} + +func (s *serverConn) sendError(id uint32, err error) error { + return s.sendPacket(statusFromError(id, err)) +} diff --git a/vendor/github.com/pkg/sftp/debug.go b/vendor/github.com/pkg/sftp/debug.go new file mode 100644 index 00000000000..f0db14d3ace --- /dev/null +++ b/vendor/github.com/pkg/sftp/debug.go @@ -0,0 +1,10 @@ +//go:build debug +// +build debug + +package sftp + +import "log" + +func debug(fmt string, args ...interface{}) { + log.Printf(fmt, args...) +} diff --git a/vendor/github.com/pkg/sftp/errno_plan9.go b/vendor/github.com/pkg/sftp/errno_plan9.go new file mode 100644 index 00000000000..cf9d39029a8 --- /dev/null +++ b/vendor/github.com/pkg/sftp/errno_plan9.go @@ -0,0 +1,42 @@ +package sftp + +import ( + "os" + "syscall" +) + +var EBADF = syscall.NewError("fd out of range or not open") + +func wrapPathError(filepath string, err error) error { + if errno, ok := err.(syscall.ErrorString); ok { + return &os.PathError{Path: filepath, Err: errno} + } + return err +} + +// translateErrno translates a syscall error number to a SFTP error code. +func translateErrno(errno syscall.ErrorString) uint32 { + switch errno { + case "": + return sshFxOk + case syscall.ENOENT: + return sshFxNoSuchFile + case syscall.EPERM: + return sshFxPermissionDenied + } + + return sshFxFailure +} + +func translateSyscallError(err error) (uint32, bool) { + switch e := err.(type) { + case syscall.ErrorString: + return translateErrno(e), true + case *os.PathError: + debug("statusFromError,pathError: error is %T %#v", e.Err, e.Err) + if errno, ok := e.Err.(syscall.ErrorString); ok { + return translateErrno(errno), true + } + } + return 0, false +} diff --git a/vendor/github.com/pkg/sftp/errno_posix.go b/vendor/github.com/pkg/sftp/errno_posix.go new file mode 100644 index 00000000000..cd87e1b5c49 --- /dev/null +++ b/vendor/github.com/pkg/sftp/errno_posix.go @@ -0,0 +1,45 @@ +//go:build !plan9 +// +build !plan9 + +package sftp + +import ( + "os" + "syscall" +) + +const EBADF = syscall.EBADF + +func wrapPathError(filepath string, err error) error { + if errno, ok := err.(syscall.Errno); ok { + return &os.PathError{Path: filepath, Err: errno} + } + return err +} + +// translateErrno translates a syscall error number to a SFTP error code. +func translateErrno(errno syscall.Errno) uint32 { + switch errno { + case 0: + return sshFxOk + case syscall.ENOENT: + return sshFxNoSuchFile + case syscall.EACCES, syscall.EPERM: + return sshFxPermissionDenied + } + + return sshFxFailure +} + +func translateSyscallError(err error) (uint32, bool) { + switch e := err.(type) { + case syscall.Errno: + return translateErrno(e), true + case *os.PathError: + debug("statusFromError,pathError: error is %T %#v", e.Err, e.Err) + if errno, ok := e.Err.(syscall.Errno); ok { + return translateErrno(errno), true + } + } + return 0, false +} diff --git a/vendor/github.com/pkg/sftp/fuzz.go b/vendor/github.com/pkg/sftp/fuzz.go new file mode 100644 index 00000000000..f2f1fc31c1d --- /dev/null +++ b/vendor/github.com/pkg/sftp/fuzz.go @@ -0,0 +1,23 @@ +//go:build gofuzz +// +build gofuzz + +package sftp + +import "bytes" + +type sinkfuzz struct{} + +func (*sinkfuzz) Close() error { return nil } +func (*sinkfuzz) Write(p []byte) (int, error) { return len(p), nil } + +var devnull = &sinkfuzz{} + +// To run: go-fuzz-build && go-fuzz +func Fuzz(data []byte) int { + c, err := NewClientPipe(bytes.NewReader(data), devnull) + if err != nil { + return 0 + } + c.Close() + return 1 +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/attrs.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/attrs.go new file mode 100644 index 00000000000..3aec937f04d --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/attrs.go @@ -0,0 +1,296 @@ +package sshfx + +// Attributes related flags. +const ( + AttrSize = 1 << iota // SSH_FILEXFER_ATTR_SIZE + AttrUIDGID // SSH_FILEXFER_ATTR_UIDGID + AttrPermissions // SSH_FILEXFER_ATTR_PERMISSIONS + AttrACModTime // SSH_FILEXFER_ACMODTIME + + AttrExtended = 1 << 31 // SSH_FILEXFER_ATTR_EXTENDED +) + +// Attributes defines the file attributes type defined in draft-ietf-secsh-filexfer-02 +// +// Defined in: https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-5 +type Attributes struct { + Flags uint32 + + // AttrSize + Size uint64 + + // AttrUIDGID + UID uint32 + GID uint32 + + // AttrPermissions + Permissions FileMode + + // AttrACmodTime + ATime uint32 + MTime uint32 + + // AttrExtended + ExtendedAttributes []ExtendedAttribute +} + +// GetSize returns the Size field and a bool that is true if and only if the value is valid/defined. +func (a *Attributes) GetSize() (size uint64, ok bool) { + return a.Size, a.Flags&AttrSize != 0 +} + +// SetSize is a convenience function that sets the Size field, +// and marks the field as valid/defined in Flags. +func (a *Attributes) SetSize(size uint64) { + a.Flags |= AttrSize + a.Size = size +} + +// GetUIDGID returns the UID and GID fields and a bool that is true if and only if the values are valid/defined. +func (a *Attributes) GetUIDGID() (uid, gid uint32, ok bool) { + return a.UID, a.GID, a.Flags&AttrUIDGID != 0 +} + +// SetUIDGID is a convenience function that sets the UID and GID fields, +// and marks the fields as valid/defined in Flags. +func (a *Attributes) SetUIDGID(uid, gid uint32) { + a.Flags |= AttrUIDGID + a.UID = uid + a.GID = gid +} + +// GetPermissions returns the Permissions field and a bool that is true if and only if the value is valid/defined. +func (a *Attributes) GetPermissions() (perms FileMode, ok bool) { + return a.Permissions, a.Flags&AttrPermissions != 0 +} + +// SetPermissions is a convenience function that sets the Permissions field, +// and marks the field as valid/defined in Flags. +func (a *Attributes) SetPermissions(perms FileMode) { + a.Flags |= AttrPermissions + a.Permissions = perms +} + +// GetACModTime returns the ATime and MTime fields and a bool that is true if and only if the values are valid/defined. +func (a *Attributes) GetACModTime() (atime, mtime uint32, ok bool) { + return a.ATime, a.MTime, a.Flags&AttrACModTime != 0 +} + +// SetACModTime is a convenience function that sets the ATime and MTime fields, +// and marks the fields as valid/defined in Flags. +func (a *Attributes) SetACModTime(atime, mtime uint32) { + a.Flags |= AttrACModTime + a.ATime = atime + a.MTime = mtime +} + +// Len returns the number of bytes a would marshal into. +func (a *Attributes) Len() int { + length := 4 + + if a.Flags&AttrSize != 0 { + length += 8 + } + + if a.Flags&AttrUIDGID != 0 { + length += 4 + 4 + } + + if a.Flags&AttrPermissions != 0 { + length += 4 + } + + if a.Flags&AttrACModTime != 0 { + length += 4 + 4 + } + + if a.Flags&AttrExtended != 0 { + length += 4 + + for _, ext := range a.ExtendedAttributes { + length += ext.Len() + } + } + + return length +} + +// MarshalInto marshals e onto the end of the given Buffer. +func (a *Attributes) MarshalInto(buf *Buffer) { + buf.AppendUint32(a.Flags) + + if a.Flags&AttrSize != 0 { + buf.AppendUint64(a.Size) + } + + if a.Flags&AttrUIDGID != 0 { + buf.AppendUint32(a.UID) + buf.AppendUint32(a.GID) + } + + if a.Flags&AttrPermissions != 0 { + buf.AppendUint32(uint32(a.Permissions)) + } + + if a.Flags&AttrACModTime != 0 { + buf.AppendUint32(a.ATime) + buf.AppendUint32(a.MTime) + } + + if a.Flags&AttrExtended != 0 { + buf.AppendUint32(uint32(len(a.ExtendedAttributes))) + + for _, ext := range a.ExtendedAttributes { + ext.MarshalInto(buf) + } + } +} + +// MarshalBinary returns a as the binary encoding of a. +func (a *Attributes) MarshalBinary() ([]byte, error) { + buf := NewBuffer(make([]byte, 0, a.Len())) + a.MarshalInto(buf) + return buf.Bytes(), nil +} + +// UnmarshalFrom unmarshals an Attributes from the given Buffer into e. +// +// NOTE: The values of fields not covered in the a.Flags are explicitly undefined. +func (a *Attributes) UnmarshalFrom(buf *Buffer) (err error) { + flags := buf.ConsumeUint32() + + return a.XXX_UnmarshalByFlags(flags, buf) +} + +// XXX_UnmarshalByFlags uses the pre-existing a.Flags field to determine which fields to decode. +// DO NOT USE THIS: it is an anti-corruption function to implement existing internal usage in pkg/sftp. +// This function is not a part of any compatibility promise. +func (a *Attributes) XXX_UnmarshalByFlags(flags uint32, buf *Buffer) (err error) { + a.Flags = flags + + // Short-circuit dummy attributes. + if a.Flags == 0 { + return buf.Err + } + + if a.Flags&AttrSize != 0 { + a.Size = buf.ConsumeUint64() + } + + if a.Flags&AttrUIDGID != 0 { + a.UID = buf.ConsumeUint32() + a.GID = buf.ConsumeUint32() + } + + if a.Flags&AttrPermissions != 0 { + a.Permissions = FileMode(buf.ConsumeUint32()) + } + + if a.Flags&AttrACModTime != 0 { + a.ATime = buf.ConsumeUint32() + a.MTime = buf.ConsumeUint32() + } + + if a.Flags&AttrExtended != 0 { + count := buf.ConsumeCount() + + a.ExtendedAttributes = make([]ExtendedAttribute, count) + for i := range a.ExtendedAttributes { + a.ExtendedAttributes[i].UnmarshalFrom(buf) + } + } + + return buf.Err +} + +// UnmarshalBinary decodes the binary encoding of Attributes into e. +func (a *Attributes) UnmarshalBinary(data []byte) error { + return a.UnmarshalFrom(NewBuffer(data)) +} + +// ExtendedAttribute defines the extended file attribute type defined in draft-ietf-secsh-filexfer-02 +// +// Defined in: https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-5 +type ExtendedAttribute struct { + Type string + Data string +} + +// Len returns the number of bytes e would marshal into. +func (e *ExtendedAttribute) Len() int { + return 4 + len(e.Type) + 4 + len(e.Data) +} + +// MarshalInto marshals e onto the end of the given Buffer. +func (e *ExtendedAttribute) MarshalInto(buf *Buffer) { + buf.AppendString(e.Type) + buf.AppendString(e.Data) +} + +// MarshalBinary returns e as the binary encoding of e. +func (e *ExtendedAttribute) MarshalBinary() ([]byte, error) { + buf := NewBuffer(make([]byte, 0, e.Len())) + e.MarshalInto(buf) + return buf.Bytes(), nil +} + +// UnmarshalFrom unmarshals an ExtendedAattribute from the given Buffer into e. +func (e *ExtendedAttribute) UnmarshalFrom(buf *Buffer) (err error) { + *e = ExtendedAttribute{ + Type: buf.ConsumeString(), + Data: buf.ConsumeString(), + } + + return buf.Err +} + +// UnmarshalBinary decodes the binary encoding of ExtendedAttribute into e. +func (e *ExtendedAttribute) UnmarshalBinary(data []byte) error { + return e.UnmarshalFrom(NewBuffer(data)) +} + +// NameEntry implements the SSH_FXP_NAME repeated data type from draft-ietf-secsh-filexfer-02 +// +// This type is incompatible with versions 4 or higher. +type NameEntry struct { + Filename string + Longname string + Attrs Attributes +} + +// Len returns the number of bytes e would marshal into. +func (e *NameEntry) Len() int { + return 4 + len(e.Filename) + 4 + len(e.Longname) + e.Attrs.Len() +} + +// MarshalInto marshals e onto the end of the given Buffer. +func (e *NameEntry) MarshalInto(buf *Buffer) { + buf.AppendString(e.Filename) + buf.AppendString(e.Longname) + + e.Attrs.MarshalInto(buf) +} + +// MarshalBinary returns e as the binary encoding of e. +func (e *NameEntry) MarshalBinary() ([]byte, error) { + buf := NewBuffer(make([]byte, 0, e.Len())) + e.MarshalInto(buf) + return buf.Bytes(), nil +} + +// UnmarshalFrom unmarshals an NameEntry from the given Buffer into e. +// +// NOTE: The values of fields not covered in the a.Flags are explicitly undefined. +func (e *NameEntry) UnmarshalFrom(buf *Buffer) (err error) { + *e = NameEntry{ + Filename: buf.ConsumeString(), + Longname: buf.ConsumeString(), + } + + return e.Attrs.UnmarshalFrom(buf) +} + +// UnmarshalBinary decodes the binary encoding of NameEntry into e. +func (e *NameEntry) UnmarshalBinary(data []byte) error { + return e.UnmarshalFrom(NewBuffer(data)) +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/buffer.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/buffer.go new file mode 100644 index 00000000000..bd4783bb880 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/buffer.go @@ -0,0 +1,340 @@ +package sshfx + +import ( + "encoding/binary" + "errors" +) + +// Various encoding errors. +var ( + ErrShortPacket = errors.New("packet too short") + ErrLongPacket = errors.New("packet too long") +) + +// Buffer wraps up the various encoding details of the SSH format. +// +// Data types are encoded as per section 4 from https://tools.ietf.org/html/draft-ietf-secsh-architecture-09#page-8 +type Buffer struct { + b []byte + off int + Err error +} + +// NewBuffer creates and initializes a new buffer using buf as its initial contents. +// The new buffer takes ownership of buf, and the caller should not use buf after this call. +// +// In most cases, new(Buffer) (or just declaring a Buffer variable) is sufficient to initialize a Buffer. +func NewBuffer(buf []byte) *Buffer { + return &Buffer{ + b: buf, + } +} + +// NewMarshalBuffer creates a new Buffer ready to start marshaling a Packet into. +// It preallocates enough space for uint32(length), uint8(type), uint32(request-id) and size more bytes. +func NewMarshalBuffer(size int) *Buffer { + return NewBuffer(make([]byte, 4+1+4+size)) +} + +// Bytes returns a slice of length b.Len() holding the unconsumed bytes in the Buffer. +// The slice is valid for use only until the next buffer modification +// (that is, only until the next call to an Append or Consume method). +func (b *Buffer) Bytes() []byte { + return b.b[b.off:] +} + +// Len returns the number of unconsumed bytes in the buffer. +func (b *Buffer) Len() int { return len(b.b) - b.off } + +// Cap returns the capacity of the buffer’s underlying byte slice, +// that is, the total space allocated for the buffer’s data. +func (b *Buffer) Cap() int { return cap(b.b) } + +// Reset resets the buffer to be empty, but it retains the underlying storage for use by future Appends. +func (b *Buffer) Reset() { + *b = Buffer{ + b: b.b[:0], + } +} + +// StartPacket resets and initializes the buffer to be ready to start marshaling a packet into. +// It truncates the buffer, reserves space for uint32(length), then appends the given packetType and requestID. +func (b *Buffer) StartPacket(packetType PacketType, requestID uint32) { + *b = Buffer{ + b: append(b.b[:0], make([]byte, 4)...), + } + + b.AppendUint8(uint8(packetType)) + b.AppendUint32(requestID) +} + +// Packet finalizes the packet started from StartPacket. +// It is expected that this will end the ownership of the underlying byte-slice, +// and so the returned byte-slices may be reused the same as any other byte-slice, +// the caller should not use this buffer after this call. +// +// It writes the packet body length into the first four bytes of the buffer in network byte order (big endian). +// The packet body length is the length of this buffer less the 4-byte length itself, plus the length of payload. +// +// It is assumed that no Consume methods have been called on this buffer, +// and so it returns the whole underlying slice. +func (b *Buffer) Packet(payload []byte) (header, payloadPassThru []byte, err error) { + b.PutLength(len(b.b) - 4 + len(payload)) + + return b.b, payload, nil +} + +// ConsumeUint8 consumes a single byte from the buffer. +// If the buffer does not have enough data, it will set Err to ErrShortPacket. +func (b *Buffer) ConsumeUint8() uint8 { + if b.Err != nil { + return 0 + } + + if b.Len() < 1 { + b.off = len(b.b) + b.Err = ErrShortPacket + return 0 + } + + var v uint8 + v, b.off = b.b[b.off], b.off+1 + return v +} + +// AppendUint8 appends a single byte into the buffer. +func (b *Buffer) AppendUint8(v uint8) { + b.b = append(b.b, v) +} + +// ConsumeBool consumes a single byte from the buffer, and returns true if that byte is non-zero. +// If the buffer does not have enough data, it will set Err to ErrShortPacket. +func (b *Buffer) ConsumeBool() bool { + return b.ConsumeUint8() != 0 +} + +// AppendBool appends a single bool into the buffer. +// It encodes it as a single byte, with false as 0, and true as 1. +func (b *Buffer) AppendBool(v bool) { + if v { + b.AppendUint8(1) + } else { + b.AppendUint8(0) + } +} + +// ConsumeUint16 consumes a single uint16 from the buffer, in network byte order (big-endian). +// If the buffer does not have enough data, it will set Err to ErrShortPacket. +func (b *Buffer) ConsumeUint16() uint16 { + if b.Err != nil { + return 0 + } + + if b.Len() < 2 { + b.off = len(b.b) + b.Err = ErrShortPacket + return 0 + } + + v := binary.BigEndian.Uint16(b.b[b.off:]) + b.off += 2 + return v +} + +// AppendUint16 appends single uint16 into the buffer, in network byte order (big-endian). +func (b *Buffer) AppendUint16(v uint16) { + b.b = append(b.b, + byte(v>>8), + byte(v>>0), + ) +} + +// unmarshalUint32 is used internally to read the packet length. +// It is unsafe, and so not exported. +// Even within this package, its use should be avoided. +func unmarshalUint32(b []byte) uint32 { + return binary.BigEndian.Uint32(b[:4]) +} + +// ConsumeUint32 consumes a single uint32 from the buffer, in network byte order (big-endian). +// If the buffer does not have enough data, it will set Err to ErrShortPacket. +func (b *Buffer) ConsumeUint32() uint32 { + if b.Err != nil { + return 0 + } + + if b.Len() < 4 { + b.off = len(b.b) + b.Err = ErrShortPacket + return 0 + } + + v := binary.BigEndian.Uint32(b.b[b.off:]) + b.off += 4 + return v +} + +// AppendUint32 appends a single uint32 into the buffer, in network byte order (big-endian). +func (b *Buffer) AppendUint32(v uint32) { + b.b = append(b.b, + byte(v>>24), + byte(v>>16), + byte(v>>8), + byte(v>>0), + ) +} + +// ConsumeCount consumes a single uint32 count from the buffer, in network byte order (big-endian) as an int. +// If the buffer does not have enough data, it will set Err to ErrShortPacket. +func (b *Buffer) ConsumeCount() int { + return int(b.ConsumeUint32()) +} + +// AppendCount appends a single int length as a uint32 into the buffer, in network byte order (big-endian). +func (b *Buffer) AppendCount(v int) { + b.AppendUint32(uint32(v)) +} + +// ConsumeUint64 consumes a single uint64 from the buffer, in network byte order (big-endian). +// If the buffer does not have enough data, it will set Err to ErrShortPacket. +func (b *Buffer) ConsumeUint64() uint64 { + if b.Err != nil { + return 0 + } + + if b.Len() < 8 { + b.off = len(b.b) + b.Err = ErrShortPacket + return 0 + } + + v := binary.BigEndian.Uint64(b.b[b.off:]) + b.off += 8 + return v +} + +// AppendUint64 appends a single uint64 into the buffer, in network byte order (big-endian). +func (b *Buffer) AppendUint64(v uint64) { + b.b = append(b.b, + byte(v>>56), + byte(v>>48), + byte(v>>40), + byte(v>>32), + byte(v>>24), + byte(v>>16), + byte(v>>8), + byte(v>>0), + ) +} + +// ConsumeInt64 consumes a single int64 from the buffer, in network byte order (big-endian) with two’s complement. +// If the buffer does not have enough data, it will set Err to ErrShortPacket. +func (b *Buffer) ConsumeInt64() int64 { + return int64(b.ConsumeUint64()) +} + +// AppendInt64 appends a single int64 into the buffer, in network byte order (big-endian) with two’s complement. +func (b *Buffer) AppendInt64(v int64) { + b.AppendUint64(uint64(v)) +} + +// ConsumeByteSlice consumes a single string of raw binary data from the buffer. +// A string is a uint32 length, followed by that number of raw bytes. +// If the buffer does not have enough data, or defines a length larger than available, it will set Err to ErrShortPacket. +// +// The returned slice aliases the buffer contents, and is valid only as long as the buffer is not reused +// (that is, only until the next call to Reset, PutLength, StartPacket, or UnmarshalBinary). +// +// In no case will any Consume calls return overlapping slice aliases, +// and Append calls are guaranteed to not disturb this slice alias. +func (b *Buffer) ConsumeByteSlice() []byte { + length := int(b.ConsumeUint32()) + if b.Err != nil { + return nil + } + + if b.Len() < length || length < 0 { + b.off = len(b.b) + b.Err = ErrShortPacket + return nil + } + + v := b.b[b.off:] + if len(v) > length || cap(v) > length { + v = v[:length:length] + } + b.off += int(length) + return v +} + +// ConsumeByteSliceCopy consumes a single string of raw binary data as a copy from the buffer. +// A string is a uint32 length, followed by that number of raw bytes. +// If the buffer does not have enough data, or defines a length larger than available, it will set Err to ErrShortPacket. +// +// The returned slice does not alias any buffer contents, +// and will therefore be valid even if the buffer is later reused. +// +// If hint has sufficient capacity to hold the data, it will be reused and overwritten, +// otherwise a new backing slice will be allocated and returned. +func (b *Buffer) ConsumeByteSliceCopy(hint []byte) []byte { + data := b.ConsumeByteSlice() + + if grow := len(data) - len(hint); grow > 0 { + hint = append(hint, make([]byte, grow)...) + } + + n := copy(hint, data) + hint = hint[:n] + return hint +} + +// AppendByteSlice appends a single string of raw binary data into the buffer. +// A string is a uint32 length, followed by that number of raw bytes. +func (b *Buffer) AppendByteSlice(v []byte) { + b.AppendUint32(uint32(len(v))) + b.b = append(b.b, v...) +} + +// ConsumeString consumes a single string of binary data from the buffer. +// A string is a uint32 length, followed by that number of raw bytes. +// If the buffer does not have enough data, or defines a length larger than available, it will set Err to ErrShortPacket. +// +// NOTE: Go implicitly assumes that strings contain UTF-8 encoded data. +// All caveats on using arbitrary binary data in Go strings applies. +func (b *Buffer) ConsumeString() string { + return string(b.ConsumeByteSlice()) +} + +// AppendString appends a single string of binary data into the buffer. +// A string is a uint32 length, followed by that number of raw bytes. +func (b *Buffer) AppendString(v string) { + b.AppendByteSlice([]byte(v)) +} + +// PutLength writes the given size into the first four bytes of the buffer in network byte order (big endian). +func (b *Buffer) PutLength(size int) { + if len(b.b) < 4 { + b.b = append(b.b, make([]byte, 4-len(b.b))...) + } + + binary.BigEndian.PutUint32(b.b, uint32(size)) +} + +// MarshalBinary returns a clone of the full internal buffer. +func (b *Buffer) MarshalBinary() ([]byte, error) { + clone := make([]byte, len(b.b)) + n := copy(clone, b.b) + return clone[:n], nil +} + +// UnmarshalBinary sets the internal buffer of b to be a clone of data, and zeros the internal offset. +func (b *Buffer) UnmarshalBinary(data []byte) error { + if grow := len(data) - len(b.b); grow > 0 { + b.b = append(b.b, make([]byte, grow)...) + } + + n := copy(b.b, data) + b.b = b.b[:n] + b.off = 0 + return nil +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/extended_packets.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/extended_packets.go new file mode 100644 index 00000000000..f717425300f --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/extended_packets.go @@ -0,0 +1,143 @@ +package sshfx + +import ( + "encoding" + "sync" +) + +// ExtendedData aliases the untyped interface composition of encoding.BinaryMarshaler and encoding.BinaryUnmarshaler. +type ExtendedData = interface { + encoding.BinaryMarshaler + encoding.BinaryUnmarshaler +} + +// ExtendedDataConstructor defines a function that returns a new(ArbitraryExtendedPacket). +type ExtendedDataConstructor func() ExtendedData + +var extendedPacketTypes = struct { + mu sync.RWMutex + constructors map[string]ExtendedDataConstructor +}{ + constructors: make(map[string]ExtendedDataConstructor), +} + +// RegisterExtendedPacketType defines a specific ExtendedDataConstructor for the given extension string. +func RegisterExtendedPacketType(extension string, constructor ExtendedDataConstructor) { + extendedPacketTypes.mu.Lock() + defer extendedPacketTypes.mu.Unlock() + + if _, exist := extendedPacketTypes.constructors[extension]; exist { + panic("encoding/ssh/filexfer: multiple registration of extended packet type " + extension) + } + + extendedPacketTypes.constructors[extension] = constructor +} + +func newExtendedPacket(extension string) ExtendedData { + extendedPacketTypes.mu.RLock() + defer extendedPacketTypes.mu.RUnlock() + + if f := extendedPacketTypes.constructors[extension]; f != nil { + return f() + } + + return new(Buffer) +} + +// ExtendedPacket defines the SSH_FXP_CLOSE packet. +type ExtendedPacket struct { + ExtendedRequest string + + Data ExtendedData +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *ExtendedPacket) Type() PacketType { + return PacketTypeExtended +} + +// MarshalPacket returns p as a two-part binary encoding of p. +// +// The Data is marshaled into binary, and returned as the payload. +func (p *ExtendedPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.ExtendedRequest) // string(extended-request) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeExtended, reqid) + buf.AppendString(p.ExtendedRequest) + + if p.Data != nil { + payload, err = p.Data.MarshalBinary() + if err != nil { + return nil, nil, err + } + } + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +// +// If p.Data is nil, and the extension has been registered, a new type will be made from the registration. +// If the extension has not been registered, then a new Buffer will be allocated. +// Then the request-specific-data will be unmarshaled from the rest of the buffer. +func (p *ExtendedPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + p.ExtendedRequest = buf.ConsumeString() + if buf.Err != nil { + return buf.Err + } + + if p.Data == nil { + p.Data = newExtendedPacket(p.ExtendedRequest) + } + + return p.Data.UnmarshalBinary(buf.Bytes()) +} + +// ExtendedReplyPacket defines the SSH_FXP_CLOSE packet. +type ExtendedReplyPacket struct { + Data ExtendedData +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *ExtendedReplyPacket) Type() PacketType { + return PacketTypeExtendedReply +} + +// MarshalPacket returns p as a two-part binary encoding of p. +// +// The Data is marshaled into binary, and returned as the payload. +func (p *ExtendedReplyPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + buf = NewMarshalBuffer(0) + } + + buf.StartPacket(PacketTypeExtendedReply, reqid) + + if p.Data != nil { + payload, err = p.Data.MarshalBinary() + if err != nil { + return nil, nil, err + } + } + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +// +// If p.Data is nil, and there is request-specific-data, +// then the request-specific-data will be wrapped in a Buffer and assigned to p.Data. +func (p *ExtendedReplyPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + if p.Data == nil { + p.Data = new(Buffer) + } + + return p.Data.UnmarshalBinary(buf.Bytes()) +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/extensions.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/extensions.go new file mode 100644 index 00000000000..c425780ca86 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/extensions.go @@ -0,0 +1,43 @@ +package sshfx + +// ExtensionPair defines the extension-pair type defined in draft-ietf-secsh-filexfer-13. +// This type is backwards-compatible with how draft-ietf-secsh-filexfer-02 defines extensions. +// +// Defined in: https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13#section-4.2 +type ExtensionPair struct { + Name string + Data string +} + +// Len returns the number of bytes e would marshal into. +func (e *ExtensionPair) Len() int { + return 4 + len(e.Name) + 4 + len(e.Data) +} + +// MarshalInto marshals e onto the end of the given Buffer. +func (e *ExtensionPair) MarshalInto(buf *Buffer) { + buf.AppendString(e.Name) + buf.AppendString(e.Data) +} + +// MarshalBinary returns e as the binary encoding of e. +func (e *ExtensionPair) MarshalBinary() ([]byte, error) { + buf := NewBuffer(make([]byte, 0, e.Len())) + e.MarshalInto(buf) + return buf.Bytes(), nil +} + +// UnmarshalFrom unmarshals an ExtensionPair from the given Buffer into e. +func (e *ExtensionPair) UnmarshalFrom(buf *Buffer) (err error) { + *e = ExtensionPair{ + Name: buf.ConsumeString(), + Data: buf.ConsumeString(), + } + + return buf.Err +} + +// UnmarshalBinary decodes the binary encoding of ExtensionPair into e. +func (e *ExtensionPair) UnmarshalBinary(data []byte) error { + return e.UnmarshalFrom(NewBuffer(data)) +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/filexfer.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/filexfer.go new file mode 100644 index 00000000000..d3009994ad4 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/filexfer.go @@ -0,0 +1,54 @@ +// Package sshfx implements the wire encoding for secsh-filexfer as described in https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt +package sshfx + +// PacketMarshaller narrowly defines packets that will only be transmitted. +// +// ExtendedPacket types will often only implement this interface, +// since decoding the whole packet body of an ExtendedPacket can only be done dependent on the ExtendedRequest field. +type PacketMarshaller interface { + // MarshalPacket is the primary intended way to encode a packet. + // The request-id for the packet is set from reqid. + // + // An optional buffer may be given in b. + // If the buffer has a minimum capacity, it shall be truncated and used to marshal the header into. + // The minimum capacity for the packet must be a constant expression, and should be at least 9. + // + // It shall return the main body of the encoded packet in header, + // and may optionally return an additional payload to be written immediately after the header. + // + // It shall encode in the first 4-bytes of the header the proper length of the rest of the header+payload. + MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) +} + +// Packet defines the behavior of a full generic SFTP packet. +// +// InitPacket, and VersionPacket are not generic SFTP packets, and instead implement (Un)MarshalBinary. +// +// ExtendedPacket types should not iplement this interface, +// since decoding the whole packet body of an ExtendedPacket can only be done dependent on the ExtendedRequest field. +type Packet interface { + PacketMarshaller + + // Type returns the SSH_FXP_xy value associated with the specific packet. + Type() PacketType + + // UnmarshalPacketBody decodes a packet body from the given Buffer. + // It is assumed that the common header values of the length, type and request-id have already been consumed. + // + // Implementations should not alias the given Buffer, + // instead they can consider prepopulating an internal buffer as a hint, + // and copying into that buffer if it has sufficient length. + UnmarshalPacketBody(buf *Buffer) error +} + +// ComposePacket converts returns from MarshalPacket into an equivalent call to MarshalBinary. +func ComposePacket(header, payload []byte, err error) ([]byte, error) { + return append(header, payload...), err +} + +// Default length values, +// Defined in draft-ietf-secsh-filexfer-02 section 3. +const ( + DefaultMaxPacketLength = 34000 + DefaultMaxDataLength = 32768 +) diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/fx.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/fx.go new file mode 100644 index 00000000000..9abcbafcbf4 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/fx.go @@ -0,0 +1,147 @@ +package sshfx + +import ( + "fmt" +) + +// Status defines the SFTP error codes used in SSH_FXP_STATUS response packets. +type Status uint32 + +// Defines the various SSH_FX_* values. +const ( + // see draft-ietf-secsh-filexfer-02 + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-7 + StatusOK = Status(iota) + StatusEOF + StatusNoSuchFile + StatusPermissionDenied + StatusFailure + StatusBadMessage + StatusNoConnection + StatusConnectionLost + StatusOPUnsupported + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-03.txt#section-7 + StatusV4InvalidHandle + StatusV4NoSuchPath + StatusV4FileAlreadyExists + StatusV4WriteProtect + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-04.txt#section-7 + StatusV4NoMedia + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-05.txt#section-7 + StatusV5NoSpaceOnFilesystem + StatusV5QuotaExceeded + StatusV5UnknownPrincipal + StatusV5LockConflict + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-06.txt#section-8 + StatusV6DirNotEmpty + StatusV6NotADirectory + StatusV6InvalidFilename + StatusV6LinkLoop + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-07.txt#section-8 + StatusV6CannotDelete + StatusV6InvalidParameter + StatusV6FileIsADirectory + StatusV6ByteRangeLockConflict + StatusV6ByteRangeLockRefused + StatusV6DeletePending + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-08.txt#section-8.1 + StatusV6FileCorrupt + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-10.txt#section-9.1 + StatusV6OwnerInvalid + StatusV6GroupInvalid + + // https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13#section-9.1 + StatusV6NoMatchingByteRangeLock +) + +func (s Status) Error() string { + return s.String() +} + +// Is returns true if the target is the same Status code, +// or target is a StatusPacket with the same Status code. +func (s Status) Is(target error) bool { + if target, ok := target.(*StatusPacket); ok { + return target.StatusCode == s + } + + return s == target +} + +func (s Status) String() string { + switch s { + case StatusOK: + return "SSH_FX_OK" + case StatusEOF: + return "SSH_FX_EOF" + case StatusNoSuchFile: + return "SSH_FX_NO_SUCH_FILE" + case StatusPermissionDenied: + return "SSH_FX_PERMISSION_DENIED" + case StatusFailure: + return "SSH_FX_FAILURE" + case StatusBadMessage: + return "SSH_FX_BAD_MESSAGE" + case StatusNoConnection: + return "SSH_FX_NO_CONNECTION" + case StatusConnectionLost: + return "SSH_FX_CONNECTION_LOST" + case StatusOPUnsupported: + return "SSH_FX_OP_UNSUPPORTED" + case StatusV4InvalidHandle: + return "SSH_FX_INVALID_HANDLE" + case StatusV4NoSuchPath: + return "SSH_FX_NO_SUCH_PATH" + case StatusV4FileAlreadyExists: + return "SSH_FX_FILE_ALREADY_EXISTS" + case StatusV4WriteProtect: + return "SSH_FX_WRITE_PROTECT" + case StatusV4NoMedia: + return "SSH_FX_NO_MEDIA" + case StatusV5NoSpaceOnFilesystem: + return "SSH_FX_NO_SPACE_ON_FILESYSTEM" + case StatusV5QuotaExceeded: + return "SSH_FX_QUOTA_EXCEEDED" + case StatusV5UnknownPrincipal: + return "SSH_FX_UNKNOWN_PRINCIPAL" + case StatusV5LockConflict: + return "SSH_FX_LOCK_CONFLICT" + case StatusV6DirNotEmpty: + return "SSH_FX_DIR_NOT_EMPTY" + case StatusV6NotADirectory: + return "SSH_FX_NOT_A_DIRECTORY" + case StatusV6InvalidFilename: + return "SSH_FX_INVALID_FILENAME" + case StatusV6LinkLoop: + return "SSH_FX_LINK_LOOP" + case StatusV6CannotDelete: + return "SSH_FX_CANNOT_DELETE" + case StatusV6InvalidParameter: + return "SSH_FX_INVALID_PARAMETER" + case StatusV6FileIsADirectory: + return "SSH_FX_FILE_IS_A_DIRECTORY" + case StatusV6ByteRangeLockConflict: + return "SSH_FX_BYTE_RANGE_LOCK_CONFLICT" + case StatusV6ByteRangeLockRefused: + return "SSH_FX_BYTE_RANGE_LOCK_REFUSED" + case StatusV6DeletePending: + return "SSH_FX_DELETE_PENDING" + case StatusV6FileCorrupt: + return "SSH_FX_FILE_CORRUPT" + case StatusV6OwnerInvalid: + return "SSH_FX_OWNER_INVALID" + case StatusV6GroupInvalid: + return "SSH_FX_GROUP_INVALID" + case StatusV6NoMatchingByteRangeLock: + return "SSH_FX_NO_MATCHING_BYTE_RANGE_LOCK" + default: + return fmt.Sprintf("SSH_FX_UNKNOWN(%d)", s) + } +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/fxp.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/fxp.go new file mode 100644 index 00000000000..780800215a5 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/fxp.go @@ -0,0 +1,169 @@ +package sshfx + +import ( + "fmt" +) + +// PacketType defines the various SFTP packet types. +type PacketType uint8 + +// Request packet types. +const ( + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3 + PacketTypeInit = PacketType(iota + 1) + PacketTypeVersion + PacketTypeOpen + PacketTypeClose + PacketTypeRead + PacketTypeWrite + PacketTypeLStat + PacketTypeFStat + PacketTypeSetstat + PacketTypeFSetstat + PacketTypeOpenDir + PacketTypeReadDir + PacketTypeRemove + PacketTypeMkdir + PacketTypeRmdir + PacketTypeRealPath + PacketTypeStat + PacketTypeRename + PacketTypeReadLink + PacketTypeSymlink + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-07.txt#section-3.3 + PacketTypeV6Link + + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-08.txt#section-3.3 + PacketTypeV6Block + PacketTypeV6Unblock +) + +// Response packet types. +const ( + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3 + PacketTypeStatus = PacketType(iota + 101) + PacketTypeHandle + PacketTypeData + PacketTypeName + PacketTypeAttrs +) + +// Extended packet types. +const ( + // https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3 + PacketTypeExtended = PacketType(iota + 200) + PacketTypeExtendedReply +) + +func (f PacketType) String() string { + switch f { + case PacketTypeInit: + return "SSH_FXP_INIT" + case PacketTypeVersion: + return "SSH_FXP_VERSION" + case PacketTypeOpen: + return "SSH_FXP_OPEN" + case PacketTypeClose: + return "SSH_FXP_CLOSE" + case PacketTypeRead: + return "SSH_FXP_READ" + case PacketTypeWrite: + return "SSH_FXP_WRITE" + case PacketTypeLStat: + return "SSH_FXP_LSTAT" + case PacketTypeFStat: + return "SSH_FXP_FSTAT" + case PacketTypeSetstat: + return "SSH_FXP_SETSTAT" + case PacketTypeFSetstat: + return "SSH_FXP_FSETSTAT" + case PacketTypeOpenDir: + return "SSH_FXP_OPENDIR" + case PacketTypeReadDir: + return "SSH_FXP_READDIR" + case PacketTypeRemove: + return "SSH_FXP_REMOVE" + case PacketTypeMkdir: + return "SSH_FXP_MKDIR" + case PacketTypeRmdir: + return "SSH_FXP_RMDIR" + case PacketTypeRealPath: + return "SSH_FXP_REALPATH" + case PacketTypeStat: + return "SSH_FXP_STAT" + case PacketTypeRename: + return "SSH_FXP_RENAME" + case PacketTypeReadLink: + return "SSH_FXP_READLINK" + case PacketTypeSymlink: + return "SSH_FXP_SYMLINK" + case PacketTypeV6Link: + return "SSH_FXP_LINK" + case PacketTypeV6Block: + return "SSH_FXP_BLOCK" + case PacketTypeV6Unblock: + return "SSH_FXP_UNBLOCK" + case PacketTypeStatus: + return "SSH_FXP_STATUS" + case PacketTypeHandle: + return "SSH_FXP_HANDLE" + case PacketTypeData: + return "SSH_FXP_DATA" + case PacketTypeName: + return "SSH_FXP_NAME" + case PacketTypeAttrs: + return "SSH_FXP_ATTRS" + case PacketTypeExtended: + return "SSH_FXP_EXTENDED" + case PacketTypeExtendedReply: + return "SSH_FXP_EXTENDED_REPLY" + default: + return fmt.Sprintf("SSH_FXP_UNKNOWN(%d)", f) + } +} + +func newPacketFromType(typ PacketType) (Packet, error) { + switch typ { + case PacketTypeOpen: + return new(OpenPacket), nil + case PacketTypeClose: + return new(ClosePacket), nil + case PacketTypeRead: + return new(ReadPacket), nil + case PacketTypeWrite: + return new(WritePacket), nil + case PacketTypeLStat: + return new(LStatPacket), nil + case PacketTypeFStat: + return new(FStatPacket), nil + case PacketTypeSetstat: + return new(SetstatPacket), nil + case PacketTypeFSetstat: + return new(FSetstatPacket), nil + case PacketTypeOpenDir: + return new(OpenDirPacket), nil + case PacketTypeReadDir: + return new(ReadDirPacket), nil + case PacketTypeRemove: + return new(RemovePacket), nil + case PacketTypeMkdir: + return new(MkdirPacket), nil + case PacketTypeRmdir: + return new(RmdirPacket), nil + case PacketTypeRealPath: + return new(RealPathPacket), nil + case PacketTypeStat: + return new(StatPacket), nil + case PacketTypeRename: + return new(RenamePacket), nil + case PacketTypeReadLink: + return new(ReadLinkPacket), nil + case PacketTypeSymlink: + return new(SymlinkPacket), nil + case PacketTypeExtended: + return new(ExtendedPacket), nil + default: + return nil, fmt.Errorf("unexpected request packet type: %v", typ) + } +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/handle_packets.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/handle_packets.go new file mode 100644 index 00000000000..44594acffdd --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/handle_packets.go @@ -0,0 +1,230 @@ +package sshfx + +// ClosePacket defines the SSH_FXP_CLOSE packet. +type ClosePacket struct { + Handle string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *ClosePacket) Type() PacketType { + return PacketTypeClose +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *ClosePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Handle) // string(handle) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeClose, reqid) + buf.AppendString(p.Handle) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *ClosePacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = ClosePacket{ + Handle: buf.ConsumeString(), + } + + return buf.Err +} + +// ReadPacket defines the SSH_FXP_READ packet. +type ReadPacket struct { + Handle string + Offset uint64 + Length uint32 +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *ReadPacket) Type() PacketType { + return PacketTypeRead +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *ReadPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + // string(handle) + uint64(offset) + uint32(len) + size := 4 + len(p.Handle) + 8 + 4 + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeRead, reqid) + buf.AppendString(p.Handle) + buf.AppendUint64(p.Offset) + buf.AppendUint32(p.Length) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *ReadPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = ReadPacket{ + Handle: buf.ConsumeString(), + Offset: buf.ConsumeUint64(), + Length: buf.ConsumeUint32(), + } + + return buf.Err +} + +// WritePacket defines the SSH_FXP_WRITE packet. +type WritePacket struct { + Handle string + Offset uint64 + Data []byte +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *WritePacket) Type() PacketType { + return PacketTypeWrite +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *WritePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + // string(handle) + uint64(offset) + uint32(len(data)); data content in payload + size := 4 + len(p.Handle) + 8 + 4 + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeWrite, reqid) + buf.AppendString(p.Handle) + buf.AppendUint64(p.Offset) + buf.AppendUint32(uint32(len(p.Data))) + + return buf.Packet(p.Data) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +// +// If p.Data is already populated, and of sufficient length to hold the data, +// then this will copy the data into that byte slice. +// +// If p.Data has a length insufficient to hold the data, +// then this will make a new slice of sufficient length, and copy the data into that. +// +// This means this _does not_ alias any of the data buffer that is passed in. +func (p *WritePacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = WritePacket{ + Handle: buf.ConsumeString(), + Offset: buf.ConsumeUint64(), + Data: buf.ConsumeByteSliceCopy(p.Data), + } + + return buf.Err +} + +// FStatPacket defines the SSH_FXP_FSTAT packet. +type FStatPacket struct { + Handle string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *FStatPacket) Type() PacketType { + return PacketTypeFStat +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *FStatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Handle) // string(handle) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeFStat, reqid) + buf.AppendString(p.Handle) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *FStatPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = FStatPacket{ + Handle: buf.ConsumeString(), + } + + return buf.Err +} + +// FSetstatPacket defines the SSH_FXP_FSETSTAT packet. +type FSetstatPacket struct { + Handle string + Attrs Attributes +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *FSetstatPacket) Type() PacketType { + return PacketTypeFSetstat +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *FSetstatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Handle) + p.Attrs.Len() // string(handle) + ATTRS(attrs) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeFSetstat, reqid) + buf.AppendString(p.Handle) + + p.Attrs.MarshalInto(buf) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *FSetstatPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = FSetstatPacket{ + Handle: buf.ConsumeString(), + } + + return p.Attrs.UnmarshalFrom(buf) +} + +// ReadDirPacket defines the SSH_FXP_READDIR packet. +type ReadDirPacket struct { + Handle string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *ReadDirPacket) Type() PacketType { + return PacketTypeReadDir +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *ReadDirPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Handle) // string(handle) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeReadDir, reqid) + buf.AppendString(p.Handle) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *ReadDirPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = ReadDirPacket{ + Handle: buf.ConsumeString(), + } + + return buf.Err +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/init_packets.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/init_packets.go new file mode 100644 index 00000000000..c553ee2e2c9 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/init_packets.go @@ -0,0 +1,99 @@ +package sshfx + +// InitPacket defines the SSH_FXP_INIT packet. +type InitPacket struct { + Version uint32 + Extensions []*ExtensionPair +} + +// MarshalBinary returns p as the binary encoding of p. +func (p *InitPacket) MarshalBinary() ([]byte, error) { + size := 1 + 4 // byte(type) + uint32(version) + + for _, ext := range p.Extensions { + size += ext.Len() + } + + b := NewBuffer(make([]byte, 4, 4+size)) + b.AppendUint8(uint8(PacketTypeInit)) + b.AppendUint32(p.Version) + + for _, ext := range p.Extensions { + ext.MarshalInto(b) + } + + b.PutLength(size) + + return b.Bytes(), nil +} + +// UnmarshalBinary unmarshals a full raw packet out of the given data. +// It is assumed that the uint32(length) has already been consumed to receive the data. +// It is also assumed that the uint8(type) has already been consumed to which packet to unmarshal into. +func (p *InitPacket) UnmarshalBinary(data []byte) (err error) { + buf := NewBuffer(data) + + *p = InitPacket{ + Version: buf.ConsumeUint32(), + } + + for buf.Len() > 0 { + var ext ExtensionPair + if err := ext.UnmarshalFrom(buf); err != nil { + return err + } + + p.Extensions = append(p.Extensions, &ext) + } + + return buf.Err +} + +// VersionPacket defines the SSH_FXP_VERSION packet. +type VersionPacket struct { + Version uint32 + Extensions []*ExtensionPair +} + +// MarshalBinary returns p as the binary encoding of p. +func (p *VersionPacket) MarshalBinary() ([]byte, error) { + size := 1 + 4 // byte(type) + uint32(version) + + for _, ext := range p.Extensions { + size += ext.Len() + } + + b := NewBuffer(make([]byte, 4, 4+size)) + b.AppendUint8(uint8(PacketTypeVersion)) + b.AppendUint32(p.Version) + + for _, ext := range p.Extensions { + ext.MarshalInto(b) + } + + b.PutLength(size) + + return b.Bytes(), nil +} + +// UnmarshalBinary unmarshals a full raw packet out of the given data. +// It is assumed that the uint32(length) has already been consumed to receive the data. +// It is also assumed that the uint8(type) has already been consumed to which packet to unmarshal into. +func (p *VersionPacket) UnmarshalBinary(data []byte) (err error) { + buf := NewBuffer(data) + + *p = VersionPacket{ + Version: buf.ConsumeUint32(), + } + + for buf.Len() > 0 { + var ext ExtensionPair + if err := ext.UnmarshalFrom(buf); err != nil { + return err + } + + p.Extensions = append(p.Extensions, &ext) + } + + return nil +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/open_packets.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/open_packets.go new file mode 100644 index 00000000000..896ba16e502 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/open_packets.go @@ -0,0 +1,86 @@ +package sshfx + +// SSH_FXF_* flags. +const ( + FlagRead = 1 << iota // SSH_FXF_READ + FlagWrite // SSH_FXF_WRITE + FlagAppend // SSH_FXF_APPEND + FlagCreate // SSH_FXF_CREAT + FlagTruncate // SSH_FXF_TRUNC + FlagExclusive // SSH_FXF_EXCL +) + +// OpenPacket defines the SSH_FXP_OPEN packet. +type OpenPacket struct { + Filename string + PFlags uint32 + Attrs Attributes +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *OpenPacket) Type() PacketType { + return PacketTypeOpen +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *OpenPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + // string(filename) + uint32(pflags) + ATTRS(attrs) + size := 4 + len(p.Filename) + 4 + p.Attrs.Len() + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeOpen, reqid) + buf.AppendString(p.Filename) + buf.AppendUint32(p.PFlags) + + p.Attrs.MarshalInto(buf) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *OpenPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = OpenPacket{ + Filename: buf.ConsumeString(), + PFlags: buf.ConsumeUint32(), + } + + return p.Attrs.UnmarshalFrom(buf) +} + +// OpenDirPacket defines the SSH_FXP_OPENDIR packet. +type OpenDirPacket struct { + Path string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *OpenDirPacket) Type() PacketType { + return PacketTypeOpenDir +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *OpenDirPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) // string(path) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeOpenDir, reqid) + buf.AppendString(p.Path) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *OpenDirPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = OpenDirPacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/fsync.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/fsync.go new file mode 100644 index 00000000000..708a4ba7ca4 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/fsync.go @@ -0,0 +1,73 @@ +package openssh + +import ( + sshfx "github.com/pkg/sftp/internal/encoding/ssh/filexfer" +) + +const extensionFSync = "fsync@openssh.com" + +// RegisterExtensionFSync registers the "fsync@openssh.com" extended packet with the encoding/ssh/filexfer package. +func RegisterExtensionFSync() { + sshfx.RegisterExtendedPacketType(extensionFSync, func() sshfx.ExtendedData { + return new(FSyncExtendedPacket) + }) +} + +// ExtensionFSync returns an ExtensionPair suitable to append into an sshfx.InitPacket or sshfx.VersionPacket. +func ExtensionFSync() *sshfx.ExtensionPair { + return &sshfx.ExtensionPair{ + Name: extensionFSync, + Data: "1", + } +} + +// FSyncExtendedPacket defines the fsync@openssh.com extend packet. +type FSyncExtendedPacket struct { + Handle string +} + +// Type returns the SSH_FXP_EXTENDED packet type. +func (ep *FSyncExtendedPacket) Type() sshfx.PacketType { + return sshfx.PacketTypeExtended +} + +// MarshalPacket returns ep as a two-part binary encoding of the full extended packet. +func (ep *FSyncExtendedPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + p := &sshfx.ExtendedPacket{ + ExtendedRequest: extensionFSync, + + Data: ep, + } + return p.MarshalPacket(reqid, b) +} + +// MarshalInto encodes ep into the binary encoding of the fsync@openssh.com extended packet-specific data. +func (ep *FSyncExtendedPacket) MarshalInto(buf *sshfx.Buffer) { + buf.AppendString(ep.Handle) +} + +// MarshalBinary encodes ep into the binary encoding of the fsync@openssh.com extended packet-specific data. +// +// NOTE: This _only_ encodes the packet-specific data, it does not encode the full extended packet. +func (ep *FSyncExtendedPacket) MarshalBinary() ([]byte, error) { + // string(handle) + size := 4 + len(ep.Handle) + + buf := sshfx.NewBuffer(make([]byte, 0, size)) + ep.MarshalInto(buf) + return buf.Bytes(), nil +} + +// UnmarshalFrom decodes the fsync@openssh.com extended packet-specific data from buf. +func (ep *FSyncExtendedPacket) UnmarshalFrom(buf *sshfx.Buffer) (err error) { + *ep = FSyncExtendedPacket{ + Handle: buf.ConsumeString(), + } + + return buf.Err +} + +// UnmarshalBinary decodes the fsync@openssh.com extended packet-specific data into ep. +func (ep *FSyncExtendedPacket) UnmarshalBinary(data []byte) (err error) { + return ep.UnmarshalFrom(sshfx.NewBuffer(data)) +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/hardlink.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/hardlink.go new file mode 100644 index 00000000000..f48d25a2be0 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/hardlink.go @@ -0,0 +1,76 @@ +package openssh + +import ( + sshfx "github.com/pkg/sftp/internal/encoding/ssh/filexfer" +) + +const extensionHardlink = "hardlink@openssh.com" + +// RegisterExtensionHardlink registers the "hardlink@openssh.com" extended packet with the encoding/ssh/filexfer package. +func RegisterExtensionHardlink() { + sshfx.RegisterExtendedPacketType(extensionHardlink, func() sshfx.ExtendedData { + return new(HardlinkExtendedPacket) + }) +} + +// ExtensionHardlink returns an ExtensionPair suitable to append into an sshfx.InitPacket or sshfx.VersionPacket. +func ExtensionHardlink() *sshfx.ExtensionPair { + return &sshfx.ExtensionPair{ + Name: extensionHardlink, + Data: "1", + } +} + +// HardlinkExtendedPacket defines the hardlink@openssh.com extend packet. +type HardlinkExtendedPacket struct { + OldPath string + NewPath string +} + +// Type returns the SSH_FXP_EXTENDED packet type. +func (ep *HardlinkExtendedPacket) Type() sshfx.PacketType { + return sshfx.PacketTypeExtended +} + +// MarshalPacket returns ep as a two-part binary encoding of the full extended packet. +func (ep *HardlinkExtendedPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + p := &sshfx.ExtendedPacket{ + ExtendedRequest: extensionHardlink, + + Data: ep, + } + return p.MarshalPacket(reqid, b) +} + +// MarshalInto encodes ep into the binary encoding of the hardlink@openssh.com extended packet-specific data. +func (ep *HardlinkExtendedPacket) MarshalInto(buf *sshfx.Buffer) { + buf.AppendString(ep.OldPath) + buf.AppendString(ep.NewPath) +} + +// MarshalBinary encodes ep into the binary encoding of the hardlink@openssh.com extended packet-specific data. +// +// NOTE: This _only_ encodes the packet-specific data, it does not encode the full extended packet. +func (ep *HardlinkExtendedPacket) MarshalBinary() ([]byte, error) { + // string(oldpath) + string(newpath) + size := 4 + len(ep.OldPath) + 4 + len(ep.NewPath) + + buf := sshfx.NewBuffer(make([]byte, 0, size)) + ep.MarshalInto(buf) + return buf.Bytes(), nil +} + +// UnmarshalFrom decodes the hardlink@openssh.com extended packet-specific data from buf. +func (ep *HardlinkExtendedPacket) UnmarshalFrom(buf *sshfx.Buffer) (err error) { + *ep = HardlinkExtendedPacket{ + OldPath: buf.ConsumeString(), + NewPath: buf.ConsumeString(), + } + + return buf.Err +} + +// UnmarshalBinary decodes the hardlink@openssh.com extended packet-specific data into ep. +func (ep *HardlinkExtendedPacket) UnmarshalBinary(data []byte) (err error) { + return ep.UnmarshalFrom(sshfx.NewBuffer(data)) +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/openssh.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/openssh.go new file mode 100644 index 00000000000..f93ff17727c --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/openssh.go @@ -0,0 +1,2 @@ +// Package openssh implements the openssh secsh-filexfer extensions as described in https://github.com/openssh/openssh-portable/blob/master/PROTOCOL +package openssh diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/posix-rename.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/posix-rename.go new file mode 100644 index 00000000000..5166489c247 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/posix-rename.go @@ -0,0 +1,76 @@ +package openssh + +import ( + sshfx "github.com/pkg/sftp/internal/encoding/ssh/filexfer" +) + +const extensionPOSIXRename = "posix-rename@openssh.com" + +// RegisterExtensionPOSIXRename registers the "posix-rename@openssh.com" extended packet with the encoding/ssh/filexfer package. +func RegisterExtensionPOSIXRename() { + sshfx.RegisterExtendedPacketType(extensionPOSIXRename, func() sshfx.ExtendedData { + return new(POSIXRenameExtendedPacket) + }) +} + +// ExtensionPOSIXRename returns an ExtensionPair suitable to append into an sshfx.InitPacket or sshfx.VersionPacket. +func ExtensionPOSIXRename() *sshfx.ExtensionPair { + return &sshfx.ExtensionPair{ + Name: extensionPOSIXRename, + Data: "1", + } +} + +// POSIXRenameExtendedPacket defines the posix-rename@openssh.com extend packet. +type POSIXRenameExtendedPacket struct { + OldPath string + NewPath string +} + +// Type returns the SSH_FXP_EXTENDED packet type. +func (ep *POSIXRenameExtendedPacket) Type() sshfx.PacketType { + return sshfx.PacketTypeExtended +} + +// MarshalPacket returns ep as a two-part binary encoding of the full extended packet. +func (ep *POSIXRenameExtendedPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + p := &sshfx.ExtendedPacket{ + ExtendedRequest: extensionPOSIXRename, + + Data: ep, + } + return p.MarshalPacket(reqid, b) +} + +// MarshalInto encodes ep into the binary encoding of the hardlink@openssh.com extended packet-specific data. +func (ep *POSIXRenameExtendedPacket) MarshalInto(buf *sshfx.Buffer) { + buf.AppendString(ep.OldPath) + buf.AppendString(ep.NewPath) +} + +// MarshalBinary encodes ep into the binary encoding of the hardlink@openssh.com extended packet-specific data. +// +// NOTE: This _only_ encodes the packet-specific data, it does not encode the full extended packet. +func (ep *POSIXRenameExtendedPacket) MarshalBinary() ([]byte, error) { + // string(oldpath) + string(newpath) + size := 4 + len(ep.OldPath) + 4 + len(ep.NewPath) + + buf := sshfx.NewBuffer(make([]byte, 0, size)) + ep.MarshalInto(buf) + return buf.Bytes(), nil +} + +// UnmarshalFrom decodes the hardlink@openssh.com extended packet-specific data from buf. +func (ep *POSIXRenameExtendedPacket) UnmarshalFrom(buf *sshfx.Buffer) (err error) { + *ep = POSIXRenameExtendedPacket{ + OldPath: buf.ConsumeString(), + NewPath: buf.ConsumeString(), + } + + return buf.Err +} + +// UnmarshalBinary decodes the hardlink@openssh.com extended packet-specific data into ep. +func (ep *POSIXRenameExtendedPacket) UnmarshalBinary(data []byte) (err error) { + return ep.UnmarshalFrom(sshfx.NewBuffer(data)) +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/statvfs.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/statvfs.go new file mode 100644 index 00000000000..51029ca0fa4 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh/statvfs.go @@ -0,0 +1,236 @@ +package openssh + +import ( + sshfx "github.com/pkg/sftp/internal/encoding/ssh/filexfer" +) + +const extensionStatVFS = "statvfs@openssh.com" + +// RegisterExtensionStatVFS registers the "statvfs@openssh.com" extended packet with the encoding/ssh/filexfer package. +func RegisterExtensionStatVFS() { + sshfx.RegisterExtendedPacketType(extensionStatVFS, func() sshfx.ExtendedData { + return new(StatVFSExtendedPacket) + }) +} + +// ExtensionStatVFS returns an ExtensionPair suitable to append into an sshfx.InitPacket or sshfx.VersionPacket. +func ExtensionStatVFS() *sshfx.ExtensionPair { + return &sshfx.ExtensionPair{ + Name: extensionStatVFS, + Data: "2", + } +} + +// StatVFSExtendedPacket defines the statvfs@openssh.com extend packet. +type StatVFSExtendedPacket struct { + Path string +} + +// Type returns the SSH_FXP_EXTENDED packet type. +func (ep *StatVFSExtendedPacket) Type() sshfx.PacketType { + return sshfx.PacketTypeExtended +} + +// MarshalPacket returns ep as a two-part binary encoding of the full extended packet. +func (ep *StatVFSExtendedPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + p := &sshfx.ExtendedPacket{ + ExtendedRequest: extensionStatVFS, + + Data: ep, + } + return p.MarshalPacket(reqid, b) +} + +// MarshalInto encodes ep into the binary encoding of the statvfs@openssh.com extended packet-specific data. +func (ep *StatVFSExtendedPacket) MarshalInto(buf *sshfx.Buffer) { + buf.AppendString(ep.Path) +} + +// MarshalBinary encodes ep into the binary encoding of the statvfs@openssh.com extended packet-specific data. +// +// NOTE: This _only_ encodes the packet-specific data, it does not encode the full extended packet. +func (ep *StatVFSExtendedPacket) MarshalBinary() ([]byte, error) { + size := 4 + len(ep.Path) // string(path) + + buf := sshfx.NewBuffer(make([]byte, 0, size)) + + ep.MarshalInto(buf) + + return buf.Bytes(), nil +} + +// UnmarshalFrom decodes the statvfs@openssh.com extended packet-specific data into ep. +func (ep *StatVFSExtendedPacket) UnmarshalFrom(buf *sshfx.Buffer) (err error) { + *ep = StatVFSExtendedPacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} + +// UnmarshalBinary decodes the statvfs@openssh.com extended packet-specific data into ep. +func (ep *StatVFSExtendedPacket) UnmarshalBinary(data []byte) (err error) { + return ep.UnmarshalFrom(sshfx.NewBuffer(data)) +} + +const extensionFStatVFS = "fstatvfs@openssh.com" + +// RegisterExtensionFStatVFS registers the "fstatvfs@openssh.com" extended packet with the encoding/ssh/filexfer package. +func RegisterExtensionFStatVFS() { + sshfx.RegisterExtendedPacketType(extensionFStatVFS, func() sshfx.ExtendedData { + return new(FStatVFSExtendedPacket) + }) +} + +// ExtensionFStatVFS returns an ExtensionPair suitable to append into an sshfx.InitPacket or sshfx.VersionPacket. +func ExtensionFStatVFS() *sshfx.ExtensionPair { + return &sshfx.ExtensionPair{ + Name: extensionFStatVFS, + Data: "2", + } +} + +// FStatVFSExtendedPacket defines the fstatvfs@openssh.com extend packet. +type FStatVFSExtendedPacket struct { + Path string +} + +// Type returns the SSH_FXP_EXTENDED packet type. +func (ep *FStatVFSExtendedPacket) Type() sshfx.PacketType { + return sshfx.PacketTypeExtended +} + +// MarshalPacket returns ep as a two-part binary encoding of the full extended packet. +func (ep *FStatVFSExtendedPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + p := &sshfx.ExtendedPacket{ + ExtendedRequest: extensionFStatVFS, + + Data: ep, + } + return p.MarshalPacket(reqid, b) +} + +// MarshalInto encodes ep into the binary encoding of the statvfs@openssh.com extended packet-specific data. +func (ep *FStatVFSExtendedPacket) MarshalInto(buf *sshfx.Buffer) { + buf.AppendString(ep.Path) +} + +// MarshalBinary encodes ep into the binary encoding of the statvfs@openssh.com extended packet-specific data. +// +// NOTE: This _only_ encodes the packet-specific data, it does not encode the full extended packet. +func (ep *FStatVFSExtendedPacket) MarshalBinary() ([]byte, error) { + size := 4 + len(ep.Path) // string(path) + + buf := sshfx.NewBuffer(make([]byte, 0, size)) + + ep.MarshalInto(buf) + + return buf.Bytes(), nil +} + +// UnmarshalFrom decodes the statvfs@openssh.com extended packet-specific data into ep. +func (ep *FStatVFSExtendedPacket) UnmarshalFrom(buf *sshfx.Buffer) (err error) { + *ep = FStatVFSExtendedPacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} + +// UnmarshalBinary decodes the statvfs@openssh.com extended packet-specific data into ep. +func (ep *FStatVFSExtendedPacket) UnmarshalBinary(data []byte) (err error) { + return ep.UnmarshalFrom(sshfx.NewBuffer(data)) +} + +// The values for the MountFlags field. +// https://github.com/openssh/openssh-portable/blob/master/PROTOCOL +const ( + MountFlagsReadOnly = 0x1 // SSH_FXE_STATVFS_ST_RDONLY + MountFlagsNoSUID = 0x2 // SSH_FXE_STATVFS_ST_NOSUID +) + +// StatVFSExtendedReplyPacket defines the extended reply packet for statvfs@openssh.com and fstatvfs@openssh.com requests. +type StatVFSExtendedReplyPacket struct { + BlockSize uint64 /* f_bsize: file system block size */ + FragmentSize uint64 /* f_frsize: fundamental fs block size / fagment size */ + Blocks uint64 /* f_blocks: number of blocks (unit f_frsize) */ + BlocksFree uint64 /* f_bfree: free blocks in filesystem */ + BlocksAvail uint64 /* f_bavail: free blocks for non-root */ + Files uint64 /* f_files: total file inodes */ + FilesFree uint64 /* f_ffree: free file inodes */ + FilesAvail uint64 /* f_favail: free file inodes for to non-root */ + FilesystemID uint64 /* f_fsid: file system id */ + MountFlags uint64 /* f_flag: bit mask of mount flag values */ + MaxNameLength uint64 /* f_namemax: maximum filename length */ +} + +// Type returns the SSH_FXP_EXTENDED_REPLY packet type. +func (ep *StatVFSExtendedReplyPacket) Type() sshfx.PacketType { + return sshfx.PacketTypeExtendedReply +} + +// MarshalPacket returns ep as a two-part binary encoding of the full extended reply packet. +func (ep *StatVFSExtendedReplyPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + p := &sshfx.ExtendedReplyPacket{ + Data: ep, + } + return p.MarshalPacket(reqid, b) +} + +// UnmarshalPacketBody returns ep as a two-part binary encoding of the full extended reply packet. +func (ep *StatVFSExtendedReplyPacket) UnmarshalPacketBody(buf *sshfx.Buffer) (err error) { + p := &sshfx.ExtendedReplyPacket{ + Data: ep, + } + return p.UnmarshalPacketBody(buf) +} + +// MarshalInto encodes ep into the binary encoding of the (f)statvfs@openssh.com extended reply packet-specific data. +func (ep *StatVFSExtendedReplyPacket) MarshalInto(buf *sshfx.Buffer) { + buf.AppendUint64(ep.BlockSize) + buf.AppendUint64(ep.FragmentSize) + buf.AppendUint64(ep.Blocks) + buf.AppendUint64(ep.BlocksFree) + buf.AppendUint64(ep.BlocksAvail) + buf.AppendUint64(ep.Files) + buf.AppendUint64(ep.FilesFree) + buf.AppendUint64(ep.FilesAvail) + buf.AppendUint64(ep.FilesystemID) + buf.AppendUint64(ep.MountFlags) + buf.AppendUint64(ep.MaxNameLength) +} + +// MarshalBinary encodes ep into the binary encoding of the (f)statvfs@openssh.com extended reply packet-specific data. +// +// NOTE: This _only_ encodes the packet-specific data, it does not encode the full extended reply packet. +func (ep *StatVFSExtendedReplyPacket) MarshalBinary() ([]byte, error) { + size := 11 * 8 // 11 × uint64(various) + + b := sshfx.NewBuffer(make([]byte, 0, size)) + ep.MarshalInto(b) + return b.Bytes(), nil +} + +// UnmarshalFrom decodes the fstatvfs@openssh.com extended reply packet-specific data into ep. +func (ep *StatVFSExtendedReplyPacket) UnmarshalFrom(buf *sshfx.Buffer) (err error) { + *ep = StatVFSExtendedReplyPacket{ + BlockSize: buf.ConsumeUint64(), + FragmentSize: buf.ConsumeUint64(), + Blocks: buf.ConsumeUint64(), + BlocksFree: buf.ConsumeUint64(), + BlocksAvail: buf.ConsumeUint64(), + Files: buf.ConsumeUint64(), + FilesFree: buf.ConsumeUint64(), + FilesAvail: buf.ConsumeUint64(), + FilesystemID: buf.ConsumeUint64(), + MountFlags: buf.ConsumeUint64(), + MaxNameLength: buf.ConsumeUint64(), + } + + return buf.Err +} + +// UnmarshalBinary decodes the fstatvfs@openssh.com extended reply packet-specific data into ep. +func (ep *StatVFSExtendedReplyPacket) UnmarshalBinary(data []byte) (err error) { + return ep.UnmarshalFrom(sshfx.NewBuffer(data)) +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/packets.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/packets.go new file mode 100644 index 00000000000..fdf65d05770 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/packets.go @@ -0,0 +1,273 @@ +package sshfx + +import ( + "errors" + "io" +) + +// smallBufferSize is an initial allocation minimal capacity. +const smallBufferSize = 64 + +// RawPacket implements the general packet format from draft-ietf-secsh-filexfer-02 +// +// RawPacket is intended for use in clients receiving responses, +// where a response will be expected to be of a limited number of types, +// and unmarshaling unknown/unexpected response packets is unnecessary. +// +// For servers expecting to receive arbitrary request packet types, +// use RequestPacket. +// +// Defined in https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3 +type RawPacket struct { + PacketType PacketType + RequestID uint32 + + Data Buffer +} + +// Type returns the Type field defining the SSH_FXP_xy type for this packet. +func (p *RawPacket) Type() PacketType { + return p.PacketType +} + +// Reset clears the pointers and reference-semantic variables of RawPacket, +// releasing underlying resources, and making them and the RawPacket suitable to be reused, +// so long as no other references have been kept. +func (p *RawPacket) Reset() { + p.Data = Buffer{} +} + +// MarshalPacket returns p as a two-part binary encoding of p. +// +// The internal p.RequestID is overridden by the reqid argument. +func (p *RawPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + buf = NewMarshalBuffer(0) + } + + buf.StartPacket(p.PacketType, reqid) + + return buf.Packet(p.Data.Bytes()) +} + +// MarshalBinary returns p as the binary encoding of p. +// +// This is a convenience implementation primarily intended for tests, +// because it is inefficient with allocations. +func (p *RawPacket) MarshalBinary() ([]byte, error) { + return ComposePacket(p.MarshalPacket(p.RequestID, nil)) +} + +// UnmarshalFrom decodes a RawPacket from the given Buffer into p. +// +// The Data field will alias the passed in Buffer, +// so the buffer passed in should not be reused before RawPacket.Reset(). +func (p *RawPacket) UnmarshalFrom(buf *Buffer) error { + *p = RawPacket{ + PacketType: PacketType(buf.ConsumeUint8()), + RequestID: buf.ConsumeUint32(), + } + + p.Data = *buf + + return buf.Err +} + +// UnmarshalBinary decodes a full raw packet out of the given data. +// It is assumed that the uint32(length) has already been consumed to receive the data. +// +// This is a convenience implementation primarily intended for tests, +// because this must clone the given data byte slice, +// as Data is not allowed to alias any part of the data byte slice. +func (p *RawPacket) UnmarshalBinary(data []byte) error { + clone := make([]byte, len(data)) + n := copy(clone, data) + return p.UnmarshalFrom(NewBuffer(clone[:n])) +} + +// readPacket reads a uint32 length-prefixed binary data packet from r. +// using the given byte slice as a backing array. +// +// If the packet length read from r is bigger than maxPacketLength, +// or greater than math.MaxInt32 on a 32-bit implementation, +// then a `ErrLongPacket` error will be returned. +// +// If the given byte slice is insufficient to hold the packet, +// then it will be extended to fill the packet size. +func readPacket(r io.Reader, b []byte, maxPacketLength uint32) ([]byte, error) { + if cap(b) < 4 { + // We will need allocate our own buffer just for reading the packet length. + + // However, we don’t really want to allocate an extremely narrow buffer (4-bytes), + // and cause unnecessary allocation churn from both length reads and small packet reads, + // so we use smallBufferSize from the bytes package as a reasonable guess. + + // But if callers really do want to force narrow throw-away allocation of every packet body, + // they can do so with a buffer of capacity 4. + b = make([]byte, smallBufferSize) + } + + if _, err := io.ReadFull(r, b[:4]); err != nil { + return nil, err + } + + length := unmarshalUint32(b) + if int(length) < 5 { + // Must have at least uint8(type) and uint32(request-id) + + if int(length) < 0 { + // Only possible when strconv.IntSize == 32, + // the packet length is longer than math.MaxInt32, + // and thus longer than any possible slice. + return nil, ErrLongPacket + } + + return nil, ErrShortPacket + } + if length > maxPacketLength { + return nil, ErrLongPacket + } + + if int(length) > cap(b) { + // We know int(length) must be positive, because of tests above. + b = make([]byte, length) + } + + n, err := io.ReadFull(r, b[:length]) + return b[:n], err +} + +// ReadFrom provides a simple functional packet reader, +// using the given byte slice as a backing array. +// +// To protect against potential denial of service attacks, +// if the read packet length is longer than maxPacketLength, +// then no packet data will be read, and ErrLongPacket will be returned. +// (On 32-bit int architectures, all packets >= 2^31 in length +// will return ErrLongPacket regardless of maxPacketLength.) +// +// If the read packet length is longer than cap(b), +// then a throw-away slice will allocated to meet the exact packet length. +// This can be used to limit the length of reused buffers, +// while still allowing reception of occasional large packets. +// +// The Data field may alias the passed in byte slice, +// so the byte slice passed in should not be reused before RawPacket.Reset(). +func (p *RawPacket) ReadFrom(r io.Reader, b []byte, maxPacketLength uint32) error { + b, err := readPacket(r, b, maxPacketLength) + if err != nil { + return err + } + + return p.UnmarshalFrom(NewBuffer(b)) +} + +// RequestPacket implements the general packet format from draft-ietf-secsh-filexfer-02 +// but also automatically decode/encodes valid request packets (2 < type < 100 || type == 200). +// +// RequestPacket is intended for use in servers receiving requests, +// where any arbitrary request may be received, and so decoding them automatically +// is useful. +// +// For clients expecting to receive specific response packet types, +// where automatic unmarshaling of the packet body does not make sense, +// use RawPacket. +// +// Defined in https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-3 +type RequestPacket struct { + RequestID uint32 + + Request Packet +} + +// Type returns the SSH_FXP_xy value associated with the underlying packet. +func (p *RequestPacket) Type() PacketType { + return p.Request.Type() +} + +// Reset clears the pointers and reference-semantic variables in RequestPacket, +// releasing underlying resources, and making them and the RequestPacket suitable to be reused, +// so long as no other references have been kept. +func (p *RequestPacket) Reset() { + p.Request = nil +} + +// MarshalPacket returns p as a two-part binary encoding of p. +// +// The internal p.RequestID is overridden by the reqid argument. +func (p *RequestPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + if p.Request == nil { + return nil, nil, errors.New("empty request packet") + } + + return p.Request.MarshalPacket(reqid, b) +} + +// MarshalBinary returns p as the binary encoding of p. +// +// This is a convenience implementation primarily intended for tests, +// because it is inefficient with allocations. +func (p *RequestPacket) MarshalBinary() ([]byte, error) { + return ComposePacket(p.MarshalPacket(p.RequestID, nil)) +} + +// UnmarshalFrom decodes a RequestPacket from the given Buffer into p. +// +// The Request field may alias the passed in Buffer, (e.g. SSH_FXP_WRITE), +// so the buffer passed in should not be reused before RequestPacket.Reset(). +func (p *RequestPacket) UnmarshalFrom(buf *Buffer) error { + typ := PacketType(buf.ConsumeUint8()) + if buf.Err != nil { + return buf.Err + } + + req, err := newPacketFromType(typ) + if err != nil { + return err + } + + *p = RequestPacket{ + RequestID: buf.ConsumeUint32(), + Request: req, + } + + return p.Request.UnmarshalPacketBody(buf) +} + +// UnmarshalBinary decodes a full request packet out of the given data. +// It is assumed that the uint32(length) has already been consumed to receive the data. +// +// This is a convenience implementation primarily intended for tests, +// because this must clone the given data byte slice, +// as Request is not allowed to alias any part of the data byte slice. +func (p *RequestPacket) UnmarshalBinary(data []byte) error { + clone := make([]byte, len(data)) + n := copy(clone, data) + return p.UnmarshalFrom(NewBuffer(clone[:n])) +} + +// ReadFrom provides a simple functional packet reader, +// using the given byte slice as a backing array. +// +// To protect against potential denial of service attacks, +// if the read packet length is longer than maxPacketLength, +// then no packet data will be read, and ErrLongPacket will be returned. +// (On 32-bit int architectures, all packets >= 2^31 in length +// will return ErrLongPacket regardless of maxPacketLength.) +// +// If the read packet length is longer than cap(b), +// then a throw-away slice will allocated to meet the exact packet length. +// This can be used to limit the length of reused buffers, +// while still allowing reception of occasional large packets. +// +// The Request field may alias the passed in byte slice, +// so the byte slice passed in should not be reused before RawPacket.Reset(). +func (p *RequestPacket) ReadFrom(r io.Reader, b []byte, maxPacketLength uint32) error { + b, err := readPacket(r, b, maxPacketLength) + if err != nil { + return err + } + + return p.UnmarshalFrom(NewBuffer(b)) +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/path_packets.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/path_packets.go new file mode 100644 index 00000000000..0180326f630 --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/path_packets.go @@ -0,0 +1,362 @@ +package sshfx + +// LStatPacket defines the SSH_FXP_LSTAT packet. +type LStatPacket struct { + Path string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *LStatPacket) Type() PacketType { + return PacketTypeLStat +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *LStatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) // string(path) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeLStat, reqid) + buf.AppendString(p.Path) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *LStatPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = LStatPacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} + +// SetstatPacket defines the SSH_FXP_SETSTAT packet. +type SetstatPacket struct { + Path string + Attrs Attributes +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *SetstatPacket) Type() PacketType { + return PacketTypeSetstat +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *SetstatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) + p.Attrs.Len() // string(path) + ATTRS(attrs) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeSetstat, reqid) + buf.AppendString(p.Path) + + p.Attrs.MarshalInto(buf) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *SetstatPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = SetstatPacket{ + Path: buf.ConsumeString(), + } + + return p.Attrs.UnmarshalFrom(buf) +} + +// RemovePacket defines the SSH_FXP_REMOVE packet. +type RemovePacket struct { + Path string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *RemovePacket) Type() PacketType { + return PacketTypeRemove +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *RemovePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) // string(path) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeRemove, reqid) + buf.AppendString(p.Path) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *RemovePacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = RemovePacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} + +// MkdirPacket defines the SSH_FXP_MKDIR packet. +type MkdirPacket struct { + Path string + Attrs Attributes +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *MkdirPacket) Type() PacketType { + return PacketTypeMkdir +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *MkdirPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) + p.Attrs.Len() // string(path) + ATTRS(attrs) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeMkdir, reqid) + buf.AppendString(p.Path) + + p.Attrs.MarshalInto(buf) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *MkdirPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = MkdirPacket{ + Path: buf.ConsumeString(), + } + + return p.Attrs.UnmarshalFrom(buf) +} + +// RmdirPacket defines the SSH_FXP_RMDIR packet. +type RmdirPacket struct { + Path string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *RmdirPacket) Type() PacketType { + return PacketTypeRmdir +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *RmdirPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) // string(path) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeRmdir, reqid) + buf.AppendString(p.Path) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *RmdirPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = RmdirPacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} + +// RealPathPacket defines the SSH_FXP_REALPATH packet. +type RealPathPacket struct { + Path string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *RealPathPacket) Type() PacketType { + return PacketTypeRealPath +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *RealPathPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) // string(path) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeRealPath, reqid) + buf.AppendString(p.Path) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *RealPathPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = RealPathPacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} + +// StatPacket defines the SSH_FXP_STAT packet. +type StatPacket struct { + Path string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *StatPacket) Type() PacketType { + return PacketTypeStat +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *StatPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) // string(path) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeStat, reqid) + buf.AppendString(p.Path) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *StatPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = StatPacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} + +// RenamePacket defines the SSH_FXP_RENAME packet. +type RenamePacket struct { + OldPath string + NewPath string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *RenamePacket) Type() PacketType { + return PacketTypeRename +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *RenamePacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + // string(oldpath) + string(newpath) + size := 4 + len(p.OldPath) + 4 + len(p.NewPath) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeRename, reqid) + buf.AppendString(p.OldPath) + buf.AppendString(p.NewPath) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *RenamePacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = RenamePacket{ + OldPath: buf.ConsumeString(), + NewPath: buf.ConsumeString(), + } + + return buf.Err +} + +// ReadLinkPacket defines the SSH_FXP_READLINK packet. +type ReadLinkPacket struct { + Path string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *ReadLinkPacket) Type() PacketType { + return PacketTypeReadLink +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *ReadLinkPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + size := 4 + len(p.Path) // string(path) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeReadLink, reqid) + buf.AppendString(p.Path) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *ReadLinkPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = ReadLinkPacket{ + Path: buf.ConsumeString(), + } + + return buf.Err +} + +// SymlinkPacket defines the SSH_FXP_SYMLINK packet. +// +// The order of the arguments to the SSH_FXP_SYMLINK method was inadvertently reversed. +// Unfortunately, the reversal was not noticed until the server was widely deployed. +// Covered in Section 4.1 of https://github.com/openssh/openssh-portable/blob/master/PROTOCOL +type SymlinkPacket struct { + LinkPath string + TargetPath string +} + +// Type returns the SSH_FXP_xy value associated with this packet type. +func (p *SymlinkPacket) Type() PacketType { + return PacketTypeSymlink +} + +// MarshalPacket returns p as a two-part binary encoding of p. +func (p *SymlinkPacket) MarshalPacket(reqid uint32, b []byte) (header, payload []byte, err error) { + buf := NewBuffer(b) + if buf.Cap() < 9 { + // string(targetpath) + string(linkpath) + size := 4 + len(p.TargetPath) + 4 + len(p.LinkPath) + buf = NewMarshalBuffer(size) + } + + buf.StartPacket(PacketTypeSymlink, reqid) + + // Arguments were inadvertently reversed. + buf.AppendString(p.TargetPath) + buf.AppendString(p.LinkPath) + + return buf.Packet(payload) +} + +// UnmarshalPacketBody unmarshals the packet body from the given Buffer. +// It is assumed that the uint32(request-id) has already been consumed. +func (p *SymlinkPacket) UnmarshalPacketBody(buf *Buffer) (err error) { + *p = SymlinkPacket{ + // Arguments were inadvertently reversed. + TargetPath: buf.ConsumeString(), + LinkPath: buf.ConsumeString(), + } + + return buf.Err +} diff --git a/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/permissions.go b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/permissions.go new file mode 100644 index 00000000000..0143ec0c99a --- /dev/null +++ b/vendor/github.com/pkg/sftp/internal/encoding/ssh/filexfer/permissions.go @@ -0,0 +1,114 @@ +package sshfx + +// FileMode represents a file’s mode and permission bits. +// The bits are defined according to POSIX standards, +// and may not apply to the OS being built for. +type FileMode uint32 + +// Permission flags, defined here to avoid potential inconsistencies in individual OS implementations. +const ( + ModePerm FileMode = 0o0777 // S_IRWXU | S_IRWXG | S_IRWXO + ModeUserRead FileMode = 0o0400 // S_IRUSR + ModeUserWrite FileMode = 0o0200 // S_IWUSR + ModeUserExec FileMode = 0o0100 // S_IXUSR + ModeGroupRead FileMode = 0o0040 // S_IRGRP + ModeGroupWrite FileMode = 0o0020 // S_IWGRP + ModeGroupExec FileMode = 0o0010 // S_IXGRP + ModeOtherRead FileMode = 0o0004 // S_IROTH + ModeOtherWrite FileMode = 0o0002 // S_IWOTH + ModeOtherExec FileMode = 0o0001 // S_IXOTH + + ModeSetUID FileMode = 0o4000 // S_ISUID + ModeSetGID FileMode = 0o2000 // S_ISGID + ModeSticky FileMode = 0o1000 // S_ISVTX + + ModeType FileMode = 0xF000 // S_IFMT + ModeNamedPipe FileMode = 0x1000 // S_IFIFO + ModeCharDevice FileMode = 0x2000 // S_IFCHR + ModeDir FileMode = 0x4000 // S_IFDIR + ModeDevice FileMode = 0x6000 // S_IFBLK + ModeRegular FileMode = 0x8000 // S_IFREG + ModeSymlink FileMode = 0xA000 // S_IFLNK + ModeSocket FileMode = 0xC000 // S_IFSOCK +) + +// IsDir reports whether m describes a directory. +// That is, it tests for m.Type() == ModeDir. +func (m FileMode) IsDir() bool { + return (m & ModeType) == ModeDir +} + +// IsRegular reports whether m describes a regular file. +// That is, it tests for m.Type() == ModeRegular +func (m FileMode) IsRegular() bool { + return (m & ModeType) == ModeRegular +} + +// Perm returns the POSIX permission bits in m (m & ModePerm). +func (m FileMode) Perm() FileMode { + return (m & ModePerm) +} + +// Type returns the type bits in m (m & ModeType). +func (m FileMode) Type() FileMode { + return (m & ModeType) +} + +// String returns a `-rwxrwxrwx` style string representing the `ls -l` POSIX permissions string. +func (m FileMode) String() string { + var buf [10]byte + + switch m.Type() { + case ModeRegular: + buf[0] = '-' + case ModeDir: + buf[0] = 'd' + case ModeSymlink: + buf[0] = 'l' + case ModeDevice: + buf[0] = 'b' + case ModeCharDevice: + buf[0] = 'c' + case ModeNamedPipe: + buf[0] = 'p' + case ModeSocket: + buf[0] = 's' + default: + buf[0] = '?' + } + + const rwx = "rwxrwxrwx" + for i, c := range rwx { + if m&(1<>24), byte(v>>16), byte(v>>8), byte(v)) +} + +func marshalUint64(b []byte, v uint64) []byte { + return marshalUint32(marshalUint32(b, uint32(v>>32)), uint32(v)) +} + +func marshalString(b []byte, v string) []byte { + return append(marshalUint32(b, uint32(len(v))), v...) +} + +func marshalFileInfo(b []byte, fi os.FileInfo) []byte { + // attributes variable struct, and also variable per protocol version + // spec version 3 attributes: + // uint32 flags + // uint64 size present only if flag SSH_FILEXFER_ATTR_SIZE + // uint32 uid present only if flag SSH_FILEXFER_ATTR_UIDGID + // uint32 gid present only if flag SSH_FILEXFER_ATTR_UIDGID + // uint32 permissions present only if flag SSH_FILEXFER_ATTR_PERMISSIONS + // uint32 atime present only if flag SSH_FILEXFER_ACMODTIME + // uint32 mtime present only if flag SSH_FILEXFER_ACMODTIME + // uint32 extended_count present only if flag SSH_FILEXFER_ATTR_EXTENDED + // string extended_type + // string extended_data + // ... more extended data (extended_type - extended_data pairs), + // so that number of pairs equals extended_count + + flags, fileStat := fileStatFromInfo(fi) + + b = marshalUint32(b, flags) + + return marshalFileStat(b, flags, fileStat) +} + +func marshalFileStat(b []byte, flags uint32, fileStat *FileStat) []byte { + if flags&sshFileXferAttrSize != 0 { + b = marshalUint64(b, fileStat.Size) + } + if flags&sshFileXferAttrUIDGID != 0 { + b = marshalUint32(b, fileStat.UID) + b = marshalUint32(b, fileStat.GID) + } + if flags&sshFileXferAttrPermissions != 0 { + b = marshalUint32(b, fileStat.Mode) + } + if flags&sshFileXferAttrACmodTime != 0 { + b = marshalUint32(b, fileStat.Atime) + b = marshalUint32(b, fileStat.Mtime) + } + + if flags&sshFileXferAttrExtended != 0 { + b = marshalUint32(b, uint32(len(fileStat.Extended))) + + for _, attr := range fileStat.Extended { + b = marshalString(b, attr.ExtType) + b = marshalString(b, attr.ExtData) + } + } + + return b +} + +func marshalStatus(b []byte, err StatusError) []byte { + b = marshalUint32(b, err.Code) + b = marshalString(b, err.msg) + b = marshalString(b, err.lang) + return b +} + +func marshal(b []byte, v interface{}) []byte { + switch v := v.(type) { + case nil: + return b + case uint8: + return append(b, v) + case uint32: + return marshalUint32(b, v) + case uint64: + return marshalUint64(b, v) + case string: + return marshalString(b, v) + case []byte: + return append(b, v...) + case os.FileInfo: + return marshalFileInfo(b, v) + default: + switch d := reflect.ValueOf(v); d.Kind() { + case reflect.Struct: + for i, n := 0, d.NumField(); i < n; i++ { + b = marshal(b, d.Field(i).Interface()) + } + return b + case reflect.Slice: + for i, n := 0, d.Len(); i < n; i++ { + b = marshal(b, d.Index(i).Interface()) + } + return b + default: + panic(fmt.Sprintf("marshal(%#v): cannot handle type %T", v, v)) + } + } +} + +func unmarshalUint32(b []byte) (uint32, []byte) { + v := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 + return v, b[4:] +} + +func unmarshalUint32Safe(b []byte) (uint32, []byte, error) { + var v uint32 + if len(b) < 4 { + return 0, nil, errShortPacket + } + v, b = unmarshalUint32(b) + return v, b, nil +} + +func unmarshalUint64(b []byte) (uint64, []byte) { + h, b := unmarshalUint32(b) + l, b := unmarshalUint32(b) + return uint64(h)<<32 | uint64(l), b +} + +func unmarshalUint64Safe(b []byte) (uint64, []byte, error) { + var v uint64 + if len(b) < 8 { + return 0, nil, errShortPacket + } + v, b = unmarshalUint64(b) + return v, b, nil +} + +func unmarshalString(b []byte) (string, []byte) { + n, b := unmarshalUint32(b) + return string(b[:n]), b[n:] +} + +func unmarshalStringSafe(b []byte) (string, []byte, error) { + n, b, err := unmarshalUint32Safe(b) + if err != nil { + return "", nil, err + } + if int64(n) > int64(len(b)) { + return "", nil, errShortPacket + } + return string(b[:n]), b[n:], nil +} + +func unmarshalAttrs(b []byte) (*FileStat, []byte, error) { + flags, b, err := unmarshalUint32Safe(b) + if err != nil { + return nil, b, err + } + return unmarshalFileStat(flags, b) +} + +func unmarshalFileStat(flags uint32, b []byte) (*FileStat, []byte, error) { + var fs FileStat + var err error + + if flags&sshFileXferAttrSize == sshFileXferAttrSize { + fs.Size, b, err = unmarshalUint64Safe(b) + if err != nil { + return nil, b, err + } + } + if flags&sshFileXferAttrUIDGID == sshFileXferAttrUIDGID { + fs.UID, b, err = unmarshalUint32Safe(b) + if err != nil { + return nil, b, err + } + fs.GID, b, err = unmarshalUint32Safe(b) + if err != nil { + return nil, b, err + } + } + if flags&sshFileXferAttrPermissions == sshFileXferAttrPermissions { + fs.Mode, b, err = unmarshalUint32Safe(b) + if err != nil { + return nil, b, err + } + } + if flags&sshFileXferAttrACmodTime == sshFileXferAttrACmodTime { + fs.Atime, b, err = unmarshalUint32Safe(b) + if err != nil { + return nil, b, err + } + fs.Mtime, b, err = unmarshalUint32Safe(b) + if err != nil { + return nil, b, err + } + } + if flags&sshFileXferAttrExtended == sshFileXferAttrExtended { + var count uint32 + count, b, err = unmarshalUint32Safe(b) + if err != nil { + return nil, b, err + } + + ext := make([]StatExtended, count) + for i := uint32(0); i < count; i++ { + var typ string + var data string + typ, b, err = unmarshalStringSafe(b) + if err != nil { + return nil, b, err + } + data, b, err = unmarshalStringSafe(b) + if err != nil { + return nil, b, err + } + ext[i] = StatExtended{ + ExtType: typ, + ExtData: data, + } + } + fs.Extended = ext + } + return &fs, b, nil +} + +func unmarshalStatus(id uint32, data []byte) error { + sid, data := unmarshalUint32(data) + if sid != id { + return &unexpectedIDErr{id, sid} + } + code, data := unmarshalUint32(data) + msg, data, _ := unmarshalStringSafe(data) + lang, _, _ := unmarshalStringSafe(data) + return &StatusError{ + Code: code, + msg: msg, + lang: lang, + } +} + +type packetMarshaler interface { + marshalPacket() (header, payload []byte, err error) +} + +func marshalPacket(m encoding.BinaryMarshaler) (header, payload []byte, err error) { + if m, ok := m.(packetMarshaler); ok { + return m.marshalPacket() + } + + header, err = m.MarshalBinary() + return +} + +// sendPacket marshals p according to RFC 4234. +func sendPacket(w io.Writer, m encoding.BinaryMarshaler) error { + header, payload, err := marshalPacket(m) + if err != nil { + return fmt.Errorf("binary marshaller failed: %w", err) + } + + length := len(header) + len(payload) - 4 // subtract the uint32(length) from the start + if debugDumpTxPacketBytes { + debug("send packet: %s %d bytes %x%x", fxp(header[4]), length, header[5:], payload) + } else if debugDumpTxPacket { + debug("send packet: %s %d bytes", fxp(header[4]), length) + } + + binary.BigEndian.PutUint32(header[:4], uint32(length)) + + if _, err := w.Write(header); err != nil { + return fmt.Errorf("failed to send packet: %w", err) + } + + if len(payload) > 0 { + if _, err := w.Write(payload); err != nil { + return fmt.Errorf("failed to send packet payload: %w", err) + } + } + + return nil +} + +func recvPacket(r io.Reader, alloc *allocator, orderID uint32) (fxp, []byte, error) { + var b []byte + if alloc != nil { + b = alloc.GetPage(orderID) + } else { + b = make([]byte, 4) + } + + if n, err := io.ReadFull(r, b[:4]); err != nil { + if err == io.EOF { + return 0, nil, err + } + + return 0, nil, fmt.Errorf("error reading packet length: %d of 4: %w", n, err) + } + + length, _ := unmarshalUint32(b) + if length > maxMsgLength { + debug("recv packet %d bytes too long", length) + return 0, nil, errLongPacket + } + if length == 0 { + debug("recv packet of 0 bytes too short") + return 0, nil, errShortPacket + } + + if alloc == nil { + b = make([]byte, length) + } + + n, err := io.ReadFull(r, b[:length]) + b = b[:n] + + if err != nil { + debug("recv packet error: %d of %d bytes: %x", n, length, b) + + // ReadFull only returns EOF if it has read no bytes. + // In this case, that means a partial packet, and thus unexpected. + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + + if n == 0 { + return 0, nil, fmt.Errorf("error reading packet body: %d of %d: %w", n, length, err) + } + + return 0, nil, fmt.Errorf("error reading packet body: %d of %d: (%s) %w", n, length, fxp(b[0]), err) + } + + typ, payload := fxp(b[0]), b[1:n] + + if debugDumpRxPacketBytes { + debug("recv packet: %s %d bytes %x", typ, length, payload) + } else if debugDumpRxPacket { + debug("recv packet: %s %d bytes", typ, length) + } + + return typ, payload, nil +} + +type extensionPair struct { + Name string + Data string +} + +func unmarshalExtensionPair(b []byte) (extensionPair, []byte, error) { + var ep extensionPair + var err error + ep.Name, b, err = unmarshalStringSafe(b) + if err != nil { + return ep, b, err + } + ep.Data, b, err = unmarshalStringSafe(b) + return ep, b, err +} + +// Here starts the definition of packets along with their MarshalBinary +// implementations. +// Manually writing the marshalling logic wins us a lot of time and +// allocation. + +type sshFxInitPacket struct { + Version uint32 + Extensions []extensionPair +} + +func (p *sshFxInitPacket) MarshalBinary() ([]byte, error) { + l := 4 + 1 + 4 // uint32(length) + byte(type) + uint32(version) + for _, e := range p.Extensions { + l += 4 + len(e.Name) + 4 + len(e.Data) + } + + b := make([]byte, 4, l) + b = append(b, sshFxpInit) + b = marshalUint32(b, p.Version) + + for _, e := range p.Extensions { + b = marshalString(b, e.Name) + b = marshalString(b, e.Data) + } + + return b, nil +} + +func (p *sshFxInitPacket) UnmarshalBinary(b []byte) error { + var err error + if p.Version, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + for len(b) > 0 { + var ep extensionPair + ep, b, err = unmarshalExtensionPair(b) + if err != nil { + return err + } + p.Extensions = append(p.Extensions, ep) + } + return nil +} + +type sshFxVersionPacket struct { + Version uint32 + Extensions []sshExtensionPair +} + +type sshExtensionPair struct { + Name, Data string +} + +func (p *sshFxVersionPacket) MarshalBinary() ([]byte, error) { + l := 4 + 1 + 4 // uint32(length) + byte(type) + uint32(version) + for _, e := range p.Extensions { + l += 4 + len(e.Name) + 4 + len(e.Data) + } + + b := make([]byte, 4, l) + b = append(b, sshFxpVersion) + b = marshalUint32(b, p.Version) + + for _, e := range p.Extensions { + b = marshalString(b, e.Name) + b = marshalString(b, e.Data) + } + + return b, nil +} + +func marshalIDStringPacket(packetType byte, id uint32, str string) ([]byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(str) + + b := make([]byte, 4, l) + b = append(b, packetType) + b = marshalUint32(b, id) + b = marshalString(b, str) + + return b, nil +} + +func unmarshalIDString(b []byte, id *uint32, str *string) error { + var err error + *id, b, err = unmarshalUint32Safe(b) + if err != nil { + return err + } + *str, _, err = unmarshalStringSafe(b) + return err +} + +type sshFxpReaddirPacket struct { + ID uint32 + Handle string +} + +func (p *sshFxpReaddirPacket) id() uint32 { return p.ID } + +func (p *sshFxpReaddirPacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpReaddir, p.ID, p.Handle) +} + +func (p *sshFxpReaddirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Handle) +} + +type sshFxpOpendirPacket struct { + ID uint32 + Path string +} + +func (p *sshFxpOpendirPacket) id() uint32 { return p.ID } + +func (p *sshFxpOpendirPacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpOpendir, p.ID, p.Path) +} + +func (p *sshFxpOpendirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpLstatPacket struct { + ID uint32 + Path string +} + +func (p *sshFxpLstatPacket) id() uint32 { return p.ID } + +func (p *sshFxpLstatPacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpLstat, p.ID, p.Path) +} + +func (p *sshFxpLstatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpStatPacket struct { + ID uint32 + Path string +} + +func (p *sshFxpStatPacket) id() uint32 { return p.ID } + +func (p *sshFxpStatPacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpStat, p.ID, p.Path) +} + +func (p *sshFxpStatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpFstatPacket struct { + ID uint32 + Handle string +} + +func (p *sshFxpFstatPacket) id() uint32 { return p.ID } + +func (p *sshFxpFstatPacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpFstat, p.ID, p.Handle) +} + +func (p *sshFxpFstatPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Handle) +} + +type sshFxpClosePacket struct { + ID uint32 + Handle string +} + +func (p *sshFxpClosePacket) id() uint32 { return p.ID } + +func (p *sshFxpClosePacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpClose, p.ID, p.Handle) +} + +func (p *sshFxpClosePacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Handle) +} + +type sshFxpRemovePacket struct { + ID uint32 + Filename string +} + +func (p *sshFxpRemovePacket) id() uint32 { return p.ID } + +func (p *sshFxpRemovePacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpRemove, p.ID, p.Filename) +} + +func (p *sshFxpRemovePacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Filename) +} + +type sshFxpRmdirPacket struct { + ID uint32 + Path string +} + +func (p *sshFxpRmdirPacket) id() uint32 { return p.ID } + +func (p *sshFxpRmdirPacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpRmdir, p.ID, p.Path) +} + +func (p *sshFxpRmdirPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpSymlinkPacket struct { + ID uint32 + + // The order of the arguments to the SSH_FXP_SYMLINK method was inadvertently reversed. + // Unfortunately, the reversal was not noticed until the server was widely deployed. + // Covered in Section 4.1 of https://github.com/openssh/openssh-portable/blob/master/PROTOCOL + + Targetpath string + Linkpath string +} + +func (p *sshFxpSymlinkPacket) id() uint32 { return p.ID } + +func (p *sshFxpSymlinkPacket) MarshalBinary() ([]byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Targetpath) + + 4 + len(p.Linkpath) + + b := make([]byte, 4, l) + b = append(b, sshFxpSymlink) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Targetpath) + b = marshalString(b, p.Linkpath) + + return b, nil +} + +func (p *sshFxpSymlinkPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Targetpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Linkpath, _, err = unmarshalStringSafe(b); err != nil { + return err + } + return nil +} + +type sshFxpHardlinkPacket struct { + ID uint32 + Oldpath string + Newpath string +} + +func (p *sshFxpHardlinkPacket) id() uint32 { return p.ID } + +func (p *sshFxpHardlinkPacket) MarshalBinary() ([]byte, error) { + const ext = "hardlink@openssh.com" + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(ext) + + 4 + len(p.Oldpath) + + 4 + len(p.Newpath) + + b := make([]byte, 4, l) + b = append(b, sshFxpExtended) + b = marshalUint32(b, p.ID) + b = marshalString(b, ext) + b = marshalString(b, p.Oldpath) + b = marshalString(b, p.Newpath) + + return b, nil +} + +type sshFxpReadlinkPacket struct { + ID uint32 + Path string +} + +func (p *sshFxpReadlinkPacket) id() uint32 { return p.ID } + +func (p *sshFxpReadlinkPacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpReadlink, p.ID, p.Path) +} + +func (p *sshFxpReadlinkPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpRealpathPacket struct { + ID uint32 + Path string +} + +func (p *sshFxpRealpathPacket) id() uint32 { return p.ID } + +func (p *sshFxpRealpathPacket) MarshalBinary() ([]byte, error) { + return marshalIDStringPacket(sshFxpRealpath, p.ID, p.Path) +} + +func (p *sshFxpRealpathPacket) UnmarshalBinary(b []byte) error { + return unmarshalIDString(b, &p.ID, &p.Path) +} + +type sshFxpNameAttr struct { + Name string + LongName string + Attrs []interface{} +} + +func (p *sshFxpNameAttr) MarshalBinary() ([]byte, error) { + var b []byte + b = marshalString(b, p.Name) + b = marshalString(b, p.LongName) + for _, attr := range p.Attrs { + b = marshal(b, attr) + } + return b, nil +} + +type sshFxpNamePacket struct { + ID uint32 + NameAttrs []*sshFxpNameAttr +} + +func (p *sshFxpNamePacket) marshalPacket() ([]byte, []byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + + b := make([]byte, 4, l) + b = append(b, sshFxpName) + b = marshalUint32(b, p.ID) + b = marshalUint32(b, uint32(len(p.NameAttrs))) + + var payload []byte + for _, na := range p.NameAttrs { + ab, err := na.MarshalBinary() + if err != nil { + return nil, nil, err + } + + payload = append(payload, ab...) + } + + return b, payload, nil +} + +func (p *sshFxpNamePacket) MarshalBinary() ([]byte, error) { + header, payload, err := p.marshalPacket() + return append(header, payload...), err +} + +type sshFxpOpenPacket struct { + ID uint32 + Path string + Pflags uint32 + Flags uint32 + Attrs interface{} +} + +func (p *sshFxpOpenPacket) id() uint32 { return p.ID } + +func (p *sshFxpOpenPacket) marshalPacket() ([]byte, []byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Path) + + 4 + 4 + + b := make([]byte, 4, l) + b = append(b, sshFxpOpen) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Path) + b = marshalUint32(b, p.Pflags) + b = marshalUint32(b, p.Flags) + + switch attrs := p.Attrs.(type) { + case []byte: + return b, attrs, nil // may as well short-ciruit this case. + case os.FileInfo: + _, fs := fileStatFromInfo(attrs) // we throw away the flags, and override with those in packet. + return b, marshalFileStat(nil, p.Flags, fs), nil + case *FileStat: + return b, marshalFileStat(nil, p.Flags, attrs), nil + } + + return b, marshal(nil, p.Attrs), nil +} + +func (p *sshFxpOpenPacket) MarshalBinary() ([]byte, error) { + header, payload, err := p.marshalPacket() + return append(header, payload...), err +} + +func (p *sshFxpOpenPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Pflags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + p.Attrs = b + return nil +} + +func (p *sshFxpOpenPacket) unmarshalFileStat(flags uint32) (*FileStat, error) { + switch attrs := p.Attrs.(type) { + case *FileStat: + return attrs, nil + case []byte: + fs, _, err := unmarshalFileStat(flags, attrs) + return fs, err + default: + return nil, fmt.Errorf("invalid type in unmarshalFileStat: %T", attrs) + } +} + +type sshFxpReadPacket struct { + ID uint32 + Len uint32 + Offset uint64 + Handle string +} + +func (p *sshFxpReadPacket) id() uint32 { return p.ID } + +func (p *sshFxpReadPacket) MarshalBinary() ([]byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Handle) + + 8 + 4 // uint64 + uint32 + + b := make([]byte, 4, l) + b = append(b, sshFxpRead) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Handle) + b = marshalUint64(b, p.Offset) + b = marshalUint32(b, p.Len) + + return b, nil +} + +func (p *sshFxpReadPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Offset, b, err = unmarshalUint64Safe(b); err != nil { + return err + } else if p.Len, _, err = unmarshalUint32Safe(b); err != nil { + return err + } + return nil +} + +// We need allocate bigger slices with extra capacity to avoid a re-allocation in sshFxpDataPacket.MarshalBinary +// So, we need: uint32(length) + byte(type) + uint32(id) + uint32(data_length) +const dataHeaderLen = 4 + 1 + 4 + 4 + +func (p *sshFxpReadPacket) getDataSlice(alloc *allocator, orderID uint32, maxTxPacket uint32) []byte { + dataLen := p.Len + if dataLen > maxTxPacket { + dataLen = maxTxPacket + } + + if alloc != nil { + // GetPage returns a slice with capacity = maxMsgLength this is enough to avoid new allocations in + // sshFxpDataPacket.MarshalBinary + return alloc.GetPage(orderID)[:dataLen] + } + + // allocate with extra space for the header + return make([]byte, dataLen, dataLen+dataHeaderLen) +} + +type sshFxpRenamePacket struct { + ID uint32 + Oldpath string + Newpath string +} + +func (p *sshFxpRenamePacket) id() uint32 { return p.ID } + +func (p *sshFxpRenamePacket) MarshalBinary() ([]byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Oldpath) + + 4 + len(p.Newpath) + + b := make([]byte, 4, l) + b = append(b, sshFxpRename) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Oldpath) + b = marshalString(b, p.Newpath) + + return b, nil +} + +func (p *sshFxpRenamePacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Oldpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Newpath, _, err = unmarshalStringSafe(b); err != nil { + return err + } + return nil +} + +type sshFxpPosixRenamePacket struct { + ID uint32 + Oldpath string + Newpath string +} + +func (p *sshFxpPosixRenamePacket) id() uint32 { return p.ID } + +func (p *sshFxpPosixRenamePacket) MarshalBinary() ([]byte, error) { + const ext = "posix-rename@openssh.com" + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(ext) + + 4 + len(p.Oldpath) + + 4 + len(p.Newpath) + + b := make([]byte, 4, l) + b = append(b, sshFxpExtended) + b = marshalUint32(b, p.ID) + b = marshalString(b, ext) + b = marshalString(b, p.Oldpath) + b = marshalString(b, p.Newpath) + + return b, nil +} + +type sshFxpWritePacket struct { + ID uint32 + Length uint32 + Offset uint64 + Handle string + Data []byte +} + +func (p *sshFxpWritePacket) id() uint32 { return p.ID } + +func (p *sshFxpWritePacket) marshalPacket() ([]byte, []byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Handle) + + 8 + // uint64 + 4 + + b := make([]byte, 4, l) + b = append(b, sshFxpWrite) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Handle) + b = marshalUint64(b, p.Offset) + b = marshalUint32(b, p.Length) + + return b, p.Data, nil +} + +func (p *sshFxpWritePacket) MarshalBinary() ([]byte, error) { + header, payload, err := p.marshalPacket() + return append(header, payload...), err +} + +func (p *sshFxpWritePacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Offset, b, err = unmarshalUint64Safe(b); err != nil { + return err + } else if p.Length, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if uint32(len(b)) < p.Length { + return errShortPacket + } + + p.Data = b[:p.Length] + return nil +} + +type sshFxpMkdirPacket struct { + ID uint32 + Flags uint32 // ignored + Path string +} + +func (p *sshFxpMkdirPacket) id() uint32 { return p.ID } + +func (p *sshFxpMkdirPacket) MarshalBinary() ([]byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Path) + + 4 // uint32 + + b := make([]byte, 4, l) + b = append(b, sshFxpMkdir) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Path) + b = marshalUint32(b, p.Flags) + + return b, nil +} + +func (p *sshFxpMkdirPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, _, err = unmarshalUint32Safe(b); err != nil { + return err + } + return nil +} + +type sshFxpSetstatPacket struct { + ID uint32 + Flags uint32 + Path string + Attrs interface{} +} + +type sshFxpFsetstatPacket struct { + ID uint32 + Flags uint32 + Handle string + Attrs interface{} +} + +func (p *sshFxpSetstatPacket) id() uint32 { return p.ID } +func (p *sshFxpFsetstatPacket) id() uint32 { return p.ID } + +func (p *sshFxpSetstatPacket) marshalPacket() ([]byte, []byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Path) + + 4 // uint32 + + b := make([]byte, 4, l) + b = append(b, sshFxpSetstat) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Path) + b = marshalUint32(b, p.Flags) + + switch attrs := p.Attrs.(type) { + case []byte: + return b, attrs, nil // may as well short-ciruit this case. + case os.FileInfo: + _, fs := fileStatFromInfo(attrs) // we throw away the flags, and override with those in packet. + return b, marshalFileStat(nil, p.Flags, fs), nil + case *FileStat: + return b, marshalFileStat(nil, p.Flags, attrs), nil + } + + return b, marshal(nil, p.Attrs), nil +} + +func (p *sshFxpSetstatPacket) MarshalBinary() ([]byte, error) { + header, payload, err := p.marshalPacket() + return append(header, payload...), err +} + +func (p *sshFxpFsetstatPacket) marshalPacket() ([]byte, []byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Handle) + + 4 // uint32 + + b := make([]byte, 4, l) + b = append(b, sshFxpFsetstat) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Handle) + b = marshalUint32(b, p.Flags) + + switch attrs := p.Attrs.(type) { + case []byte: + return b, attrs, nil // may as well short-ciruit this case. + case os.FileInfo: + _, fs := fileStatFromInfo(attrs) // we throw away the flags, and override with those in packet. + return b, marshalFileStat(nil, p.Flags, fs), nil + case *FileStat: + return b, marshalFileStat(nil, p.Flags, attrs), nil + } + + return b, marshal(nil, p.Attrs), nil +} + +func (p *sshFxpFsetstatPacket) MarshalBinary() ([]byte, error) { + header, payload, err := p.marshalPacket() + return append(header, payload...), err +} + +func (p *sshFxpSetstatPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Path, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + p.Attrs = b + return nil +} + +func (p *sshFxpSetstatPacket) unmarshalFileStat(flags uint32) (*FileStat, error) { + switch attrs := p.Attrs.(type) { + case *FileStat: + return attrs, nil + case []byte: + fs, _, err := unmarshalFileStat(flags, attrs) + return fs, err + default: + return nil, fmt.Errorf("invalid type in unmarshalFileStat: %T", attrs) + } +} + +func (p *sshFxpFsetstatPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Handle, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Flags, b, err = unmarshalUint32Safe(b); err != nil { + return err + } + p.Attrs = b + return nil +} + +func (p *sshFxpFsetstatPacket) unmarshalFileStat(flags uint32) (*FileStat, error) { + switch attrs := p.Attrs.(type) { + case *FileStat: + return attrs, nil + case []byte: + fs, _, err := unmarshalFileStat(flags, attrs) + return fs, err + default: + return nil, fmt.Errorf("invalid type in unmarshalFileStat: %T", attrs) + } +} + +type sshFxpHandlePacket struct { + ID uint32 + Handle string +} + +func (p *sshFxpHandlePacket) MarshalBinary() ([]byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(p.Handle) + + b := make([]byte, 4, l) + b = append(b, sshFxpHandle) + b = marshalUint32(b, p.ID) + b = marshalString(b, p.Handle) + + return b, nil +} + +type sshFxpStatusPacket struct { + ID uint32 + StatusError +} + +func (p *sshFxpStatusPacket) MarshalBinary() ([]byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + + 4 + len(p.StatusError.msg) + + 4 + len(p.StatusError.lang) + + b := make([]byte, 4, l) + b = append(b, sshFxpStatus) + b = marshalUint32(b, p.ID) + b = marshalStatus(b, p.StatusError) + + return b, nil +} + +type sshFxpDataPacket struct { + ID uint32 + Length uint32 + Data []byte +} + +func (p *sshFxpDataPacket) marshalPacket() ([]byte, []byte, error) { + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + + b := make([]byte, 4, l) + b = append(b, sshFxpData) + b = marshalUint32(b, p.ID) + b = marshalUint32(b, p.Length) + + return b, p.Data, nil +} + +// MarshalBinary encodes the receiver into a binary form and returns the result. +// To avoid a new allocation the Data slice must have a capacity >= Length + 9 +// +// This is hand-coded rather than just append(header, payload...), +// in order to try and reuse the r.Data backing store in the packet. +func (p *sshFxpDataPacket) MarshalBinary() ([]byte, error) { + b := append(p.Data, make([]byte, dataHeaderLen)...) + copy(b[dataHeaderLen:], p.Data[:p.Length]) + // b[0:4] will be overwritten with the length in sendPacket + b[4] = sshFxpData + binary.BigEndian.PutUint32(b[5:9], p.ID) + binary.BigEndian.PutUint32(b[9:13], p.Length) + return b, nil +} + +func (p *sshFxpDataPacket) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.Length, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if uint32(len(b)) < p.Length { + return errShortPacket + } + + p.Data = b[:p.Length] + return nil +} + +type sshFxpStatvfsPacket struct { + ID uint32 + Path string +} + +func (p *sshFxpStatvfsPacket) id() uint32 { return p.ID } + +func (p *sshFxpStatvfsPacket) MarshalBinary() ([]byte, error) { + const ext = "statvfs@openssh.com" + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(ext) + + 4 + len(p.Path) + + b := make([]byte, 4, l) + b = append(b, sshFxpExtended) + b = marshalUint32(b, p.ID) + b = marshalString(b, ext) + b = marshalString(b, p.Path) + + return b, nil +} + +// A StatVFS contains statistics about a filesystem. +type StatVFS struct { + ID uint32 + Bsize uint64 /* file system block size */ + Frsize uint64 /* fundamental fs block size */ + Blocks uint64 /* number of blocks (unit f_frsize) */ + Bfree uint64 /* free blocks in file system */ + Bavail uint64 /* free blocks for non-root */ + Files uint64 /* total file inodes */ + Ffree uint64 /* free file inodes */ + Favail uint64 /* free file inodes for to non-root */ + Fsid uint64 /* file system id */ + Flag uint64 /* bit mask of f_flag values */ + Namemax uint64 /* maximum filename length */ +} + +// TotalSpace calculates the amount of total space in a filesystem. +func (p *StatVFS) TotalSpace() uint64 { + return p.Frsize * p.Blocks +} + +// FreeSpace calculates the amount of free space in a filesystem. +func (p *StatVFS) FreeSpace() uint64 { + return p.Frsize * p.Bfree +} + +// marshalPacket converts to ssh_FXP_EXTENDED_REPLY packet binary format +func (p *StatVFS) marshalPacket() ([]byte, []byte, error) { + header := []byte{0, 0, 0, 0, sshFxpExtendedReply} + + var buf bytes.Buffer + err := binary.Write(&buf, binary.BigEndian, p) + + return header, buf.Bytes(), err +} + +// MarshalBinary encodes the StatVFS as an SSH_FXP_EXTENDED_REPLY packet. +func (p *StatVFS) MarshalBinary() ([]byte, error) { + header, payload, err := p.marshalPacket() + return append(header, payload...), err +} + +type sshFxpFsyncPacket struct { + ID uint32 + Handle string +} + +func (p *sshFxpFsyncPacket) id() uint32 { return p.ID } + +func (p *sshFxpFsyncPacket) MarshalBinary() ([]byte, error) { + const ext = "fsync@openssh.com" + l := 4 + 1 + 4 + // uint32(length) + byte(type) + uint32(id) + 4 + len(ext) + + 4 + len(p.Handle) + + b := make([]byte, 4, l) + b = append(b, sshFxpExtended) + b = marshalUint32(b, p.ID) + b = marshalString(b, ext) + b = marshalString(b, p.Handle) + + return b, nil +} + +type sshFxpExtendedPacket struct { + ID uint32 + ExtendedRequest string + SpecificPacket interface { + serverRespondablePacket + readonly() bool + } +} + +func (p *sshFxpExtendedPacket) id() uint32 { return p.ID } +func (p *sshFxpExtendedPacket) readonly() bool { + if p.SpecificPacket == nil { + return true + } + return p.SpecificPacket.readonly() +} + +func (p *sshFxpExtendedPacket) respond(svr *Server) responsePacket { + if p.SpecificPacket == nil { + return statusFromError(p.ID, nil) + } + return p.SpecificPacket.respond(svr) +} + +func (p *sshFxpExtendedPacket) UnmarshalBinary(b []byte) error { + var err error + bOrig := b + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.ExtendedRequest, _, err = unmarshalStringSafe(b); err != nil { + return err + } + + // specific unmarshalling + switch p.ExtendedRequest { + case "statvfs@openssh.com": + p.SpecificPacket = &sshFxpExtendedPacketStatVFS{} + case "posix-rename@openssh.com": + p.SpecificPacket = &sshFxpExtendedPacketPosixRename{} + case "hardlink@openssh.com": + p.SpecificPacket = &sshFxpExtendedPacketHardlink{} + default: + return fmt.Errorf("packet type %v: %w", p.SpecificPacket, errUnknownExtendedPacket) + } + + return p.SpecificPacket.UnmarshalBinary(bOrig) +} + +type sshFxpExtendedPacketStatVFS struct { + ID uint32 + ExtendedRequest string + Path string +} + +func (p *sshFxpExtendedPacketStatVFS) id() uint32 { return p.ID } +func (p *sshFxpExtendedPacketStatVFS) readonly() bool { return true } +func (p *sshFxpExtendedPacketStatVFS) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.ExtendedRequest, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Path, _, err = unmarshalStringSafe(b); err != nil { + return err + } + return nil +} + +type sshFxpExtendedPacketPosixRename struct { + ID uint32 + ExtendedRequest string + Oldpath string + Newpath string +} + +func (p *sshFxpExtendedPacketPosixRename) id() uint32 { return p.ID } +func (p *sshFxpExtendedPacketPosixRename) readonly() bool { return false } +func (p *sshFxpExtendedPacketPosixRename) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.ExtendedRequest, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Oldpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Newpath, _, err = unmarshalStringSafe(b); err != nil { + return err + } + return nil +} + +func (p *sshFxpExtendedPacketPosixRename) respond(s *Server) responsePacket { + err := os.Rename(s.toLocalPath(p.Oldpath), s.toLocalPath(p.Newpath)) + return statusFromError(p.ID, err) +} + +type sshFxpExtendedPacketHardlink struct { + ID uint32 + ExtendedRequest string + Oldpath string + Newpath string +} + +// https://github.com/openssh/openssh-portable/blob/master/PROTOCOL +func (p *sshFxpExtendedPacketHardlink) id() uint32 { return p.ID } +func (p *sshFxpExtendedPacketHardlink) readonly() bool { return true } +func (p *sshFxpExtendedPacketHardlink) UnmarshalBinary(b []byte) error { + var err error + if p.ID, b, err = unmarshalUint32Safe(b); err != nil { + return err + } else if p.ExtendedRequest, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Oldpath, b, err = unmarshalStringSafe(b); err != nil { + return err + } else if p.Newpath, _, err = unmarshalStringSafe(b); err != nil { + return err + } + return nil +} + +func (p *sshFxpExtendedPacketHardlink) respond(s *Server) responsePacket { + err := os.Link(s.toLocalPath(p.Oldpath), s.toLocalPath(p.Newpath)) + return statusFromError(p.ID, err) +} diff --git a/vendor/github.com/pkg/sftp/pool.go b/vendor/github.com/pkg/sftp/pool.go new file mode 100644 index 00000000000..36126290657 --- /dev/null +++ b/vendor/github.com/pkg/sftp/pool.go @@ -0,0 +1,79 @@ +package sftp + +// bufPool provides a pool of byte-slices to be reused in various parts of the package. +// It is safe to use concurrently through a pointer. +type bufPool struct { + ch chan []byte + blen int +} + +func newBufPool(depth, bufLen int) *bufPool { + return &bufPool{ + ch: make(chan []byte, depth), + blen: bufLen, + } +} + +func (p *bufPool) Get() []byte { + if p.blen <= 0 { + panic("bufPool: new buffer creation length must be greater than zero") + } + + for { + select { + case b := <-p.ch: + if cap(b) < p.blen { + // just in case: throw away any buffer with insufficient capacity. + continue + } + + return b[:p.blen] + + default: + return make([]byte, p.blen) + } + } +} + +func (p *bufPool) Put(b []byte) { + if p == nil { + // functional default: no reuse. + return + } + + if cap(b) < p.blen || cap(b) > p.blen*2 { + // DO NOT reuse buffers with insufficient capacity. + // This could cause panics when resizing to p.blen. + + // DO NOT reuse buffers with excessive capacity. + // This could cause memory leaks. + return + } + + select { + case p.ch <- b: + default: + } +} + +type resChanPool chan chan result + +func newResChanPool(depth int) resChanPool { + return make(chan chan result, depth) +} + +func (p resChanPool) Get() chan result { + select { + case ch := <-p: + return ch + default: + return make(chan result, 1) + } +} + +func (p resChanPool) Put(ch chan result) { + select { + case p <- ch: + default: + } +} diff --git a/vendor/github.com/pkg/sftp/release.go b/vendor/github.com/pkg/sftp/release.go new file mode 100644 index 00000000000..9ecedc44101 --- /dev/null +++ b/vendor/github.com/pkg/sftp/release.go @@ -0,0 +1,6 @@ +//go:build !debug +// +build !debug + +package sftp + +func debug(fmt string, args ...interface{}) {} diff --git a/vendor/github.com/pkg/sftp/request-attrs.go b/vendor/github.com/pkg/sftp/request-attrs.go new file mode 100644 index 00000000000..476c56518ce --- /dev/null +++ b/vendor/github.com/pkg/sftp/request-attrs.go @@ -0,0 +1,57 @@ +package sftp + +// Methods on the Request object to make working with the Flags bitmasks and +// Attr(ibutes) byte blob easier. Use Pflags() when working with an Open/Write +// request and AttrFlags() and Attributes() when working with SetStat requests. + +// FileOpenFlags defines Open and Write Flags. Correlate directly with with os.OpenFile flags +// (https://golang.org/pkg/os/#pkg-constants). +type FileOpenFlags struct { + Read, Write, Append, Creat, Trunc, Excl bool +} + +func newFileOpenFlags(flags uint32) FileOpenFlags { + return FileOpenFlags{ + Read: flags&sshFxfRead != 0, + Write: flags&sshFxfWrite != 0, + Append: flags&sshFxfAppend != 0, + Creat: flags&sshFxfCreat != 0, + Trunc: flags&sshFxfTrunc != 0, + Excl: flags&sshFxfExcl != 0, + } +} + +// Pflags converts the bitmap/uint32 from SFTP Open packet pflag values, +// into a FileOpenFlags struct with booleans set for flags set in bitmap. +func (r *Request) Pflags() FileOpenFlags { + return newFileOpenFlags(r.Flags) +} + +// FileAttrFlags that indicate whether SFTP file attributes were passed. When a flag is +// true the corresponding attribute should be available from the FileStat +// object returned by Attributes method. Used with SetStat. +type FileAttrFlags struct { + Size, UidGid, Permissions, Acmodtime bool +} + +func newFileAttrFlags(flags uint32) FileAttrFlags { + return FileAttrFlags{ + Size: (flags & sshFileXferAttrSize) != 0, + UidGid: (flags & sshFileXferAttrUIDGID) != 0, + Permissions: (flags & sshFileXferAttrPermissions) != 0, + Acmodtime: (flags & sshFileXferAttrACmodTime) != 0, + } +} + +// AttrFlags returns a FileAttrFlags boolean struct based on the +// bitmap/uint32 file attribute flags from the SFTP packaet. +func (r *Request) AttrFlags() FileAttrFlags { + return newFileAttrFlags(r.Flags) +} + +// Attributes parses file attributes byte blob and return them in a +// FileStat object. +func (r *Request) Attributes() *FileStat { + fs, _, _ := unmarshalFileStat(r.Flags, r.Attrs) + return fs +} diff --git a/vendor/github.com/pkg/sftp/request-errors.go b/vendor/github.com/pkg/sftp/request-errors.go new file mode 100644 index 00000000000..6505b5c74f1 --- /dev/null +++ b/vendor/github.com/pkg/sftp/request-errors.go @@ -0,0 +1,54 @@ +package sftp + +type fxerr uint32 + +// Error types that match the SFTP's SSH_FXP_STATUS codes. Gives you more +// direct control of the errors being sent vs. letting the library work them +// out from the standard os/io errors. +const ( + ErrSSHFxOk = fxerr(sshFxOk) + ErrSSHFxEOF = fxerr(sshFxEOF) + ErrSSHFxNoSuchFile = fxerr(sshFxNoSuchFile) + ErrSSHFxPermissionDenied = fxerr(sshFxPermissionDenied) + ErrSSHFxFailure = fxerr(sshFxFailure) + ErrSSHFxBadMessage = fxerr(sshFxBadMessage) + ErrSSHFxNoConnection = fxerr(sshFxNoConnection) + ErrSSHFxConnectionLost = fxerr(sshFxConnectionLost) + ErrSSHFxOpUnsupported = fxerr(sshFxOPUnsupported) +) + +// Deprecated error types, these are aliases for the new ones, please use the new ones directly +const ( + ErrSshFxOk = ErrSSHFxOk + ErrSshFxEof = ErrSSHFxEOF + ErrSshFxNoSuchFile = ErrSSHFxNoSuchFile + ErrSshFxPermissionDenied = ErrSSHFxPermissionDenied + ErrSshFxFailure = ErrSSHFxFailure + ErrSshFxBadMessage = ErrSSHFxBadMessage + ErrSshFxNoConnection = ErrSSHFxNoConnection + ErrSshFxConnectionLost = ErrSSHFxConnectionLost + ErrSshFxOpUnsupported = ErrSSHFxOpUnsupported +) + +func (e fxerr) Error() string { + switch e { + case ErrSSHFxOk: + return "OK" + case ErrSSHFxEOF: + return "EOF" + case ErrSSHFxNoSuchFile: + return "no such file" + case ErrSSHFxPermissionDenied: + return "permission denied" + case ErrSSHFxBadMessage: + return "bad message" + case ErrSSHFxNoConnection: + return "no connection" + case ErrSSHFxConnectionLost: + return "connection lost" + case ErrSSHFxOpUnsupported: + return "operation unsupported" + default: + return "failure" + } +} diff --git a/vendor/github.com/pkg/sftp/request-example.go b/vendor/github.com/pkg/sftp/request-example.go new file mode 100644 index 00000000000..f003e7114c0 --- /dev/null +++ b/vendor/github.com/pkg/sftp/request-example.go @@ -0,0 +1,647 @@ +package sftp + +// This serves as an example of how to implement the request server handler as +// well as a dummy backend for testing. It implements an in-memory backend that +// works as a very simple filesystem with simple flat key-value lookup system. + +import ( + "errors" + "io" + "os" + "path" + "sort" + "strings" + "sync" + "syscall" + "time" +) + +const maxSymlinkFollows = 5 + +var errTooManySymlinks = errors.New("too many symbolic links") + +// InMemHandler returns a Handlers object with the test handlers. +func InMemHandler() Handlers { + root := &root{ + rootFile: &memFile{name: "/", modtime: time.Now(), isdir: true}, + files: make(map[string]*memFile), + } + return Handlers{root, root, root, root} +} + +// Example Handlers +func (fs *root) Fileread(r *Request) (io.ReaderAt, error) { + flags := r.Pflags() + if !flags.Read { + // sanity check + return nil, os.ErrInvalid + } + + return fs.OpenFile(r) +} + +func (fs *root) Filewrite(r *Request) (io.WriterAt, error) { + flags := r.Pflags() + if !flags.Write { + // sanity check + return nil, os.ErrInvalid + } + + return fs.OpenFile(r) +} + +func (fs *root) OpenFile(r *Request) (WriterAtReaderAt, error) { + if fs.mockErr != nil { + return nil, fs.mockErr + } + _ = r.WithContext(r.Context()) // initialize context for deadlock testing + + fs.mu.Lock() + defer fs.mu.Unlock() + + return fs.openfile(r.Filepath, r.Flags) +} + +func (fs *root) putfile(pathname string, file *memFile) error { + pathname, err := fs.canonName(pathname) + if err != nil { + return err + } + + if !strings.HasPrefix(pathname, "/") { + return os.ErrInvalid + } + + if _, err := fs.lfetch(pathname); err != os.ErrNotExist { + return os.ErrExist + } + + file.name = pathname + fs.files[pathname] = file + + return nil +} + +func (fs *root) openfile(pathname string, flags uint32) (*memFile, error) { + pflags := newFileOpenFlags(flags) + + file, err := fs.fetch(pathname) + if err == os.ErrNotExist { + if !pflags.Creat { + return nil, os.ErrNotExist + } + + var count int + // You can create files through dangling symlinks. + link, err := fs.lfetch(pathname) + for err == nil && link.symlink != "" { + if pflags.Excl { + // unless you also passed in O_EXCL + return nil, os.ErrInvalid + } + + if count++; count > maxSymlinkFollows { + return nil, errTooManySymlinks + } + + pathname = link.symlink + link, err = fs.lfetch(pathname) + } + + file := &memFile{ + modtime: time.Now(), + } + + if err := fs.putfile(pathname, file); err != nil { + return nil, err + } + + return file, nil + } + + if err != nil { + return nil, err + } + + if pflags.Creat && pflags.Excl { + return nil, os.ErrExist + } + + if file.IsDir() { + return nil, os.ErrInvalid + } + + if pflags.Trunc { + if err := file.Truncate(0); err != nil { + return nil, err + } + } + + return file, nil +} + +func (fs *root) Filecmd(r *Request) error { + if fs.mockErr != nil { + return fs.mockErr + } + _ = r.WithContext(r.Context()) // initialize context for deadlock testing + + fs.mu.Lock() + defer fs.mu.Unlock() + + switch r.Method { + case "Setstat": + file, err := fs.openfile(r.Filepath, sshFxfWrite) + if err != nil { + return err + } + + if r.AttrFlags().Size { + return file.Truncate(int64(r.Attributes().Size)) + } + + return nil + + case "Rename": + // SFTP-v2: "It is an error if there already exists a file with the name specified by newpath." + // This varies from the POSIX specification, which allows limited replacement of target files. + if fs.exists(r.Target) { + return os.ErrExist + } + + return fs.rename(r.Filepath, r.Target) + + case "Rmdir": + return fs.rmdir(r.Filepath) + + case "Remove": + // IEEE 1003.1 remove explicitly can unlink files and remove empty directories. + // We use instead here the semantics of unlink, which is allowed to be restricted against directories. + return fs.unlink(r.Filepath) + + case "Mkdir": + return fs.mkdir(r.Filepath) + + case "Link": + return fs.link(r.Filepath, r.Target) + + case "Symlink": + // NOTE: r.Filepath is the target, and r.Target is the linkpath. + return fs.symlink(r.Filepath, r.Target) + } + + return errors.New("unsupported") +} + +func (fs *root) rename(oldpath, newpath string) error { + file, err := fs.lfetch(oldpath) + if err != nil { + return err + } + + newpath, err = fs.canonName(newpath) + if err != nil { + return err + } + + if !strings.HasPrefix(newpath, "/") { + return os.ErrInvalid + } + + target, err := fs.lfetch(newpath) + if err != os.ErrNotExist { + if target == file { + // IEEE 1003.1: if oldpath and newpath are the same directory entry, + // then return no error, and perform no further action. + return nil + } + + switch { + case file.IsDir(): + // IEEE 1003.1: if oldpath is a directory, and newpath exists, + // then newpath must be a directory, and empty. + // It is to be removed prior to rename. + if err := fs.rmdir(newpath); err != nil { + return err + } + + case target.IsDir(): + // IEEE 1003.1: if oldpath is not a directory, and newpath exists, + // then newpath may not be a directory. + return syscall.EISDIR + } + } + + fs.files[newpath] = file + + if file.IsDir() { + dirprefix := file.name + "/" + + for name, file := range fs.files { + if strings.HasPrefix(name, dirprefix) { + newname := path.Join(newpath, strings.TrimPrefix(name, dirprefix)) + + fs.files[newname] = file + file.name = newname + delete(fs.files, name) + } + } + } + + file.name = newpath + delete(fs.files, oldpath) + + return nil +} + +func (fs *root) PosixRename(r *Request) error { + if fs.mockErr != nil { + return fs.mockErr + } + _ = r.WithContext(r.Context()) // initialize context for deadlock testing + + fs.mu.Lock() + defer fs.mu.Unlock() + + return fs.rename(r.Filepath, r.Target) +} + +func (fs *root) StatVFS(r *Request) (*StatVFS, error) { + if fs.mockErr != nil { + return nil, fs.mockErr + } + + return getStatVFSForPath(r.Filepath) +} + +func (fs *root) mkdir(pathname string) error { + dir := &memFile{ + modtime: time.Now(), + isdir: true, + } + + return fs.putfile(pathname, dir) +} + +func (fs *root) rmdir(pathname string) error { + // IEEE 1003.1: If pathname is a symlink, then rmdir should fail with ENOTDIR. + dir, err := fs.lfetch(pathname) + if err != nil { + return err + } + + if !dir.IsDir() { + return syscall.ENOTDIR + } + + // use the dir‘s internal name not the pathname we passed in. + // the dir.name is always the canonical name of a directory. + pathname = dir.name + + for name := range fs.files { + if path.Dir(name) == pathname { + return errors.New("directory not empty") + } + } + + delete(fs.files, pathname) + + return nil +} + +func (fs *root) link(oldpath, newpath string) error { + file, err := fs.lfetch(oldpath) + if err != nil { + return err + } + + if file.IsDir() { + return errors.New("hard link not allowed for directory") + } + + return fs.putfile(newpath, file) +} + +// symlink() creates a symbolic link named `linkpath` which contains the string `target`. +// NOTE! This would be called with `symlink(req.Filepath, req.Target)` due to different semantics. +func (fs *root) symlink(target, linkpath string) error { + link := &memFile{ + modtime: time.Now(), + symlink: target, + } + + return fs.putfile(linkpath, link) +} + +func (fs *root) unlink(pathname string) error { + // does not follow symlinks! + file, err := fs.lfetch(pathname) + if err != nil { + return err + } + + if file.IsDir() { + // IEEE 1003.1: implementations may opt out of allowing the unlinking of directories. + // SFTP-v2: SSH_FXP_REMOVE may not remove directories. + return os.ErrInvalid + } + + // DO NOT use the file’s internal name. + // because of hard-links files cannot have a single canonical name. + delete(fs.files, pathname) + + return nil +} + +type listerat []os.FileInfo + +// Modeled after strings.Reader's ReadAt() implementation +func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) { + var n int + if offset >= int64(len(f)) { + return 0, io.EOF + } + n = copy(ls, f[offset:]) + if n < len(ls) { + return n, io.EOF + } + return n, nil +} + +func (fs *root) Filelist(r *Request) (ListerAt, error) { + if fs.mockErr != nil { + return nil, fs.mockErr + } + _ = r.WithContext(r.Context()) // initialize context for deadlock testing + + fs.mu.Lock() + defer fs.mu.Unlock() + + switch r.Method { + case "List": + files, err := fs.readdir(r.Filepath) + if err != nil { + return nil, err + } + return listerat(files), nil + + case "Stat": + file, err := fs.fetch(r.Filepath) + if err != nil { + return nil, err + } + return listerat{file}, nil + } + + return nil, errors.New("unsupported") +} + +func (fs *root) readdir(pathname string) ([]os.FileInfo, error) { + dir, err := fs.fetch(pathname) + if err != nil { + return nil, err + } + + if !dir.IsDir() { + return nil, syscall.ENOTDIR + } + + var files []os.FileInfo + + for name, file := range fs.files { + if path.Dir(name) == dir.name { + files = append(files, file) + } + } + + sort.Slice(files, func(i, j int) bool { return files[i].Name() < files[j].Name() }) + + return files, nil +} + +func (fs *root) Readlink(pathname string) (string, error) { + file, err := fs.lfetch(pathname) + if err != nil { + return "", err + } + + if file.symlink == "" { + return "", os.ErrInvalid + } + + return file.symlink, nil +} + +// implements LstatFileLister interface +func (fs *root) Lstat(r *Request) (ListerAt, error) { + if fs.mockErr != nil { + return nil, fs.mockErr + } + _ = r.WithContext(r.Context()) // initialize context for deadlock testing + + fs.mu.Lock() + defer fs.mu.Unlock() + + file, err := fs.lfetch(r.Filepath) + if err != nil { + return nil, err + } + return listerat{file}, nil +} + +// In memory file-system-y thing that the Handlers live on +type root struct { + rootFile *memFile + mockErr error + + mu sync.Mutex + files map[string]*memFile +} + +// Set a mocked error that the next handler call will return. +// Set to nil to reset for no error. +func (fs *root) returnErr(err error) { + fs.mockErr = err +} + +func (fs *root) lfetch(path string) (*memFile, error) { + if path == "/" { + return fs.rootFile, nil + } + + file, ok := fs.files[path] + if file == nil { + if ok { + delete(fs.files, path) + } + + return nil, os.ErrNotExist + } + + return file, nil +} + +// canonName returns the “canonical” name of a file, that is: +// if the directory of the pathname is a symlink, it follows that symlink to the valid directory name. +// this is relatively easy, since `dir.name` will be the only valid canonical path for a directory. +func (fs *root) canonName(pathname string) (string, error) { + dirname, filename := path.Dir(pathname), path.Base(pathname) + + dir, err := fs.fetch(dirname) + if err != nil { + return "", err + } + + if !dir.IsDir() { + return "", syscall.ENOTDIR + } + + return path.Join(dir.name, filename), nil +} + +func (fs *root) exists(path string) bool { + path, err := fs.canonName(path) + if err != nil { + return false + } + + _, err = fs.lfetch(path) + + return err != os.ErrNotExist +} + +func (fs *root) fetch(pathname string) (*memFile, error) { + file, err := fs.lfetch(pathname) + if err != nil { + return nil, err + } + + var count int + for file.symlink != "" { + if count++; count > maxSymlinkFollows { + return nil, errTooManySymlinks + } + + linkTarget := file.symlink + if !path.IsAbs(linkTarget) { + linkTarget = path.Join(path.Dir(file.name), linkTarget) + } + + file, err = fs.lfetch(linkTarget) + if err != nil { + return nil, err + } + } + + return file, nil +} + +// Implements os.FileInfo, io.ReaderAt and io.WriterAt interfaces. +// These are the 3 interfaces necessary for the Handlers. +// Implements the optional interface TransferError. +type memFile struct { + name string + modtime time.Time + symlink string + isdir bool + + mu sync.RWMutex + content []byte + err error +} + +// These are helper functions, they must be called while holding the memFile.mu mutex +func (f *memFile) size() int64 { return int64(len(f.content)) } +func (f *memFile) grow(n int64) { f.content = append(f.content, make([]byte, n)...) } + +// Have memFile fulfill os.FileInfo interface +func (f *memFile) Name() string { return path.Base(f.name) } +func (f *memFile) Size() int64 { + f.mu.Lock() + defer f.mu.Unlock() + + return f.size() +} +func (f *memFile) Mode() os.FileMode { + if f.isdir { + return os.FileMode(0755) | os.ModeDir + } + if f.symlink != "" { + return os.FileMode(0777) | os.ModeSymlink + } + return os.FileMode(0644) +} +func (f *memFile) ModTime() time.Time { return f.modtime } +func (f *memFile) IsDir() bool { return f.isdir } +func (f *memFile) Sys() interface{} { + return fakeFileInfoSys() +} + +func (f *memFile) ReadAt(b []byte, off int64) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + + if f.err != nil { + return 0, f.err + } + + if off < 0 { + return 0, errors.New("memFile.ReadAt: negative offset") + } + + if off >= f.size() { + return 0, io.EOF + } + + n := copy(b, f.content[off:]) + if n < len(b) { + return n, io.EOF + } + + return n, nil +} + +func (f *memFile) WriteAt(b []byte, off int64) (int, error) { + // fmt.Println(string(p), off) + // mimic write delays, should be optional + time.Sleep(time.Microsecond * time.Duration(len(b))) + + f.mu.Lock() + defer f.mu.Unlock() + + if f.err != nil { + return 0, f.err + } + + grow := int64(len(b)) + off - f.size() + if grow > 0 { + f.grow(grow) + } + + return copy(f.content[off:], b), nil +} + +func (f *memFile) Truncate(size int64) error { + f.mu.Lock() + defer f.mu.Unlock() + + if f.err != nil { + return f.err + } + + grow := size - f.size() + if grow <= 0 { + f.content = f.content[:size] + } else { + f.grow(grow) + } + + return nil +} + +func (f *memFile) TransferError(err error) { + f.mu.Lock() + defer f.mu.Unlock() + + f.err = err +} diff --git a/vendor/github.com/pkg/sftp/request-interfaces.go b/vendor/github.com/pkg/sftp/request-interfaces.go new file mode 100644 index 00000000000..13e7577e3d6 --- /dev/null +++ b/vendor/github.com/pkg/sftp/request-interfaces.go @@ -0,0 +1,159 @@ +package sftp + +import ( + "io" + "os" +) + +// WriterAtReaderAt defines the interface to return when a file is to +// be opened for reading and writing +type WriterAtReaderAt interface { + io.WriterAt + io.ReaderAt +} + +// Interfaces are differentiated based on required returned values. +// All input arguments are to be pulled from Request (the only arg). + +// The Handler interfaces all take the Request object as its only argument. +// All the data you should need to handle the call are in the Request object. +// The request.Method attribute is initially the most important one as it +// determines which Handler gets called. + +// FileReader should return an io.ReaderAt for the filepath +// Note in cases of an error, the error text will be sent to the client. +// Called for Methods: Get +type FileReader interface { + Fileread(*Request) (io.ReaderAt, error) +} + +// FileWriter should return an io.WriterAt for the filepath. +// +// The request server code will call Close() on the returned io.WriterAt +// object if an io.Closer type assertion succeeds. +// Note in cases of an error, the error text will be sent to the client. +// Note when receiving an Append flag it is important to not open files using +// O_APPEND if you plan to use WriteAt, as they conflict. +// Called for Methods: Put, Open +type FileWriter interface { + Filewrite(*Request) (io.WriterAt, error) +} + +// OpenFileWriter is a FileWriter that implements the generic OpenFile method. +// You need to implement this optional interface if you want to be able +// to read and write from/to the same handle. +// Called for Methods: Open +type OpenFileWriter interface { + FileWriter + OpenFile(*Request) (WriterAtReaderAt, error) +} + +// FileCmder should return an error +// Note in cases of an error, the error text will be sent to the client. +// Called for Methods: Setstat, Rename, Rmdir, Mkdir, Link, Symlink, Remove +type FileCmder interface { + Filecmd(*Request) error +} + +// PosixRenameFileCmder is a FileCmder that implements the PosixRename method. +// If this interface is implemented PosixRename requests will call it +// otherwise they will be handled in the same way as Rename +type PosixRenameFileCmder interface { + FileCmder + PosixRename(*Request) error +} + +// StatVFSFileCmder is a FileCmder that implements the StatVFS method. +// You need to implement this interface if you want to handle statvfs requests. +// Please also be sure that the statvfs@openssh.com extension is enabled +type StatVFSFileCmder interface { + FileCmder + StatVFS(*Request) (*StatVFS, error) +} + +// FileLister should return an object that fulfils the ListerAt interface +// Note in cases of an error, the error text will be sent to the client. +// Called for Methods: List, Stat, Readlink +// +// Since Filelist returns an os.FileInfo, this can make it non-ideal for implementing Readlink. +// This is because the Name receiver method defined by that interface defines that it should only return the base name. +// However, Readlink is required to be capable of returning essentially any arbitrary valid path relative or absolute. +// In order to implement this more expressive requirement, implement [ReadlinkFileLister] which will then be used instead. +type FileLister interface { + Filelist(*Request) (ListerAt, error) +} + +// LstatFileLister is a FileLister that implements the Lstat method. +// If this interface is implemented Lstat requests will call it +// otherwise they will be handled in the same way as Stat +type LstatFileLister interface { + FileLister + Lstat(*Request) (ListerAt, error) +} + +// RealPathFileLister is a FileLister that implements the Realpath method. +// The built-in RealPath implementation does not resolve symbolic links. +// By implementing this interface you can customize the returned path +// and, for example, resolve symbolinc links if needed for your use case. +// You have to return an absolute POSIX path. +// +// Up to v1.13.5 the signature for the RealPath method was: +// +// # RealPath(string) string +// +// we have added a legacyRealPathFileLister that implements the old method +// to ensure that your code does not break. +// You should use the new method signature to avoid future issues +type RealPathFileLister interface { + FileLister + RealPath(string) (string, error) +} + +// ReadlinkFileLister is a FileLister that implements the Readlink method. +// By implementing the Readlink method, it is possible to return any arbitrary valid path relative or absolute. +// This allows giving a better response than via the default FileLister (which is limited to os.FileInfo, whose Name method should only return the base name of a file) +type ReadlinkFileLister interface { + FileLister + Readlink(string) (string, error) +} + +// This interface is here for backward compatibility only +type legacyRealPathFileLister interface { + FileLister + RealPath(string) string +} + +// NameLookupFileLister is a FileLister that implmeents the LookupUsername and LookupGroupName methods. +// If this interface is implemented, then longname ls formatting will use these to convert usernames and groupnames. +type NameLookupFileLister interface { + FileLister + LookupUserName(string) string + LookupGroupName(string) string +} + +// ListerAt does for file lists what io.ReaderAt does for files, i.e. a []os.FileInfo buffer is passed to the ListAt function +// and the entries that are populated in the buffer will be passed to the client. +// +// ListAt should return the number of entries copied and an io.EOF error if at end of list. +// This is testable by comparing how many you copied to how many could be copied (eg. n < len(ls) below). +// The copy() builtin is best for the copying. +// +// Uid and gid information will on unix systems be retrieved from [os.FileInfo.Sys] +// if this function returns a [syscall.Stat_t] when called on a populated entry. +// Alternatively, if the entry implements [FileInfoUidGid], it will be used for uid and gid information. +// +// If a populated entry implements [FileInfoExtendedData], extended attributes will also be returned to the client. +// +// The request server code will call Close() on ListerAt if an io.Closer type assertion succeeds. +// +// Note in cases of an error, the error text will be sent to the client. +type ListerAt interface { + ListAt([]os.FileInfo, int64) (int, error) +} + +// TransferError is an optional interface that readerAt and writerAt +// can implement to be notified about the error causing Serve() to exit +// with the request still open +type TransferError interface { + TransferError(err error) +} diff --git a/vendor/github.com/pkg/sftp/request-plan9.go b/vendor/github.com/pkg/sftp/request-plan9.go new file mode 100644 index 00000000000..38f91bcde28 --- /dev/null +++ b/vendor/github.com/pkg/sftp/request-plan9.go @@ -0,0 +1,16 @@ +//go:build plan9 +// +build plan9 + +package sftp + +import ( + "syscall" +) + +func fakeFileInfoSys() interface{} { + return &syscall.Dir{} +} + +func testOsSys(sys interface{}) error { + return nil +} diff --git a/vendor/github.com/pkg/sftp/request-readme.md b/vendor/github.com/pkg/sftp/request-readme.md new file mode 100644 index 00000000000..f8b81f3aa37 --- /dev/null +++ b/vendor/github.com/pkg/sftp/request-readme.md @@ -0,0 +1,53 @@ +# Request Based SFTP API + +The request based API allows for custom backends in a way similar to the http +package. In order to create a backend you need to implement 4 handler +interfaces; one for reading, one for writing, one for misc commands and one for +listing files. Each has 1 required method and in each case those methods take +the Request as the only parameter and they each return something different. +These 4 interfaces are enough to handle all the SFTP traffic in a simplified +manner. + +The Request structure has 5 public fields which you will deal with. + +- Method (string) - string name of incoming call +- Filepath (string) - POSIX path of file to act on +- Flags (uint32) - 32bit bitmask value of file open/create flags +- Attrs ([]byte) - byte string of file attribute data +- Target (string) - target path for renames and sym-links + +Below are the methods and a brief description of what they need to do. + +### Fileread(*Request) (io.Reader, error) + +Handler for "Get" method and returns an io.Reader for the file which the server +then sends to the client. + +### Filewrite(*Request) (io.Writer, error) + +Handler for "Put" method and returns an io.Writer for the file which the server +then writes the uploaded file to. The file opening "pflags" are currently +preserved in the Request.Flags field as a 32bit bitmask value. See the [SFTP +spec](https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt#section-6.3) for +details. + +### Filecmd(*Request) error + +Handles "SetStat", "Rename", "Rmdir", "Mkdir" and "Symlink" methods. Makes the +appropriate changes and returns nil for success or an filesystem like error +(eg. os.ErrNotExist). The attributes are currently propagated in their raw form +([]byte) and will need to be unmarshalled to be useful. See the respond method +on sshFxpSetstatPacket for example of you might want to do this. + +### Fileinfo(*Request) ([]os.FileInfo, error) + +Handles "List", "Stat", "Readlink" methods. Gathers/creates FileInfo structs +with the data on the files and returns in a list (list of 1 for Stat and +Readlink). + + +## TODO + +- Add support for API users to see trace/debugging info of what is going on +inside SFTP server. +- Unmarshal the file attributes into a structure on the Request object. diff --git a/vendor/github.com/pkg/sftp/request-server.go b/vendor/github.com/pkg/sftp/request-server.go new file mode 100644 index 00000000000..08c24d7f6fa --- /dev/null +++ b/vendor/github.com/pkg/sftp/request-server.go @@ -0,0 +1,355 @@ +package sftp + +import ( + "context" + "errors" + "io" + "path" + "path/filepath" + "strconv" + "sync" +) + +const defaultMaxTxPacket uint32 = 1 << 15 + +// Handlers contains the 4 SFTP server request handlers. +type Handlers struct { + FileGet FileReader + FilePut FileWriter + FileCmd FileCmder + FileList FileLister +} + +// RequestServer abstracts the sftp protocol with an http request-like protocol +type RequestServer struct { + Handlers Handlers + + *serverConn + pktMgr *packetManager + + startDirectory string + maxTxPacket uint32 + + mu sync.RWMutex + handleCount int + openRequests map[string]*Request +} + +// A RequestServerOption is a function which applies configuration to a RequestServer. +type RequestServerOption func(*RequestServer) + +// WithRSAllocator enable the allocator. +// After processing a packet we keep in memory the allocated slices +// and we reuse them for new packets. +// The allocator is experimental +func WithRSAllocator() RequestServerOption { + return func(rs *RequestServer) { + alloc := newAllocator() + rs.pktMgr.alloc = alloc + rs.conn.alloc = alloc + } +} + +// WithStartDirectory sets a start directory to use as base for relative paths. +// If unset the default is "/" +func WithStartDirectory(startDirectory string) RequestServerOption { + return func(rs *RequestServer) { + rs.startDirectory = cleanPath(startDirectory) + } +} + +// WithRSMaxTxPacket sets the maximum size of the payload returned to the client, +// measured in bytes. The default value is 32768 bytes, and this option +// can only be used to increase it. Setting this option to a larger value +// should be safe, because the client decides the size of the requested payload. +// +// The default maximum packet size is 32768 bytes. +func WithRSMaxTxPacket(size uint32) RequestServerOption { + return func(rs *RequestServer) { + if size < defaultMaxTxPacket { + return + } + + rs.maxTxPacket = size + } +} + +// NewRequestServer creates/allocates/returns new RequestServer. +// Normally there will be one server per user-session. +func NewRequestServer(rwc io.ReadWriteCloser, h Handlers, options ...RequestServerOption) *RequestServer { + svrConn := &serverConn{ + conn: conn{ + Reader: rwc, + WriteCloser: rwc, + }, + } + rs := &RequestServer{ + Handlers: h, + + serverConn: svrConn, + pktMgr: newPktMgr(svrConn), + + startDirectory: "/", + maxTxPacket: defaultMaxTxPacket, + + openRequests: make(map[string]*Request), + } + + for _, o := range options { + o(rs) + } + return rs +} + +// New Open packet/Request +func (rs *RequestServer) nextRequest(r *Request) string { + rs.mu.Lock() + defer rs.mu.Unlock() + + rs.handleCount++ + + r.handle = strconv.Itoa(rs.handleCount) + rs.openRequests[r.handle] = r + + return r.handle +} + +// Returns Request from openRequests, bool is false if it is missing. +// +// The Requests in openRequests work essentially as open file descriptors that +// you can do different things with. What you are doing with it are denoted by +// the first packet of that type (read/write/etc). +func (rs *RequestServer) getRequest(handle string) (*Request, bool) { + rs.mu.RLock() + defer rs.mu.RUnlock() + + r, ok := rs.openRequests[handle] + return r, ok +} + +// Close the Request and clear from openRequests map +func (rs *RequestServer) closeRequest(handle string) error { + rs.mu.Lock() + defer rs.mu.Unlock() + + if r, ok := rs.openRequests[handle]; ok { + delete(rs.openRequests, handle) + return r.close() + } + + return EBADF +} + +// Close the read/write/closer to trigger exiting the main server loop +func (rs *RequestServer) Close() error { return rs.conn.Close() } + +func (rs *RequestServer) serveLoop(pktChan chan<- orderedRequest) error { + defer close(pktChan) // shuts down sftpServerWorkers + + var err error + var pkt requestPacket + var pktType fxp + var pktBytes []byte + + for { + pktType, pktBytes, err = rs.serverConn.recvPacket(rs.pktMgr.getNextOrderID()) + if err != nil { + // we don't care about releasing allocated pages here, the server will quit and the allocator freed + return err + } + + pkt, err = makePacket(rxPacket{pktType, pktBytes}) + if err != nil { + switch { + case errors.Is(err, errUnknownExtendedPacket): + // do nothing + default: + debug("makePacket err: %v", err) + rs.conn.Close() // shuts down recvPacket + return err + } + } + + pktChan <- rs.pktMgr.newOrderedRequest(pkt) + } +} + +// Serve requests for user session +func (rs *RequestServer) Serve() error { + defer func() { + if rs.pktMgr.alloc != nil { + rs.pktMgr.alloc.Free() + } + }() + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + var wg sync.WaitGroup + runWorker := func(ch chan orderedRequest) { + wg.Add(1) + go func() { + defer wg.Done() + if err := rs.packetWorker(ctx, ch); err != nil { + rs.conn.Close() // shuts down recvPacket + } + }() + } + pktChan := rs.pktMgr.workerChan(runWorker) + + err := rs.serveLoop(pktChan) + + wg.Wait() // wait for all workers to exit + + rs.mu.Lock() + defer rs.mu.Unlock() + + // make sure all open requests are properly closed + // (eg. possible on dropped connections, client crashes, etc.) + for handle, req := range rs.openRequests { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + req.transferError(err) + + delete(rs.openRequests, handle) + req.close() + } + + return err +} + +func (rs *RequestServer) packetWorker(ctx context.Context, pktChan chan orderedRequest) error { + for pkt := range pktChan { + orderID := pkt.orderID() + if epkt, ok := pkt.requestPacket.(*sshFxpExtendedPacket); ok { + if epkt.SpecificPacket != nil { + pkt.requestPacket = epkt.SpecificPacket + } + } + + var rpkt responsePacket + switch pkt := pkt.requestPacket.(type) { + case *sshFxInitPacket: + rpkt = &sshFxVersionPacket{Version: sftpProtocolVersion, Extensions: sftpExtensions} + case *sshFxpClosePacket: + handle := pkt.getHandle() + rpkt = statusFromError(pkt.ID, rs.closeRequest(handle)) + case *sshFxpRealpathPacket: + var realPath string + var err error + + switch pather := rs.Handlers.FileList.(type) { + case RealPathFileLister: + realPath, err = pather.RealPath(pkt.getPath()) + case legacyRealPathFileLister: + realPath = pather.RealPath(pkt.getPath()) + default: + realPath = cleanPathWithBase(rs.startDirectory, pkt.getPath()) + } + if err != nil { + rpkt = statusFromError(pkt.ID, err) + } else { + rpkt = cleanPacketPath(pkt, realPath) + } + case *sshFxpOpendirPacket: + request := requestFromPacket(ctx, pkt, rs.startDirectory) + handle := rs.nextRequest(request) + rpkt = request.opendir(rs.Handlers, pkt) + if _, ok := rpkt.(*sshFxpHandlePacket); !ok { + // if we return an error we have to remove the handle from the active ones + rs.closeRequest(handle) + } + case *sshFxpOpenPacket: + request := requestFromPacket(ctx, pkt, rs.startDirectory) + handle := rs.nextRequest(request) + rpkt = request.open(rs.Handlers, pkt) + if _, ok := rpkt.(*sshFxpHandlePacket); !ok { + // if we return an error we have to remove the handle from the active ones + rs.closeRequest(handle) + } + case *sshFxpFstatPacket: + handle := pkt.getHandle() + request, ok := rs.getRequest(handle) + if !ok { + rpkt = statusFromError(pkt.ID, EBADF) + } else { + request = &Request{ + Method: "Stat", + Filepath: cleanPathWithBase(rs.startDirectory, request.Filepath), + } + rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID, rs.maxTxPacket) + } + case *sshFxpFsetstatPacket: + handle := pkt.getHandle() + request, ok := rs.getRequest(handle) + if !ok { + rpkt = statusFromError(pkt.ID, EBADF) + } else { + request = &Request{ + Method: "Setstat", + Filepath: cleanPathWithBase(rs.startDirectory, request.Filepath), + } + rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID, rs.maxTxPacket) + } + case *sshFxpExtendedPacketPosixRename: + request := &Request{ + Method: "PosixRename", + Filepath: cleanPathWithBase(rs.startDirectory, pkt.Oldpath), + Target: cleanPathWithBase(rs.startDirectory, pkt.Newpath), + } + rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID, rs.maxTxPacket) + case *sshFxpExtendedPacketStatVFS: + request := &Request{ + Method: "StatVFS", + Filepath: cleanPathWithBase(rs.startDirectory, pkt.Path), + } + rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID, rs.maxTxPacket) + case hasHandle: + handle := pkt.getHandle() + request, ok := rs.getRequest(handle) + if !ok { + rpkt = statusFromError(pkt.id(), EBADF) + } else { + rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID, rs.maxTxPacket) + } + case hasPath: + request := requestFromPacket(ctx, pkt, rs.startDirectory) + rpkt = request.call(rs.Handlers, pkt, rs.pktMgr.alloc, orderID, rs.maxTxPacket) + request.close() + default: + rpkt = statusFromError(pkt.id(), ErrSSHFxOpUnsupported) + } + + rs.pktMgr.readyPacket( + rs.pktMgr.newOrderedResponse(rpkt, orderID)) + } + return nil +} + +// clean and return name packet for file +func cleanPacketPath(pkt *sshFxpRealpathPacket, realPath string) responsePacket { + return &sshFxpNamePacket{ + ID: pkt.id(), + NameAttrs: []*sshFxpNameAttr{ + { + Name: realPath, + LongName: realPath, + Attrs: emptyFileStat, + }, + }, + } +} + +// Makes sure we have a clean POSIX (/) absolute path to work with +func cleanPath(p string) string { + return cleanPathWithBase("/", p) +} + +func cleanPathWithBase(base, p string) string { + p = filepath.ToSlash(filepath.Clean(p)) + if !path.IsAbs(p) { + return path.Join(base, p) + } + return p +} diff --git a/vendor/github.com/pkg/sftp/request-unix.go b/vendor/github.com/pkg/sftp/request-unix.go new file mode 100644 index 00000000000..e3e037d6069 --- /dev/null +++ b/vendor/github.com/pkg/sftp/request-unix.go @@ -0,0 +1,24 @@ +//go:build !windows && !plan9 +// +build !windows,!plan9 + +package sftp + +import ( + "errors" + "syscall" +) + +func fakeFileInfoSys() interface{} { + return &syscall.Stat_t{Uid: 65534, Gid: 65534} +} + +func testOsSys(sys interface{}) error { + fstat := sys.(*FileStat) + if fstat.UID != uint32(65534) { + return errors.New("Uid failed to match") + } + if fstat.GID != uint32(65534) { + return errors.New("Gid failed to match") + } + return nil +} diff --git a/vendor/github.com/pkg/sftp/request.go b/vendor/github.com/pkg/sftp/request.go new file mode 100644 index 00000000000..e7c47a9c9b1 --- /dev/null +++ b/vendor/github.com/pkg/sftp/request.go @@ -0,0 +1,670 @@ +package sftp + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "sync" + "syscall" +) + +// MaxFilelist is the max number of files to return in a readdir batch. +var MaxFilelist int64 = 100 + +// state encapsulates the reader/writer/readdir from handlers. +type state struct { + mu sync.RWMutex + + writerAt io.WriterAt + readerAt io.ReaderAt + writerAtReaderAt WriterAtReaderAt + listerAt ListerAt + lsoffset int64 +} + +// copy returns a shallow copy the state. +// This is broken out to specific fields, +// because we have to copy around the mutex in state. +func (s *state) copy() state { + s.mu.RLock() + defer s.mu.RUnlock() + + return state{ + writerAt: s.writerAt, + readerAt: s.readerAt, + writerAtReaderAt: s.writerAtReaderAt, + listerAt: s.listerAt, + lsoffset: s.lsoffset, + } +} + +func (s *state) setReaderAt(rd io.ReaderAt) { + s.mu.Lock() + defer s.mu.Unlock() + + s.readerAt = rd +} + +func (s *state) getReaderAt() io.ReaderAt { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.readerAt +} + +func (s *state) setWriterAt(rd io.WriterAt) { + s.mu.Lock() + defer s.mu.Unlock() + + s.writerAt = rd +} + +func (s *state) getWriterAt() io.WriterAt { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.writerAt +} + +func (s *state) setWriterAtReaderAt(rw WriterAtReaderAt) { + s.mu.Lock() + defer s.mu.Unlock() + + s.writerAtReaderAt = rw +} + +func (s *state) getWriterAtReaderAt() WriterAtReaderAt { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.writerAtReaderAt +} + +func (s *state) getAllReaderWriters() (io.ReaderAt, io.WriterAt, WriterAtReaderAt) { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.readerAt, s.writerAt, s.writerAtReaderAt +} + +// Returns current offset for file list +func (s *state) lsNext() int64 { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.lsoffset +} + +// Increases next offset +func (s *state) lsInc(offset int64) { + s.mu.Lock() + defer s.mu.Unlock() + + s.lsoffset += offset +} + +// manage file read/write state +func (s *state) setListerAt(la ListerAt) { + s.mu.Lock() + defer s.mu.Unlock() + + s.listerAt = la +} + +func (s *state) getListerAt() ListerAt { + s.mu.RLock() + defer s.mu.RUnlock() + + return s.listerAt +} + +func (s *state) closeListerAt() error { + s.mu.Lock() + defer s.mu.Unlock() + + var err error + + if s.listerAt != nil { + if c, ok := s.listerAt.(io.Closer); ok { + err = c.Close() + } + s.listerAt = nil + } + + return err +} + +// Request contains the data and state for the incoming service request. +type Request struct { + // Get, Put, Setstat, Stat, Rename, Remove + // Rmdir, Mkdir, List, Readlink, Link, Symlink + Method string + Filepath string + Flags uint32 + Attrs []byte // convert to sub-struct + Target string // for renames and sym-links + handle string + + // reader/writer/readdir from handlers + state + + // context lasts duration of request + ctx context.Context + cancelCtx context.CancelFunc +} + +// NewRequest creates a new Request object. +func NewRequest(method, path string) *Request { + return &Request{ + Method: method, + Filepath: cleanPath(path), + } +} + +// copy returns a shallow copy of existing request. +// This is broken out to specific fields, +// because we have to copy around the mutex in state. +func (r *Request) copy() *Request { + return &Request{ + Method: r.Method, + Filepath: r.Filepath, + Flags: r.Flags, + Attrs: r.Attrs, + Target: r.Target, + handle: r.handle, + + state: r.state.copy(), + + ctx: r.ctx, + cancelCtx: r.cancelCtx, + } +} + +// New Request initialized based on packet data +func requestFromPacket(ctx context.Context, pkt hasPath, baseDir string) *Request { + request := &Request{ + Method: requestMethod(pkt), + Filepath: cleanPathWithBase(baseDir, pkt.getPath()), + } + request.ctx, request.cancelCtx = context.WithCancel(ctx) + + switch p := pkt.(type) { + case *sshFxpOpenPacket: + request.Flags = p.Pflags + request.Attrs = p.Attrs.([]byte) + case *sshFxpSetstatPacket: + request.Flags = p.Flags + request.Attrs = p.Attrs.([]byte) + case *sshFxpRenamePacket: + request.Target = cleanPathWithBase(baseDir, p.Newpath) + case *sshFxpSymlinkPacket: + // NOTE: given a POSIX compliant signature: symlink(target, linkpath string) + // this makes Request.Target the linkpath, and Request.Filepath the target. + request.Target = cleanPathWithBase(baseDir, p.Linkpath) + request.Filepath = p.Targetpath + case *sshFxpExtendedPacketHardlink: + request.Target = cleanPathWithBase(baseDir, p.Newpath) + } + return request +} + +// Context returns the request's context. To change the context, +// use WithContext. +// +// The returned context is always non-nil; it defaults to the +// background context. +// +// For incoming server requests, the context is canceled when the +// request is complete or the client's connection closes. +func (r *Request) Context() context.Context { + if r.ctx != nil { + return r.ctx + } + return context.Background() +} + +// WithContext returns a copy of r with its context changed to ctx. +// The provided ctx must be non-nil. +func (r *Request) WithContext(ctx context.Context) *Request { + if ctx == nil { + panic("nil context") + } + r2 := r.copy() + r2.ctx = ctx + r2.cancelCtx = nil + return r2 +} + +// Close reader/writer if possible +func (r *Request) close() error { + defer func() { + if r.cancelCtx != nil { + r.cancelCtx() + } + }() + + err := r.state.closeListerAt() + + rd, wr, rw := r.getAllReaderWriters() + + // Close errors on a Writer are far more likely to be the important one. + // As they can be information that there was a loss of data. + if c, ok := wr.(io.Closer); ok { + if err2 := c.Close(); err == nil { + // update error if it is still nil + err = err2 + } + } + + if c, ok := rw.(io.Closer); ok { + if err2 := c.Close(); err == nil { + // update error if it is still nil + err = err2 + + r.setWriterAtReaderAt(nil) + } + } + + if c, ok := rd.(io.Closer); ok { + if err2 := c.Close(); err == nil { + // update error if it is still nil + err = err2 + } + } + + return err +} + +// Notify transfer error if any +func (r *Request) transferError(err error) { + if err == nil { + return + } + + rd, wr, rw := r.getAllReaderWriters() + + if t, ok := wr.(TransferError); ok { + t.TransferError(err) + } + + if t, ok := rw.(TransferError); ok { + t.TransferError(err) + } + + if t, ok := rd.(TransferError); ok { + t.TransferError(err) + } +} + +// called from worker to handle packet/request +func (r *Request) call(handlers Handlers, pkt requestPacket, alloc *allocator, orderID uint32, maxTxPacket uint32) responsePacket { + switch r.Method { + case "Get": + return fileget(handlers.FileGet, r, pkt, alloc, orderID, maxTxPacket) + case "Put": + return fileput(handlers.FilePut, r, pkt, alloc, orderID, maxTxPacket) + case "Open": + return fileputget(handlers.FilePut, r, pkt, alloc, orderID, maxTxPacket) + case "Setstat", "Rename", "Rmdir", "Mkdir", "Link", "Symlink", "Remove", "PosixRename", "StatVFS": + return filecmd(handlers.FileCmd, r, pkt) + case "List": + return filelist(handlers.FileList, r, pkt) + case "Stat", "Lstat": + return filestat(handlers.FileList, r, pkt) + case "Readlink": + if readlinkFileLister, ok := handlers.FileList.(ReadlinkFileLister); ok { + return readlink(readlinkFileLister, r, pkt) + } + return filestat(handlers.FileList, r, pkt) + default: + return statusFromError(pkt.id(), fmt.Errorf("unexpected method: %s", r.Method)) + } +} + +// Additional initialization for Open packets +func (r *Request) open(h Handlers, pkt requestPacket) responsePacket { + flags := r.Pflags() + + id := pkt.id() + + switch { + case flags.Write, flags.Append, flags.Creat, flags.Trunc: + if flags.Read { + if openFileWriter, ok := h.FilePut.(OpenFileWriter); ok { + r.Method = "Open" + rw, err := openFileWriter.OpenFile(r) + if err != nil { + return statusFromError(id, err) + } + + r.setWriterAtReaderAt(rw) + + return &sshFxpHandlePacket{ + ID: id, + Handle: r.handle, + } + } + } + + r.Method = "Put" + wr, err := h.FilePut.Filewrite(r) + if err != nil { + return statusFromError(id, err) + } + + r.setWriterAt(wr) + + case flags.Read: + r.Method = "Get" + rd, err := h.FileGet.Fileread(r) + if err != nil { + return statusFromError(id, err) + } + + r.setReaderAt(rd) + + default: + return statusFromError(id, errors.New("bad file flags")) + } + + return &sshFxpHandlePacket{ + ID: id, + Handle: r.handle, + } +} + +func (r *Request) opendir(h Handlers, pkt requestPacket) responsePacket { + r.Method = "List" + la, err := h.FileList.Filelist(r) + if err != nil { + return statusFromError(pkt.id(), wrapPathError(r.Filepath, err)) + } + + r.setListerAt(la) + + return &sshFxpHandlePacket{ + ID: pkt.id(), + Handle: r.handle, + } +} + +// wrap FileReader handler +func fileget(h FileReader, r *Request, pkt requestPacket, alloc *allocator, orderID uint32, maxTxPacket uint32) responsePacket { + rd := r.getReaderAt() + if rd == nil { + return statusFromError(pkt.id(), errors.New("unexpected read packet")) + } + + data, offset, _ := packetData(pkt, alloc, orderID, maxTxPacket) + + n, err := rd.ReadAt(data, offset) + // only return EOF error if no data left to read + if err != nil && (err != io.EOF || n == 0) { + return statusFromError(pkt.id(), err) + } + + return &sshFxpDataPacket{ + ID: pkt.id(), + Length: uint32(n), + Data: data[:n], + } +} + +// wrap FileWriter handler +func fileput(h FileWriter, r *Request, pkt requestPacket, alloc *allocator, orderID uint32, maxTxPacket uint32) responsePacket { + wr := r.getWriterAt() + if wr == nil { + return statusFromError(pkt.id(), errors.New("unexpected write packet")) + } + + data, offset, _ := packetData(pkt, alloc, orderID, maxTxPacket) + + _, err := wr.WriteAt(data, offset) + return statusFromError(pkt.id(), err) +} + +// wrap OpenFileWriter handler +func fileputget(h FileWriter, r *Request, pkt requestPacket, alloc *allocator, orderID uint32, maxTxPacket uint32) responsePacket { + rw := r.getWriterAtReaderAt() + if rw == nil { + return statusFromError(pkt.id(), errors.New("unexpected write and read packet")) + } + + switch p := pkt.(type) { + case *sshFxpReadPacket: + data, offset := p.getDataSlice(alloc, orderID, maxTxPacket), int64(p.Offset) + + n, err := rw.ReadAt(data, offset) + // only return EOF error if no data left to read + if err != nil && (err != io.EOF || n == 0) { + return statusFromError(pkt.id(), err) + } + + return &sshFxpDataPacket{ + ID: pkt.id(), + Length: uint32(n), + Data: data[:n], + } + + case *sshFxpWritePacket: + data, offset := p.Data, int64(p.Offset) + + _, err := rw.WriteAt(data, offset) + return statusFromError(pkt.id(), err) + + default: + return statusFromError(pkt.id(), errors.New("unexpected packet type for read or write")) + } +} + +// file data for additional read/write packets +func packetData(p requestPacket, alloc *allocator, orderID uint32, maxTxPacket uint32) (data []byte, offset int64, length uint32) { + switch p := p.(type) { + case *sshFxpReadPacket: + return p.getDataSlice(alloc, orderID, maxTxPacket), int64(p.Offset), p.Len + case *sshFxpWritePacket: + return p.Data, int64(p.Offset), p.Length + } + return +} + +// wrap FileCmder handler +func filecmd(h FileCmder, r *Request, pkt requestPacket) responsePacket { + switch p := pkt.(type) { + case *sshFxpFsetstatPacket: + r.Flags = p.Flags + r.Attrs = p.Attrs.([]byte) + } + + switch r.Method { + case "PosixRename": + if posixRenamer, ok := h.(PosixRenameFileCmder); ok { + err := posixRenamer.PosixRename(r) + return statusFromError(pkt.id(), err) + } + + // PosixRenameFileCmder not implemented handle this request as a Rename + r.Method = "Rename" + err := h.Filecmd(r) + return statusFromError(pkt.id(), err) + + case "StatVFS": + if statVFSCmdr, ok := h.(StatVFSFileCmder); ok { + stat, err := statVFSCmdr.StatVFS(r) + if err != nil { + return statusFromError(pkt.id(), err) + } + stat.ID = pkt.id() + return stat + } + + return statusFromError(pkt.id(), ErrSSHFxOpUnsupported) + } + + err := h.Filecmd(r) + return statusFromError(pkt.id(), err) +} + +// wrap FileLister handler +func filelist(h FileLister, r *Request, pkt requestPacket) responsePacket { + lister := r.getListerAt() + if lister == nil { + return statusFromError(pkt.id(), errors.New("unexpected dir packet")) + } + + offset := r.lsNext() + finfo := make([]os.FileInfo, MaxFilelist) + n, err := lister.ListAt(finfo, offset) + r.lsInc(int64(n)) + // ignore EOF as we only return it when there are no results + finfo = finfo[:n] // avoid need for nil tests below + + switch r.Method { + case "List": + if err != nil && (err != io.EOF || n == 0) { + return statusFromError(pkt.id(), err) + } + + nameAttrs := make([]*sshFxpNameAttr, 0, len(finfo)) + + // If the type conversion fails, we get untyped `nil`, + // which is handled by not looking up any names. + idLookup, _ := h.(NameLookupFileLister) + + for _, fi := range finfo { + nameAttrs = append(nameAttrs, &sshFxpNameAttr{ + Name: fi.Name(), + LongName: runLs(idLookup, fi), + Attrs: []interface{}{fi}, + }) + } + + return &sshFxpNamePacket{ + ID: pkt.id(), + NameAttrs: nameAttrs, + } + + default: + err = fmt.Errorf("unexpected method: %s", r.Method) + return statusFromError(pkt.id(), err) + } +} + +func filestat(h FileLister, r *Request, pkt requestPacket) responsePacket { + var lister ListerAt + var err error + + if r.Method == "Lstat" { + if lstatFileLister, ok := h.(LstatFileLister); ok { + lister, err = lstatFileLister.Lstat(r) + } else { + // LstatFileLister not implemented handle this request as a Stat + r.Method = "Stat" + lister, err = h.Filelist(r) + } + } else { + lister, err = h.Filelist(r) + } + if err != nil { + return statusFromError(pkt.id(), err) + } + finfo := make([]os.FileInfo, 1) + n, err := lister.ListAt(finfo, 0) + finfo = finfo[:n] // avoid need for nil tests below + + switch r.Method { + case "Stat", "Lstat": + if err != nil && err != io.EOF { + return statusFromError(pkt.id(), err) + } + if n == 0 { + err = &os.PathError{ + Op: strings.ToLower(r.Method), + Path: r.Filepath, + Err: syscall.ENOENT, + } + return statusFromError(pkt.id(), err) + } + return &sshFxpStatResponse{ + ID: pkt.id(), + info: finfo[0], + } + case "Readlink": + if err != nil && err != io.EOF { + return statusFromError(pkt.id(), err) + } + if n == 0 { + err = &os.PathError{ + Op: "readlink", + Path: r.Filepath, + Err: syscall.ENOENT, + } + return statusFromError(pkt.id(), err) + } + filename := finfo[0].Name() + return &sshFxpNamePacket{ + ID: pkt.id(), + NameAttrs: []*sshFxpNameAttr{ + { + Name: filename, + LongName: filename, + Attrs: emptyFileStat, + }, + }, + } + default: + err = fmt.Errorf("unexpected method: %s", r.Method) + return statusFromError(pkt.id(), err) + } +} + +func readlink(readlinkFileLister ReadlinkFileLister, r *Request, pkt requestPacket) responsePacket { + resolved, err := readlinkFileLister.Readlink(r.Filepath) + if err != nil { + return statusFromError(pkt.id(), err) + } + return &sshFxpNamePacket{ + ID: pkt.id(), + NameAttrs: []*sshFxpNameAttr{ + { + Name: resolved, + LongName: resolved, + Attrs: emptyFileStat, + }, + }, + } +} + +// init attributes of request object from packet data +func requestMethod(p requestPacket) (method string) { + switch p.(type) { + case *sshFxpReadPacket, *sshFxpWritePacket, *sshFxpOpenPacket: + // set in open() above + case *sshFxpOpendirPacket, *sshFxpReaddirPacket: + // set in opendir() above + case *sshFxpSetstatPacket, *sshFxpFsetstatPacket: + method = "Setstat" + case *sshFxpRenamePacket: + method = "Rename" + case *sshFxpSymlinkPacket: + method = "Symlink" + case *sshFxpRemovePacket: + method = "Remove" + case *sshFxpStatPacket, *sshFxpFstatPacket: + method = "Stat" + case *sshFxpLstatPacket: + method = "Lstat" + case *sshFxpRmdirPacket: + method = "Rmdir" + case *sshFxpReadlinkPacket: + method = "Readlink" + case *sshFxpMkdirPacket: + method = "Mkdir" + case *sshFxpExtendedPacketHardlink: + method = "Link" + } + return method +} diff --git a/vendor/github.com/pkg/sftp/request_windows.go b/vendor/github.com/pkg/sftp/request_windows.go new file mode 100644 index 00000000000..bd1d6864573 --- /dev/null +++ b/vendor/github.com/pkg/sftp/request_windows.go @@ -0,0 +1,13 @@ +package sftp + +import ( + "syscall" +) + +func fakeFileInfoSys() interface{} { + return syscall.Win32FileAttributeData{} +} + +func testOsSys(sys interface{}) error { + return nil +} diff --git a/vendor/github.com/pkg/sftp/server.go b/vendor/github.com/pkg/sftp/server.go new file mode 100644 index 00000000000..7735c42c4e5 --- /dev/null +++ b/vendor/github.com/pkg/sftp/server.go @@ -0,0 +1,657 @@ +package sftp + +// sftp server counterpart + +import ( + "encoding" + "errors" + "fmt" + "io" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "sync" + "syscall" + "time" +) + +const ( + // SftpServerWorkerCount defines the number of workers for the SFTP server + SftpServerWorkerCount = 8 +) + +type file interface { + Stat() (os.FileInfo, error) + ReadAt(b []byte, off int64) (int, error) + WriteAt(b []byte, off int64) (int, error) + Readdir(int) ([]os.FileInfo, error) + Name() string + Truncate(int64) error + Chmod(mode fs.FileMode) error + Chown(uid, gid int) error + Close() error +} + +// Server is an SSH File Transfer Protocol (sftp) server. +// This is intended to provide the sftp subsystem to an ssh server daemon. +// This implementation currently supports most of sftp server protocol version 3, +// as specified at https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt +type Server struct { + *serverConn + debugStream io.Writer + readOnly bool + pktMgr *packetManager + openFiles map[string]file + openFilesLock sync.RWMutex + handleCount int + workDir string + winRoot bool + maxTxPacket uint32 +} + +func (svr *Server) nextHandle(f file) string { + svr.openFilesLock.Lock() + defer svr.openFilesLock.Unlock() + svr.handleCount++ + handle := strconv.Itoa(svr.handleCount) + svr.openFiles[handle] = f + return handle +} + +func (svr *Server) closeHandle(handle string) error { + svr.openFilesLock.Lock() + defer svr.openFilesLock.Unlock() + if f, ok := svr.openFiles[handle]; ok { + delete(svr.openFiles, handle) + return f.Close() + } + + return EBADF +} + +func (svr *Server) getHandle(handle string) (file, bool) { + svr.openFilesLock.RLock() + defer svr.openFilesLock.RUnlock() + f, ok := svr.openFiles[handle] + return f, ok +} + +type serverRespondablePacket interface { + encoding.BinaryUnmarshaler + id() uint32 + respond(svr *Server) responsePacket +} + +// NewServer creates a new Server instance around the provided streams, serving +// content from the root of the filesystem. Optionally, ServerOption +// functions may be specified to further configure the Server. +// +// A subsequent call to Serve() is required to begin serving files over SFTP. +func NewServer(rwc io.ReadWriteCloser, options ...ServerOption) (*Server, error) { + svrConn := &serverConn{ + conn: conn{ + Reader: rwc, + WriteCloser: rwc, + }, + } + s := &Server{ + serverConn: svrConn, + debugStream: ioutil.Discard, + pktMgr: newPktMgr(svrConn), + openFiles: make(map[string]file), + maxTxPacket: defaultMaxTxPacket, + } + + for _, o := range options { + if err := o(s); err != nil { + return nil, err + } + } + + return s, nil +} + +// A ServerOption is a function which applies configuration to a Server. +type ServerOption func(*Server) error + +// WithDebug enables Server debugging output to the supplied io.Writer. +func WithDebug(w io.Writer) ServerOption { + return func(s *Server) error { + s.debugStream = w + return nil + } +} + +// ReadOnly configures a Server to serve files in read-only mode. +func ReadOnly() ServerOption { + return func(s *Server) error { + s.readOnly = true + return nil + } +} + +// WindowsRootEnumeratesDrives configures a Server to serve a virtual '/' for windows that lists all drives +func WindowsRootEnumeratesDrives() ServerOption { + return func(s *Server) error { + s.winRoot = true + return nil + } +} + +// WithAllocator enable the allocator. +// After processing a packet we keep in memory the allocated slices +// and we reuse them for new packets. +// The allocator is experimental +func WithAllocator() ServerOption { + return func(s *Server) error { + alloc := newAllocator() + s.pktMgr.alloc = alloc + s.conn.alloc = alloc + return nil + } +} + +// WithServerWorkingDirectory sets a working directory to use as base +// for relative paths. +// If unset the default is current working directory (os.Getwd). +func WithServerWorkingDirectory(workDir string) ServerOption { + return func(s *Server) error { + s.workDir = cleanPath(workDir) + return nil + } +} + +// WithMaxTxPacket sets the maximum size of the payload returned to the client, +// measured in bytes. The default value is 32768 bytes, and this option +// can only be used to increase it. Setting this option to a larger value +// should be safe, because the client decides the size of the requested payload. +// +// The default maximum packet size is 32768 bytes. +func WithMaxTxPacket(size uint32) ServerOption { + return func(s *Server) error { + if size < defaultMaxTxPacket { + return errors.New("size must be greater than or equal to 32768") + } + + s.maxTxPacket = size + + return nil + } +} + +type rxPacket struct { + pktType fxp + pktBytes []byte +} + +// Up to N parallel servers +func (svr *Server) sftpServerWorker(pktChan chan orderedRequest) error { + for pkt := range pktChan { + // readonly checks + readonly := true + switch pkt := pkt.requestPacket.(type) { + case notReadOnly: + readonly = false + case *sshFxpOpenPacket: + readonly = pkt.readonly() + case *sshFxpExtendedPacket: + readonly = pkt.readonly() + } + + // If server is operating read-only and a write operation is requested, + // return permission denied + if !readonly && svr.readOnly { + svr.pktMgr.readyPacket( + svr.pktMgr.newOrderedResponse(statusFromError(pkt.id(), syscall.EPERM), pkt.orderID()), + ) + continue + } + + if err := handlePacket(svr, pkt); err != nil { + return err + } + } + return nil +} + +func handlePacket(s *Server, p orderedRequest) error { + var rpkt responsePacket + orderID := p.orderID() + switch p := p.requestPacket.(type) { + case *sshFxInitPacket: + rpkt = &sshFxVersionPacket{ + Version: sftpProtocolVersion, + Extensions: sftpExtensions, + } + case *sshFxpStatPacket: + // stat the requested file + info, err := os.Stat(s.toLocalPath(p.Path)) + rpkt = &sshFxpStatResponse{ + ID: p.ID, + info: info, + } + if err != nil { + rpkt = statusFromError(p.ID, err) + } + case *sshFxpLstatPacket: + // stat the requested file + info, err := s.lstat(s.toLocalPath(p.Path)) + rpkt = &sshFxpStatResponse{ + ID: p.ID, + info: info, + } + if err != nil { + rpkt = statusFromError(p.ID, err) + } + case *sshFxpFstatPacket: + f, ok := s.getHandle(p.Handle) + var err error = EBADF + var info os.FileInfo + if ok { + info, err = f.Stat() + rpkt = &sshFxpStatResponse{ + ID: p.ID, + info: info, + } + } + if err != nil { + rpkt = statusFromError(p.ID, err) + } + case *sshFxpMkdirPacket: + // TODO FIXME: ignore flags field + err := os.Mkdir(s.toLocalPath(p.Path), 0o755) + rpkt = statusFromError(p.ID, err) + case *sshFxpRmdirPacket: + err := os.Remove(s.toLocalPath(p.Path)) + rpkt = statusFromError(p.ID, err) + case *sshFxpRemovePacket: + err := os.Remove(s.toLocalPath(p.Filename)) + rpkt = statusFromError(p.ID, err) + case *sshFxpRenamePacket: + err := os.Rename(s.toLocalPath(p.Oldpath), s.toLocalPath(p.Newpath)) + rpkt = statusFromError(p.ID, err) + case *sshFxpSymlinkPacket: + err := os.Symlink(s.toLocalPath(p.Targetpath), s.toLocalPath(p.Linkpath)) + rpkt = statusFromError(p.ID, err) + case *sshFxpClosePacket: + rpkt = statusFromError(p.ID, s.closeHandle(p.Handle)) + case *sshFxpReadlinkPacket: + f, err := os.Readlink(s.toLocalPath(p.Path)) + rpkt = &sshFxpNamePacket{ + ID: p.ID, + NameAttrs: []*sshFxpNameAttr{ + { + Name: f, + LongName: f, + Attrs: emptyFileStat, + }, + }, + } + if err != nil { + rpkt = statusFromError(p.ID, err) + } + case *sshFxpRealpathPacket: + f, err := filepath.Abs(s.toLocalPath(p.Path)) + f = cleanPath(f) + rpkt = &sshFxpNamePacket{ + ID: p.ID, + NameAttrs: []*sshFxpNameAttr{ + { + Name: f, + LongName: f, + Attrs: emptyFileStat, + }, + }, + } + if err != nil { + rpkt = statusFromError(p.ID, err) + } + case *sshFxpOpendirPacket: + lp := s.toLocalPath(p.Path) + + if stat, err := s.stat(lp); err != nil { + rpkt = statusFromError(p.ID, err) + } else if !stat.IsDir() { + rpkt = statusFromError(p.ID, &os.PathError{ + Path: lp, Err: syscall.ENOTDIR, + }) + } else { + rpkt = (&sshFxpOpenPacket{ + ID: p.ID, + Path: p.Path, + Pflags: sshFxfRead, + }).respond(s) + } + case *sshFxpReadPacket: + var err error = EBADF + f, ok := s.getHandle(p.Handle) + if ok { + err = nil + data := p.getDataSlice(s.pktMgr.alloc, orderID, s.maxTxPacket) + n, _err := f.ReadAt(data, int64(p.Offset)) + if _err != nil && (_err != io.EOF || n == 0) { + err = _err + } + rpkt = &sshFxpDataPacket{ + ID: p.ID, + Length: uint32(n), + Data: data[:n], + // do not use data[:n:n] here to clamp the capacity, we allocated extra capacity above to avoid reallocations + } + } + if err != nil { + rpkt = statusFromError(p.ID, err) + } + + case *sshFxpWritePacket: + f, ok := s.getHandle(p.Handle) + var err error = EBADF + if ok { + _, err = f.WriteAt(p.Data, int64(p.Offset)) + } + rpkt = statusFromError(p.ID, err) + case *sshFxpExtendedPacket: + if p.SpecificPacket == nil { + rpkt = statusFromError(p.ID, ErrSSHFxOpUnsupported) + } else { + rpkt = p.respond(s) + } + case serverRespondablePacket: + rpkt = p.respond(s) + default: + return fmt.Errorf("unexpected packet type %T", p) + } + + s.pktMgr.readyPacket(s.pktMgr.newOrderedResponse(rpkt, orderID)) + return nil +} + +// Serve serves SFTP connections until the streams stop or the SFTP subsystem +// is stopped. It returns nil if the server exits cleanly. +func (svr *Server) Serve() error { + defer func() { + if svr.pktMgr.alloc != nil { + svr.pktMgr.alloc.Free() + } + }() + var wg sync.WaitGroup + runWorker := func(ch chan orderedRequest) { + wg.Add(1) + go func() { + defer wg.Done() + if err := svr.sftpServerWorker(ch); err != nil { + svr.conn.Close() // shuts down recvPacket + } + }() + } + pktChan := svr.pktMgr.workerChan(runWorker) + + var err error + var pkt requestPacket + var pktType fxp + var pktBytes []byte + for { + pktType, pktBytes, err = svr.serverConn.recvPacket(svr.pktMgr.getNextOrderID()) + if err != nil { + // Check whether the connection terminated cleanly in-between packets. + if err == io.EOF { + err = nil + } + // we don't care about releasing allocated pages here, the server will quit and the allocator freed + break + } + + pkt, err = makePacket(rxPacket{pktType, pktBytes}) + if err != nil { + switch { + case errors.Is(err, errUnknownExtendedPacket): + //if err := svr.serverConn.sendError(pkt, ErrSshFxOpUnsupported); err != nil { + // debug("failed to send err packet: %v", err) + // svr.conn.Close() // shuts down recvPacket + // break + //} + default: + debug("makePacket err: %v", err) + svr.conn.Close() // shuts down recvPacket + break + } + } + + pktChan <- svr.pktMgr.newOrderedRequest(pkt) + } + + close(pktChan) // shuts down sftpServerWorkers + wg.Wait() // wait for all workers to exit + + // close any still-open files + for handle, file := range svr.openFiles { + fmt.Fprintf(svr.debugStream, "sftp server file with handle %q left open: %v\n", handle, file.Name()) + file.Close() + } + return err // error from recvPacket +} + +type ider interface { + id() uint32 +} + +// The init packet has no ID, so we just return a zero-value ID +func (p *sshFxInitPacket) id() uint32 { return 0 } + +type sshFxpStatResponse struct { + ID uint32 + info os.FileInfo +} + +func (p *sshFxpStatResponse) marshalPacket() ([]byte, []byte, error) { + l := 4 + 1 + 4 // uint32(length) + byte(type) + uint32(id) + + b := make([]byte, 4, l) + b = append(b, sshFxpAttrs) + b = marshalUint32(b, p.ID) + + var payload []byte + payload = marshalFileInfo(payload, p.info) + + return b, payload, nil +} + +func (p *sshFxpStatResponse) MarshalBinary() ([]byte, error) { + header, payload, err := p.marshalPacket() + return append(header, payload...), err +} + +var emptyFileStat = []interface{}{uint32(0)} + +func (p *sshFxpOpenPacket) readonly() bool { + return !p.hasPflags(sshFxfWrite) +} + +func (p *sshFxpOpenPacket) hasPflags(flags ...uint32) bool { + for _, f := range flags { + if p.Pflags&f == 0 { + return false + } + } + return true +} + +func (p *sshFxpOpenPacket) respond(svr *Server) responsePacket { + var osFlags int + if p.hasPflags(sshFxfRead, sshFxfWrite) { + osFlags |= os.O_RDWR + } else if p.hasPflags(sshFxfWrite) { + osFlags |= os.O_WRONLY + } else if p.hasPflags(sshFxfRead) { + osFlags |= os.O_RDONLY + } else { + // how are they opening? + return statusFromError(p.ID, syscall.EINVAL) + } + + // Don't use O_APPEND flag as it conflicts with WriteAt. + // The sshFxfAppend flag is a no-op here as the client sends the offsets. + + if p.hasPflags(sshFxfCreat) { + osFlags |= os.O_CREATE + } + if p.hasPflags(sshFxfTrunc) { + osFlags |= os.O_TRUNC + } + if p.hasPflags(sshFxfExcl) { + osFlags |= os.O_EXCL + } + + mode := os.FileMode(0o644) + // Like OpenSSH, we only handle permissions here, and only when the file is being created. + // Otherwise, the permissions are ignored. + if p.Flags&sshFileXferAttrPermissions != 0 { + fs, err := p.unmarshalFileStat(p.Flags) + if err != nil { + return statusFromError(p.ID, err) + } + mode = fs.FileMode() & os.ModePerm + } + + f, err := svr.openfile(svr.toLocalPath(p.Path), osFlags, mode) + if err != nil { + return statusFromError(p.ID, err) + } + + handle := svr.nextHandle(f) + return &sshFxpHandlePacket{ID: p.ID, Handle: handle} +} + +func (p *sshFxpReaddirPacket) respond(svr *Server) responsePacket { + f, ok := svr.getHandle(p.Handle) + if !ok { + return statusFromError(p.ID, EBADF) + } + + dirents, err := f.Readdir(128) + if err != nil { + return statusFromError(p.ID, err) + } + + idLookup := osIDLookup{} + + ret := &sshFxpNamePacket{ID: p.ID} + for _, dirent := range dirents { + ret.NameAttrs = append(ret.NameAttrs, &sshFxpNameAttr{ + Name: dirent.Name(), + LongName: runLs(idLookup, dirent), + Attrs: []interface{}{dirent}, + }) + } + return ret +} + +func (p *sshFxpSetstatPacket) respond(svr *Server) responsePacket { + path := svr.toLocalPath(p.Path) + + debug("setstat name %q", path) + + fs, err := p.unmarshalFileStat(p.Flags) + + if err == nil && (p.Flags&sshFileXferAttrSize) != 0 { + err = os.Truncate(path, int64(fs.Size)) + } + if err == nil && (p.Flags&sshFileXferAttrPermissions) != 0 { + err = os.Chmod(path, fs.FileMode()) + } + if err == nil && (p.Flags&sshFileXferAttrUIDGID) != 0 { + err = os.Chown(path, int(fs.UID), int(fs.GID)) + } + if err == nil && (p.Flags&sshFileXferAttrACmodTime) != 0 { + err = os.Chtimes(path, fs.AccessTime(), fs.ModTime()) + } + + return statusFromError(p.ID, err) +} + +func (p *sshFxpFsetstatPacket) respond(svr *Server) responsePacket { + f, ok := svr.getHandle(p.Handle) + if !ok { + return statusFromError(p.ID, EBADF) + } + + path := f.Name() + + debug("fsetstat name %q", path) + + fs, err := p.unmarshalFileStat(p.Flags) + + if err == nil && (p.Flags&sshFileXferAttrSize) != 0 { + err = f.Truncate(int64(fs.Size)) + } + if err == nil && (p.Flags&sshFileXferAttrPermissions) != 0 { + err = f.Chmod(fs.FileMode()) + } + if err == nil && (p.Flags&sshFileXferAttrUIDGID) != 0 { + err = f.Chown(int(fs.UID), int(fs.GID)) + } + if err == nil && (p.Flags&sshFileXferAttrACmodTime) != 0 { + type chtimer interface { + Chtimes(atime, mtime time.Time) error + } + + switch f := interface{}(f).(type) { + case chtimer: + // future-compatible, for when/if *os.File supports Chtimes. + err = f.Chtimes(fs.AccessTime(), fs.ModTime()) + default: + err = os.Chtimes(path, fs.AccessTime(), fs.ModTime()) + } + } + + return statusFromError(p.ID, err) +} + +func statusFromError(id uint32, err error) *sshFxpStatusPacket { + ret := &sshFxpStatusPacket{ + ID: id, + StatusError: StatusError{ + // sshFXOk = 0 + // sshFXEOF = 1 + // sshFXNoSuchFile = 2 ENOENT + // sshFXPermissionDenied = 3 + // sshFXFailure = 4 + // sshFXBadMessage = 5 + // sshFXNoConnection = 6 + // sshFXConnectionLost = 7 + // sshFXOPUnsupported = 8 + Code: sshFxOk, + }, + } + if err == nil { + return ret + } + + debug("statusFromError: error is %T %#v", err, err) + ret.StatusError.Code = sshFxFailure + ret.StatusError.msg = err.Error() + + if os.IsNotExist(err) { + ret.StatusError.Code = sshFxNoSuchFile + return ret + } + if code, ok := translateSyscallError(err); ok { + ret.StatusError.Code = code + return ret + } + + if errors.Is(err, io.EOF) { + ret.StatusError.Code = sshFxEOF + return ret + } + + var e fxerr + if errors.As(err, &e) { + ret.StatusError.Code = uint32(e) + return ret + } + + return ret +} diff --git a/vendor/github.com/pkg/sftp/server_plan9.go b/vendor/github.com/pkg/sftp/server_plan9.go new file mode 100644 index 00000000000..4e8ed067895 --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_plan9.go @@ -0,0 +1,27 @@ +package sftp + +import ( + "path" + "path/filepath" +) + +func (s *Server) toLocalPath(p string) string { + if s.workDir != "" && !path.IsAbs(p) { + p = path.Join(s.workDir, p) + } + + lp := filepath.FromSlash(p) + + if path.IsAbs(p) { + tmp := lp[1:] + + if filepath.IsAbs(tmp) { + // If the FromSlash without any starting slashes is absolute, + // then we have a filepath encoded with a prefix '/'. + // e.g. "/#s/boot" to "#s/boot" + return tmp + } + } + + return lp +} diff --git a/vendor/github.com/pkg/sftp/server_posix.go b/vendor/github.com/pkg/sftp/server_posix.go new file mode 100644 index 00000000000..c07d70a043e --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_posix.go @@ -0,0 +1,21 @@ +//go:build !windows +// +build !windows + +package sftp + +import ( + "io/fs" + "os" +) + +func (s *Server) openfile(path string, flag int, mode fs.FileMode) (file, error) { + return os.OpenFile(path, flag, mode) +} + +func (s *Server) lstat(name string) (os.FileInfo, error) { + return os.Lstat(name) +} + +func (s *Server) stat(name string) (os.FileInfo, error) { + return os.Stat(name) +} diff --git a/vendor/github.com/pkg/sftp/server_statvfs_darwin.go b/vendor/github.com/pkg/sftp/server_statvfs_darwin.go new file mode 100644 index 00000000000..8c01dac52d0 --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_statvfs_darwin.go @@ -0,0 +1,21 @@ +package sftp + +import ( + "syscall" +) + +func statvfsFromStatfst(stat *syscall.Statfs_t) (*StatVFS, error) { + return &StatVFS{ + Bsize: uint64(stat.Bsize), + Frsize: uint64(stat.Bsize), // fragment size is a linux thing; use block size here + Blocks: stat.Blocks, + Bfree: stat.Bfree, + Bavail: stat.Bavail, + Files: stat.Files, + Ffree: stat.Ffree, + Favail: stat.Ffree, // not sure how to calculate Favail + Fsid: uint64(uint64(stat.Fsid.Val[1])<<32 | uint64(stat.Fsid.Val[0])), // endianness? + Flag: uint64(stat.Flags), // assuming POSIX? + Namemax: 1024, // man 2 statfs shows: #define MAXPATHLEN 1024 + }, nil +} diff --git a/vendor/github.com/pkg/sftp/server_statvfs_impl.go b/vendor/github.com/pkg/sftp/server_statvfs_impl.go new file mode 100644 index 00000000000..a5470798cf3 --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_statvfs_impl.go @@ -0,0 +1,30 @@ +//go:build darwin || linux +// +build darwin linux + +// fill in statvfs structure with OS specific values +// Statfs_t is different per-kernel, and only exists on some unixes (not Solaris for instance) + +package sftp + +import ( + "syscall" +) + +func (p *sshFxpExtendedPacketStatVFS) respond(svr *Server) responsePacket { + retPkt, err := getStatVFSForPath(p.Path) + if err != nil { + return statusFromError(p.ID, err) + } + retPkt.ID = p.ID + + return retPkt +} + +func getStatVFSForPath(name string) (*StatVFS, error) { + var stat syscall.Statfs_t + if err := syscall.Statfs(name, &stat); err != nil { + return nil, err + } + + return statvfsFromStatfst(&stat) +} diff --git a/vendor/github.com/pkg/sftp/server_statvfs_linux.go b/vendor/github.com/pkg/sftp/server_statvfs_linux.go new file mode 100644 index 00000000000..615c4157a50 --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_statvfs_linux.go @@ -0,0 +1,23 @@ +//go:build linux +// +build linux + +package sftp + +import ( + "syscall" +) + +func statvfsFromStatfst(stat *syscall.Statfs_t) (*StatVFS, error) { + return &StatVFS{ + Bsize: uint64(stat.Bsize), + Frsize: uint64(stat.Frsize), + Blocks: stat.Blocks, + Bfree: stat.Bfree, + Bavail: stat.Bavail, + Files: stat.Files, + Ffree: stat.Ffree, + Favail: stat.Ffree, // not sure how to calculate Favail + Flag: uint64(stat.Flags), // assuming POSIX? + Namemax: uint64(stat.Namelen), + }, nil +} diff --git a/vendor/github.com/pkg/sftp/server_statvfs_plan9.go b/vendor/github.com/pkg/sftp/server_statvfs_plan9.go new file mode 100644 index 00000000000..e71a27d37b7 --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_statvfs_plan9.go @@ -0,0 +1,13 @@ +package sftp + +import ( + "syscall" +) + +func (p *sshFxpExtendedPacketStatVFS) respond(svr *Server) responsePacket { + return statusFromError(p.ID, syscall.EPLAN9) +} + +func getStatVFSForPath(name string) (*StatVFS, error) { + return nil, syscall.EPLAN9 +} diff --git a/vendor/github.com/pkg/sftp/server_statvfs_stubs.go b/vendor/github.com/pkg/sftp/server_statvfs_stubs.go new file mode 100644 index 00000000000..dd4705bb4f2 --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_statvfs_stubs.go @@ -0,0 +1,16 @@ +//go:build !darwin && !linux && !plan9 +// +build !darwin,!linux,!plan9 + +package sftp + +import ( + "syscall" +) + +func (p *sshFxpExtendedPacketStatVFS) respond(svr *Server) responsePacket { + return statusFromError(p.ID, syscall.ENOTSUP) +} + +func getStatVFSForPath(name string) (*StatVFS, error) { + return nil, syscall.ENOTSUP +} diff --git a/vendor/github.com/pkg/sftp/server_unix.go b/vendor/github.com/pkg/sftp/server_unix.go new file mode 100644 index 00000000000..495b397c06b --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_unix.go @@ -0,0 +1,16 @@ +//go:build !windows && !plan9 +// +build !windows,!plan9 + +package sftp + +import ( + "path" +) + +func (s *Server) toLocalPath(p string) string { + if s.workDir != "" && !path.IsAbs(p) { + p = path.Join(s.workDir, p) + } + + return p +} diff --git a/vendor/github.com/pkg/sftp/server_windows.go b/vendor/github.com/pkg/sftp/server_windows.go new file mode 100644 index 00000000000..e940dba101c --- /dev/null +++ b/vendor/github.com/pkg/sftp/server_windows.go @@ -0,0 +1,193 @@ +package sftp + +import ( + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "time" + + "golang.org/x/sys/windows" +) + +func (s *Server) toLocalPath(p string) string { + if s.workDir != "" && !path.IsAbs(p) { + p = path.Join(s.workDir, p) + } + + lp := filepath.FromSlash(p) + + if path.IsAbs(p) { // starts with '/' + if len(p) == 1 && s.winRoot { + return `\\.\` // for openfile + } + + tmp := lp + for len(tmp) > 0 && tmp[0] == '\\' { + tmp = tmp[1:] + } + + if filepath.IsAbs(tmp) { + // If the FromSlash without any starting slashes is absolute, + // then we have a filepath encoded with a prefix '/'. + // e.g. "/C:/Windows" to "C:\\Windows" + return tmp + } + + tmp += "\\" + + if filepath.IsAbs(tmp) { + // If the FromSlash without any starting slashes but with extra end slash is absolute, + // then we have a filepath encoded with a prefix '/' and a dropped '/' at the end. + // e.g. "/C:" to "C:\\" + return tmp + } + + if s.winRoot { + // Make it so that "/Windows" is not found, and "/c:/Windows" has to be used + return `\\.\` + tmp + } + } + + return lp +} + +func bitsToDrives(bitmap uint32) []string { + var drive rune = 'a' + var drives []string + + for bitmap != 0 && drive <= 'z' { + if bitmap&1 == 1 { + drives = append(drives, string(drive)+":") + } + drive++ + bitmap >>= 1 + } + + return drives +} + +func getDrives() ([]string, error) { + mask, err := windows.GetLogicalDrives() + if err != nil { + return nil, fmt.Errorf("GetLogicalDrives: %w", err) + } + return bitsToDrives(mask), nil +} + +type driveInfo struct { + fs.FileInfo + name string +} + +func (i *driveInfo) Name() string { + return i.name // since the Name() returned from a os.Stat("C:\\") is "\\" +} + +type winRoot struct { + drives []string +} + +func newWinRoot() (*winRoot, error) { + drives, err := getDrives() + if err != nil { + return nil, err + } + return &winRoot{ + drives: drives, + }, nil +} + +func (f *winRoot) Readdir(n int) ([]os.FileInfo, error) { + drives := f.drives + if n > 0 && len(drives) > n { + drives = drives[:n] + } + f.drives = f.drives[len(drives):] + if len(drives) == 0 { + return nil, io.EOF + } + + var infos []os.FileInfo + for _, drive := range drives { + fi, err := os.Stat(drive + `\`) + if err != nil { + return nil, err + } + + di := &driveInfo{ + FileInfo: fi, + name: drive, + } + infos = append(infos, di) + } + + return infos, nil +} + +func (f *winRoot) Stat() (os.FileInfo, error) { + return rootFileInfo, nil +} +func (f *winRoot) ReadAt(b []byte, off int64) (int, error) { + return 0, os.ErrPermission +} +func (f *winRoot) WriteAt(b []byte, off int64) (int, error) { + return 0, os.ErrPermission +} +func (f *winRoot) Name() string { + return "/" +} +func (f *winRoot) Truncate(int64) error { + return os.ErrPermission +} +func (f *winRoot) Chmod(mode fs.FileMode) error { + return os.ErrPermission +} +func (f *winRoot) Chown(uid, gid int) error { + return os.ErrPermission +} +func (f *winRoot) Close() error { + f.drives = nil + return nil +} + +func (s *Server) openfile(path string, flag int, mode fs.FileMode) (file, error) { + if path == `\\.\` && s.winRoot { + return newWinRoot() + } + return os.OpenFile(path, flag, mode) +} + +type winRootFileInfo struct { + name string + modTime time.Time +} + +func (w *winRootFileInfo) Name() string { return w.name } +func (w *winRootFileInfo) Size() int64 { return 0 } +func (w *winRootFileInfo) Mode() fs.FileMode { return fs.ModeDir | 0555 } // read+execute for all +func (w *winRootFileInfo) ModTime() time.Time { return w.modTime } +func (w *winRootFileInfo) IsDir() bool { return true } +func (w *winRootFileInfo) Sys() interface{} { return nil } + +// Create a new root FileInfo +var rootFileInfo = &winRootFileInfo{ + name: "/", + modTime: time.Now(), +} + +func (s *Server) lstat(name string) (os.FileInfo, error) { + if name == `\\.\` && s.winRoot { + return rootFileInfo, nil + } + return os.Lstat(name) +} + +func (s *Server) stat(name string) (os.FileInfo, error) { + if name == `\\.\` && s.winRoot { + return rootFileInfo, nil + } + return os.Stat(name) +} diff --git a/vendor/github.com/pkg/sftp/sftp.go b/vendor/github.com/pkg/sftp/sftp.go new file mode 100644 index 00000000000..1e698bb2782 --- /dev/null +++ b/vendor/github.com/pkg/sftp/sftp.go @@ -0,0 +1,258 @@ +// Package sftp implements the SSH File Transfer Protocol as described in +// https://filezilla-project.org/specs/draft-ietf-secsh-filexfer-02.txt +package sftp + +import ( + "fmt" +) + +const ( + sshFxpInit = 1 + sshFxpVersion = 2 + sshFxpOpen = 3 + sshFxpClose = 4 + sshFxpRead = 5 + sshFxpWrite = 6 + sshFxpLstat = 7 + sshFxpFstat = 8 + sshFxpSetstat = 9 + sshFxpFsetstat = 10 + sshFxpOpendir = 11 + sshFxpReaddir = 12 + sshFxpRemove = 13 + sshFxpMkdir = 14 + sshFxpRmdir = 15 + sshFxpRealpath = 16 + sshFxpStat = 17 + sshFxpRename = 18 + sshFxpReadlink = 19 + sshFxpSymlink = 20 + sshFxpStatus = 101 + sshFxpHandle = 102 + sshFxpData = 103 + sshFxpName = 104 + sshFxpAttrs = 105 + sshFxpExtended = 200 + sshFxpExtendedReply = 201 +) + +const ( + sshFxOk = 0 + sshFxEOF = 1 + sshFxNoSuchFile = 2 + sshFxPermissionDenied = 3 + sshFxFailure = 4 + sshFxBadMessage = 5 + sshFxNoConnection = 6 + sshFxConnectionLost = 7 + sshFxOPUnsupported = 8 + + // see draft-ietf-secsh-filexfer-13 + // https://tools.ietf.org/html/draft-ietf-secsh-filexfer-13#section-9.1 + sshFxInvalidHandle = 9 + sshFxNoSuchPath = 10 + sshFxFileAlreadyExists = 11 + sshFxWriteProtect = 12 + sshFxNoMedia = 13 + sshFxNoSpaceOnFilesystem = 14 + sshFxQuotaExceeded = 15 + sshFxUnknownPrincipal = 16 + sshFxLockConflict = 17 + sshFxDirNotEmpty = 18 + sshFxNotADirectory = 19 + sshFxInvalidFilename = 20 + sshFxLinkLoop = 21 + sshFxCannotDelete = 22 + sshFxInvalidParameter = 23 + sshFxFileIsADirectory = 24 + sshFxByteRangeLockConflict = 25 + sshFxByteRangeLockRefused = 26 + sshFxDeletePending = 27 + sshFxFileCorrupt = 28 + sshFxOwnerInvalid = 29 + sshFxGroupInvalid = 30 + sshFxNoMatchingByteRangeLock = 31 +) + +const ( + sshFxfRead = 0x00000001 + sshFxfWrite = 0x00000002 + sshFxfAppend = 0x00000004 + sshFxfCreat = 0x00000008 + sshFxfTrunc = 0x00000010 + sshFxfExcl = 0x00000020 +) + +var ( + // supportedSFTPExtensions defines the supported extensions + supportedSFTPExtensions = []sshExtensionPair{ + {"hardlink@openssh.com", "1"}, + {"posix-rename@openssh.com", "1"}, + {"statvfs@openssh.com", "2"}, + } + sftpExtensions = supportedSFTPExtensions +) + +type fxp uint8 + +func (f fxp) String() string { + switch f { + case sshFxpInit: + return "SSH_FXP_INIT" + case sshFxpVersion: + return "SSH_FXP_VERSION" + case sshFxpOpen: + return "SSH_FXP_OPEN" + case sshFxpClose: + return "SSH_FXP_CLOSE" + case sshFxpRead: + return "SSH_FXP_READ" + case sshFxpWrite: + return "SSH_FXP_WRITE" + case sshFxpLstat: + return "SSH_FXP_LSTAT" + case sshFxpFstat: + return "SSH_FXP_FSTAT" + case sshFxpSetstat: + return "SSH_FXP_SETSTAT" + case sshFxpFsetstat: + return "SSH_FXP_FSETSTAT" + case sshFxpOpendir: + return "SSH_FXP_OPENDIR" + case sshFxpReaddir: + return "SSH_FXP_READDIR" + case sshFxpRemove: + return "SSH_FXP_REMOVE" + case sshFxpMkdir: + return "SSH_FXP_MKDIR" + case sshFxpRmdir: + return "SSH_FXP_RMDIR" + case sshFxpRealpath: + return "SSH_FXP_REALPATH" + case sshFxpStat: + return "SSH_FXP_STAT" + case sshFxpRename: + return "SSH_FXP_RENAME" + case sshFxpReadlink: + return "SSH_FXP_READLINK" + case sshFxpSymlink: + return "SSH_FXP_SYMLINK" + case sshFxpStatus: + return "SSH_FXP_STATUS" + case sshFxpHandle: + return "SSH_FXP_HANDLE" + case sshFxpData: + return "SSH_FXP_DATA" + case sshFxpName: + return "SSH_FXP_NAME" + case sshFxpAttrs: + return "SSH_FXP_ATTRS" + case sshFxpExtended: + return "SSH_FXP_EXTENDED" + case sshFxpExtendedReply: + return "SSH_FXP_EXTENDED_REPLY" + default: + return "unknown" + } +} + +type fx uint8 + +func (f fx) String() string { + switch f { + case sshFxOk: + return "SSH_FX_OK" + case sshFxEOF: + return "SSH_FX_EOF" + case sshFxNoSuchFile: + return "SSH_FX_NO_SUCH_FILE" + case sshFxPermissionDenied: + return "SSH_FX_PERMISSION_DENIED" + case sshFxFailure: + return "SSH_FX_FAILURE" + case sshFxBadMessage: + return "SSH_FX_BAD_MESSAGE" + case sshFxNoConnection: + return "SSH_FX_NO_CONNECTION" + case sshFxConnectionLost: + return "SSH_FX_CONNECTION_LOST" + case sshFxOPUnsupported: + return "SSH_FX_OP_UNSUPPORTED" + default: + return "unknown" + } +} + +type unexpectedPacketErr struct { + want, got fxp +} + +func (u *unexpectedPacketErr) Error() string { + return fmt.Sprintf("sftp: unexpected packet: want %v, got %v", u.want, u.got) +} + +func unimplementedPacketErr(u fxp) error { + return fmt.Errorf("sftp: unimplemented packet type: got %v", u) +} + +type unexpectedIDErr struct{ want, got uint32 } + +func (u *unexpectedIDErr) Error() string { + return fmt.Sprintf("sftp: unexpected id: want %d, got %d", u.want, u.got) +} + +func unimplementedSeekWhence(whence int) error { + return fmt.Errorf("sftp: unimplemented seek whence %d", whence) +} + +func unexpectedCount(want, got uint32) error { + return fmt.Errorf("sftp: unexpected count: want %d, got %d", want, got) +} + +type unexpectedVersionErr struct{ want, got uint32 } + +func (u *unexpectedVersionErr) Error() string { + return fmt.Sprintf("sftp: unexpected server version: want %v, got %v", u.want, u.got) +} + +// A StatusError is returned when an SFTP operation fails, and provides +// additional information about the failure. +type StatusError struct { + Code uint32 + msg, lang string +} + +func (s *StatusError) Error() string { + return fmt.Sprintf("sftp: %q (%v)", s.msg, fx(s.Code)) +} + +// FxCode returns the error code typed to match against the exported codes +func (s *StatusError) FxCode() fxerr { + return fxerr(s.Code) +} + +func getSupportedExtensionByName(extensionName string) (sshExtensionPair, error) { + for _, supportedExtension := range supportedSFTPExtensions { + if supportedExtension.Name == extensionName { + return supportedExtension, nil + } + } + return sshExtensionPair{}, fmt.Errorf("unsupported extension: %s", extensionName) +} + +// SetSFTPExtensions allows to customize the supported server extensions. +// See the variable supportedSFTPExtensions for supported extensions. +// This method accepts a slice of sshExtensionPair names for example 'hardlink@openssh.com'. +// If an invalid extension is given an error will be returned and nothing will be changed +func SetSFTPExtensions(extensions ...string) error { + tempExtensions := []sshExtensionPair{} + for _, extension := range extensions { + sftpExtension, err := getSupportedExtensionByName(extension) + if err != nil { + return err + } + tempExtensions = append(tempExtensions, sftpExtension) + } + sftpExtensions = tempExtensions + return nil +} diff --git a/vendor/github.com/pkg/sftp/stat.go b/vendor/github.com/pkg/sftp/stat.go new file mode 100644 index 00000000000..2bb2c137039 --- /dev/null +++ b/vendor/github.com/pkg/sftp/stat.go @@ -0,0 +1,94 @@ +package sftp + +import ( + "os" + + sshfx "github.com/pkg/sftp/internal/encoding/ssh/filexfer" +) + +// isRegular returns true if the mode describes a regular file. +func isRegular(mode uint32) bool { + return sshfx.FileMode(mode)&sshfx.ModeType == sshfx.ModeRegular +} + +// toFileMode converts sftp filemode bits to the os.FileMode specification +func toFileMode(mode uint32) os.FileMode { + var fm = os.FileMode(mode & 0777) + + switch sshfx.FileMode(mode) & sshfx.ModeType { + case sshfx.ModeDevice: + fm |= os.ModeDevice + case sshfx.ModeCharDevice: + fm |= os.ModeDevice | os.ModeCharDevice + case sshfx.ModeDir: + fm |= os.ModeDir + case sshfx.ModeNamedPipe: + fm |= os.ModeNamedPipe + case sshfx.ModeSymlink: + fm |= os.ModeSymlink + case sshfx.ModeRegular: + // nothing to do + case sshfx.ModeSocket: + fm |= os.ModeSocket + } + + if sshfx.FileMode(mode)&sshfx.ModeSetUID != 0 { + fm |= os.ModeSetuid + } + if sshfx.FileMode(mode)&sshfx.ModeSetGID != 0 { + fm |= os.ModeSetgid + } + if sshfx.FileMode(mode)&sshfx.ModeSticky != 0 { + fm |= os.ModeSticky + } + + return fm +} + +// fromFileMode converts from the os.FileMode specification to sftp filemode bits +func fromFileMode(mode os.FileMode) uint32 { + ret := sshfx.FileMode(mode & os.ModePerm) + + switch mode & os.ModeType { + case os.ModeDevice | os.ModeCharDevice: + ret |= sshfx.ModeCharDevice + case os.ModeDevice: + ret |= sshfx.ModeDevice + case os.ModeDir: + ret |= sshfx.ModeDir + case os.ModeNamedPipe: + ret |= sshfx.ModeNamedPipe + case os.ModeSymlink: + ret |= sshfx.ModeSymlink + case 0: + ret |= sshfx.ModeRegular + case os.ModeSocket: + ret |= sshfx.ModeSocket + } + + if mode&os.ModeSetuid != 0 { + ret |= sshfx.ModeSetUID + } + if mode&os.ModeSetgid != 0 { + ret |= sshfx.ModeSetGID + } + if mode&os.ModeSticky != 0 { + ret |= sshfx.ModeSticky + } + + return uint32(ret) +} + +const ( + s_ISUID = uint32(sshfx.ModeSetUID) + s_ISGID = uint32(sshfx.ModeSetGID) + s_ISVTX = uint32(sshfx.ModeSticky) +) + +// S_IFMT is a legacy export, and was brought in to support GOOS environments whose sysconfig.S_IFMT may be different from the value used internally by SFTP standards. +// There should be no reason why you need to import it, or use it, but unexporting it could cause code to break in a way that cannot be readily fixed. +// As such, we continue to export this value as the value used in the SFTP standard. +// +// Deprecated: Remove use of this value, and avoid any future use as well. +// There is no alternative provided, you should never need to access this value. +const S_IFMT = uint32(sshfx.ModeType) diff --git a/vendor/modules.txt b/vendor/modules.txt index 55976da6658..6b53788c291 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -196,6 +196,9 @@ github.com/klauspost/compress/zstd/internal/xxhash # github.com/klauspost/pgzip v1.2.6 ## explicit github.com/klauspost/pgzip +# github.com/kr/fs v0.1.0 +## explicit +github.com/kr/fs # github.com/manifoldco/promptui v0.9.0 ## explicit; go 1.12 github.com/manifoldco/promptui @@ -334,6 +337,11 @@ github.com/openshift/imagebuilder/strslice # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors +# github.com/pkg/sftp v1.13.10 +## explicit; go 1.23.0 +github.com/pkg/sftp +github.com/pkg/sftp/internal/encoding/ssh/filexfer +github.com/pkg/sftp/internal/encoding/ssh/filexfer/openssh # github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 ## explicit; go 1.20 github.com/planetscale/vtprotobuf/protohelpers