diff --git a/Cargo.lock b/Cargo.lock index 7977421..bf1e9df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,24 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "deno_error" version = "0.5.2" @@ -30,6 +48,8 @@ dependencies = [ "deno_error", "percent-encoding", "pretty_assertions", + "sys_traits", + "tempfile", "thiserror", "url", ] @@ -51,6 +71,22 @@ dependencies = [ "syn", ] +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -60,6 +96,17 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "icu_collections" version = "1.5.0" @@ -201,9 +248,15 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.164" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "linux-raw-sys" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "litemap" @@ -211,6 +264,45 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -245,20 +337,48 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_syscall" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustix" +version = "0.38.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", @@ -279,9 +399,9 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" [[package]] name = "syn" -version = "2.0.89" +version = "2.0.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" dependencies = [ "proc-macro2", "quote", @@ -299,20 +419,43 @@ dependencies = [ "syn", ] +[[package]] +name = "sys_traits" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72001c0d1b690f17bb18ea7960821389199fd59ce6784f883fc76d0f3fbb6236" +dependencies = [ + "getrandom", + "parking_lot", +] + +[[package]] +name = "tempfile" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys", +] + [[package]] name = "thiserror" -version = "2.0.3" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c006c85c7651b3cf2ada4584faa36773bd07bac24acfb39f3c431b36d7e667aa" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.3" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f077553d607adc1caf65430528a576c757a71ed73944b66ebb58ef2bbd243568" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", @@ -358,6 +501,85 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "write16" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 85e1b33..086f3c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,8 +12,17 @@ repository = "https://github.com/denoland/deno_path_util" [dependencies] percent-encoding = "2.3.0" thiserror = "2" +sys_traits.workspace = true deno_error = "0.5.2" -url = { version = "2.5.1" } +url = "2.5.1" [dev-dependencies] pretty_assertions = "1.4.0" +sys_traits = { workspace = true, features = ["getrandom", "memory", "real"] } +tempfile = "3.4.0" + +[workspace] +members = ["."] + +[workspace.dependencies] +sys_traits = "0.1.0" diff --git a/src/fs.rs b/src/fs.rs new file mode 100644 index 0000000..d5864b3 --- /dev/null +++ b/src/fs.rs @@ -0,0 +1,207 @@ +// Copyright 2018-2024 the Deno authors. MIT license. + +use std::io::Error; +use std::io::ErrorKind; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use sys_traits::FsCanonicalize; +use sys_traits::FsCreateDirAll; +use sys_traits::FsMetadata; +use sys_traits::FsOpen; +use sys_traits::FsRemoveFile; +use sys_traits::FsRename; +use sys_traits::OpenOptions; +use sys_traits::SystemRandom; +use sys_traits::ThreadSleep; + +use crate::get_atomic_path; +use crate::normalize_path; + +/// Canonicalizes a path which might be non-existent by going up the +/// ancestors until it finds a directory that exists, canonicalizes +/// that path, then adds back the remaining path components. +/// +/// Note: When using this, you should be aware that a symlink may +/// subsequently be created along this path by some other code. +pub fn canonicalize_path_maybe_not_exists( + sys: &impl FsCanonicalize, + path: &Path, +) -> std::io::Result { + let path = normalize_path(path); + let mut path = path.as_path(); + let mut names_stack = Vec::new(); + loop { + match sys.fs_canonicalize(path) { + Ok(mut canonicalized_path) => { + for name in names_stack.into_iter().rev() { + canonicalized_path = canonicalized_path.join(name); + } + return Ok(canonicalized_path); + } + Err(err) if err.kind() == ErrorKind::NotFound => { + names_stack.push(match path.file_name() { + Some(name) => name.to_owned(), + None => return Err(err), + }); + path = match path.parent() { + Some(parent) => parent, + None => return Err(err), + }; + } + Err(err) => return Err(err), + } + } +} + +pub fn atomic_write_file_with_retries< + TSys: FsCreateDirAll + + FsMetadata + + FsOpen + + FsRemoveFile + + FsRename + + ThreadSleep + + SystemRandom, +>( + sys: &TSys, + file_path: &Path, + data: &[u8], + mode: u32, +) -> std::io::Result<()> { + let mut count = 0; + loop { + match atomic_write_file(sys, file_path, data, mode) { + Ok(()) => return Ok(()), + Err(err) => { + if count >= 5 { + // too many retries, return the error + return Err(err); + } + count += 1; + let sleep_ms = std::cmp::min(50, 10 * count); + sys.thread_sleep(std::time::Duration::from_millis(sleep_ms)); + } + } + } +} + +/// Writes the file to the file system at a temporary path, then +/// renames it to the destination in a single sys call in order +/// to never leave the file system in a corrupted state. +/// +/// This also handles creating the directory if a NotFound error +/// occurs. +pub fn atomic_write_file< + TSys: FsCreateDirAll + FsMetadata + FsOpen + FsRemoveFile + FsRename + SystemRandom, +>( + sys: &TSys, + file_path: &Path, + data: &[u8], + mode: u32, +) -> std::io::Result<()> { + fn atomic_write_file_raw( + sys: &TSys, + temp_file_path: &Path, + file_path: &Path, + data: &[u8], + mode: u32, + ) -> std::io::Result<()> { + let mut options = OpenOptions::write(); + options.mode = Some(mode); + let mut file = sys.fs_open(temp_file_path, &options)?; + file.write_all(data)?; + sys + .fs_rename(temp_file_path, file_path) + .inspect_err(|_err| { + // clean up the created temp file on error + let _ = sys.fs_remove_file(temp_file_path); + }) + } + + let temp_file_path = get_atomic_path(sys, file_path); + + if let Err(write_err) = + atomic_write_file_raw(sys, &temp_file_path, file_path, data, mode) + { + if write_err.kind() == ErrorKind::NotFound { + let parent_dir_path = file_path.parent().unwrap(); + match sys.fs_create_dir_all(parent_dir_path) { + Ok(()) => { + return atomic_write_file_raw( + sys, + &temp_file_path, + file_path, + data, + mode, + ) + .map_err(|err| add_file_context_to_err(file_path, err)); + } + Err(create_err) => { + if !sys.fs_exists(parent_dir_path).unwrap_or(false) { + return Err(Error::new( + create_err.kind(), + format!( + "{:#} (for '{}')\nCheck the permission of the directory.", + create_err, + parent_dir_path.display() + ), + )); + } + } + } + } + return Err(add_file_context_to_err(file_path, write_err)); + } + Ok(()) +} + +fn add_file_context_to_err(file_path: &Path, err: Error) -> Error { + Error::new( + err.kind(), + format!("{:#} (for '{}')", err, file_path.display()), + ) +} + +#[cfg(test)] +mod test { + use std::path::PathBuf; + + use sys_traits::impls::InMemorySys; + use sys_traits::impls::RealSys; + use sys_traits::EnvSetCurrentDir; + use sys_traits::FsCreateDirAll; + use sys_traits::FsRead; + + use super::atomic_write_file_with_retries; + use super::canonicalize_path_maybe_not_exists; + + #[test] + fn test_canonicalize_path_maybe_not_exists() { + let sys = InMemorySys::default(); + sys.fs_create_dir_all("/a/b/c").unwrap(); + sys.env_set_current_dir("/a/b").unwrap(); + let path = + canonicalize_path_maybe_not_exists(&sys, &PathBuf::from("./c")).unwrap(); + assert_eq!(path, PathBuf::from("/a/b/c")); + let path = + canonicalize_path_maybe_not_exists(&sys, &PathBuf::from("./c/d/e")) + .unwrap(); + assert_eq!(path, PathBuf::from("/a/b/c/d/e")); + } + + #[test] + fn test_atomic_write_file() { + let sys = RealSys; + let temp_dir = tempfile::tempdir().unwrap(); + let path = temp_dir.path().join("a/b/c"); + atomic_write_file_with_retries(&sys, &path, b"data", 0o644).unwrap(); + assert_eq!(sys.fs_read_to_string(&path).unwrap(), "data"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let file = std::fs::metadata(path).unwrap(); + assert_eq!(file.permissions().mode(), 0o100644); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index fd18295..3951d47 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -5,13 +5,15 @@ #![deny(clippy::unused_async)] #![deny(clippy::unnecessary_wraps)] -use std::io::ErrorKind; use std::path::Component; use std::path::Path; use std::path::PathBuf; +use sys_traits::SystemRandom; use thiserror::Error; use url::Url; +pub mod fs; + /// Gets the parent of this url. pub fn url_parent(url: &Url) -> Url { let mut url = url.clone(); @@ -310,40 +312,18 @@ pub fn strip_unc_prefix(path: PathBuf) -> PathBuf { } } -/// Canonicalizes a path which might be non-existent by going up the -/// ancestors until it finds a directory that exists, canonicalizes -/// that path, then adds back the remaining path components. -/// -/// Note: When using this, you should be aware that a symlink may -/// subsequently be created along this path by some other code. -pub fn canonicalize_path_maybe_not_exists( - path: &Path, - canonicalize: &impl Fn(&Path) -> std::io::Result, -) -> std::io::Result { - let path = normalize_path(path); - let mut path = path.as_path(); - let mut names_stack = Vec::new(); - loop { - match canonicalize(path) { - Ok(mut canonicalized_path) => { - for name in names_stack.into_iter().rev() { - canonicalized_path = canonicalized_path.join(name); - } - return Ok(canonicalized_path); - } - Err(err) if err.kind() == ErrorKind::NotFound => { - names_stack.push(match path.file_name() { - Some(name) => name.to_owned(), - None => return Err(err), - }); - path = match path.parent() { - Some(parent) => parent, - None => return Err(err), - }; - } - Err(err) => return Err(err), - } - } +pub fn get_atomic_path(sys: &impl SystemRandom, path: &Path) -> PathBuf { + let rand = gen_rand_path_component(sys); + let extension = format!("{rand}.tmp"); + path.with_extension(extension) +} + +fn gen_rand_path_component(sys: &impl SystemRandom) -> String { + use std::fmt::Write; + (0..4).fold(String::with_capacity(8), |mut output, _| { + write!(&mut output, "{:02x}", sys.sys_random_u8().unwrap()).unwrap(); + output + }) } #[cfg(test)] @@ -524,4 +504,14 @@ mod tests { ); } } + + #[test] + fn test_atomic_path() { + let sys = sys_traits::impls::InMemorySys::default(); + sys.set_seed(Some(10)); + let path = Path::new("/a/b/c.txt"); + let atomic_path = get_atomic_path(&sys, path); + assert_eq!(atomic_path.parent().unwrap(), path.parent().unwrap()); + assert_eq!(atomic_path.file_name().unwrap(), "c.3d3d3d3d.tmp"); + } }