feat: add file loader (#15)

.github/dependabot.yml

@@ -3,8 +3,8 @@ updates:
   - package-ecosystem: github-actions
     directory: /
     schedule:
-      interval: weekly
+      interval: monthly
   - package-ecosystem: cargo
     directory: /
     schedule:
-      interval: weekly
+      interval: monthly

Cargo.toml

@@ -12,25 +12,25 @@ description = "Automatically load secrets from your preferred vault as environme
 license = "MPL-2.0"
 
 [dependencies]
-anyhow = "1.0.69"
+anyhow = "1.0.70"
 self_update = { version = "0.36.0", features = [
   "archive-tar",
   "archive-zip",
   "compression-flate2",
   "compression-zip-deflate",
   "compression-zip-bzip2",
 ] }
-serde = { version = "1.0.156", features = ["derive"] }
+serde = { version = "1.0.159", features = ["derive"] }
 serde_yaml = "0.9.19"
-clap = { version = "4.1.8", features = ["derive"] }
-regex = "1.7.1"
+clap = { version = "4.2.1", features = ["derive"] }
+regex = "1.7.3"
 lade-sdk = { path = "./sdk", version = "0.4.1-beta.1" }
 tokio = { version = "1", features = ["full"] }
-indexmap = { version = "1.9.2", features = ["serde"] }
-clap-verbosity-flag = "2.0.0"
+indexmap = { version = "1.9.3", features = ["serde"] }
+clap-verbosity-flag = "2.0.1"
 env_logger = "0.10.0"
-openssl = { version = "0.10.46", features = ["vendored"] }
-serde_json = "1.0.94"
+openssl = { version = "0.10.49", features = ["vendored"] }
+serde_json = "1.0.95"
 
 [dev-dependencies]
-assert_cmd = "2.0.9"
+assert_cmd = "2.0.10"

README.md

@@ -106,6 +106,15 @@ command regex:
     EXPORTED_ENV_VAR: vault://DOMAIN/MOUNT/KEY/FIELD
 ```
 
+### File loader
+
+Supports INI, JSON, YAML and TOML files.
+
+```yaml
+command regex:
+    EXPORTED_ENV_VAR: file://PATH?query=.fields[0].field
+```
+
 ### Raw loader
 
 ```yaml

examples/sources/config.ini

@@ -0,0 +1,3 @@
+format=ini
+[nested]
+foo=bar

examples/sources/config.json

@@ -0,0 +1,7 @@
+{
+  "format": "json",
+  "nested": {
+    "foo": "bar"
+  },
+  "list": ["a"]
+}

examples/sources/config.toml

@@ -0,0 +1,4 @@
+format = "toml"
+[nested]
+foo = "bar"
+list = [ "a" ]

examples/sources/config.yaml

@@ -0,0 +1,5 @@
+format: yaml
+nested:
+  foo: bar
+list:
+  - a

lade.yml

@@ -13,3 +13,7 @@
   D2: op://my.1password.eu/Personal/Lade/password
   E1: vault://127.0.0.1:8200/secret/password/value1
   E2: vault://127.0.0.1:8200/secret/password/value2
+  F1: file://../lade/examples/sources/config.ini?query=.format
+  F2: file://./examples/sources/config.json?query=.format
+  F3: file://~/Documents/github/lade/examples/sources/config.toml?query=.format
+  F4: file://$HOME/Documents/github/lade/examples/sources/config.yaml?query=.format

sdk/Cargo.toml

@@ -6,16 +6,21 @@ description = "Lade SDK"
 license = "MPL-2.0"
 
 [dependencies]
-anyhow = "1.0.69"
+access-json = "0.1.0"
+anyhow = "1.0.70"
 async-process = "1.6.0"
-async-trait = "0.1.66"
-futures = "0.3.27"
+async-trait = "0.1.68"
+futures = "0.3.28"
 itertools = "0.10.5"
 log = "0.4.17"
 once_cell = "1.17.1"
-regex = "1.7.1"
-serde = { version = "1.0.156", features = ["derive"] }
-serde_json = "1.0.94"
-tempfile = "3.4.0"
+regex = "1.7.3"
+rust-ini = "0.18.0"
+serde = { version = "1.0.159", features = ["derive"] }
+serde_json = "1.0.95"
+serde_yaml = "0.9.19"
+tempfile = "3.5.0"
+tokio = { version = "1.27.0", features = ["fs"] }
+toml = "0.7.3"
 url = "2.3.1"
 uuid = { version = "1.3.0", features = ["v4"] }

sdk/src/providers/file.rs

@@ -0,0 +1,160 @@
+use std::{collections::HashMap, env, path::PathBuf, str::FromStr};
+
+use access_json::JSONQuery;
+use anyhow::{bail, Context, Ok, Result};
+use async_trait::async_trait;
+use futures::future::try_join_all;
+use ini::Ini;
+use itertools::Itertools;
+use serde_json::Value;
+use tokio::fs;
+use url::Url;
+
+use super::Provider;
+use crate::Hydration;
+
+#[derive(Default)]
+pub struct File {
+    urls: Vec<String>,
+}
+
+impl File {
+    pub fn new() -> Self {
+        Default::default()
+    }
+}
+
+#[async_trait]
+impl Provider for File {
+    fn add(&mut self, value: String) -> Result<()> {
+        match Url::parse(&value) {
+            std::result::Result::Ok(url)
+                if url.scheme() == "file"
+                    && url.query_pairs().into_iter().any(|(k, _v)| k == "query") =>
+            {
+                self.urls.push(value);
+                Ok(())
+            }
+            _ => bail!("Not an file scheme or missing ?query=.field part"),
+        }
+    }
+    async fn resolve(&self) -> Result<Hydration> {
+        let home = env::var("HOME").context("getting $HOME")?;
+        let fetches = self
+            .urls
+            .iter()
+            .into_group_map_by(|u| {
+                let u = Url::parse(u).unwrap();
+                let port = match u.port() {
+                    Some(port) => format!(":{}", port),
+                    None => "".to_string(),
+                };
+                format!(
+                    "{}{}{}",
+                    u.host()
+                        .map(|h| h.to_string().replace("$home", &home).replace('~', &home))
+                        .unwrap_or("".to_string()),
+                    port,
+                    u.path()
+                )
+            })
+            .into_iter()
+            .map(|(file, group)| {
+                let path = PathBuf::from_str(&file).unwrap();
+                let format = file
+                    .split('.')
+                    .last()
+                    .expect("no file format found")
+                    .to_string();
+
+                async move {
+                    let str = fs::read_to_string(&path)
+                        .await
+                        .unwrap_or_else(|_| panic!("cannot read file {}", path.display()));
+                    let json = match format.as_str() {
+                        "yaml" | "yml" => serde_yaml::from_str::<Value>(&str)?,
+                        "json" => serde_json::from_str::<Value>(&str)?,
+                        "toml" => {
+                            let values = toml::from_str::<toml::Value>(&str)?;
+                            toml2json(values)
+                        }
+                        "ini" => {
+                            let values = Ini::load_from_str(&str)?;
+                            ini2json(values)
+                        }
+                        _ => bail!("unsupported file format: {}", format),
+                    };
+
+                    let hydration = group
+                        .into_iter()
+                        .map(|u| {
+                            let url = Url::parse(u).unwrap();
+                            let query = url
+                                .query_pairs()
+                                .into_iter()
+                                .find(|(k, _v)| k == "query")
+                                .unwrap()
+                                .1;
+
+                            let compiled = JSONQuery::parse(&query)
+                                .unwrap_or_else(|_| panic!("cannot compile query {}", query));
+                            let res = compiled
+                                .execute(&json)
+                                .unwrap()
+                                .unwrap_or_else(|| panic!("no query result for {}", query));
+
+                            let output = match res {
+                                Value::String(s) => s,
+                                x => x.to_string(),
+                            };
+
+                            (u.to_string(), output)
+                        })
+                        .collect::<Hydration>();
+
+                    Ok(hydration)
+                }
+            })
+            .collect::<Vec<_>>();
+
+        Ok(try_join_all(fetches).await?.into_iter().flatten().collect())
+    }
+}
+
+fn ini2json(ini: Ini) -> Value {
+    let mut map = HashMap::new();
+    for (section, properties) in ini.iter() {
+        let mut section_map = HashMap::new();
+        for (key, value) in properties.iter() {
+            section_map.insert(key.to_string(), Value::String(value.to_string()));
+        }
+        match section {
+            Some(section) => {
+                map.insert(
+                    section.to_string(),
+                    Value::Object(section_map.into_iter().collect()),
+                );
+            }
+            None => {
+                map.extend(section_map);
+            }
+        }
+    }
+    Value::Object(map.into_iter().collect())
+}
+
+fn toml2json(toml: toml::Value) -> Value {
+    match toml {
+        toml::Value::String(s) => Value::String(s),
+        toml::Value::Integer(i) => Value::Number(i.into()),
+        toml::Value::Float(f) => {
+            Value::Number(serde_json::Number::from_f64(f).expect("nan not allowed"))
+        }
+        toml::Value::Boolean(b) => Value::Bool(b),
+        toml::Value::Array(arr) => Value::Array(arr.into_iter().map(toml2json).collect()),
+        toml::Value::Table(table) => {
+            Value::Object(table.into_iter().map(|(k, v)| (k, toml2json(v))).collect())
+        }
+        toml::Value::Datetime(dt) => Value::String(dt.to_string()),
+    }
+}

sdk/src/providers/mod.rs

@@ -4,6 +4,7 @@ use async_trait::async_trait;
 use crate::Hydration;
 
 mod doppler;
+mod file;
 mod infisical;
 mod onepassword;
 mod raw;
@@ -21,6 +22,7 @@ pub fn providers() -> Vec<Box<dyn Provider + Send>> {
         Box::new(infisical::Infisical::new()),
         Box::new(onepassword::OnePassword::new()),
         Box::new(vault::Vault::new()),
+        Box::new(file::File::new()),
         Box::new(raw::Raw::new()),
     ]
 }

sdk/src/providers/raw.rs

@@ -18,6 +18,11 @@ impl Raw {
 #[async_trait]
 impl Provider for Raw {
     fn add(&mut self, value: String) -> Result<()> {
+        let mut value = value;
+        // escape the first ! if it exists
+        if value.starts_with('!') {
+            value.remove(0);
+        }
         self.values.push(value);
         Ok(())
     }