V0.3.0

.github/workflows/CI.yml

@@ -1,23 +1,19 @@
 name: Validation
 
-on: [push, pull_request]
+on: [pull_request]
 
 jobs:
   conventions:
     name: Conventions
-    strategy:
-      matrix:
-        os: ["ubuntu-latest"]
-        toolchain: ["stable", "beta", "nightly"]
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
     steps:
       - uses: actions/checkout@v2
 
       - name: Install the Rust toolchain
         uses: actions-rs/toolchain@v1
         id: toolchain
         with:
-          toolchain: ${{ matrix.toolchain }}
+          toolchain: stable
           components: rustfmt, clippy
           override: true
           default: true
@@ -29,7 +25,7 @@ jobs:
         uses: actions/cache@v1
         with:
           path: target
-          key: ${{ matrix.os }}-${{ steps.toolchain.outputs.rustc_hash }}-build-${{ hashFiles('Cargo.lock') }}
+          key: ubuntu-latest-${{ steps.toolchain.outputs.rustc_hash }}-build-${{ hashFiles('Cargo.lock') }}
 
       - name: Perform clippy checks.
         run: cargo clippy --all-targets --all-features -- -D warnings
@@ -38,8 +34,8 @@ jobs:
     name: Testing
     strategy:
       matrix:
-        os: ["windows-2019", "ubuntu-16.04", "ubuntu-18.04", "macOS-latest"]
-        toolchain: ["stable", "beta", "nightly"]
+        os: ["windows-2019", "ubuntu-latest", "macOS-latest"]
+        toolchain: ["stable"]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
@@ -87,4 +83,4 @@ jobs:
           args: "--verbose --all-features --workspace"
 
       - name: Upload to codecov.io
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v2

Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "rsef-rs"
-version = "0.2.0"
-authors = ["Christian Veenman <chris_veenman@hotmail.com>"]
+version = "0.3.0"
+authors = ["Christian <devqps@protonmail.com>"]
 edition = '2018'
 readme = "README.md"
 keywords = ["rsef", "parser", "rir"]
@@ -12,8 +12,6 @@ documentation = "https://docs.rs/rsef-rs"
 description = "A library for downloading and parsing RIR Statistics Exchange Format (RSEF) listings."
 license = "GPL-3.0"
 exclude = [
-    "README.md",
-    "tests/*",
     ".github/*"
 ]
 
@@ -23,13 +21,14 @@ maintenance = { status = "actively-developed" }
 
 [features]
 # Allows a user to download the RSEF listings.
-download = ["reqwest", "bzip2", "libflate", "chrono"]
+download = ["reqwest", "bzip2", "libflate", "chrono", "tokio"]
 
 # No feature is included in the default distribution.
-default = []
+default = ["download"]
 
 [dependencies]
 reqwest = { version = "0.9", optional = true }
-bzip2 = { version = "0.3", optional = true }
-libflate = { version = "1.0", optional = true }
-chrono = { version = "0.4", optional = true }
+bzip2 = { version = "0.4", optional = true }
+libflate = { version = "1", optional = true }
+chrono = { version = "0.4", optional = true }
+tokio = { version = "1", optional = true, features = ["macros", "rt", "rt-multi-thread"]}

README.md

@@ -1,31 +1,33 @@
 # RIR Statistics Exchange Format in Rust (rsef-rs)
-[![Build Status](https://github.com/DevQps/rsef-rs/workflows/Validation/badge.svg)](https://github.com/DevQps/mrt-rs)
-[![codecov](https://codecov.io/gh/DevQps/rsef-rs/branch/master/graph/badge.svg)](https://codecov.io/gh/DevQps/mrt-rs)
-[![Crates](https://img.shields.io/crates/v/rsef_rs.svg)](https://crates.io/crates/mrt-rs)
+[![Daily Scheduled Test](https://github.com/DevQps/rsef-rs/workflows/Scheduled/badge.svg)](https://github.com/DevQps/rsef-rs)
+[![codecov](https://codecov.io/gh/DevQps/rsef-rs/branch/master/graph/badge.svg)](https://codecov.io/gh/DevQps/rsef-rs)
+[![Crates](https://img.shields.io/crates/v/rsef_rs.svg)](https://crates.io/crates/rsef-rs)
 
 A library for downloading and parsing RIR Statistics Exchange Format (RSEF) listings in Rust.
 
 ## Features
-rsef-rs optionally includes the `download` feature which allows you to download listings from Regional Internet Registries with a single statement.
-In order to enable the `download` feature you can add the following to your dependencies section in your Cargo.toml:
-
-```no_run
-[dependencies]
-rsef-rs = { version = "0.2", features = ["download"] }
-```
+The `download` feature, which is enabled by default, allows you to download extended listings from Regional Internet Registries with a single statement.
 
 ## Examples & Documentation
-
 **Downloading and parsing an RSEF Listing**
 
-If you enabled the `download` feature, you can download listings as shown below:
+If the `download` feature is enabled, you can download listings as shown below:
 
 ```
 use rsef_rs::{Registry, Line};
+use tokio::runtime::Runtime;
 
 // Friday 1 February 2019 21:22:48
 let timestamp = 1_549_056_168;
-let stream = Registry::RIPE.download(timestamp).unwrap();
+
+// Download the listing from RIPE.
+let future = Registry::download(Registry::RIPE, timestamp);
+
+// Tokio's block_on method is used to wait for future to complete.
+// Use `future.await.unwrap()` in async contexts.
+let rt = Runtime::new().expect("Failure to construct Tokio runtime.");
+let stream = rt.block_on(future).unwrap();
+
 let records = rsef_rs::read_all(stream).unwrap();
 
 for x in records {

codecov.yml

@@ -0,0 +1,14 @@
+codecov:
+  require_ci_to_pass: yes
+
+coverage:
+  status:
+    project:
+      default:
+        # basic
+        target: 75%
+        threshold: 0%
+        flags:
+          - unit
+        paths:
+          - "src"

src/download.rs

@@ -2,15 +2,29 @@
 //! When the `download` feature is enabled, functionality is provided that allows a user to download
 //! RSEF listings from a specific date and parse them.
 //!
+//! Additional Note:
+//! - RIPE and LACNIC latest listings are from yesterday, whereas APNIC, AFRINIC and ARIN provide listings for today. Remember this when attempting to retrieve RSEF contents.
+//! - It will attempt to download the 'extended' listings with opaque identifiers.
+//!
 //! # Examples
 //!
 //! ## Downloading and parsing an RSEF Listing
-//! ```
+//! ```rust
 //! use rsef_rs::{Registry, Line};
+//! use tokio::runtime::Runtime;
 //!
 //! // Friday 1 February 2019 21:22:48
 //! let timestamp = 1_549_056_168;
-//! let stream = Registry::RIPE.download(timestamp).unwrap();
+//!
+//! // Download the listing from RIPE.
+//! let future = Registry::download(Registry::RIPE, timestamp);
+//!
+//! // Tokio's block_on method is used to wait for future to complete.
+//! // Use `future.await.unwrap()` in async contexts.
+//! let rt = Runtime::new().expect("Failure to construct Tokio runtime.");
+//! let stream = rt.block_on(future).unwrap();
+//!
+//! // Parse the stream into records.
 //! let records = rsef_rs::read_all(stream).unwrap();
 //!
 //! for x in records {
@@ -23,10 +37,7 @@
 //! ```
 
 use bzip2::read::BzDecoder;
-use chrono::DateTime;
-use chrono::Datelike;
 use chrono::NaiveDateTime;
-use chrono::Utc;
 use libflate::gzip::Decoder;
 
 use std::error::Error;
@@ -47,63 +58,86 @@ impl Registry {
     /// Downloads the RSEF listings of a specific Regional Internet Registry at a specific moment.
     /// The timestamp should be an UNIX Epoch. Returns a decoded stream that can be read from.
     /// Only the year, month and day wll be used to select the listing for that day.
-    pub fn download(&self, timestamp: i64) -> Result<Box<dyn Read>, Box<dyn Error>> {
-        let datetime: DateTime<Utc> =
-            DateTime::from_utc(NaiveDateTime::from_timestamp(timestamp, 0), Utc);
-        let year = datetime.year();
-        let month = if datetime.month() < 10 {
-            format!("0{}", datetime.month())
-        } else {
-            format!("{}", datetime.month())
-        };
-        let day = if datetime.month() < 10 {
-            format!("0{}", datetime.day())
-        } else {
-            format!("{}", datetime.day())
-        };
-
-        match self {
+    pub async fn download(
+        registry: Registry,
+        timestamp_seconds: i64,
+    ) -> Result<Box<dyn Read>, Box<dyn Error>> {
+        let timestamp = NaiveDateTime::from_timestamp(timestamp_seconds, 0);
+        let today = chrono::Utc::now().naive_utc().date();
+
+        if today < timestamp.date() {
+            return Err("Error: The date provided to Registry::download is in the future.".into());
+        }
+
+        match registry {
+            // AFRINIC does have a listing for "today".
             Registry::AFRINIC => {
-                let url = format!(
-                    "https://ftp.afrinic.net/pub/stats/afrinic/{}/delegated-afrinic-extended-{}{}{}",
-                    year, year, month, day
-                );
+                let url = if timestamp.date() == chrono::Utc::now().naive_utc().date() {
+                    timestamp
+                        .format("https://ftp.afrinic.net/pub/stats/afrinic/delegated-afrinic-extended-%Y%m%d")
+                        .to_string()
+                } else {
+                    timestamp
+                        .format("https://ftp.afrinic.net/pub/stats/afrinic/%Y/delegated-afrinic-extended-%Y%m%d")
+                        .to_string()
+                };
 
                 let response = reqwest::get(url.as_str())?;
                 Ok(Box::new(response))
             }
+            // APNIC does have a listing for 'today'.
             Registry::APNIC => {
-                let url = format!(
-                    "https://ftp.apnic.net/stats/apnic/{}/delegated-apnic-extended-{}{}{}.gz",
-                    year, year, month, day
-                );
-
-                let response = reqwest::get(url.as_str())?;
-                Ok(Box::new(Decoder::new(response)?))
+                if timestamp.date() == chrono::Utc::now().naive_utc().date() {
+                    let url = timestamp
+                        .format("https://ftp.apnic.net/stats/apnic/delegated-apnic-extended-%Y%m%d")
+                        .to_string();
+
+                    let response = reqwest::get(url.as_str())?;
+                    Ok(Box::new(response))
+                } else {
+                    let url = timestamp
+                        .format("https://ftp.apnic.net/stats/apnic/%Y/delegated-apnic-extended-%Y%m%d.gz")
+                        .to_string();
+
+                    let response = reqwest::get(url.as_str())?;
+                    Ok(Box::new(Decoder::new(response)?))
+                }
             }
+
+            // ARIN does have a listing for "today".
             Registry::ARIN => {
-                let url = format!(
-                    "https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-{}{}{}",
-                    year, month, day
-                );
+                let url = timestamp
+                    .format("https://ftp.arin.net/pub/stats/arin/delegated-arin-extended-%Y%m%d")
+                    .to_string();
 
                 let response = reqwest::get(url.as_str())?;
                 Ok(Box::new(response))
             }
+            // LACNIC does not have a listing for 'today'.
             Registry::LACNIC => {
-                let url = format!(
-                    "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-{}{}{}",
-                    year, month, day
-                );
+                if timestamp.date() == today {
+                    return Err("Error: LACNIC does not provide RSEF listings for today.".into());
+                }
+
+                let url = timestamp
+                    .format(
+                        "https://ftp.lacnic.net/pub/stats/lacnic/delegated-lacnic-extended-%Y%m%d",
+                    )
+                    .to_string();
 
                 let response = reqwest::get(url.as_str())?;
                 Ok(Box::new(response))
             }
+
+            // RIPE does not have a listing for "today".
             Registry::RIPE => {
-                let url = format!(
-                    "https://ftp.ripe.net/pub/stats/ripencc/{}/delegated-ripencc-extended-{}{}{}.bz2",
-                    year, year, month, day
-                );
+                if timestamp.date() == today {
+                    return Err("Error: RIPE does not provide RSEF listings for today.".into());
+                }
+
+                let url = timestamp
+                    .format("https://ftp.ripe.net/pub/stats/ripencc/%Y/delegated-ripencc-extended-%Y%m%d.bz2")
+                    .to_string();
 
                 let response = reqwest::get(url.as_str())?;
                 Ok(Box::new(BzDecoder::new(response)))
@@ -117,29 +151,36 @@ mod tests {
     // Note this useful idiom: importing names from outer (for mod tests) scope.
     use crate::Registry;
 
-    #[test]
-    fn test_download() {
-        // Friday 1 February 2019 21:22:48
-        let timestamp = 1_549_056_168;
+    #[tokio::test]
+    async fn test_download() {
+        // Test it with the date of yesterday as RIPE and LACNIC do not provide listings for today.
+        const DAY: i64 = 60 * 60 * 24;
+        let timestamp = chrono::Utc::now().timestamp() - DAY;
 
         println!("Downloading from AFRINIC");
-        let stream = Registry::AFRINIC.download(timestamp).unwrap();
+        let stream = Registry::download(Registry::AFRINIC, timestamp)
+            .await
+            .unwrap();
         let _ = crate::read_all(stream).unwrap();
 
         println!("Downloading from APNIC");
-        let stream = Registry::APNIC.download(timestamp).unwrap();
+        let stream = Registry::download(Registry::APNIC, timestamp)
+            .await
+            .unwrap();
         let _ = crate::read_all(stream).unwrap();
 
         println!("Downloading from ARIN");
-        let stream = Registry::ARIN.download(timestamp).unwrap();
+        let stream = Registry::download(Registry::ARIN, timestamp).await.unwrap();
         let _ = crate::read_all(stream).unwrap();
 
         println!("Downloading from LACNIC");
-        let stream = Registry::LACNIC.download(timestamp).unwrap();
+        let stream = Registry::download(Registry::LACNIC, timestamp)
+            .await
+            .unwrap();
         let _ = crate::read_all(stream).unwrap();
 
         println!("Downloading from RIPE");
-        let stream = Registry::RIPE.download(timestamp).unwrap();
+        let stream = Registry::download(Registry::RIPE, timestamp).await.unwrap();
         let _ = crate::read_all(stream).unwrap();
     }
 }

src/lib.rs

@@ -198,3 +198,15 @@ pub fn read_all(read: impl Read) -> Result<impl Iterator<Item = Line>, Box<dyn E
 
     Ok(lines.into_iter())
 }
+
+#[cfg(doctest)]
+mod test_readme {
+    macro_rules! external_doc_test {
+        ($x:expr) => {
+            #[doc = $x]
+            extern "C" {}
+        };
+    }
+
+    external_doc_test!(include_str!("../README.md"));
+}