Merge pull request #162 from DomainTools/2.4

Merge 2.4 changes

Author: jbabac

VERSION

@@ -1 +1 @@
-2.3.0
+2.4.0

domaintools/_version.py

@@ -20,4 +20,4 @@
 
 """
 
-current = "2.3.0"
+current = "2.4.0"

domaintools/api.py

@@ -411,38 +411,6 @@ def whois_history(self, query, mode=None, sort=None, offset=None, limit=None, **
             **kwargs,
         )
 
-    def phisheye(self, query, days_back=None, **kwargs):
-        """Returns domain results for the specified term for today or the specified number of days_back.
-        Terms must be setup for monitoring via the web interface: https://research.domaintools.com/phisheye.
-
-        NOTE: Properties of a domain are only provided if we have been able to obtain them.
-              Many domains will have incomplete data because that information isn't available in their Whois records,
-              or they don't have DNS results for a name server or IP address.
-        """
-        return self._results(
-            "phisheye",
-            "/v1/phisheye",
-            query=query,
-            days_back=days_back,
-            items_path=("domains",),
-            **kwargs,
-        )
-
-    def phisheye_term_list(self, include_inactive=False, **kwargs):
-        """Provides a list of terms that are set up for this account.
-        This call is not charged against your API usage limit.
-
-        NOTE: The terms must be configured in the PhishEye web interface: https://research.domaintools.com/phisheye.
-              There is no API call to set up the terms.
-        """
-        return self._results(
-            "phisheye_term_list",
-            "/v1/phisheye/term-list",
-            include_inactive=include_inactive,
-            items_path=("terms",),
-            **kwargs,
-        )
-
     def iris(
         self,
         domain=None,
@@ -1065,7 +1033,21 @@ def iris_detect_ignored_domains(
         )
 
     def nod(self, **kwargs) -> FeedsResults:
-        """Returns back list of the newly observed domains feed"""
+        """Returns back list of the newly observed domains feed.
+        Apex-level domains (e.g. example.com but not www.example.com) that we observe for the first time, and have not observed previously with our global DNS sensor network.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint)
@@ -1082,7 +1064,22 @@ def nod(self, **kwargs) -> FeedsResults:
         )
 
     def nad(self, **kwargs) -> FeedsResults:
-        """Returns back list of the newly active domains feed"""
+        """Returns back list of the newly active domains feed. Contains domains that have been observed after having not been seen for at least 10 days in passive DNS.
+        Apex-level domains (e.g. example.com but not www.example.com) that we observe based on the latest lifecycle of the domain. A domain may be seen either for the first time ever, or again after at least 10 days of inactivity (no observed resolutions in DNS).
+        Populated with our global passive DNS (pDNS) sensor network.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
@@ -1099,7 +1096,22 @@ def nad(self, **kwargs) -> FeedsResults:
         )
 
     def domainrdap(self, **kwargs) -> FeedsResults:
-        """Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP)"""
+        """Returns changes to global domain registration information, populated by the Registration Data Access Protocol (RDAP).
+        Compliments the 5-Minute WHOIS Feed as registries and registrars switch from Whois to RDAP.
+        Contains parsed and raw RDAP-format domain registration data, emitted as soon as they are collected and parsed into a normalized structure.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
@@ -1113,7 +1125,22 @@ def domainrdap(self, **kwargs) -> FeedsResults:
         )
 
     def domaindiscovery(self, **kwargs) -> FeedsResults:
-        """Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties"""
+        """Returns new domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties".
+        Contains domains that are newly-discovered by Domain Tools in both passive and active DNS sources, emitted as soon as they are first observed.
+        New domains as they are either discovered in domain registration information, observed by our global sensor network, or reported by trusted third parties.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
         validate_feeds_parameters(kwargs)
         endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
         source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
@@ -1128,3 +1155,35 @@ def domaindiscovery(self, **kwargs) -> FeedsResults:
             cls=FeedsResults,
             **kwargs,
         )
+
+    def noh(self, **kwargs) -> FeedsResults:
+        """Returns back list of the newly observed hostnames feed.
+        Contains fully qualified domain names (i.e. host names) that have never been seen before in passive DNS, emitted as soon as they are first observed.
+        Hostname resolutions that we observe for the first time with our global DNS sensor network.
+
+        domain: str: Filter for an exact domain or a substring contained within a domain by prefixing or suffixing your substring with "*". Check the documentation for examples
+
+        before: str: Filter for records before the given time value inclusive or time offset relative to now
+
+        after: str: Filter for records after the given time value inclusive or time offset relative to now
+
+        headers: bool: Use in combination with Accept: text/csv headers to control if headers are sent or not
+
+        sessionID: str: A custom string to distinguish between different sessions
+
+        top: int: Limit the number of results to the top N, where N is the value of this parameter.
+        """
+        validate_feeds_parameters(kwargs)
+        endpoint = kwargs.pop("endpoint", Endpoint.FEED.value)
+        source = ENDPOINT_TO_SOURCE_MAP.get(endpoint).value
+        if endpoint == Endpoint.DOWNLOAD.value or kwargs.get("output_format", OutputFormat.JSONL.value) != OutputFormat.CSV.value:
+            # headers param is allowed only in Feed API and CSV format
+            kwargs.pop("headers", None)
+
+        return self._results(
+            f"newly-observed-hosts-feed-({source})",
+            f"v1/{endpoint}/noh/",
+            response_path=(),
+            cls=FeedsResults,
+            **kwargs,
+        )

domaintools/cli/commands/__init__.py

@@ -2,6 +2,5 @@
 from .iris import *
 from .domains import *
 from .ips import *
-from .phisheye import *
 from .detects import *
 from .feeds import *

domaintools/cli/commands/feeds.py

@@ -314,3 +314,83 @@ def feeds_domaindiscovery(
     ),
 ):
     DTCLICommand.run(name=c.FEEDS_DOMAINDISCOVERY, params=ctx.params)
+
+
+@dt_cli.command(
+    name=c.FEEDS_NOH,
+    help=get_cli_helptext_by_name(command_name=c.FEEDS_NOH),
+)
+def feeds_noh(
+    ctx: typer.Context,
+    user: str = typer.Option(None, "-u", "--user", help="Domaintools API Username."),
+    key: str = typer.Option(None, "-k", "--key", help="DomainTools API key"),
+    creds_file: str = typer.Option(
+        "~/.dtapi",
+        "-c",
+        "--credfile",
+        help="Optional file with API username and API key, one per line.",
+    ),
+    no_verify_ssl: bool = typer.Option(
+        False,
+        "--no-verify-ssl",
+        help="Skip verification of SSL certificate when making HTTPs API calls",
+    ),
+    no_sign_api_key: bool = typer.Option(
+        False,
+        "--no-sign-api-key",
+        help="Skip signing of api key",
+    ),
+    header_authentication: bool = typer.Option(
+        True,
+        "--no-header-auth",
+        help="Don't use header authentication",
+    ),
+    output_format: str = typer.Option(
+        "jsonl",
+        "-f",
+        "--format",
+        help=f"Output format in [{OutputFormat.JSONL.value}, {OutputFormat.CSV.value}]",
+        callback=DTCLICommand.validate_feeds_format_input,
+    ),
+    endpoint: str = typer.Option(
+        Endpoint.FEED.value,
+        "-e",
+        "--endpoint",
+        help=f"Valid endpoints: [{Endpoint.FEED.value}, {Endpoint.DOWNLOAD.value}]",
+        callback=DTCLICommand.validate_endpoint_input,
+    ),
+    sessionID: str = typer.Option(
+        None,
+        "--session-id",
+        help="Unique identifier for the session",
+    ),
+    after: str = typer.Option(
+        None,
+        "--after",
+        help="Start of the time window, relative to the current time in seconds, for which data will be provided",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    before: str = typer.Option(
+        None,
+        "--before",
+        help="The end of the query window in seconds, relative to the current time, inclusive",
+        callback=DTCLICommand.validate_after_or_before_input,
+    ),
+    domain: str = typer.Option(
+        None,
+        "-d",
+        "--domain",
+        help="A string value used to filter feed results",
+    ),
+    headers: bool = typer.Option(
+        False,
+        "--headers",
+        help="Adds a header to the first line of response when text/csv is set in header parameters",
+    ),
+    top: str = typer.Option(
+        None,
+        "--top",
+        help="Number of results to return in the response payload. This is ignored in download endpoint",
+    ),
+):
+    DTCLICommand.run(name=c.FEEDS_NOH, params=ctx.params)

domaintools/cli/commands/phisheye.py

@@ -39,12 +39,9 @@
 IRIS_DETECT_ESCALATE_DOMAINS = "iris_detect_escalate_domains"
 IRIS_DETECT_IGNORED_DOMAINS = "iris_detect_ignored_domains"
 
-# phisheye
-PHISHEYE = "phisheye"
-PHISHEYE_TERM_LIST = "phisheye_term_list"
-
 # feeds
 FEEDS_NAD = "nad"
 FEEDS_NOD = "nod"
+FEEDS_NOH = "noh"
 FEEDS_DOMAINRDAP = "domainrdap"
 FEEDS_DOMAINDISCOVERY = "domaindiscovery"