Prevent loading relative reference URI

Trying to load a relative reference URI (no scheme and host/authority,
only path) via the `HttpLoader` now immediately logs (or throws when
`loadOrFail()` is used) an error instead of trying to actually load it.

Author: otsch

CHANGELOG.md

@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.2.4] - 2025-02-25
+### Fixed
+* Trying to load a relative reference URI (no scheme and host/authority, only path) via the `HttpLoader` now immediately logs (or throws when `loadOrFail()` is used) an error instead of trying to actually load it.
+
 ## [3.2.3] - 2025-01-28
 ### Fixed
 * Fix deprecation warning triggered in the `DomQuery` class, when trying to get the value of an HTML/XML attribute that does not exist on the element.

src/Loader/Http/HttpLoader.php

@@ -123,8 +123,10 @@ public function load(mixed $subject): ?RespondedRequest
 
         try {
             $request = $this->validateSubjectType($subject);
-        } catch (InvalidArgumentException) {
-            $this->logger->error('Invalid input URL: ' . var_export($subject, true));
+        } catch (InvalidArgumentException|Exception $exception) {
+            $url = $subject instanceof RequestInterface ? (string) $subject->getUri() : (string) $subject;
+
+            $this->logger->error('Invalid input URL: ' . $url . ' - ' . $exception->getMessage());
 
             return null;
         }
@@ -164,7 +166,7 @@ public function load(mixed $subject): ?RespondedRequest
     }
 
     /**
-     * @throws LoadingException
+     * @throws LoadingException|InvalidArgumentException|Exception
      */
     public function loadOrFail(mixed $subject): RespondedRequest
     {
@@ -592,16 +594,29 @@ protected function shouldRequestBeServedFromCache(RequestInterface $request): bo
     }
 
     /**
-     * @throws InvalidArgumentException
+     * @throws InvalidArgumentException|Exception
      */
     protected function validateSubjectType(RequestInterface|string $requestOrUri): RequestInterface
     {
         if (is_string($requestOrUri)) {
             try {
-                return new Request('GET', Url::parsePsr7($requestOrUri));
+                $url = Url::parse($requestOrUri);
+
+                if ($url->isRelativeReference()) {
+                    throw new InvalidArgumentException(
+                        'The URI is a relative reference and therefore can\'t be loaded.',
+                    );
+                }
+
+                return new Request('GET', $url->toPsr7());
             } catch (InvalidUrlException) {
                 throw new InvalidArgumentException('Invalid URL.');
             }
+        } elseif (
+            empty(trim($requestOrUri->getUri()->getScheme())) &&
+            Url::parse($requestOrUri->getUri())->isRelativeReference()
+        ) {
+            throw new InvalidArgumentException('The URI is a relative reference and therefore can\'t be loaded.');
         }
 
         return $requestOrUri;

tests/Loader/Http/HttpLoaderTest.php

@@ -9,29 +9,26 @@
 use Crwlr\Crawler\Loader\Http\Politeness\Throttler;
 use Crwlr\Crawler\Steps\Filters\Filter;
 use Crwlr\Crawler\UserAgents\BotUserAgent;
-use Crwlr\Crawler\UserAgents\UserAgent;
 use Exception;
 use GuzzleHttp\Client;
 use GuzzleHttp\Psr7\Request;
 use GuzzleHttp\Psr7\Response;
+use InvalidArgumentException;
 use Mockery;
 use PHPUnit\Framework\TestCase;
 use Psr\Http\Client\ClientInterface;
 use Psr\Http\Message\RequestInterface;
 use Psr\Http\Message\UriInterface;
 use Psr\SimpleCache\CacheInterface;
+use tests\_Stubs\DummyLogger;
 use tests\_Stubs\RespondedRequestChild;
 use Throwable;
 
 use function tests\helper_cachedir;
 use function tests\helper_getFastLoader;
+use function tests\helper_nonBotUserAgent;
 use function tests\helper_resetCacheDir;
 
-function helper_nonBotUserAgent(): UserAgent
-{
-    return new UserAgent('Mozilla/5.0 (compatible; FooBot)');
-}
-
 afterEach(function () {
     helper_resetCacheDir();
 });
@@ -50,6 +47,27 @@ function helper_nonBotUserAgent(): UserAgent
     $httpLoader->loadOrFail('https://www.crwlr.software');
 });
 
+it('fails and logs an error when invoked with a relative reference URI', function () {
+    $logger = new DummyLogger();
+
+    $httpLoader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);
+
+    $httpLoader->load('/foo');
+
+    expect($logger->messages)->not->toBeEmpty()
+        ->and($logger->messages[0]['message'])->toBe(
+            'Invalid input URL: /foo - The URI is a relative reference and therefore can\'t be loaded.',
+        );
+});
+
+it('fails and throws an exception when loadOrFail() is called with a relative reference URI', function () {
+    $logger = new DummyLogger();
+
+    $httpLoader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);
+
+    $httpLoader->loadOrFail('/foo');
+})->throws(InvalidArgumentException::class);
+
 it('accepts RequestInterface as argument to load', function () {
     $httpClient = Mockery::mock(ClientInterface::class);
 
@@ -62,6 +80,31 @@ function helper_nonBotUserAgent(): UserAgent
     $httpLoader->loadOrFail(new Request('GET', 'https://www.crwlr.software'));
 });
 
+it('fails and logs an error when invoked with a RequestInterface object having a relative reference URI', function () {
+    $logger = new DummyLogger();
+
+    $httpLoader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);
+
+    $httpLoader->load(new Request('GET', '/foo'));
+
+    expect($logger->messages)->not->toBeEmpty()
+        ->and($logger->messages[0]['message'])->toBe(
+            'Invalid input URL: /foo - The URI is a relative reference and therefore can\'t be loaded.',
+        );
+});
+
+it(
+    'fails and throws an exception when loadOrFail() is called with a RequestInterface object having a relative ' .
+    'reference URI',
+    function () {
+        $logger = new DummyLogger();
+
+        $httpLoader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);
+
+        $httpLoader->loadOrFail(new Request('GET', '/foo'));
+    },
+)->throws(InvalidArgumentException::class);
+
 it(
     'calls the before and after load hooks regardless whether the response was successful or not',
     function ($responseStatusCode) {
@@ -217,15 +260,15 @@ function ($responseStatusCode) {
     $httpLoader = new class (new BotUserAgent('Foo'), $httpClient) extends HttpLoader {
         public function isAllowedToBeLoaded(UriInterface $uri, bool $throwsException = false): bool
         {
-            return $uri->__toString() === '/foo';
+            return $uri->__toString() === 'https://www.example.com/foo';
         }
     };
 
-    $response = $httpLoader->load('/foo');
+    $response = $httpLoader->load('https://www.example.com/foo');
 
     expect($response)->toBeInstanceOf(RespondedRequest::class);
 
-    $response = $httpLoader->load('/bar');
+    $response = $httpLoader->load('https://www.example.com/bar');
 
     expect($response)->toBeNull();
 });

tests/Loader/Http/Politeness/RobotsTxtHandlerTest.php

@@ -124,7 +124,7 @@ function () {
 
     $robotsTxt = new RobotsTxtHandler($loader, new CliLogger());
 
-    expect($robotsTxt->isAllowed('/anything'))->toBeTrue();
+    expect($robotsTxt->isAllowed('https://www.example.com/anything'))->toBeTrue();
 
     $logOutput = $this->getActualOutputForAssertion();
 

tests/Pest.php

@@ -291,6 +291,11 @@ protected function loader(UserAgentInterface $userAgent, LoggerInterface $logger
     };
 }
 
+function helper_nonBotUserAgent(): UserAgent
+{
+    return new UserAgent('Mozilla/5.0 (compatible; FooBot)');
+}
+
 function helper_getMinThrottler(): Throttler
 {
     return new Throttler(new MultipleOf(0.0001), new MultipleOf(0.0002), Microseconds::fromSeconds(0.0001));

tests/Steps/Loading/HttpTest.php

@@ -10,13 +10,15 @@
 use GuzzleHttp\Psr7\Request;
 use GuzzleHttp\Psr7\Response;
 use GuzzleHttp\Psr7\Utils;
+use InvalidArgumentException;
 use Mockery;
 use Psr\Http\Message\RequestInterface;
 use stdClass;
 use tests\_Stubs\DummyLogger;
 
 use function tests\helper_getRespondedRequest;
 use function tests\helper_invokeStepWithInput;
+use function tests\helper_nonBotUserAgent;
 use function tests\helper_traverseIterable;
 
 it('can be invoked with a string as input', function () {
@@ -55,6 +57,33 @@
         ->and($logger->messages[0]['message'])->toEndWith('. The invalid input is of type object.');
 });
 
+it('logs an error message when invoked with a relative reference URI', function () {
+    $logger = new DummyLogger();
+
+    $loader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);
+
+    $step = (new Http('GET'))->setLoader($loader)->addLogger($logger);
+
+    helper_invokeStepWithInput($step, '/foo/bar');
+
+    expect($logger->messages)->not->toBeEmpty()
+        ->and($logger->messages[0]['message'])->toBe(
+            'Invalid input URL: /foo/bar - The URI is a relative reference and therefore can\'t be loaded.',
+        );
+});
+
+it('throws an exception when invoked with a relative reference URI and stopOnErrorResponse() was called', function () {
+    $logger = new DummyLogger();
+
+    $loader = new HttpLoader(helper_nonBotUserAgent(), logger: $logger);
+
+    $step = (new Http('GET'))->setLoader($loader)->addLogger($logger);
+
+    $step->stopOnErrorResponse();
+
+    helper_invokeStepWithInput($step, '/foo/bar');
+})->throws(InvalidArgumentException::class);
+
 test('You can set the request method via constructor', function (string $httpMethod) {
     $loader = Mockery::mock(HttpLoader::class);