[!!!][FEATURE] Don't write crawling response body by default

Author: eliashaeussler

docs/config-reference/crawler-options.md

@@ -324,3 +324,49 @@ CACHE_WARMUP_CRAWLER_OPTIONS='{"request_options": {"delay": 500, "timeout": 10}}
 ```
 
 :::
+
+### `write_response_body` <Badge type="tip" text="4.0+" />
+
+<small>🎨&nbsp;Type: `bool` &middot; 🐝&nbsp;Default: `false`</small>
+
+> Define whether or not to write response body of crawled URLs to the corresponding
+> response object.
+
+::: warning
+Enabling this option may significantly increase memory consumption during cache warmup.
+:::
+
+::: code-group
+
+```bash [CLI]
+./cache-warmup.phar --crawler-options '{"write_response_body": true}'
+```
+
+```json [JSON]
+{
+    "crawlerOptions": {
+        "write_response_body": true
+    }
+}
+```
+
+```php [PHP]
+use EliasHaeussler\CacheWarmup;
+
+return static function (CacheWarmup\Config\CacheWarmupConfig $config) {
+    $config->setCrawlerOption('write_response_body', true);
+
+    return $config;
+};
+```
+
+```yaml [YAML]
+crawlerOptions:
+  write_response_body: true
+```
+
+```bash [.env]
+CACHE_WARMUP_CRAWLER_OPTIONS='{"write_response_body": true}'
+```
+
+:::

src/Crawler/ConcurrentCrawler.php

@@ -43,6 +43,7 @@
  *     request_headers: array<string, string>,
  *     request_options: array<string, mixed>,
  *     client_config: array<string, mixed>,
+ *     write_response_body: bool,
  * }>
  */
 final class ConcurrentCrawler extends AbstractConfigurableCrawler implements LoggingCrawler, StoppableCrawler

src/Crawler/ConcurrentCrawlerTrait.php

@@ -26,9 +26,13 @@
 use EliasHaeussler\CacheWarmup\Http;
 use GuzzleHttp\ClientInterface;
 use GuzzleHttp\Pool;
+use GuzzleHttp\RequestOptions;
 use Psr\Http\Message;
 use Symfony\Component\OptionsResolver;
 
+use function array_key_exists;
+use function fopen;
+
 /**
  * ConcurrentCrawlerTrait.
  *
@@ -65,6 +69,11 @@ protected function configureOptions(OptionsResolver\OptionsResolver $optionsReso
             ->allowedTypes('array')
             ->default([])
         ;
+
+        $optionsResolver->define('write_response_body')
+            ->allowedTypes('bool')
+            ->default(false)
+        ;
     }
 
     /**
@@ -81,11 +90,19 @@ protected function createPool(
             $this->options['request_method'],
             $this->options['request_headers'],
         );
+        $options = $this->options['request_options'];
+
+        if (!$this->options['write_response_body'] && !array_key_exists(RequestOptions::SINK, $options)) {
+            Http\Message\Stream\NullStream::register();
+            $options[RequestOptions::SINK] = fopen('null:///', 'w+');
+        } else {
+            Http\Message\Stream\NullStream::unregister();
+        }
 
         return Http\Message\RequestPoolFactory::create($requestFactory->buildIterable($urls))
             ->withClient($client)
             ->withConcurrency($this->options['concurrency'])
-            ->withOptions($this->options['request_options'])
+            ->withOptions($options)
             ->withResponseHandler(...$handlers)
             ->withStopOnFailure($stopOnFailure)
             ->createPool()

src/Crawler/OutputtingCrawler.php

@@ -46,6 +46,7 @@
  *     request_headers: array<string, string>,
  *     request_options: array<string, mixed>,
  *     client_config: array<string, mixed>,
+ *     write_response_body: bool,
  * }>
  */
 final class OutputtingCrawler extends AbstractConfigurableCrawler implements LoggingCrawler, StoppableCrawler, VerboseCrawler

src/Http/Message/Stream/NullStream.php

@@ -0,0 +1,110 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the Composer package "eliashaeussler/cache-warmup".
+ *
+ * Copyright (C) 2020-2024 Elias Häußler <[email protected]>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ */
+
+namespace EliasHaeussler\CacheWarmup\Http\Message\Stream;
+
+use function in_array;
+use function stream_get_wrappers;
+use function stream_wrapper_register;
+use function stream_wrapper_unregister;
+
+/**
+ * NullStream.
+ *
+ * @author Elias Häußler <[email protected]>
+ * @license GPL-3.0-or-later
+ */
+final class NullStream
+{
+    private const PROTOCOL = 'null';
+
+    /**
+     * @var resource|null
+     */
+    public $context;
+
+    public static function register(): void
+    {
+        if (!self::isRegistered()) {
+            stream_wrapper_register(self::PROTOCOL, self::class);
+        }
+    }
+
+    public static function unregister(): void
+    {
+        if (self::isRegistered()) {
+            stream_wrapper_unregister(self::PROTOCOL);
+        }
+    }
+
+    private static function isRegistered(): bool
+    {
+        return in_array(self::PROTOCOL, stream_get_wrappers(), true);
+    }
+
+    public function stream_close(): void {}
+
+    public function stream_eof(): bool
+    {
+        return true;
+    }
+
+    public function stream_flush(): bool
+    {
+        return true;
+    }
+
+    public function stream_open(string $path, string $mode, int $options, ?string &$opened_path): bool
+    {
+        return true;
+    }
+
+    public function stream_read(int $count): string
+    {
+        return '';
+    }
+
+    public function stream_seek(int $count, int $whence = SEEK_SET): bool
+    {
+        return true;
+    }
+
+    /**
+     * @return array{}
+     */
+    public function stream_stat(): array
+    {
+        return [];
+    }
+
+    public function stream_tell(): int
+    {
+        return 0;
+    }
+
+    public function stream_write(string $data): int
+    {
+        // 1 is enough for curl handler to not fail during writing
+        return '' === $data ? 0 : 1;
+    }
+}

tests/unit/Crawler/ConcurrentCrawlerTest.php

@@ -28,10 +28,13 @@
 use EliasHaeussler\TransientLogger;
 use GuzzleHttp\Client;
 use GuzzleHttp\Psr7;
+use GuzzleHttp\RequestOptions;
 use PHPUnit\Framework;
 use Psr\Http\Message;
 use Psr\Log;
 
+use function stream_get_meta_data;
+
 /**
  * ConcurrentCrawlerTest.
  *
@@ -94,6 +97,45 @@ public function crawlIgnoresGivenClientConfigIfInstantiatedClientIsPassed(): voi
         self::assertNull($this->mockHandler->getLastRequest());
     }
 
+    #[Framework\Attributes\Test]
+    public function crawlDoesNotWritesResponseBodyByDefault(): void
+    {
+        $this->mockHandler->append(new Psr7\Response());
+
+        $this->subject->crawl([new Psr7\Uri('https://www.example.org')]);
+
+        $lastOptions = $this->mockHandler->getLastOptions();
+        $sink = $lastOptions[RequestOptions::SINK] ?? null;
+
+        self::assertIsResource($sink);
+        self::assertInstanceOf(
+            Src\Http\Message\Stream\NullStream::class,
+            stream_get_meta_data($sink)['wrapper_data'],
+        );
+    }
+
+    #[Framework\Attributes\Test]
+    public function crawlWritesResponseBodyIfConfigured(): void
+    {
+        $this->mockHandler->append(new Psr7\Response());
+
+        $subject = new Src\Crawler\ConcurrentCrawler(
+            [
+                'client_config' => [
+                    'handler' => $this->mockHandler,
+                ],
+                'write_response_body' => true,
+            ],
+        );
+
+        $subject->crawl([new Psr7\Uri('https://www.example.org')]);
+
+        $lastOptions = $this->mockHandler->getLastOptions();
+        $sink = $lastOptions[RequestOptions::SINK] ?? null;
+
+        self::assertNull($sink);
+    }
+
     #[Framework\Attributes\Test]
     public function crawlSendsRequestToAllGivenUrls(): void
     {

tests/unit/Crawler/OutputtingCrawlerTest.php

@@ -28,11 +28,14 @@
 use EliasHaeussler\TransientLogger;
 use GuzzleHttp\Client;
 use GuzzleHttp\Psr7;
+use GuzzleHttp\RequestOptions;
 use PHPUnit\Framework;
 use Psr\Http\Message;
 use Psr\Log;
 use Symfony\Component\Console;
 
+use function stream_get_meta_data;
+
 /**
  * OutputtingCrawlerTest.
  *
@@ -100,6 +103,45 @@ public function crawlIgnoresGivenClientConfigIfInstantiatedClientIsPassed(): voi
         self::assertNull($this->mockHandler->getLastRequest());
     }
 
+    #[Framework\Attributes\Test]
+    public function crawlDoesNotWritesResponseBodyByDefault(): void
+    {
+        $this->mockHandler->append(new Psr7\Response());
+
+        $this->subject->crawl([new Psr7\Uri('https://www.example.org')]);
+
+        $lastOptions = $this->mockHandler->getLastOptions();
+        $sink = $lastOptions[RequestOptions::SINK] ?? null;
+
+        self::assertIsResource($sink);
+        self::assertInstanceOf(
+            Src\Http\Message\Stream\NullStream::class,
+            stream_get_meta_data($sink)['wrapper_data'],
+        );
+    }
+
+    #[Framework\Attributes\Test]
+    public function crawlWritesResponseBodyIfConfigured(): void
+    {
+        $this->mockHandler->append(new Psr7\Response());
+
+        $subject = new Src\Crawler\ConcurrentCrawler(
+            [
+                'client_config' => [
+                    'handler' => $this->mockHandler,
+                ],
+                'write_response_body' => true,
+            ],
+        );
+
+        $subject->crawl([new Psr7\Uri('https://www.example.org')]);
+
+        $lastOptions = $this->mockHandler->getLastOptions();
+        $sink = $lastOptions[RequestOptions::SINK] ?? null;
+
+        self::assertNull($sink);
+    }
+
     #[Framework\Attributes\Test]
     public function crawlSendsRequestToAllGivenUrls(): void
     {