Created action to dynamically build the robots.txt

module/Core/config/dependencies.config.php

@@ -48,6 +48,7 @@ return [
             Action\RedirectAction::class => ConfigAbstractFactory::class,
             Action\PixelAction::class => ConfigAbstractFactory::class,
             Action\QrCodeAction::class => ConfigAbstractFactory::class,
+            Action\RobotsAction::class => ConfigAbstractFactory::class,
 
             ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => ConfigAbstractFactory::class,
             ShortUrl\Helper\ShortUrlStringifier::class => ConfigAbstractFactory::class,
@@ -57,6 +58,8 @@ return [
             Mercure\MercureUpdatesGenerator::class => ConfigAbstractFactory::class,
 
             Importer\ImportedLinksProcessor::class => ConfigAbstractFactory::class,
+
+            Crawling\CrawlingHelper::class => InvokableFactory::class,
         ],
 
         'aliases' => [
@@ -129,6 +132,7 @@ return [
             ShortUrl\Helper\ShortUrlStringifier::class,
             'Logger_Shlink',
         ],
+        Action\RobotsAction::class => [Crawling\CrawlingHelper::class],
 
         ShortUrl\Resolver\PersistenceShortUrlRelationResolver::class => ['em'],
         ShortUrl\Helper\ShortUrlStringifier::class => ['config.url_shortener.domain', 'config.router.base_path'],

module/Core/config/routes.config.php

@@ -9,6 +9,14 @@ use Shlinkio\Shlink\Core\Action;
 return [
 
     'routes' => [
+        [
+            'name' => Action\RobotsAction::class,
+            'path' => '/robots.txt',
+            'middleware' => [
+                Action\RobotsAction::class,
+            ],
+            'allowed_methods' => [RequestMethod::METHOD_GET],
+        ],
         [
             'name' => Action\RedirectAction::class,
             'path' => '/{shortCode}',

module/Core/src/Action/RobotsAction.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Action;
+
+use Fig\Http\Message\StatusCodeInterface;
+use GuzzleHttp\Psr7\Response;
+use Psr\Http\Message\ResponseInterface;
+use Psr\Http\Message\ServerRequestInterface;
+use Psr\Http\Server\RequestHandlerInterface;
+use Shlinkio\Shlink\Core\Crawling\CrawlingHelperInterface;
+
+use function sprintf;
+
+use const PHP_EOL;
+
+class RobotsAction implements RequestHandlerInterface, StatusCodeInterface
+{
+    private CrawlingHelperInterface $crawlingHelper;
+
+    public function __construct(CrawlingHelperInterface $crawlingHelper)
+    {
+        $this->crawlingHelper = $crawlingHelper;
+    }
+
+    public function handle(ServerRequestInterface $request): ResponseInterface
+    {
+        return new Response(self::STATUS_OK, ['Content-type' => 'text/plain'], $this->buildRobots());
+    }
+
+    private function buildRobots(): iterable
+    {
+        yield <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+
+        ROBOTS;
+
+        $shortCodes = $this->crawlingHelper->listCrawlableShortCodes();
+        foreach ($shortCodes as $shortCode) {
+            yield sprintf('Allow: /%s%s', $shortCode, PHP_EOL);
+        }
+
+        yield 'Disallow: /';
+    }
+}

module/Core/src/Crawling/CrawlingHelper.php

@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Crawling;
+
+class CrawlingHelper implements CrawlingHelperInterface
+{
+    public function listCrawlableShortCodes(): iterable
+    {
+        return [];
+    }
+}

module/Core/src/Crawling/CrawlingHelperInterface.php

@@ -0,0 +1,13 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Shlinkio\Shlink\Core\Crawling;
+
+interface CrawlingHelperInterface
+{
+    /**
+     * @return string[]|iterable
+     */
+    public function listCrawlableShortCodes(): iterable;
+}

module/Core/test/Action/RobotsActionTest.php

@@ -0,0 +1,75 @@
+<?php
+
+declare(strict_types=1);
+
+namespace ShlinkioTest\Shlink\Core\Action;
+
+use Laminas\Diactoros\ServerRequestFactory;
+use PHPUnit\Framework\TestCase;
+use Prophecy\PhpUnit\ProphecyTrait;
+use Prophecy\Prophecy\ObjectProphecy;
+use Shlinkio\Shlink\Core\Action\RobotsAction;
+use Shlinkio\Shlink\Core\Crawling\CrawlingHelperInterface;
+
+class RobotsActionTest extends TestCase
+{
+    use ProphecyTrait;
+
+    private RobotsAction $action;
+    private ObjectProphecy $helper;
+
+    protected function setUp(): void
+    {
+        $this->helper = $this->prophesize(CrawlingHelperInterface::class);
+        $this->action = new RobotsAction($this->helper->reveal());
+    }
+
+    /**
+     * @test
+     * @dataProvider provideShortCodes
+     */
+    public function buildsRobotsLinesFromCrawlableShortCodes(array $shortCodes, string $expected): void
+    {
+        $getShortCodes = $this->helper->listCrawlableShortCodes()->willReturn($shortCodes);
+
+        $response = $this->action->handle(ServerRequestFactory::fromGlobals());
+
+        self::assertEquals(200, $response->getStatusCode());
+        self::assertEquals($expected, $response->getBody()->__toString());
+        self::assertEquals('text/plain', $response->getHeaderLine('Content-Type'));
+        $getShortCodes->shouldHaveBeenCalledOnce();
+    }
+
+    public function provideShortCodes(): iterable
+    {
+        yield 'three short codes' => [['foo', 'bar', 'baz'], <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+        Allow: /foo
+        Allow: /bar
+        Allow: /baz
+        Disallow: /
+        ROBOTS];
+        yield 'five short codes' => [['foo', 'bar', 'some', 'thing', 'baz'], <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+        Allow: /foo
+        Allow: /bar
+        Allow: /some
+        Allow: /thing
+        Allow: /baz
+        Disallow: /
+        ROBOTS];
+        yield 'no short codes' => [[], <<<ROBOTS
+        # For more information about the robots.txt standard, see:
+        # https://www.robotstxt.org/orig.html
+
+        User-agent: *
+        Disallow: /
+        ROBOTS];
+    }
+}

public/robots.txt

@@ -1,5 +0,0 @@
-# For more information about the robots.txt standard, see:
-# http://www.robotstxt.org/orig.html
-
-User-agent: *
-Disallow: /