contao · leofeyer · Sep 3, 2019 · Sep 3, 2019 · Sep 3, 2019 · Sep 3, 2019
diff --git a/composer.json b/composer.json
@@ -100,7 +100,8 @@
         "terminal42/service-annotation-bundle": "^1.0",
         "toflar/psr6-symfony-http-cache-store": "^2.0",
         "true/punycode": "^2.1",
-        "twig/twig": "^2.7"
+        "twig/twig": "^2.7",
+        "webignition/robots-txt-file": "^2.1"
     },
     "replace": {
         "contao/calendar-bundle": "self.version",

diff --git a/core-bundle/composer.json b/core-bundle/composer.json
@@ -87,7 +87,8 @@
         "symfony/yaml": "4.2.* || 4.3.*",
         "terminal42/service-annotation-bundle": "^1.0",
         "true/punycode": "^2.1",
-        "twig/twig": "^2.7"
+        "twig/twig": "^2.7",
+        "webignition/robots-txt-file": "^2.1"
     },
     "conflict": {
         "contao-community-alliance/composer-plugin": "<3.0",

diff --git a/core-bundle/src/Controller/FaviconController.php b/core-bundle/src/Controller/FaviconController.php
@@ -0,0 +1,83 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of Contao.
+ *
+ * (c) Leo Feyer
+ *
+ * @license LGPL-3.0-or-later
+ */
+
+namespace Contao\CoreBundle\Controller;
+
+use Contao\CoreBundle\Framework\ContaoFramework;
+use Contao\FilesModel;
+use Contao\PageModel;
+use FOS\HttpCache\ResponseTagger;
+use Symfony\Component\HttpFoundation\BinaryFileResponse;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\Routing\Annotation\Route;
+
+/**
+ * @Route(defaults={"_scope" = "frontend"})
+ */
+class FaviconController
+{
+    /**
+     * @var ContaoFramework
+     */
+    private $contaoFramework;
+
+    /**
+     * @var ResponseTagger|null
+     */
+    private $responseTagger;
+
+    public function __construct(ContaoFramework $contaoFramework, ResponseTagger $responseTagger = null)
+    {
+        $this->contaoFramework = $contaoFramework;
+        $this->responseTagger = $responseTagger;
+    }
+
+    /**
+     * @Route("/favicon.ico", name="contao_favicon")
+     */
+    public function __invoke(Request $request): Response
+    {
+        $this->contaoFramework->initialize();
+
+        /** @var PageModel $pageModel */
+        $pageModel = $this->contaoFramework->getAdapter(PageModel::class);
+
+        /** @var PageModel|null $rootPage */
+        $rootPage = $pageModel->findPublishedFallbackByHostname(
+            $request->server->get('HTTP_HOST'),
+            ['fallbackToEmpty' => true]
+        );
+
+        if (null === $rootPage || null === ($favicon = $rootPage->favicon)) {
+            return new Response('', Response::HTTP_NOT_FOUND);
+        }
+
+        /** @var FilesModel $filesModel */
+        $filesModel = $this->contaoFramework->getAdapter(FilesModel::class);
+        $faviconModel = $filesModel->findByUuid($favicon);
+
+        if (null === $faviconModel) {
+            return new Response('', Response::HTTP_NOT_FOUND);
+        }
+
+        // Cache the response for 1 year and tag it so it is invalidated when the settings are edited
+        $response = new BinaryFileResponse($faviconModel->path);
+        $response->setSharedMaxAge(31556952);
+
+        if (null !== $this->responseTagger) {
+            $this->responseTagger->addTags(['contao.db.tl_page.'.$rootPage->id]);
+        }
+
+        return $response;
+    }
+}
diff --git a/core-bundle/src/Controller/RobotsTxtController.php b/core-bundle/src/Controller/RobotsTxtController.php
@@ -0,0 +1,74 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of Contao.
+ *
+ * (c) Leo Feyer
+ *
+ * @license LGPL-3.0-or-later
+ */
+
+namespace Contao\CoreBundle\Controller;
+
+use Contao\CoreBundle\Event\ContaoCoreEvents;
+use Contao\CoreBundle\Event\RobotsTxtEvent;
+use Contao\CoreBundle\Framework\ContaoFramework;
+use Contao\PageModel;
+use Symfony\Component\EventDispatcher\EventDispatcherInterface;
+use Symfony\Component\HttpFoundation\Request;
+use Symfony\Component\HttpFoundation\Response;
+use Symfony\Component\Routing\Annotation\Route;
+use webignition\RobotsTxt\File\Parser;
+
+/**
+ * @Route(defaults={"_scope" = "frontend"})
+ */
+class RobotsTxtController
+{
+    /**
+     * @var ContaoFramework
+     */
+    private $contaoFramework;
+
+    /**
+     * @var EventDispatcherInterface
+     */
+    private $eventDispatcher;
+
+    public function __construct(ContaoFramework $contaoFramework, EventDispatcherInterface $eventDispatcher)
+    {
+        $this->contaoFramework = $contaoFramework;
+        $this->eventDispatcher = $eventDispatcher;
+    }
+
+    /**
+     * @Route("/robots.txt", name="contao_robots_txt")
+     */
+    public function __invoke(Request $request): Response
+    {
+        $this->contaoFramework->initialize();
+
+        /** @var PageModel $pageModel */
+        $pageModel = $this->contaoFramework->getAdapter(PageModel::class);
+
+        /** @var PageModel|null $rootPage */
+        $rootPage = $pageModel->findPublishedFallbackByHostname(
+            $request->server->get('HTTP_HOST'),
+            ['fallbackToEmpty' => true]
+        );
+
+        if (null === $rootPage) {
+            return new Response('', Response::HTTP_NOT_FOUND);
+        }
+
+        $parser = new Parser();
+        $parser->setSource($rootPage->robotsTxt);
+        $file = $parser->getFile();
+
+        $this->eventDispatcher->dispatch(ContaoCoreEvents::ROBOTS_TXT, new RobotsTxtEvent($file, $request, $rootPage));
+
+        return new Response((string) $file, 200, ['Content-Type' => 'text/plain; charset=UTF-8']);
+    }
+}
diff --git a/core-bundle/src/Event/ContaoCoreEvents.php b/core-bundle/src/Event/ContaoCoreEvents.php
@@ -68,6 +68,15 @@ final class ContaoCoreEvents
      */
     public const PREVIEW_URL_CONVERT = 'contao.preview_url_convert';
 
+    /**
+     * The contao.robots_txt event is triggered when the /robots.txt route is called.
+     *
+     * @var string
+     *
+     * @see RobotsTxtEvent
+     */
+    public const ROBOTS_TXT = 'contao.robots_txt';
+
     /**
      * The contao.slug_valid_characters event is triggered when the valid slug characters options are generated.
      *

diff --git a/core-bundle/src/Event/RobotsTxtEvent.php b/core-bundle/src/Event/RobotsTxtEvent.php
@@ -0,0 +1,58 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of Contao.
+ *
+ * (c) Leo Feyer
+ *
+ * @license LGPL-3.0-or-later
+ */
+
+namespace Contao\CoreBundle\Event;
+
+use Contao\PageModel;
+use Symfony\Component\EventDispatcher\Event;
+use Symfony\Component\HttpFoundation\Request;
+use webignition\RobotsTxt\File\File;
+
+class RobotsTxtEvent extends Event
+{
+    /**
+     * @var File
+     */
+    private $file;
+
+    /**
+     * @var Request
+     */
+    private $request;
+
+    /**
+     * @var PageModel
+     */
+    private $rootPage;
+
+    public function __construct(File $file, Request $request, PageModel $rootPage)
+    {
+        $this->file = $file;
+        $this->request = $request;
+        $this->rootPage = $rootPage;
+    }
+
+    public function getFile(): File
+    {
+        return $this->file;
+    }
+
+    public function getRequest(): Request
+    {
+        return $this->request;
+    }
+
+    public function getRootPage(): PageModel
+    {
+        return $this->rootPage;
+    }
+}
diff --git a/core-bundle/src/EventListener/RobotsTxtListener.php b/core-bundle/src/EventListener/RobotsTxtListener.php
@@ -0,0 +1,85 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of Contao.
+ *
+ * (c) Leo Feyer
+ *
+ * @license LGPL-3.0-or-later
+ */
+
+namespace Contao\CoreBundle\EventListener;
+
+use Contao\CoreBundle\Event\RobotsTxtEvent;
+use Contao\CoreBundle\Framework\ContaoFramework;
+use Contao\PageModel;
+use webignition\RobotsTxt\Directive\Directive;
+use webignition\RobotsTxt\Inspector\Inspector;
+use webignition\RobotsTxt\Record\Record;
+
+class RobotsTxtListener
+{
+    /**
+     * @var ContaoFramework
+     */
+    private $contaoFramework;
+
+    public function __construct(ContaoFramework $contaoFramework)
+    {
+        $this->contaoFramework = $contaoFramework;
+    }
+
+    public function onRobotsTxt(RobotsTxtEvent $event): void
+    {
+        $this->contaoFramework->initialize();
+
+        $file = $event->getFile();
+        $inspector = new Inspector($file);
+
+        // Get all directives for user-agent: *
+        $directiveList = $inspector->getDirectives();
+
+        // If no directive for user-agent: * exists, we add the record
+        if (0 === $directiveList->getLength()) {
+            $record = new Record();
+            $this->addContaoDisallowDirectivesToRecord($record);
+            $file->addRecord($record);
+        }
+
+        $records = $file->getRecords();
+
+        foreach ($records as $record) {
+            $this->addContaoDisallowDirectivesToRecord($record);
+        }
+
+        /** @var PageModel $pageModel */
+        $pageModel = $this->contaoFramework->getAdapter(PageModel::class);
+        $rootPages = $pageModel->findPublishedRootPages(['dns' => $event->getRootPage()->dns]);
+
+        // Generate the sitemaps
+        foreach ($rootPages as $rootPage) {
+            if (!$rootPage->createSitemap) {
+                continue;
+            }
+
+            $sitemap = sprintf(
+                '%s%s/share/%s.xml',
+                $rootPage->useSSL ? 'https://' : 'http://',
+                $rootPage->dns ?: $event->getRequest()->server->get('HTTP_HOST'),
+                $rootPage->sitemapName
+            );
+
+            $event->getFile()->getNonGroupDirectives()->add(new Directive('Sitemap', $sitemap));
+        }
+    }
+
+    private function addContaoDisallowDirectivesToRecord(Record $record): void
+    {
+        $directiveList = $record->getDirectiveList();
+        $directiveList->add(new Directive('Disallow', '/contao$'));
+        $directiveList->add(new Directive('Disallow', '/contao?'));
+        $directiveList->add(new Directive('Disallow', '/contao/'));
+    }
+}
diff --git a/core-bundle/src/Resources/config/listener.yml b/core-bundle/src/Resources/config/listener.yml
@@ -8,6 +8,12 @@ services:
             calls:
                 - ['setContainer', ['@service_container']]
 
+    Contao\CoreBundle\EventListener\RobotsTxtListener:
+        arguments:
+            - '@contao.framework'
+        tags:
+            - { name: kernel.event_listener, event: contao.robots_txt, method: onRobotsTxt }
+
     contao.listener.add_to_search_index:
         class: Contao\CoreBundle\EventListener\AddToSearchIndexListener
         arguments:

diff --git a/core-bundle/src/Resources/config/services.yml b/core-bundle/src/Resources/config/services.yml
@@ -28,6 +28,22 @@ services:
             - { name: container.service_subscriber }
         public: true
 
+    Contao\CoreBundle\Controller\FaviconController:
+        arguments:
+            - '@contao.framework'
+            - '@?fos_http_cache.http.symfony_response_tagger'
+        tags:
+            - controller.service_arguments
+        public: true
+
+    Contao\CoreBundle\Controller\RobotsTxtController:
+        arguments:
+            - '@contao.framework'
+            - '@event_dispatcher'
+        tags:
+            - controller.service_arguments
+        public: true
+
     contao.assets.assets_context:
         class: Contao\CoreBundle\Asset\ContaoContext
         arguments: