From 019abb75d81351ad2c2969fed5558d8e110637a1 Mon Sep 17 00:00:00 2001 From: Daniel Kirkland Date: Thu, 22 Oct 2026 04:40:11 -0700 Subject: [PATCH 1/4] docs: Fix robots.txt extension examples Co-Authored-By: Claude Sonnet 5.5 --- .../plugins/content/seo/extend-robots-txt.md | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index d7eec4d575..edbafd0ded 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -53,9 +53,10 @@ namespace Swag\Example\Listener; use Psr\Log\LoggerInterface; use Shopware\Core\Framework\Log\Package; use Shopware\Storefront\Page\Robots\Event\RobotsDirectiveParsingEvent; -use Shopware\Storefront\Page\Robots\ValueObject\RobotsDirective; -use Shopware\Storefront\Page\Robots\ValueObject\RobotsDirectiveType; -use Shopware\Storefront\Page\Robots\ValueObject\RobotsUserAgentBlock; +use Shopware\Storefront\Page\Robots\Struct\RobotsDirective; +use Shopware\Storefront\Page\Robots\Struct\RobotsDirectiveType; +use Shopware\Storefront\Page\Robots\Struct\RobotsUserAgentBlock; +use Shopware\Storefront\Page\Robots\Parser\ParsedRobots; #[Package('storefront')] class RobotsExtensionListener @@ -67,22 +68,29 @@ class RobotsExtensionListener public function __invoke(RobotsDirectiveParsingEvent $event): void { - $parsedRobots = $event->getParsedRobots(); + $parsedRobots = $event->parsedResult; // Add restrictions for AI crawlers $aiCrawlers = ['GPTBot', 'ChatGPT-User', 'CCBot', 'anthropic-ai']; - $aiBlock = new RobotsUserAgentBlock( - userAgents: $aiCrawlers, - directives: [ - new RobotsDirective( - type: RobotsDirectiveType::DISALLOW, - value: '/checkout/', - ), - ], - ); + $aiBlocks = []; + foreach ($aiCrawlers as $userAgent) { + $aiBlocks[] = new RobotsUserAgentBlock( + userAgent: $userAgent, + directives: [ + new RobotsDirective( + type: RobotsDirectiveType::DISALLOW, + value: '/checkout/', + ), + ], + ); + } - $parsedRobots->addUserAgentBlock($aiBlock); + $event->parsedResult = new ParsedRobots( + userAgentBlocks: [...$parsedRobots->userAgentBlocks, ...$aiBlocks], + orphanedPathDirectives: $parsedRobots->orphanedPathDirectives, + issues: $parsedRobots->issues, + ); $this->logger->info('Extended robots.txt with AI crawler rules'); } @@ -184,14 +192,14 @@ use Shopware\Core\Framework\Log\Package; use Shopware\Storefront\Page\Robots\Event\RobotsDirectiveParsingEvent; use Shopware\Storefront\Page\Robots\Parser\ParseIssue; use Shopware\Storefront\Page\Robots\Parser\ParseIssueSeverity; -use Shopware\Storefront\Page\Robots\ValueObject\RobotsDirectiveType; +use Shopware\Storefront\Page\Robots\Struct\RobotsDirectiveType; #[Package('storefront')] class RobotsValidationListener { public function __invoke(RobotsDirectiveParsingEvent $event): void { - $parsedRobots = $event->getParsedRobots(); + $parsedRobots = $event->parsedResult; // Validate crawl-delay values foreach ($parsedRobots->getUserAgentBlocks() as $block) { From f036caa64011ab9fd884c3dc230b0832083e6ce6 Mon Sep 17 00:00:00 2001 From: Daniel Kirkland Date: Thu, 22 Oct 2026 05:52:48 -0700 Subject: [PATCH 2/4] docs: fix robots.txt event examples Co-Authored-By: Claude Sonnet 5.5 --- .../plugins/content/seo/extend-robots-txt.md | 91 +++++++++++++------ 1 file changed, 65 insertions(+), 26 deletions(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index edbafd0ded..ec5ddfe16b 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -147,8 +147,8 @@ class CustomDirectiveListener // Support Google and Yandex specific directives $knownCustomDirectives = ['noimageindex', 'noarchive', 'clean-param']; - if (in_array(strtolower($event->getDirectiveName()), $knownCustomDirectives, true)) { - $event->setHandled(true); // Prevent "unknown directive" warning + if (in_array(strtolower($event->directiveType), $knownCustomDirectives, true)) { + $event->handled = true; // Prevent "unknown directive" warning } } } @@ -192,6 +192,7 @@ use Shopware\Core\Framework\Log\Package; use Shopware\Storefront\Page\Robots\Event\RobotsDirectiveParsingEvent; use Shopware\Storefront\Page\Robots\Parser\ParseIssue; use Shopware\Storefront\Page\Robots\Parser\ParseIssueSeverity; +use Shopware\Storefront\Page\Robots\Parser\ParsedRobots; use Shopware\Storefront\Page\Robots\Struct\RobotsDirectiveType; #[Package('storefront')] @@ -200,57 +201,95 @@ class RobotsValidationListener public function __invoke(RobotsDirectiveParsingEvent $event): void { $parsedRobots = $event->parsedResult; + $issues = $parsedRobots->issues; + $lineNumbers = []; + + foreach (explode("\n", $event->text) as $lineIndex => $line) { + $parts = explode(':', trim($line), 2); + if (count($parts) !== 2) { + continue; + } + + $type = RobotsDirectiveType::tryFromInsensitive(trim($parts[0])); + if ($type !== null) { + $key = $type->value . ': ' . trim($parts[1]); + $lineNumbers[$key][] = [$lineIndex + 1, trim($line)]; + } + } + + $lineFor = static function (RobotsDirectiveType $type, string $value) use (&$lineNumbers): array { + $key = $type->value . ': ' . $value; + $matchingLines = $lineNumbers[$key] ?? []; + if ($matchingLines === []) { + return [0, '']; + } + + $line = array_shift($matchingLines); + $lineNumbers[$key] = $matchingLines; + + return $line; + }; // Validate crawl-delay values - foreach ($parsedRobots->getUserAgentBlocks() as $block) { - foreach ($block->getDirectives() as $directive) { - if ($directive->getType() === RobotsDirectiveType::CRAWL_DELAY) { - $value = (int) $directive->getValue(); + foreach ($parsedRobots->userAgentBlocks as $block) { + foreach ($block->directives as $directive) { + if ($directive->type === RobotsDirectiveType::CRAWL_DELAY) { + $value = (int) $directive->value; + [$lineNumber, $lineContent] = $lineFor($directive->type, $directive->value); if ($value <= 0) { - $event->addIssue(new ParseIssue( + $issues[] = new ParseIssue( severity: ParseIssueSeverity::ERROR, - message: 'Invalid crawl-delay value: must be a positive integer', - lineNumber: null, - )); + lineNumber: $lineNumber, + lineContent: $lineContent, + reason: 'Invalid crawl-delay value: must be a positive integer', + ); } if ($value > 10) { - $event->addIssue(new ParseIssue( + $issues[] = new ParseIssue( severity: ParseIssueSeverity::WARNING, - message: 'Crawl-delay value is very high. This may significantly slow down indexing.', - lineNumber: null, - )); + lineNumber: $lineNumber, + lineContent: $lineContent, + reason: 'Crawl-delay value is very high. This may significantly slow down indexing.', + ); } } } } // Check for conflicting Allow/Disallow directives - foreach ($parsedRobots->getUserAgentBlocks() as $block) { + foreach ($parsedRobots->userAgentBlocks as $block) { $disallowed = []; $allowed = []; - foreach ($block->getDirectives() as $directive) { - if ($directive->getType() === RobotsDirectiveType::DISALLOW) { - $disallowed[] = $directive->getValue(); - } elseif ($directive->getType() === RobotsDirectiveType::ALLOW) { - $allowed[] = $directive->getValue(); + foreach ($block->directives as $directive) { + if ($directive->type === RobotsDirectiveType::DISALLOW) { + $disallowed[] = [$directive->value, ...$lineFor($directive->type, $directive->value)]; + } elseif ($directive->type === RobotsDirectiveType::ALLOW) { + $allowed[] = [$directive->value, ...$lineFor($directive->type, $directive->value)]; } } - foreach ($allowed as $allowPath) { - foreach ($disallowed as $disallowPath) { + foreach ($allowed as [$allowPath, $allowLineNumber, $allowLineContent]) { + foreach ($disallowed as [$disallowPath]) { if ($allowPath === $disallowPath) { - $event->addIssue(new ParseIssue( + $issues[] = new ParseIssue( severity: ParseIssueSeverity::WARNING, - message: sprintf('Conflicting directives: Path "%s" is both allowed and disallowed', $allowPath), - lineNumber: null, - )); + lineNumber: $allowLineNumber, + lineContent: $allowLineContent, + reason: sprintf('Conflicting directives: Path "%s" is both allowed and disallowed', $allowPath), + ); } } } } + + $event->parsedResult = new ParsedRobots( + userAgentBlocks: $parsedRobots->userAgentBlocks, + orphanedPathDirectives: $parsedRobots->orphanedPathDirectives, + issues: $issues, + ); } } ``` From 1a5560b9991b2d80bd6f688541b987578d7d5487 Mon Sep 17 00:00:00 2001 From: Daniel Kirkland Date: Thu, 22 Oct 2026 05:53:40 -0700 Subject: [PATCH 3/4] docs: Simplify the robots.txt validation example Replace the line-number reconstruction with the plain issue fields, so the example stays short and matches the source. Co-Authored-By: Claude Sonnet 5.5 --- .../plugins/content/seo/extend-robots-txt.md | 54 +++++-------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index ec5ddfe16b..769303d45d 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -202,56 +202,28 @@ class RobotsValidationListener { $parsedRobots = $event->parsedResult; $issues = $parsedRobots->issues; - $lineNumbers = []; - - foreach (explode("\n", $event->text) as $lineIndex => $line) { - $parts = explode(':', trim($line), 2); - if (count($parts) !== 2) { - continue; - } - - $type = RobotsDirectiveType::tryFromInsensitive(trim($parts[0])); - if ($type !== null) { - $key = $type->value . ': ' . trim($parts[1]); - $lineNumbers[$key][] = [$lineIndex + 1, trim($line)]; - } - } - - $lineFor = static function (RobotsDirectiveType $type, string $value) use (&$lineNumbers): array { - $key = $type->value . ': ' . $value; - $matchingLines = $lineNumbers[$key] ?? []; - if ($matchingLines === []) { - return [0, '']; - } - - $line = array_shift($matchingLines); - $lineNumbers[$key] = $matchingLines; - - return $line; - }; // Validate crawl-delay values foreach ($parsedRobots->userAgentBlocks as $block) { foreach ($block->directives as $directive) { if ($directive->type === RobotsDirectiveType::CRAWL_DELAY) { $value = (int) $directive->value; - [$lineNumber, $lineContent] = $lineFor($directive->type, $directive->value); if ($value <= 0) { $issues[] = new ParseIssue( - severity: ParseIssueSeverity::ERROR, - lineNumber: $lineNumber, - lineContent: $lineContent, + lineNumber: 0, + lineContent: $directive->render(), reason: 'Invalid crawl-delay value: must be a positive integer', + severity: ParseIssueSeverity::ERROR, ); } if ($value > 10) { $issues[] = new ParseIssue( - severity: ParseIssueSeverity::WARNING, - lineNumber: $lineNumber, - lineContent: $lineContent, + lineNumber: 0, + lineContent: $directive->render(), reason: 'Crawl-delay value is very high. This may significantly slow down indexing.', + severity: ParseIssueSeverity::WARNING, ); } } @@ -265,20 +237,20 @@ class RobotsValidationListener foreach ($block->directives as $directive) { if ($directive->type === RobotsDirectiveType::DISALLOW) { - $disallowed[] = [$directive->value, ...$lineFor($directive->type, $directive->value)]; + $disallowed[] = $directive->value; } elseif ($directive->type === RobotsDirectiveType::ALLOW) { - $allowed[] = [$directive->value, ...$lineFor($directive->type, $directive->value)]; + $allowed[] = $directive->value; } } - foreach ($allowed as [$allowPath, $allowLineNumber, $allowLineContent]) { - foreach ($disallowed as [$disallowPath]) { + foreach ($allowed as $allowPath) { + foreach ($disallowed as $disallowPath) { if ($allowPath === $disallowPath) { $issues[] = new ParseIssue( - severity: ParseIssueSeverity::WARNING, - lineNumber: $allowLineNumber, - lineContent: $allowLineContent, + lineNumber: 0, + lineContent: $allowPath, reason: sprintf('Conflicting directives: Path "%s" is both allowed and disallowed', $allowPath), + severity: ParseIssueSeverity::WARNING, ); } } From da44ddd051a51d0cf7290afe8a7fde8fc925f577 Mon Sep 17 00:00:00 2001 From: Su <112690947+sushmangupta@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:36:40 +0200 Subject: [PATCH 4/4] Apply suggestion from @nfortier-shopware Co-authored-by: Nicolas Fortier <164332867+nfortier-shopware@users.noreply.github.com> --- guides/plugins/plugins/content/seo/extend-robots-txt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guides/plugins/plugins/content/seo/extend-robots-txt.md b/guides/plugins/plugins/content/seo/extend-robots-txt.md index 769303d45d..f491e8dfef 100644 --- a/guides/plugins/plugins/content/seo/extend-robots-txt.md +++ b/guides/plugins/plugins/content/seo/extend-robots-txt.md @@ -53,10 +53,10 @@ namespace Swag\Example\Listener; use Psr\Log\LoggerInterface; use Shopware\Core\Framework\Log\Package; use Shopware\Storefront\Page\Robots\Event\RobotsDirectiveParsingEvent; +use Shopware\Storefront\Page\Robots\Parser\ParsedRobots; use Shopware\Storefront\Page\Robots\Struct\RobotsDirective; use Shopware\Storefront\Page\Robots\Struct\RobotsDirectiveType; use Shopware\Storefront\Page\Robots\Struct\RobotsUserAgentBlock; -use Shopware\Storefront\Page\Robots\Parser\ParsedRobots; #[Package('storefront')] class RobotsExtensionListener