Issue #3086075 by jhodgdon, andypost, Charlie ChX Negyesi, Spokje: Use Twig to...

Issue #3086075 by jhodgdon, andypost, Charlie ChX Negyesi, Spokje: Use Twig to strip Twig syntax from help topics files in the syntax checker

(cherry picked from commit 79c009db)

core/modules/help_topics/tests/modules/help_topics_test/bad_help_topics/syntax/bad_help_topics.locale.html.twig

+---
+label: 'Help topic with locale-unsafe tag'
+top_level: true
+---
+<p>{% trans %}some translated text and a <script>alert('hello')</script>{% endtrans %}</p>

core/modules/help_topics/tests/modules/help_topics_test/bad_help_topics/syntax/bad_help_topics.translated.html.twig

@@ -3,3 +3,4 @@ label: 'Help topic with untranslated text'
 top_level: true
 ---
 <p>Body goes here</p>
+<p>{% trans %}some translated text too{% endtrans %}</p>

core/modules/help_topics/tests/modules/help_topics_twig_tester/help_topics_twig_tester.info.yml

+name: 'Help Topics Twig Tester'
+type: module
+description: 'Support module for help testing.'
+package: Testing
+dependencies:
+  - drupal:help_topics

core/modules/help_topics/tests/modules/help_topics_twig_tester/help_topics_twig_tester.services.yml

+services:
+  help_test_twig.extension:
+    class: Drupal\help_topics_twig_tester\HelpTestTwigExtension
+    arguments: []
+    tags:
+      - { name: twig.extension, priority: 500 }

core/modules/help_topics/tests/modules/help_topics_twig_tester/src/HelpTestTwigExtension.php

+<?php
+
+namespace Drupal\help_topics_twig_tester;
+
+use Twig\Extension\AbstractExtension;
+
+/**
+ * Defines and registers Drupal Twig extensions for testing help topics.
+ */
+class HelpTestTwigExtension extends AbstractExtension {
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getNodeVisitors() {
+    return [
+      new HelpTestTwigNodeVisitor(),
+    ];
+  }
+
+}

core/modules/help_topics/tests/modules/help_topics_twig_tester/src/HelpTestTwigNodeVisitor.php

+<?php
+
+namespace Drupal\help_topics_twig_tester;
+
+use Drupal\Core\Template\TwigNodeTrans;
+use Twig\Environment;
+use Twig\Node\Node;
+use Twig\Node\PrintNode;
+use Twig\Node\SetNode;
+use Twig\Node\TextNode;
+use Twig\Node\Expression\AbstractExpression;
+use Twig\NodeVisitor\AbstractNodeVisitor;
+
+/**
+ * Defines a Twig node visitor for testing help topics.
+ *
+ * See static::setStateValue() for information on the special processing
+ * this class can do.
+ */
+class HelpTestTwigNodeVisitor extends AbstractNodeVisitor {
+
+  /**
+   * Delimiter placed around single translated chunks.
+   */
+  public const DELIMITER = 'Not Likely To Be Inside A Template';
+
+  /**
+   * Name used in \Drupal::state() for saving state information.
+   */
+  protected const STATE_NAME = 'help_test_twig_node_visitor';
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function doEnterNode(Node $node, Environment $env) {
+    return $node;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  protected function doLeaveNode(Node $node, Environment $env) {
+    $processing = static::getState();
+    if (!$processing['manner']) {
+      return $node;
+    }
+
+    // For all special processing, we want to remove variables, set statements,
+    // and assorted Twig expression calls (if, do, etc.).
+    if ($node instanceof SetNode || $node instanceof PrintNode ||
+       $node instanceof AbstractExpression) {
+      return NULL;
+    }
+
+    if ($node instanceof TwigNodeTrans) {
+      // Count the number of translated chunks.
+      $this_chunk = $processing['chunk_count'] + 1;
+      static::setStateValue('chunk_count', $this_chunk);
+      if ($this_chunk > $processing['max_chunk']) {
+        static::setStateValue('max_chunk', $this_chunk);
+      }
+
+      if ($processing['manner'] == 'remove_translated') {
+        // Remove all translated text.
+        return NULL;
+      }
+      elseif ($processing['manner'] == 'replace_translated') {
+        // Replace with a dummy string.
+        $node = new TextNode('dummy', 0);
+      }
+      elseif ($processing['manner'] == 'translated_chunk') {
+        // Return the text only if it's the next chunk we're supposed to return.
+        // Add a wrapper, because non-translated nodes will still be returned.
+        if ($this_chunk == $processing['return_chunk']) {
+          return new TextNode(static::DELIMITER . $this->extractText($node) . static::DELIMITER, 0);
+        }
+        else {
+          return NULL;
+        }
+      }
+    }
+
+    if ($processing['manner'] == 'remove_translated' && $node instanceof TextNode) {
+      // For this processing, we also want to remove all HTML tags and
+      // whitespace from TextNodes.
+      $text = $node->getAttribute('data');
+      $text = strip_tags($text);
+      $text = preg_replace('|\s+|', '', $text);
+      return new TextNode($text, 0);
+    }
+
+    return $node;
+  }
+
+  /**
+   * {@inheritdoc}
+   */
+  public function getPriority() {
+    return -100;
+  }
+
+  /**
+   * Extracts the text from a translated text object.
+   *
+   * @param \Drupal\Core\Template\TwigNodeTrans $node
+   *   Translated text node.
+   *
+   * @return string
+   *   Text in the node.
+   */
+  protected function extractText(TwigNodeTrans $node) {
+    // Extract the singular/body and optional plural text from the
+    // TwigNodeTrans object.
+    $bodies = $node->getNode('body');
+    if (!count($bodies)) {
+      $bodies = [$bodies];
+    }
+    if ($node->hasNode('plural')) {
+      $plural = $node->getNode('plural');
+      if (!count($plural)) {
+        $bodies[] = $plural;
+      }
+      else {
+        foreach ($plural as $item) {
+          $bodies[] = $item;
+        }
+      }
+    }
+
+    // Extract the text from each component of the singular/plural strings.
+    $text = '';
+    foreach ($bodies as $body) {
+      if ($body->hasAttribute('data')) {
+        $text .= $body->getAttribute('data');
+      }
+    }
+    return trim($text);
+  }
+
+  /**
+   * Returns the state information.
+   *
+   * @return array
+   *   The state information.
+   */
+  public static function getState() {
+    return \Drupal::state()->get(static::STATE_NAME, ['manner' => 0]);
+  }
+
+  /**
+   * Sets state information.
+   *
+   * @param string $key
+   *   Key to set. Possible keys:
+   *   - manner: Type of special processing to do when rendering. Values:
+   *     - 0: No processing.
+   *     - remove_translated: Remove all translated text, HTML tags, and
+   *       whitespace.
+   *     - replace_translated: Replace all translated text with dummy text.
+   *     - translated_chunk: Remove all translated text except one designated
+   *       chunk (see return_chunk below).
+   *     - bare_body (or any other non-zero value): Remove variables, set
+   *       statements, and Twig programming, but leave everything else intact.
+   *   - chunk_count: Current index of translated chunks. Reset to -1 before
+   *     each rendering run. (Used internally by this class.)
+   *   - max_chunk: Maximum index of translated chunks. Reset to -1 before
+   *     each rendering run.
+   *   - return_chunk: Chunk index to keep intact for translated_chunk
+   *     processing. All others are removed.
+   * @param $value
+   *   Value to set for $key.
+   */
+  public static function setStateValue(string $key, $value) {
+    $state = \Drupal::state();
+    $values = $state->get(static::STATE_NAME, ['manner' => 0]);
+    $values[$key] = $value;
+    $state->set(static::STATE_NAME, $values);
+  }
+
+}

core/modules/help_topics/tests/src/Functional/HelpTopicsSyntaxTest.php

@@ -3,8 +3,10 @@
 namespace Drupal\Tests\help_topics\Functional;
 
 use Drupal\Core\Extension\ExtensionLifecycle;
+use Drupal\Component\FrontMatter\FrontMatter;
 use Drupal\Tests\BrowserTestBase;
 use Drupal\help_topics\HelpTopicDiscovery;
+use Drupal\help_topics_twig_tester\HelpTestTwigNodeVisitor;
 use PHPUnit\Framework\ExpectationFailedException;
 use PHPUnit\Framework\AssertionFailedError;
 
@@ -27,6 +29,7 @@ class HelpTopicsSyntaxTest extends BrowserTestBase {
   protected static $modules = [
     'help',
     'help_topics',
+    'help_topics_twig_tester',
     'locale',
   ];
 
@@ -98,6 +101,7 @@ public function testHelpTopics() {
    */
   protected function verifyTopic($id, $definitions, $response = 200) {
     $definition = $definitions[$id];
+    HelpTestTwigNodeVisitor::setStateValue('manner', 0);
 
     // Visit the URL for the topic.
     $this->drupalGet('admin/help/topic/' . $id);
@@ -114,7 +118,7 @@ protected function verifyTopic($id, $definitions, $response = 200) {
     $has_top_level_related = FALSE;
     if (isset($definition['related'])) {
       foreach ($definition['related'] as $related_id) {
-        $this->assertArrayHasKey($related_id, $definitions, 'Topic ' . $id . ' is only related to topics that exist (' . $related_id . ')');
+        $this->assertArrayHasKey($related_id, $definitions, 'Topic ' . $id . ' is only related to topics that exist: ' . $related_id);
         $has_top_level_related = $has_top_level_related || !empty($definitions[$related_id]['top_level']);
       }
     }
@@ -125,42 +129,56 @@ protected function verifyTopic($id, $definitions, $response = 200) {
     // Verify that the label is not empty.
     $this->assertNotEmpty($definition['label'], 'Topic ' . $id . ' has a non-empty label');
 
-    // Read in the file so we can run some tests on that.
-    $body = file_get_contents($definition[HelpTopicDiscovery::FILE_KEY]);
-    $this->assertNotEmpty($body, 'Topic ' . $id . ' has a non-empty Twig file');
+    // Test the syntax and contents of the Twig file (without the front
+    // matter, which is tested in other ways above). We need to render the
+    // template several times with variations, so read it in once.
+    $template = file_get_contents($definition[HelpTopicDiscovery::FILE_KEY]);
+    $template_text = FrontMatter::create($template)->getContent();
 
-    // Remove the front matter data (already tested above), and Twig set and
-    // variable printouts from the file.
-    $body = preg_replace('|---.*---|sU', '', $body);
-    $body = preg_replace('|\{\{.*\}\}|sU', '', $body);
-    $body = preg_replace('|\{\% set.*\%\}|sU', '', $body);
-    $body = preg_replace('|\{\% endset \%\}|sU', '', $body);
-    $body = trim($body);
-    $this->assertNotEmpty($body, 'Topic ' . $id . ' Twig file contains some text outside of front matter');
-
-    // Verify that if we remove all the translated text, whitespace, and
-    // HTML tags, there is nothing left (that is, all text is translated).
-    $text = preg_replace('|\{\% trans \%\}.*\{\% endtrans \%\}|sU', '', $body);
-    $text = strip_tags($text);
-    $text = preg_replace('|\s+|', '', $text);
-    $this->assertEmpty($text, 'Topic ' . $id . ' Twig file has all of its text translated');
-
-    // Verify that all of the translated text is locale-safe and valid HTML.
-    $matches = [];
-    preg_match_all('|\{\% trans \%\}(.*)\{\% endtrans \%\}|sU', $body, $matches, PREG_PATTERN_ORDER);
-    foreach ($matches[1] as $string) {
-      $this->assertTrue(locale_string_is_safe($string), 'Topic ' . $id . ' Twig file translatable strings are all exportable');
-      $this->validateHtml($string, $id);
+    // Verify that the body is not empty and is valid HTML.
+    $text = $this->renderHelpTopic($template_text, 'bare_body');
+    $this->assertNotEmpty($text, 'Topic ' . $id . ' contains some text outside of front matter');
+    $this->validateHtml($text, $id);
+    $max_chunk_num = HelpTestTwigNodeVisitor::getState()['max_chunk'];
+    $this->assertTrue($max_chunk_num >= 0, 'Topic ' . $id . ' has at least one translated chunk');
+
+    // Verify that each chunk of the translated text is locale-safe and
+    // valid HTML.
+    $chunk_num = 0;
+    $number_checked = 0;
+    while ($chunk_num <= $max_chunk_num) {
+      $chunk_str = $id . ' section ' . $chunk_num;
+
+      // Render the topic, asking for just one chunk, and extract the chunk.
+      // Note that some chunks may not actually get rendered, if they are inside
+      // set statements, because we skip rendering variable output.
+      HelpTestTwigNodeVisitor::setStateValue('return_chunk', $chunk_num);
+      $text = $this->renderHelpTopic($template_text, 'translated_chunk');
+      $matches = [];
+      $matched = preg_match('|' . HelpTestTwigNodeVisitor::DELIMITER . '(.*)' . HelpTestTwigNodeVisitor::DELIMITER . '|', $text, $matches);
+      if ($matched) {
+        $number_checked++;
+        $text = $matches[1];
+        $this->assertNotEmpty($text, 'Topic ' . $chunk_str . ' contains text');
+
+        // Verify the chunk is OK.
+        $this->assertTrue(locale_string_is_safe($text), 'Topic ' . $chunk_str . ' translatable string is locale-safe');
+        $this->validateHtml($text, $chunk_str);
+      }
+      $chunk_num++;
     }
-
-    // Validate the HTML in the body as a whole.
-    $this->validateHtml($body, $id);
+    $this->assertTrue($number_checked > 0, 'Tested at least one translated chunk in ' . $id);
 
     // Validate the HTML in the body with the translated text replaced by a
     // dummy string, to verify that HTML syntax is not partly in and partly out
     // of the translated text.
-    $text = preg_replace('|\{\% trans \%\}.*\{\% endtrans \%\}|sU', 'dummy', $body);
+    $text = $this->renderHelpTopic($template_text, 'replace_translated');
     $this->validateHtml($text, $id);
+
+    // Verify that if we remove all the translated text, whitespace, and
+    // HTML tags, there is nothing left (that is, all text is translated).
+    $text = preg_replace('|\s+|', '', $this->renderHelpTopic($template_text, 'remove_translated'));
+    $this->assertEmpty($text, 'Topic ' . $id . ' Twig file has all of its text translated');
   }
 
   /**
@@ -245,6 +263,10 @@ protected function verifyBadTopic($id, $definitions) {
           $this->assertStringContainsString('Twig file has all of its text translated', $message);
           break;
 
+        case 'locale':
+          $this->assertStringContainsString('translatable string is locale-safe', $message);
+          break;
+
         case 'h1':
           $this->assertStringContainsString('has no H1 tag', $message);
           break;
@@ -297,4 +319,30 @@ protected function listDirectories($type) {
     return $directories;
   }
 
+  /**
+   * Renders a help topic in a special manner.
+   *
+   * @param string $content
+   *   Template text, without the front matter.
+   * @param string $manner
+   *   The special processing choice for topic rendering.
+   *
+   * @return string
+   *   The rendered topic.
+   */
+  protected function renderHelpTopic(string $content, string $manner) {
+    // Set up the special state variables for rendering.
+    HelpTestTwigNodeVisitor::setStateValue('manner', $manner);
+    HelpTestTwigNodeVisitor::setStateValue('max_chunk', -1);
+    HelpTestTwigNodeVisitor::setStateValue('chunk_count', -1);
+
+    // Add a random comment to the end, to thwart caching, and render. We need
+    // the HelpTestTwigNodeVisitor class to hit it each time we render.
+    $build = [
+      '#type' => 'inline_template',
+      '#template' => $content . "\n{# " . rand() . " #}",
+    ];
+    return (string) \Drupal::service('renderer')->renderPlain($build);
+  }
+
 }