Issue #3151364 by Charlie ChX Negyesi, amateescu, alexpott, jhodgdon:...

Issue #3151364 by Charlie ChX Negyesi, amateescu, alexpott, jhodgdon: diacritics are not removed from ǢǣǼǽǮǯ

core/lib/Drupal/Component/Transliteration/PhpTransliteration.php

@@ -58,6 +58,21 @@ class PhpTransliteration implements TransliterationInterface {
    */
   protected $genericMap = [];
 
+  /**
+   * Special characters for ::removeDiacritics().
+   *
+   * Characters which have accented variants but their base character
+   * transliterates to more than one ASCII character require special
+   * treatment: we want to remove their accent and use the un-
+   * transliterated base character.
+   */
+  protected $fixTransliterateForRemoveDiacritics = [
+    'AE' => 'Æ',
+    'ae' => 'æ',
+    'ZH' => 'Ʒ',
+    'zh' => 'ʒ',
+  ];
+
   /**
    * Constructs a transliteration object.
    *
@@ -93,6 +108,9 @@ public function removeDiacritics($string) {
         if (strlen($to_add) === 1) {
           $replacement = $to_add;
         }
+        elseif (isset($this->fixTransliterateForRemoveDiacritics[$to_add])) {
+          $replacement = $this->fixTransliterateForRemoveDiacritics[$to_add];
+        }
       }
 
       $result .= $replacement;

core/modules/search/search.post_update.php

@@ -13,3 +13,14 @@ function search_removed_post_updates() {
     'search_post_update_block_page' => '9.0.0',
   ];
 }
+
+/**
+ * Mark everything for reindexing after diacritics removal rule change.
+ */
+function search_post_update_reindex_after_diacritics_rule_change() {
+  $search_page_repository = \Drupal::service('search.search_page_repository');
+  foreach ($search_page_repository->getIndexableSearchPages() as $entity) {
+    $entity->getPlugin()->markForReindex();
+  }
+  return t("Content has been marked for re-indexing for all active search pages. Searching will continue to work, but new content won't be indexed until all existing content has been re-indexed.");
+}

core/tests/Drupal/Tests/Component/Transliteration/PhpTransliterationTest.php

@@ -59,8 +59,8 @@ public function providerTestPhpTransliterationRemoveDiacritics() {
       // Test all characters in the Unicode range 0x01CD to 0x024F.
       ['ǍǎǏ', 'AaI'],
       ['ǐǑǒǓǔǕǖǗǘǙǚǛǜǝǞǟ', 'iOoUuUuUuUuUuǝAa'],
-      ['ǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯ', 'AaǢǣGgGgKkOoOoǮǯ'],
-      ['ǰDZDzdzǴǵǶǷǸǹǺǻǼǽǾǿ', 'jDZDzdzGgǶǷNnAaǼǽOo'],
+      ['ǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯ', 'AaÆæGgGgKkOoOoƷʒ'],
+      ['ǰDZDzdzǴǵǶǷǸǹǺǻǼǽǾǿ', 'jDZDzdzGgǶǷNnAaÆæOo'],
       ['ȀȁȂȃȄȅȆȇȈȉȊȋȌȍȎȏ', 'AaAaEeEeIiIiOoOo'],
       ['ȐȑȒȓȔȕȖȗȘșȚțȜȝȞȟ', 'RrRrUuUuSsTtȜȝHh'],
       ['ȠȡȢȣȤȥȦȧȨȩȪȫȬȭȮȯ', 'ȠȡȢȣZzAaEeOoOoOo'],