From 695d2db311d4cf587bc0d63ed2b80b35bae69bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Goetz?= Date: Sun, 29 Dec 2019 20:41:13 +0100 Subject: [PATCH] Use transliteration for slug #183 --- libs/DauxHelper.php | 145 +----------------- .../ContentTypes/Markdown/TOC/Processor.php | 18 +-- tests/Format/HTML/TableOfContentsTest.php | 12 +- tests/Tree/BuilderTest.php | 5 +- 4 files changed, 13 insertions(+), 167 deletions(-) diff --git a/libs/DauxHelper.php b/libs/DauxHelper.php index c902a33..a601ff7 100644 --- a/libs/DauxHelper.php +++ b/libs/DauxHelper.php @@ -224,9 +224,8 @@ class DauxHelper public static function slug($title) { // Convert to ASCII - foreach (static::charsArray() as $key => $value) { - $title = str_replace($value, $key, $title); - } + $title = transliterator_transliterate("Any-Latin; NFD; [:Nonspacing Mark:] Remove; NFC;", $title); + $title = iconv("utf-8", "ASCII//TRANSLIT//IGNORE", $title); // Remove unsupported characters $title = preg_replace('/[^\x20-\x7E]/u', '', $title); @@ -245,146 +244,6 @@ class DauxHelper return trim($title, $separator); } - /** - * Returns the replacements for the slug() method. - * - * Taken from Stringy - * - * @return array An array of replacements. - */ - public static function charsArray() - { - static $charsArray; - - if (isset($charsArray)) { - return $charsArray; - } - - return $charsArray = [ - 'a' => [ - 'à', 'á', 'ả', 'ã', 'ạ', 'ă', 'ắ', 'ằ', 'ẳ', 'ẵ', - 'ặ', 'â', 'ấ', 'ầ', 'ẩ', 'ẫ', 'ậ', 'ä', 'ā', 'ą', - 'å', 'α', 'ά', 'ἀ', 'ἁ', 'ἂ', 'ἃ', 'ἄ', 'ἅ', 'ἆ', - 'ἇ', 'ᾀ', 'ᾁ', 'ᾂ', 'ᾃ', 'ᾄ', 'ᾅ', 'ᾆ', 'ᾇ', 'ὰ', - 'ά', 'ᾰ', 'ᾱ', 'ᾲ', 'ᾳ', 'ᾴ', 'ᾶ', 'ᾷ', 'а', 'أ', ], - 'b' => ['б', 'β', 'Ъ', 'Ь', 'ب'], - 'c' => ['ç', 'ć', 'č', 'ĉ', 'ċ'], - 'd' => ['ď', 'ð', 'đ', 'ƌ', 'ȡ', 'ɖ', 'ɗ', 'ᵭ', 'ᶁ', 'ᶑ', - 'д', 'δ', 'د', 'ض', ], - 'e' => ['é', 'è', 'ẻ', 'ẽ', 'ẹ', 'ê', 'ế', 'ề', 'ể', 'ễ', - 'ệ', 'ë', 'ē', 'ę', 'ě', 'ĕ', 'ė', 'ε', 'έ', 'ἐ', - 'ἑ', 'ἒ', 'ἓ', 'ἔ', 'ἕ', 'ὲ', 'έ', 'е', 'ё', 'э', - 'є', 'ə', ], - 'f' => ['ф', 'φ', 'ف'], - 'g' => ['ĝ', 'ğ', 'ġ', 'ģ', 'г', 'ґ', 'γ', 'ج'], - 'h' => ['ĥ', 'ħ', 'η', 'ή', 'ح', 'ه'], - 'i' => ['í', 'ì', 'ỉ', 'ĩ', 'ị', 'î', 'ï', 'ī', 'ĭ', 'į', - 'ı', 'ι', 'ί', 'ϊ', 'ΐ', 'ἰ', 'ἱ', 'ἲ', 'ἳ', 'ἴ', - 'ἵ', 'ἶ', 'ἷ', 'ὶ', 'ί', 'ῐ', 'ῑ', 'ῒ', 'ΐ', 'ῖ', - 'ῗ', 'і', 'ї', 'и', ], - 'j' => ['ĵ', 'ј', 'Ј'], - 'k' => ['ķ', 'ĸ', 'к', 'κ', 'Ķ', 'ق', 'ك'], - 'l' => ['ł', 'ľ', 'ĺ', 'ļ', 'ŀ', 'л', 'λ', 'ل'], - 'm' => ['м', 'μ', 'م'], - 'n' => ['ñ', 'ń', 'ň', 'ņ', 'ʼn', 'ŋ', 'ν', 'н', 'ن'], - 'o' => ['ó', 'ò', 'ỏ', 'õ', 'ọ', 'ô', 'ố', 'ồ', 'ổ', 'ỗ', - 'ộ', 'ơ', 'ớ', 'ờ', 'ở', 'ỡ', 'ợ', 'ø', 'ō', 'ő', - 'ŏ', 'ο', 'ὀ', 'ὁ', 'ὂ', 'ὃ', 'ὄ', 'ὅ', 'ὸ', 'ό', - 'ö', 'о', 'و', 'θ', ], - 'p' => ['п', 'π'], - 'r' => ['ŕ', 'ř', 'ŗ', 'р', 'ρ', 'ر'], - 's' => ['ś', 'š', 'ş', 'с', 'σ', 'ș', 'ς', 'س', 'ص'], - 't' => ['ť', 'ţ', 'т', 'τ', 'ț', 'ت', 'ط'], - 'u' => ['ú', 'ù', 'ủ', 'ũ', 'ụ', 'ư', 'ứ', 'ừ', 'ử', 'ữ', - 'ự', 'ü', 'û', 'ū', 'ů', 'ű', 'ŭ', 'ų', 'µ', 'у', ], - 'v' => ['в'], - 'w' => ['ŵ', 'ω', 'ώ'], - 'x' => ['χ'], - 'y' => ['ý', 'ỳ', 'ỷ', 'ỹ', 'ỵ', 'ÿ', 'ŷ', 'й', 'ы', 'υ', - 'ϋ', 'ύ', 'ΰ', 'ي', ], - 'z' => ['ź', 'ž', 'ż', 'з', 'ζ', 'ز'], - 'aa' => ['ع'], - 'ae' => ['æ'], - 'ch' => ['ч'], - 'dj' => ['ђ', 'đ'], - 'dz' => ['џ'], - 'gh' => ['غ'], - 'kh' => ['х', 'خ'], - 'lj' => ['љ'], - 'nj' => ['њ'], - 'oe' => ['œ'], - 'ps' => ['ψ'], - 'sh' => ['ш'], - 'shch' => ['щ'], - 'ss' => ['ß'], - 'th' => ['þ', 'ث', 'ذ', 'ظ'], - 'ts' => ['ц'], - 'ya' => ['я'], - 'yu' => ['ю'], - 'zh' => ['ж'], - '(c)' => ['©'], - 'A' => ['Á', 'À', 'Ả', 'Ã', 'Ạ', 'Ă', 'Ắ', 'Ằ', 'Ẳ', 'Ẵ', - 'Ặ', 'Â', 'Ấ', 'Ầ', 'Ẩ', 'Ẫ', 'Ậ', 'Ä', 'Å', 'Ā', - 'Ą', 'Α', 'Ά', 'Ἀ', 'Ἁ', 'Ἂ', 'Ἃ', 'Ἄ', 'Ἅ', 'Ἆ', - 'Ἇ', 'ᾈ', 'ᾉ', 'ᾊ', 'ᾋ', 'ᾌ', 'ᾍ', 'ᾎ', 'ᾏ', 'Ᾰ', - 'Ᾱ', 'Ὰ', 'Ά', 'ᾼ', 'А', ], - 'B' => ['Б', 'Β'], - 'C' => ['Ç', 'Ć', 'Č', 'Ĉ', 'Ċ'], - 'D' => ['Ď', 'Ð', 'Đ', 'Ɖ', 'Ɗ', 'Ƌ', 'ᴅ', 'ᴆ', 'Д', 'Δ'], - 'E' => ['É', 'È', 'Ẻ', 'Ẽ', 'Ẹ', 'Ê', 'Ế', 'Ề', 'Ể', 'Ễ', - 'Ệ', 'Ë', 'Ē', 'Ę', 'Ě', 'Ĕ', 'Ė', 'Ε', 'Έ', 'Ἐ', - 'Ἑ', 'Ἒ', 'Ἓ', 'Ἔ', 'Ἕ', 'Έ', 'Ὲ', 'Е', 'Ё', 'Э', - 'Є', 'Ə', ], - 'F' => ['Ф', 'Φ'], - 'G' => ['Ğ', 'Ġ', 'Ģ', 'Г', 'Ґ', 'Γ'], - 'H' => ['Η', 'Ή'], - 'I' => ['Í', 'Ì', 'Ỉ', 'Ĩ', 'Ị', 'Î', 'Ï', 'Ī', 'Ĭ', 'Į', - 'İ', 'Ι', 'Ί', 'Ϊ', 'Ἰ', 'Ἱ', 'Ἳ', 'Ἴ', 'Ἵ', 'Ἶ', - 'Ἷ', 'Ῐ', 'Ῑ', 'Ὶ', 'Ί', 'И', 'І', 'Ї', ], - 'K' => ['К', 'Κ'], - 'L' => ['Ĺ', 'Ł', 'Л', 'Λ', 'Ļ'], - 'M' => ['М', 'Μ'], - 'N' => ['Ń', 'Ñ', 'Ň', 'Ņ', 'Ŋ', 'Н', 'Ν'], - 'O' => ['Ó', 'Ò', 'Ỏ', 'Õ', 'Ọ', 'Ô', 'Ố', 'Ồ', 'Ổ', 'Ỗ', - 'Ộ', 'Ơ', 'Ớ', 'Ờ', 'Ở', 'Ỡ', 'Ợ', 'Ö', 'Ø', 'Ō', - 'Ő', 'Ŏ', 'Ο', 'Ό', 'Ὀ', 'Ὁ', 'Ὂ', 'Ὃ', 'Ὄ', 'Ὅ', - 'Ὸ', 'Ό', 'О', 'Θ', 'Ө', ], - 'P' => ['П', 'Π'], - 'R' => ['Ř', 'Ŕ', 'Р', 'Ρ'], - 'S' => ['Ş', 'Ŝ', 'Ș', 'Š', 'Ś', 'С', 'Σ'], - 'T' => ['Ť', 'Ţ', 'Ŧ', 'Ț', 'Т', 'Τ'], - 'U' => ['Ú', 'Ù', 'Ủ', 'Ũ', 'Ụ', 'Ư', 'Ứ', 'Ừ', 'Ử', 'Ữ', - 'Ự', 'Û', 'Ü', 'Ū', 'Ů', 'Ű', 'Ŭ', 'Ų', 'У', ], - 'V' => ['В'], - 'W' => ['Ω', 'Ώ'], - 'X' => ['Χ'], - 'Y' => ['Ý', 'Ỳ', 'Ỷ', 'Ỹ', 'Ỵ', 'Ÿ', 'Ῠ', 'Ῡ', 'Ὺ', 'Ύ', - 'Ы', 'Й', 'Υ', 'Ϋ', ], - 'Z' => ['Ź', 'Ž', 'Ż', 'З', 'Ζ'], - 'AE' => ['Æ'], - 'CH' => ['Ч'], - 'DJ' => ['Ђ'], - 'DZ' => ['Џ'], - 'KH' => ['Х'], - 'LJ' => ['Љ'], - 'NJ' => ['Њ'], - 'PS' => ['Ψ'], - 'SH' => ['Ш'], - 'SHCH' => ['Щ'], - 'SS' => ['ẞ'], - 'TH' => ['Þ'], - 'TS' => ['Ц'], - 'YA' => ['Я'], - 'YU' => ['Ю'], - 'ZH' => ['Ж'], - ' ' => ["\xC2\xA0", "\xE2\x80\x80", "\xE2\x80\x81", - "\xE2\x80\x82", "\xE2\x80\x83", "\xE2\x80\x84", - "\xE2\x80\x85", "\xE2\x80\x86", "\xE2\x80\x87", - "\xE2\x80\x88", "\xE2\x80\x89", "\xE2\x80\x8A", - "\xE2\x80\xAF", "\xE2\x81\x9F", "\xE3\x80\x80", ], - ]; - } - /** * @param string $from * @param string $to diff --git a/libs/Format/HTML/ContentTypes/Markdown/TOC/Processor.php b/libs/Format/HTML/ContentTypes/Markdown/TOC/Processor.php index e4db127..0abc9ec 100644 --- a/libs/Format/HTML/ContentTypes/Markdown/TOC/Processor.php +++ b/libs/Format/HTML/ContentTypes/Markdown/TOC/Processor.php @@ -14,6 +14,7 @@ use League\CommonMark\Node\Node; use ReflectionMethod; use Todaymade\Daux\Config; use Todaymade\Daux\ContentTypes\Markdown\TableOfContents; +use Todaymade\Daux\DauxHelper; class Processor { @@ -71,21 +72,6 @@ class Processor } } - /** - * Get an escaped version of the link - * @param string $url - * @return string - */ - protected function escaped($url) { - $url = trim($url); - $url = preg_replace('~[^\\pL0-9_]+~u', '-', $url); - $url = trim($url, "-"); - $url = iconv("utf-8", "ASCII//TRANSLIT//IGNORE", $url); - $url = preg_replace('~[^-a-zA-Z0-9_]+~', '', $url); - - return $url; - } - protected function getUniqueId(Document $document, $proposed) { if ($proposed == "page_") { $proposed = "page_section_" . (count($document->heading_ids) + 1); @@ -138,7 +124,7 @@ class Processor } } - $node->data['attributes']['id'] = $this->getUniqueId($document, 'page_' . $this->escaped($text)); + $node->data['attributes']['id'] = $this->getUniqueId($document, 'page_' . DauxHelper::slug($text)); } /** diff --git a/tests/Format/HTML/TableOfContentsTest.php b/tests/Format/HTML/TableOfContentsTest.php index d2b50c6..e425efd 100644 --- a/tests/Format/HTML/TableOfContentsTest.php +++ b/tests/Format/HTML/TableOfContentsTest.php @@ -57,14 +57,14 @@ EXPECTED; $expected = <<
  • -

    基础操作

    +

    基础操作

  • -

    操作基础

    +

    操作基础

    • -

    基础操作

    -

    操作基础

    +

    基础操作

    +

    操作基础

    EXPECTED; @@ -99,10 +99,10 @@ EXPECTED; $expected = <<
  • -

    TEST : Test

    +

    TEST : Test

    • -

    TEST : Test

    +

    TEST : Test

    EXPECTED; diff --git a/tests/Tree/BuilderTest.php b/tests/Tree/BuilderTest.php index c667781..ec5f682 100644 --- a/tests/Tree/BuilderTest.php +++ b/tests/Tree/BuilderTest.php @@ -85,7 +85,7 @@ class BuilderTest extends TestCase // File, Url, Uri, Title ['A Page.md', 'dir/A_Page.html', 'A_Page.html', 'A Page'], ['Page#1.md', 'dir/Page1.html', 'Page1.html', 'Page#1'], - ['你好世界.md', 'dir/你好世界.html', '你好世界.html', '你好世界'] + ['你好世界.md', 'dir/ni_hao_shi_jie.html', 'ni_hao_shi_jie.html', '你好世界'] ]; } @@ -163,6 +163,7 @@ class BuilderTest extends TestCase $structure = [ 'Page.md' => 'another page', 'Button.md' => 'another page', + '你好世界.md' => 'another page', '22.png' => '' ]; $root = vfsStream::setup('root', null, $structure); @@ -176,7 +177,7 @@ class BuilderTest extends TestCase Builder::build($tree, []); $this->assertEquals( - ['22.png', 'Button.html', 'Page.html'], + ['22.png', 'Button.html', 'Page.html', 'ni_hao_shi_jie.html'], array_keys($tree->getEntries()) ); }