Merge pull request #535 from Crozzers/fix-non-ascii-header-ids

Update `_slugify` to use utf-8 encoding (issue #534)

Author: nicholasserra

CHANGES.md

@@ -6,6 +6,7 @@
 - [pull #527] Fix base64 images being corrupted in safe mode (issue #526)
 - [pull #529] Add `breaks` extra with ability to hard break on backslashes (issue #525)
 - [pull #532] Fix #493 persisting when `code-friendly` extra enabled
+- [pull #535] Update `_slugify` to use utf-8 encoding (issue #534)
 
 ## python-markdown2 2.4.10
 

lib/markdown2.py

@@ -2843,7 +2843,7 @@ def _slugify(value):
     From Django's "django/template/defaultfilters.py".
     """
     import unicodedata
-    value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode()
+    value = unicodedata.normalize('NFKD', value).encode('utf-8', 'ignore').decode()
     value = _slugify_strip_re.sub('', value).strip().lower()
     return _slugify_hyphenate_re.sub('-', value)
 ## end of http://code.activestate.com/recipes/577257/ }}}

test/tm-cases/header_ids_4.html

@@ -1,6 +1,6 @@
 <!-- -*- coding: utf-8 -*- -->
 
-<h1 id="fruit-really-likes">Fruit заголовок <em>really</em> likes</h1>
+<h1 id="fruit-заголовок-really-likes">Fruit заголовок <em>really</em> likes</h1>
 
 <ul>
 <li>apples</li>

test/tm-cases/toc_4.html

@@ -1,6 +1,6 @@
 <h1 id="python">Python</h1>
 
-<h2 id="-1">蟒蛇</h2>
+<h2 id="蟒蛇">蟒蛇</h2>
 
 <ul>
 <li>外形特性</li>
@@ -9,7 +9,7 @@ <h2 id="-1">蟒蛇</h2>
 
 <h2 id="markdown">Markdown</h2>
 
-<h2 id="-2">标记语言</h2>
+<h2 id="标记语言">标记语言</h2>
 
 <ul>
 <li>类型</li>

test/tm-cases/toc_4.toc_html

@@ -1,8 +1,8 @@
 <ul>
   <li><a href="#python">Python</a>
   <ul>
-    <li><a href="#-1">蟒蛇</a></li>
+    <li><a href="#蟒蛇">蟒蛇</a></li>
     <li><a href="#markdown">Markdown</a></li>
-    <li><a href="#-2">标记语言</a></li>
+    <li><a href="#标记语言">标记语言</a></li>
   </ul></li>
 </ul>