LgKd3rcc

1#!/usr/bin/python
2# -*- coding: utf-8 -*-
3
4# This source code is distributed under GNU GPL v2 license
5# written by Victor Stinner <victor.stinner AT gmail.com>
6# creatied: 2006-08-14 -- last change: 2007-08-17
7
8# Convert any unicode string to ASCII string:
9#  - Remove diacriticals
10#  - Replace special letter with similar ASCII character (similar glyph)
11#
12# Support greek, cyrillic, some latin letters and some signs.
13
14from unicodedata import normalize
15
16UNICODE_TO_ASCII = {
17    # Latin letters
18    u"Æ": u"AE",   # U+00C6 (latin capital ligature ae)
19    u"Ø": u"O",    # U+00D8 (latin capital letter o with stroke)
20    u"ß": u"ss",   # U+00DF (latin small letter sharp s)
21    u"æ": u"ae",   # U+00E6 (latin small ligature ae)
22    u"ø": u"o",    # U+00F8 (latin small letter o with stroke)
23    u"Å‚": u"l",    # U+0142 (latin small letter l with stroke)
24    u"Å’": u"OE",   # U+0152 (latin capital ligature oe)
25    u"Å“": u"oe",   # U+0153 (latin small ligature oe)
26
27    # Various signs
28    u"¡": u"!",    # U+00A1 (inverted exclamation mark)
29    u"©": u"(c)",  # U+00A9 (copyright sign)
30    u"«": u'"',    # U+00AB (left-pointing double angle quotation mark)
31    u"®": u"(r)",  # U+00AE (registred sign)
32    u"²": u"2",    # U+00B2 (superscript two)
33    u"»": u'"',    # U+00BB (right-pointing double angle quotation mark)
34    u"⁄": u"/",    # U+2044 (fraction slash)
35
36    # Greek
37    u"Α": u"A",    # U+0391 (capital alpha)
38    u"Î’": u"B",    # U+0392 (capital beta)
39    u"Ε": u"E",    # U+0395 (capital epsilon)
40    u"Ζ": u"Z",    # U+0396 (capital zeta)
41    u"Η": u"H",    # U+0397 (capital eta)
42    u"Θ": u"O",    # U+0398 (captial theta)
43    u"ÃŽâ„¢": u"I",    # U+0399 (capital iota)
44    u"ÃŽÅ¡": u"K",    # U+039A (capital kappa)
45    u"ÃŽÅ“": u"M",    # U+039C (capital mu)
46    u"Ν": u"N",    # U+039D (capital nu)
47    u"Ο": u"O",    # U+039F (capital omicron)
48    u"Ρ": u"P",    # U+03A1 (capital rho)
49    u"Τ": u"T",    # U+03A4 (capital tau)
50    u"ÃŽÂ¥": u"Y",    # U+03A5 (capital upsilon)
51    u"Χ": u"X",    # U+03A7 (capital chi)
52    u"α": u"a",    # U+03B1 (small alpha)
53    u"β": u"b",    # U+03B2 (small beta)
54    u"γ": u"y",    # U+03B2 (small gamma)
55    u"ε": u"e",    # U+03B5 (small espilon)
56    u"η": u"n",    # U+03B7 (small eta)
57    u"ο": u"o",    # U+03BF (small omicron)
58    u"ρ": u"p",    # U+03C1 (small rho)
59    u"Ï…": u"v",    # U+03C1 (small upsilon)
60
61    # Cyrillic
62    u"І": u"I",    # U+0406 (capital byelorussian-ukrainian i)
63    u"Ј": u"J",    # U+0408 (capital je)
64    u"Ð’": u"B",    # U+0412 (capital ve)
65    u"Е": u"E",    # U+0415 (capital ie)
66    u"И": u"N",    # U+0418 (capital i)
67    u"З": u"3",    # U+0417 (capital ze)
68    u"К": u"K",    # U+041A (capital ka)
69    u"М": u"M",    # U+041C (capital em)
70    u"Н": u"H",    # U+041D (capital en)
71    u"О": u"O",    # U+041E (capital o)
72    u"Ð ": u"P",    # U+0420 (capital er)
73    u"С": u"C",    # U+0421 (capital es)
74    u"Т": u"T",    # U+0422 (capital te)
75    u"У": u"Y",    # U+0423 (capital u)
76    u"Ð¥": u"X",    # U+0425 (capital ha)
77    u"Я": u"R",    # U+042F (capital ya)
78    u"а": u"a",    # U+0430 (small a)
79    u"в": u"b",    # U+0432 (small ve)
80    u"е": u"e",    # U+0435 (small ie)
81    u"з": u"3",    # U+0437 (small ze)
82    u"к": u"k",    # U+043A (small ka)
83    u"м": u"m",    # U+043C (small em)
84    u"н": u"h",    # U+043D (small en)
85    u"о": u"o",    # U+043E (small o)
86    u"Ñ€": u"p",    # U+0440 (small er)
87    u"с": u"c",    # U+0441 (small es)
88    u"Ñ‚": u"T",    # U+0442 (small te)
89    u"у": u"y",    # U+0443 (small u)
90    u"Ñ…": u"x",    # U+0445 (small ha)
91    u"я": u"R",    # U+044F (small ya)
92    u"Ñ–": u"i",    # U+0456 (small byelorussian-ukrainian i)
93    u"ј": u"j",    # U+0458 (small je)
94}
95
96def unicode2ascii(text, replace=None):
97    """
98    Convert an unicode string (type 'unicode') to ascii string (type 'str').
99    Try to keep same visual result.
100
101    You can specify an ASCII character to replace non-ASCII character
102    in 'replace' argument (eg. replace='?').
103
104    >>> unicode2ascii(unicode("¡ Hé hø « español » ! Pythøn", "UTF-8"))
105    '! He ho " espanol " ! Python'
106    >>> unicode2ascii(unicode("L'œuf de læticia", "UTF-8"))
107    "L'oeuf de laeticia"
108    >>> unicode2ascii(unicode("ῙΈΌΑΒΓΔΕΖΗΘΙΚΛΝΜΞΟΥάήαγδεζημ", "UTF-8"), u'?')
109    'IEOAB??EZHOIK?NM?OYanay?e?n?'
110    >>> unicode2ascii(unicode("ЀЁЄЅІЇЈЌЍАВЕЗИКМНОРСТУХавезмнопрстухѐёіїјк", "UTF-8"), u'?')
111    'EE??IIJKN?BE3NKMHOPCTYXabe3mho?pcTyxeeiijk'
112    """
113    assert isinstance(text, unicode)
114    if replace:
115        if isinstance(replace, str):
116            replace = unicode(replace, "latin-1")
117        if not isinstance(replace, unicode) \
118        or len(replace) != 1 \
119        or not (32 <= ord(replace) <= 127):
120            raise ValueError(
121                "invalid replace character (%r): "
122                "need one ascii printable character" % replace)
123
124    ascii = []
125    for char in text:
126        # Remove diacriticals
127        char = normalize("NFKD", char)[0]
128
129        # Known values
130        if char in UNICODE_TO_ASCII:
131            ascii.append(UNICODE_TO_ASCII[char])
132            continue
133
134        if ord(char) <= 127:
135            # Add valid ASCII
136            ascii.append(char)
137        elif replace:
138            # non-ASCII character
139            ascii.append(replace)
140        # else: ignore it
141
142    text = ''.join(ascii)
143    return text.encode("ascii", "strict")
144
145if __name__ == "__main__":
146    from doctest import testmod
147    from sys import exit
148    failure, total = testmod()
149    if failure:
150        print "%s failure on %s tests" % (failure, total)
151        exit(1)
152    else:
153        print "All tests are OK (count=%s)" % total