From acda716d2d955ba2107a06a11df2e7938d0dfd4d Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Fri, 29 Apr 2011 16:14:49 -0400 Subject: [PATCH] Bug 648966 - Update g_unichar_iswide and g_unichar_iswide_cjk Update to Unicode 6.0. Also attach Python script that generates the tables. --- glib/Makefile.am | 1 + glib/gen-iswide-table.py | 58 ++++++++++++++++++++++++++++ glib/guniprop.c | 81 ++++++++++++++++++++++++---------------- 3 files changed, 107 insertions(+), 33 deletions(-) create mode 100755 glib/gen-iswide-table.py diff --git a/glib/Makefile.am b/glib/Makefile.am index 5a9ab53a6..9bcfe2bfa 100644 --- a/glib/Makefile.am +++ b/glib/Makefile.am @@ -82,6 +82,7 @@ MIRRORING_TAB_SOURCE = \ EXTRA_DIST += \ makefile.msc.in \ glib.rc.in \ + gen-iswide-table.py \ gen-unicode-tables.pl \ gen-script-table.pl \ glibconfig.h.win32.in \ diff --git a/glib/gen-iswide-table.py b/glib/gen-iswide-table.py new file mode 100755 index 000000000..292addc3d --- /dev/null +++ b/glib/gen-iswide-table.py @@ -0,0 +1,58 @@ +#!/usr/bin/python + +import sys + +W = {} +W['A'] = [] +W['W'] = [] +W['F'] = W['W'] + +for line in sys.stdin: + i = line.find ('#') + if i >= 0: + line = line[:i] + line = line.strip () + if not len (line): + continue + + fields = [x.strip () for x in line.split (';')] + chars = fields[0] + width = fields[1] + + if width not in ['A', 'W', 'F']: + continue + + if chars.find ('..') > 0: + (start,end) = chars.split ('..') + else: + start = chars + end = chars + start, end = int(start,16), int(end,16) + + for i in range (start, end+1): + W[width].append (i) + + +def write_intervals (S): + S.sort () + start = S[0]; + end = start - 1 + for c in S: + if c == end+1: + end += 1 + continue + else: + print "{0x%04X, 0x%04X}, " % (start, end) + start = c + end = start + print "{0x%04X, 0x%04X} " % (start, end) + + + +print "table for g_unichar_iswide():" +print +write_intervals (W['W']) +print +print "table for g_unichar_iswide_cjk():" +print +write_intervals (W['A']) diff --git a/glib/guniprop.c b/glib/guniprop.c index 639057434..923478d3d 100644 --- a/glib/guniprop.c +++ b/glib/guniprop.c @@ -440,6 +440,20 @@ interval_compare (const void *key, const void *elt) return 0; } + +/* + * NOTE: + * + * The tables for g_unichar_iswide() and g_unichar_iswide_cjk() are + * generated from the Unicode Character Database's file + * extracted/DerivedEastAsianWidth.txt using the gen-iswide-table.py + * in this way: + * + * ./gen-iswide-table.py < path/to/ucd/extracted/DerivedEastAsianWidth.txt | fmt + * + * Last update for Unicode 6.0. + */ + /** * g_unichar_iswide: * @c: a Unicode character @@ -452,19 +466,18 @@ interval_compare (const void *key, const void *elt) gboolean g_unichar_iswide (gunichar c) { - /* sorted list of intervals of East_Asian_Width = W and F characters - * from Unicode 5.1.0. produced by mungling output of: - * grep ';[FW]\>' EastAsianWidth.txt */ + /* See NOTE earlier for how to update this table. */ static const struct Interval wide[] = { - {0x1100, 0x1159}, {0x115F, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x2E99}, - {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, {0x3000, 0x303E}, - {0x3041, 0x3096}, {0x3099, 0x30FF}, {0x3105, 0x312D}, {0x3131, 0x318E}, - {0x3190, 0x31B7}, {0x31C0, 0x31E3}, {0x31F0, 0x321E}, {0x3220, 0x3243}, - {0x3250, 0x32FE}, {0x3300, 0x4DB5}, {0x4E00, 0x9FC3}, {0xA000, 0xA48C}, - {0xA490, 0xA4C6}, {0xAC00, 0xD7A3}, {0xF900, 0xFA2D}, {0xFA30, 0xFA6A}, - {0xFA70, 0xFAD9}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52}, {0xFE54, 0xFE66}, - {0xFE68, 0xFE6B}, {0xFF01, 0xFF60}, {0xFFE0, 0xFFE6}, {0x20000, 0x2FFFD}, - {0x30000, 0x3FFFD} + {0x1100, 0x115F}, {0x11A3, 0x11A7}, {0x11FA, 0x11FF}, {0x2329, 0x232A}, + {0x2E80, 0x2E99}, {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, + {0x3000, 0x303E}, {0x3041, 0x3096}, {0x3099, 0x30FF}, {0x3105, 0x312D}, + {0x3131, 0x318E}, {0x3190, 0x31BA}, {0x31C0, 0x31E3}, {0x31F0, 0x321E}, + {0x3220, 0x3247}, {0x3250, 0x32FE}, {0x3300, 0x4DBF}, {0x4E00, 0xA48C}, + {0xA490, 0xA4C6}, {0xA960, 0xA97C}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, + {0xD7CB, 0xD7FB}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52}, + {0xFE54, 0xFE66}, {0xFE68, 0xFE6B}, {0xFF01, 0xFF60}, {0xFFE0, 0xFFE6}, + {0x1B000, 0x1B001}, {0x1F200, 0x1F202}, {0x1F210, 0x1F23A}, {0x1F240, + 0x1F248}, {0x1F250, 0x1F251}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD} }; if (bsearch (GUINT_TO_POINTER (c), wide, G_N_ELEMENTS (wide), sizeof wide[0], @@ -497,9 +510,7 @@ g_unichar_iswide (gunichar c) gboolean g_unichar_iswide_cjk (gunichar c) { - /* sorted list of intervals of East_Asian_Width = A and F characters - * from Unicode 5.1.0. produced by mungling output of: - * grep ';[A]\>' EastAsianWidth.txt */ + /* See NOTE earlier for how to update this table. */ static const struct Interval ambiguous[] = { {0x00A1, 0x00A1}, {0x00A4, 0x00A4}, {0x00A7, 0x00A8}, {0x00AA, 0x00AA}, {0x00AD, 0x00AE}, {0x00B0, 0x00B4}, {0x00B6, 0x00BA}, {0x00BC, 0x00BF}, @@ -523,24 +534,28 @@ g_unichar_iswide_cjk (gunichar c) {0x2103, 0x2103}, {0x2105, 0x2105}, {0x2109, 0x2109}, {0x2113, 0x2113}, {0x2116, 0x2116}, {0x2121, 0x2122}, {0x2126, 0x2126}, {0x212B, 0x212B}, {0x2153, 0x2154}, {0x215B, 0x215E}, {0x2160, 0x216B}, {0x2170, 0x2179}, - {0x2190, 0x2199}, {0x21B8, 0x21B9}, {0x21D2, 0x21D2}, {0x21D4, 0x21D4}, - {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203}, {0x2207, 0x2208}, - {0x220B, 0x220B}, {0x220F, 0x220F}, {0x2211, 0x2211}, {0x2215, 0x2215}, - {0x221A, 0x221A}, {0x221D, 0x2220}, {0x2223, 0x2223}, {0x2225, 0x2225}, - {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237}, {0x223C, 0x223D}, - {0x2248, 0x2248}, {0x224C, 0x224C}, {0x2252, 0x2252}, {0x2260, 0x2261}, - {0x2264, 0x2267}, {0x226A, 0x226B}, {0x226E, 0x226F}, {0x2282, 0x2283}, - {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299}, {0x22A5, 0x22A5}, - {0x22BF, 0x22BF}, {0x2312, 0x2312}, {0x2460, 0x24E9}, {0x24EB, 0x254B}, - {0x2550, 0x2573}, {0x2580, 0x258F}, {0x2592, 0x2595}, {0x25A0, 0x25A1}, - {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7}, {0x25BC, 0x25BD}, - {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, {0x25CB, 0x25CB}, {0x25CE, 0x25D1}, - {0x25E2, 0x25E5}, {0x25EF, 0x25EF}, {0x2605, 0x2606}, {0x2609, 0x2609}, - {0x260E, 0x260F}, {0x2614, 0x2615}, {0x261C, 0x261C}, {0x261E, 0x261E}, - {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661}, {0x2663, 0x2665}, - {0x2667, 0x266A}, {0x266C, 0x266D}, {0x266F, 0x266F}, {0x273D, 0x273D}, - {0x2776, 0x277F}, {0xE000, 0xF8FF}, {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, - {0xE0100, 0xE01EF}, {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD} + {0x2189, 0x2189}, {0x2190, 0x2199}, {0x21B8, 0x21B9}, {0x21D2, 0x21D2}, + {0x21D4, 0x21D4}, {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203}, + {0x2207, 0x2208}, {0x220B, 0x220B}, {0x220F, 0x220F}, {0x2211, 0x2211}, + {0x2215, 0x2215}, {0x221A, 0x221A}, {0x221D, 0x2220}, {0x2223, 0x2223}, + {0x2225, 0x2225}, {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237}, + {0x223C, 0x223D}, {0x2248, 0x2248}, {0x224C, 0x224C}, {0x2252, 0x2252}, + {0x2260, 0x2261}, {0x2264, 0x2267}, {0x226A, 0x226B}, {0x226E, 0x226F}, + {0x2282, 0x2283}, {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299}, + {0x22A5, 0x22A5}, {0x22BF, 0x22BF}, {0x2312, 0x2312}, {0x2460, 0x24E9}, + {0x24EB, 0x254B}, {0x2550, 0x2573}, {0x2580, 0x258F}, {0x2592, 0x2595}, + {0x25A0, 0x25A1}, {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7}, + {0x25BC, 0x25BD}, {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, {0x25CB, 0x25CB}, + {0x25CE, 0x25D1}, {0x25E2, 0x25E5}, {0x25EF, 0x25EF}, {0x2605, 0x2606}, + {0x2609, 0x2609}, {0x260E, 0x260F}, {0x2614, 0x2615}, {0x261C, 0x261C}, + {0x261E, 0x261E}, {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661}, + {0x2663, 0x2665}, {0x2667, 0x266A}, {0x266C, 0x266D}, {0x266F, 0x266F}, + {0x269E, 0x269F}, {0x26BE, 0x26BF}, {0x26C4, 0x26CD}, {0x26CF, 0x26E1}, + {0x26E3, 0x26E3}, {0x26E8, 0x26FF}, {0x273D, 0x273D}, {0x2757, 0x2757}, + {0x2776, 0x277F}, {0x2B55, 0x2B59}, {0x3248, 0x324F}, {0xE000, 0xF8FF}, + {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A}, {0x1F110, + 0x1F12D}, {0x1F130, 0x1F169}, {0x1F170, 0x1F19A}, {0xE0100, 0xE01EF}, + {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD} }; if (g_unichar_iswide (c))