Bug 648966 - Update g_unichar_iswide and g_unichar_iswide_cjk

Update to Unicode 6.0. Also attach Python script that generates the tables.
2025-10-31 00:12:19 +01:00 · 2011-04-29 16:14:49 -04:00
parent 426e1b6159
commit acda716d2d
3 changed files with 107 additions and 33 deletions
--- a/glib/Makefile.am
+++ b/glib/Makefile.am
@@ -82,6 +82,7 @@ MIRRORING_TAB_SOURCE = 				\
 EXTRA_DIST += 			\
 	makefile.msc.in		\
 	glib.rc.in		\
+	gen-iswide-table.py	\
 	gen-unicode-tables.pl	\
 	gen-script-table.pl	\
 	glibconfig.h.win32.in	\
--- a/glib/gen-iswide-table.py
+++ b/glib/gen-iswide-table.py
@@ -0,0 +1,58 @@
+#!/usr/bin/python
+
+import sys
+
+W = {}
+W['A'] = []
+W['W'] = []
+W['F'] = W['W']
+
+for line in sys.stdin:
+	i = line.find ('#')
+	if i >= 0:
+		line = line[:i]
+	line = line.strip ()
+	if not len (line):
+		continue
+
+	fields = [x.strip () for x in line.split (';')]
+	chars = fields[0]
+	width = fields[1]
+
+	if width not in ['A', 'W', 'F']:
+		continue
+
+	if chars.find ('..') > 0:
+		(start,end) = chars.split ('..')
+	else:
+		start = chars
+		end = chars
+	start, end = int(start,16), int(end,16)
+
+	for i in range (start, end+1):
+		W[width].append (i)
+
+
+def write_intervals (S):
+	S.sort ()
+	start = S[0];
+	end = start - 1
+	for c in S:
+		if c == end+1:
+			end += 1
+			continue
+		else:
+			print "{0x%04X, 0x%04X}, " % (start, end)
+			start = c
+			end = start
+	print "{0x%04X, 0x%04X} " % (start, end)
+
+
+
+print "table for g_unichar_iswide():"
+print
+write_intervals (W['W'])
+print
+print "table for g_unichar_iswide_cjk():"
+print
+write_intervals (W['A'])
--- a/glib/guniprop.c
+++ b/glib/guniprop.c
@@ -440,6 +440,20 @@ interval_compare (const void *key, const void *elt)
  return 0;
 }

+
+/*
+ * NOTE:
+ *
+ * The tables for g_unichar_iswide() and g_unichar_iswide_cjk() are
+ * generated from the Unicode Character Database's file
+ * extracted/DerivedEastAsianWidth.txt using the gen-iswide-table.py
+ * in this way:
+ *
+ *   ./gen-iswide-table.py < path/to/ucd/extracted/DerivedEastAsianWidth.txt | fmt
+ *
+ * Last update for Unicode 6.0.
+ */
+
 /**
 * g_unichar_iswide:
 * @c: a Unicode character
@@ -452,19 +466,18 @@ interval_compare (const void *key, const void *elt)
 gboolean
 g_unichar_iswide (gunichar c)
 {
-  /* sorted list of intervals of East_Asian_Width = W and F characters
-   * from Unicode 5.1.0.  produced by mungling output of:
-   * grep ';[FW]\>' EastAsianWidth.txt */
+  /* See NOTE earlier for how to update this table. */
  static const struct Interval wide[] = {
-    {0x1100, 0x1159}, {0x115F, 0x115F}, {0x2329, 0x232A}, {0x2E80, 0x2E99},
-    {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB}, {0x3000, 0x303E},
-    {0x3041, 0x3096}, {0x3099, 0x30FF}, {0x3105, 0x312D}, {0x3131, 0x318E},
-    {0x3190, 0x31B7}, {0x31C0, 0x31E3}, {0x31F0, 0x321E}, {0x3220, 0x3243},
-    {0x3250, 0x32FE}, {0x3300, 0x4DB5}, {0x4E00, 0x9FC3}, {0xA000, 0xA48C},
-    {0xA490, 0xA4C6}, {0xAC00, 0xD7A3}, {0xF900, 0xFA2D}, {0xFA30, 0xFA6A},
-    {0xFA70, 0xFAD9}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52}, {0xFE54, 0xFE66},
-    {0xFE68, 0xFE6B}, {0xFF01, 0xFF60}, {0xFFE0, 0xFFE6}, {0x20000, 0x2FFFD},
-    {0x30000, 0x3FFFD}
+    {0x1100, 0x115F}, {0x11A3, 0x11A7}, {0x11FA, 0x11FF}, {0x2329, 0x232A},
+    {0x2E80, 0x2E99}, {0x2E9B, 0x2EF3}, {0x2F00, 0x2FD5}, {0x2FF0, 0x2FFB},
+    {0x3000, 0x303E}, {0x3041, 0x3096}, {0x3099, 0x30FF}, {0x3105, 0x312D},
+    {0x3131, 0x318E}, {0x3190, 0x31BA}, {0x31C0, 0x31E3}, {0x31F0, 0x321E},
+    {0x3220, 0x3247}, {0x3250, 0x32FE}, {0x3300, 0x4DBF}, {0x4E00, 0xA48C},
+    {0xA490, 0xA4C6}, {0xA960, 0xA97C}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6},
+    {0xD7CB, 0xD7FB}, {0xF900, 0xFAFF}, {0xFE10, 0xFE19}, {0xFE30, 0xFE52},
+    {0xFE54, 0xFE66}, {0xFE68, 0xFE6B}, {0xFF01, 0xFF60}, {0xFFE0, 0xFFE6},
+    {0x1B000, 0x1B001}, {0x1F200, 0x1F202}, {0x1F210, 0x1F23A}, {0x1F240,
+    0x1F248}, {0x1F250, 0x1F251}, {0x20000, 0x2FFFD}, {0x30000, 0x3FFFD}
  };

  if (bsearch (GUINT_TO_POINTER (c), wide, G_N_ELEMENTS (wide), sizeof wide[0],
@@ -497,9 +510,7 @@ g_unichar_iswide (gunichar c)
 gboolean
 g_unichar_iswide_cjk (gunichar c)
 {
-  /* sorted list of intervals of East_Asian_Width = A and F characters
-   * from Unicode 5.1.0.  produced by mungling output of:
-   * grep ';[A]\>' EastAsianWidth.txt */
+  /* See NOTE earlier for how to update this table. */
  static const struct Interval ambiguous[] = {
    {0x00A1, 0x00A1}, {0x00A4, 0x00A4}, {0x00A7, 0x00A8}, {0x00AA, 0x00AA},
    {0x00AD, 0x00AE}, {0x00B0, 0x00B4}, {0x00B6, 0x00BA}, {0x00BC, 0x00BF},
@@ -523,24 +534,28 @@ g_unichar_iswide_cjk (gunichar c)
    {0x2103, 0x2103}, {0x2105, 0x2105}, {0x2109, 0x2109}, {0x2113, 0x2113},
    {0x2116, 0x2116}, {0x2121, 0x2122}, {0x2126, 0x2126}, {0x212B, 0x212B},
    {0x2153, 0x2154}, {0x215B, 0x215E}, {0x2160, 0x216B}, {0x2170, 0x2179},
-    {0x2190, 0x2199}, {0x21B8, 0x21B9}, {0x21D2, 0x21D2}, {0x21D4, 0x21D4},
-    {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203}, {0x2207, 0x2208},
-    {0x220B, 0x220B}, {0x220F, 0x220F}, {0x2211, 0x2211}, {0x2215, 0x2215},
-    {0x221A, 0x221A}, {0x221D, 0x2220}, {0x2223, 0x2223}, {0x2225, 0x2225},
-    {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237}, {0x223C, 0x223D},
-    {0x2248, 0x2248}, {0x224C, 0x224C}, {0x2252, 0x2252}, {0x2260, 0x2261},
-    {0x2264, 0x2267}, {0x226A, 0x226B}, {0x226E, 0x226F}, {0x2282, 0x2283},
-    {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299}, {0x22A5, 0x22A5},
-    {0x22BF, 0x22BF}, {0x2312, 0x2312}, {0x2460, 0x24E9}, {0x24EB, 0x254B},
-    {0x2550, 0x2573}, {0x2580, 0x258F}, {0x2592, 0x2595}, {0x25A0, 0x25A1},
-    {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7}, {0x25BC, 0x25BD},
-    {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, {0x25CB, 0x25CB}, {0x25CE, 0x25D1},
-    {0x25E2, 0x25E5}, {0x25EF, 0x25EF}, {0x2605, 0x2606}, {0x2609, 0x2609},
-    {0x260E, 0x260F}, {0x2614, 0x2615}, {0x261C, 0x261C}, {0x261E, 0x261E},
-    {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661}, {0x2663, 0x2665},
-    {0x2667, 0x266A}, {0x266C, 0x266D}, {0x266F, 0x266F}, {0x273D, 0x273D},
-    {0x2776, 0x277F}, {0xE000, 0xF8FF}, {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD},
-    {0xE0100, 0xE01EF}, {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}
+    {0x2189, 0x2189}, {0x2190, 0x2199}, {0x21B8, 0x21B9}, {0x21D2, 0x21D2},
+    {0x21D4, 0x21D4}, {0x21E7, 0x21E7}, {0x2200, 0x2200}, {0x2202, 0x2203},
+    {0x2207, 0x2208}, {0x220B, 0x220B}, {0x220F, 0x220F}, {0x2211, 0x2211},
+    {0x2215, 0x2215}, {0x221A, 0x221A}, {0x221D, 0x2220}, {0x2223, 0x2223},
+    {0x2225, 0x2225}, {0x2227, 0x222C}, {0x222E, 0x222E}, {0x2234, 0x2237},
+    {0x223C, 0x223D}, {0x2248, 0x2248}, {0x224C, 0x224C}, {0x2252, 0x2252},
+    {0x2260, 0x2261}, {0x2264, 0x2267}, {0x226A, 0x226B}, {0x226E, 0x226F},
+    {0x2282, 0x2283}, {0x2286, 0x2287}, {0x2295, 0x2295}, {0x2299, 0x2299},
+    {0x22A5, 0x22A5}, {0x22BF, 0x22BF}, {0x2312, 0x2312}, {0x2460, 0x24E9},
+    {0x24EB, 0x254B}, {0x2550, 0x2573}, {0x2580, 0x258F}, {0x2592, 0x2595},
+    {0x25A0, 0x25A1}, {0x25A3, 0x25A9}, {0x25B2, 0x25B3}, {0x25B6, 0x25B7},
+    {0x25BC, 0x25BD}, {0x25C0, 0x25C1}, {0x25C6, 0x25C8}, {0x25CB, 0x25CB},
+    {0x25CE, 0x25D1}, {0x25E2, 0x25E5}, {0x25EF, 0x25EF}, {0x2605, 0x2606},
+    {0x2609, 0x2609}, {0x260E, 0x260F}, {0x2614, 0x2615}, {0x261C, 0x261C},
+    {0x261E, 0x261E}, {0x2640, 0x2640}, {0x2642, 0x2642}, {0x2660, 0x2661},
+    {0x2663, 0x2665}, {0x2667, 0x266A}, {0x266C, 0x266D}, {0x266F, 0x266F},
+    {0x269E, 0x269F}, {0x26BE, 0x26BF}, {0x26C4, 0x26CD}, {0x26CF, 0x26E1},
+    {0x26E3, 0x26E3}, {0x26E8, 0x26FF}, {0x273D, 0x273D}, {0x2757, 0x2757},
+    {0x2776, 0x277F}, {0x2B55, 0x2B59}, {0x3248, 0x324F}, {0xE000, 0xF8FF},
+    {0xFE00, 0xFE0F}, {0xFFFD, 0xFFFD}, {0x1F100, 0x1F10A}, {0x1F110,
+    0x1F12D}, {0x1F130, 0x1F169}, {0x1F170, 0x1F19A}, {0xE0100, 0xE01EF},
+    {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD}
  };

  if (g_unichar_iswide (c))