unicode: Add API to access data from GraphemeBreakProperty

2025-11-30 22:08:19 +01:00 · 2012-09-02 22:25:53 +02:00
parent 7c7fe30485
commit aa610d0faa
9 changed files with 25077 additions and 6 deletions
--- a/Makefile.am
+++ b/Makefile.am
@@ -6,7 +6,7 @@ include $(top_srcdir)/Makefile.decl

 ACLOCAL_AMFLAGS = -I m4macros ${ACLOCAL_FLAGS}

-SUBDIRS = . m4macros glib gmodule gthread gobject gio po docs
+SUBDIRS = . m4macros glib gmodule gthread gobject gio po
 DIST_SUBDIRS = $(SUBDIRS) build
 if BUILD_MODULAR_TESTS
 SUBDIRS += tests 
--- a/docs/reference/glib/glib-sections.txt
+++ b/docs/reference/glib/glib-sections.txt
@@ -2739,6 +2739,7 @@ g_unichar_break_type
 g_unichar_combining_class
 g_unicode_canonical_ordering
 g_unicode_canonical_decomposition
+g_unichar_grapheme_cluster_break_type
 g_unichar_get_mirror_char
 GUnicodeScript
 g_unichar_get_script
--- a/glib/Makefile.am
+++ b/glib/Makefile.am
@@ -200,6 +200,8 @@ libglib_2_0_la_SOURCES = 	\
 	gunidecomp.h		\
 	gunidecomp.c		\
 	gunicodeprivate.h	\
+	gunigraphemebreak.h	\
+	gunigraphemebreak.c	\
 	gurifuncs.c 		\
 	gutils.c		\
 	gvariant.h		\
--- a/glib/gen-unicode-tables.pl
+++ b/glib/gen-unicode-tables.pl
@@ -153,6 +153,23 @@ $FOLDING_MAPPING = 2;
     'ZW' => "G_UNICODE_BREAK_ZERO_WIDTH_SPACE"
     );

+%grapheme_break_mappings =
+    (
+     'XX'                 => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_OTHER",
+     'CR'                 => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_CR",
+     'LF'                 => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_LF",
+     'Control'            => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_CONTROL",
+     'SpacingMark'        => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_SPACING_MARK",
+     'Extend'             => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_EXTEND",
+     'Prepend'            => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_PREPEND",
+     'Regional_Indicator' => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_REGIONAL_INDICATOR",
+     'L'                  => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_L",
+     'V'                  => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_V",
+     'T'                  => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_T",
+     'LV'                 => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LV",
+     'LVT'                => "G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LVT"
+    );
+
 # Title case mappings.
 %title_to_lower = ();
 %title_to_upper = ();
@@ -179,10 +196,10 @@ elsif (@ARGV && $ARGV[0] eq '-both')

 if (@ARGV != 2) {
    $0 =~ s@.*/@@;
-    die "\nUsage: $0 [-decomp | -both] UNICODE-VERSION DIRECTORY\n\n       DIRECTORY should contain the following Unicode data files:\n       UnicodeData.txt, LineBreak.txt, SpecialCasing.txt, CaseFolding.txt,\n       CompositionExclusions.txt\n\n";
+    die "\nUsage: $0 [-decomp | -both] UNICODE-VERSION DIRECTORY\n\n       DIRECTORY should contain the following Unicode data files:\n       UnicodeData.txt, LineBreak.txt, SpecialCasing.txt, CaseFolding.txt,\n       CompositionExclusions.txt, auxiliary/GraphemeBreakProperty.txt\n\n";
 }

-my ($unicodedatatxt, $linebreaktxt, $specialcasingtxt, $casefoldingtxt, $compositionexclusionstxt);
+my ($unicodedatatxt, $linebreaktxt, $graphemebreaktxt, $specialcasingtxt, $casefoldingtxt, $compositionexclusionstxt);

 my $d = $ARGV[1];
 opendir (my $dir, $d) or die "Cannot open Unicode data dir $d: $!\n";
@@ -195,8 +212,16 @@ for my $f (readdir ($dir))
    $compositionexclusionstxt = "$d/$f" if ($f =~ /^CompositionExclusions.*\.txt/);
 }

+$d = "$ARGV[1]/auxiliary";
+opendir ($dir, $d) or die "Cannot open Unicode auxiliary data dir $d: $!\n";
+for my $f (readdir ($dir))
+{
+    $graphemebreaktxt = "$d/$f" if ($f =~ /^GraphemeBreakProperty.*\.txt/);
+}
+
 defined $unicodedatatxt or die "Did not find UnicodeData file";
 defined $linebreaktxt or die "Did not find LineBreak file";
+defined $graphemebreaktxt or die "Did not find GraphemeBreakProperty file";
 defined $specialcasingtxt or die "Did not find SpecialCasing file";
 defined $casefoldingtxt or die "Did not find CaseFolding file";
 defined $compositionexclusionstxt or die "Did not find CompositionExclusions file";
@@ -362,6 +387,57 @@ for (++$last_code; $last_code <= 0x10FFFF; ++$last_code)

 print STDERR "Last code is not 0x10FFFF" if ($last_code != 0x10FFFF);

+print "Creating grapheme break table\n";
+
+print "Line grapheme break data from $graphemebreaktxt\n";
+
+open (INPUT, "< $graphemebreaktxt") || exit 1;
+
+while (<INPUT>)
+{
+    my ($start_code, $end_code);
+
+    chop;
+
+    next if /^#/;
+    next if /^$/;
+
+    s/\s*#.*//;
+
+    @fields = split (';', $_, 30);
+    if ($#fields != 1)
+    {
+        printf STDERR ("Entry for $fields[$CODE] has wrong number of fields (%d)\n", $#fields);
+        next;
+    }
+
+    $fields[$CODE] =~ s/\s+//;
+    if ($fields[$CODE] =~ /([A-F0-9]{4,6})\.\.([A-F0-9]{4,6})/)
+    {
+        $start_code = hex ($1);
+        $end_code = hex ($2);
+    } else {
+        $start_code = $end_code = hex ($fields[$CODE]);
+    }
+
+    $fields[$BREAK_PROPERTY] =~ s/\s+//;
+
+    for ($last_code = $start_code; $last_code <= $end_code; $last_code++)
+    {
+        $grapheme_break_props[$last_code] = $fields[$BREAK_PROPERTY];
+    }
+
+    $last_code = $end_code;
+}
+
+close INPUT;
+
+# All other characters have value Other (XX)
+for ($last_code = 0x0; $last_code <= 0x10FFFF; ++$last_code)
+{
+    $grapheme_break_props[$last_code] = 'XX' unless defined($grapheme_break_props[$last_code]);
+}
+
 print "Reading special-casing table for case conversion\n";

 open (INPUT, "< $specialcasingtxt") || exit 1;
@@ -491,6 +567,8 @@ while (<INPUT>)

 close INPUT;

+$last_code = 0x10FFFF;
+
 if ($do_props) {
    &print_tables ($last_code)
 }
@@ -501,6 +579,8 @@ if ($do_decomp) {

 &print_line_break ($last_code);

+&print_grapheme_break ($last_code);
+
 exit 0;


@@ -971,6 +1051,72 @@ sub print_line_break
    printf STDERR "Generated %d bytes in break tables\n", $bytes_out;
 }

+sub print_grapheme_break
+{
+    my ($last) = @_;
+    my ($outfile) = "gunigraphemebreak.h";
+
+    local ($bytes_out) = 0;
+
+    print "Writing $outfile...\n";
+
+    open (OUT, "> $outfile");
+
+    print OUT "/* This file is automatically generated.  DO NOT EDIT!\n";
+    print OUT "   Instead, edit gen-unicode-tables.pl and re-run.  */\n\n";
+
+    print OUT "#ifndef G_UNIGRAPHEMEBREAKTABLES_H\n";
+    print OUT "#define G_UNIGRAPHEMEBREAKTABLES_H\n\n";
+
+    print OUT "#include <glib/gtypes.h>\n";
+    print OUT "#include <glib/gunicode.h>\n\n";
+
+    print OUT "#define G_UNICODE_DATA_VERSION \"$ARGV[0]\"\n\n";
+
+    printf OUT "#define G_UNICODE_LAST_CHAR 0x%04X\n\n", $last;
+
+    printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 10000\n\n";
+
+    my $last_part1 = ($pages_before_e0000 * 256) - 1;
+    printf OUT "/* the last code point that should be looked up in grapheme_break_property_table_part1 */\n";
+    printf OUT "#define G_UNICODE_LAST_CHAR_PART1 0x%04X\n\n", $last_part1;
+
+    $table_index = 0;
+    printf OUT "static const gint8 grapheme_break_property_data[][256] = {\n";
+    for ($count = 0; $count <= $last; $count += 256)
+    {
+        $row[$count / 256] = &print_row ($count, 1, \&fetch_grapheme_break_type);
+    }
+    printf OUT "\n};\n\n";
+
+    printf OUT "/* U+0000 through U+%04X */\n", $last_part1;
+    print OUT "static const gint16 grapheme_break_property_table_part1[$pages_before_e0000] = {\n";
+    for ($count = 0; $count <= $last_part1; $count += 256)
+    {
+        print OUT ",\n" if $count > 0;
+        print OUT "  ", $row[$count / 256];
+        $bytes_out += 2;
+    }
+    print OUT "\n};\n\n";
+
+    printf OUT "/* U+E0000 through U+%04X */\n", $last;
+    print OUT "static const gint16 grapheme_break_property_table_part2[768] = {\n";
+    for ($count = 0xE0000; $count <= $last; $count += 256)
+    {
+        print OUT ",\n" if $count > 0xE0000;
+        print OUT "  ", $row[$count / 256];
+        $bytes_out += 2;
+    }
+    print OUT "\n};\n\n";
+
+
+    print OUT "#endif /* G_UNIGRAPHEMEBREAKTABLES_H */\n";
+
+    close (OUT);
+
+    printf STDERR "Generated %d bytes in grapheme break tables\n", $bytes_out;
+}
+

 # A fetch function for the break properties table.
 sub fetch_break_type
@@ -979,6 +1125,13 @@ sub fetch_break_type
    return $break_mappings{$break_props[$index]};
 }

+# A fetch function for the grapheme break properties table.
+sub fetch_grapheme_break_type
+{
+    my ($index) = @_;
+    return $grapheme_break_mappings{$grapheme_break_props[$index]};
+}
+
 # Fetcher for combining class.
 sub fetch_cclass
 {
@@ -1335,6 +1488,3 @@ EOT
   my $recordlen = (2+$casefoldlen+1) & ~1;
   printf "Generated %d bytes for casefold table\n", $recordlen * @casefold;
 }
-
-			     
-
--- a/glib/glib.symbols
+++ b/glib/glib.symbols
@@ -1285,6 +1285,7 @@ g_unichar_xdigit_value
 g_unichar_type
 g_unicode_canonical_decomposition
 g_unicode_canonical_ordering
+g_unichar_grapheme_cluster_break_type
 g_unicode_script_from_iso15924
 g_unicode_script_to_iso15924
 g_utf8_casefold
--- a/glib/gunicode.h
+++ b/glib/gunicode.h
@@ -506,6 +506,44 @@ typedef enum
  G_UNICODE_SCRIPT_TAKRI                   /* Takr */
 } GUnicodeScript;

+/**
+ * GUnicodeGraphemeClusterBreakType:
+ * 
+ * These are the grapheme cluster break types.
+ * 
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_OTHER:
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_CR: Carriage Return
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_LF: Line Feed
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_CONTROL: Control
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_SPACING_MARK: Spacing Mark
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_EXTEND: Extend
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_PREPEND: Prepend
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_REGIONAL_INDICATOR: Regional Indicator
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_L: Hangul L Syllable
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_V: Hangul T Syllable
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_T: Hangul V Syllable
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LV: Hangul LV Syllable
+ * @G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LVT: Hangul LVT Syllable
+ *
+ * Since: 2.36
+ */
+typedef enum
+{
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_OTHER = 0,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_CR,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_LF,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_CONTROL,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_SPACING_MARK,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_EXTEND,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_PREPEND,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_REGIONAL_INDICATOR,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_L,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_V,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_T,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LV,
+  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LVT
+} GUnicodeGraphemeClusterBreakType;
+
 guint32        g_unicode_script_to_iso15924   (GUnicodeScript script);
 GUnicodeScript g_unicode_script_from_iso15924 (guint32        iso15924);

@@ -547,6 +585,9 @@ GUnicodeType g_unichar_type (gunichar c) G_GNUC_CONST;
 /* Return the line break property for a given character */
 GUnicodeBreakType g_unichar_break_type (gunichar c) G_GNUC_CONST;

+/* Return the grapheme cluster break property for a given character */
+GUnicodeGraphemeClusterBreakType g_unichar_grapheme_cluster_break_type (gunichar c) G_GNUC_CONST;
+
 /* Returns the combining class for a given character */
 gint g_unichar_combining_class (gunichar uc) G_GNUC_CONST;

--- a/glib/gunigraphemebreak.c
+++ b/glib/gunigraphemebreak.c
@@ -0,0 +1,58 @@
+/* gunibreak.c - line break properties
+ *
+ *  Copyright 2000 Red Hat, Inc.
+ *
+ * The Gnome Library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * The Gnome Library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with the Gnome Library; see the file COPYING.LIB.  If not,
+ * write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ *   Boston, MA 02111-1307, USA.
+ */
+
+#include "config.h"
+
+#include <stdlib.h>
+
+#include "gunigraphemebreak.h"
+
+#define TPROP_PART1(Page, Char) \
+  ((grapheme_break_property_table_part1[Page] >= G_UNICODE_MAX_TABLE_INDEX) \
+   ? (grapheme_break_property_table_part1[Page] - G_UNICODE_MAX_TABLE_INDEX) \
+   : (grapheme_break_property_data[grapheme_break_property_table_part1[Page]][Char]))
+
+#define TPROP_PART2(Page, Char) \
+  ((grapheme_break_property_table_part2[Page] >= G_UNICODE_MAX_TABLE_INDEX) \
+   ? (grapheme_break_property_table_part2[Page] - G_UNICODE_MAX_TABLE_INDEX) \
+   : (grapheme_break_property_data[grapheme_break_property_table_part2[Page]][Char]))
+
+#define PROP(Char) \
+  (((Char) <= G_UNICODE_LAST_CHAR_PART1) \
+   ? TPROP_PART1 ((Char) >> 8, (Char) & 0xff) \
+   : (((Char) >= 0xe0000 && (Char) <= G_UNICODE_LAST_CHAR) \
+      ? TPROP_PART2 (((Char) - 0xe0000) >> 8, (Char) & 0xff) \
+      : G_UNICODE_BREAK_UNKNOWN))
+
+/**
+ * g_unichar_grapheme_cluster_break_type:
+ * @c: a Unicode character
+ *
+ * Determines the grapeme cluster break type of @c.
+ *
+ * Return value: the grapheme cluster break type of @c
+ *
+ * Since: 2.36
+ **/
+GUnicodeGraphemeClusterBreakType
+g_unichar_grapheme_cluster_break_type (gunichar c)
+{
+  return PROP (c);
+}
--- a/glib/gunigraphemebreak.h
+++ b/glib/gunigraphemebreak.h
--- a/glib/tests/unicode.c
+++ b/glib/tests/unicode.c
@@ -140,6 +140,42 @@ test_unichar_break_type (void)
    }
 }

+static void
+test_unichar_grapheme_break_type (void)
+{
+  guint i;
+  const struct {
+    gunichar c;
+    GUnicodeGraphemeClusterBreakType type;
+  } const examples[] = {
+    { 0x000D,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_CR                  },
+    { 0x000A,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_LF                  },
+    { 0x200C,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_EXTEND              },
+    { 0x200D,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_EXTEND              },
+    { 0x007F,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_CONTROL             },
+    { 0x0300,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_EXTEND              },
+    { 0x1F1E6, G_UNICODE_GRAPHEME_CLUSTER_BREAK_REGIONAL_INDICATOR  },
+    { 0x1F1FF, G_UNICODE_GRAPHEME_CLUSTER_BREAK_REGIONAL_INDICATOR  },
+    { 0x0903,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_SPACING_MARK        },
+    { 0x1D16D, G_UNICODE_GRAPHEME_CLUSTER_BREAK_SPACING_MARK        },
+    { 0x1100,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_L   },
+    { 0x115F,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_L   },
+    { 0x1160,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_V   },
+    { 0x11A7,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_V   },
+    { 0x11A8,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_T   },
+    { 0x11FF,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_T   },
+    { 0xAC00,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LV  },
+    { 0xD788,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LV  },
+    { 0xAC01,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LVT },
+    { 0xD7A3,  G_UNICODE_GRAPHEME_CLUSTER_BREAK_HANGUL_SYLLABLE_LVT }
+  };
+
+  for (i = 0; i < G_N_ELEMENTS (examples); i++)
+    {
+      g_assert_cmpint (g_unichar_grapheme_cluster_break_type (examples[i].c), ==, examples[i].type);
+    }
+}
+
 static void
 test_unichar_script (void)
 {
@@ -839,6 +875,7 @@ main (int   argc,
  g_test_add_func ("/unicode/validate", test_unichar_validate);
  g_test_add_func ("/unicode/character-type", test_unichar_character_type);
  g_test_add_func ("/unicode/break-type", test_unichar_break_type);
+  g_test_add_func ("/unicode/grapheme-break-type", test_unichar_grapheme_break_type);
  g_test_add_func ("/unicode/script", test_unichar_script);
  g_test_add_func ("/unicode/combining-class", test_combining_class);
  g_test_add_func ("/unicode/mirror", test_mirror);