luc14n0/glib
1
0
Fork 0
mirror of https://gitlab.gnome.org/GNOME/glib.git synced 2025-04-02 13:53:06 +02:00
Code Issues Packages Projects Releases Wiki Activity

glib-mkenums: best effort attempt on non-utf8 encoded files.

Browse Source 
Some source files aren't valid utf-8 containing for example
iso8859-1 accented characters in author's names.
Replace invalid data with a replacement '?' character and print a
warning to keep things working.
Based on a patch from Christoph Reiter in
https://bugzilla.gnome.org/show_bug.cgi?id=785113#c20
This commit is contained in:
Patrick Welche 2017-10-23 13:59:58 +01:00 committed by Philip Withnall
parent b6d4edbae8
commit 2fc3948ba7
1 changed files with 28 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
gobject/glib-mkenums.in
View File

@ -26,14 +26,6 @@ the GNU General Public License which can be found in the
GLib source package. Sources, examples and contact GLib source package. Sources, examples and contact
information are available at http://www.gtk.org''' information are available at http://www.gtk.org'''
# Python 2 defaults to ASCII in case stdout is redirected.
# This should make it match Python 3, which uses the locale encoding.
if sys.stdout.encoding is None:
output_stream = codecs.getwriter(
locale.getpreferredencoding())(sys.stdout)
else:
output_stream = sys.stdout
# pylint: disable=too-few-public-methods # pylint: disable=too-few-public-methods
class Color: class Color:
'''ANSI Terminal colors''' '''ANSI Terminal colors'''
@ -81,6 +73,29 @@ def write_output(output):
global output_stream global output_stream
print(output, file=output_stream) print(output, file=output_stream)
# Python 2 defaults to ASCII in case stdout is redirected.
# This should make it match Python 3, which uses the locale encoding.
if sys.stdout.encoding is None:
output_stream = codecs.getwriter(
locale.getpreferredencoding())(sys.stdout)
else:
output_stream = sys.stdout
# Some source files aren't UTF-8 and the old perl version didn't care.
# Replace invalid data with a replacement character to keep things working.
# https://bugzilla.gnome.org/show_bug.cgi?id=785113#c20
def replace_and_warn(err):
# 7 characters of context either side of the offending character
print_warning('UnicodeWarning: {} at {} ({})'.format(
err.reason, err.start,
err.object[err.start - 7:err.end + 7]))
return ('?', err.end)
codecs.register_error('replace_and_warn', replace_and_warn)
# glib-mkenums.py # glib-mkenums.py
# Information about the current enumeration # Information about the current enumeration
flags = None # Is enumeration a bitmask? flags = None # Is enumeration a bitmask?
@ -157,7 +172,8 @@ def parse_entries(file, file_name):
m = re.match(r'\#include\s*<([^>]*)>', line) m = re.match(r'\#include\s*<([^>]*)>', line)
if m: if m:
newfilename = os.path.join("..", m.group(1)) newfilename = os.path.join("..", m.group(1))
newfile = io.open(newfilename, encoding="utf-8") newfile = io.open(newfilename, encoding="utf-8",
errors="replace_and_warn")
if not parse_entries(newfile, newfilename): if not parse_entries(newfile, newfilename):
return False return False
@ -253,7 +269,7 @@ def read_template_file(file):
} }
in_ = 'junk' in_ = 'junk'
ifile = io.open(file, encoding="utf-8") ifile = io.open(file, encoding="utf-8", errors="replace_and_warn")
for line in ifile: for line in ifile:
m = re.match(r'\/\*\*\*\s+(BEGIN|END)\s+([\w-]+)\s+\*\*\*\/', line) m = re.match(r'\/\*\*\*\s+(BEGIN|END)\s+([\w-]+)\s+\*\*\*\/', line)
if m: if m:
@ -413,7 +429,8 @@ def process_file(curfilename):
firstenum = True firstenum = True
try: try:
curfile = io.open(curfilename, encoding="utf-8") curfile = io.open(curfilename, encoding="utf-8",
errors="replace_and_warn")
except IOError as e: except IOError as e:
if e.errno == errno.ENOENT: if e.errno == errno.ENOENT:
print_warning('No file "{}" found.'.format(curfilename)) print_warning('No file "{}" found.'.format(curfilename))