ksh/ksh93-shift_ijs.dif

--- src/cmd/ksh93/edit/edit.c
+++ src/cmd/ksh93/edit/edit.c	2007-03-28 18:22:27.380621000 +0200
@@ -28,6 +28,7 @@
  */
 
 #include	<ast.h>
+#include	<ast_wchar.h>
 #include	<errno.h>
 #include	<ccode.h>
 #include	<ctype.h>
@@ -54,8 +55,20 @@
 static char CURSOR_UP[20] = { ESC, '[', 'A', 0 };
 
 #if SHOPT_MULTIBYTE
-#   define is_cntrl(c)	((c<=STRIP) && iscntrl(c))
-#   define is_print(c)	((c&~STRIP) || isprint(c))
+# if _hdr_wctype
+#   include <wctype.h>
+#   define is_print(c)  iswprint((c))
+#   define is_cntrl(c)  iswcntrl((c))
+# else
+#   define is_cntrl(c)	(((c)<=STRIP) && iscntrl((c)))
+#   define is_print(c)	(((c)&~STRIP) || isprint((c)))
+# endif
+# if !_lib_iswcntrl && !defined(iswcntrl)
+#   define iswcntrl(c)  (((c)<=STRIP) && iscntrl((c)))
+# endif
+# if !_lib_iswprint && !defined(iswprint)
+#   define iswprint(c)  (((c)&~STRIP) || isprint((c)))
+# endif
 #else
 #   define is_cntrl(c)	iscntrl(c)
 #   define is_print(c)	isprint(c)
--- src/cmd/ksh93/edit/emacs.c
+++ src/cmd/ksh93/edit/emacs.c	2007-04-03 12:46:28.354368552 +0200
@@ -62,6 +62,7 @@ One line screen editor for any program
  */
 
 #include	<ast.h>
+#include	<ast_wchar.h>
 #include	<ctype.h>
 #include	"FEATURE/cmds"
 #if KSHELL
@@ -83,6 +84,9 @@ One line screen editor for any program
 
 
 #if SHOPT_MULTIBYTE
+#   if _hdr_wctype
+#	include <wctype.h>
+#   endif 
 #   define gencpy(a,b)	ed_gencpy(a,b)
 #   define genncpy(a,b,n)	ed_genncpy(a,b,n)
 #   define genlen(str)	ed_genlen(str)
@@ -1434,11 +1438,19 @@ static void setcursor(register Emacs_t *
 #if SHOPT_MULTIBYTE
 static int print(register int c)
 {
+#if _lib_iswprint || defined(iswprint)
+	return(iswprint(c));
+#else
 	return((c&~STRIP)==0 && isprint(c));
+#endif
 }
 
 static int _isword(register int c)
 {
+#if _lib_iswalnum || defined(iswalnum)
+	return(iswalnum(c) || (c == '_'));
+#else
 	return((c&~STRIP) || isalnum(c) || c=='_');
+#endif
 }
 #endif /* SHOPT_MULTIBYTE */
--- src/cmd/ksh93/edit/vi.c
+++ src/cmd/ksh93/edit/vi.c	2007-03-28 18:22:42.862649000 +0200
@@ -28,6 +28,8 @@
  *		cbosgd!pds
 -*/
 
+#include <ast.h>
+#include <ast_wchar.h>
 
 #if KSHELL
 #   include	"defs.h"
@@ -65,10 +67,19 @@
 #   define gencpy(a,b)	ed_gencpy(a,b)
 #   define genncpy(a,b,n)	ed_genncpy(a,b,n)
 #   define genlen(str)	ed_genlen(str)
-#   define digit(c)	((c&~STRIP)==0 && isdigit(c))
-#   define is_print(c)	((c&~STRIP) || isprint(c))
+#   if _hdr_wctype
+#	include <wctype.h>
+#	define digit(c)     	iswdigit((c))
+#	define is_print(c)	iswprint((c))
+#   else
+#	define digit(c)     	(((c)&~STRIP)==0 && isdigit((c)))
+#	define is_print(c)	(((c)&~STRIP) || isprint((c)))
+#   endif
+#   if !_lib_iswdigit && !defined(iswdigit)
+#	define iswdigit(c)	(((c)&~STRIP)==0 && isdigit((c)))
+#   endif
 #   if !_lib_iswprint && !defined(iswprint)
-#	define iswprint(c)	((c&~0177) || isprint(c))
+#	define iswprint(c)	(((c)&~STRIP) || isprint((c)))
 #   endif
     static int _isalph(int);
     static int _ismetach(int);
--- src/cmd/ksh93/sh/macro.c
+++ src/cmd/ksh93/sh/macro.c	2007-11-20 12:09:31.000000000 +0000
@@ -1860,13 +1860,20 @@ static void comsubst(Mac_t *mp,int type)
 	return;
 }
 
+#if SHOPT_MULTIBYTE
+#define ismbstate(c,s)	((state[*(unsigned char*)(c)]==(s))&&(!mbwide()||(mbsize(c)<2)))
+#else
+#define ismbstate(c,s)	(state[*(unsigned char*)(c)]==(s))
+#endif
+
+
 /*
  * copy <str> onto the stack
  */
 static void mac_copy(register Mac_t *mp,register const char *str, register int size)
 {
 	register char		*state;
-	register const char	*cp=str;
+	register const char	*cp=str, *sp;
 	register int		c,n,nopat;
 	nopat = (mp->quote||mp->assign==1||mp->arith);
 	if(mp->zeros)
@@ -1885,14 +1892,25 @@ static void mac_copy(register Mac_t *mp,
 		/* insert \ before file expansion characters */
 		while(size-->0)
 		{
+			sp = cp;
 			c = state[n= *(unsigned char*)cp++];
+#if SHOPT_MULTIBYTE
+			if(mbwide())
+			{	ssize_t len = mbsize(sp);
+				if (len-- > 1)
+				{	cp += len;
+					size -= len;
+					continue;
+				}
+			}
+#endif /* SHOPT_MULTIBYTE */
 			if(nopat&&(c==S_PAT||c==S_ESC||c==S_BRACT||c==S_ENDCH) && mp->pattern!=3)
 				c=1;
 			else if(mp->pattern==4 && (c==S_ESC||c==S_BRACT||c==S_ENDCH || isastchar(n)))
 				c=1;
 			else if(mp->pattern==2 && c==S_SLASH)
 				c=1;
-			else if(mp->pattern==3 && c==S_ESC && (state[*(unsigned char*)cp]==S_DIG||(*cp==ESCAPE)))
+			else if(mp->pattern==3 && c==S_ESC && (ismbstate(cp,S_DIG)||(*cp==ESCAPE)))
 			{
 				if(!(c=mp->quote))
 					cp++;
@@ -1916,14 +1934,14 @@ static void mac_copy(register Mac_t *mp,
 		state = sh.ifstable;
 		if(mp->pattern)
 		{
-			char *sp = "&|()";
-			while(c = *sp++)
+			sp = "&|()";
+			while((c = *sp++))
 			{
 				if(state[c]==0)
 					state[c] = S_EPAT;
 			}
 			sp = "*?[{";
-			while(c = *sp++)
+			while((c = *sp++))
 			{
 				if(state[c]==0)
 					state[c] = S_PAT;
@@ -1933,7 +1951,20 @@ static void mac_copy(register Mac_t *mp,
 		}
 		while(size-->0)
 		{
-			if((n=state[c= *(unsigned char*)cp++])==S_ESC || n==S_EPAT)
+			sp = cp;
+			n = state[c= *(unsigned char*)cp++];
+#if SHOPT_MULTIBYTE
+			if(mbwide() && (n!=S_MBYTE))
+			{	ssize_t len = mbsize(sp);
+				if (len-- > 1)
+				{	cp += len;
+					size -= len;
+					stakwrite(sp, len+1);
+					continue;
+				}
+			}
+#endif /* SHOPT_MULTIBYTE */
+			if(n==S_ESC || n==S_EPAT)
 			{
 				/* don't allow extended patterns in this case */
 				mp->patfound = mp->pattern;
--- src/cmd/ksh93/sh/string.c
+++ src/cmd/ksh93/sh/string.c	2007-03-28 18:23:11.582410000 +0200
@@ -42,7 +42,7 @@
 #endif
 
 #if !_lib_iswprint && !defined(iswprint)
-#   define iswprint(c)		(((c)&~0377) || isprint(c))
+#   define iswprint(c)		(((c)&~STRIP) || isprint(c))
 #endif
 
 
@@ -250,12 +250,13 @@ void	sh_trim(register char *sp)
 	if(sp)
 	{
 		dp = sp;
-		while(c= *sp)
+		while((c = *sp))
 		{
 #if SHOPT_MULTIBYTE
 			int len;
 			if(mbwide() && (len=mbsize(sp))>1)
 			{
+				memmove(dp, sp, len);
 				dp += len;
 				sp += len;
 				continue;
--- src/cmd/ksh93/tests/sjis.sh
+++ src/cmd/ksh93/tests/sjis.sh	2007-11-20 13:07:57.000000000 +0000
@@ -0,0 +1,63 @@
+########################################################################
+#                                                                      #
+#   Copyright (c) 2007 SuSE Linux Products GmbH, Nuernberg, Germany    #
+#                                                                      #
+#   This library is free software; you can redistribute it and/or      #
+#   modify it under the terms of the GNU Lesser General Public         #
+#   License as published by the Free Software Foundation;              #
+#   version 2.1 of the License.                                        #
+#                                                                      #
+#   This library is distributed in the hope that it will be useful,    #
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of     #
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the      #
+#   GNU Lesser General Public License at                               #
+#   http://www.gnu.org/licenses/lgpl-2.1.html for more details         #
+#                                                                      #
+#   Author: Werner Fink <werner@suse.de>                               #
+#                                                                      #
+########################################################################
+
+#
+# Byte ranges for Shift-JIS encoding (hexadecimal):
+# First byte:   81-9F, E0-EF
+# Second byte:  40-7E, 80-FC
+#
+# Now test out some multi byte characters which
+# include 7bit aka ASCII bytes with 0x81 0x{40-7E}
+#
+
+typeset -i chr=0
+typeset -i err=0
+typeset printf=$(type -p printf 2>/dev/null)
+
+unset LC_ALL
+unset LC_CTYPE
+export LANG=ja_JP.SJIS
+
+for second in $(seq 64 126); do
+    : $((chr++))
+    second=$(printf '%x' ${second})
+    mbchar="$(printf "\x81\x${second}")"
+    if test -z "${mbchar}" ; then
+	: $((err++))		# ERROR in builtin printf
+	continue
+    fi
+    if test -x "${printf}" ; then
+	if test $(${printf} "\x81\x${second}") != ${mbchar} ; then
+	    : $((err++))	# ERROR in builtin printf
+	    continue
+	fi
+    fi
+    uq=$(echo ${mbchar})
+    dq=$(echo "${mbchar}")
+    test "$uq" != "$dq" && let err+=1
+    test ${#uq} -ne 1 -o ${#dq} -ne 1 && let err+=1
+done
+
+if test $err -ne 0 ; then
+    : err_exit
+    : err_exit
+    print -u2 -n "\t"
+    print -u2 -r ${0##*/}[$LINENO]: "Shift-JIS encoding failed"
+fi
+exit $err
--- src/lib/libast/comp/setlocale.c
+++ src/lib/libast/comp/setlocale.c	2007-03-28 18:24:16.098860000 +0200
@@ -32,6 +32,7 @@
 
 #include "lclib.h"
 
+#include <ast.h>
 #include <ast_wchar.h>
 #include <ctype.h>
 #include <mc.h>
--- src/lib/libast/comp/wc.c
+++ src/lib/libast/comp/wc.c	2006-04-19 12:17:59.000000000 +0200
@@ -26,6 +26,7 @@
  */
 
 #include <ast.h>
+#include <ast_wchar.h>
 #include <wchar.h>
 
 #define STUB	1
--- src/lib/libast/features/wchar
+++ src/lib/libast/features/wchar	2006-04-19 12:25:08.000000000 +0200
@@ -1,5 +1,6 @@
 set	prototyped
-lib	mbstowcs,wctomb,wcrtomb,wcslen,wcstombs,wcwidth stdlib.h stdio.h wchar.h
+lib	mbstowcs,wctomb,wcrtomb,wcslen,wcstombs,wcscpy,wcwidth stdlib.h stdio.h wchar.h wctype.h
+lib	iswprint,iswalpha,iswalnum,iswdigit,iswcntrl stdlib.h stdio.h ctype.h wctype.h
 lib	towlower,towupper stdlib.h stdio.h wchar.h
 typ	mbstate_t stdlib.h stdio.h wchar.h
 nxt	wchar
@@ -31,6 +32,12 @@ cat <<!
 	#undef	putwc
 	#undef	putwchar
 	#undef	ungetwc
+	#undef	fwprintf
+	#undef	swprintf
+	#undef	vfwprintf
+	#undef	vswprintf
+	#undef	vwprintf
+	#undef	wprintf
 
 	#define fgetwc		_ast_fgetwc
 	#define fgetws		_ast_fgetws
@@ -81,6 +88,12 @@ cat <<!
 	#if !_lib_wcstombs
 	extern size_t		wcstombs(char*, const wchar_t*, size_t);
 	#endif
+	#if !_lib_wcscpy
+	extern wchar_t		*wcscpy(wchar_t*t, const wchar_t*);
+	#endif
+	#if !_lib_wcwidth
+	extern int		int wcwidth(wchar_t c);
+	#endif
 
 	extern int		fwprintf(FILE*, const wchar_t*, ...);
 	extern int		fwscanf(FILE*, const wchar_t*, ...);
--- src/lib/libast/include/ast.h
+++ src/lib/libast/include/ast.h	2007-11-20 11:55:01.000000000 +0000
@@ -176,13 +176,16 @@ typedef struct
 #define mbcoll()	(ast.mb_xfrm!=0)
 #define mbwide()	(mbmax()>1)
 
-#define mbchar(p)	(mbwide()?((ast.tmp_int=(*ast.mb_towc)(&ast.tmp_wchar,(char*)(p),mbmax()))>0?((p+=ast.tmp_int),ast.tmp_wchar):(p++,ast.tmp_int)):(*(unsigned char*)(p++)))
 #define mbinit()	(mbwide()?(*ast.mb_towc)((wchar_t*)0,(char*)0,mbmax()):0)
 #define mbsize(p)	(mbwide()?(*ast.mb_len)((char*)(p),mbmax()):((p),1))
 #define mbconv(s,w)	(ast.mb_conv?(*ast.mb_conv)(s,w):((*(s)=(w)),1))
 #define mbwidth(w)	(ast.mb_width&&((ast.tmp_int=(*ast.mb_width)(w))>=0||(w)>UCHAR_MAX)?ast.tmp_int:1)
 #define mbxfrm(t,f,n)	(mbcoll()?(*ast.mb_xfrm)((char*)(t),(char*)(f),n):0)
-
+#define mbchar(p)	((mbsize(p)>1) ? \
+				(((ast.tmp_int=(*ast.mb_towc)(&ast.tmp_wchar,(char*)(p),mbmax()))>0) ? \
+					((p+=ast.tmp_int),ast.tmp_wchar) : \
+					(*(unsigned char*)(p++))) : \
+				(*(unsigned char*)(p++)))
 /*
  * common macros
  */
--- src/lib/libast/regex/reglib.h
+++ src/lib/libast/regex/reglib.h	2006-04-19 12:17:59.000000000 +0200
@@ -57,6 +57,7 @@ typedef struct regsubop_s
 	char		re_rhs[1];	/* substitution rhs		*/
 
 #include <ast.h>
+#include <ast_wchar.h>
 #include <cdt.h>
 #include <stk.h>
 
--- src/lib/libcmd/Mamfile
+++ src/lib/libcmd/Mamfile	2007-03-28 18:25:32.624884000 +0200
@@ -454,7 +454,7 @@ make cat.o
 prev cat.c
 meta cat.o %.c>%.o cat.c cat
 prev cat.c
-exec - ${CC} ${mam_cc_FLAGS} ${CCFLAGS} -I. -I${PACKAGE_ast_INCLUDE} -DERROR_CATALOG=\""libcmd"\" -DUSAGE_LICENSE=\""[-author?Glenn Fowler <gsf@research.att.com>][-author?David Korn <dgk@research.att.com>][-copyright?Copyright (c) 1992-2007 AT&T Knowledge Ventures][-license?http://www.opensource.org/licenses/cpl1.0.txt][--catalog?libcmd]"\" -D_BLD_cmd -D_PACKAGE_ast -c cat.c
+exec - ${CC} ${mam_cc_FLAGS} ${CCFLAGS} -I. -I${PACKAGE_ast_INCLUDE} -DERROR_CATALOG=\""libcmd"\" -DUSAGE_LICENSE=\""[-author?Glenn Fowler <gsf@research.att.com>][-author?David Korn <dgk@research.att.com>][-copyright?Copyright (c) 1992-2007 AT&T Knowledge Ventures][-license?http://www.opensource.org/licenses/cpl1.0.txt][--catalog?libcmd]"\" -D_BLD_cmd -D_PACKAGE_ast -DSHOPT_MULTIBYTE -c cat.c
 done cat.o generated
 make chgrp.o
 prev chgrp.c
--- src/lib/libcmd/cat.c
+++ src/lib/libcmd/cat.c	2007-03-28 18:28:43.841928000 +0200
@@ -136,9 +136,11 @@ vcat(register char* states, Sfio_t *fdin
 		{
 			cpold = cp;
 			/* skip over printable characters */
+#if SHOPT_MULTIBYTE
 			if (mbwide())
 				while ((n = (m = mbsize(cp)) < 2 ? states[*cp++] : (cp += m, states['a'])) == 0);
 			else
+#endif
 				while ((n = states[*cp++]) == 0);
 			if (n==T_ENDBUF)
 			{