Re: Latest CVS + ZLE_UNICODE_SUPPORT on RHEL3

[Andrey Borzenkov <arvidjaar@xxxxxxxxxx>]

On Monday 21 February 2005 17:29, Peter Stephenson wrote:
> Borzenkov Andrey wrote:
> > The problem seems to be that currently zle returns to zsh array of
> > wchar_t; a nd zsh of course has no idea what to do with it (and treats it
> > as array of ch ar anyway). I started to attempt to fix it but then I
> > realized I do not know (and there is nothing in patch) what was intended:
> >
> > - convert wchar_t to something else (mbchar back? UTF-8?) before passing
> > it t o main zsh code.
>
> This should happen already.  zleread() now calls zlegetline() (instead
> of just returning the line it had already assembled) which calls
> zlelineasstring() in zle_utils.c.  For ZLE_UNICODE_SUPPORT this should
> convert the wchar_t array to a metafied multibyte string as used by the
> rest of the shell.
>

right, missed this.

> I haven't tested the combined effect of the patches so far at all, since
> I know there are bits that aren't fixed up yet.

attached patch makes it so far as being able to start zsh -f and really input 
something. The relevant parts are actually selfinsert (wchar_t != wint_t) and 
zlelineasstring (length was computed incorrectly and final NULL metafied). 
Everything else is just part of yesterday's futile attempts :)

Oh, and for some reasons mbrtowc(&outchar, &cnull, 1, &ps) segfaulted on me 
(Mandrake cooker).

I can't commit it as for some reason ssh access to CVS hangs. 

-andrey

Index: Src/system.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/system.h,v
retrieving revision 1.28
diff -u -p -r1.28 system.h
--- Src/system.h	18 Feb 2005 13:57:27 -0000	1.28
+++ Src/system.h	21 Feb 2005 19:08:51 -0000
@@ -689,6 +689,7 @@ extern short ospeed;
  */
 #if defined(HAVE_WCHAR_H) && defined(HAVE_WCTOMB) && defined (__STDC_ISO_10646__)
 # include <wchar.h>
+# include <wctype.h>
 
 /*
  * More stringent requirements to enable complete Unicode conversion
Index: Src/Zle/zle.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle.h,v
retrieving revision 1.8
diff -u -p -r1.8 zle.h
--- Src/Zle/zle.h	21 Feb 2005 11:20:24 -0000	1.8
+++ Src/Zle/zle.h	21 Feb 2005 19:08:52 -0000
@@ -47,6 +47,7 @@ typedef wint_t   ZLE_INT_T;
 #define ZLENL	L'\n'
 #define ZLENUL	L'\0'
 #define ZLETAB	L'\t'
+#define ZLESPC	L' '
 
 #define DIGIT_1		L'1'
 #define DIGIT_9		L'9'
@@ -75,6 +76,7 @@ typedef int ZLE_INT_T;
 #define ZLENL	'\n'
 #define ZLENUL	'\0'
 #define ZLETAB	'\t'
+#define ZLESPC	' '
 
 #define DIGIT_1		'1'
 #define DIGIT_9		'9'
Index: Src/Zle/zle_main.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_main.c,v
retrieving revision 1.59
diff -u -p -r1.59 zle_main.c
--- Src/Zle/zle_main.c	18 Feb 2005 13:57:28 -0000	1.59
+++ Src/Zle/zle_main.c	21 Feb 2005 19:08:53 -0000
@@ -752,6 +752,7 @@ getrestchar(int inchar)
     char buf[MB_CUR_MAX], *ptr;
     wchar_t outchar;
     int ret;
+    mbstate_t ps;
 
     /*
      * We are guaranteed to set a valid wide last character,
@@ -764,7 +765,8 @@ getrestchar(int inchar)
 	return lastchar_wide = WEOF;
 
     /* reset shift state by converting null */
-    mbrtowc(&outchar, &cnull, 1, &ps);
+    //mbrtowc(&outchar, &cnull, 1, &ps);
+    memset (&ps, '\0', sizeof (ps));
 
     ptr = buf;
     *ptr++ = inchar;
@@ -785,7 +787,7 @@ getrestchar(int inchar)
 	    return lastchar_wide = WEOF;
 	*ptr++ = inchar;
     }
-    return lastchar_wide = (wint_t)outchar;
+    return lastchar_wide = (ZLE_INT_T)outchar;
 }
 /**/
 #endif
Index: Src/Zle/zle_misc.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_misc.c,v
retrieving revision 1.16
diff -u -p -r1.16 zle_misc.c
--- Src/Zle/zle_misc.c	18 Feb 2005 17:31:11 -0000	1.16
+++ Src/Zle/zle_misc.c	21 Feb 2005 19:08:53 -0000
@@ -61,9 +61,13 @@ mod_export int
 selfinsert(UNUSED(char **args))
 {
 #ifdef ZLE_UNICODE_SUPPORT
+    /* wint_t and wchar_t not neccessarily the same size */
+    wchar_t tmp;
+
     if (!lastchar_wide_valid)
 	getrestchar(lastchar);
-    doinsert(&lastchar_wide, 1);
+    tmp = lastchar_wide;
+    doinsert(&tmp, 1);
 #else
     char s = lastchar;
     doinsert(&s, 1);
@@ -921,7 +925,7 @@ executenamedcommand(char *prmt)
 #ifdef ZLE_UNICODE_SUPPORT
 		    if (!lastchar_wide_valid)
 			getrestchar(0);
-		    if (iswcntrl(lastchar))
+		    if (iswcntrl(lastchar_wide))
 #else
 		    if (icntrl(lastchar))
 #endif
Index: Src/Zle/zle_tricky.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_tricky.c,v
retrieving revision 1.49
diff -u -p -r1.49 zle_tricky.c
--- Src/Zle/zle_tricky.c	18 Feb 2005 17:31:11 -0000	1.49
+++ Src/Zle/zle_tricky.c	21 Feb 2005 19:08:55 -0000
@@ -159,12 +159,12 @@ int hascompwidgets;
 static int
 usetab(void)
 {
-    unsigned char *s = zleline + zlecs - 1;
+    ZLE_STRING_T s = zleline + zlecs - 1;
 
     if (keybuf[0] != '\t' || keybuf[1])
 	return 0;
-    for (; s >= zleline && *s != '\n'; s--)
-	if (*s != '\t' && *s != ' ')
+    for (; s >= zleline && *s != ZLENL; s--)
+	if (*s != ZLETAB && *s != ZLESPC)
 	    return 0;
     if (compfunc) {
 	wouldinstab = 1;
@@ -866,7 +866,7 @@ addx(char **ptmp)
 	(instring && (zleline[zlecs] == '"' || zleline[zlecs] == '\'')) ||
 	(addspace = (comppref && !iblank(zleline[zlecs])))) {
 	*ptmp = (char *)zleline;
-	zleline = (unsigned char *)zhalloc(strlen((char *)zleline) + 3 +
+	zleline = (ZLE_STRING_T)zhalloc(strlen((char *)zleline) + 3 +
 					   addspace);
 	memcpy(zleline, *ptmp, zlecs);
 	zleline[zlecs] = 'x';
Index: Src/Zle/zle_utils.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_utils.c,v
retrieving revision 1.18
diff -u -p -r1.18 zle_utils.c
--- Src/Zle/zle_utils.c	18 Feb 2005 13:57:28 -0000	1.18
+++ Src/Zle/zle_utils.c	21 Feb 2005 19:08:55 -0000
@@ -111,22 +111,22 @@ zlelineasstring(ZLE_STRING_T instr, int 
 {
 #ifdef ZLE_UNICODE_SUPPORT
     char *s;
-    char *mb_cursor;
     int i, j;
     size_t mb_len = 0;
 
-    mb_cursor = s = zalloc(inll * MB_CUR_MAX);
+    s = zalloc(inll * MB_CUR_MAX + 1);
 
-    for(i=0;i<=inll;i++) {
+    for(i=0; i < inll; i++) {
 	if (outcs != NULL && i == incs)
 	    *outcs = mb_len;
-	j = wctomb(mb_cursor, instr[i]);
+	j = wctomb(s + mb_len, instr[i]);
 	if (j == -1) {
 	    /* invalid char; what to do? */
 	} else {
 	    mb_len += j;
 	}
     }
+    s[mb_len] = '\0';
 
     if (outll != NULL)
 	*outll = mb_len;
@@ -135,7 +135,7 @@ zlelineasstring(ZLE_STRING_T instr, int 
 	unsigned char *ret =
 	    (unsigned char *) metafy((char *) s, mb_len, META_HEAPDUP);
 
-	zfree((char *)s, inll * MB_CUR_MAX);
+	zfree((char *)s, inll * MB_CUR_MAX + 1);
 
 	return ret;
     }