Re: PATCH: (New) bug with multibyte characters

[Peter Stephenson <pws@xxxxxxx>]

Peter Stephenson <pws@xxxxxxx> wrote:
> My change of INULL() was too wide-ranging and caused strange features,
> particularly with Cyrillic characters where bytes were matched by INULL()
> but not by imeta().

Come to think of it, it would be a much better plan to use the same form of
macro as every other test.

Index: Src/exec.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/exec.c,v
retrieving revision 1.93
diff -u -r1.93 exec.c
--- Src/exec.c	23 Sep 2005 17:03:17 -0000	1.93
+++ Src/exec.c	13 Oct 2005 16:19:52 -0000
@@ -2864,7 +2864,7 @@
     char *s, *t, *bptr, c;
 
     for (s = str; *s; s++)
-	if (INULL(*s)) {
+	if (inull(*s)) {
 	    qt = 1;
 	    break;
 	}
Index: Src/glob.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/glob.c,v
retrieving revision 1.47
diff -u -r1.47 glob.c
--- Src/glob.c	11 Oct 2005 16:48:06 -0000	1.47
+++ Src/glob.c	13 Oct 2005 16:19:53 -0000
@@ -2588,13 +2588,13 @@
 		 * pattern matching.
 		 */
 		continue;
-	    } else if (INULL(c)) {
+	    } else if (inull(c)) {
 		char *t = s - 1;
 
 		while ((c = *s++)) {
 		    if (c == Bnullkeep)
 			*t++ = '\\';
-		    else if (!INULL(c))
+		    else if (!inull(c))
 			*t++ = c;
 		}
 		*t = '\0';
Index: Src/params.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/params.c,v
retrieving revision 1.104
diff -u -r1.104 params.c
--- Src/params.c	20 Sep 2005 15:10:27 -0000	1.104
+++ Src/params.c	13 Oct 2005 16:19:53 -0000
@@ -1043,8 +1043,8 @@
     for (t = s, i = 0;
 	 (c = *t) && ((c != Outbrack &&
 		       (ishash || c != ',')) || i); t++) {
-	/* Untokenize INULL() except before brackets and double-quotes */
-	if (INULL(c)) {
+	/* Untokenize inull() except before brackets and double-quotes */
+	if (inull(c)) {
 	    c = t[1];
 	    if (c == '[' || c == ']' ||
 		c == '(' || c == ')' ||
@@ -1070,7 +1070,7 @@
 	return 0;
     s = dupstrpfx(s, t - s);
     *str = tt = t;
-    /* If we're NOT reverse subscripting, strip the INULL()s so brackets *
+    /* If we're NOT reverse subscripting, strip the inull()s so brackets *
      * are not backslashed after parsestr().  Otherwise leave them alone *
      * so that the brackets will be escaped when we patcompile() or when *
      * subscript arithmetic is performed (for nested subscripts).        */
@@ -1305,11 +1305,11 @@
 
     *s++ = '[';
     s = parse_subscript(s, dq);	/* Error handled after untokenizing */
-    /* Now we untokenize everything except INULL() markers so we can check *
-     * for the '*' and '@' special subscripts.  The INULL()s are removed  *
+    /* Now we untokenize everything except inull() markers so we can check *
+     * for the '*' and '@' special subscripts.  The inull()s are removed  *
      * in getarg() after we know whether we're doing reverse indexing.    */
     for (tbrack = *pptr + 1; *tbrack && tbrack != s; tbrack++) {
-	if (INULL(*tbrack) && !*++tbrack)
+	if (inull(*tbrack) && !*++tbrack)
 	    break;
 	if (itok(*tbrack))	/* Need to check for Nularg here? */
 	    *tbrack = ztokens[*tbrack - Pound];
Index: Src/subst.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/subst.c,v
retrieving revision 1.41
diff -u -r1.41 subst.c
--- Src/subst.c	11 Oct 2005 16:48:06 -0000	1.41
+++ Src/subst.c	13 Oct 2005 16:19:53 -0000
@@ -1406,7 +1406,7 @@
 		zerr("bad substitution", NULL, 0);
 		return NULL;
 	    }
-	} else if (inbrace && INULL(*s)) {
+	} else if (inbrace && inull(*s)) {
 	    /*
 	     * Handles things like ${(f)"$(<file)"} by skipping 
 	     * the double quotes.  We don't need to know what was
@@ -1460,7 +1460,7 @@
 	 * This tests for the second double quote in an expression
 	 * like ${(f)"$(<file)"}, compare above.
 	 */
-	while (INULL(*s))
+	while (inull(*s))
 	    s++;
 	v = (Value) NULL;
     } else if (aspar) {
@@ -1850,7 +1850,7 @@
 	 * we didn't have a subexpression, e.g. ${"foo"}.
 	 * This form is pointless, but logically it ought to work.
 	 */
-	while (INULL(*s))
+	while (inull(*s))
 	    s++;
     }
     /*
@@ -2846,11 +2846,11 @@
 		}
 		zsfree(hsubr);
 		for (tt = hsubl; *tt; tt++)
-		    if (INULL(*tt) && *tt != Bnullkeep)
+		    if (inull(*tt) && *tt != Bnullkeep)
 			chuck(tt--);
 		untokenize(hsubl);
 		for (tt = hsubr = ztrdup(ptr2); *tt; tt++)
-		    if (INULL(*tt) && *tt != Bnullkeep)
+		    if (inull(*tt) && *tt != Bnullkeep)
 			chuck(tt--);
 		ptr2[-1] = del;
 		if (sav)
Index: Src/utils.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/utils.c,v
retrieving revision 1.98
diff -u -r1.98 utils.c
--- Src/utils.c	3 Oct 2005 09:09:59 -0000	1.98
+++ Src/utils.c	13 Oct 2005 16:19:53 -0000
@@ -2562,8 +2562,10 @@
     typtab['\0'] |= IMETA;
     typtab[STOUC(Meta)  ] |= IMETA;
     typtab[STOUC(Marker)] |= IMETA;
-    for (t0 = (int)STOUC(Pound); t0 <= (int)STOUC(Nularg); t0++)
+    for (t0 = (int)STOUC(Pound); t0 <= (int)STOUC(Comma); t0++)
 	typtab[t0] |= ITOK | IMETA;
+    for (t0 = (int)STOUC(Snull); t0 <= (int)STOUC(Nularg); t0++)
+	typtab[t0] |= ITOK | IMETA | INULL;
     for (s = ifs ? ifs : DEFAULT_IFS; *s; s++) {
 	if (inblank(*s)) {
 	    if (s[1] == *s)
Index: Src/zsh.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/zsh.h,v
retrieving revision 1.78
diff -u -r1.78 zsh.h
--- Src/zsh.h	13 Oct 2005 13:19:30 -0000	1.78
+++ Src/zsh.h	13 Oct 2005 16:19:54 -0000
@@ -163,8 +163,6 @@
  */
 #define Nularg		((char) 0x9c)
 
-#define INULL(x)	((x) >= Snull && (x) <= Nularg)
-
 /*
  * Take care to update the use of IMETA appropriately when adding
  * tokens here.
Index: Src/ztype.h
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/ztype.h,v
retrieving revision 1.1.1.1
diff -u -r1.1.1.1 ztype.h
--- Src/ztype.h	15 Apr 1999 18:05:39 -0000	1.1.1.1
+++ Src/ztype.h	13 Oct 2005 16:19:54 -0000
@@ -41,6 +41,7 @@
 #define ISPECIAL (1 << 11)
 #define IMETA    (1 << 12)
 #define IWSEP    (1 << 13)
+#define INULL    (1 << 14)
 #define _icom(X,Y) (typtab[STOUC(X)] & Y)
 #define idigit(X) _icom(X,IDIGIT)
 #define ialnum(X) _icom(X,IALNUM)
@@ -56,3 +57,4 @@
 #define ispecial(X) _icom(X,ISPECIAL)
 #define imeta(X) _icom(X,IMETA)
 #define iwsep(X) _icom(X,IWSEP)
+#define inull(X) _icom(X,INULL)
Index: Src/Zle/compcore.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/compcore.c,v
retrieving revision 1.76
diff -u -r1.76 compcore.c
--- Src/Zle/compcore.c	18 Aug 2005 18:19:29 -0000	1.76
+++ Src/Zle/compcore.c	13 Oct 2005 16:19:54 -0000
@@ -1461,7 +1461,7 @@
 	autoq = NULL;
     }
     for (p = ns, i = swb; *p; p++, i++) {
-	if (INULL(*p)) {
+	if (inull(*p)) {
 	    if (i < scs) {
 		if (*p == Bnull) {
                     if (p[1] && remq)
Index: Src/Zle/compctl.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/compctl.c,v
retrieving revision 1.23
diff -u -r1.23 compctl.c
--- Src/Zle/compctl.c	10 Aug 2005 10:56:41 -0000	1.23
+++ Src/Zle/compctl.c	13 Oct 2005 16:19:54 -0000
@@ -2862,7 +2862,7 @@
 	autoq = "";
     }
     for (p = ns, i = swb; *p; p++, i++) {
-	if (INULL(*p)) {
+	if (inull(*p)) {
 	    if (i < scs) {
 		soffs--;
 		if (remq && *p == Bnull && p[1])
Index: Src/Zle/zle_tricky.c
===================================================================
RCS file: /cvsroot/zsh/zsh/Src/Zle/zle_tricky.c,v
retrieving revision 1.56
diff -u -r1.56 zle_tricky.c
--- Src/Zle/zle_tricky.c	29 Sep 2005 17:32:38 -0000	1.56
+++ Src/Zle/zle_tricky.c	13 Oct 2005 16:19:55 -0000
@@ -775,7 +775,7 @@
 	    char *w = dupstring(origword), *x, *q, *ox;
 
 	    for (q = w; *q; q++)
-		if (INULL(*q))
+		if (inull(*q))
 		    *q = Nularg;
 	    zlemetacs = wb;
 	    foredel(we - wb);
@@ -1015,7 +1015,7 @@
 has_real_token(const char *s)
 {
     while (*s) {
-	if (itok(*s) && !INULL(*s))
+	if (itok(*s) && !inull(*s))
 	    return 1;
 	s++;
     }
@@ -1487,7 +1487,7 @@
     }
     /* While building the quoted form, we also clean up the command line. */
     for (p = s, i = wb, j = 0; *p; p++, i++)
-	if (INULL(*p)) {
+	if (inull(*p)) {
 	    if (i < zlemetacs)
 		offs--;
 	    if (*p == Snull && isset(RCQUOTES))


-- 
Peter Stephenson <pws@xxxxxxx>                  Software Engineer
CSR PLC, Churchill House, Cambridge Business Park, Cowley Road
Cambridge, CB4 0WZ, UK                          Tel: +44 (0)1223 692070


This message has been scanned for viruses by BlackSpider MailControl - www.blackspider.com