gap/gap-ttyname-null.diff

parent 96502953fcae60a727bae2866243bd1f95756d33 ()
commit 94d7cff5aad36fb2f4ad8580fb813e2650905719
Author: Jan Engelhardt <jengelh@medozas.de>
Date:   Mon Jun 27 10:45:18 2011 +0200

system: fix crash related to ttyname

There are several problems with this code.

1. ttyname can return NULL, which invokes undefined behavior
when passed to strcmp.

2. ttyname may be using a static buffer, the comparison with strcmp
could potentially always yield true, so the result needs to be stored
away first.
---
 src/system.c |   36 ++++++++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 8 deletions(-)

Index: gap4r6/src/system.c
===================================================================
--- gap4r6.orig/src/system.c
+++ gap4r6/src/system.c
@@ -1819,12 +1819,22 @@ void InitSystem (
     syBuf[0].fp = fileno(stdin);
     syBuf[0].bufno = -1;
     if ( isatty( fileno(stdin) ) ) {
-        if ( isatty( fileno(stdout) )
-          && ! strcmp( ttyname(fileno(stdin)), ttyname(fileno(stdout)) ) )
+	char *in, *out;
+
+	in = ttyname(fileno(stdin));
+	if (in != NULL)
+		in = strdup(in);
+	out = isatty(fileno(stdout)) ? ttyname(fileno(stdout)) : NULL;
+	if (out != NULL)
+		out = strdup(out);
+
+	if (in != NULL && out != NULL && strcmp(in, out) == 0)
             syBuf[0].echo = fileno(stdout);
-        else
-            syBuf[0].echo = open( ttyname(fileno(stdin)), O_WRONLY );
+	else if (in != NULL)
+            syBuf[0].echo = open(in, O_WRONLY);
         syBuf[0].isTTY = 1;
+	free(in);
+	free(out);
     }
     else {
         syBuf[0].echo = fileno(stdout);
@@ -1833,13 +1843,23 @@ void InitSystem (
     syBuf[1].echo = syBuf[1].fp = fileno(stdout); 
     syBuf[1].bufno = -1;
     if ( isatty( fileno(stderr) ) ) {
-        if ( isatty( fileno(stdin) )
-          && ! strcmp( ttyname(fileno(stdin)), ttyname(fileno(stderr)) ) )
+	char *in, *err;
+
+	in = isatty(fileno(stdin)) ? ttyname(fileno(stdin)) : NULL;
+	if (in != NULL)
+		in = strdup(in);
+	err = ttyname(fileno(stderr));
+	if (err != NULL)
+		err = strdup(err);
+
+	if (in != NULL && err != NULL && strcmp(in, err) == 0)
             syBuf[2].fp = fileno(stdin);
-        else
-            syBuf[2].fp = open( ttyname(fileno(stderr)), O_RDONLY );
+	else if (err != NULL)
+            syBuf[2].fp = open(err, O_RDONLY);
         syBuf[2].echo = fileno(stderr);
         syBuf[2].isTTY = 1;
+	free(in);
+	free(err);
     }
     else
       syBuf[2].isTTY = 0;