gcc12/gcc12-fifo-jobserver-support.patch

From 1270ccda70ca09f7d4fe76b5156dca8992bd77a6 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 9 Aug 2022 13:59:32 +0200
Subject: [PATCH 1/3] Factor out jobserver_active_p.

gcc/ChangeLog:

	* gcc.cc (driver::detect_jobserver): Remove and move to
	jobserver.h.
	* lto-wrapper.cc (jobserver_active_p): Likewise.
	(run_gcc): Likewise.
	* opts-jobserver.h: New file.
	* opts-common.cc (jobserver_info::jobserver_info): New function.
---
 gcc/gcc.cc           | 37 +++++--------------------------------
 gcc/lto-wrapper.cc   | 44 +++++++++-----------------------------------
 gcc/opts-common.cc   | 41 +++++++++++++++++++++++++++++++++++++++++
 gcc/opts-jobserver.h | 44 ++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 99 insertions(+), 67 deletions(-)
 create mode 100644 gcc/opts-jobserver.h

diff --git a/gcc/gcc.cc b/gcc/gcc.cc
index 5cbb38560b2..cac11c1a117 100644
--- a/gcc/gcc.cc
+++ b/gcc/gcc.cc
@@ -27,6 +27,7 @@ CC recognizes how to compile each input file by suffixes in the file names.
 Once it knows which kind of compilation to perform, the procedure for
 compilation is specified by a string called a "spec".  */
 
+#define INCLUDE_STRING
 #include "config.h"
 #include "system.h"
 #include "coretypes.h"
@@ -43,6 +44,7 @@ compilation is specified by a string called a "spec".  */
 #include "opts.h"
 #include "filenames.h"
 #include "spellcheck.h"
+#include "opts-jobserver.h"
 
 
 
@@ -9178,38 +9180,9 @@ driver::final_actions () const
 void
 driver::detect_jobserver () const
 {
-  /* Detect jobserver and drop it if it's not working.  */
-  const char *makeflags = env.get ("MAKEFLAGS");
-  if (makeflags != NULL)
-    {
-      const char *needle = "--jobserver-auth=";
-      const char *n = strstr (makeflags, needle);
-      if (n != NULL)
-	{
-	  int rfd = -1;
-	  int wfd = -1;
-
-	  bool jobserver
-	    = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
-	       && rfd > 0
-	       && wfd > 0
-	       && is_valid_fd (rfd)
-	       && is_valid_fd (wfd));
-
-	  /* Drop the jobserver if it's not working now.  */
-	  if (!jobserver)
-	    {
-	      unsigned offset = n - makeflags;
-	      char *dup = xstrdup (makeflags);
-	      dup[offset] = '\0';
-
-	      const char *space = strchr (makeflags + offset, ' ');
-	      if (space != NULL)
-		strcpy (dup + offset, space);
-	      xputenv (concat ("MAKEFLAGS=", dup, NULL));
-	    }
-	}
-    }
+  jobserver_info jinfo;
+  if (!jinfo.is_active && !jinfo.skipped_makeflags.empty ())
+    xputenv (jinfo.skipped_makeflags.c_str ());
 }
 
 /* Determine what the exit code of the driver should be.  */
diff --git a/gcc/lto-wrapper.cc b/gcc/lto-wrapper.cc
index 795ab74555c..1e8eba16dfb 100644
--- a/gcc/lto-wrapper.cc
+++ b/gcc/lto-wrapper.cc
@@ -37,6 +37,7 @@ along with GCC; see the file COPYING3.  If not see
    ./ccCJuXGv.lto.ltrans.o
 */
 
+#define INCLUDE_STRING
 #include "config.h"
 #include "system.h"
 #include "coretypes.h"
@@ -49,6 +50,8 @@ along with GCC; see the file COPYING3.  If not see
 #include "lto-section-names.h"
 #include "collect-utils.h"
 #include "opts-diagnostic.h"
+#include "opt-suggestions.h"
+#include "opts-jobserver.h"
 
 /* Environment variable, used for passing the names of offload targets from GCC
    driver to lto-wrapper.  */
@@ -1336,35 +1339,6 @@ init_num_threads (void)
 #endif
 }
 
-/* Test and return reason why a jobserver cannot be detected.  */
-
-static const char *
-jobserver_active_p (void)
-{
-  #define JS_PREFIX "jobserver is not available: "
-  #define JS_NEEDLE "--jobserver-auth="
-
-  const char *makeflags = getenv ("MAKEFLAGS");
-  if (makeflags == NULL)
-    return JS_PREFIX "%<MAKEFLAGS%> environment variable is unset";
-
-  const char *n = strstr (makeflags, JS_NEEDLE);
-  if (n == NULL)
-    return JS_PREFIX "%<" JS_NEEDLE "%> is not present in %<MAKEFLAGS%>";
-
-  int rfd = -1;
-  int wfd = -1;
-
-  if (sscanf (n + strlen (JS_NEEDLE), "%d,%d", &rfd, &wfd) == 2
-      && rfd > 0
-      && wfd > 0
-      && is_valid_fd (rfd)
-      && is_valid_fd (wfd))
-    return NULL;
-  else
-    return JS_PREFIX "cannot access %<" JS_NEEDLE "%> file descriptors";
-}
-
 /* Print link to -flto documentation with a hint message.  */
 
 void
@@ -1422,7 +1396,6 @@ run_gcc (unsigned argc, char *argv[])
   bool jobserver_requested = false;
   int auto_parallel = 0;
   bool no_partition = false;
-  const char *jobserver_error = NULL;
   bool fdecoded_options_first = true;
   vec<cl_decoded_option> fdecoded_options;
   fdecoded_options.create (16);
@@ -1653,14 +1626,14 @@ run_gcc (unsigned argc, char *argv[])
     }
   else
     {
-      jobserver_error = jobserver_active_p ();
-      if (jobserver && jobserver_error != NULL)
+      jobserver_info jinfo;
+      if (jobserver && !jinfo.is_active)
 	{
 	  /* Fall back to auto parallelism.  */
 	  jobserver = 0;
 	  auto_parallel = 1;
 	}
-      else if (!jobserver && jobserver_error == NULL)
+      else if (!jobserver && jinfo.is_active)
 	{
 	  parallel = 1;
 	  jobserver = 1;
@@ -1971,9 +1944,10 @@ cont:
 
       if (nr > 1)
 	{
-	  if (jobserver_requested && jobserver_error != NULL)
+	  jobserver_info jinfo;
+	  if (jobserver_requested && !jinfo.is_active)
 	    {
-	      warning (0, jobserver_error);
+	      warning (0, jinfo.error_msg.c_str ());
 	      print_lto_docs_link ();
 	    }
 	  else if (parallel == 0)
diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index 8097c058c72..4d4f424df13 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -17,6 +17,7 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
+#define INCLUDE_STRING
 #include "config.h"
 #include "system.h"
 #include "intl.h"
@@ -25,6 +26,7 @@ along with GCC; see the file COPYING3.  If not see
 #include "options.h"
 #include "diagnostic.h"
 #include "spellcheck.h"
+#include "opts-jobserver.h"
 
 static void prune_options (struct cl_decoded_option **, unsigned int *);
 
@@ -2005,3 +2007,42 @@ void prepend_xassembler_to_collect_as_options (const char *collect_as_options,
       obstack_1grow (o, '\'');
     }
 }
+
+jobserver_info::jobserver_info ()
+{
+  /* Detect jobserver and drop it if it's not working.  */
+  string js_needle = "--jobserver-auth=";
+
+  const char *envval = getenv ("MAKEFLAGS");
+  if (envval != NULL)
+    {
+      string makeflags = envval;
+      size_t n = makeflags.rfind (js_needle);
+      if (n != string::npos)
+	{
+	  if (sscanf (makeflags.c_str () + n + js_needle.size (),
+		      "%d,%d", &rfd, &wfd) == 2
+	      && rfd > 0
+	      && wfd > 0
+	      && is_valid_fd (rfd)
+	      && is_valid_fd (wfd))
+	    is_active = true;
+	  else
+	    {
+	      string dup = makeflags.substr (0, n);
+	      size_t pos = makeflags.find (' ', n);
+	      if (pos != string::npos)
+		dup += makeflags.substr (pos);
+	      skipped_makeflags = "MAKEFLAGS=" + dup;
+	      error_msg
+		= "cannot access %<" + js_needle + "%> file descriptors";
+	    }
+	}
+      error_msg = "%<" + js_needle + "%> is not present in %<MAKEFLAGS%>";
+    }
+  else
+    error_msg = "%<MAKEFLAGS%> environment variable is unset";
+
+  if (!error_msg.empty ())
+    error_msg = "jobserver is not available: " + error_msg;
+}
diff --git a/gcc/opts-jobserver.h b/gcc/opts-jobserver.h
new file mode 100644
index 00000000000..68ce188b84a
--- /dev/null
+++ b/gcc/opts-jobserver.h
@@ -0,0 +1,44 @@
+/* GNU make's jobserver related functionality.
+   Copyright (C) 2022 Free Software Foundation, Inc.
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3.  If not see
+<http://www.gnu.org/licenses/>.
+
+See dbgcnt.def for usage information.  */
+
+#ifndef GCC_JOBSERVER_H
+#define GCC_JOBSERVER_H
+
+using namespace std;
+
+struct jobserver_info
+{
+  /* Default constructor.  */
+  jobserver_info ();
+
+  /* Error message if there is a problem.  */
+  string error_msg = "";
+  /* Skipped MAKEFLAGS where --jobserver-auth is skipped.  */
+  string skipped_makeflags = "";
+  /* File descriptor for reading used for jobserver communication.  */
+  int rfd = -1;
+  /* File descriptor for writing used for jobserver communication.  */
+  int wfd = -1;
+  /* Return true if jobserver is active.  */
+  bool is_active = false;
+};
+
+#endif /* GCC_JOBSERVER_H */
-- 
2.37.2

From 53e3b2bf16a486c15c20991c6095f7be09012b55 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 9 Aug 2022 13:59:36 +0200
Subject: [PATCH 2/3] lto: support --jobserver-style=fifo for recent GNU make

gcc/ChangeLog:

	* opts-jobserver.h: Add one member.
	* opts-common.cc (jobserver_info::jobserver_info): Parse FIFO
	format of --jobserver-auth.
---
 gcc/opts-common.cc   | 17 +++++++++++++++--
 gcc/opts-jobserver.h |  2 ++
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index 4d4f424df13..c2993f9140a 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -2010,8 +2010,14 @@ void prepend_xassembler_to_collect_as_options (const char *collect_as_options,
 
 jobserver_info::jobserver_info ()
 {
+  /* Traditionally, GNU make uses opened pipes for jobserver-auth,
+    e.g. --jobserver-auth=3,4.
+    Starting with GNU make 4.4, one can use --jobserver-style=fifo
+    and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta.  */
+
   /* Detect jobserver and drop it if it's not working.  */
   string js_needle = "--jobserver-auth=";
+  string fifo_prefix = "fifo:";
 
   const char *envval = getenv ("MAKEFLAGS");
   if (envval != NULL)
@@ -2020,8 +2026,15 @@ jobserver_info::jobserver_info ()
       size_t n = makeflags.rfind (js_needle);
       if (n != string::npos)
 	{
-	  if (sscanf (makeflags.c_str () + n + js_needle.size (),
-		      "%d,%d", &rfd, &wfd) == 2
+	  string ending = makeflags.substr (n + js_needle.size ());
+	  if (ending.find (fifo_prefix) == 0)
+	    {
+	      ending = ending.substr (fifo_prefix.size ());
+	      pipe_path = ending.substr (0, ending.find (' '));
+	      is_active = true;
+	    }
+	  else if (sscanf (makeflags.c_str () + n + js_needle.size (),
+			   "%d,%d", &rfd, &wfd) == 2
 	      && rfd > 0
 	      && wfd > 0
 	      && is_valid_fd (rfd)
diff --git a/gcc/opts-jobserver.h b/gcc/opts-jobserver.h
index 68ce188b84a..98ea2579962 100644
--- a/gcc/opts-jobserver.h
+++ b/gcc/opts-jobserver.h
@@ -37,6 +37,8 @@ struct jobserver_info
   int rfd = -1;
   /* File descriptor for writing used for jobserver communication.  */
   int wfd = -1;
+  /* Named pipe path.  */
+  string pipe_path = "";
   /* Return true if jobserver is active.  */
   bool is_active = false;
 };
-- 
2.37.2

From fed766af32ed6cd371016cc24e931131e19b4eb1 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 9 Aug 2022 13:59:39 +0200
Subject: [PATCH 3/3] lto: respect jobserver in parallel WPA streaming

	PR lto/106328

gcc/ChangeLog:

	* opts-jobserver.h (struct jobserver_info): Add pipefd.
	(jobserver_info::connect): New.
	(jobserver_info::disconnect): Likewise.
	(jobserver_info::get_token): Likewise.
	(jobserver_info::return_token): Likewise.
	* opts-common.cc: Implement the new functions.

gcc/lto/ChangeLog:

	* lto.cc (wait_for_child): Decrement nruns once a process
	finishes.
	(stream_out_partitions): Use job server if active.
	(do_whole_program_analysis): Likewise.
---
 gcc/lto/lto.cc       | 58 +++++++++++++++++++++++++++++++++-----------
 gcc/opts-common.cc   | 40 ++++++++++++++++++++++++++++++
 gcc/opts-jobserver.h | 14 +++++++++++
 3 files changed, 98 insertions(+), 14 deletions(-)

diff --git a/gcc/lto/lto.cc b/gcc/lto/lto.cc
index 31b0c1862f7..c82307f4f7e 100644
--- a/gcc/lto/lto.cc
+++ b/gcc/lto/lto.cc
@@ -18,6 +18,7 @@ You should have received a copy of the GNU General Public License
 along with GCC; see the file COPYING3.  If not see
 <http://www.gnu.org/licenses/>.  */
 
+#define INCLUDE_STRING
 #include "config.h"
 #include "system.h"
 #include "coretypes.h"
@@ -54,11 +55,17 @@ along with GCC; see the file COPYING3.  If not see
 #include "attribs.h"
 #include "builtins.h"
 #include "lto-common.h"
+#include "opts-jobserver.h"
 
-
-/* Number of parallel tasks to run, -1 if we want to use GNU Make jobserver.  */
+/* Number of parallel tasks to run.  */
 static int lto_parallelism;
 
+/* Number of active WPA streaming processes.  */
+static int nruns = 0;
+
+/* GNU make's jobserver info.  */
+static jobserver_info *jinfo = NULL;
+
 /* Return true when NODE has a clone that is analyzed (i.e. we need
    to load its body even if the node itself is not needed).  */
 
@@ -205,6 +212,12 @@ wait_for_child ()
 		     "streaming subprocess was killed by signal");
     }
   while (!WIFEXITED (status) && !WIFSIGNALED (status));
+
+    --nruns;
+
+    /* Return token to the jobserver if active.  */
+    if (jinfo != NULL && jinfo->is_active)
+      jinfo->return_token ();
 }
 #endif
 
@@ -228,25 +241,35 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max,
 		       bool ARG_UNUSED (last))
 {
 #ifdef HAVE_WORKING_FORK
-  static int nruns;
-
   if (lto_parallelism <= 1)
     {
       stream_out_partitions_1 (temp_filename, blen, min, max);
       return;
     }
 
-  /* Do not run more than LTO_PARALLELISM streamings
-     FIXME: we ignore limits on jobserver.  */
   if (lto_parallelism > 0 && nruns >= lto_parallelism)
-    {
-      wait_for_child ();
-      nruns --;
-    }
+    wait_for_child ();
+
   /* If this is not the last parallel partition, execute new
      streaming process.  */
   if (!last)
     {
+      if (jinfo != NULL && jinfo->is_active)
+	while (true)
+	  {
+	    if (jinfo->get_token ())
+	      break;
+	    if (nruns > 0)
+	      wait_for_child ();
+	    else
+	      {
+		/* There are no free tokens, lets do the job outselves.  */
+		stream_out_partitions_1 (temp_filename, blen, min, max);
+		asm_nodes_output = true;
+		return;
+	      }
+	  }
+
       pid_t cpid = fork ();
 
       if (!cpid)
@@ -264,10 +287,12 @@ stream_out_partitions (char *temp_filename, int blen, int min, int max,
   /* Last partition; stream it and wait for all children to die.  */
   else
     {
-      int i;
       stream_out_partitions_1 (temp_filename, blen, min, max);
-      for (i = 0; i < nruns; i++)
+      while (nruns > 0)
 	wait_for_child ();
+
+      if (jinfo != NULL && jinfo->is_active)
+	jinfo->disconnect ();
     }
   asm_nodes_output = true;
 #else
@@ -460,9 +485,14 @@ do_whole_program_analysis (void)
 
   lto_parallelism = 1;
 
-  /* TODO: jobserver communication is not supported, yet.  */
   if (!strcmp (flag_wpa, "jobserver"))
-    lto_parallelism = param_max_lto_streaming_parallelism;
+    {
+      jinfo = new jobserver_info ();
+      if (jinfo->is_active)
+	jinfo->connect ();
+
+      lto_parallelism = param_max_lto_streaming_parallelism;
+    }
   else
     {
       lto_parallelism = atoi (flag_wpa);
diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index c2993f9140a..5d79f174a38 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -2059,3 +2059,43 @@ jobserver_info::jobserver_info ()
   if (!error_msg.empty ())
     error_msg = "jobserver is not available: " + error_msg;
 }
+
+void
+jobserver_info::connect ()
+{
+  if (!pipe_path.empty ())
+    pipefd = open (pipe_path.c_str (), O_RDWR);
+}
+
+void
+jobserver_info::disconnect ()
+{
+  if (!pipe_path.empty ())
+    {
+      gcc_assert (close (pipefd) == 0);
+      pipefd = -1;
+    }
+}
+
+bool
+jobserver_info::get_token ()
+{
+  int fd = pipe_path.empty () ? rfd : pipefd;
+  char c;
+  unsigned n = read (fd, &c, 1);
+  if (n != 1)
+    {
+      gcc_assert (errno == EAGAIN);
+      return false;
+    }
+  else
+    return true;
+}
+
+void
+jobserver_info::return_token ()
+{
+  int fd = pipe_path.empty () ? wfd : pipefd;
+  char c = 'G';
+  gcc_assert (write (fd, &c, 1) == 1);
+}
diff --git a/gcc/opts-jobserver.h b/gcc/opts-jobserver.h
index 98ea2579962..76c1d9b2882 100644
--- a/gcc/opts-jobserver.h
+++ b/gcc/opts-jobserver.h
@@ -29,6 +29,18 @@ struct jobserver_info
   /* Default constructor.  */
   jobserver_info ();
 
+  /* Connect to the server.  */
+  void connect ();
+
+  /* Disconnect from the server.  */
+  void disconnect ();
+
+  /* Get token from the server.  */
+  bool get_token ();
+
+  /* Return token to the server.  */
+  void return_token ();
+
   /* Error message if there is a problem.  */
   string error_msg = "";
   /* Skipped MAKEFLAGS where --jobserver-auth is skipped.  */
@@ -39,6 +51,8 @@ struct jobserver_info
   int wfd = -1;
   /* Named pipe path.  */
   string pipe_path = "";
+  /* Pipe file descriptor.  */
+  int pipefd = -1;
   /* Return true if jobserver is active.  */
   bool is_active = false;
 };
-- 
2.37.2

From 3f1c2f89f6b8b8d23a9072f8549b0a2c1de06b03 Mon Sep 17 00:00:00 2001
From: Martin Liska <mliska@suse.cz>
Date: Tue, 16 Aug 2022 08:22:29 +0200
Subject: [PATCH] jobserver: fix fifo mode by opening pipe in proper mode

The current jobserver_info relies on non-blocking FDs,
thus one the pipe in such mode.

gcc/ChangeLog:

	* opts-common.cc (jobserver_info::connect): Open fifo
	in non-blocking mode.
---
 gcc/opts-common.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gcc/opts-common.cc b/gcc/opts-common.cc
index 5d79f174a38..4dec9f94447 100644
--- a/gcc/opts-common.cc
+++ b/gcc/opts-common.cc
@@ -2064,7 +2064,7 @@ void
 jobserver_info::connect ()
 {
   if (!pipe_path.empty ())
-    pipefd = open (pipe_path.c_str (), O_RDWR);
+    pipefd = open (pipe_path.c_str (), O_RDWR | O_NONBLOCK);
 }
 
 void
-- 
2.37.2