Use a pool of ABC processes.

Doing ABC runs in parallel can actually make things slower when every ABC run requires
spawning an ABC subprocess --- especially when using popen(), which on glibc does not
use vfork(). What seems to happen is that constant fork()ing keeps making the main
process data pages copy-on-write, so the main process code that is setting up each ABC
call takes a lot of minor page-faults, slowing it down.

The solution is pretty straightforward although a little tricky to implement.
We just reuse ABC subprocesses. Instead of passing the ABC script name on the command
line, we spawn an ABC REPL and pipe a command into it to source the script. When that's
done we echo an `ABC_DONE` token instead of exiting. Yosys then puts the ABC process
onto a stack which we can pull from the next time we do an ABC run.

For one of our large designs, this is an additional 5x speedup of the primary AbcPass.
It does 5155 ABC runs, all very small; runtime of the AbcPass goes from 760s to 149s
(not very scientific benchmarking but the effect size is large).

Author: rocallahan

kernel/threading.h

@@ -154,6 +154,33 @@ class ThreadPool
 #endif
 };
 
+template <class T>
+class ConcurrentStack
+{
+public:
+	void push_back(T &&t) {
+#ifdef YOSYS_ENABLE_THREADS
+		std::lock_guard<std::mutex> lock(mutex);
+#endif
+		contents.push_back(std::move(t));
+	}
+	std::optional<T> try_pop_back() {
+#ifdef YOSYS_ENABLE_THREADS
+		std::lock_guard<std::mutex> lock(mutex);
+#endif
+		if (contents.empty())
+			return std::nullopt;
+		T result = std::move(contents.back());
+		contents.pop_back();
+		return result;
+	}
+private:
+#ifdef YOSYS_ENABLE_THREADS
+	std::mutex mutex;
+#endif
+	std::vector<T> contents;
+};
+
 YOSYS_NAMESPACE_END
 
 #endif // YOSYS_THREADING_H

passes/techmap/abc.cc

@@ -59,6 +59,11 @@
 #include <memory>
 #include <vector>
 
+#ifdef __linux__
+#  include <fcntl.h>
+#  include <spawn.h>
+#  include <sys/wait.h>
+#endif
 #ifndef _WIN32
 #  include <unistd.h>
 #  include <dirent.h>
@@ -153,6 +158,121 @@ struct AbcSigVal {
 	}
 };
 
+#if defined(__linux__) && !defined(YOSYS_DISABLE_SPAWN)
+struct AbcProcess
+{
+	pid_t pid;
+	int to_child_pipe;
+	int from_child_pipe;
+
+	AbcProcess() : pid(0), to_child_pipe(-1), from_child_pipe(-1) {}
+	AbcProcess(AbcProcess &&other) {
+		pid = other.pid;
+		to_child_pipe = other.to_child_pipe;
+		from_child_pipe = other.from_child_pipe;
+		other.pid = 0;
+		other.to_child_pipe = other.from_child_pipe = -1;
+	}
+	AbcProcess &operator=(AbcProcess &&other) {
+		if (this != &other) {
+			pid = other.pid;
+			to_child_pipe = other.to_child_pipe;
+			from_child_pipe = other.from_child_pipe;
+			other.pid = 0;
+			other.to_child_pipe = other.from_child_pipe = -1;
+		}
+		return *this;
+	}
+	~AbcProcess() {
+		if (pid == 0)
+			return;
+		if (to_child_pipe >= 0)
+			close(to_child_pipe);
+		int status;
+		int ret = waitpid(pid, &status, 0);
+		if (ret != pid) {
+			log_error("waitpid(%d) failed", pid);
+		}
+		if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) {
+			log_error("ABC failed with status %X", status);
+		}
+		if (from_child_pipe >= 0)
+			close(from_child_pipe);
+	}
+};
+
+std::optional<AbcProcess> spawn_abc(const char* abc_exe, DeferredLogs &logs) {
+	// Open pipes O_CLOEXEC so we don't leak any of the fds into racing
+	// fork()s.
+	int to_child_pipe[2];
+	if (pipe2(to_child_pipe, O_CLOEXEC) != 0) {
+		logs.log_error("pipe failed");
+		return std::nullopt;
+	}
+	int from_child_pipe[2];
+	if (pipe2(from_child_pipe, O_CLOEXEC) != 0) {
+		logs.log_error("pipe failed");
+		return std::nullopt;
+	}
+
+	AbcProcess result;
+	result.to_child_pipe = to_child_pipe[1];
+	result.from_child_pipe = from_child_pipe[0];
+	// Allow the child side of the pipes to be inherited.
+	fcntl(to_child_pipe[0], F_SETFD, 0);
+	fcntl(from_child_pipe[1], F_SETFD, 0);
+
+	posix_spawn_file_actions_t file_actions;
+	if (posix_spawn_file_actions_init(&file_actions) != 0) {
+		logs.log_error("posix_spawn_file_actions_init failed");
+		return std::nullopt;
+	}
+
+	if (posix_spawn_file_actions_addclose(&file_actions, to_child_pipe[1]) != 0) {
+		logs.log_error("posix_spawn_file_actions_addclose failed");
+		return std::nullopt;
+	}
+	if (posix_spawn_file_actions_addclose(&file_actions, from_child_pipe[0]) != 0) {
+		logs.log_error("posix_spawn_file_actions_addclose failed");
+		return std::nullopt;
+	}
+	if (posix_spawn_file_actions_adddup2(&file_actions, to_child_pipe[0], STDIN_FILENO) != 0) {
+		logs.log_error("posix_spawn_file_actions_adddup2 failed");
+		return std::nullopt;
+	}
+	if (posix_spawn_file_actions_adddup2(&file_actions, from_child_pipe[1], STDOUT_FILENO) != 0) {
+		logs.log_error("posix_spawn_file_actions_adddup2 failed");
+		return std::nullopt;
+	}
+	if (posix_spawn_file_actions_adddup2(&file_actions, from_child_pipe[1], STDERR_FILENO) != 0) {
+		logs.log_error("posix_spawn_file_actions_adddup2 failed");
+		return std::nullopt;
+	}
+	if (posix_spawn_file_actions_addclose(&file_actions, to_child_pipe[0]) != 0) {
+		logs.log_error("posix_spawn_file_actions_addclose failed");
+		return std::nullopt;
+	}
+	if (posix_spawn_file_actions_addclose(&file_actions, from_child_pipe[1]) != 0) {
+		logs.log_error("posix_spawn_file_actions_addclose failed");
+		return std::nullopt;
+	}
+
+	char arg1[] = "-s";
+	char* argv[] = { strdup(abc_exe), arg1, nullptr };
+	if (0 != posix_spawn(&result.pid, abc_exe, &file_actions, nullptr, argv, environ)) {
+		logs.log_error("posix_spawn %s failed", abc_exe);
+		return std::nullopt;
+	}
+	free(argv[0]);
+	posix_spawn_file_actions_destroy(&file_actions);
+	close(to_child_pipe[0]);
+	close(from_child_pipe[1]);
+	return result;
+}
+#else
+struct AbcProcess {};
+#endif
+
 using AbcSigMap = SigValMap<AbcSigVal>;
 
 // Used by off-main-threads. Contains no direct or indirect access to RTLIL.
@@ -167,7 +287,7 @@ struct RunAbcState {
 	dict<int, std::string> pi_map, po_map;
 
 	RunAbcState(const AbcConfig &config) : config(config) {}
-	void run();
+	void run(ConcurrentStack<AbcProcess> &process_pool);
 };
 
 struct AbcModuleState {
@@ -1010,7 +1130,42 @@ void AbcModuleState::prepare_module(RTLIL::Design *design, RTLIL::Module *module
 	handle_loops(assign_map, module);
 }
 
-void RunAbcState::run()
+bool read_until_abc_done(abc_output_filter &filt, int fd, DeferredLogs &logs) {
+	std::string line;
+	char buf[1024];
+	while (true) {
+		int ret = read(fd, buf, sizeof(buf) - 1);
+		if (ret < 0) {
+			logs.log_error("Failed to read from ABC, errno=%d", errno);
+			return false;
+		}
+		if (ret == 0) {
+			logs.log_error("ABC exited prematurely");
+			return false;
+		}
+		char *start = buf;
+		char *end = buf + ret;
+		while (start < end) {
+			char *p = static_cast<char*>(memchr(start, '\n', end - start));
+			if (p == nullptr) {
+				break;
+			}
+			line.append(start, p + 1 - start);
+			// ABC seems to actually print "ABC_DONE \n", but we probably shouldn't
+			// rely on that extra space being output.
+			if (line.substr(0, 8) == "ABC_DONE") {
+				// Ignore any leftover output, there should only be a prompt perhaps
+				return true;
+			}
+			filt.next_line(line);
+			line.clear();
+			start = p + 1;
+		}
+		line.append(start, end - start);
+	}
+}
+
+void RunAbcState::run(ConcurrentStack<AbcProcess> &process_pool)
 {
 	std::string buffer = stringf("%s/input.blif", tempdir_name);
 	FILE *f = fopen(buffer.c_str(), "wt");
@@ -1137,14 +1292,12 @@ void RunAbcState::run()
 			count_gates, GetSize(signal_list), count_input, count_output);
 	if (count_output > 0)
 	{
-		buffer = stringf("\"%s\" -s -f %s/abc.script 2>&1", config.exe_file, tempdir_name);
-		logs.log("Running ABC command: %s\n", replace_tempdir(buffer, tempdir_name, config.show_tempdir));
+		std::string tmp_script_name = stringf("%s/abc.script", tempdir_name);
+		logs.log("Running ABC script: %s\n", replace_tempdir(tmp_script_name, tempdir_name, config.show_tempdir));
 
 		errno = 0;
-#ifndef YOSYS_LINK_ABC
 		abc_output_filter filt(*this, tempdir_name, config.show_tempdir);
-		int ret = run_command(buffer, std::bind(&abc_output_filter::next_line, filt, std::placeholders::_1));
-#else
+#ifdef YOSYS_LINK_ABC
 		string temp_stdouterr_name = stringf("%s/stdouterr.txt", tempdir_name);
 		FILE *temp_stdouterr_w = fopen(temp_stdouterr_name.c_str(), "w");
 		if (temp_stdouterr_w == NULL)
@@ -1165,7 +1318,6 @@ void RunAbcState::run()
 		fclose(temp_stdouterr_w);
 		// These needs to be mutable, supposedly due to getopt
 		char *abc_argv[5];
-		string tmp_script_name = stringf("%s/abc.script", tempdir_name);
 		abc_argv[0] = strdup(config.exe_file.c_str());
 		abc_argv[1] = strdup("-s");
 		abc_argv[2] = strdup("-f");
@@ -1183,13 +1335,40 @@ void RunAbcState::run()
 		fclose(old_stdout);
 		fclose(old_stderr);
 		std::ifstream temp_stdouterr_r(temp_stdouterr_name);
-		abc_output_filter filt(*this, tempdir_name, config.show_tempdir);
 		for (std::string line; std::getline(temp_stdouterr_r, line); )
 			filt.next_line(line + "\n");
 		temp_stdouterr_r.close();
+#elif defined(__linux__) && !defined(YOSYS_DISABLE_SPAWN)
+		AbcProcess process;
+		if (std::optional<AbcProcess> process_opt = process_pool.try_pop_back()) {
+			process = std::move(process_opt.value());
+		} else if (std::optional<AbcProcess> process_opt = spawn_abc(config.exe_file.c_str(), logs)) {
+			process = std::move(process_opt.value());
+		} else {
+			return;
+		}
+		std::string cmd = stringf(
+				// This makes ABC switch stdout to line buffering, which we need
+				// to see our ABC_DONE message.
+				"set abcout /dev/stdout\n"
+				"empty\n"
+				"source %s\n"
+				"echo \"ABC_DONE\"\n", tmp_script_name);
+		int ret = write(process.to_child_pipe, cmd.c_str(), cmd.size());
+		if (ret != static_cast<int>(cmd.size())) {
+			logs.log_error("write failed");
+			return;
+		}
+		ret = read_until_abc_done(filt, process.from_child_pipe, logs) ? 0 : 1;
+		if (ret == 0) {
+			process_pool.push_back(std::move(process));
+		}
+#else
+		std::string cmd = stringf("\"%s\" -s -f %s/abc.script 2>&1", config.exe_file.c_str(), tempdir_name.c_str());
+		int ret = run_command(cmd, std::bind(&abc_output_filter::next_line, filt, std::placeholders::_1));
 #endif
 		if (ret != 0) {
-			logs.log_error("ABC: execution of command \"%s\" failed: return code %d (errno=%d).\n", buffer, ret, errno);
+			logs.log_error("ABC: execution of script \"%s\" failed: return code %d (errno=%d).\n", tmp_script_name, ret, errno);
 			return;
 		}
 		did_run = true;
@@ -2205,7 +2384,8 @@ struct AbcPass : public Pass {
 
 				AbcModuleState state(config, initvals);
 				state.prepare_module(design, mod, assign_map, cells, dff_mode, clk_str);
-				state.run_abc.run();
+				ConcurrentStack<AbcProcess> process_pool;
+				state.run_abc.run(process_pool);
 				state.extract(assign_map, design, mod);
 				continue;
 			}
@@ -2379,11 +2559,12 @@ struct AbcPass : public Pass {
 			ConcurrentQueue<std::unique_ptr<AbcModuleState>> work_queue(num_worker_threads);
 			ConcurrentQueue<std::unique_ptr<AbcModuleState>> work_finished_queue;
 			int work_finished_count = 0;
+			ConcurrentStack<AbcProcess> process_pool;
 			ThreadPool worker_threads(num_worker_threads, [&](int){
 					while (std::optional<std::unique_ptr<AbcModuleState>> work =
 							work_queue.pop_front()) {
 						// Only the `run_abc` component is safe to touch here!
-						(*work)->run_abc.run();
+						(*work)->run_abc.run(process_pool);
 						work_finished_queue.push_back(std::move(*work));
 					}
 				});
@@ -2409,7 +2590,7 @@ struct AbcPass : public Pass {
 					work_queue.push_back(std::move(state));
 				} else {
 					// Just run everything on the main thread.
-					state->run_abc.run();
+					state->run_abc.run(process_pool);
 					work_finished_queue.push_back(std::move(state));
 				}
 			}