WIP: PG-1710 Create helpers for decrypting/encrypting archived WAL

contrib/pg_tde/.gitignore

@@ -11,7 +11,9 @@ __pycache__
 /configure~
 /log
 /results
-/src/pg_tde_change_key_provider
+/src/bin/pg_tde_archive_decrypt
+/src/bin/pg_tde_change_key_provider
+/src/bin/pg_tde_restore_encrypt
 /t/results
 /tmp_check
 

contrib/pg_tde/Makefile

@@ -51,10 +51,16 @@ src/libkmip/libkmip/src/kmip_bio.o \
 src/libkmip/libkmip/src/kmip_locate.o \
 src/libkmip/libkmip/src/kmip_memset.o
 
-SCRIPTS_built = src/pg_tde_change_key_provider
+SCRIPTS_built = src/bin/pg_tde_archive_decrypt \
+src/bin/pg_tde_change_key_provider \
+src/bin/pg_tde_restore_encrypt
 
 EXTRA_INSTALL += contrib/pg_buffercache contrib/test_decoding
-EXTRA_CLEAN += src/pg_tde_change_key_provider.o
+EXTRA_CLEAN += src/bin/pg_tde_archive_decrypt.o \
+src/bin/pg_tde_change_key_provider.o \
+src/bin/pg_tde_restore_encrypt.o \
+xlogreader.c \
+xlogreader.o
 
 ifdef USE_PGXS
 PG_CONFIG = pg_config
@@ -71,9 +77,21 @@ endif
 
 override SHLIB_LINK += -lcurl -lcrypto -lssl
 
-src/pg_tde_change_key_provider: src/pg_tde_change_key_provider.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtde.a
+src/bin/pg_tde_change_key_provider: src/bin/pg_tde_change_key_provider.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtde.a
 	$(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
 
+src/bin/pg_tde_archive_decrypt: src/bin/pg_tde_archive_decrypt.o xlogreader.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtdexlog.a $(top_builddir)/src/libtde/libtde.a
+	$(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+
+src/bin/pg_tde_restore_encrypt: src/bin/pg_tde_restore_encrypt.o xlogreader.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtdexlog.a $(top_builddir)/src/libtde/libtde.a
+	$(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+
+xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/%
+	rm -f $@ && $(LN_S) $< .
+
+xlogreader.o: xlogreader.c
+	$(CC) $(CPPFLAGS) -DFRONTEND -c $< -o $@
+
 # Fetches typedefs list for PostgreSQL core and merges it with typedefs defined in this project.
 # https://wiki.postgresql.org/wiki/Running_pgindent_on_non-core_code_or_development_code
 update-typedefs:

contrib/pg_tde/meson.build

@@ -113,6 +113,7 @@ tap_tests = [
   't/rotate_key.pl',
   't/tde_heap.pl',
   't/unlogged_tables.pl',
+  't/wal_archiving.pl',
   't/wal_encrypt.pl',
 ]
 
@@ -142,10 +143,10 @@ pg_tde_frontend = static_library('pg_tde_frontend',
 )
 
 pg_tde_change_key_provider_sources = files(
-  'src/pg_tde_change_key_provider.c',
+  'src/bin/pg_tde_change_key_provider.c',
 )
 
-pg_tde_change_key_provider = executable('pg_tde_change_key_provider',
+pg_tde_change_key_provider = executable('bin/pg_tde_change_key_provider',
   pg_tde_change_key_provider_sources,
   dependencies: [frontend_code, lz4, zstd],
   c_args: ['-DFRONTEND'], # needed for xlogreader et al
@@ -154,3 +155,31 @@ pg_tde_change_key_provider = executable('pg_tde_change_key_provider',
   link_with: [pg_tde_frontend]
 )
 contrib_targets += pg_tde_change_key_provider
+
+pg_tde_archive_decrypt_sources = files(
+  'src/bin/pg_tde_archive_decrypt.c',
+) + xlogreader_sources
+
+pg_tde_archive_decrypt = executable('bin/pg_tde_archive_decrypt',
+  pg_tde_archive_decrypt_sources,
+  dependencies: [frontend_code],
+  c_args: ['-DFRONTEND'], # needed for xlogreader et al
+  kwargs: default_bin_args,
+  include_directories: [postgres_inc, pg_tde_inc],
+  link_with: [pg_tde_frontend]
+)
+contrib_targets += pg_tde_archive_decrypt
+
+pg_tde_restore_encrypt_sources = files(
+  'src/bin/pg_tde_restore_encrypt.c',
+) + xlogreader_sources
+
+pg_tde_restore_encrypt = executable('bin/pg_tde_restore_encrypt',
+  pg_tde_restore_encrypt_sources,
+  dependencies: [frontend_code],
+  c_args: ['-DFRONTEND'], # needed for xlogreader et al
+  kwargs: default_bin_args,
+  include_directories: [postgres_inc, pg_tde_inc],
+  link_with: [pg_tde_frontend]
+)
+contrib_targets += pg_tde_restore_encrypt

contrib/pg_tde/src/bin/pg_tde_archive_decrypt.c

@@ -0,0 +1,182 @@
+#include "postgres_fe.h"
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "access/xlog_internal.h"
+#include "access/xlog_smgr.h"
+#include "common/logging.h"
+
+#include "access/pg_tde_fe_init.h"
+#include "access/pg_tde_xlog_smgr.h"
+
+static void
+write_decrypted_segment(const char *segpath, const char *segname, const char *fifopath)
+{
+	int			fd;
+	int			fifo;
+	off_t		fsize;
+	int			r;
+	TimeLineID	tli;
+	XLogSegNo	segno;
+	PGAlignedXLogBlock buf;
+	off_t		pos = 0;
+
+	fd = open(segpath, O_RDONLY | PG_BINARY, 0);
+	if (fd < 0)
+		pg_fatal("could not open file \"%s\": %m", segname);
+
+	fifo = open(fifopath, O_WRONLY | PG_BINARY, 0);
+	if (fifo < 0)
+		pg_fatal("could not open pipe \"%s\": %m", fifopath);
+
+	/*
+	 * WalSegSz extracted from the first page header but it might be
+	 * encrypted. But we need to know the segment seize to decrypt it (it's
+	 * required for encryption offset calculations). So we get the segment
+	 * size from the file's actual size. XLogLongPageHeaderData->xlp_seg_size
+	 * there is "just as a cross-check" anyway.
+	 */
+	fsize = lseek(fd, 0, SEEK_END);
+	XLogFromFileName(segname, &tli, &segno, fsize);
+
+	r = xlog_smgr->seg_read(fd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize);
+
+	if (r == XLOG_BLCKSZ)
+	{
+		XLogLongPageHeader longhdr = (XLogLongPageHeader) buf.data;
+		int			WalSegSz = longhdr->xlp_seg_size;
+
+		if (WalSegSz != fsize)
+			pg_fatal("mismatch of segment size in WAL file \"%s\" (header: %d bytes, file size: %ld bytes)",
+					 segname, WalSegSz, fsize);
+
+		if (!IsValidWalSegSize(WalSegSz))
+		{
+			pg_log_error(ngettext("invalid WAL segment size in WAL file \"%s\" (%d byte)",
+								  "invalid WAL segment size in WAL file \"%s\" (%d bytes)",
+								  WalSegSz),
+						 segname, WalSegSz);
+			pg_log_error_detail("The WAL segment size must be a power of two between 1 MB and 1 GB.");
+			exit(1);
+		}
+	}
+	else if (r < 0)
+		pg_fatal("could not read file \"%s\": %m",
+				 segname);
+	else
+		pg_fatal("could not read file \"%s\": read %d of %d",
+				 segname, r, XLOG_BLCKSZ);
+
+	pos += XLOG_BLCKSZ;
+
+	/* TODO: handle interrupted write */
+	r = write(fifo, buf.data, XLOG_BLCKSZ);
+	if (r != XLOG_BLCKSZ)
+		pg_fatal("could not read pipe: %m");
+
+	while (1)
+	{
+		r = xlog_smgr->seg_read(fd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize);
+
+		if (r == 0)
+			break;
+		else if (r < 0)
+			pg_fatal("could not read file \"%s\": %m", segname);
+		else if (r != XLOG_BLCKSZ)
+			pg_fatal("could not read file \"%s\": read %d of %d",
+					 segname, r, XLOG_BLCKSZ);
+
+		pos += XLOG_BLCKSZ;
+
+		/* TODO: handle interrupted write */
+		r = write(fifo, buf.data, XLOG_BLCKSZ);
+		if (r != XLOG_BLCKSZ)
+			pg_fatal("could not read pipe: %m");
+	}
+
+	close(fifo);
+	close(fd);
+}
+
+int
+main(int argc, char *argv[])
+{
+	char	   *segpath;
+	char	   *sep;
+	char	   *segname;
+	char		fifodir[MAXPGPATH] = "/tmp/pgtdewrapXXXXXX";
+	char		fifopath[MAXPGPATH];
+	char	   *s;
+	bool		issegment;
+	pid_t		child;
+	int			status;
+	int			r;
+
+	pg_logging_init(argv[0]);
+
+	if (argc < 3)
+		pg_fatal("too few arguments");
+
+	segpath = argv[1];
+
+	pg_tde_fe_init("pg_tde");
+	TDEXLogSmgrInit();
+
+	sep = strrchr(segpath, '/');
+
+	if (sep != NULL)
+		segname = sep + 1;
+	else
+		segname = segpath;
+
+	issegment = strlen(segname) == 24;
+
+	if (issegment)
+	{
+		if (mkdtemp(fifodir) == NULL)
+			pg_fatal("could not create temporary directory \"%s\": %m", fifodir);
+
+		/* TODO: Handle truncation */
+		s = strcpy(fifopath, fifodir);
+		s = strcpy(s, "/");
+		strcpy(s, segname);
+
+		if (mkfifo(fifopath, 0600) < 0)
+			pg_fatal("could not create fifo \"%s\": %m", fifopath);
+
+		for (int i = 2; i < argc; i++)
+			if (strcmp(segpath, argv[i]) == 0)
+				argv[i] = fifopath;
+	}
+
+	child = fork();
+	if (child == 0)
+	{
+		if (execvp(argv[2], argv + 2) < 0)
+			pg_fatal("exec failed: %m");
+	}
+	else if (child < 0)
+		pg_fatal("could not create background process: %m");
+
+	if (issegment)
+		write_decrypted_segment(segpath, segname, fifopath);
+
+	r = waitpid(child, &status, 0);
+	if (r == (pid_t) -1)
+		pg_fatal("could not wait for child process: %m");
+	if (r != child)
+		pg_fatal("child %d died, expected %d", (int) r, (int) child);
+	if (status != 0)
+		pg_fatal("%s", wait_result_to_str(status));
+
+	if (issegment && unlink(fifopath) < 0)
+		pg_log_warning("could not remove file \"%s\": %m", fifopath);
+	if (issegment && rmdir(fifodir) < 0)
+		pg_log_warning("could not remove directory \"%s\": %m", fifodir);
+
+	return 0;
+}