Skip to content

Commit 4becad7

Browse files
jeltzjeltz
committed
WIP: PG-1710 Create helpers for decrypting/encrypting archived WAL
1 parent 0b9538a commit 4becad7

File tree

6 files changed

+481
-2
lines changed

6 files changed

+481
-2
lines changed

‎contrib/pg_tde/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,9 @@ __pycache__
1111
/configure~
1212
/log
1313
/results
14+
/src/bin/pg_tde_archive_decrypt
1415
/src/bin/pg_tde_change_key_provider
16+
/src/bin/pg_tde_restore_encrypt
1517
/t/results
1618
/tmp_check
1719

‎contrib/pg_tde/Makefile

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,16 @@ src/libkmip/libkmip/src/kmip_bio.o \
5151
src/libkmip/libkmip/src/kmip_locate.o \
5252
src/libkmip/libkmip/src/kmip_memset.o
5353

54-
SCRIPTS_built = src/bin/pg_tde_change_key_provider
54+
SCRIPTS_built = src/bin/pg_tde_archive_decrypt \
55+
src/bin/pg_tde_change_key_provider \
56+
src/bin/pg_tde_restore_encrypt
5557

5658
EXTRA_INSTALL += contrib/pg_buffercache contrib/test_decoding
57-
EXTRA_CLEAN += src/bin/pg_tde_change_key_provider.o
59+
EXTRA_CLEAN += src/bin/pg_tde_archive_decrypt.o \
60+
src/bin/pg_tde_change_key_provider.o \
61+
src/bin/pg_tde_restore_encrypt.o \
62+
xlogreader.c \
63+
xlogreader.o
5864

5965
ifdef USE_PGXS
6066
PG_CONFIG = pg_config
@@ -74,6 +80,18 @@ override SHLIB_LINK += -lcurl -lcrypto -lssl
7480
src/bin/pg_tde_change_key_provider: src/bin/pg_tde_change_key_provider.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtde.a
7581
$(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
7682

83+
src/bin/pg_tde_archive_decrypt: src/bin/pg_tde_archive_decrypt.o xlogreader.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtdexlog.a $(top_builddir)/src/libtde/libtde.a
84+
$(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
85+
86+
src/bin/pg_tde_restore_encrypt: src/bin/pg_tde_restore_encrypt.o xlogreader.o $(top_srcdir)/src/fe_utils/simple_list.o $(top_builddir)/src/libtde/libtdexlog.a $(top_builddir)/src/libtde/libtde.a
87+
$(CC) -DFRONTEND $(CFLAGS) $^ $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
88+
89+
xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/%
90+
rm -f $@ && $(LN_S) $< .
91+
92+
xlogreader.o: xlogreader.c
93+
$(CC) $(CPPFLAGS) -DFRONTEND -c $< -o $@
94+
7795
# Fetches typedefs list for PostgreSQL core and merges it with typedefs defined in this project.
7896
# https://wiki.postgresql.org/wiki/Running_pgindent_on_non-core_code_or_development_code
7997
update-typedefs:

‎contrib/pg_tde/meson.build

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ tap_tests = [
113113
't/rotate_key.pl',
114114
't/tde_heap.pl',
115115
't/unlogged_tables.pl',
116+
't/wal_archiving.pl',
116117
't/wal_encrypt.pl',
117118
]
118119

@@ -154,3 +155,31 @@ pg_tde_change_key_provider = executable('bin/pg_tde_change_key_provider',
154155
link_with: [pg_tde_frontend]
155156
)
156157
contrib_targets += pg_tde_change_key_provider
158+
159+
pg_tde_archive_decrypt_sources = files(
160+
'src/bin/pg_tde_archive_decrypt.c',
161+
) + xlogreader_sources
162+
163+
pg_tde_archive_decrypt = executable('bin/pg_tde_archive_decrypt',
164+
pg_tde_archive_decrypt_sources,
165+
dependencies: [frontend_code],
166+
c_args: ['-DFRONTEND'], # needed for xlogreader et al
167+
kwargs: default_bin_args,
168+
include_directories: [postgres_inc, pg_tde_inc],
169+
link_with: [pg_tde_frontend]
170+
)
171+
contrib_targets += pg_tde_archive_decrypt
172+
173+
pg_tde_restore_encrypt_sources = files(
174+
'src/bin/pg_tde_restore_encrypt.c',
175+
) + xlogreader_sources
176+
177+
pg_tde_restore_encrypt = executable('bin/pg_tde_restore_encrypt',
178+
pg_tde_restore_encrypt_sources,
179+
dependencies: [frontend_code],
180+
c_args: ['-DFRONTEND'], # needed for xlogreader et al
181+
kwargs: default_bin_args,
182+
include_directories: [postgres_inc, pg_tde_inc],
183+
link_with: [pg_tde_frontend]
184+
)
185+
contrib_targets += pg_tde_restore_encrypt

‎contrib/pg_tde/src/bin/pg_tde_archive_decrypt.c

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
#include "postgres_fe.h"
2+
3+
#include <fcntl.h>
4+
#include <stdlib.h>
5+
#include <sys/stat.h>
6+
#include <sys/wait.h>
7+
#include <unistd.h>
8+
9+
#include "access/xlog_internal.h"
10+
#include "access/xlog_smgr.h"
11+
#include "common/logging.h"
12+
13+
#include "access/pg_tde_fe_init.h"
14+
#include "access/pg_tde_xlog_smgr.h"
15+
16+
static void
17+
write_decrypted_segment(const char *segpath, const char *segname, const char *fifopath)
18+
{
19+
int fd;
20+
int fifo;
21+
off_t fsize;
22+
int r;
23+
TimeLineID tli;
24+
XLogSegNo segno;
25+
PGAlignedXLogBlock buf;
26+
off_t pos = 0;
27+
28+
fd = open(segpath, O_RDONLY | PG_BINARY, 0);
29+
if (fd < 0)
30+
pg_fatal("could not open file \"%s\": %m", segname);
31+
32+
fifo = open(fifopath, O_WRONLY | PG_BINARY, 0);
33+
if (fifo < 0)
34+
pg_fatal("could not open pipe \"%s\": %m", fifopath);
35+
36+
/*
37+
* WalSegSz extracted from the first page header but it might be
38+
* encrypted. But we need to know the segment seize to decrypt it (it's
39+
* required for encryption offset calculations). So we get the segment
40+
* size from the file's actual size. XLogLongPageHeaderData->xlp_seg_size
41+
* there is "just as a cross-check" anyway.
42+
*/
43+
fsize = lseek(fd, 0, SEEK_END);
44+
XLogFromFileName(segname, &tli, &segno, fsize);
45+
46+
r = xlog_smgr->seg_read(fd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize);
47+
48+
if (r == XLOG_BLCKSZ)
49+
{
50+
XLogLongPageHeader longhdr = (XLogLongPageHeader) buf.data;
51+
int WalSegSz = longhdr->xlp_seg_size;
52+
53+
if (WalSegSz != fsize)
54+
pg_fatal("mismatch of segment size in WAL file \"%s\" (header: %d bytes, file size: %ld bytes)",
55+
segname, WalSegSz, fsize);
56+
57+
if (!IsValidWalSegSize(WalSegSz))
58+
{
59+
pg_log_error(ngettext("invalid WAL segment size in WAL file \"%s\" (%d byte)",
60+
"invalid WAL segment size in WAL file \"%s\" (%d bytes)",
61+
WalSegSz),
62+
segname, WalSegSz);
63+
pg_log_error_detail("The WAL segment size must be a power of two between 1 MB and 1 GB.");
64+
exit(1);
65+
}
66+
}
67+
else if (r < 0)
68+
pg_fatal("could not read file \"%s\": %m",
69+
segname);
70+
else
71+
pg_fatal("could not read file \"%s\": read %d of %d",
72+
segname, r, XLOG_BLCKSZ);
73+
74+
pos += XLOG_BLCKSZ;
75+
76+
/* TODO: handle interrupted write */
77+
r = write(fifo, buf.data, XLOG_BLCKSZ);
78+
if (r != XLOG_BLCKSZ)
79+
pg_fatal("could not read pipe: %m");
80+
81+
while (1)
82+
{
83+
r = xlog_smgr->seg_read(fd, buf.data, XLOG_BLCKSZ, pos, tli, segno, fsize);
84+
85+
if (r == 0)
86+
break;
87+
else if (r < 0)
88+
pg_fatal("could not read file \"%s\": %m", segname);
89+
else if (r != XLOG_BLCKSZ)
90+
pg_fatal("could not read file \"%s\": read %d of %d",
91+
segname, r, XLOG_BLCKSZ);
92+
93+
pos += XLOG_BLCKSZ;
94+
95+
/* TODO: handle interrupted write */
96+
r = write(fifo, buf.data, XLOG_BLCKSZ);
97+
if (r != XLOG_BLCKSZ)
98+
pg_fatal("could not read pipe: %m");
99+
}
100+
101+
close(fifo);
102+
close(fd);
103+
}
104+
105+
int
106+
main(int argc, char *argv[])
107+
{
108+
char *segpath;
109+
char *sep;
110+
char *segname;
111+
char fifodir[MAXPGPATH] = "/tmp/pgtdewrapXXXXXX";
112+
char fifopath[MAXPGPATH];
113+
char *s;
114+
bool issegment;
115+
pid_t child;
116+
int status;
117+
int r;
118+
119+
pg_logging_init(argv[0]);
120+
121+
if (argc < 3)
122+
pg_fatal("too few arguments");
123+
124+
segpath = argv[1];
125+
126+
pg_tde_fe_init("pg_tde");
127+
TDEXLogSmgrInit();
128+
129+
sep = strrchr(segpath, '/');
130+
131+
if (sep != NULL)
132+
segname = sep + 1;
133+
else
134+
segname = segpath;
135+
136+
issegment = strlen(segname) == 24;
137+
138+
if (issegment)
139+
{
140+
if (mkdtemp(fifodir) == NULL)
141+
pg_fatal("could not create temporary directory \"%s\": %m", fifodir);
142+
143+
/* TODO: Handle truncation */
144+
s = strcpy(fifopath, fifodir);
145+
s = strcpy(s, "/");
146+
strcpy(s, segname);
147+
148+
if (mkfifo(fifopath, 0600) < 0)
149+
pg_fatal("could not create fifo \"%s\": %m", fifopath);
150+
151+
for (int i = 2; i < argc; i++)
152+
if (strcmp(segpath, argv[i]) == 0)
153+
argv[i] = fifopath;
154+
}
155+
156+
child = fork();
157+
if (child == 0)
158+
{
159+
if (execvp(argv[2], argv + 2) < 0)
160+
pg_fatal("exec failed: %m");
161+
}
162+
else if (child < 0)
163+
pg_fatal("could not create background process: %m");
164+
165+
if (issegment)
166+
write_decrypted_segment(segpath, segname, fifopath);
167+
168+
r = waitpid(child, &status, 0);
169+
if (r == (pid_t) -1)
170+
pg_fatal("could not wait for child process: %m");
171+
if (r != child)
172+
pg_fatal("child %d died, expected %d", (int) r, (int) child);
173+
if (status != 0)
174+
pg_fatal("%s", wait_result_to_str(status));
175+
176+
if (issegment && unlink(fifopath) < 0)
177+
pg_log_warning("could not remove file \"%s\": %m", fifopath);
178+
if (issegment && rmdir(fifodir) < 0)
179+
pg_log_warning("could not remove directory \"%s\": %m", fifodir);
180+
181+
return 0;
182+
}

0 commit comments

Comments
 (0)