Skip to content

Commit 2354546

Browse files
waitingsongwaitingsong
committed
feat: add parameter pg_fs_wal (G) for changing pg_wal dir
new pg_wal style is `<pg_fs_wal>/<pg_cluster>-<pg_version>/pg_wal` when pg_fs_wal defined pg_fs_wal validatiton: pg_fs_wal must start with '/' and not contain spaces modify pg_wal of current pg_cluster: 1. edit and save pigsty.yml by setting the global `pg_wal` 2. run command `./pgsql.yml -t pg_wal` tested scenarios: - `bin/pgsql-add pg-a` - pg_fs_wal undefined - pg_fs_wal defined - `./pgsql.yml -t pg_wal` after full installation - set pg_fs_wal=/wal - set pg_fs_wal=/wal2 (from /wal) - remove pg_fs_wal (from /wal2) - multiple call `./pgsql.yml -t pg_wal` - keep pg_fs_wal undefined or blank - keep pg_fs_wal=/wal `pg-back full` will be executed automatically if pg_fs_wal changed and apply this settings: - full installation with valid pg_fs_wal (no backup operation if pg_fs_wal undefined or blank) - pg_fs_wal changed after installation - undefined or empty to `/wal` - `/wal` to `/wal2` - `/wal2` to undefined or empty
1 parent b0c1c2e commit 2354546

File tree

11 files changed

+416
-6
lines changed

11 files changed

+416
-6
lines changed

‎roles/pg_remove/tasks/main.yml

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,13 @@
9090
- /pg/data
9191
- "{{ pg_fs_main }}/postgres"
9292

93+
- name: remove pg_fs_wal/?/pg_wal (if pg_fs_wal defined)
94+
tags: pg_data
95+
when: pg_clean|bool and (pg_fs_wal | length > 0)
96+
ignore_errors: yes
97+
file: path={{ item }} state=absent
98+
with_items:
99+
- "{{ pg_fs_wal }}/{{ pg_cluster }}-{{ pg_version }}"
93100

94101
#--------------------------------------------------------------#
95102
# remove packages [pg_pkg]
@@ -120,12 +127,12 @@
120127
pg_packages_list: |-
121128
{% set pkg_map = package_map | default({}) %}
122129
{% for pkg_list in pg_packages %}{% for pkg in pkg_list | trim | regex_replace('(,|\\s)+', ',') | regex_findall('([^,\\s]+)') | default([]) %}{% if pkg != '' %}{% if pkg in pkg_map %}{% set pkg = pkg_map[pkg] %}{% endif %}{% if not loop.first %},{% endif %}{{ pkg|replace('${pg_version}', pg_version|string)|replace('$v', pg_version|string) }}{% endif %}{% endfor %}
123-
130+
124131
{% endfor %}
125132
pg_extension_list: |-
126133
{% set pkg_map = package_map | default({}) %}
127134
{% for ext_list in pg_extensions %}{% for ext in ext_list | trim | regex_replace('(,|\\s)+', ',') | regex_findall('([^,\\s]+)') | default([]) %}{% if ext != '' %}{% if ext in pkg_map %}{% set ext = pkg_map[ext] %}{% endif %}{% if not loop.first %},{% endif %}{{ ext|replace('${pg_version}', pg_version|string)|replace('$v', pg_version|string) }}{% endif %}{% endfor %}
128-
135+
129136
{% endfor %}
130137
131138
# uninstall extensions first

‎roles/pgsql/defaults/main.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ pg_safeguard: false # prevent purging running postgres instance? f
5656
pg_clean: true # purging existing postgres during pgsql init? true by default
5757
pg_data: /pg/data # postgres data directory, `/pg/data` by default
5858
pg_fs_main: /data # mountpoint/path for postgres main data, `/data` by default
59+
pg_fs_wal: '' # mountpoint/path for pg_wal dir, default blank, new pg_wal will be <pg_fs_wal>/<pg_cluster>-<pg_version>/pg_wal
5960
pg_fs_bkup: /data/backups # mountpoint/path for pg backup data, `/data/backup` by default
6061
pg_storage_type: SSD # storage type for pg main data, SSD,HDD, SSD by default
6162
pg_dummy_filesize: 64MiB # size of `/pg/dummy`, hold 64MB disk space for emergency use

‎roles/pgsql/tasks/clean.yml

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
ignore_errors: false
2020
fail:
2121
msg: >-
22-
Abort because the node {{ inventory_hostname }} seems not managed by pigsty, use ./node.yml to init node first.
22+
Abort because the node {{ inventory_hostname }} seems not managed by pigsty, use ./node.yml to init node first.
2323
2424
# check postgres by listening port
2525
- name: check postgres exists
@@ -41,7 +41,7 @@
4141
ignore_errors: false
4242
fail:
4343
msg: >-
44-
Abort because pg instance {{ pg_cluster }}-{{ pg_seq }} @ {{ inventory_hostname }} exists.
44+
Abort because pg instance {{ pg_cluster }}-{{ pg_seq }} @ {{ inventory_hostname }} exists.
4545
{% if pg_safeguard|bool %}pg_safeguard enabled, please disable it before purging.{% endif %}
4646
{% if not pg_clean|bool %}pg_clean = false, use -e pg_safeguard=false to override.{% endif %}
4747
@@ -94,7 +94,7 @@
9494
rm -rf /lib/systemd/system/[email protected] /lib/systemd/system/postgresql.service /etc/init.d/postgresql
9595
systemctl daemon-reload
9696
{% endif %}
97-
97+
9898
if ps -u postgres -o pid:1,command | grep -E 'postmaster|postgres:|-D' ; then
9999
{{ pg_bin_dir }}/pg_ctl -D {{ pg_data }} stop --mode=immediate
100100
fi
@@ -121,7 +121,7 @@
121121
{% else %}
122122
META_DIR="{{ pg_namespace|default('/pg') }}/{{ pg_cluster }}"
123123
{% endif %}
124-
124+
125125
export ETCDCTL_API=3
126126
export ETCDCTL_ENDPOINTS="{% for ip in groups['etcd']|sort %}{% if not loop.first %},{% endif %}https://{{ ip }}:{{ etcd_port }}{% endfor %}"
127127
export ETCDCTL_CACERT=/etc/pki/ca.crt
@@ -145,4 +145,10 @@
145145
- /etc/pgbouncer
146146
- /var/run/pgbouncer
147147

148+
- name: remove pg_fs_wal/?/pg_wal (if pg_fs_wal defined)
149+
tags: pg_clean_data
150+
when: pg_fs_wal | length > 0
151+
file: path={{ item }} state=absent
152+
with_items:
153+
- "{{ pg_fs_wal }}/{{ pg_cluster }}-{{ pg_version }}"
148154
...

‎roles/pgsql/tasks/dir.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,10 @@
4242
- "{{ pg_backup_dir }}/backup"
4343
- "/var/run/postgresql"
4444

45+
- name: List files in a directory
46+
ansible.builtin.command:
47+
cmd: ls "{{ pg_cluster_dir }}/data"
48+
4549
# make sure the /var/run/postgresql is created
4650
- name: create /var/run/postgresql tmpfiles.d
4751
copy:

‎roles/pgsql/tasks/main.yml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,12 @@
8383
tags: pgbackrest
8484
when: pgbackrest_enabled|bool
8585

86+
#--------------------------------------------------------------#
87+
# WAL [pg_wal]
88+
#--------------------------------------------------------------#
89+
- import_tasks: pg_wal.yml
90+
tags: pg_wal
91+
8692
#--------------------------------------------------------------#
8793
# Pgbouncer [pgbouncer]
8894
#--------------------------------------------------------------#

‎roles/pgsql/tasks/pg_wal.yml

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#!/usr/bin/ansible-playbook
2+
---
3+
#--------------------------------------------------------------#
4+
# Modify pg_wal destination if pg_fs_wal defined [pg_wal]
5+
# new pg_wal is <pg_fs_wal>/<pg_cluster>-<pg_version>/pg_wal
6+
#--------------------------------------------------------------#
7+
8+
9+
- name: validate variable pg_fs_wal
10+
tags: [ validate_pg_fs_wal ]
11+
when: pg_fs_wal | length > 0
12+
block:
13+
- name: validate pg_fs_wal "{{ pg_fs_wal }}"
14+
fail:
15+
msg: "The variable pg_fs_wal must start with '/' and not contain spaces"
16+
when: >
17+
(pg_fs_wal[0] != '/') or (' ' in pg_fs_wal)
18+
19+
- name: retrieve path of pg_wal directory
20+
become: yes
21+
block:
22+
- name: validate variable pg_data
23+
assert:
24+
that: pg_data is defined and pg_data | length > 0
25+
msg: "pg_data must be defined"
26+
changed_when: false
27+
28+
- name: get current pg_wal real path
29+
command: realpath -m "{{ pg_data }}/pg_wal"
30+
register: real_curr_pg_wal_cmd
31+
changed_when: false
32+
33+
- name: calculate new pg_data path (parent of target pg_wal)
34+
set_fact:
35+
target_new_pg_data: >-
36+
{% if pg_fs_wal is defined and pg_fs_wal %}
37+
{{ pg_fs_wal | trim }}/{{ pg_cluster }}-{{ pg_version }}
38+
{% else %}
39+
{{ pg_data }}
40+
{% endif %}
41+
changed_when: false
42+
43+
- name: get target pg_wal real path
44+
command: "realpath -m {{ target_new_pg_data }}" # do not quote the var like "{{ target_new_pg_data }}"
45+
register: new_pg_data_cmd
46+
changed_when: false
47+
48+
- name: Set final path facts
49+
set_fact:
50+
new_pg_data: "{{ new_pg_data_cmd.stdout }}"
51+
real_curr_pg_wal: "{{ real_curr_pg_wal_cmd.stdout }}"
52+
pg_wal_dst: "{{ new_pg_data_cmd.stdout }}/pg_wal"
53+
changed_when: false
54+
55+
- name: get current pg_wal stat
56+
stat:
57+
path: "{{ pg_data }}/pg_wal" # do not use real_curr_pg_wal, use pg_data instead
58+
register: real_curr_pg_wal_stat
59+
changed_when: false
60+
61+
- name: Set real_curr_pg_wal_is_link variable
62+
set_fact:
63+
real_curr_pg_wal_is_link: "{{ real_curr_pg_wal_stat.stat.exists and real_curr_pg_wal_stat.stat.islnk }}"
64+
changed_when: false
65+
66+
- name: variable output for pg_wal
67+
debug:
68+
msg: |
69+
real_curr_pg_wal: {{ real_curr_pg_wal }}, pg_wal_dst: {{ pg_wal_dst }},
70+
real_curr_pg_wal_is_link: {{ real_curr_pg_wal_is_link }},
71+
pg_data: {{ pg_data }}, new_pg_data: {{ new_pg_data }},
72+
pg_fs_wal: {{ pg_fs_wal }},
73+
changed_when: false
74+
75+
76+
77+
#--------------------------------------------------------------#
78+
# Modify WAL [modify_pg_wal]
79+
#--------------------------------------------------------------#
80+
- include_tasks: pg_wal/pg_wal_modification.yml
81+
tags: modify_pg_wal
82+
when: (real_curr_pg_wal != pg_wal_dst) and (pg_fs_wal | length > 0)
83+
84+
85+
#--------------------------------------------------------------#
86+
# Restore WAL [restore_pg_wal]
87+
#--------------------------------------------------------------#
88+
- include_tasks: pg_wal/pg_wal_restoration.yml
89+
tags: restore_pg_wal
90+
when: (real_curr_pg_wal != pg_wal_dst) and (pg_fs_wal is not defined or (pg_fs_wal | trim | length == 0))
91+
92+
93+
...

‎roles/pgsql/tasks/pg_wal/pg_wal_after.yml

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
#!/usr/bin/ansible-playbook
2+
---
3+
4+
- name: operations after updating pg_wal dir
5+
vars:
6+
tmp_wal_dir: "pg_wal.tmp-backup"
7+
pg_data_lock: "{{ pg_data }}.locked"
8+
pg_wal_src_tmp: "{{ pg_data_lock }}/{{ tmp_wal_dir }}"
9+
dbsu: "{{ pg_dbsu|default('postgres') }}"
10+
block:
11+
- name: check if postmaster.pid exists and is not empty
12+
stat:
13+
path: "{{ pg_data }}/postmaster.pid"
14+
register: postmaster_pid_stat
15+
16+
- name: start postgres primary after modifying pg_wal
17+
become_user: "{{ dbsu }}"
18+
when: (pg_role == 'primary') and (postmaster_pid_stat.stat.exists == false or postmaster_pid_stat.stat.size == 0)
19+
args: { executable: /bin/bash }
20+
shell: |
21+
{{ pg_bin_dir }}/pg_ctl -D {{ pg_data }} start
22+
23+
- name: start postgres replica after modifying pg_wal
24+
become_user: "{{ dbsu }}"
25+
when: (pg_role != 'primary') and (postmaster_pid_stat.stat.exists == false or postmaster_pid_stat.stat.size == 0)
26+
args: { executable: /bin/bash }
27+
shell: |
28+
{{ pg_bin_dir }}/pg_ctl -D {{ pg_data }} start
29+
30+
- name: wait for postgres primary after modifying pg_wal
31+
when: pg_role == 'primary'
32+
wait_for: host={{ inventory_hostname }} port={{ pg_port }} state=started timeout=60
33+
34+
- name: check postgres primary ready after modifying pg_wal
35+
become_user: "{{ dbsu }}"
36+
when: pg_role == 'primary'
37+
shell: |
38+
{{ pg_bin_dir }}/pg_isready -t 5 -p {{ pg_port }}
39+
register: result
40+
retries: 6
41+
until: result.rc == 0
42+
delay: 5
43+
44+
- name: start patroni after modifying pg_wal
45+
args: { executable: /bin/bash }
46+
shell: |
47+
systemctl start patroni
48+
49+
- name: resume patroni after modifying pg_wal
50+
become_user: "{{ dbsu }}"
51+
when: pg_role == 'primary'
52+
command: /usr/bin/patronictl -c /pg/bin/patroni.yml resume
53+
54+
- name: resume replica patroni after modifying pg_wal (should fail)
55+
become_user: "{{ dbsu }}"
56+
when: pg_role != 'primary'
57+
command: /usr/bin/patronictl -c /pg/bin/patroni.yml resume
58+
ignore_errors: yes
59+
60+
- name: restart postgres via patroni after modifying pg_wal
61+
become_user: "{{ dbsu }}"
62+
when: pg_role == 'primary'
63+
args: { executable: /bin/bash }
64+
shell: |
65+
{% if pg_mode|default('pgsql') == 'citus' %}
66+
/usr/bin/patronictl -c /etc/patroni/patroni.yml restart {{ pg_shard }} --group {{ pg_group }} --force
67+
{% else %}
68+
/usr/bin/patronictl -c /etc/patroni/patroni.yml restart {{ pg_cluster }} --force
69+
{% endif %}
70+
71+
- name: clean tmp directory after modifying pg_wal
72+
ignore_errors: yes
73+
file: path={{ item }} state=absent
74+
with_items:
75+
- "{{ pg_data }}/{{ tmp_wal_dir }}"
76+
- "{{ pg_wal_src_tmp }}"
77+
78+
- name: clean tmp directory after modifying pg_wal
79+
ignore_errors: yes
80+
when: (real_curr_pg_wal != pg_wal_dst) and (real_curr_pg_wal_is_link == true)
81+
file: path={{ item }} state=absent
82+
with_items:
83+
- "{{ real_curr_pg_wal }}"
84+
85+
...

‎roles/pgsql/tasks/pg_wal/pg_wal_backup.yml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
#!/usr/bin/ansible-playbook
2+
---
3+
4+
- name: backup
5+
when: pgbackrest_enabled|bool
6+
vars:
7+
dbsu: "{{ pg_dbsu|default('postgres') }}"
8+
block:
9+
- name: full backup cls {{ pg_cluster }}
10+
become_user: "{{ dbsu }}"
11+
when: pg_role == 'primary'
12+
command: /pg/bin/pg-backup full
13+
register: back_ret_cmd
14+
15+
- name: show backup result for {{ pg_cluster }}
16+
when: pg_role == 'primary'
17+
debug:
18+
msg: |
19+
STDOUT {{ back_ret_cmd.stdout_lines }}
20+
STDERR {{ back_ret_cmd.stderr_lines }}
21+
changed_when: false
22+
23+
...

‎roles/pgsql/tasks/pg_wal/pg_wal_before.yml

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
#!/usr/bin/ansible-playbook
2+
---
3+
4+
- name: operations before updating pg_wal dir
5+
vars:
6+
tmp_wal_dir: "pg_wal.tmp-backup"
7+
pg_data_lock: "{{ pg_data }}.locked"
8+
pg_wal_src_tmp: "{{ pg_data_lock }}/{{ tmp_wal_dir }}"
9+
dbsu: "{{ pg_dbsu|default('postgres') }}"
10+
block:
11+
- import_tasks: pg_wal_backup.yml
12+
13+
- name: pause patroni before modifying pg_wal
14+
become_user: "{{ dbsu }}"
15+
when: pg_role == 'primary'
16+
command: /usr/bin/patronictl -c /pg/bin/patroni.yml pause
17+
ignore_errors: yes
18+
19+
- name: pause replica patroni before modifying pg_wal (should fail)
20+
become_user: "{{ dbsu }}"
21+
when: pg_role != 'primary'
22+
command: /usr/bin/patronictl -c /pg/bin/patroni.yml pause
23+
ignore_errors: yes
24+
25+
- name: stop patroni before modifying pg_wal
26+
args: { executable: /bin/bash }
27+
shell: |
28+
systemctl stop patroni
29+
30+
- name: stop postgres replica before modifying pg_wal
31+
become_user: "{{ dbsu }}"
32+
when: pg_role != 'primary'
33+
args: { executable: /bin/bash }
34+
shell: |
35+
{{ pg_bin_dir }}/pg_ctl -D {{ pg_data }} stop
36+
sync; sync;
37+
sleep 1
38+
39+
- name: stop postgres primary before modifying pg_wal
40+
become_user: "{{ dbsu }}"
41+
when: pg_role == 'primary'
42+
args: { executable: /bin/bash }
43+
shell: |
44+
{{ pg_bin_dir }}/pg_ctl -D {{ pg_data }} stop
45+
sync; sync;
46+
sleep 1
47+
48+
- name: clean tmp dir before modify pg_wal
49+
file: path={{ item }} state=absent
50+
with_items:
51+
- "{{ pg_data_lock }}"
52+
- "{{ pg_wal_src_tmp }}"
53+
54+
...

0 commit comments

Comments
 (0)