Skip to content

Commit 2690b05

Browse files
derrickmehaffyderrickmehaffy
authored
Merge pull request strapi#54 from strapi/35-environment-variables-schema-not-working
Add support for schemas in PG
2 parents 072c02d + 37def27 commit 2690b05

File tree

13 files changed

+454
-418
lines changed

13 files changed

+454
-418
lines changed

‎v3-sql-v4-sql/config/database.js‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ if (process.env.DATABASE_CLIENT === "pg") {
2727
user: process.env.DATABASE_V3_USER,
2828
password: process.env.DATABASE_V3_PASSWORD,
2929
database: process.env.DATABASE_V3_DATABASE,
30-
schema: process.env.DATABASE_V3_SCHEMA
3130
},
3231
};
3332

@@ -39,7 +38,6 @@ if (process.env.DATABASE_CLIENT === "pg") {
3938
user: process.env.DATABASE_V4_USER,
4039
password: process.env.DATABASE_V4_PASSWORD,
4140
database: process.env.DATABASE_V4_DATABASE,
42-
schema: process.env.DATABASE_V4_SCHEMA
4341
},
4442
};
4543
}
@@ -51,7 +49,7 @@ if (process.env.DATABASE_CLIENT === "mysql") {
5149
port: process.env.DATABASE_V3_PORT,
5250
user: process.env.DATABASE_V3_USER,
5351
password: process.env.DATABASE_V3_PASSWORD,
54-
database: process.env.DATABASE_V3_DATABASE
52+
database: process.env.DATABASE_V3_DATABASE,
5553
},
5654
};
5755

@@ -62,7 +60,7 @@ if (process.env.DATABASE_CLIENT === "mysql") {
6260
port: process.env.DATABASE_V4_PORT,
6361
user: process.env.DATABASE_V4_USER,
6462
password: process.env.DATABASE_V4_PASSWORD,
65-
database: process.env.DATABASE_V4_DATABASE
63+
database: process.env.DATABASE_V4_DATABASE,
6664
},
6765
};
6866
}

‎v3-sql-v4-sql/migrate/helpers/adminHelpers.js‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const { migrateItems } = require("./migrateFields");
1111
const { migrateUids } = require("./migrateValues");
1212
const pluralize = require("pluralize");
1313
const { camelCase } = require("lodash");
14+
const { resolveDestTableName, resolveSourceTableName } = require("./tableNameHelpers");
1415

1516
const extraV4Permissions = [
1617
{ action: "admin::api-tokens.create", properties: {}, conditions: [] },
@@ -37,19 +38,20 @@ function migrateProperties(properties) {
3738
return properties;
3839
}
3940

41+
4042
async function migrateAdminPermissions() {
4143
const source = "strapi_permission";
4244
const destination = "admin_permissions";
4345
const destinationLinks = "admin_permissions_role_links";
4446
const count =
45-
(await dbV3(source).count().first()).count ||
46-
(await dbV3(source).count().first())["count(*)"];
47+
(await dbV3(resolveSourceTableName(source)).count().first()).count ||
48+
(await dbV3(resolveSourceTableName(source)).count().first())["count(*)"];
4749
console.log(`Migrating ${count} items from ${source} to ${destination}`);
48-
await dbV4(destinationLinks).del();
49-
await dbV4(destination).del();
50+
await dbV4(resolveDestTableName(destinationLinks)).del();
51+
await dbV4(resolveDestTableName(destination)).del();
5052
for (var page = 0; page * BATCH_SIZE < count; page++) {
5153
console.log(`${source} batch #${page + 1}`);
52-
const items = await dbV3(source)
54+
const items = await dbV3(resolveSourceTableName(source))
5355
.limit(BATCH_SIZE)
5456
.offset(page * BATCH_SIZE);
5557
const migratedItems = migrateItems(items, ({ role, ...item }) => ({
@@ -63,19 +65,19 @@ async function migrateAdminPermissions() {
6365
permission_id: item.id,
6466
role_id: item.role,
6567
}));
66-
await dbV4(destination).insert(migratedItems);
67-
await dbV4(destinationLinks).insert(roleLinks);
68+
await dbV4(resolveDestTableName(destination)).insert(migratedItems);
69+
await dbV4(resolveDestTableName(destinationLinks)).insert(roleLinks);
6870
}
6971
await resetTableSequence(destination);
7072

7173
let ids = [];
7274

7375
if (isPGSQL) {
74-
ids = await dbV4(destination).insert(extraV4Permissions).returning("id");
76+
ids = await dbV4(resolveDestTableName(destination)).insert(extraV4Permissions).returning("id");
7577
}
7678

7779
if (isSQLITE) {
78-
ids = await dbV4(destination).insert(
80+
ids = await dbV4(resolveDestTableName(destination)).insert(
7981
extraV4Permissions.map((item) => ({
8082
...item,
8183
properties: JSON.stringify(item.properties),
@@ -85,7 +87,7 @@ async function migrateAdminPermissions() {
8587
}
8688

8789
if (isMYSQL) {
88-
ids = await dbV4(destination).insert(
90+
ids = await dbV4(resolveDestTableName(destination)).insert(
8991
extraV4Permissions.map((item) => ({
9092
...item,
9193
properties: JSON.stringify(item.properties),
@@ -94,7 +96,7 @@ async function migrateAdminPermissions() {
9496
);
9597
}
9698

97-
await dbV4(destinationLinks).insert(
99+
await dbV4(resolveDestTableName(destinationLinks)).insert(
98100
ids.map((id) => ({ permission_id: id.id, role_id: SUPER_ADMIN }))
99101
);
100102
}

‎v3-sql-v4-sql/migrate/helpers/constants.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
const { isPGSQL } = require("../../config/database");
2+
13
const BATCH_SIZE = process.env.BATCH_SIZE || 50;
24
const SUPER_ADMIN = 1;
35

‎v3-sql-v4-sql/migrate/helpers/migrate.js‎

Lines changed: 52 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,12 @@
1-
const {
2-
dbV3,
3-
dbV4,
4-
isPGSQL,
5-
isMYSQL,
6-
isSQLITE,
7-
} = require("../../config/database");
8-
const { BATCH_SIZE } = require("./constants");
9-
const { migrateItems } = require("./migrateFields");
10-
const { pick } = require("lodash");
1+
const { dbV3, dbV4, isPGSQL, isMYSQL, isSQLITE } = require('../../config/database');
2+
const { BATCH_SIZE } = require('./constants');
3+
const { migrateItems } = require('./migrateFields');
4+
const { pick } = require('lodash');
5+
const { resolveDestTableName, resolveSourceTableName } = require('./tableNameHelpers');
116

127
async function migrate(source, destination, itemMapper = undefined) {
138
if (isMYSQL) {
14-
const sourceNotExists =
15-
(await dbV3.raw(`SHOW TABLES LIKE '%${source}%';`))[0].length === 0;
9+
const sourceNotExists = (await dbV3.raw(`SHOW TABLES LIKE '%${source}%';`))[0].length === 0;
1610
const destinationNotExists =
1711
(await dbV4.raw(`SHOW TABLES LIKE '%${destination}%';`))[0].length === 0;
1812

@@ -32,23 +26,23 @@ async function migrate(source, destination, itemMapper = undefined) {
3226

3327
const sourceNotExists =
3428
(
35-
await dbV3("sqlite_master")
36-
.select("name")
37-
.where("type", "table")
38-
.where("name", source)
29+
await dbV3('sqlite_master')
30+
.select('name')
31+
.where('type', 'table')
32+
.where('name', source)
3933
.first()
4034
.count()
41-
)["count(*)"] === 0;
35+
)['count(*)'] === 0;
4236

4337
const destinationNotExists =
4438
(
45-
await dbV4("sqlite_master")
46-
.select("name")
47-
.where("type", "table")
48-
.where("name", destination)
39+
await dbV4('sqlite_master')
40+
.select('name')
41+
.where('type', 'table')
42+
.where('name', destination)
4943
.first()
5044
.count()
51-
)["count(*)"] === 0;
45+
)['count(*)'] === 0;
5246

5347
if (sourceNotExists) {
5448
console.log(`SOURCE TABLE ${source} DOES NOT EXISTS`);
@@ -68,18 +62,18 @@ async function migrate(source, destination, itemMapper = undefined) {
6862

6963
const sourceNotExists =
7064
(
71-
await dbV3("information_schema.tables")
72-
.select("table_name")
73-
.where("table_schema", "public")
74-
.where("table_name", source)
65+
await dbV3('information_schema.tables')
66+
.select('table_name')
67+
.where('table_schema', process.env.DATABASE_V3_SCHEMA)
68+
.where('table_name', source)
7569
).length === 0;
7670

7771
const destinationNotExists =
7872
(
79-
await dbV4("information_schema.tables")
80-
.select("table_name")
81-
.where("table_schema", "public")
82-
.where("table_name", destination)
73+
await dbV4('information_schema.tables')
74+
.select('table_name')
75+
.where('table_schema', process.env.DATABASE_V4_SCHEMA)
76+
.where('table_name', destination)
8377
).length === 0;
8478

8579
if (sourceNotExists) {
@@ -94,26 +88,31 @@ async function migrate(source, destination, itemMapper = undefined) {
9488
}
9589

9690
const count =
97-
(await dbV3(source).count().first()).count ||
98-
(await dbV3(source).count().first())["count(*)"];
99-
const columnsInfo = await dbV3(source).columnInfo();
91+
(await dbV3(resolveSourceTableName(source)).count().first()).count ||
92+
(await dbV3(resolveSourceTableName(source)).count().first())['count(*)'];
93+
const columnsInfo = await dbV3(resolveSourceTableName(source)).columnInfo();
10094

10195
const jsonFields = Object.keys(columnsInfo).filter((column) => {
102-
return columnsInfo[column].type === "jsonb";
96+
return columnsInfo[column].type === 'jsonb';
10397
});
10498

10599
console.log(`Migrating ${count} items from ${source} to ${destination}`);
106-
await dbV4(destination).del();
100+
await dbV4(resolveDestTableName(destination)).del();
107101

108-
console.log("DBV4 ITEMS");
102+
let tableColumnsInfo = await dbV4(destination).columnInfo();
109103

110-
const tableColumnsInfo = await dbV4(destination).columnInfo();
104+
if (isPGSQL) {
105+
// https://github.com/knex/knex/issues/1490
106+
tableColumnsInfo = await dbV4(destination)
107+
.withSchema(process.env.DATABASE_V4_SCHEMA)
108+
.columnInfo();
109+
}
111110

112111
const tableColumns = Object.keys(tableColumnsInfo);
113112

114113
for (let page = 0; page * BATCH_SIZE < count; page++) {
115114
console.log(`${source} batch #${page + 1}`);
116-
const items = await dbV3(source)
115+
const items = await dbV3(resolveSourceTableName(source))
117116
.limit(BATCH_SIZE)
118117
.offset(page * BATCH_SIZE);
119118

@@ -127,29 +126,24 @@ async function migrate(source, destination, itemMapper = undefined) {
127126
return item;
128127
});
129128

130-
const migratedItems = migrateItems(withParsedJsonFields, itemMapper).map(
131-
(item) => {
132-
const filteredItems = pick(item, tableColumns);
133-
134-
if (Object.keys(item).length !== Object.keys(filteredItems).length) {
135-
const filteredColumns = Object.keys(item).filter(function (obj) {
136-
return Object.keys(filteredItems).indexOf(obj) == -1;
137-
});
129+
const migratedItems = migrateItems(withParsedJsonFields, itemMapper).map((item) => {
130+
const filteredItems = pick(item, tableColumns);
138131

139-
console.log(
140-
"WARNING - items of " +
141-
destination +
142-
" was filtered " +
143-
JSON.stringify(filteredColumns)
144-
);
145-
}
132+
if (Object.keys(item).length !== Object.keys(filteredItems).length) {
133+
const filteredColumns = Object.keys(item).filter(function (obj) {
134+
return Object.keys(filteredItems).indexOf(obj) == -1;
135+
});
146136

147-
return filteredItems;
137+
console.log(
138+
'WARNING - items of ' + destination + ' was filtered ' + JSON.stringify(filteredColumns)
139+
);
148140
}
149-
);
141+
142+
return filteredItems;
143+
});
150144

151145
if (migratedItems.length > 0) {
152-
await dbV4(destination).insert(migratedItems);
146+
await dbV4(resolveDestTableName(destination)).insert(migratedItems);
153147
}
154148
}
155149

@@ -158,12 +152,10 @@ async function migrate(source, destination, itemMapper = undefined) {
158152

159153
async function resetTableSequence(destination) {
160154
if (isPGSQL) {
161-
const hasId = await dbV4.schema.hasColumn(destination, "id");
155+
const hasId = await dbV4.schema.hasColumn(destination, 'id');
162156
if (hasId) {
163157
const seq = `${destination.slice(0, 56)}_id_seq`;
164-
await dbV4.raw(
165-
`SELECT SETVAL ('${seq}', (SELECT MAX(id) + 1 FROM ${destination}))`
166-
);
158+
await dbV4.raw(`SELECT SETVAL ('${seq}', (SELECT MAX(id) + 1 FROM ${destination}))`);
167159
}
168160
}
169161
}

‎v3-sql-v4-sql/migrate/helpers/relationHelpers.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,7 @@ async function migrateRelations(tables, relations) {
146146
v4Tables = (
147147
await dbV4("information_schema.tables")
148148
.select("table_name")
149-
.where("table_schema", "public")
149+
.where("table_schema", process.env.DATABASE_V4_SCHEMA)
150150
).map((row) => row.table_name);
151151
}
152152

‎v3-sql-v4-sql/migrate/helpers/tableNameHelpers.js‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
const { isPGSQL } = require("../../config/database");
2+
3+
const resolveSourceTableName = (name) => {
4+
if (isPGSQL) {
5+
return process.env.DATABASE_V3_SCHEMA + "." + name;
6+
}
7+
8+
return name;
9+
};
10+
11+
const resolveDestTableName = (name) => {
12+
if (isPGSQL) {
13+
return process.env.DATABASE_V4_SCHEMA + "." + name;
14+
}
15+
16+
return name;
17+
};
18+
19+
module.exports = {
20+
resolveDestTableName,
21+
resolveSourceTableName,
22+
};

‎v3-sql-v4-sql/migrate/index.js‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,12 @@ const migrations = [
2727

2828
async function migrate() {
2929
if (isPGSQL) {
30-
await dbV4.raw("set session_replication_role to replica;");
30+
try {
31+
await dbV4.raw("set session_replication_role to replica;");
32+
} catch (error) {
33+
console.log("Error setting session_replication_role to replica, you may get foreign key constraint errors");
34+
console.log("Replication role requires specific admin permissions");
35+
}
3136
}
3237

3338
if (isMYSQL) {
@@ -39,7 +44,7 @@ async function migrate() {
3944
tables = (
4045
await dbV3("information_schema.tables")
4146
.select("table_name")
42-
.where("table_schema", "public")
47+
.where("table_schema", process.env.DATABASE_V3_SCHEMA)
4348
).map((row) => row.table_name);
4449
}
4550

‎v3-sql-v4-sql/migrate/migrateComponents.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ const {
1515
processRelation,
1616
migrateRelations,
1717
} = require("./helpers/relationHelpers");
18+
const { resolveSourceTableName } = require("./helpers/tableNameHelpers");
1819

1920
var relations = [];
2021
const skipAttributes = ["created_by", "updated_by"];
@@ -23,7 +24,7 @@ const processedTables = [];
2324
async function migrateTables(tables) {
2425
console.log("Migrating components");
2526

26-
const modelsDefs = await dbV3("core_store").where(
27+
const modelsDefs = await dbV3(resolveSourceTableName("core_store")).where(
2728
"key",
2829
"like",
2930
"model_def_%"
@@ -51,7 +52,7 @@ async function migrateTables(tables) {
5152
componentRelationsTables = (
5253
await dbV3("information_schema.tables")
5354
.select("table_name")
54-
.where("table_schema", "public")
55+
.where("table_schema", process.env.DATABASE_V3_SCHEMA)
5556
.where("table_name", "like", "%_components")
5657
)
5758
.map((row) => row.table_name)

0 commit comments

Comments
 (0)