Skip to content

Commit 74d9a8d

Browse files
derrickmehaffyderrickmehaffy
committed
Merge branch 'main' into 47-cannot-read-properties-of-undefined-reading-includes
2 parents fb06be3 + 2690b05 commit 74d9a8d

File tree

13 files changed

+467
-422
lines changed

13 files changed

+467
-422
lines changed

‎v3-sql-v4-sql/config/database.js‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,6 @@ if (process.env.DATABASE_CLIENT === "pg") {
2727
user: process.env.DATABASE_V3_USER,
2828
password: process.env.DATABASE_V3_PASSWORD,
2929
database: process.env.DATABASE_V3_DATABASE,
30-
schema: process.env.DATABASE_V3_SCHEMA
3130
},
3231
};
3332

@@ -39,7 +38,6 @@ if (process.env.DATABASE_CLIENT === "pg") {
3938
user: process.env.DATABASE_V4_USER,
4039
password: process.env.DATABASE_V4_PASSWORD,
4140
database: process.env.DATABASE_V4_DATABASE,
42-
schema: process.env.DATABASE_V4_SCHEMA
4341
},
4442
};
4543
}
@@ -51,7 +49,7 @@ if (process.env.DATABASE_CLIENT === "mysql") {
5149
port: process.env.DATABASE_V3_PORT,
5250
user: process.env.DATABASE_V3_USER,
5351
password: process.env.DATABASE_V3_PASSWORD,
54-
database: process.env.DATABASE_V3_DATABASE
52+
database: process.env.DATABASE_V3_DATABASE,
5553
},
5654
};
5755

@@ -62,7 +60,7 @@ if (process.env.DATABASE_CLIENT === "mysql") {
6260
port: process.env.DATABASE_V4_PORT,
6361
user: process.env.DATABASE_V4_USER,
6462
password: process.env.DATABASE_V4_PASSWORD,
65-
database: process.env.DATABASE_V4_DATABASE
63+
database: process.env.DATABASE_V4_DATABASE,
6664
},
6765
};
6866
}

‎v3-sql-v4-sql/migrate/helpers/adminHelpers.js‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ const { migrateItems } = require("./migrateFields");
1111
const { migrateUids } = require("./migrateValues");
1212
const pluralize = require("pluralize");
1313
const { camelCase } = require("lodash");
14+
const { resolveDestTableName, resolveSourceTableName } = require("./tableNameHelpers");
1415

1516
const extraV4Permissions = [
1617
{ action: "admin::api-tokens.create", properties: {}, conditions: [] },
@@ -37,19 +38,20 @@ function migrateProperties(properties) {
3738
return properties;
3839
}
3940

41+
4042
async function migrateAdminPermissions() {
4143
const source = "strapi_permission";
4244
const destination = "admin_permissions";
4345
const destinationLinks = "admin_permissions_role_links";
4446
const count =
45-
(await dbV3(source).count().first()).count ||
46-
(await dbV3(source).count().first())["count(*)"];
47+
(await dbV3(resolveSourceTableName(source)).count().first()).count ||
48+
(await dbV3(resolveSourceTableName(source)).count().first())["count(*)"];
4749
console.log(`Migrating ${count} items from ${source} to ${destination}`);
48-
await dbV4(destinationLinks).del();
49-
await dbV4(destination).del();
50+
await dbV4(resolveDestTableName(destinationLinks)).del();
51+
await dbV4(resolveDestTableName(destination)).del();
5052
for (var page = 0; page * BATCH_SIZE < count; page++) {
5153
console.log(`${source} batch #${page + 1}`);
52-
const items = await dbV3(source)
54+
const items = await dbV3(resolveSourceTableName(source))
5355
.limit(BATCH_SIZE)
5456
.offset(page * BATCH_SIZE);
5557
const migratedItems = migrateItems(items, ({ role, ...item }) => ({
@@ -63,19 +65,19 @@ async function migrateAdminPermissions() {
6365
permission_id: item.id,
6466
role_id: item.role,
6567
}));
66-
await dbV4(destination).insert(migratedItems);
67-
await dbV4(destinationLinks).insert(roleLinks);
68+
await dbV4(resolveDestTableName(destination)).insert(migratedItems);
69+
await dbV4(resolveDestTableName(destinationLinks)).insert(roleLinks);
6870
}
6971
await resetTableSequence(destination);
7072

7173
let ids = [];
7274

7375
if (isPGSQL) {
74-
ids = await dbV4(destination).insert(extraV4Permissions).returning("id");
76+
ids = await dbV4(resolveDestTableName(destination)).insert(extraV4Permissions).returning("id");
7577
}
7678

7779
if (isSQLITE) {
78-
ids = await dbV4(destination).insert(
80+
ids = await dbV4(resolveDestTableName(destination)).insert(
7981
extraV4Permissions.map((item) => ({
8082
...item,
8183
properties: JSON.stringify(item.properties),
@@ -85,7 +87,7 @@ async function migrateAdminPermissions() {
8587
}
8688

8789
if (isMYSQL) {
88-
ids = await dbV4(destination).insert(
90+
ids = await dbV4(resolveDestTableName(destination)).insert(
8991
extraV4Permissions.map((item) => ({
9092
...item,
9193
properties: JSON.stringify(item.properties),
@@ -94,7 +96,7 @@ async function migrateAdminPermissions() {
9496
);
9597
}
9698

97-
await dbV4(destinationLinks).insert(
99+
await dbV4(resolveDestTableName(destinationLinks)).insert(
98100
ids.map((id) => ({ permission_id: id.id, role_id: SUPER_ADMIN }))
99101
);
100102
}

‎v3-sql-v4-sql/migrate/helpers/constants.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
const { isPGSQL } = require("../../config/database");
2+
13
const BATCH_SIZE = process.env.BATCH_SIZE || 50;
24
const SUPER_ADMIN = 1;
35

‎v3-sql-v4-sql/migrate/helpers/migrate.js‎

Lines changed: 52 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,12 @@
1-
const {
2-
dbV3,
3-
dbV4,
4-
isPGSQL,
5-
isMYSQL,
6-
isSQLITE,
7-
} = require("../../config/database");
8-
const { BATCH_SIZE } = require("./constants");
9-
const { migrateItems } = require("./migrateFields");
10-
const { pick } = require("lodash");
1+
const { dbV3, dbV4, isPGSQL, isMYSQL, isSQLITE } = require('../../config/database');
2+
const { BATCH_SIZE } = require('./constants');
3+
const { migrateItems } = require('./migrateFields');
4+
const { pick } = require('lodash');
5+
const { resolveDestTableName, resolveSourceTableName } = require('./tableNameHelpers');
116

127
async function migrate(source, destination, itemMapper = undefined) {
138
if (isMYSQL) {
14-
const sourceNotExists =
15-
(await dbV3.raw(`SHOW TABLES LIKE '%${source}%';`))[0].length === 0;
9+
const sourceNotExists = (await dbV3.raw(`SHOW TABLES LIKE '%${source}%';`))[0].length === 0;
1610
const destinationNotExists =
1711
(await dbV4.raw(`SHOW TABLES LIKE '%${destination}%';`))[0].length === 0;
1812

@@ -32,23 +26,23 @@ async function migrate(source, destination, itemMapper = undefined) {
3226

3327
const sourceNotExists =
3428
(
35-
await dbV3("sqlite_master")
36-
.select("name")
37-
.where("type", "table")
38-
.where("name", source)
29+
await dbV3('sqlite_master')
30+
.select('name')
31+
.where('type', 'table')
32+
.where('name', source)
3933
.first()
4034
.count()
41-
)["count(*)"] === 0;
35+
)['count(*)'] === 0;
4236

4337
const destinationNotExists =
4438
(
45-
await dbV4("sqlite_master")
46-
.select("name")
47-
.where("type", "table")
48-
.where("name", destination)
39+
await dbV4('sqlite_master')
40+
.select('name')
41+
.where('type', 'table')
42+
.where('name', destination)
4943
.first()
5044
.count()
51-
)["count(*)"] === 0;
45+
)['count(*)'] === 0;
5246

5347
if (sourceNotExists) {
5448
console.log(`SOURCE TABLE ${source} DOES NOT EXISTS`);
@@ -68,18 +62,18 @@ async function migrate(source, destination, itemMapper = undefined) {
6862

6963
const sourceNotExists =
7064
(
71-
await dbV3("information_schema.tables")
72-
.select("table_name")
73-
.where("table_schema", "public")
74-
.where("table_name", source)
65+
await dbV3('information_schema.tables')
66+
.select('table_name')
67+
.where('table_schema', process.env.DATABASE_V3_SCHEMA)
68+
.where('table_name', source)
7569
).length === 0;
7670

7771
const destinationNotExists =
7872
(
79-
await dbV4("information_schema.tables")
80-
.select("table_name")
81-
.where("table_schema", "public")
82-
.where("table_name", destination)
73+
await dbV4('information_schema.tables')
74+
.select('table_name')
75+
.where('table_schema', process.env.DATABASE_V4_SCHEMA)
76+
.where('table_name', destination)
8377
).length === 0;
8478

8579
if (sourceNotExists) {
@@ -94,26 +88,31 @@ async function migrate(source, destination, itemMapper = undefined) {
9488
}
9589

9690
const count =
97-
(await dbV3(source).count().first()).count ||
98-
(await dbV3(source).count().first())["count(*)"];
99-
const columnsInfo = await dbV3(source).columnInfo();
91+
(await dbV3(resolveSourceTableName(source)).count().first()).count ||
92+
(await dbV3(resolveSourceTableName(source)).count().first())['count(*)'];
93+
const columnsInfo = await dbV3(resolveSourceTableName(source)).columnInfo();
10094

10195
const jsonFields = Object.keys(columnsInfo).filter((column) => {
102-
return columnsInfo[column].type === "jsonb";
96+
return columnsInfo[column].type === 'jsonb';
10397
});
10498

10599
console.log(`Migrating ${count} items from ${source} to ${destination}`);
106-
await dbV4(destination).del();
100+
await dbV4(resolveDestTableName(destination)).del();
107101

108-
console.log("DBV4 ITEMS");
102+
let tableColumnsInfo = await dbV4(destination).columnInfo();
109103

110-
const tableColumnsInfo = await dbV4(destination).columnInfo();
104+
if (isPGSQL) {
105+
// https://github.com/knex/knex/issues/1490
106+
tableColumnsInfo = await dbV4(destination)
107+
.withSchema(process.env.DATABASE_V4_SCHEMA)
108+
.columnInfo();
109+
}
111110

112111
const tableColumns = Object.keys(tableColumnsInfo);
113112

114113
for (let page = 0; page * BATCH_SIZE < count; page++) {
115114
console.log(`${source} batch #${page + 1}`);
116-
const items = await dbV3(source)
115+
const items = await dbV3(resolveSourceTableName(source))
117116
.limit(BATCH_SIZE)
118117
.offset(page * BATCH_SIZE);
119118

@@ -127,29 +126,24 @@ async function migrate(source, destination, itemMapper = undefined) {
127126
return item;
128127
});
129128

130-
const migratedItems = migrateItems(withParsedJsonFields, itemMapper).map(
131-
(item) => {
132-
const filteredItems = pick(item, tableColumns);
133-
134-
if (Object.keys(item).length !== Object.keys(filteredItems).length) {
135-
const filteredColumns = Object.keys(item).filter(function (obj) {
136-
return Object.keys(filteredItems).indexOf(obj) == -1;
137-
});
129+
const migratedItems = migrateItems(withParsedJsonFields, itemMapper).map((item) => {
130+
const filteredItems = pick(item, tableColumns);
138131

139-
console.log(
140-
"WARNING - items of " +
141-
destination +
142-
" was filtered " +
143-
JSON.stringify(filteredColumns)
144-
);
145-
}
132+
if (Object.keys(item).length !== Object.keys(filteredItems).length) {
133+
const filteredColumns = Object.keys(item).filter(function (obj) {
134+
return Object.keys(filteredItems).indexOf(obj) == -1;
135+
});
146136

147-
return filteredItems;
137+
console.log(
138+
'WARNING - items of ' + destination + ' was filtered ' + JSON.stringify(filteredColumns)
139+
);
148140
}
149-
);
141+
142+
return filteredItems;
143+
});
150144

151145
if (migratedItems.length > 0) {
152-
await dbV4(destination).insert(migratedItems);
146+
await dbV4(resolveDestTableName(destination)).insert(migratedItems);
153147
}
154148
}
155149

@@ -158,12 +152,10 @@ async function migrate(source, destination, itemMapper = undefined) {
158152

159153
async function resetTableSequence(destination) {
160154
if (isPGSQL) {
161-
const hasId = await dbV4.schema.hasColumn(destination, "id");
155+
const hasId = await dbV4.schema.hasColumn(destination, 'id');
162156
if (hasId) {
163157
const seq = `${destination.slice(0, 56)}_id_seq`;
164-
await dbV4.raw(
165-
`SELECT SETVAL ('${seq}', (SELECT MAX(id) + 1 FROM ${destination}))`
166-
);
158+
await dbV4.raw(`SELECT SETVAL ('${seq}', (SELECT MAX(id) + 1 FROM ${destination}))`);
167159
}
168160
}
169161
}

‎v3-sql-v4-sql/migrate/helpers/relationHelpers.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ async function migrateRelations(tables, relations) {
135135

136136
if (isPGSQL) {
137137
v4Tables = (
138-
await dbV4('information_schema.tables').select('table_name').where('table_schema', 'public')
138+
await dbV4('information_schema.tables').select('table_name').where('table_schema', process.env.DATABASE_V4_SCHEMA)
139139
).map((row) => row.table_name);
140140
}
141141

‎v3-sql-v4-sql/migrate/helpers/tableNameHelpers.js‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
const { isPGSQL } = require("../../config/database");
2+
3+
const resolveSourceTableName = (name) => {
4+
if (isPGSQL) {
5+
return process.env.DATABASE_V3_SCHEMA + "." + name;
6+
}
7+
8+
return name;
9+
};
10+
11+
const resolveDestTableName = (name) => {
12+
if (isPGSQL) {
13+
return process.env.DATABASE_V4_SCHEMA + "." + name;
14+
}
15+
16+
return name;
17+
};
18+
19+
module.exports = {
20+
resolveDestTableName,
21+
resolveSourceTableName,
22+
};

‎v3-sql-v4-sql/migrate/index.js‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,12 @@ const migrations = [
2121

2222
async function migrate() {
2323
if (isPGSQL) {
24-
await dbV4.raw('set session_replication_role to replica;');
24+
try {
25+
await dbV4.raw("set session_replication_role to replica;");
26+
} catch (error) {
27+
console.log("Error setting session_replication_role to replica, you may get foreign key constraint errors");
28+
console.log("Replication role requires specific admin permissions");
29+
}
2530
}
2631

2732
if (isMYSQL) {
@@ -31,7 +36,9 @@ async function migrate() {
3136

3237
if (isPGSQL) {
3338
tables = (
34-
await dbV3('information_schema.tables').select('table_name').where('table_schema', 'public')
39+
await dbV3("information_schema.tables")
40+
.select("table_name")
41+
.where("table_schema", process.env.DATABASE_V3_SCHEMA)
3542
).map((row) => row.table_name);
3643
}
3744

‎v3-sql-v4-sql/migrate/migrateComponents.js‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,11 @@ const { singular } = require('pluralize');
55
const { migrateUids } = require('./helpers/migrateValues');
66
const { migrateItem } = require('./helpers/migrateFields');
77

8-
const { processRelation, migrateRelations } = require('./helpers/relationHelpers');
8+
const {
9+
processRelation,
10+
migrateRelations,
11+
} = require("./helpers/relationHelpers");
12+
const { resolveSourceTableName } = require("./helpers/tableNameHelpers");
913

1014
var relations = [];
1115
const skipAttributes = ['created_by', 'updated_by'];
@@ -14,7 +18,11 @@ const processedTables = [];
1418
async function migrateTables(tables) {
1519
console.log('Migrating components');
1620

17-
const modelsDefs = await dbV3('core_store').where('key', 'like', 'model_def_%');
21+
const modelsDefs = await dbV3(resolveSourceTableName("core_store")).where(
22+
"key",
23+
"like",
24+
"model_def_%"
25+
);
1826

1927
const componentsToMigrate = modelsDefs
2028
.filter((item) => {
@@ -36,10 +44,10 @@ async function migrateTables(tables) {
3644

3745
if (isPGSQL) {
3846
componentRelationsTables = (
39-
await dbV3('information_schema.tables')
40-
.select('table_name')
41-
.where('table_schema', 'public')
42-
.where('table_name', 'like', '%_components')
47+
await dbV3("information_schema.tables")
48+
.select("table_name")
49+
.where("table_schema", process.env.DATABASE_V3_SCHEMA)
50+
.where("table_name", "like", "%_components")
4351
)
4452
.map((row) => row.table_name)
4553
.filter((item) => !componentsToMigrate.includes(item));

0 commit comments

Comments
 (0)