Skip to content

Commit 08c7953

Browse files
chrisbobbechrisbobbe
committed
tools: Add check-messages-en, to find dangling messages in messages_en.json
As discussed on CZO: https://chat.zulip.org/#narrow/stream/243-mobile-team/topic/Strings.20in.20messages_en.2Ejson.20but.20not.20in.20the.20app/near/1425641 Run with: tools/check-messages-en This gives just one result, which we'll address next. We add flow-parser at <0.159.0. We'd like a later version -- probably ideal would be to sync it with flow-bin, which is ^0.170.0 right now -- but facebook/flow@5de4ea57e, released in v0.159.0, was a change that isn't yet handled by any version of ast-types [1]. A fix for ast-types is open as the issue benjamn/ast-types#728 and PR benjamn/ast-types#727. [1] I get this error output: /Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/types.js:669 throw new Error("did not recognize object of type " + ^ Error: did not recognize object of type "PropertyDefinition" at Object.getFieldNames (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/types.js:669:19) at visitChildren (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path-visitor.js:185:36) at Visitor.PVp.visitWithoutReset (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path-visitor.js:167:20) at NodePath.each (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path.js:88:26) at visitChildren (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path-visitor.js:179:18) at Visitor.PVp.visitWithoutReset (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path-visitor.js:167:20) at visitChildren (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path-visitor.js:204:25) at Visitor.PVp.visitWithoutReset (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path-visitor.js:167:20) at visitChildren (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path-visitor.js:204:25) at Visitor.PVp.visitWithoutReset (/Users/chrisbobbe/dev/zulip-mobile/node_modules/ast-types/lib/path-visitor.js:167:20)
1 parent 77b7e9c commit 08c7953

File tree

3 files changed

+263
-4
lines changed

3 files changed

+263
-4
lines changed

‎package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@
9292
"@rollup/plugin-node-resolve": "^13.0.4",
9393
"@types/react-native": "~0.67.6",
9494
"@vusion/webfonts-generator": "^0.8.0",
95+
"ast-types": "^0.16.1",
9596
"babel-plugin-transform-flow-enums": "^0.0.2",
9697
"core-js": "^3.1.4",
9798
"deep-freeze": "^0.0.1",
@@ -107,6 +108,7 @@
107108
"eslint-plugin-react-hooks": "^4.5.0",
108109
"flow-bin": "^0.170.0",
109110
"flow-coverage-report": "^0.8.0",
111+
"flow-parser": "<0.159.0",
110112
"flow-typed": "^3.3.1",
111113
"hermes-eslint": "^0.9.0",
112114
"immutable-devtools": "^0.1.5",
@@ -124,6 +126,7 @@
124126
"prettier-eslint-cli": "^6.0.1",
125127
"react-native-cli": "^2.0.1",
126128
"react-test-renderer": "17.0.2",
129+
"recast": "^0.22.0",
127130
"redux-mock-store": "^1.5.1",
128131
"rollup": "^2.26.5",
129132
"sqlite3": "^5.0.2",

‎tools/check-messages-en‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
#!/usr/bin/env node
2+
3+
// TODO: Type-check this file
4+
5+
// Sadly our auto-format doesn't seem to run eslint on this file; give in to
6+
// Prettier here.
7+
// TODO: debug
8+
/* eslint-disable operator-linebreak */
9+
10+
/**
11+
* Tests messages_en.json for strings that seem to not appear in the UI.
12+
*/
13+
14+
const fs = require('fs');
15+
const path = require('path');
16+
const { namedTypes: n, visit } = require('ast-types');
17+
const flowParser = require('flow-parser');
18+
const { parse } = require('recast');
19+
20+
const messages_en = require('../static/translations/messages_en.json');
21+
22+
/**
23+
* Make a list of files that might contain UI strings, by recursing in src/.
24+
*/
25+
function getPossibleUiStringFilePaths() {
26+
const result = [];
27+
const kSrcDirName = 'src/';
28+
function walk(dir, _dirName = '') {
29+
let dirent;
30+
// eslint-disable-next-line no-cond-assign
31+
while ((dirent = dir.readSync())) {
32+
// To reduce false negatives, `continue` when nothing in `dirent` can
33+
// cause UI strings to appear in the app.
34+
35+
if (dirent.isFile()) {
36+
if (!dirent.name.endsWith('.js')) {
37+
// Non-JS code, and Flow type definitions in .js.flow files.
38+
continue;
39+
}
40+
41+
result.push(path.join(kSrcDirName, _dirName, dirent.name));
42+
} else if (dirent.isDirectory()) {
43+
const subdirName = path.join(_dirName, dirent.name);
44+
45+
// e.g., …/__tests__ and …/__flow-tests__
46+
if (subdirName.endsWith('tests__')) {
47+
// Test code.
48+
continue;
49+
}
50+
51+
walk(fs.opendirSync(path.join(kSrcDirName, subdirName)), subdirName);
52+
} else {
53+
// Something we don't expect to find under src/, probably containing
54+
// no UI strings. (symlinks? fifos, sockets, devices??)
55+
continue;
56+
}
57+
}
58+
}
59+
walk(fs.opendirSync(kSrcDirName));
60+
return result;
61+
}
62+
63+
const parseOptions = {
64+
parser: {
65+
parse(src) {
66+
return flowParser.parse(src, {
67+
// Comments can't cause UI strings to appear in the app; ignore them.
68+
all_comments: false,
69+
comments: false,
70+
71+
// We use Flow enums; the parser shouldn't crash on them.
72+
enums: true,
73+
74+
// Set `tokens: true` just to work around a mysterious error.
75+
//
76+
// From the doc for this option:
77+
//
78+
// > include a list of all parsed tokens in a top-level tokens
79+
// > property
80+
//
81+
// We don't actually want this list of tokens. String literals do
82+
// get represented in the list, but as tokens, i.e., meaningful
83+
// chunks of the literal source code. They come with surrounding
84+
// quotes, escape syntax, etc:
85+
//
86+
// 'doesn\'t'
87+
// "doesn't"
88+
//
89+
// What we really want is the *value* of a string literal:
90+
//
91+
// doesn't
92+
//
93+
// and we get that from the AST.
94+
//
95+
// Anyway, we set `true` for this because otherwise I've been seeing
96+
// `parse` throw an error:
97+
//
98+
// Error: Line 72: Invalid regular expression: missing /
99+
//
100+
// TODO: Debug and/or file an issue upstream.
101+
tokens: true,
102+
});
103+
},
104+
},
105+
};
106+
107+
/**
108+
* Look at all given files and collect all strings that might be UI strings.
109+
*
110+
* The result will include non-UI strings because we can't realistically
111+
* filter them all out. That means, when the caller checks messages_en
112+
* against these strings, it could get false negatives: perhaps messages_en
113+
* has a string 'message-empty' (why would it, though), and that string
114+
* won't be flagged because it appears as an enum value in ComposeBox.
115+
*
116+
* To reduce these false negatives, we filter out low-hanging fruit, like
117+
* the string 'foo' in `import Foo from 'foo'`.
118+
*/
119+
function getPossibleUiStrings(possibleUiStringFilePaths) {
120+
const result = new Set();
121+
possibleUiStringFilePaths.forEach(filePath => {
122+
const source = fs.readFileSync(filePath).toString();
123+
const ast = parse(source, parseOptions);
124+
125+
visit(ast, {
126+
// Find nodes with type "Literal" in the AST.
127+
/* eslint-disable no-shadow */
128+
visitLiteral(path) {
129+
const { value } = path.value;
130+
131+
if (
132+
// (Non-string literals include numbers, booleans, etc.)
133+
typeof value === 'string' &&
134+
// This string isn't like 'react' in `import React from 'react'`.
135+
!n.ImportDeclaration.check(path.parent.value)
136+
) {
137+
result.add(value);
138+
}
139+
140+
// A literal is always a leaf, right? No need to call this.traverse.
141+
return false;
142+
},
143+
144+
// Find nodes with type "TemplateLiteral" in the AST. We sometimes use
145+
// template literals in UI strings for readability.
146+
/* eslint-disable no-shadow */
147+
visitTemplateLiteral(path) {
148+
if (
149+
// Translatable UI strings are unlikely to contain
150+
// sub-expressions.
151+
//
152+
// Also, if a template literal has nontrivial sub-expressions, we
153+
// can't reasonably interpret them here anyway.
154+
path.value.quasis.length === 1 &&
155+
path.value.expressions.length === 0
156+
) {
157+
result.add(path.value.quasis[0].value.cooked);
158+
}
159+
160+
return this.traverse(path);
161+
},
162+
});
163+
});
164+
return result;
165+
}
166+
167+
function main() {
168+
const possibleUiStrings = getPossibleUiStrings(getPossibleUiStringFilePaths());
169+
170+
// Check each key ("message ID" in formatjs's lingo) against
171+
// possibleUiStrings, and make a list of any that aren't found.
172+
const danglingMessageIds = Object.keys(messages_en).filter(
173+
messageId => !possibleUiStrings.has(messageId),
174+
);
175+
176+
if (danglingMessageIds.length > 0) {
177+
console.error(
178+
"Found message IDs in static/translations/messages_en.json that don't seem to be used in the app:",
179+
);
180+
console.error(danglingMessageIds);
181+
process.exit(1);
182+
}
183+
}
184+
185+
main();

‎yarn.lock‎

Lines changed: 75 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3199,6 +3199,16 @@ asap@~2.0.3, asap@~2.0.6:
31993199
resolved "https://registry.yarnpkg.com/asap/-/asap-2.0.6.tgz#e50347611d7e690943208bbdafebcbc2fb866d46"
32003200
integrity sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==
32013201

3202+
assert@^2.0.0:
3203+
version "2.0.0"
3204+
resolved "https://registry.yarnpkg.com/assert/-/assert-2.0.0.tgz#95fc1c616d48713510680f2eaf2d10dd22e02d32"
3205+
integrity sha512-se5Cd+js9dXJnu6Ag2JFc00t+HmHOen+8Q+L7O9zI0PqQXr20uk2J0XQqMxZEeo5U50o8Nvmmx7dZrl+Ufr35A==
3206+
dependencies:
3207+
es6-object-assign "^1.1.0"
3208+
is-nan "^1.2.1"
3209+
object-is "^1.0.1"
3210+
util "^0.12.0"
3211+
32023212
assign-symbols@^1.0.0:
32033213
version "1.0.0"
32043214
resolved "https://registry.yarnpkg.com/assign-symbols/-/assign-symbols-1.0.0.tgz#59667f41fadd4f20ccbc2bb96b8d4f7f78ec0367"
@@ -3216,6 +3226,20 @@ [email protected]:
32163226
dependencies:
32173227
tslib "^2.0.1"
32183228

3229+
[email protected]:
3230+
version "0.15.2"
3231+
resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.15.2.tgz#39ae4809393c4b16df751ee563411423e85fb49d"
3232+
integrity sha512-c27loCv9QkZinsa5ProX751khO9DJl/AcB5c2KNtA6NRvHKS0PgLfcftz72KVq504vB0Gku5s2kUZzDBvQWvHg==
3233+
dependencies:
3234+
tslib "^2.0.1"
3235+
3236+
ast-types@^0.16.1:
3237+
version "0.16.1"
3238+
resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.16.1.tgz#7a9da1617c9081bc121faafe91711b4c8bb81da2"
3239+
integrity sha512-6t10qk83GOG8p0vKmaCr8eiilZwO171AvbROMtvvNiwrTly62t+7XkA8RdIIVbpMhCASAsxgAzdRSwh6nw/5Dg==
3240+
dependencies:
3241+
tslib "^2.0.1"
3242+
32193243
"ast-types@npm:@gregprice/ast-types@^0.15.3-0.tsflower.5":
32203244
version "0.15.3-0.tsflower.5"
32213245
resolved "https://registry.yarnpkg.com/@gregprice/ast-types/-/ast-types-0.15.3-0.tsflower.5.tgz#59bfaf5b776dc8bf3e9de7d94d5a9546f3a1b6df"
@@ -4934,6 +4958,11 @@ es-to-primitive@^1.2.1:
49344958
is-date-object "^1.0.1"
49354959
is-symbol "^1.0.2"
49364960

4961+
es6-object-assign@^1.1.0:
4962+
version "1.1.0"
4963+
resolved "https://registry.yarnpkg.com/es6-object-assign/-/es6-object-assign-1.1.0.tgz#c2c3582656247c39ea107cb1e6652b6f9f24523c"
4964+
integrity sha512-MEl9uirslVwqQU369iHNWZXsI8yaZYGg/D65aOgZkeyFJwHYSxilf7rQzXKI7DdDuBPrBXbfk3sl9hJhmd5AUw==
4965+
49374966
escalade@^3.1.1:
49384967
version "3.1.1"
49394968
resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
@@ -5823,6 +5852,11 @@ flow-parser@0.*:
58235852
resolved "https://registry.yarnpkg.com/flow-parser/-/flow-parser-0.192.0.tgz#e2aa03e0c6a844c4d6ccdb4af2bc83cc589d9c8c"
58245853
integrity sha512-FLyei0ikf4ab9xlg+05WNmdpOODiH9XVBuw7iI9OZyjIo+cX2L2OUPTovjbWLYLlI41oGTcprbKdB/f9XwBnKw==
58255854

5855+
flow-parser@<0.159.0:
5856+
version "0.158.0"
5857+
resolved "https://registry.yarnpkg.com/flow-parser/-/flow-parser-0.158.0.tgz#d845f167c722babe880110fc3681c44f21823399"
5858+
integrity sha512-0hMsPkBTRrkII/0YiG9ehOxFXy4gOWdk8RSRze5WbfeKAQpL5kC2K4BmumyTfU9o5gr7/llgElF3UpSSrjzQAA==
5859+
58265860
flow-parser@^0.121.0:
58275861
version "0.121.0"
58285862
resolved "https://registry.yarnpkg.com/flow-parser/-/flow-parser-0.121.0.tgz#9f9898eaec91a9f7c323e9e992d81ab5c58e618f"
@@ -6741,7 +6775,7 @@ is-accessor-descriptor@^1.0.0:
67416775
dependencies:
67426776
kind-of "^6.0.0"
67436777

6744-
is-arguments@^1.1.0, is-arguments@^1.1.1:
6778+
is-arguments@^1.0.4, is-arguments@^1.1.0, is-arguments@^1.1.1:
67456779
version "1.1.1"
67466780
resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.1.1.tgz#15b3f88fda01f2a97fec84ca761a560f123efa9b"
67476781
integrity sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==
@@ -6898,6 +6932,13 @@ is-generator-fn@^2.0.0:
68986932
resolved "https://registry.yarnpkg.com/is-generator-fn/-/is-generator-fn-2.1.0.tgz#7d140adc389aaf3011a8f2a2a4cfa6faadffb118"
68996933
integrity sha512-cTIB4yPYL/Grw0EaSzASzg6bBy9gqCofvWN8okThAYIxKJZC+udlRAmGbM0XLeniEJSs8uEgHPGuHSe1XsOLSQ==
69006934

6935+
is-generator-function@^1.0.7:
6936+
version "1.0.10"
6937+
resolved "https://registry.yarnpkg.com/is-generator-function/-/is-generator-function-1.0.10.tgz#f1558baf1ac17e0deea7c0415c438351ff2b3c72"
6938+
integrity sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==
6939+
dependencies:
6940+
has-tostringtag "^1.0.0"
6941+
69016942
is-glob@^2.0.0:
69026943
version "2.0.1"
69036944
resolved "https://registry.yarnpkg.com/is-glob/-/is-glob-2.0.1.tgz#d096f926a3ded5600f3fdfd91198cb0888c2d863"
@@ -6939,6 +6980,14 @@ is-module@^1.0.0:
69396980
resolved "https://registry.yarnpkg.com/is-module/-/is-module-1.0.0.tgz#3258fb69f78c14d5b815d664336b4cffb6441591"
69406981
integrity sha512-51ypPSPCoTEIN9dy5Oy+h4pShgJmPCygKfyRCISBI+JoWT/2oJvK8QPxmwv7b/p239jXrm9M1mlQbyKJ5A152g==
69416982

6983+
is-nan@^1.2.1:
6984+
version "1.3.2"
6985+
resolved "https://registry.yarnpkg.com/is-nan/-/is-nan-1.3.2.tgz#043a54adea31748b55b6cd4e09aadafa69bd9e1d"
6986+
integrity sha512-E+zBKpQ2t6MEo1VsonYmluk9NxGrbzpeeLC2xIViuO2EjU2xsXsBPwTr3Ykv9l08UYEVEdWeRZNouaZqF6RN0w==
6987+
dependencies:
6988+
call-bind "^1.0.0"
6989+
define-properties "^1.1.3"
6990+
69426991
is-negative-zero@^2.0.2:
69436992
version "2.0.2"
69446993
resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.2.tgz#7bf6f03a28003b8b3965de3ac26f664d765f3150"
@@ -7049,7 +7098,7 @@ is-symbol@^1.0.2, is-symbol@^1.0.3:
70497098
dependencies:
70507099
has-symbols "^1.0.2"
70517100

7052-
is-typed-array@^1.1.10:
7101+
is-typed-array@^1.1.10, is-typed-array@^1.1.3:
70537102
version "1.1.10"
70547103
resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.10.tgz#36a5b5cb4189b575d1a3e4b08536bfb485801e3f"
70557104
integrity sha512-PJqgEHiWZvMpaFZ3uTc8kHPM4+4ADTlDniuQL7cU/UDA0Ql7F70yGfHph3cLNe+c9toaigv+DFzTJKhc2CtO6A==
@@ -9379,7 +9428,7 @@ object-inspect@^1.12.2, object-inspect@^1.9.0:
93799428
resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.12.2.tgz#c0641f26394532f28ab8d796ab954e43c009a8ea"
93809429
integrity sha512-z+cPxW0QGUp0mcqcsgQyLVRDoXFQbXOwBaqyF7VIgI4TWNQsDHrBpUQslRmIfAoYWdYzs6UlKJtB2XJpTaNSpQ==
93819430

9382-
object-is@^1.1.5:
9431+
object-is@^1.0.1, object-is@^1.1.5:
93839432
version "1.1.5"
93849433
resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.5.tgz#b9deeaa5fc7f1846a0faecdceec138e5778f53ac"
93859434
integrity sha512-3cyDsyHgtmi7I7DfSSI2LDp6SK2lwvtbg0p0R1e0RvTqF5ceGx+K2dfSjm1bKDMVCFEDAQvy+o8c6a7VujOddw==
@@ -10453,6 +10502,17 @@ recast@^0.20.4:
1045310502
source-map "~0.6.1"
1045410503
tslib "^2.0.1"
1045510504

10505+
recast@^0.22.0:
10506+
version "0.22.0"
10507+
resolved "https://registry.yarnpkg.com/recast/-/recast-0.22.0.tgz#1dd3bf1b86e5eb810b044221a1a734234ed3e9c0"
10508+
integrity sha512-5AAx+mujtXijsEavc5lWXBPQqrM4+Dl5qNH96N2aNeuJFUzpiiToKPsxQD/zAIJHspz7zz0maX0PCtCTFVlixQ==
10509+
dependencies:
10510+
assert "^2.0.0"
10511+
ast-types "0.15.2"
10512+
esprima "~4.0.0"
10513+
source-map "~0.6.1"
10514+
tslib "^2.0.1"
10515+
1045610516
"recast@npm:@gregprice/recast@^0.21.2-0.tsflower.8":
1045710517
version "0.21.2-0.tsflower.8"
1045810518
resolved "https://registry.yarnpkg.com/@gregprice/recast/-/recast-0.21.2-0.tsflower.8.tgz#576b7329a3cb6fcd1e406ebded7daf3cd5d9f573"
@@ -12230,6 +12290,17 @@ util.promisify@~1.0.0:
1223012290
has-symbols "^1.0.1"
1223112291
object.getownpropertydescriptors "^2.1.0"
1223212292

12293+
util@^0.12.0:
12294+
version "0.12.5"
12295+
resolved "https://registry.yarnpkg.com/util/-/util-0.12.5.tgz#5f17a6059b73db61a875668781a1c2b136bd6fbc"
12296+
integrity sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==
12297+
dependencies:
12298+
inherits "^2.0.3"
12299+
is-arguments "^1.0.4"
12300+
is-generator-function "^1.0.7"
12301+
is-typed-array "^1.1.3"
12302+
which-typed-array "^1.1.2"
12303+
1223312304
[email protected]:
1223412305
version "0.2.1"
1223512306
resolved "https://registry.yarnpkg.com/utile/-/utile-0.2.1.tgz#930c88e99098d6220834c356cbd9a770522d90d7"
@@ -12435,7 +12506,7 @@ which-module@^2.0.0:
1243512506
resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
1243612507
integrity sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==
1243712508

12438-
which-typed-array@^1.1.8:
12509+
which-typed-array@^1.1.2, which-typed-array@^1.1.8:
1243912510
version "1.1.9"
1244012511
resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.9.tgz#307cf898025848cf995e795e8423c7f337efbde6"
1244112512
integrity sha512-w9c4xkx6mPidwp7180ckYWfMmvxpjlZuIudNtDf4N/tTAUB8VJbX25qZoAsrtGuYNnGw3pa0AXgbGKRB8/EceA==

0 commit comments

Comments
 (0)