Skip to content

Commit 520b2ca

Browse files
tgphollytgpholly
committed
Cache Database requests & decode uri better
1 parent c8b96b1 commit 520b2ca

File tree

1 file changed

+61
-42
lines changed

1 file changed

+61
-42
lines changed

‎EUS.js

+61-42
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,15 @@ const config = require("../config/config.json"), crypto = require("crypto"), emo
44
const MODULE_FUNCTION = "handle_requests",
55

66
// Base path for module folder creation and navigation
7-
BASE_PATH = "/EUS";
7+
BASE_PATH = "/EUS",
8+
API_CACHE_LIFESPAN = 3600000;
89

910
let node_modules = {};
1011

1112
let eusConfig = {},
1213
useUploadKey = true,
1314
cacheJSON = "",
1415
startupFinished = false,
15-
diskRunningOnSize = 0,
1616
timeSinceLastCache = Date.now();
1717

1818
class Database {
@@ -100,16 +100,11 @@ function init() {
100100
node_modules["chalk"] = require("chalk");
101101
node_modules["busboy"] = require("connect-busboy");
102102
node_modules["randomstring"] = require("randomstring");
103-
node_modules["diskUsage"] = require("diskusage");
104103
node_modules["streamMeter"] = require("stream-meter");
105104
node_modules["mysql2"] = require("mysql2");
106105

107106
// Only ran on startup so using sync functions is fine
108107

109-
// Fetch total size of disk on startup, this will never change during runtime
110-
// if it does something seriously wrong has happened.
111-
diskRunningOnSize = node_modules.diskUsage.checkSync(__dirname).total;
112-
113108
// Makes the folder for files of the module
114109
if (!fs.existsSync(__dirname + BASE_PATH)) {
115110
fs.mkdirSync(__dirname + BASE_PATH);
@@ -126,6 +121,11 @@ function init() {
126121
console.log(`[EUS] Made EUS images folder`);
127122
}
128123

124+
if (!fs.existsSync("/tmp/EUS_UPLOADS")) {
125+
fs.mkdirSync("/tmp/EUS_UPLOADS");
126+
console.log("[EUS] Made EUS temp upload folder");
127+
}
128+
129129
// Makes the config file
130130
if (!fs.existsSync(__dirname + BASE_PATH + "/config.json")) {
131131
// Config doesn't exist, make it.
@@ -151,6 +151,7 @@ function init() {
151151
// Cache for the file count and space usage, this takes a while to do so it's best to cache the result
152152
let cacheIsReady = false;
153153
function cacheFilesAndSpace() {
154+
timeSinceLastCache = Date.now();
154155
return new Promise(async (resolve, reject) => {
155156
const startCacheTime = Date.now();
156157
cacheIsReady = false;
@@ -166,20 +167,14 @@ function cacheFilesAndSpace() {
166167
});
167168
cachedFilesAndSpace["filesTotal"] = totalFiles;
168169

169-
cachedFilesAndSpace["space"] = {usage: {}, total:{}};
170+
cachedFilesAndSpace["size"] = {};
170171

171-
const dbSize = (await dbConnection.query(`SELECT SUM(imageSize) FROM images LIMIT 1`))["SUM(imageSize)"];
172+
const dbSize = (await dbConnection.query(`SELECT SUM(fileSize) FROM images LIMIT 1`))["SUM(fileSize)"];
172173
const totalSizeBytes = dbSize == null ? 0 : dbSize;
173174
const mbSize = totalSizeBytes / 1024 / 1024;
174-
cachedFilesAndSpace["space"]["usage"]["mb"] = parseFloat(mbSize.toFixed(4));
175-
cachedFilesAndSpace["space"]["usage"]["gb"] = parseFloat((mbSize / 1024).toFixed(4));
176-
cachedFilesAndSpace["space"]["usage"]["string"] = await spaceToLowest(totalSizeBytes, true);
177-
178-
//totalDiskSize
179-
const totalMBSize = diskRunningOnSize / 1024 / 1024;
180-
cachedFilesAndSpace["space"]["total"]["mb"] = parseFloat(totalMBSize.toFixed(4));
181-
cachedFilesAndSpace["space"]["total"]["gb"] = parseFloat((totalMBSize / 1024).toFixed(4));
182-
cachedFilesAndSpace["space"]["total"]["string"] = await spaceToLowest(diskRunningOnSize, true);
175+
cachedFilesAndSpace["size"]["mb"] = parseFloat(mbSize.toFixed(4));
176+
cachedFilesAndSpace["size"]["gb"] = parseFloat((mbSize / 1024).toFixed(4));
177+
cachedFilesAndSpace["size"]["string"] = await spaceToLowest(totalSizeBytes, true);
183178

184179
resolve(cachedFilesAndSpace);
185180
global.modules.consoleHelper.printInfo(emoji.folder, `Stats api cache took ${Date.now() - startCacheTime}ms`);
@@ -250,7 +245,7 @@ function validateConfig(json) {
250245

251246
// Check if server needs to be shutdown
252247
if (performShutdownAfterValidation) {
253-
console.error("EUS config properties are missing, refer to docs for more details (https://wiki.eusv.ml)");
248+
console.error("EUS config properties are missing, refer to example config on GitHub (https://github.com/tgpholly/EUS)");
254249
process.exit(1);
255250
}
256251
else return true;
@@ -277,7 +272,7 @@ function regularFile(req, res, urs = "", startTime = 0) {
277272
global.modules.consoleHelper.printInfo(emoji.cross, `${req.method}: ${node_modules.chalk.red("[404]")} ${req.url} ${Date.now() - startTime}ms`);
278273
} else {
279274
// File does exist, send it back to the client.
280-
res.sendFile(__dirname + BASE_PATH + "/files"+req.url);
275+
res.sendFile(`${__dirname}${BASE_PATH}/files${req.url}`);
281276
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: ${node_modules.chalk.green("[200]")} ${req.url} ${Date.now() - startTime}ms`);
282277
}
283278
});
@@ -298,8 +293,6 @@ module.exports = {
298293
// Setup express to use busboy
299294
global.app.use(node_modules.busboy());
300295
startupFinished = true;
301-
//cacheJSON = JSON.stringify(await cacheFilesAndSpace());
302-
//cacheIsReady = true;
303296
},
304297
get:async function(req, res) {
305298
/*
@@ -316,7 +309,7 @@ module.exports = {
316309
res.set("X-Frame-Options", "SAMEORIGIN");
317310
res.set("X-Content-Type-Options", "nosniff");
318311

319-
req.url = cleanURL(req.url);
312+
req.url = decodeURIComponent(req.url.split("?")[0]);
320313

321314
// The Funny Response
322315
if (req.url.includes(".php") || req.url.includes("/wp-")) {
@@ -337,16 +330,20 @@ module.exports = {
337330

338331
if (dbAble) {
339332
if (urs in existanceCache) {
340-
imageFile(req, res, `${urs}.${existanceCache[urs]}`, startTime);
333+
const cachedFile = existanceCache[urs];
334+
imageFile(req, res, `${cachedFile.fileHash}.${cachedFile.fileType}`, startTime);
341335
} else {
342336
if (dbConnection.dbActive) {
343337
// Try to get what we think is an image's details from the DB
344-
const dbEntry = await dbConnection.query(`SELECT imageType FROM images WHERE imageId = ? LIMIT 1`, [urs]);
338+
const dbEntry = await dbConnection.query(`SELECT hash, imageType FROM images WHERE imageId = ? LIMIT 1`, [urs]);
345339

346340
// There's an entry in the DB for this, send the file back.
347341
if (dbEntry != null) {
348-
existanceCache[urs] = dbEntry.imageType;
349-
imageFile(req, res, `${urs}.${dbEntry.imageType}`, startTime);
342+
existanceCache[urs] = {
343+
fileHash: dbEntry.hash,
344+
fileType: dbEntry.imageType
345+
};
346+
imageFile(req, res, `${dbEntry.hash}.${dbEntry.imageType}`, startTime);
350347
}
351348
// There's no entry, so treat this as a regular file.
352349
else regularFile(req, res, urs, startTime);
@@ -396,7 +393,7 @@ module.exports = {
396393
return;
397394
}
398395
// Create a write stream for the file
399-
let fstream = fs.createWriteStream(__dirname + BASE_PATH + "/i/" + fileOutName + "." + thefe);
396+
let fstream = fs.createWriteStream("/tmp/EUS_UPLOADS/" + fileOutName);
400397
file.pipe(fstream);
401398

402399
// Get all file data for the file MD5
@@ -416,19 +413,41 @@ module.exports = {
416413
hash.write(md5Buffer);
417414
hash.end();
418415

419-
// Add to the existance cache
420-
existanceCache[fileOutName] = thefe[0];
421-
422-
// Store image data in db
423-
await dbConnection.query(`INSERT INTO images (id, imageId, imageType, hash, imageSize) VALUES (NULL, ?, ?, ?, ?)`, [fileOutName, thefe[0], hash.read(), md5Buffer.length]);
424-
425-
// Send URL of the uploaded image to the client
426-
res.end(eusConfig.baseURL + fileOutName);
427-
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${Date.now() - startTime}ms`);
428-
429-
// Update cached files & space
430-
cacheJSON = JSON.stringify(await cacheFilesAndSpace());
431-
cacheIsReady = true;
416+
const fileHash = hash.read();
417+
const dataOnHash = await dbConnection.query("SELECT imageId FROM images WHERE hash = ? LIMIT 1", [fileHash]);
418+
if (dataOnHash !== undefined)
419+
{
420+
fs.unlink(`/tmp/EUS_UPLOADS/${fileOutName}`, () => {});
421+
res.end(`${eusConfig.baseURL}${dataOnHash.imageId}`);
422+
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Hash matched! Sending ${dataOnHash.imageId} instead. Took ${Date.now() - startTime}ms`);
423+
return;
424+
} else {
425+
fs.rename(`/tmp/EUS_UPLOADS/${fileOutName}`, `${__dirname}${BASE_PATH}/i/${fileHash}.${thefe[0]}`, async () => {
426+
// Add to the existance cache
427+
existanceCache[fileOutName] = {
428+
fileHash: fileHash,
429+
fileType: thefe[0]
430+
};
431+
432+
// Store image data in db
433+
await dbConnection.query(`INSERT INTO images (id, imageId, imageType, hash, fileSize) VALUES (NULL, ?, ?, ?, ?)`, [fileOutName, thefe[0], fileHash, md5Buffer.length]);
434+
435+
// Send URL of the uploaded image to the client
436+
res.end(eusConfig.baseURL + fileOutName);
437+
global.modules.consoleHelper.printInfo(emoji.heavy_check, `${req.method}: Upload of ${fileOutName} finished. Took ${Date.now() - startTime}ms`);
438+
439+
// Update cached files & space
440+
if ((Date.now() - timeSinceLastCache) >= API_CACHE_LIFESPAN) {
441+
cacheJSON = JSON.stringify(await cacheFilesAndSpace());
442+
} else {
443+
const tempJson = JSON.parse(cacheJSON);
444+
tempJson.fileCounts[thefe[0]]++;
445+
cacheJSON = JSON.stringify(tempJson);
446+
global.modules.consoleHelper.printInfo(emoji.folder, `Skiped api cache`);
447+
}
448+
cacheIsReady = true;
449+
});
450+
}
432451
});
433452

434453
/*fstream.on('close', async () => {
@@ -529,5 +548,5 @@ module.exports.MOD_FUNC = MODULE_FUNCTION;
529548

530549
module.exports.REQUIRED_NODE_MODULES = [
531550
"chalk", "connect-busboy", "randomstring",
532-
"diskusage", "stream-meter", "mysql2"
551+
"stream-meter", "mysql2"
533552
];

0 commit comments

Comments
 (0)