Skip to content

Commit 169d571

Browse files
annebanneb
committed
globalize cache, add column_stats
1 parent efe0c9d commit 169d571

File tree

7 files changed

+223
-54
lines changed

7 files changed

+223
-54
lines changed

‎README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ If you don't have git, you can donwload [a zip file](https://github.com/anneb/pg
1919
npm install
2020
cp config/dbconfig.example.json config/dbconfig.json
2121
# now edit config/dbconfig.json for your PostGis database
22-
npm start
22+
node pgserver.js
2323
# point your browser to localost:8090 for more info
2424

2525
For interactive data browsing, preview, administration and api documentation, head to [http://localhost:8090](http://localhost:8090).

‎column_stats.js

+144
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
const sqlTableName = require('./utils/sqltablename.js');
2+
3+
const sql = (params, query) => {
4+
return `
5+
select count(1)::integer as "count", ${params.column} as "value"
6+
from ${params.table}
7+
where ${query.geom_column} is not null
8+
group by ${params.column} order by count(1) desc limit 2000;
9+
`
10+
} // TODO, use sql place holders $1, $2 etc. instead of inserting user-parameters into query
11+
12+
module.exports = function(app, pool, cache) {
13+
14+
let cacheMiddleWare = async(req, res, next) => {
15+
if (!cache) {
16+
next();
17+
return;
18+
}
19+
const cacheDir = `${req.params.table}/attrstats/`;
20+
const key = ((req.query.geom_column?req.query.geom_column:'geom') + (req.params.column?','+req.params.column:''))
21+
.replace(/[\W]+/g, '_');
22+
23+
const stats = await cache.getCachedFile(cacheDir, key);
24+
if (stats) {
25+
console.log(`stats cache hit for ${cacheDir}?${key}`);
26+
if (stats.length === 0) {
27+
res.status(204)
28+
}
29+
res.header('Content-Type', 'application/json').send(stats);
30+
return;
31+
} else {
32+
res.sendResponse = res.send;
33+
res.send = (body) => {
34+
if (res.statusCode >= 200 && res.statusCode < 300) {
35+
cache.setCachedFile(cacheDir, key, body);
36+
}
37+
res.sendResponse(body);
38+
}
39+
next();
40+
}
41+
}
42+
43+
/**
44+
* @swagger
45+
*
46+
* /data/{table}/colstats/{column}:
47+
* get:
48+
* description: get statistics for column
49+
* tags: ['meta']
50+
* produces:
51+
* - application/json
52+
* parameters:
53+
* - name: table
54+
* description: name of postgis table
55+
* in: path
56+
* required: true
57+
* type: string
58+
* - name: column
59+
* description: name of column
60+
* in: path
61+
* type: string
62+
* required: true
63+
* - name: geom_column
64+
* description: name of geometry column (default 'geom')
65+
* in: query
66+
* required: false
67+
* responses:
68+
* 200:
69+
* description: json statistics
70+
* content:
71+
* application/json
72+
* schema:
73+
* type: object
74+
* properties:
75+
* table:
76+
* type: string
77+
* description: name of table
78+
* column:
79+
* type: string
80+
* description: name of column
81+
* numvalues:
82+
* description: number of different values, null means unknown (>2000)
83+
* type: integer
84+
* uniquevalues:
85+
* description: whether or not all values are unique
86+
* type: boolean
87+
* values:
88+
* description: array of values sorted by highest count first
89+
* type: array
90+
* maxItems: 2000
91+
* items:
92+
* type: object
93+
* properties:
94+
* value:
95+
* description: encountered value for column (any type)
96+
* type: string
97+
* count:
98+
* description: number of geometries with this value
99+
* type: integer
100+
* 204:
101+
* description: no data
102+
* 422:
103+
* description: invalid table or column
104+
* 500:
105+
* description: unexpected error
106+
*/
107+
app.get('/data/:table/colstats/:column', cacheMiddleWare, async (req, res)=>{
108+
if (!req.query.geom_column) {
109+
req.query.geom_column = 'geom'; // default
110+
}
111+
const sqlString = sql(req.params, req.query);
112+
//console.log(sqlString);
113+
try {
114+
const result = await pool.query(sqlString);
115+
const stats = result.rows
116+
if (stats.length === 0) {
117+
res.status(204).json({});
118+
return;
119+
}
120+
res.json({
121+
table: req.params.table,
122+
column: req.params.column,
123+
numvalues: stats.length < 2000?stats.length:null,
124+
uniquevalues: stats[0].value !== null?stats[0].count === 1:stats.length>1?stats[1].count === 1:false,
125+
values: stats
126+
})
127+
} catch(err) {
128+
console.log(err);
129+
let status = 500;
130+
switch (err.code) {
131+
case '42P01':
132+
// table does not exist
133+
status = 422;
134+
break;
135+
case '42703':
136+
// column does not exist
137+
status = 422;
138+
break;
139+
default:
140+
}
141+
res.status(status).json({error:err.message})
142+
}
143+
})
144+
}

‎mvt.js

+37-30
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,16 @@
11
// based on https://raw.githubusercontent.com/tobinbradley/dirt-simple-postgis-http-api/master/routes/mvt.js
22
const sqlTableName = require('./utils/sqltablename.js');
33

4-
const DirCache = require('./utils/dircache.js')
5-
const cache = new DirCache('./cache');
6-
74
const sm = require('@mapbox/sphericalmercator');
85
const merc = new sm({
96
size: 256
107
})
118

12-
let cacheMiddleWare = async(req, res, next) => {
13-
const cacheDir = `${req.params.datasource}/mvt/${req.params.z}/${req.params.x}/${req.params.y}`;
14-
const key = ((req.query.geom_column?req.query.geom_column:'geom') + (req.query.columns?','+req.query.columns:''))
15-
.replace(/[\W]+/g, '_');
16-
17-
const mvt = await cache.getCachedFile(cacheDir, key);
18-
if (mvt) {
19-
console.log(`cache hit for ${cacheDir}?${key}`);
20-
if (mvt.length === 0) {
21-
res.status(204)
22-
}
23-
res.header('Content-Type', 'application/x-protobuf').send(mvt);
24-
return;
25-
} else {
26-
res.sendResponse = res.send;
27-
res.send = (body) => {
28-
if (res.statusCode >= 200 && res.statusCode < 300) {
29-
cache.setCachedFile(cacheDir, key, body);
30-
}
31-
res.sendResponse(body);
32-
}
33-
next();
34-
}
9+
function queryColumnsNotNull(queryColumns) {
10+
if (queryColumns) {
11+
return ` and (${queryColumns.split(',').map(column=>`${column} is not null`).join(' or ')})`
12+
}
13+
return ''
3514
}
3615

3716
const sql = (params, query) => {
@@ -57,17 +36,45 @@ const sql = (params, query) => {
5736
ST_MakeEnvelope(${bounds.join()}, 3857),
5837
srid
5938
) && ${query.geom_column}
39+
${queryColumnsNotNull(query.columns)}
6040
-- Optional Filter
6141
${query.filter ? `AND ${query.filter}` : ''}
6242
) r
6343
) q
6444
`
6545
} // TODO, use sql place holders $1, $2 etc. instead of inserting user-parameters into query
66-
6746

68-
// TODO add flat-cache
47+
module.exports = function(app, pool, cache) {
6948

70-
module.exports = function(app, pool) {
49+
let cacheMiddleWare = async(req, res, next) => {
50+
if (!cache) {
51+
next();
52+
return;
53+
}
54+
const cacheDir = `${req.params.datasource}/mvt/${req.params.z}/${req.params.x}/${req.params.y}`;
55+
const key = ((req.query.geom_column?req.query.geom_column:'geom') + (req.query.columns?','+req.query.columns:''))
56+
.replace(/[\W]+/g, '_');
57+
58+
const mvt = await cache.getCachedFile(cacheDir, key);
59+
if (mvt) {
60+
console.log(`cache hit for ${cacheDir}?${key}`);
61+
if (mvt.length === 0) {
62+
res.status(204)
63+
}
64+
res.header('Content-Type', 'application/x-protobuf').send(mvt);
65+
return;
66+
} else {
67+
res.sendResponse = res.send;
68+
res.send = (body) => {
69+
if (res.statusCode >= 200 && res.statusCode < 300) {
70+
cache.setCachedFile(cacheDir, key, body);
71+
}
72+
res.sendResponse(body);
73+
}
74+
next();
75+
}
76+
}
77+
7178
/**
7279
* @swagger
7380
*
@@ -121,7 +128,7 @@ module.exports = function(app, pool) {
121128
}
122129
req.params.table = req.params.datasource;
123130
const sqlString = sql(req.params, req.query);
124-
//console.log(sqlString);
131+
// console.log(sqlString);
125132
try {
126133
const result = await pool.query(sqlString);
127134
const mvt = result.rows[0].st_asmvt

‎pgserver.js

+12-5
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,25 @@ const dbconfig = require('./config/dbconfig.json');
1515
const readOnlyPool = new Pool(dbconfig);
1616
readOnlyPool.connect();
1717

18+
const DirCache = require('./utils/dircache.js')
19+
const cache = new DirCache(`./cache/${dbconfig.database?dbconfig.database:process.env.PGDATABASE?process.env.PGDATABASE:''}`);
20+
21+
1822
const swagger = require('./swagger.js')(app);
1923
const login = require('./login.js')(app);
2024
const upload = require('./upload.js')(app);
21-
const mvt = require('./mvt.js')(app, readOnlyPool);
25+
const mvt = require('./mvt.js')(app, readOnlyPool, cache);
2226
const geojson = require('./geojson.js')(app, readOnlyPool);
2327
const geobuf = require('./geobuf.js')(app, readOnlyPool);
24-
const list_layers = require('./list_layers.js')(app, readOnlyPool);
25-
const layer_columns = require('./layer_columns.js')(app, readOnlyPool);
26-
const bbox = require('./bbox.js')(app, readOnlyPool);
28+
const listLayers = require('./list_layers.js')(app, readOnlyPool);
29+
const layerColumns = require('./layer_columns.js')(app, readOnlyPool);
30+
const bbox = require('./bbox.js')(app, readOnlyPool, cache);
2731
const query = require('./query.js')(app, readOnlyPool);
32+
const columnStats = require('./column_stats.js')(app, readOnlyPool, cache);
33+
34+
const server = app.listen(pgserverconfig.port);
35+
server.setTimeout(600000);
2836

29-
app.listen(pgserverconfig.port);
3037
console.log(`pgserver listening on port ${pgserverconfig.port}`);
3138

3239
module.exports = app;

‎public/attrinfo.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,7 @@
245245
const geomColumn = urlParams.get('geom_column');
246246
document.querySelector('#tablename').innerHTML = fullTableName;
247247
document.querySelector('#columnname').innerHTML = `${attrName} (${attrType})`;
248-
document.querySelector('#back').innerHTML = `<a href="tableinfo.html?table=${fullTableName}">Terug naar layer informatie</a>`
248+
document.querySelector('#back').innerHTML = `<a href="tableinfo.html?table=${fullTableName}&geom_column=${geomColumn}">Terug naar layer informatie</a>`
249249
const parts = fullTableName.split('.');
250250
const tableName = (parts.length > 1) ? parts[1] : parts[0];
251251
fetch(`data/query/${fullTableName}?columns=count("${attrName}"),count(distinct+"${attrName}")+as+distinct,min("${attrName}"),max("${attrName}")`).then(response=>{

‎public/tableinfo.html

+17-16
Original file line numberDiff line numberDiff line change
@@ -33,29 +33,30 @@
3333
li.innerHTML = `<a href="./attrinfo.html?table=${fullTableName}&geom_column=${geomcolumn?geomcolumn:''}&column=${item.field_name}&columntype=${item.field_type}&geomtype=${geomType}"><b>${item.field_name}</b></a> (${item.field_type})`
3434
list.appendChild(li);
3535
}
36+
fetch(`api/bbox/${fullTableName}${geomcolumn?`?geom_column=${geomcolumn}`:''}`).then(response=>{
37+
const bbox = document.querySelector('#bbox');
38+
if (response.ok) {
39+
response.json().then(json=> {
40+
addBboxllToLinks(json.bboxll);
41+
bbox.innerHTML = `number of rows: ${json.allrows}<br>
42+
number of geometries: ${json.geomrows}<br>
43+
srid: EPSG:${json.srid}<br>
44+
bbox lon/lat: ${json.bboxll?`sw: ${json.bboxll[0][0]},${json.bboxll[0][1]}, ne: ${json.bboxll[1][0]},${json.bboxll[1][1]}`: 'not defined'}<br>
45+
bbox (EPSG:${json.srid}): ${json.srid?`ll: ${json.bboxsrid[0][0]},${json.bboxsrid[0][1]}, tr: ${json.bboxsrid[1][0]},${json.bboxsrid[1][1]}`: 'not defined'}<br>
46+
`
47+
})
48+
} else {
49+
bbox.innerHTML = `Error getting bbox, response: response: ${response.status} ${response.statusText?response.statusText:''} ${response.url}`
50+
}
51+
})
3652
})
3753
} else {
3854
const li = document.createElement('li');
3955
li.innerHTML = `Error getting list, response: ${response.status} ${response.statusText?response.statusText:''} ${response.url}`
4056
list.appendChild(li);
4157
}
4258
})
43-
fetch(`api/bbox/${fullTableName}${geomcolumn?`?geom_column=${geomcolumn}`:''}`).then(response=>{
44-
const bbox = document.querySelector('#bbox');
45-
if (response.ok) {
46-
response.json().then(json=> {
47-
addBboxllToLinks(json.bboxll);
48-
bbox.innerHTML = `number of rows: ${json.allrows}<br>
49-
number of geometries: ${json.geomrows}<br>
50-
srid: EPSG:${json.srid}<br>
51-
bbox lon/lat: ${json.bboxll?`sw: ${json.bboxll[0][0]},${json.bboxll[0][1]}, ne: ${json.bboxll[1][0]},${json.bboxll[1][1]}`: 'not defined'}<br>
52-
bbox (EPSG:${json.srid}): ${json.srid?`ll: ${json.bboxsrid[0][0]},${json.bboxsrid[0][1]}, tr: ${json.bboxsrid[1][0]},${json.bboxsrid[1][1]}`: 'not defined'}<br>
53-
`
54-
})
55-
} else {
56-
bbox.innerHTML = `Error getting bbox, response: response: ${response.status} ${response.statusText?response.statusText:''} ${response.url}`
57-
}
58-
})
59+
5960
}
6061
</script>
6162
</head>

‎swagger.js

+11-1
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,17 @@ const swaggerDefinition = {
2323

2424
const swaggerJSDocOptions = {
2525
swaggerDefinition,
26-
apis: ['./login.js', './mvt.js', './list_layers.js', './layer_columns.js', './bbox.js', './geojson.js', './geobuf.js', './query.js']
26+
apis: [
27+
'./login.js',
28+
'./mvt.js',
29+
'./list_layers.js',
30+
'./layer_columns.js',
31+
'./bbox.js',
32+
'./geojson.js',
33+
'./geobuf.js',
34+
'./query.js',
35+
'./column_stats.js'
36+
]
2737
}
2838

2939
const swaggerSpec = swaggerJSDoc(swaggerJSDocOptions);

0 commit comments

Comments
 (0)