geolocation api rate-limit: add per-provider prefix to cached response, refactor geocoding provider code
This commit is contained in:
sedum
135
server/api/controller/geocoding.js
Normal file
135
server/api/controller/geocoding.js
Normal file
@@ -0,0 +1,135 @@
|
||||
const rateLimit = require('express-rate-limit');
|
||||
const log = require('../../log')
|
||||
const nominatim = require('../../services/geocoding/nominatim')
|
||||
const photon = require('../../services/geocoding/photon')
|
||||
const axios = require('axios')
|
||||
const { version } = require('../../../package.json')
|
||||
const cache = require('memory-cache');
|
||||
let d = 0 // departure time
|
||||
let h = 0 // hit geocoding provider time (aka Latency)
|
||||
|
||||
const geocodingController = {
|
||||
/**
|
||||
* TODO: replace/merge with a general 'instance rate-limiter' or 'instance api-related rate-limiter' when this will be defined
|
||||
*/
|
||||
instanceApiRateLimiter: rateLimit({
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
max: 100, // Limit each IP to 100 requests per `window` (here, per 15 minutes)
|
||||
standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
|
||||
legacyHeaders: false, // Disable the `X-RateLimit-*` headers
|
||||
}),
|
||||
|
||||
/**
|
||||
* Limit provider api usage.
|
||||
* From https://operations.osmfoundation.org/policies/nominatim/
|
||||
* [Requirements] No heavy uses (an absolute maximum of 1 request per second).
|
||||
* [Websites and Apps]
|
||||
* - Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits.
|
||||
* - If at all possible, set up a proxy and also enable caching of requests.
|
||||
*/
|
||||
providerRateLimit (req, res, next, providerCommonName) {
|
||||
let a = Date.now(); // arrival time
|
||||
let dprev = d
|
||||
d = dprev + 1000 + h
|
||||
|
||||
console.log('a: ' + a)
|
||||
console.log('dprev: ' + dprev)
|
||||
console.log('d: ' + d)
|
||||
|
||||
// if the same request was already cached skip the delay mechanism
|
||||
if (cache.get(providerCommonName + '_' + req.params.place_details)) {
|
||||
if (a < d) {
|
||||
log.warn('More than 1 request per second to geocoding api. This from ' + req.ip + ' . The response data is served from memory-cache')
|
||||
}
|
||||
console.log('yet cached')
|
||||
// reset departure time because there is no need to ask provider
|
||||
d = dprev
|
||||
return next()
|
||||
}
|
||||
|
||||
if (d === 0 || a > d) {
|
||||
// no-queue or old-queue
|
||||
console.log('No queue or Old queue')
|
||||
// arrival time + 10ms estimated computing time
|
||||
d = a + 10
|
||||
next()
|
||||
} else {
|
||||
// fresh queue
|
||||
console.log('Fresh queue')
|
||||
let wait = d - a
|
||||
console.log('Waiting '+ wait)
|
||||
log.warn('More than 1 request per second to geocoding api. This from ' + req.ip + ' . Applying ToS padding before asking to provider. The response data is now cached.')
|
||||
|
||||
setTimeout(() => {
|
||||
next()
|
||||
}, wait)
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
async nominatimRateLimit(req, res, next) {
|
||||
geocodingController.providerRateLimit(req, res, next, nominatim.commonName)
|
||||
},
|
||||
|
||||
async photonRateLimit(req, res, next) {
|
||||
geocodingController.providerRateLimit(req, res, next, photon.commonName)
|
||||
},
|
||||
|
||||
async checkInCache (req, res, details, providerCommonName) {
|
||||
if (cache.get(providerCommonName + '_' + details)) {
|
||||
const ret = await cache.get(providerCommonName + '_' + details)
|
||||
return ret
|
||||
} else {
|
||||
console.log('Not in cache')
|
||||
return
|
||||
}
|
||||
},
|
||||
|
||||
async queryProvider(req, res, details, provider) {
|
||||
let RTTstart = Date.now()
|
||||
console.log('Asking Provider: ' + RTTstart)
|
||||
|
||||
const ret = await axios.get(`${provider.endpoint(req, res)}`, {
|
||||
params: provider.getParams(req, res),
|
||||
headers: { 'User-Agent': `gancio ${version}` }
|
||||
})
|
||||
|
||||
if (ret) {
|
||||
let RTTend = Date.now()
|
||||
console.log('Asking Provider: ' + RTTend)
|
||||
// Save the hit time (aka Latency)
|
||||
console.log('Saving latency h: ' + h)
|
||||
h = (RTTend - RTTstart) / 2
|
||||
}
|
||||
|
||||
// Cache the response data
|
||||
cache.put(provider.commonName + '_' + details, ret.data, 1000 * 60 * 60 * 24);
|
||||
console.log(cache.keys())
|
||||
console.log(cache.exportJson())
|
||||
return ret.data
|
||||
},
|
||||
|
||||
|
||||
async _nominatim (req, res) {
|
||||
const details = req.params.place_details
|
||||
|
||||
const ret = await geocodingController.checkInCache(req, res, details, nominatim.commonName) ||
|
||||
await geocodingController.queryProvider(req, res, details, nominatim)
|
||||
|
||||
return res.json(ret)
|
||||
|
||||
},
|
||||
|
||||
async _photon (req, res) {
|
||||
const details = req.params.place_details
|
||||
|
||||
const ret = await geocodingController.checkInCache(req, res, details, photon.commonName) ||
|
||||
await geocodingController.queryProvider(req, res, details, photon)
|
||||
|
||||
return res.json(ret)
|
||||
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
module.exports = geocodingController
|
||||
@@ -1,130 +0,0 @@
|
||||
const rateLimit = require('express-rate-limit');
|
||||
const log = require('../../log')
|
||||
const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search'
|
||||
const PHOTON_URL = 'https://photon.komoot.io/api/'
|
||||
const axios = require('axios')
|
||||
const { version } = require('../../../package.json')
|
||||
const cache = require('memory-cache');
|
||||
let d = 0 // departure time
|
||||
let h = 0 // hit geocoding provider time (aka Latency)
|
||||
|
||||
const geolocationController = {
|
||||
/**
|
||||
* TODO: replace/merge with a general 'instance rate-limiter' or 'instance api-related rate-limiter' when this will be defined
|
||||
*/
|
||||
instanceApiRateLimiter: rateLimit({
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
max: 100, // Limit each IP to 100 requests per `window` (here, per 15 minutes)
|
||||
standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
|
||||
legacyHeaders: false, // Disable the `X-RateLimit-*` headers
|
||||
}),
|
||||
|
||||
/**
|
||||
* Limit api usage
|
||||
* From https://operations.osmfoundation.org/policies/nominatim/
|
||||
* [Requirements] No heavy uses (an absolute maximum of 1 request per second).
|
||||
* [Websites and Apps]
|
||||
* - Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits.
|
||||
* - If at all possible, set up a proxy and also enable caching of requests.
|
||||
*/
|
||||
providerRateLimit (req, res, next) {
|
||||
let a = Date.now(); // arrival time
|
||||
let dprev = d
|
||||
d = dprev + 1000 + h
|
||||
|
||||
console.log('a: ' + a)
|
||||
console.log('dprev: ' + dprev)
|
||||
console.log('d: ' + d)
|
||||
|
||||
// if the same request was already cached skip the delay mechanism
|
||||
if (cache.get(req.params.place_details)) {
|
||||
if (a < d) {
|
||||
log.warn('More than 1 request per second to geolocation api. This from ' + req.ip + ' . The response data is served from memory-cache')
|
||||
}
|
||||
// reset departure time because there is no need to ask provider
|
||||
d = dprev
|
||||
return next()
|
||||
}
|
||||
|
||||
if (d === 0 || a > d) {
|
||||
// no-queue or old-queue
|
||||
console.log('No queue or Old queue')
|
||||
// arrival time + 10ms estimated computing time
|
||||
d = a + 10
|
||||
next()
|
||||
} else {
|
||||
// fresh queue
|
||||
console.log('Fresh queue')
|
||||
let wait = d - a
|
||||
console.log('Waiting '+ wait)
|
||||
log.warn('More than 1 request per second to geolocation api. This from ' + req.ip + ' . Applying ToS padding before asking to provider. The response data is now cached.')
|
||||
|
||||
setTimeout(() => {
|
||||
next()
|
||||
}, wait)
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
async _nominatim (req, res) {
|
||||
const details = req.params.place_details
|
||||
const countrycodes = res.locals.settings.geocoding_countrycodes || []
|
||||
const geocoding_provider = res.locals.settings.geocoding_provider || NOMINATIM_URL
|
||||
// ?limit=3&format=json&namedetails=1&addressdetails=1&q=
|
||||
|
||||
const ret = await axios.get(`${geocoding_provider}`, {
|
||||
params: {
|
||||
countrycodes: countrycodes.join(','),
|
||||
q: details,
|
||||
limit: 3,
|
||||
format: 'json',
|
||||
addressdetails: 1,
|
||||
namedetails: 1,
|
||||
},
|
||||
headers: { 'User-Agent': `gancio ${version}` }
|
||||
})
|
||||
|
||||
return res.json(ret.data)
|
||||
|
||||
},
|
||||
|
||||
async _photon (req, res) {
|
||||
const details = req.params.place_details
|
||||
const geocoding_provider = res.locals.settings.geocoding_provider || PHOTON_URL
|
||||
|
||||
if (cache.get(details)) {
|
||||
console.log('Retrieving data from cache')
|
||||
const ret = {
|
||||
data: await cache.get(details)
|
||||
}
|
||||
return res.json(ret.data)
|
||||
} else {
|
||||
let RTTstart = Date.now()
|
||||
|
||||
console.log('Asking Provider: ' + RTTstart)
|
||||
const ret = await axios.get(`${geocoding_provider}`, {
|
||||
params: {
|
||||
q: details,
|
||||
limit: 3,
|
||||
},
|
||||
headers: { 'User-Agent': `gancio ${version}` }
|
||||
})
|
||||
|
||||
if (ret) {
|
||||
let RTTend = Date.now()
|
||||
console.log('Response arrived: ' + RTTend)
|
||||
// Save the hit time (aka Latency)
|
||||
h = (RTTend - RTTstart) / 2
|
||||
console.log('Saving latency h: ' + h)
|
||||
}
|
||||
|
||||
console.log('Caching results')
|
||||
cache.put(details, ret.data);
|
||||
return res.json(ret.data)
|
||||
}
|
||||
|
||||
},
|
||||
|
||||
}
|
||||
|
||||
module.exports = geolocationController
|
||||
@@ -19,7 +19,7 @@ const resourceController = require('./controller/resource')
|
||||
const oauthController = require('./controller/oauth')
|
||||
const announceController = require('./controller/announce')
|
||||
const pluginController = require('./controller/plugins')
|
||||
const geolocationController = require('./controller/geolocation')
|
||||
const geocodingController = require('./controller/geocoding')
|
||||
const helpers = require('../helpers')
|
||||
const storage = require('./storage')
|
||||
|
||||
@@ -173,8 +173,8 @@ module.exports = () => {
|
||||
api.put('/place', isAdmin, placeController.updatePlace)
|
||||
|
||||
// - GEOCODING
|
||||
api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geolocationController.instanceApiRateLimiter, geolocationController.providerRateLimit, geolocationController._nominatim)
|
||||
api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geolocationController.instanceApiRateLimiter, geolocationController.providerRateLimit, geolocationController._photon)
|
||||
api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geocodingController.instanceApiRateLimiter, geocodingController.nominatimRateLimit, geocodingController._nominatim)
|
||||
api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geocodingController.instanceApiRateLimiter, geocodingController.photonRateLimit, geocodingController._photon)
|
||||
|
||||
// - TAGS
|
||||
api.get('/tags', isAdmin, tagController.getAll)
|
||||
|
||||
23
server/services/geocoding/nominatim.js
Normal file
23
server/services/geocoding/nominatim.js
Normal file
@@ -0,0 +1,23 @@
|
||||
|
||||
module.exports = {
|
||||
commonName: 'Nominatim',
|
||||
DEFAULT_ENDPOINT: 'https://nominatim.openstreetmap.org/search',
|
||||
endpoint: (req, res) => {
|
||||
return res.locals.settings.geocoding_provider || this.DEFAULT_ENDPOINT
|
||||
},
|
||||
|
||||
getParams (req, res) {
|
||||
const countrycodes = res.locals.settings.geocoding_countrycodes || []
|
||||
const details = req.params.place_details
|
||||
|
||||
return {
|
||||
countrycodes: countrycodes.join(','),
|
||||
q: details,
|
||||
limit: 3,
|
||||
format: 'json',
|
||||
addressdetails: 1,
|
||||
namedetails: 1,
|
||||
}
|
||||
},
|
||||
|
||||
}
|
||||
18
server/services/geocoding/photon.js
Normal file
18
server/services/geocoding/photon.js
Normal file
@@ -0,0 +1,18 @@
|
||||
|
||||
module.exports = {
|
||||
commonName: 'Photon',
|
||||
DEFAULT_ENDPOINT: 'https://photon.komoot.io/api/',
|
||||
endpoint: (req, res) => {
|
||||
return res.locals.settings.geocoding_provider || this.DEFAULT_ENDPOINT
|
||||
},
|
||||
|
||||
getParams (req, res) {
|
||||
const details = req.params.place_details
|
||||
|
||||
return {
|
||||
q: details,
|
||||
limit: 3,
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user