From cc4bb69667ea9ab905532fd5856ac10e7312187e Mon Sep 17 00:00:00 2001 From: sedum Date: Thu, 19 Jan 2023 12:06:23 +0100 Subject: [PATCH] geolocation api rate-limit: add per-provider prefix to cached response, refactor geocoding provider code --- server/api/controller/geocoding.js | 135 +++++++++++++++++++++++++ server/api/controller/geolocation.js | 130 ------------------------ server/api/index.js | 6 +- server/services/geocoding/nominatim.js | 23 +++++ server/services/geocoding/photon.js | 18 ++++ 5 files changed, 179 insertions(+), 133 deletions(-) create mode 100644 server/api/controller/geocoding.js delete mode 100644 server/api/controller/geolocation.js create mode 100644 server/services/geocoding/nominatim.js create mode 100644 server/services/geocoding/photon.js diff --git a/server/api/controller/geocoding.js b/server/api/controller/geocoding.js new file mode 100644 index 00000000..5e382640 --- /dev/null +++ b/server/api/controller/geocoding.js @@ -0,0 +1,135 @@ +const rateLimit = require('express-rate-limit'); +const log = require('../../log') +const nominatim = require('../../services/geocoding/nominatim') +const photon = require('../../services/geocoding/photon') +const axios = require('axios') +const { version } = require('../../../package.json') +const cache = require('memory-cache'); +let d = 0 // departure time +let h = 0 // hit geocoding provider time (aka Latency) + +const geocodingController = { + /** + * TODO: replace/merge with a general 'instance rate-limiter' or 'instance api-related rate-limiter' when this will be defined + */ + instanceApiRateLimiter: rateLimit({ + windowMs: 15 * 60 * 1000, // 15 minutes + max: 100, // Limit each IP to 100 requests per `window` (here, per 15 minutes) + standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers + legacyHeaders: false, // Disable the `X-RateLimit-*` headers + }), + + /** + * Limit provider api usage. + * From https://operations.osmfoundation.org/policies/nominatim/ + * [Requirements] No heavy uses (an absolute maximum of 1 request per second). + * [Websites and Apps] + * - Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits. + * - If at all possible, set up a proxy and also enable caching of requests. + */ + providerRateLimit (req, res, next, providerCommonName) { + let a = Date.now(); // arrival time + let dprev = d + d = dprev + 1000 + h + + console.log('a: ' + a) + console.log('dprev: ' + dprev) + console.log('d: ' + d) + + // if the same request was already cached skip the delay mechanism + if (cache.get(providerCommonName + '_' + req.params.place_details)) { + if (a < d) { + log.warn('More than 1 request per second to geocoding api. This from ' + req.ip + ' . The response data is served from memory-cache') + } + console.log('yet cached') + // reset departure time because there is no need to ask provider + d = dprev + return next() + } + + if (d === 0 || a > d) { + // no-queue or old-queue + console.log('No queue or Old queue') + // arrival time + 10ms estimated computing time + d = a + 10 + next() + } else { + // fresh queue + console.log('Fresh queue') + let wait = d - a + console.log('Waiting '+ wait) + log.warn('More than 1 request per second to geocoding api. This from ' + req.ip + ' . Applying ToS padding before asking to provider. The response data is now cached.') + + setTimeout(() => { + next() + }, wait) + } + + }, + + async nominatimRateLimit(req, res, next) { + geocodingController.providerRateLimit(req, res, next, nominatim.commonName) + }, + + async photonRateLimit(req, res, next) { + geocodingController.providerRateLimit(req, res, next, photon.commonName) + }, + + async checkInCache (req, res, details, providerCommonName) { + if (cache.get(providerCommonName + '_' + details)) { + const ret = await cache.get(providerCommonName + '_' + details) + return ret + } else { + console.log('Not in cache') + return + } + }, + + async queryProvider(req, res, details, provider) { + let RTTstart = Date.now() + console.log('Asking Provider: ' + RTTstart) + + const ret = await axios.get(`${provider.endpoint(req, res)}`, { + params: provider.getParams(req, res), + headers: { 'User-Agent': `gancio ${version}` } + }) + + if (ret) { + let RTTend = Date.now() + console.log('Asking Provider: ' + RTTend) + // Save the hit time (aka Latency) + console.log('Saving latency h: ' + h) + h = (RTTend - RTTstart) / 2 + } + + // Cache the response data + cache.put(provider.commonName + '_' + details, ret.data, 1000 * 60 * 60 * 24); + console.log(cache.keys()) + console.log(cache.exportJson()) + return ret.data + }, + + + async _nominatim (req, res) { + const details = req.params.place_details + + const ret = await geocodingController.checkInCache(req, res, details, nominatim.commonName) || + await geocodingController.queryProvider(req, res, details, nominatim) + + return res.json(ret) + + }, + + async _photon (req, res) { + const details = req.params.place_details + + const ret = await geocodingController.checkInCache(req, res, details, photon.commonName) || + await geocodingController.queryProvider(req, res, details, photon) + + return res.json(ret) + + }, + +} + +module.exports = geocodingController \ No newline at end of file diff --git a/server/api/controller/geolocation.js b/server/api/controller/geolocation.js deleted file mode 100644 index 0233e919..00000000 --- a/server/api/controller/geolocation.js +++ /dev/null @@ -1,130 +0,0 @@ -const rateLimit = require('express-rate-limit'); -const log = require('../../log') -const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search' -const PHOTON_URL = 'https://photon.komoot.io/api/' -const axios = require('axios') -const { version } = require('../../../package.json') -const cache = require('memory-cache'); -let d = 0 // departure time -let h = 0 // hit geocoding provider time (aka Latency) - -const geolocationController = { - /** - * TODO: replace/merge with a general 'instance rate-limiter' or 'instance api-related rate-limiter' when this will be defined - */ - instanceApiRateLimiter: rateLimit({ - windowMs: 15 * 60 * 1000, // 15 minutes - max: 100, // Limit each IP to 100 requests per `window` (here, per 15 minutes) - standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers - legacyHeaders: false, // Disable the `X-RateLimit-*` headers - }), - - /** - * Limit api usage - * From https://operations.osmfoundation.org/policies/nominatim/ - * [Requirements] No heavy uses (an absolute maximum of 1 request per second). - * [Websites and Apps] - * - Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits. - * - If at all possible, set up a proxy and also enable caching of requests. - */ - providerRateLimit (req, res, next) { - let a = Date.now(); // arrival time - let dprev = d - d = dprev + 1000 + h - - console.log('a: ' + a) - console.log('dprev: ' + dprev) - console.log('d: ' + d) - - // if the same request was already cached skip the delay mechanism - if (cache.get(req.params.place_details)) { - if (a < d) { - log.warn('More than 1 request per second to geolocation api. This from ' + req.ip + ' . The response data is served from memory-cache') - } - // reset departure time because there is no need to ask provider - d = dprev - return next() - } - - if (d === 0 || a > d) { - // no-queue or old-queue - console.log('No queue or Old queue') - // arrival time + 10ms estimated computing time - d = a + 10 - next() - } else { - // fresh queue - console.log('Fresh queue') - let wait = d - a - console.log('Waiting '+ wait) - log.warn('More than 1 request per second to geolocation api. This from ' + req.ip + ' . Applying ToS padding before asking to provider. The response data is now cached.') - - setTimeout(() => { - next() - }, wait) - } - - }, - - async _nominatim (req, res) { - const details = req.params.place_details - const countrycodes = res.locals.settings.geocoding_countrycodes || [] - const geocoding_provider = res.locals.settings.geocoding_provider || NOMINATIM_URL - // ?limit=3&format=json&namedetails=1&addressdetails=1&q= - - const ret = await axios.get(`${geocoding_provider}`, { - params: { - countrycodes: countrycodes.join(','), - q: details, - limit: 3, - format: 'json', - addressdetails: 1, - namedetails: 1, - }, - headers: { 'User-Agent': `gancio ${version}` } - }) - - return res.json(ret.data) - - }, - - async _photon (req, res) { - const details = req.params.place_details - const geocoding_provider = res.locals.settings.geocoding_provider || PHOTON_URL - - if (cache.get(details)) { - console.log('Retrieving data from cache') - const ret = { - data: await cache.get(details) - } - return res.json(ret.data) - } else { - let RTTstart = Date.now() - - console.log('Asking Provider: ' + RTTstart) - const ret = await axios.get(`${geocoding_provider}`, { - params: { - q: details, - limit: 3, - }, - headers: { 'User-Agent': `gancio ${version}` } - }) - - if (ret) { - let RTTend = Date.now() - console.log('Response arrived: ' + RTTend) - // Save the hit time (aka Latency) - h = (RTTend - RTTstart) / 2 - console.log('Saving latency h: ' + h) - } - - console.log('Caching results') - cache.put(details, ret.data); - return res.json(ret.data) - } - - }, - -} - -module.exports = geolocationController \ No newline at end of file diff --git a/server/api/index.js b/server/api/index.js index a2cecb4f..1dfae154 100644 --- a/server/api/index.js +++ b/server/api/index.js @@ -19,7 +19,7 @@ const resourceController = require('./controller/resource') const oauthController = require('./controller/oauth') const announceController = require('./controller/announce') const pluginController = require('./controller/plugins') -const geolocationController = require('./controller/geolocation') +const geocodingController = require('./controller/geocoding') const helpers = require('../helpers') const storage = require('./storage') @@ -173,8 +173,8 @@ module.exports = () => { api.put('/place', isAdmin, placeController.updatePlace) // - GEOCODING - api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geolocationController.instanceApiRateLimiter, geolocationController.providerRateLimit, geolocationController._nominatim) - api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geolocationController.instanceApiRateLimiter, geolocationController.providerRateLimit, geolocationController._photon) + api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geocodingController.instanceApiRateLimiter, geocodingController.nominatimRateLimit, geocodingController._nominatim) + api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geocodingController.instanceApiRateLimiter, geocodingController.photonRateLimit, geocodingController._photon) // - TAGS api.get('/tags', isAdmin, tagController.getAll) diff --git a/server/services/geocoding/nominatim.js b/server/services/geocoding/nominatim.js new file mode 100644 index 00000000..8b55153a --- /dev/null +++ b/server/services/geocoding/nominatim.js @@ -0,0 +1,23 @@ + +module.exports = { + commonName: 'Nominatim', + DEFAULT_ENDPOINT: 'https://nominatim.openstreetmap.org/search', + endpoint: (req, res) => { + return res.locals.settings.geocoding_provider || this.DEFAULT_ENDPOINT + }, + + getParams (req, res) { + const countrycodes = res.locals.settings.geocoding_countrycodes || [] + const details = req.params.place_details + + return { + countrycodes: countrycodes.join(','), + q: details, + limit: 3, + format: 'json', + addressdetails: 1, + namedetails: 1, + } + }, + +} \ No newline at end of file diff --git a/server/services/geocoding/photon.js b/server/services/geocoding/photon.js new file mode 100644 index 00000000..29a4dfda --- /dev/null +++ b/server/services/geocoding/photon.js @@ -0,0 +1,18 @@ + +module.exports = { + commonName: 'Photon', + DEFAULT_ENDPOINT: 'https://photon.komoot.io/api/', + endpoint: (req, res) => { + return res.locals.settings.geocoding_provider || this.DEFAULT_ENDPOINT + }, + + getParams (req, res) { + const details = req.params.place_details + + return { + q: details, + limit: 3, + } + } + +} \ No newline at end of file