diff --git a/package.json b/package.json index e2c0d2fc..7046e50a 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "dompurify": "^2.4.1", "email-templates": "^10.0.1", "express": "^4.18.1", + "express-rate-limit": "^6.7.0", "http-signature": "^1.3.6", "https-proxy-agent": "^5.0.1", "ical.js": "^1.5.0", @@ -61,6 +62,7 @@ "linkifyjs": "4.0.2", "lodash": "^4.17.21", "mariadb": "^3.0.1", + "memory-cache": "^0.2.0", "microformat-node": "^2.0.1", "minify-css-string": "^1.0.0", "mkdirp": "^1.0.4", diff --git a/server/api/controller/geocoding.js b/server/api/controller/geocoding.js new file mode 100644 index 00000000..a86b1ca3 --- /dev/null +++ b/server/api/controller/geocoding.js @@ -0,0 +1,132 @@ +const rateLimit = require('express-rate-limit'); +const log = require('../../log') +const nominatim = require('../../services/geocoding/nominatim') +const photon = require('../../services/geocoding/photon') +const axios = require('axios') +const { version } = require('../../../package.json') +let d = 0 // departure time +let h = 0 // hit geocoding provider time (aka Latency) + +const geocodingController = { + /** + * TODO: replace/merge with a general 'instance rate-limiter' or 'instance api-related rate-limiter' when this will be defined + */ + instanceApiRateLimiter: rateLimit({ + windowMs: 15 * 60 * 1000, // 15 minutes + max: 100, // Limit each IP to 100 requests per `window` (here, per 15 minutes) + standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers + legacyHeaders: false, // Disable the `X-RateLimit-*` headers + }), + + /** + * Limit provider api usage. + * From https://operations.osmfoundation.org/policies/nominatim/ + * [Requirements] No heavy uses (an absolute maximum of 1 request per second). + * [Websites and Apps] + * - Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits. + * - If at all possible, set up a proxy and also enable caching of requests. + */ + providerRateLimit (req, res, next, providerCache) { + let a = Date.now(); // arrival time + let dprev = d + d = dprev + 1000 + h + + // console.log('a: ' + a) + // console.log('dprev: ' + dprev) + // console.log('d: ' + d) + + // if the same request was already cached skip the delay mechanism + if (providerCache.get(req.params.place_details)) { + if (a < d) { + log.warn('More than 1 request per second to geocoding api. This from ' + req.ip + ' . The response data is served from memory-cache.') + } + // reset departure time because there is no need to ask provider + d = dprev + return next() + } + + if (d === 0 || a > d) { + // no-queue or old-queue + // console.log('No queue or Old queue') + // arrival time + 10ms estimated computing time + d = a + 10 + next() + } else { + // fresh queue + // console.log('Fresh queue') + let wait = d - a + // console.log('Waiting '+ wait) + log.warn('More than 1 request per second to geocoding api. This from ' + req.ip + ' . Applying ToS padding before asking to provider. The response data is now cached.') + + setTimeout(() => { + next() + }, wait) + } + + }, + + async nominatimRateLimit(req, res, next) { + geocodingController.providerRateLimit(req, res, next, nominatim.cache) + }, + + async photonRateLimit(req, res, next) { + geocodingController.providerRateLimit(req, res, next, photon.cache) + }, + + async checkInCache (req, res, details, providerCache) { + const ret = await providerCache.get(details) + if (ret) { + return ret + } else { + return + } + }, + + async queryProvider(req, res, details, provider) { + let RTTstart = Date.now() + // console.log('Asking Provider: ' + RTTstart) + + const ret = await axios.get(`${provider.endpoint(req, res)}`, { + params: provider.getParams(req, res), + headers: { 'User-Agent': `gancio ${version}` } + }) + + if (ret) { + let RTTend = Date.now() + // console.log('Asking Provider: ' + RTTend) + // Save the hit time (aka Latency) + // console.log('Saving latency h: ' + h) + h = (RTTend - RTTstart) / 2 + } + + // Cache the response data + provider.cache.put(details, ret.data, 1000 * 60 * 60 * 24); + // console.log(cache.keys()) + // console.log(cache.exportJson()) + return ret.data + }, + + + async _nominatim (req, res) { + const details = req.params.place_details + + const ret = await geocodingController.checkInCache(req, res, details, nominatim.cache) || + await geocodingController.queryProvider(req, res, details, nominatim) + + return res.json(ret) + + }, + + async _photon (req, res) { + const details = req.params.place_details + + const ret = await geocodingController.checkInCache(req, res, details, photon.cache) || + await geocodingController.queryProvider(req, res, details, photon) + + return res.json(ret) + + }, + +} + +module.exports = geocodingController \ No newline at end of file diff --git a/server/api/controller/place.js b/server/api/controller/place.js index 8fd1aa3b..47f71b34 100644 --- a/server/api/controller/place.js +++ b/server/api/controller/place.js @@ -7,9 +7,6 @@ const { version } = require('../../../package.json') const log = require('../../log') const { Op, where, col, fn, cast } = require('sequelize') -const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search' -const PHOTON_URL = 'https://photon.komoot.io/api/' -const axios = require('axios') module.exports = { @@ -75,45 +72,6 @@ module.exports = { // TOFIX: don't know why limit does not work return res.json(places.slice(0, 10)) - }, - - async _nominatim (req, res) { - const details = req.params.place_details - const countrycodes = res.locals.settings.geocoding_countrycodes || [] - const geocoding_provider = res.locals.settings.geocoding_provider || NOMINATIM_URL - // ?limit=3&format=json&namedetails=1&addressdetails=1&q= - - const ret = await axios.get(`${geocoding_provider}`, { - params: { - countrycodes: countrycodes.join(','), - q: details, - limit: 3, - format: 'json', - addressdetails: 1, - namedetails: 1, - }, - headers: { 'User-Agent': `gancio ${version}` } - }) - - return res.json(ret.data) - - }, - - async _photon (req, res) { - const details = req.params.place_details - const geocoding_provider = res.locals.settings.geocoding_provider || PHOTON_URL - - const ret = await axios.get(`${geocoding_provider}`, { - params: { - q: details, - limit: 3, - }, - headers: { 'User-Agent': `gancio ${version}` } - }) - - // console.log(ret) - return res.json(ret.data) - - }, + } } diff --git a/server/api/index.js b/server/api/index.js index 2abd999c..f6c8bf6b 100644 --- a/server/api/index.js +++ b/server/api/index.js @@ -19,6 +19,7 @@ const resourceController = require('./controller/resource') const oauthController = require('./controller/oauth') const announceController = require('./controller/announce') const pluginController = require('./controller/plugins') +const geocodingController = require('./controller/geocoding') const helpers = require('../helpers') const storage = require('./storage') @@ -65,7 +66,6 @@ module.exports = () => { api.get('/ping', (_req, res) => res.sendStatus(200)) api.get('/user', isAuth, (req, res) => res.json(req.user)) - api.post('/user/recover', userController.forgotPassword) api.post('/user/check_recover_code', userController.checkRecoverCode) api.post('/user/recover_password', userController.updatePasswordWithRecoverCode) @@ -173,8 +173,8 @@ module.exports = () => { api.put('/place', isAdmin, placeController.updatePlace) // - GEOCODING - api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, placeController._nominatim) - api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, placeController._photon) + api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geocodingController.instanceApiRateLimiter, geocodingController.nominatimRateLimit, geocodingController._nominatim) + api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geocodingController.instanceApiRateLimiter, geocodingController.photonRateLimit, geocodingController._photon) // - TAGS api.get('/tags', isAdmin, tagController.getAll) diff --git a/server/services/geocoding/nominatim.js b/server/services/geocoding/nominatim.js new file mode 100644 index 00000000..9e8b09fd --- /dev/null +++ b/server/services/geocoding/nominatim.js @@ -0,0 +1,28 @@ +const cache = require('memory-cache') +const providerCache = new cache.Cache() + +const nominatim = { + commonName: 'Nominatim', + DEFAULT_ENDPOINT: 'https://nominatim.openstreetmap.org/search', + endpoint: (req, res) => { + return res.locals.settings.geocoding_provider || nominatim.DEFAULT_ENDPOINT + }, + cache: providerCache, + + getParams (req, res) { + const countrycodes = res.locals.settings.geocoding_countrycodes || [] + const details = req.params.place_details + + return { + countrycodes: countrycodes.join(','), + q: details, + limit: 3, + format: 'json', + addressdetails: 1, + namedetails: 1, + } + }, + +} + +module.exports = nominatim \ No newline at end of file diff --git a/server/services/geocoding/photon.js b/server/services/geocoding/photon.js new file mode 100644 index 00000000..4eced7db --- /dev/null +++ b/server/services/geocoding/photon.js @@ -0,0 +1,23 @@ +const cache = require('memory-cache') +const providerCache = new cache.Cache() + +const photon = { + commonName: 'Photon', + DEFAULT_ENDPOINT: 'https://photon.komoot.io/api/', + endpoint: (req, res) => { + return res.locals.settings.geocoding_provider || photon.DEFAULT_ENDPOINT + }, + cache: providerCache, + + getParams (req, res) { + const details = req.params.place_details + + return { + q: details, + limit: 3, + } + } + +} + +module.exports = photon \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 231181ec..ba411d2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5232,6 +5232,11 @@ expect@^29.4.0: jest-message-util "^29.4.0" jest-util "^29.4.0" +express-rate-limit@^6.7.0: + version "6.7.0" + resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-6.7.0.tgz#6aa8a1bd63dfe79702267b3af1161a93afc1d3c2" + integrity sha512-vhwIdRoqcYB/72TK3tRZI+0ttS8Ytrk24GfmsxDXK9o9IhHNO5bXRiXQSExPQ4GbaE5tvIS7j1SGrxsuWs+sGA== + express@^4.18.1: version "4.18.2" resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59" @@ -8018,6 +8023,11 @@ memoizee@^0.4.15: next-tick "^1.1.0" timers-ext "^0.1.7" +memory-cache@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/memory-cache/-/memory-cache-0.2.0.tgz#7890b01d52c00c8ebc9d533e1f8eb17e3034871a" + integrity sha512-OcjA+jzjOYzKmKS6IQVALHLVz+rNTMPoJvCztFaZxwG14wtAW7VRZjwTQu06vKCYOxh4jVnik7ya0SXTB0W+xA== + memory-fs@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552" @@ -12709,10 +12719,8 @@ watchpack@^1.7.4: resolved "https://registry.yarnpkg.com/watchpack/-/watchpack-1.7.5.tgz#1267e6c55e0b9b5be44c2023aed5437a2c26c453" integrity sha512-9P3MWk6SrKjHsGkLT2KHXdQ/9SNkyoJbabxnKOoJepsvJjJG8uYTR3yTPxPQvNDI3w4Nz1xnE0TLHK4RIVe/MQ== dependencies: - chokidar "^3.4.1" graceful-fs "^4.1.2" neo-async "^2.5.0" - watchpack-chokidar2 "^2.0.1" optionalDependencies: chokidar "^3.4.1" watchpack-chokidar2 "^2.0.1"