geolocation api rate-limit: improve the delay mechanism to be sure to don't hit provider more than 1 time/s, add memory-cache to save response data.

package.json

@@ -62,6 +62,7 @@
     "linkifyjs": "4.0.2",
     "lodash": "^4.17.21",
     "mariadb": "^3.0.1",
+    "memory-cache": "^0.2.0",
     "microformat-node": "^2.0.1",
     "minify-css-string": "^1.0.0",
     "mkdirp": "^1.0.4",

server/api/controller/geolocation.js

@@ -1,9 +1,18 @@
 const rateLimit = require('express-rate-limit');
 const log = require('../../log')
-let d // departure time
+const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search'
+const PHOTON_URL = 'https://photon.komoot.io/api/'
+const axios = require('axios')
+const { version } = require('../../../package.json')
+const cache = require('memory-cache');
+let d = 0 // departure time
+let h = 0 // hit geocoding provider time (aka Latency)
 
 const geolocationController = {
-  rateLimiter: rateLimit({
+  /**
+   * TODO: replace/merge with a general 'instance rate-limiter' or 'instance api-related rate-limiter' when this will be defined
+   */  
+  instanceApiRateLimiter: rateLimit({
     windowMs: 15 * 60 * 1000, // 15 minutes
     max: 100, // Limit each IP to 100 requests per `window` (here, per 15 minutes)
     standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
@@ -14,33 +23,108 @@ const geolocationController = {
    * Limit api usage
    * From https://operations.osmfoundation.org/policies/nominatim/
    * [Requirements] No heavy uses (an absolute maximum of 1 request per second).
-   * [Websites and Apps] Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits.
+   * [Websites and Apps] 
+   * - Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits.
+   * - If at all possible, set up a proxy and also enable caching of requests. 
    */
-  apiLimit (req, res, next) {
+  providerRateLimit (req, res, next) {
-    let dprev = d // departure time of previous
+    let a = Date.now(); // arrival time
-    let a = Date.now() // arrival time
+    let dprev = d
+    d = dprev + 1000 + h
 
-    if (typeof dprev !== 'undefined') {
+    console.log('a: ' + a)
-      d = dprev + 1000
+    console.log('dprev: ' + dprev)
+    console.log('d: ' + d)
 
-      if (a > d) {
+    // if the same request was already cached skip the delay mechanism
-        d = a + 10
+    if (cache.get(req.params.place_details)) {
-        geolocationController.rateLimiter(req, res, next)
+      if (a < d) {
-      } else {
+        log.warn('More than 1 request per second to geolocation api. This from ' + req.ip + ' . The response data is served from memory-cache')
-        let wait = d - a
-        log.warn('More than 1 request per second to geolocation api. This from ' + req.ip)
-
-        setTimeout(() => {
-          geolocationController.rateLimiter(req, res, next)
-        }, wait)
       }
+      // reset departure time because there is no need to ask provider
+      d = dprev
+      return next()
+    }
+
+    if (d === 0 || a > d) {
+      // no-queue or old-queue
+      console.log('No queue or Old queue')
+      // arrival time + 10ms estimated computing time
+      d = a + 10 
+      next()
     } else {
-      d = a + 10 // add 10ms
+      // fresh queue
-      geolocationController.rateLimiter(req, res, next)
+      console.log('Fresh queue')
+      let wait = d - a
+      console.log('Waiting '+ wait)
+      log.warn('More than 1 request per second to geolocation api. This from ' + req.ip + ' . Applying ToS padding before asking to provider. The response data is now cached.')
+      
+      setTimeout(() => {
+        next()
+      }, wait)
     }
     
-  }
+  },
-  
+
+  async _nominatim (req, res) {
+    const details = req.params.place_details
+    const countrycodes = res.locals.settings.geocoding_countrycodes || []
+    const geocoding_provider = res.locals.settings.geocoding_provider || NOMINATIM_URL
+    // ?limit=3&format=json&namedetails=1&addressdetails=1&q=
+
+    const ret = await axios.get(`${geocoding_provider}`, {
+      params: {
+        countrycodes: countrycodes.join(','),
+        q: details,
+        limit: 3,
+        format: 'json',
+        addressdetails: 1,
+        namedetails: 1,
+      },
+      headers: { 'User-Agent': `gancio ${version}` }
+    })
+
+    return res.json(ret.data)
+
+  },
+
+  async _photon (req, res) {
+    const details = req.params.place_details
+    const geocoding_provider = res.locals.settings.geocoding_provider || PHOTON_URL
+
+    if (cache.get(details)) { 
+      console.log('Retrieving data from cache')
+      const ret = {
+        data: await cache.get(details) 
+      }
+      return res.json(ret.data) 
+    } else {
+      let RTTstart = Date.now()
+
+      console.log('Asking Provider: ' + RTTstart)
+      const ret = await axios.get(`${geocoding_provider}`, {
+        params: {
+          q: details,
+          limit: 3,
+        },
+        headers: { 'User-Agent': `gancio ${version}` }
+      })
+
+      if (ret) { 
+        let RTTend = Date.now()
+        console.log('Response arrived: ' + RTTend)
+        // Save the hit time (aka Latency)
+        h = (RTTend - RTTstart) / 2
+        console.log('Saving latency h: ' + h)
+      }
+
+      console.log('Caching results')
+      cache.put(details, ret.data);
+      return res.json(ret.data)
+    }
+ 
+  },
+
 }
 
 module.exports = geolocationController

server/api/controller/place.js

@@ -7,9 +7,6 @@ const { version } = require('../../../package.json')
 
 const log = require('../../log')
 const { Op, where, col, fn, cast } = require('sequelize')
-const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search'
-const PHOTON_URL = 'https://photon.komoot.io/api/'
-const axios = require('axios')
 
 module.exports = {
 
@@ -75,45 +72,6 @@ module.exports = {
 
     // TOFIX: don't know why limit does not work
     return res.json(places.slice(0, 10))
-  },
+  }
-
-  async _nominatim (req, res) {
-    const details = req.params.place_details
-    const countrycodes = res.locals.settings.geocoding_countrycodes || []
-    const geocoding_provider = res.locals.settings.geocoding_provider || NOMINATIM_URL
-    // ?limit=3&format=json&namedetails=1&addressdetails=1&q=
-
-    const ret = await axios.get(`${geocoding_provider}`, {
-      params: {
-        countrycodes: countrycodes.join(','),
-        q: details,
-        limit: 3,
-        format: 'json',
-        addressdetails: 1,
-        namedetails: 1,
-      },
-      headers: { 'User-Agent': `gancio ${version}` }
-    })
-
-    return res.json(ret.data)
-
-  },
-
-  async _photon (req, res) {
-    const details = req.params.place_details
-    const geocoding_provider = res.locals.settings.geocoding_provider || PHOTON_URL
-
-    const ret = await axios.get(`${geocoding_provider}`, {
-      params: {
-        q: details,
-        limit: 3,
-      },
-      headers: { 'User-Agent': `gancio ${version}` }
-    })
-
-    // console.log(ret)
-    return res.json(ret.data)
-
-  },
 
 }

server/api/index.js

@@ -173,8 +173,8 @@ module.exports = () => {
     api.put('/place', isAdmin, placeController.updatePlace)
 
     // - GEOCODING
-    api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geolocationController.apiLimit, placeController._nominatim)
+    api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geolocationController.instanceApiRateLimiter, geolocationController.providerRateLimit, geolocationController._nominatim)
-    api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geolocationController.apiLimit, placeController._photon)
+    api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geolocationController.instanceApiRateLimiter, geolocationController.providerRateLimit, geolocationController._photon)
   
     // - TAGS
     api.get('/tags', isAdmin, tagController.getAll)

yarn.lock

@@ -7950,6 +7950,11 @@ memoizee@^0.4.15:
     next-tick "^1.1.0"
     timers-ext "^0.1.7"
 
+memory-cache@^0.2.0:
+  version "0.2.0"
+  resolved "https://registry.yarnpkg.com/memory-cache/-/memory-cache-0.2.0.tgz#7890b01d52c00c8ebc9d533e1f8eb17e3034871a"
+  integrity sha512-OcjA+jzjOYzKmKS6IQVALHLVz+rNTMPoJvCztFaZxwG14wtAW7VRZjwTQu06vKCYOxh4jVnik7ya0SXTB0W+xA==
+
 memory-fs@^0.4.1:
   version "0.4.1"
   resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552"