Adding Automatic Retry for GIS Apis

wzdx/tools/cdot_geospatial_api.py

@@ -18,7 +18,11 @@ def __init__(
         setCachedRequest: Callable[[str, str], None] = lambda url, response: None,
         BASE_URL: str = os.getenv(
             "CDOT_GEOSPATIAL_API_BASE_URL",
-            "https://dtdapps.codot.gov/server/rest/services/LRS/Routes_withDEC/MapServer/exts/LrsServerRounded",
+            "https://dtdapps.codot.gov/arcgis/rest/services/LRS/Routes_withDEC/MapServer/exts/LrsServerRounded",
+        ),
+        BACKUP_BASE_URL: str = os.getenv(
+            "CDOT_GEOSPATIAL_API_BACKUP_BASE_URL",
+            "https://dtdapps.coloradodot.info/arcgis/rest/services/LRS/Routes/MapServer/exts/LrsServerRounded",
         ),
     ):
         """Initialize the Geospatial API
@@ -27,10 +31,12 @@ def __init__(
             getCachedRequest ((url: str) => cached_response: str, optional): Optional method to enable custom caching. This method is called with a request url to retrieve the cached result.
             setCachedRequest ((url: str, response: str) => None, optional): Optional method to enable custom caching. This method is called with a request url and response to write the cached result.
             BASE_URL (str, optional): Optional override of GIS server base url, should end with CdotLrsAccessRounded. Defaults first to the env variable CDOT_GEOSPATIAL_API_BASE_URL, then to https://dtdapps.codot.gov/server/rest/services/LRS/Routes_withDEC/MapServer/exts/CdotLrsAccessRounded.
+            BACKUP_BASE_URL (str, optional): Optional override of GIS server backup base url, should end with CdotLrsAccessRounded. Defaults first to the env variable CDOT_GEOSPATIAL_API_BACKUP_BASE_URL, then to https://dtdapps.coloradodot.info/arcgis/rest/services/LRS/Routes/MapServer/exts/CdotLrsAccessRounded.
         """
         self.getCachedRequest = getCachedRequest
         self.setCachedRequest = setCachedRequest
         self.BASE_URL = BASE_URL
+        self.BACKUP_BASE_URL = BACKUP_BASE_URL
         self.ROUTE_BETWEEN_MEASURES_API = "RouteBetweenMeasures"
         self.GET_ROUTE_AND_MEASURE_API = "MeasureAtPoint"
         self.GET_POINT_AT_MEASURE_API = "PointAtMeasure"
@@ -135,7 +141,6 @@ def get_route_and_measure(
         parameters.append("f=pjson")
 
         url = f"{self.BASE_URL}/{self.GET_ROUTE_AND_MEASURE_API}?{'&'.join(parameters)}"
-        logging.debug(url)
 
         # https://dtdapps.coloradodot.info/arcgis/rest/services/LRS/Routes/MapServer/exts/CdotLrsAccessRounded/MeasureAtPoint?x=-105&y=39.5&inSR=4326&tolerance=10000&outSR=&f=html
         resp = self._make_cached_web_request(url)
@@ -372,6 +377,7 @@ def _make_cached_web_request(
         timeout: int = 15,
         retryOnTimeout: bool = False,
         source: str = "cdot_geospatial_api",
+        use_backup_endpoint: bool = True,
     ) -> Any:
         """Make a GET request and cache the response
 
@@ -396,22 +402,47 @@ def _make_cached_web_request(
                 f"Average Response Time: {sum(self.responseTimes)/len(self.responseTimes)}"
             )
             logging.debug("Max Response Time: " + str(max(self.responseTimes)))
-            return json.loads(response)
+
+            try:
+                return json.loads(response)
+            except json.JSONDecodeError as e:
+                logging.error(
+                    f"Failed to decode JSON response for {source} with url: {url}. Timeout: {timeout}. Error: {e}. Response content: {response}"
+                )
+                raise e
         except requests.exceptions.Timeout as e:
             if retryOnTimeout:
                 logging.debug(
-                    f"Geospatial Request Timed Out for {source} with url : {url}. Timeout: {timeout}. Retrying with double timeout"
+                    f"Geospatial Request Timed Out for {source} with url: {url}. Timeout: {timeout}. Retrying with double timeout"
                 )
                 return self._make_cached_web_request(
                     url, timeout=timeout * 2, retryOnTimeout=False
                 )
             else:
-                logging.warning(
-                    f"Geospatial Request Timed Out for {source} with url : {url}. Timeout: {timeout}. Error: {e}"
+                logging.error(
+                    f"Geospatial Request Timed Out for {source} with url: {url}. Timeout: {timeout}. Error: {e}"
                 )
                 return None
         except requests.exceptions.RequestException as e:
-            logging.warning(
-                f"Geospatial Request Failed for {source} with url : {url}. Timeout: {timeout}. Error: {e}"
+            logging.error(
+                f"Geospatial Request Failed for {source} with url: {url}. Timeout: {timeout}. RequestException Error: {e}"
             )
             return None
+        except Exception as e:
+            if use_backup_endpoint and self.BACKUP_BASE_URL:
+                logging.warning(
+                    f"Geospatial Request Failed for {source} with url: {url}. Timeout: {timeout}. Error: {e}. Retrying with backup endpoint."
+                )
+                backup_url = url.replace(self.BASE_URL, self.BACKUP_BASE_URL)
+                return self._make_cached_web_request(
+                    backup_url,
+                    timeout=timeout,
+                    retryOnTimeout=retryOnTimeout,
+                    source=source + "_backup",
+                    use_backup_endpoint=False,
+                )
+            else:
+                logging.error(
+                    f"Geospatial Request Failed for {source} with url: {url}. Timeout: {timeout}. Unknown Error: {e}. No backup endpoint available or already tried."
+                )
+            return None