Continued:

- renamed 'res' to 'response'
- catch exception, log error and raise it again

diff --git a/api.py b/api.py
index 177eb5f947702a35bdbc5b62f725539170d57904..94fdc3895368e7a6ad2bf77b97fa4cec9dfa15ca 100644 (file)
--- a/api.py
+++ b/api.py
@@ -133,9 +133,9 @@ def mutual(domains: list[str] = Query()):
             "bw": "*." + domains[1],
         },
     )
-    res = fba.cursor.fetchone()
+    response = fba.cursor.fetchone()
 
-    if res is not None:
+    if response is not None:
         # Blocks found
         return JSONResponse(status_code=418, content={})
 
@@ -147,22 +147,22 @@ def index(request: Request, blockers: int = None, blocked: int = None, reference
     scores = None
 
     if blockers != None and blockers > 0:
-        res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blockers={blockers}")
+        response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blockers={blockers}")
     elif blocked != None and blocked > 0:
-        res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blocked={blocked}")
+        response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blocked={blocked}")
     elif reference != None and reference > 0:
-        res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?reference={reference}")
+        response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?reference={reference}")
     elif software != None and software > 0:
-        res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?software={software}")
+        response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?software={software}")
     elif originator != None and originator > 0:
-        res = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?originator={originator}")
+        response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?originator={originator}")
     else:
         raise HTTPException(status_code=400, detail="No filter specified")
 
-    if res == None:
+    if response == None:
         raise HTTPException(status_code=500, detail="Could not determine scores")
-    elif not res.ok:
-        raise HTTPException(status_code=res.status_code, detail=res.text)
+    elif not response.ok:
+        raise HTTPException(status_code=response.status_code, detail=response.text)
 
     return templates.TemplateResponse("scoreboard.html", {
         "base_url"  : config.get("base_url"),
@@ -174,7 +174,7 @@ def index(request: Request, blockers: int = None, blocked: int = None, reference
         "reference" : reference,
         "software"  : software,
         "originator": originator,
-        "scores"    : fba.json_from_response(res)
+        "scores"    : fba.json_from_response(response)
     })
 
 @router.get(config.get("base_url") + "/")

diff --git a/fba/fba.py b/fba/fba.py
index e81e478a80b1167254c7c6ad6ad617088a844844..d8abc9ee5c911078756b811b42607555226163b9 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -371,30 +371,30 @@ def update_last_blocked(domain: str):
 
     # DEBUG: print("DEBUG: EXIT!")
 
-def log_error(domain: str, res: any):
-    # DEBUG: print("DEBUG: domain,res[]:", domain, type(res))
+def log_error(domain: str, response: requests.models.Response):
+    # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
     if type(domain) != str:
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
     try:
-        # DEBUG: print("DEBUG: BEFORE res[]:", type(res))
-        if isinstance(res, BaseException) or isinstance(res, json.decoder.JSONDecodeError):
-            res = str(res)
+        # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
+        if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
+            response = str(response)
 
-        # DEBUG: print("DEBUG: AFTER res[]:", type(res))
-        if type(res) is str:
+        # DEBUG: print("DEBUG: AFTER response[]:", type(response))
+        if type(response) is str:
             cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[
                 domain,
-                res,
+                response,
                 time.time()
             ])
         else:
             cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[
                 domain,
-                res.status_code,
-                res.reason,
+                response.status_code,
+                response.reason,
                 time.time()
             ])
 
@@ -407,32 +407,32 @@ def log_error(domain: str, res: any):
 
     # DEBUG: print("DEBUG: EXIT!")
 
-def update_last_error(domain: str, res: any):
-    # DEBUG: print("DEBUG: domain,res[]:", domain, type(res))
+def update_last_error(domain: str, response: requests.models.Response):
+    # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
     if type(domain) != str:
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
         raise ValueError(f"Parameter 'domain' cannot be empty")
 
-    # DEBUG: print("DEBUG: BEFORE res[]:", type(res))
-    if isinstance(res, BaseException) or isinstance(res, json.decoder.JSONDecodeError):
-        res = f"{type}:str(res)"
+    # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
+    if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
+        response = f"{type}:str(response)"
 
-    # DEBUG: print("DEBUG: AFTER res[]:", type(res))
-    if type(res) is str:
-        # DEBUG: print(f"DEBUG: Setting last_error_details='{res}'");
+    # DEBUG: print("DEBUG: AFTER response[]:", type(response))
+    if type(response) is str:
+        # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'");
         instances.set("last_status_code"  , domain, 999)
-        instances.set("last_error_details", domain, res)
+        instances.set("last_error_details", domain, response)
     else:
-        # DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'");
-        instances.set("last_status_code"  , domain, res.status_code)
-        instances.set("last_error_details", domain, res.reason)
+        # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'");
+        instances.set("last_status_code"  , domain, response.status_code)
+        instances.set("last_error_details", domain, response.reason)
 
     # Running pending updated
     # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
     instances.update_instance_data(domain)
 
-    log_error(domain, res)
+    log_error(domain, response)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -561,15 +561,15 @@ def get_peers(domain: str, software: str) -> list:
     elif software == "lemmy":
         # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...")
         try:
-            res = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+            response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-            data = json_from_response(res)
+            data = json_from_response(response)
 
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
-            if not res.ok or res.status_code >= 400:
+            # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'")
+            if not response.ok or response.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
-                update_last_error(domain, res)
-            elif res.ok and isinstance(data, list):
+                update_last_error(domain, response)
+            elif response.ok and isinstance(data, list):
                 # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
             elif "federated_instances" in data:
@@ -578,7 +578,7 @@ def get_peers(domain: str, software: str) -> list:
                 # DEBUG: print("DEBUG: Added instance(s) to peers")
             else:
                 print("WARNING: JSON response does not contain 'federated_instances':", domain)
-                update_last_error(domain, res)
+                update_last_error(domain, response)
 
         except BaseException as e:
             print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
@@ -599,11 +599,11 @@ def get_peers(domain: str, software: str) -> list:
             # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'")
             while True:
                 try:
-                    res = get_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout")))
+                    response = get_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-                    data = json_from_response(res)
-                    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
-                    if res.ok and isinstance(data, dict):
+                    data = json_from_response(response)
+                    # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'")
+                    if response.ok and isinstance(data, dict):
                         # DEBUG: print("DEBUG: Success, data:", len(data))
                         if "data" in data:
                             # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).")
@@ -636,21 +636,21 @@ def get_peers(domain: str, software: str) -> list:
 
     # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...")
     try:
-        res = get_response(domain, get_peers_url, api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+        response = get_response(domain, get_peers_url, api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-        data = json_from_response(res)
+        data = json_from_response(response)
 
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
-        if not res.ok or res.status_code >= 400:
+        # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+        if not response.ok or response.status_code >= 400:
             # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...")
-            res = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+            response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-            data = json_from_response(res)
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
-            if not res.ok or res.status_code >= 400:
+            data = json_from_response(response)
+            # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+            if not response.ok or response.status_code >= 400:
                 print("WARNING: Could not reach any JSON API:", domain)
-                update_last_error(domain, res)
-            elif res.ok and isinstance(data, list):
+                update_last_error(domain, response)
+            elif response.ok and isinstance(data, list):
                 # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
             elif "federated_instances" in data:
@@ -659,7 +659,7 @@ def get_peers(domain: str, software: str) -> list:
                 # DEBUG: print("DEBUG: Added instance(s) to peers")
             else:
                 print("WARNING: JSON response does not contain 'federated_instances':", domain)
-                update_last_error(domain, res)
+                update_last_error(domain, response)
         else:
             # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
             peers = data
@@ -692,13 +692,13 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict =
     # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers)
     data = {}
     try:
-        res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
+        response = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
 
-        data = json_from_response(res)
-        # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
-        if not res.ok or res.status_code >= 400:
-            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'")
-            update_last_error(domain, res)
+        data = json_from_response(response)
+        # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+        if not response.ok or response.status_code >= 400:
+            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'")
+            update_last_error(domain, response)
 
     except BaseException as e:
         print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'")
@@ -740,21 +740,21 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list:
 
         try:
             # DEBUG: print("DEBUG: Fetching request:", request)
-            res = reqto.get(request, headers=api_headers, timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
+            response = reqto.get(request, headers=api_headers, timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
 
-            data = json_from_response(res)
-            # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
-            if res.ok and isinstance(data, dict):
+            data = json_from_response(response)
+            # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+            if response.ok and isinstance(data, dict):
                 # DEBUG: print("DEBUG: Success:", request)
                 instances.set("detection_mode", domain, "STATIC_CHECK")
                 instances.set("nodeinfo_url"  , domain, request)
                 break
-            elif res.ok and isinstance(data, list):
+            elif response.ok and isinstance(data, list):
                 # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
                 sys.exit(255)
-            elif not res.ok or res.status_code >= 400:
+            elif not response.ok or response.status_code >= 400:
                 print("WARNING: Failed fetching nodeinfo from domain:", domain)
-                update_last_error(domain, res)
+                update_last_error(domain, response)
                 continue
 
         except BaseException as e:
@@ -776,11 +776,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
     data = {}
 
     try:
-        res = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
+        response = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
 
-        data = json_from_response(res)
-        # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data))
-        if res.ok and isinstance(data, dict):
+        data = json_from_response(response)
+        # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data))
+        if response.ok and isinstance(data, dict):
             nodeinfo = data
             # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain)
             if "links" in nodeinfo:
@@ -789,11 +789,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
                     # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"])
                     if link["rel"] in nodeinfo_identifier:
                         # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"])
-                        res = reqto.get(link["href"])
+                        response = reqto.get(link["href"])
 
-                        data = json_from_response(res)
-                        # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code)
-                        if res.ok and isinstance(data, dict):
+                        data = json_from_response(response)
+                        # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code)
+                        if response.ok and isinstance(data, dict):
                             # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data))
                             instances.set("detection_mode", domain, "AUTO_DISCOVERY")
                             instances.set("nodeinfo_url"  , domain, link["href"])
@@ -829,10 +829,10 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
         # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
         response = get_response(domain, path, headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-        # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text))
-        if res.ok and res.status_code < 300 and len(res.text) > 0:
+        # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text))
+        if response.ok and response.status_code < 300 and len(response.text) > 0:
             # DEBUG: print("DEBUG: Search for <meta name='generator'>:", domain)
-            doc = bs4.BeautifulSoup(res.text, "html.parser")
+            doc = bs4.BeautifulSoup(response.text, "html.parser")
 
             # DEBUG: print("DEBUG: doc[]:", type(doc))
             generator = doc.find("meta", {"name": "generator"})
@@ -1481,8 +1481,13 @@ def get_response(domain: str, path: str, headers: dict, timeout: list) -> reques
     elif type(path) != str:
         raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
 
-    # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...")
-    response = reqto.get(f"https://{domain}{path}", headers=headers, timeout=timeout);
+    try:
+        # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...")
+        response = reqto.get(f"https://{domain}{path}", headers=headers, timeout=timeout);
+    except requests.exceptions.ConnectionError as e:
+        # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(e)}]='{str(e)}'")
+        update_last_error(domain, e)
+        raise e
 
     # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
     return response

diff --git a/fetch_blocks.py b/fetch_blocks.py
index 424f61ac0af8c7341559f3fc8c4da3b978153497..bb03061bdde2e2b6deb054f2aca4e188838aad3c 100755 (executable)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -115,7 +115,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                             nodeinfo_url = searchres[1]
                             # DEBUG: print("DEBUG: Looked up domain:", blocked)
                         elif not validators.domain(blocked):
-                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
+                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
                             continue
 
                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
@@ -188,7 +188,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                             origin = searchres[1]
                             nodeinfo_url = searchres[2]
                         elif not validators.domain(blocked):
-                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
+                            print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
                             continue
 
                         # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
@@ -312,7 +312,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                             # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
                             fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
                     elif not validators.domain(blocked):
-                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
                         continue
 
                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
@@ -414,7 +414,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                         origin = searchres[1]
                         nodeinfo_url = searchres[2]
                     elif not validators.domain(blocked):
-                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
                         continue
 
                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
@@ -482,7 +482,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                         origin = searchres[1]
                         nodeinfo_url = searchres[2]
                     elif not validators.domain(blocked):
-                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domai name - skipped!")
+                        print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
                         continue
 
                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)

diff --git a/fetch_fba_rss.py b/fetch_fba_rss.py
index 6553bde56f683ff04d50e766d9eb0e9ddca57288..c7c28e6d0f6102cfd1cfed1917ac0e636bd8d5e9 100755 (executable)
--- a/fetch_fba_rss.py
+++ b/fetch_fba_rss.py
@@ -29,12 +29,12 @@ feed = sys.argv[1]
 domains = list()
 try:
     print(f"INFO: Fetch FBA-specific RSS feed='{feed}' ...")
-    res = reqto.get(feed, headers=fba.headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
+    response = reqto.get(feed, headers=fba.headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
 
-    # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.text()={len(res.text)}")
-    if res.ok and res.status_code < 300 and len(res.text) > 0:
+    # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
+    if response.ok and response.status_code < 300 and len(response.text) > 0:
         # DEBUG: print(f"DEBUG: Parsing RSS feed ...")
-        rss = rss_parser.Parser.parse(res.text)
+        rss = rss_parser.Parser.parse(response.text)
         for item in rss.channel.items:
             # DEBUG: print(f"DEBUG: item.link={item.link}")
             domain = item.link.split("=")[1]