Continued:

- need to handle requests.exceptions.InvalidSchema as well
- if variable link is not dict or does not contain 'rel' attribute for what
  reason, then skip this

diff --git a/fba/federation.py b/fba/federation.py
index e79c69e0e24f3ce6bafb7e856f4e8cbf1ea77fda..eebbfd72d5a0ade9e98c947be34c59d8e7d74861 100644 (file)
--- a/fba/federation.py
+++ b/fba/federation.py
@@ -311,8 +311,10 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict:
         if "links" in nodeinfo:
             # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"]))
             for link in nodeinfo["links"]:
-                # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"])
-                if link["rel"] in nodeinfo_identifier:
+                # DEBUG: print(f"DEBUG: link[{type(link)}]='{link}'")
+                if not isinstance(link, dict) or not "rel" in link:
+                    print(f"WARNING: link[]='{type(link)}' is not 'dict' or no element 'rel' found")
+                elif link["rel"] in nodeinfo_identifier:
                     # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"])
                     data = network.fetch_api_url(
                         link["href"],
@@ -367,7 +369,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
             software = tidyup.domain(generator.get("content"))
             print(f"INFO: domain='{domain}' is generated by '{software}'")
             instances.set_detection_mode(domain, "GENERATOR")
-        elif isinstance(site_name, bs4.element.Tag):
+        elif isinstance(site_name, bs4.element.Tag) and isinstance(site_name.get("content"), str):
             # DEBUG: print("DEBUG: Found property=og:site_name:", domain)
             sofware = tidyup.domain(site_name.get("content"))
             print(f"INFO: domain='{domain}' has og:site_name='{software}'")

diff --git a/fba/network.py b/fba/network.py
index 5a404a40a9deb60c5e62dfb18075f34badbaab6c..38e37f1f3130b49f668282cce2adb65fd2cf9703 100644 (file)
--- a/fba/network.py
+++ b/fba/network.py
@@ -38,6 +38,7 @@ api_headers = {
 exceptions = (
     requests.exceptions.ChunkedEncodingError,
     requests.exceptions.ConnectionError,
+    requests.exceptions.InvalidSchema,
     requests.exceptions.Timeout,
     requests.exceptions.TooManyRedirects,
     UnicodeEncodeError