Continued:

- handle typical network exceptions here, too
- don't sys.exit() here anymore, exit function properly
- don't 'break' here, there was no loop, better return proper error code
- fixed a few problems, found by pylint

diff --git a/fba/commands.py b/fba/commands.py
index e746deeb6cc2e7d35e00918c28842ad2395773c0..28d14126a9277cf5db175ef9fe1664e78ade013c 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -17,7 +17,6 @@
 import csv
 import inspect
 import json
-import sys
 import time
 
 import argparse
@@ -25,7 +24,6 @@ import atoma
 import bs4
 import markdown
 import reqto
-import requests
 import validators
 
 from fba import blacklist
@@ -62,7 +60,7 @@ def check_instance(args: argparse.Namespace) -> int:
     # DEBUG: print(f"DEBUG: status={status} - EXIT!")
     return status
 
-def fetch_bkali(args: argparse.Namespace):
+def fetch_bkali(args: argparse.Namespace) -> int:
     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
     domains = list()
     try:
@@ -70,15 +68,27 @@ def fetch_bkali(args: argparse.Namespace):
             "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
         }))
 
-        # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'")
-        if len(fetched) == 0:
+        # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
+        if "error_message" in fetched:
+            print(f"WARNING: post_json_api() for 'gql.api.bka.li' returned error message: {fetched['error_message']}")
+            instances.update_last_error(domain, fetched)
+            return 100
+        elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+            print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
+            instances.update_last_error(domain, fetched["json"]["error"]["message"])
+            return 101
+
+        rows = fetched["json"]
+
+        # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
+        if len(rows) == 0:
             raise Exception("WARNING: Returned no records")
-        elif "data" not in fetched:
-            raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'")
-        elif "nodeinfo" not in fetched["data"]:
-            raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'")
+        elif "data" not in rows:
+            raise Exception(f"WARNING: rows()={len(rows)} does not contain key 'data'")
+        elif "nodeinfo" not in rows["data"]:
+            raise Exception(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'")
 
-        for entry in fetched["data"]["nodeinfo"]:
+        for entry in rows["data"]["nodeinfo"]:
             # DEBUG: print(f"DEBUG: entry['{type(entry)}']='{entry}'")
             if not "domain" in entry:
                 print(f"WARNING: entry()={len(entry)} does not contain 'domain' - SKIPPED!")
@@ -97,8 +107,8 @@ def fetch_bkali(args: argparse.Namespace):
             domains.append(entry["domain"])
 
     except network.exceptions as exception:
-        print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'")
-        sys.exit(255)
+        print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}' - EXIT!")
+        return 102
 
     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
     if len(domains) > 0:
@@ -114,6 +124,7 @@ def fetch_bkali(args: argparse.Namespace):
                 instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
+    return 0
 
 def fetch_blocks(args: argparse.Namespace):
     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")

diff --git a/fba/instances.py b/fba/instances.py
index 595dd038b3231a8deeaa346af7dee65a5b18103d..8eb1496d45c43214ff17047d2b77cec994d8671b 100644 (file)
--- a/fba/instances.py
+++ b/fba/instances.py
@@ -283,7 +283,7 @@ def update_last_error(domain: str, error: dict):
         set_data("last_status_code"  , domain, error["status_code"])
         set_data("last_error_details", domain, error["json"]["error"])
     else:
-       raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'")
+        raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'")
 
     # Running pending updated
     # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")

diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py
index 0fac01a4f7288327226af700b896a6c65bcd0269..6fedb472dfafc02990281fd902f449cf32e28b6b 100644 (file)
--- a/fba/networks/friendica.py
+++ b/fba/networks/friendica.py
@@ -17,6 +17,7 @@
 import bs4
 
 from fba import config
+from fba import instances
 from fba import network
 
 from fba.helpers import tidyup
@@ -28,26 +29,32 @@ def fetch_blocks(domain: str) -> dict:
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
 
-    # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain)
     blocked = list()
+    blocklist = None
 
-    doc = bs4.BeautifulSoup(
-        network.fetch_response(
-            domain,
-            "/friendica",
-            network.web_headers,
-            (config.get("connection_timeout"), config.get("read_timeout"))
-        ).text,
-        "html.parser",
-    )
-    # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'")
+    try:
+        # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain)
+        doc = bs4.BeautifulSoup(
+            network.fetch_response(
+                domain,
+                "/friendica",
+                network.web_headers,
+                (config.get("connection_timeout"), config.get("read_timeout"))
+            ).text,
+            "html.parser",
+        )
+        # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'")
 
-    blocklist = doc.find(id="about_blocklist")
+        blocklist = doc.find(id="about_blocklist")
+    except network.exceptions as exception:
+        print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+        instances.update_last_error(domain, exception)
+        return dict()
 
     # Prevents exceptions:
     if blocklist is None:
         # DEBUG: print("DEBUG: Instance has no block list:", domain)
-        return {}
+        return dict()
 
     table = blocklist.find("table")
 

diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py
index 201946758f73968f36da7c42102fe3ae3b7d5d6b..0dc62a007d387ba890ae7c0a5ce0e2e772738c8b 100644 (file)
--- a/fba/networks/mastodon.py
+++ b/fba/networks/mastodon.py
@@ -17,7 +17,6 @@
 import inspect
 
 import bs4
-import requests
 import validators
 
 from fba import blacklist

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index 0ae00d727bdcaff936e196f67be9ffba579bdd78..25531e191f091737dfd1b28e5196e346f3ae4b53 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -16,8 +16,6 @@
 
 import json
 
-import requests
-
 from fba import blacklist
 from fba import config
 from fba import csrf