Continued:

- caught more specific exception and not broad BaseException (still some are
  left)
- renamed variable blocks -> blocking to avoid confusion with imported
  fba.blocks

diff --git a/fba/commands.py b/fba/commands.py
index 6554e2ddc5b79e3e32fd7e17ee750889bf32c099..ef13214c84d1b37d22fda63883892ead220b6062 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -96,7 +96,7 @@ def fetch_bkali(args: argparse.Namespace):
             # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
             domains.append(entry["domain"])
 
-    except BaseException as exception:
+    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
         print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'")
         sys.exit(255)
 
@@ -168,13 +168,15 @@ def fetch_blocks(args: argparse.Namespace):
             mastodon.fetch_blocks(blocker, origin, nodeinfo_url)
         elif software == "friendica" or software == "misskey":
             print(f"INFO: blocker='{blocker}',software='{software}'")
+
+            blocking = list()
             if software == "friendica":
-                blocks = friendica.fetch_blocks(blocker)
+                blocking = friendica.fetch_blocks(blocker)
             elif software == "misskey":
-                blocks = misskey.fetch_blocks(blocker)
+                blocking = misskey.fetch_blocks(blocker)
 
-            print(f"INFO: Checking {len(blocks.items())} entries from blocker='{blocker}',software='{software}' ...")
-            for block_level, blocklist in blocks.items():
+            print(f"INFO: Checking {len(blocking.items())} entries from blocker='{blocker}',software='{software}' ...")
+            for block_level, blocklist in blocking.items():
                 # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist))
                 block_level = tidyup.domain(block_level)
                 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
@@ -208,8 +210,8 @@ def fetch_blocks(args: argparse.Namespace):
                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
                             continue
 
-                        blocked = searchres[0]
-                        origin = searchres[1]
+                        blocked      = searchres[0]
+                        origin       = searchres[1]
                         nodeinfo_url = searchres[2]
                     elif blocked.count("?") > 0:
                         # Some obscure them with question marks, not sure if that's dependent on version or not
@@ -223,8 +225,8 @@ def fetch_blocks(args: argparse.Namespace):
                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
                             continue
 
-                        blocked = searchres[0]
-                        origin = searchres[1]
+                        blocked      = searchres[0]
+                        origin       = searchres[1]
                         nodeinfo_url = searchres[2]
                     elif not validators.domain(blocked):
                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")

diff --git a/fba/fba.py b/fba/fba.py
index 27c678b3869c55ec9c4bf652c2a7cfaaac7dc0ef..9ea0db030060da51c053f646d61089867deb3e08 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -17,7 +17,6 @@ import hashlib
 import re
 import json
 import sqlite3
-import sys
 import time
 
 from urllib.parse import urlparse

diff --git a/fba/federation.py b/fba/federation.py
index 2a834b66f25b05c7dc56efd5f9b68585c8ca1784..77fd00a1df7bb476c311a26f02381ac2f4bc09dc 100644 (file)
--- a/fba/federation.py
+++ b/fba/federation.py
@@ -13,8 +13,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-import sys
-
 import bs4
 import validators
 
@@ -146,7 +144,7 @@ def fetch_peers(domain: str, software: str) -> list:
             # DEBUG: print("DEBUG: Added instance(s) to peers")
         else:
             print("WARNING: JSON response does not contain 'federated_instances':", domain)
-            instances.update_last_error(domain, response)
+            instances.update_last_error(domain, data)
     else:
         # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
         peers = data["json"]

diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py
index f2cc4e42f1595530a0bcace8f24236ac5aa68ba9..5963227fbb80e1c16cbf86822899826e6edf7c1f 100644 (file)
--- a/fba/networks/lemmy.py
+++ b/fba/networks/lemmy.py
@@ -38,7 +38,7 @@ def fetch_peers(domain: str) -> list:
         # DEBUG: print(f"DEBUG: data['{type(data)}']='{data}'")
         if "error_message" in data:
             print("WARNING: Could not reach any JSON API:", domain)
-            instances.update_last_error(domain, response)
+            instances.update_last_error(domain, data)
         elif "federated_instances" in data["json"]:
             # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'")
             peers = peers + federation.add_peers(data["json"]["federated_instances"])

diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py
index 25d11f416595afee42a0238b6a7032e35565389b..7a1f3eb6b07b9c16e3867ffa6d87e2cd95a0a7c8 100644 (file)
--- a/fba/networks/mastodon.py
+++ b/fba/networks/mastodon.py
@@ -17,6 +17,7 @@
 import inspect
 
 import bs4
+import requests
 import validators
 
 from fba import blacklist
@@ -131,32 +132,35 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
     try:
         # json endpoint for newer mastodongs
         blockdict = list()
-        try:
-            rows = {
-                "reject"        : [],
-                "media_removal" : [],
-                "followers_only": [],
-                "report_removal": [],
-            }
+        rows = {
+            "reject"        : [],
+            "media_removal" : [],
+            "followers_only": [],
+            "report_removal": [],
+        }
 
-            # DEBUG: print("DEBUG: Querying API domain_blocks:", domain)
-            data = network.get_json_api(
-                domain,
-                "/api/v1/instance/domain_blocks",
-                (config.get("connection_timeout"), config.get("read_timeout"))
-            )
-
-            if "error_message" in data:
-                print(f"WARNING: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'")
-                instances.update_last_error(domain, data)
+        # DEBUG: print("DEBUG: Querying API domain_blocks:", domain)
+        data = network.get_json_api(
+            domain,
+            "/api/v1/instance/domain_blocks",
+            (config.get("connection_timeout"), config.get("read_timeout"))
+        )
 
+        if "error_message" in data:
+            print(f"WARNING: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'")
+            instances.update_last_error(domain, data)
+        else:
+            # Getting blocklist
             blocklist = data["json"]
+
+        if len(blocklist) > 0:
             print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...")
             for block in blocklist:
+                # Map block -> entry
                 entry = {
-                    'domain': block['domain'],
-                    'hash'  : block['digest'],
-                    'reason': block['comment']
+                    "domain": block["domain"],
+                    "hash"  : block["digest"],
+                    "reason": block["comment"]
                 }
 
                 # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
@@ -174,9 +178,8 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                     rows['report_removal'].append(entry)
                 else:
                     print("WARNING: Unknown severity:", block['severity'], block['domain'])
-
-        except BaseException as exception:
-            # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: domain='{domain}',exception[{type(exception)}]={str(exception)}")
+        else:
+            # DEBUG: print(f"DEBUG: domain='{domain}' has returned zero rows, trying /about/more page ...")
             rows = fetch_blocks_from_about(domain)
 
         print(f"INFO: Checking {len(rows.items())} entries from domain='{domain}',software='mastodon' ...")
@@ -258,7 +261,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
 
         # DEBUG: print("DEBUG: Committing changes ...")
         fba.connection.commit()
-    except BaseException as exception:
+    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
         print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'")
 
     # DEBUG: print("DEBUG: EXIT!")

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index 2a098a71689d8a213f1e519ba6567046085917aa..7a32f8e738cb11ab763add8e461c0cd9eefc8753 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -16,6 +16,8 @@
 
 import json
 
+import requests
+
 from fba import blacklist
 from fba import config
 from fba import instances
@@ -186,8 +188,8 @@ def fetch_blocks(domain: str) -> dict:
                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
                 break
 
-        except BaseException as exception:
-            print("WARNING: Caught error, exiting loop:", domain, exception)
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
+            print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
             instances.update_last_error(domain, exception)
             offset = 0
             break
@@ -243,8 +245,8 @@ def fetch_blocks(domain: str) -> dict:
                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
                 break
 
-        except BaseException as exception:
-            print("ERROR: Exception during POST:", domain, exception)
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
+            print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
             instances.update_last_error(domain, exception)
             offset = 0
             break