From f71083f2182908ee7bd6e8c24bdeeb3778cb9b41 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sat, 10 Jun 2023 18:50:26 +0200 Subject: [PATCH] Continued: - caught more specific exception and not broad BaseException (still some are left) - renamed variable blocks -> blocking to avoid confusion with imported fba.blocks --- fba/commands.py | 20 +++++++++------- fba/fba.py | 1 - fba/federation.py | 4 +--- fba/networks/lemmy.py | 2 +- fba/networks/mastodon.py | 51 +++++++++++++++++++++------------------- fba/networks/misskey.py | 10 ++++---- 6 files changed, 46 insertions(+), 42 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 6554e2d..ef13214 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -96,7 +96,7 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...") domains.append(entry["domain"]) - except BaseException as exception: + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'") sys.exit(255) @@ -168,13 +168,15 @@ def fetch_blocks(args: argparse.Namespace): mastodon.fetch_blocks(blocker, origin, nodeinfo_url) elif software == "friendica" or software == "misskey": print(f"INFO: blocker='{blocker}',software='{software}'") + + blocking = list() if software == "friendica": - blocks = friendica.fetch_blocks(blocker) + blocking = friendica.fetch_blocks(blocker) elif software == "misskey": - blocks = misskey.fetch_blocks(blocker) + blocking = misskey.fetch_blocks(blocker) - print(f"INFO: Checking {len(blocks.items())} entries from blocker='{blocker}',software='{software}' ...") - for block_level, blocklist in blocks.items(): + print(f"INFO: Checking {len(blocking.items())} entries from blocker='{blocker}',software='{software}' ...") + for block_level, blocklist in blocking.items(): # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist)) block_level = tidyup.domain(block_level) # DEBUG: print("DEBUG: AFTER-block_level:", block_level) @@ -208,8 +210,8 @@ def fetch_blocks(args: argparse.Namespace): print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue - blocked = searchres[0] - origin = searchres[1] + blocked = searchres[0] + origin = searchres[1] nodeinfo_url = searchres[2] elif blocked.count("?") > 0: # Some obscure them with question marks, not sure if that's dependent on version or not @@ -223,8 +225,8 @@ def fetch_blocks(args: argparse.Namespace): print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue - blocked = searchres[0] - origin = searchres[1] + blocked = searchres[0] + origin = searchres[1] nodeinfo_url = searchres[2] elif not validators.domain(blocked): print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") diff --git a/fba/fba.py b/fba/fba.py index 27c678b..9ea0db0 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -17,7 +17,6 @@ import hashlib import re import json import sqlite3 -import sys import time from urllib.parse import urlparse diff --git a/fba/federation.py b/fba/federation.py index 2a834b6..77fd00a 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -13,8 +13,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import sys - import bs4 import validators @@ -146,7 +144,7 @@ def fetch_peers(domain: str, software: str) -> list: # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) - instances.update_last_error(domain, response) + instances.update_last_error(domain, data) else: # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) peers = data["json"] diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index f2cc4e4..5963227 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -38,7 +38,7 @@ def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: data['{type(data)}']='{data}'") if "error_message" in data: print("WARNING: Could not reach any JSON API:", domain) - instances.update_last_error(domain, response) + instances.update_last_error(domain, data) elif "federated_instances" in data["json"]: # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") peers = peers + federation.add_peers(data["json"]["federated_instances"]) diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 25d11f4..7a1f3eb 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -17,6 +17,7 @@ import inspect import bs4 +import requests import validators from fba import blacklist @@ -131,32 +132,35 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): try: # json endpoint for newer mastodongs blockdict = list() - try: - rows = { - "reject" : [], - "media_removal" : [], - "followers_only": [], - "report_removal": [], - } + rows = { + "reject" : [], + "media_removal" : [], + "followers_only": [], + "report_removal": [], + } - # DEBUG: print("DEBUG: Querying API domain_blocks:", domain) - data = network.get_json_api( - domain, - "/api/v1/instance/domain_blocks", - (config.get("connection_timeout"), config.get("read_timeout")) - ) - - if "error_message" in data: - print(f"WARNING: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'") - instances.update_last_error(domain, data) + # DEBUG: print("DEBUG: Querying API domain_blocks:", domain) + data = network.get_json_api( + domain, + "/api/v1/instance/domain_blocks", + (config.get("connection_timeout"), config.get("read_timeout")) + ) + if "error_message" in data: + print(f"WARNING: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'") + instances.update_last_error(domain, data) + else: + # Getting blocklist blocklist = data["json"] + + if len(blocklist) > 0: print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...") for block in blocklist: + # Map block -> entry entry = { - 'domain': block['domain'], - 'hash' : block['digest'], - 'reason': block['comment'] + "domain": block["domain"], + "hash" : block["digest"], + "reason": block["comment"] } # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment']) @@ -174,9 +178,8 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): rows['report_removal'].append(entry) else: print("WARNING: Unknown severity:", block['severity'], block['domain']) - - except BaseException as exception: - # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: domain='{domain}',exception[{type(exception)}]={str(exception)}") + else: + # DEBUG: print(f"DEBUG: domain='{domain}' has returned zero rows, trying /about/more page ...") rows = fetch_blocks_from_about(domain) print(f"INFO: Checking {len(rows.items())} entries from domain='{domain}',software='mastodon' ...") @@ -258,7 +261,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() - except BaseException as exception: + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'") # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 2a098a7..7a32f8e 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -16,6 +16,8 @@ import json +import requests + from fba import blacklist from fba import config from fba import instances @@ -186,8 +188,8 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") break - except BaseException as exception: - print("WARNING: Caught error, exiting loop:", domain, exception) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: + print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) offset = 0 break @@ -243,8 +245,8 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") break - except BaseException as exception: - print("ERROR: Exception during POST:", domain, exception) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: + print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) offset = 0 break -- 2.39.5