From 91992eab9dea91fbadf5cfd2a2049495b25beb21 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 11 Jun 2023 14:49:07 +0200 Subject: [PATCH] Continued: - handle typical network exceptions here, too - don't sys.exit() here anymore, exit function properly - don't 'break' here, there was no loop, better return proper error code - fixed a few problems, found by pylint --- fba/commands.py | 35 +++++++++++++++++++++++------------ fba/instances.py | 2 +- fba/networks/friendica.py | 33 ++++++++++++++++++++------------- fba/networks/mastodon.py | 1 - fba/networks/misskey.py | 2 -- 5 files changed, 44 insertions(+), 29 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index e746dee..28d1412 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -17,7 +17,6 @@ import csv import inspect import json -import sys import time import argparse @@ -25,7 +24,6 @@ import atoma import bs4 import markdown import reqto -import requests import validators from fba import blacklist @@ -62,7 +60,7 @@ def check_instance(args: argparse.Namespace) -> int: # DEBUG: print(f"DEBUG: status={status} - EXIT!") return status -def fetch_bkali(args: argparse.Namespace): +def fetch_bkali(args: argparse.Namespace) -> int: # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") domains = list() try: @@ -70,15 +68,27 @@ def fetch_bkali(args: argparse.Namespace): "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}" })) - # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'") - if len(fetched) == 0: + # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") + if "error_message" in fetched: + print(f"WARNING: post_json_api() for 'gql.api.bka.li' returned error message: {fetched['error_message']}") + instances.update_last_error(domain, fetched) + return 100 + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + instances.update_last_error(domain, fetched["json"]["error"]["message"]) + return 101 + + rows = fetched["json"] + + # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'") + if len(rows) == 0: raise Exception("WARNING: Returned no records") - elif "data" not in fetched: - raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'") - elif "nodeinfo" not in fetched["data"]: - raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'") + elif "data" not in rows: + raise Exception(f"WARNING: rows()={len(rows)} does not contain key 'data'") + elif "nodeinfo" not in rows["data"]: + raise Exception(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'") - for entry in fetched["data"]["nodeinfo"]: + for entry in rows["data"]["nodeinfo"]: # DEBUG: print(f"DEBUG: entry['{type(entry)}']='{entry}'") if not "domain" in entry: print(f"WARNING: entry()={len(entry)} does not contain 'domain' - SKIPPED!") @@ -97,8 +107,8 @@ def fetch_bkali(args: argparse.Namespace): domains.append(entry["domain"]) except network.exceptions as exception: - print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'") - sys.exit(255) + print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}' - EXIT!") + return 102 # DEBUG: print(f"DEBUG: domains()={len(domains)}") if len(domains) > 0: @@ -114,6 +124,7 @@ def fetch_bkali(args: argparse.Namespace): instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") + return 0 def fetch_blocks(args: argparse.Namespace): # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") diff --git a/fba/instances.py b/fba/instances.py index 595dd03..8eb1496 100644 --- a/fba/instances.py +++ b/fba/instances.py @@ -283,7 +283,7 @@ def update_last_error(domain: str, error: dict): set_data("last_status_code" , domain, error["status_code"]) set_data("last_error_details", domain, error["json"]["error"]) else: - raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'") + raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'") # Running pending updated # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...") diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index 0fac01a..6fedb47 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -17,6 +17,7 @@ import bs4 from fba import config +from fba import instances from fba import network from fba.helpers import tidyup @@ -28,26 +29,32 @@ def fetch_blocks(domain: str) -> dict: elif domain == "": raise ValueError("Parameter 'domain' is empty") - # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) blocked = list() + blocklist = None - doc = bs4.BeautifulSoup( - network.fetch_response( - domain, - "/friendica", - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ).text, - "html.parser", - ) - # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'") + try: + # DEBUG: print("DEBUG: Fetching friendica blocks from domain:", domain) + doc = bs4.BeautifulSoup( + network.fetch_response( + domain, + "/friendica", + network.web_headers, + (config.get("connection_timeout"), config.get("read_timeout")) + ).text, + "html.parser", + ) + # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'") - blocklist = doc.find(id="about_blocklist") + blocklist = doc.find(id="about_blocklist") + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + instances.update_last_error(domain, exception) + return dict() # Prevents exceptions: if blocklist is None: # DEBUG: print("DEBUG: Instance has no block list:", domain) - return {} + return dict() table = blocklist.find("table") diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 2019467..0dc62a0 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -17,7 +17,6 @@ import inspect import bs4 -import requests import validators from fba import blacklist diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 0ae00d7..25531e1 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -16,8 +16,6 @@ import json -import requests - from fba import blacklist from fba import config from fba import csrf -- 2.39.5