From 55951df9798a5735cb167f24256cdb60d841a79c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 11 Jun 2023 00:35:17 +0200 Subject: [PATCH] Continued: - some misskey (Chinese) instance returned broken UTF-8, so I had to catch UnicodeEncodeError, too --- fba/commands.py | 44 ++++++++++++++++++++-------------------- fba/networks/mastodon.py | 2 +- fba/networks/misskey.py | 4 ++-- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 6b20a7b..d8917de 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -96,7 +96,7 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...") domains.append(entry["domain"]) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'") sys.exit(255) @@ -109,9 +109,9 @@ def fetch_bkali(args: argparse.Namespace): try: print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex: - print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{domain}'") - instances.update_last_error(domain, ex) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -322,9 +322,9 @@ def fetch_cs(args: argparse.Namespace): try: print(f"INFO: Fetching instances from domain='{row['domain']}' ...") federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex: - print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{row['domain']}'") - instances.update_last_error(row["domain"], ex) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'") + instances.update_last_error(row["domain"], exception) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() @@ -370,9 +370,9 @@ def fetch_fba_rss(args: argparse.Namespace): try: print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex: - print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{domain}'") - instances.update_last_error(domain, ex) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -423,9 +423,9 @@ def fetch_fbabot_atom(args: argparse.Namespace): try: print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex: - print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{domain}'") - instances.update_last_error(domain, ex) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -437,9 +437,9 @@ def fetch_instances(args: argparse.Namespace): try: print(f"INFO: Fetching instances from args.domain='{args.domain}' ...") federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex: - print(f"WARNING: Exception '{type(ex)}' during fetching instances from args.domain='{args.domain}'") - instances.update_last_error(args.domain, ex) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'") + instances.update_last_error(args.domain, exception) return if args.single: @@ -462,9 +462,9 @@ def fetch_instances(args: argparse.Namespace): try: print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex: - print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{row[0]}'") - instances.update_last_error(row[0], ex) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'") + instances.update_last_error(row[0], exception) # DEBUG: print("DEBUG: EXIT!") @@ -495,8 +495,8 @@ def fetch_federater(args: argparse.Namespace): try: print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex: - print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{row['#domain']}'") - instances.update_last_error(row["#domain"], ex) + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'") + instances.update_last_error(row["#domain"], exception) # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 6b1b967..4337bf4 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -266,7 +266,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'") # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 68d3b26..0327cfc 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -200,7 +200,7 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") break - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) offset = 0 @@ -253,7 +253,7 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") break - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: + except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) offset = 0 -- 2.39.5