Continued:

- some misskey (Chinese) instance returned broken UTF-8, so I had to catch
  UnicodeEncodeError, too

diff --git a/fba/commands.py b/fba/commands.py
index 6b20a7bfcc1dbaa233bfd07c154b54c4ebdf1da1..d8917de89919700bc135c7dc7088de402eaf988a 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -96,7 +96,7 @@ def fetch_bkali(args: argparse.Namespace):
             # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
             domains.append(entry["domain"])
 
-    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
+    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
         print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'")
         sys.exit(255)
 
@@ -109,9 +109,9 @@ def fetch_bkali(args: argparse.Namespace):
             try:
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex:
-                print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{domain}'")
-                instances.update_last_error(domain, ex)
+            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -322,9 +322,9 @@ def fetch_cs(args: argparse.Namespace):
                     try:
                         print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
                         federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
-                    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex:
-                        print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{row['domain']}'")
-                        instances.update_last_error(row["domain"], ex)
+                    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+                        print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'")
+                        instances.update_last_error(row["domain"], exception)
 
         # DEBUG: print("DEBUG: Committing changes ...")
         fba.connection.commit()
@@ -370,9 +370,9 @@ def fetch_fba_rss(args: argparse.Namespace):
             try:
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex:
-                print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{domain}'")
-                instances.update_last_error(domain, ex)
+            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -423,9 +423,9 @@ def fetch_fbabot_atom(args: argparse.Namespace):
             try:
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex:
-                print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{domain}'")
-                instances.update_last_error(domain, ex)
+            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -437,9 +437,9 @@ def fetch_instances(args: argparse.Namespace):
     try:
         print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
         federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
-    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex:
-        print(f"WARNING: Exception '{type(ex)}' during fetching instances from args.domain='{args.domain}'")
-        instances.update_last_error(args.domain, ex)
+    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+        print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'")
+        instances.update_last_error(args.domain, exception)
         return
 
     if args.single:
@@ -462,9 +462,9 @@ def fetch_instances(args: argparse.Namespace):
         try:
             print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
             federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
-        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex:
-            print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{row[0]}'")
-            instances.update_last_error(row[0], ex)
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+            print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'")
+            instances.update_last_error(row[0], exception)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -495,8 +495,8 @@ def fetch_federater(args: argparse.Namespace):
             try:
                 print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
                 federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as ex:
-                print(f"WARNING: Exception '{type(ex)}' during fetching instances from domain='{row['#domain']}'")
-                instances.update_last_error(row["#domain"], ex)
+            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'")
+                instances.update_last_error(row["#domain"], exception)
 
     # DEBUG: print("DEBUG: EXIT!")

diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py
index 6b1b967f7418a97380faa2953d23b69324943b8c..4337bf45629cf4c2e788a9c275f3f4b2584a81a5 100644 (file)
--- a/fba/networks/mastodon.py
+++ b/fba/networks/mastodon.py
@@ -266,7 +266,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
 
         # DEBUG: print("DEBUG: Committing changes ...")
         fba.connection.commit()
-    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
+    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
         print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'")
 
     # DEBUG: print("DEBUG: EXIT!")

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index 68d3b261c4136a142334b74106da1fc4d0216e7f..0327cfc130abd3c374552a1c305706d2a84e4290 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -200,7 +200,7 @@ def fetch_blocks(domain: str) -> dict:
                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
                 break
 
-        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
             instances.update_last_error(domain, exception)
             offset = 0
@@ -253,7 +253,7 @@ def fetch_blocks(domain: str) -> dict:
                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
                 break
 
-        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
+        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
             instances.update_last_error(domain, exception)
             offset = 0