Continued:

- forgot to copy-paste error_message check for "blocked"
- also needed to take fetched['json'] for fetched rows

diff --git a/fba/commands.py b/fba/commands.py
index d8917de89919700bc135c7dc7088de402eaf988a..401f6a4701bc089f3b8f47c02ded8125e10a6d83 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -96,7 +96,7 @@ def fetch_bkali(args: argparse.Namespace):
             # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
             domains.append(entry["domain"])
 
-    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+    except network.exceptions as exception:
         print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'")
         sys.exit(255)
 
@@ -109,7 +109,7 @@ def fetch_bkali(args: argparse.Namespace):
             try:
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+            except network.exceptions as exception:
                 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
                 instances.update_last_error(domain, exception)
 
@@ -206,6 +206,7 @@ def fetch_blocks(args: argparse.Namespace):
 
                         searchres = fba.cursor.fetchone()
 
+                        print(f"DEBUG: searchres[]='{type(searchres)}'")
                         if searchres is None:
                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
                             continue
@@ -221,6 +222,7 @@ def fetch_blocks(args: argparse.Namespace):
 
                         searchres = fba.cursor.fetchone()
 
+                        print(f"DEBUG: searchres[]='{type(searchres)}'")
                         if searchres is None:
                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
                             continue
@@ -322,7 +324,7 @@ def fetch_cs(args: argparse.Namespace):
                     try:
                         print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
                         federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
-                    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+                    except network.exceptions as exception:
                         print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'")
                         instances.update_last_error(row["domain"], exception)
 
@@ -370,7 +372,7 @@ def fetch_fba_rss(args: argparse.Namespace):
             try:
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+            except network.exceptions as exception:
                 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
                 instances.update_last_error(domain, exception)
 
@@ -423,7 +425,7 @@ def fetch_fbabot_atom(args: argparse.Namespace):
             try:
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+            except network.exceptions as exception:
                 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
                 instances.update_last_error(domain, exception)
 
@@ -437,7 +439,7 @@ def fetch_instances(args: argparse.Namespace):
     try:
         print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
         federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
-    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+    except network.exceptions as exception:
         print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'")
         instances.update_last_error(args.domain, exception)
         return
@@ -462,7 +464,7 @@ def fetch_instances(args: argparse.Namespace):
         try:
             print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
             federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
-        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+        except network.exceptions as exception:
             print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'")
             instances.update_last_error(row[0], exception)
 
@@ -495,7 +497,7 @@ def fetch_federater(args: argparse.Namespace):
             try:
                 print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
                 federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
-            except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+            except network.exceptions as exception:
                 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'")
                 instances.update_last_error(row["#domain"], exception)
 

diff --git a/fba/network.py b/fba/network.py
index 7c0bb8a1ab5ddd0741c002b869f920940ffb1b13..759e8e76fdbf8fdf52ca1479c12f4f3a6fbe0770 100644 (file)
--- a/fba/network.py
+++ b/fba/network.py
@@ -33,6 +33,14 @@ api_headers = {
     "Content-Type": "application/json",
 }
 
+# Exceptions to always catch
+exceptions = (
+    requests.exceptions.ConnectionError,
+    requests.exceptions.Timeout,
+    requests.exceptions.TooManyRedirects,
+    UnicodeEncodeError
+)
+
 def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict:
     # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',data='{data}',headers()={len(headers)} - CALLED!")
     if not isinstance(domain, str):

diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py
index 4337bf45629cf4c2e788a9c275f3f4b2584a81a5..424a0236e7c9275975192fc148241e3c59aaf749 100644 (file)
--- a/fba/networks/mastodon.py
+++ b/fba/networks/mastodon.py
@@ -27,6 +27,7 @@ from fba import csrf
 from fba import fba
 from fba import instances
 from fba import network
+
 from fba.helpers import tidyup
 
 language_mapping = {
@@ -135,7 +136,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
 
     try:
         # json endpoint for newer mastodongs
-        blockdict = list()
+        found_blocks = list()
+        blocklist = list()
+
         rows = {
             "reject"        : [],
             "media_removal" : [],
@@ -219,13 +222,14 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                     )
                     searchres = fba.cursor.fetchone()
 
+                    print(f"DEBUG: searchres[]='{type(searchres)}'")
                     if searchres is None:
                         print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!")
                         continue
 
                     # DEBUG: print("DEBUG: Updating domain: ", searchres[0])
-                    blocked = searchres[0]
-                    origin = searchres[1]
+                    blocked      = searchres[0]
+                    origin       = searchres[1]
                     nodeinfo_url = searchres[2]
 
                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
@@ -255,7 +259,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                     blocks.add_instance(domain, blocking, reason, block_level)
 
                     if block_level == "reject":
-                        blockdict.append({
+                        found_blocks.append({
                             "blocked": blocked,
                             "reason" : reason
                         })
@@ -266,7 +270,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
 
         # DEBUG: print("DEBUG: Committing changes ...")
         fba.connection.commit()
-    except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+    except network.exceptions as exception:
         print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'")
 
     # DEBUG: print("DEBUG: EXIT!")

diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py
index 0327cfc130abd3c374552a1c305706d2a84e4290..f3a1eb30dcd7a53cd6a42f2c4255b20325dfc5bb 100644 (file)
--- a/fba/networks/misskey.py
+++ b/fba/networks/misskey.py
@@ -134,10 +134,9 @@ def fetch_blocks(domain: str) -> dict:
     step    = config.get("misskey_limit")
     headers = csrf.determine(domain, {"Origin": domain})
 
+    # iterating through all "suspended" (follow-only in its terminology)
+    # instances page-by-page since it doesn't support sending them all at once
     while True:
-        # iterating through all "suspended" (follow-only in its terminology)
-        # instances page-by-page, since that troonware doesn't support
-        # sending them all at once
         try:
             # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
             if offset == 0:
@@ -158,7 +157,6 @@ def fetch_blocks(domain: str) -> dict:
                     "offset"   : offset - 1
                 }), headers)
 
-            # Check records
             # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
             if "error_message" in fetched:
                 print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
@@ -171,7 +169,7 @@ def fetch_blocks(domain: str) -> dict:
 
             rows = fetched["json"]
 
-            # DEBUG: print("DEBUG: rows():", len(rows))
+            # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
             if len(rows) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
@@ -185,7 +183,8 @@ def fetch_blocks(domain: str) -> dict:
             count = 0
             for instance in rows:
                 # Is it there?
-                if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance):
+                # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
+                if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
                     count = count + 1
                     blocklist["suspended"].append(
                         {
@@ -200,48 +199,61 @@ def fetch_blocks(domain: str) -> dict:
                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
                 break
 
-        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+        except network.exceptions as exception:
             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
             instances.update_last_error(domain, exception)
             offset = 0
             break
 
     while True:
-        # same shit, different asshole ("blocked" aka full suspend)
+        # Fetch blocked (full suspended) instances
         try:
             if offset == 0:
                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
-                fetched = network.post_json_api(domain, "/api/federation/instances", headers, json.dumps({
+                fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
                     "sort"   : "+pubAt",
                     "host"   : None,
                     "blocked": True,
                     "limit"  : step
-                }))
+                }), headers)
             else:
                 # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
-                fetched = network.post_json_api(domain, "/api/federation/instances", headers, json.dumps({
+                fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
                     "sort"   : "+pubAt",
                     "host"   : None,
                     "blocked": True,
                     "limit"  : step,
                     "offset" : offset - 1
-                }))
+                }), headers)
+
+            # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}")
+            if "error_message" in fetched:
+                print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
+                instances.update_last_error(domain, fetched)
+                break
+            elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+                print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
+                instances.update_last_error(domain, fetched["json"]["error"]["message"])
+                break
+
+            rows = fetched["json"]
 
-            # DEBUG: print("DEBUG: fetched():", len(fetched))
-            if len(fetched) == 0:
+            # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
+            if len(rows) == 0:
                 # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
                 break
-            elif len(fetched) != config.get("misskey_limit"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
-                offset = offset + (config.get("misskey_limit") - len(fetched))
+            elif len(rows) != config.get("misskey_limit"):
+                # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(rows))
             else:
                 # DEBUG: print("DEBUG: Raising offset by step:", step)
                 offset = offset + step
 
             count = 0
-            for instance in fetched:
+            for instance in rows:
                 # Is it there?
-                if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance):
+                # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
+                if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
                     count = count + 1
                     blocklist["blocked"].append({
                         "domain": tidyup.domain(instance["host"]),
@@ -253,7 +265,7 @@ def fetch_blocks(domain: str) -> dict:
                 # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
                 break
 
-        except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception:
+        except network.exceptions as exception:
             print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
             instances.update_last_error(domain, exception)
             offset = 0

diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py
index b997dc6fb39e6bb304f1f7caa0119a291a48afb7..86f69cdee9cd47809de2801908d7f9cfd55856d1 100644 (file)
--- a/fba/networks/pleroma.py
+++ b/fba/networks/pleroma.py
@@ -94,8 +94,8 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                             "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
                         )
                         searchres = fba.cursor.fetchone()
-                        # DEBUG: print("DEBUG: searchres[]:", type(searchres))
 
+                        print(f"DEBUG: searchres[]='{type(searchres)}'")
                         if searchres is None:
                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
                             continue
@@ -173,6 +173,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                         )
                         searchres = fba.cursor.fetchone()
 
+                        print(f"DEBUG: searchres[]='{type(searchres)}'")
                         if searchres is None:
                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
                             continue