Continued:

- moved fba.fetch_misskey_blocks() to misskey.fetch_blocks()
- avoided calling tidyup_reason() on NoneType

diff --git a/fba/commands.py b/fba/commands.py
index 53ad51838e89ec064fcf64a1df0e85402e34217e..799d6db58161e041f2a4c192a96405d58a9c182b 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -159,7 +159,7 @@ def fetch_blocks(args: argparse.Namespace):
                 if software == "friendica":
                     json = fba.fetch_friendica_blocks(blocker)
                 elif software == "misskey":
-                    json = fba.fetch_misskey_blocks(blocker)
+                    json = misskey.fetch_blocks(blocker)
 
                 print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
                 for block_level, blocklist in json.items():
@@ -175,7 +175,7 @@ def fetch_blocks(args: argparse.Namespace):
                         blocked, reason = block.values()
                         # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!")
                         blocked = fba.tidyup_domain(blocked)
-                        reason  = fba.tidyup_reason(reason)
+                        reason  = fba.tidyup_reason(reason) if reason != None else None
                         # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
 
                         if blocked == "":

diff --git a/fba/fba.py b/fba/fba.py
index e52ddc36eeaaad55f4621d5bcc437359c993a89c..1eddf5d5aae7af0919831df1b59a5d9ed5164635 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -793,154 +793,10 @@ def fetch_friendica_blocks(domain: str) -> dict:
         "reject": blocked
     }
 
-def fetch_misskey_blocks(domain: str) -> dict:
-    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
-    if type(domain) != str:
-        raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
-    elif domain == "":
-        raise ValueError(f"Parameter 'domain' is empty")
-
-    # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain)
-    blocklist = {
-        "suspended": [],
-        "blocked"  : []
-    }
-
-    offset = 0
-    step = config.get("misskey_limit")
-    while True:
-        # iterating through all "suspended" (follow-only in its terminology)
-        # instances page-by-page, since that troonware doesn't support
-        # sending them all at once
-        try:
-            # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
-            if offset == 0:
-                # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
-                fetched = post_json_api(domain, "/api/federation/instances", json.dumps({
-                    "sort"     : "+pubAt",
-                    "host"     : None,
-                    "suspended": True,
-                    "limit"    : step
-                }), {
-                    "Origin": domain
-                })
-            else:
-                # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
-                fetched = post_json_api(domain, "/api/federation/instances", json.dumps({
-                    "sort"     : "+pubAt",
-                    "host"     : None,
-                    "suspended": True,
-                    "limit"    : step,
-                    "offset"   : offset - 1
-                }), {
-                    "Origin": domain
-                })
-
-            # DEBUG: print("DEBUG: fetched():", len(fetched))
-            if len(fetched) == 0:
-                # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
-                break
-            elif len(fetched) != config.get("misskey_limit"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
-                offset = offset + (config.get("misskey_limit") - len(fetched))
-            else:
-                # DEBUG: print("DEBUG: Raising offset by step:", step)
-                offset = offset + step
-
-            count = 0
-            for instance in fetched:
-                # Is it there?
-                if instance["isSuspended"] and not has_key(blocklist["suspended"], "domain", instance):
-                    count = count + 1
-                    blocklist["suspended"].append(
-                        {
-                            "domain": tidyup_domain(instance["host"]),
-                            # no reason field, nothing
-                            "reason": None
-                        }
-                    )
-
-            # DEBUG: print(f"DEBUG: count={count}")
-            if count == 0:
-                # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!")
-                break
-
-        except BaseException as e:
-            print("WARNING: Caught error, exiting loop:", domain, e)
-            instances.update_last_error(domain, e)
-            offset = 0
-            break
-
-    while True:
-        # same shit, different asshole ("blocked" aka full suspend)
-        try:
-            if offset == 0:
-                # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
-                fetched = post_json_api(domain,"/api/federation/instances", json.dumps({
-                    "sort"   : "+pubAt",
-                    "host"   : None,
-                    "blocked": True,
-                    "limit"  : step
-                }), {
-                    "Origin": domain
-                })
-            else:
-                # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
-                fetched = post_json_api(domain,"/api/federation/instances", json.dumps({
-                    "sort"   : "+pubAt",
-                    "host"   : None,
-                    "blocked": True,
-                    "limit"  : step,
-                    "offset" : offset - 1
-                }), {
-                    "Origin": domain
-                })
-
-            # DEBUG: print("DEBUG: fetched():", len(fetched))
-            if len(fetched) == 0:
-                # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
-                break
-            elif len(fetched) != config.get("misskey_limit"):
-                # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
-                offset = offset + (config.get("misskey_limit") - len(fetched))
-            else:
-                # DEBUG: print("DEBUG: Raising offset by step:", step)
-                offset = offset + step
-
-            count = 0
-            for instance in fetched:
-                # Is it there?
-                if instance["isBlocked"] and not has_key(blocklist["blocked"], "domain", instance):
-                    count = count + 1
-                    blocklist["blocked"].append({
-                        "domain": tidyup_domain(instance["host"]),
-                        "reason": None
-                    })
-
-            # DEBUG: print(f"DEBUG: count={count}")
-            if count == 0:
-                # DEBUG: print(f"DEBUG: API is no more returning new instances, aborting loop!")
-                break
-
-        except BaseException as e:
-            print("ERROR: Exception during POST:", domain, e)
-            instances.update_last_error(domain, e)
-            offset = 0
-            break
-
-    # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
-    instances.update_last_instance_fetch(domain)
-
-    # DEBUG: print("DEBUG: Returning for domain,blocked(),suspended():", domain, len(blocklist["blocked"]), len(blocklist["suspended"]))
-    return {
-        "reject"        : blocklist["blocked"],
-        "followers_only": blocklist["suspended"]
-    }
-
 def tidyup_reason(reason: str) -> str:
     # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!")
     if type(reason) != str:
-        raise ValueError(f"Parameter reason[]={type(reason)} is not expected")
+        raise ValueError(f"Parameter reason[]={type(reason)} is not 'str'")
 
     # Strip string
     reason = reason.strip()
@@ -954,7 +810,7 @@ def tidyup_reason(reason: str) -> str:
 def tidyup_domain(domain: str) -> str:
     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
     if type(domain) != str:
-        raise ValueError(f"Parameter domain[]={type(domain)} is not expected")
+        raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
 
     # All lower-case and strip spaces out + last dot
     domain = domain.lower().strip().rstrip(".")

diff --git a/fba/federation/misskey.py b/fba/federation/misskey.py
index 8ea0b2b44b18a1bcfbe8eee590f97d794d6a5d23..598e80e0edc8b9ea6f3ff8f8118ddb1d10aca371 100644 (file)
--- a/fba/federation/misskey.py
+++ b/fba/federation/misskey.py
@@ -106,3 +106,147 @@ def fetch_peers(domain: str) -> list:
 
     # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
     return peers
+
+def fetch_blocks(domain: str) -> dict:
+    print(f"DEBUG: domain='{domain}' - CALLED!")
+    if type(domain) != str:
+        raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+    elif domain == "":
+        raise ValueError(f"Parameter 'domain' is empty")
+
+    print("DEBUG: Fetching misskey blocks from domain:", domain)
+    blocklist = {
+        "suspended": [],
+        "blocked"  : []
+    }
+
+    offset = 0
+    step = config.get("misskey_limit")
+    while True:
+        # iterating through all "suspended" (follow-only in its terminology)
+        # instances page-by-page, since that troonware doesn't support
+        # sending them all at once
+        try:
+            print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
+            if offset == 0:
+                print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
+                fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
+                    "sort"     : "+pubAt",
+                    "host"     : None,
+                    "suspended": True,
+                    "limit"    : step
+                }), {
+                    "Origin": domain
+                })
+            else:
+                print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
+                fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
+                    "sort"     : "+pubAt",
+                    "host"     : None,
+                    "suspended": True,
+                    "limit"    : step,
+                    "offset"   : offset - 1
+                }), {
+                    "Origin": domain
+                })
+
+            print("DEBUG: fetched():", len(fetched))
+            if len(fetched) == 0:
+                print("DEBUG: Returned zero bytes, exiting loop:", domain)
+                break
+            elif len(fetched) != config.get("misskey_limit"):
+                print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
+            else:
+                print("DEBUG: Raising offset by step:", step)
+                offset = offset + step
+
+            count = 0
+            for instance in fetched:
+                # Is it there?
+                if instance["isSuspended"] and not fba.has_key(blocklist["suspended"], "domain", instance):
+                    count = count + 1
+                    blocklist["suspended"].append(
+                        {
+                            "domain": fba.tidyup_domain(instance["host"]),
+                            # no reason field, nothing
+                            "reason": None
+                        }
+                    )
+
+            print(f"DEBUG: count={count}")
+            if count == 0:
+                print(f"DEBUG: API is no more returning new instances, aborting loop!")
+                break
+
+        except BaseException as e:
+            print("WARNING: Caught error, exiting loop:", domain, e)
+            instances.update_last_error(domain, e)
+            offset = 0
+            break
+
+    while True:
+        # same shit, different asshole ("blocked" aka full suspend)
+        try:
+            if offset == 0:
+                print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
+                fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
+                    "sort"   : "+pubAt",
+                    "host"   : None,
+                    "blocked": True,
+                    "limit"  : step
+                }), {
+                    "Origin": domain
+                })
+            else:
+                print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
+                fetched = fba.post_json_api(domain, "/api/federation/instances", json.dumps({
+                    "sort"   : "+pubAt",
+                    "host"   : None,
+                    "blocked": True,
+                    "limit"  : step,
+                    "offset" : offset - 1
+                }), {
+                    "Origin": domain
+                })
+
+            print("DEBUG: fetched():", len(fetched))
+            if len(fetched) == 0:
+                print("DEBUG: Returned zero bytes, exiting loop:", domain)
+                break
+            elif len(fetched) != config.get("misskey_limit"):
+                print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+                offset = offset + (config.get("misskey_limit") - len(fetched))
+            else:
+                print("DEBUG: Raising offset by step:", step)
+                offset = offset + step
+
+            count = 0
+            for instance in fetched:
+                # Is it there?
+                if instance["isBlocked"] and not fba.has_key(blocklist["blocked"], "domain", instance):
+                    count = count + 1
+                    blocklist["blocked"].append({
+                        "domain": fba.tidyup_domain(instance["host"]),
+                        "reason": None
+                    })
+
+            print(f"DEBUG: count={count}")
+            if count == 0:
+                print(f"DEBUG: API is no more returning new instances, aborting loop!")
+                break
+
+        except BaseException as e:
+            print("ERROR: Exception during POST:", domain, e)
+            instances.update_last_error(domain, e)
+            offset = 0
+            break
+
+    print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
+    instances.update_last_instance_fetch(domain)
+
+    print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
+    return {
+        "reject"        : blocklist["blocked"],
+        "followers_only": blocklist["suspended"]
+    }

diff --git a/fba/federation/pleroma.py b/fba/federation/pleroma.py
index 7094ae462106e161dedb887463393a1d5758c8ce..a9345725e53f18768fdebeab4a24fe942d323f11 100644 (file)
--- a/fba/federation/pleroma.py
+++ b/fba/federation/pleroma.py
@@ -152,7 +152,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
                 for blocked, reason in info.items():
                     # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!")
                     blocked = fba.tidyup_domain(blocked)
-                    reason = fba.tidyup_reason(reason)
+                    reason  = fba.tidyup_reason(reason) if reason != None else None
                     # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
 
                     if blocked == "":