WIP:

- more refacturing away from "generic except blocks"

diff --git a/api.py b/api.py
index 1275d8ff6adf5884d1c8ad646f5a13e4494e71cf..a9c0e4dd428598f932507b0ab006bf6eecb50a4a 100644 (file)
--- a/api.py
+++ b/api.py
@@ -31,6 +31,7 @@ import validators
 
 from fba import config
 from fba import fba
+from fba import network
 
 router = fastapi.FastAPI(docs_url=config.get("base_url") + "/docs", redoc_url=config.get("base_url") + "/redoc")
 templates = Jinja2Templates(directory="templates")
@@ -157,6 +158,8 @@ def api_mutual(domains: list[str] = Query()):
 
 @router.get(config.get("base_url") + "/scoreboard")
 def scoreboard(request: Request, blockers: int = None, blocked: int = None, reference: int = None, software: int = None, command: int = None, error_code: int = None):
+    response = None
+
     if blockers is not None and blockers > 0:
         response = requests.get(f"http://{config.get('host')}:{config.get('port')}{config.get('base_url')}/api/top.json?blockers={blockers}")
     elif blocked is not None and blocked > 0:
@@ -188,7 +191,7 @@ def scoreboard(request: Request, blockers: int = None, blocked: int = None, refe
         "software"  : software,
         "command"   : command,
         "error_code": error_code,
-        "scores"    : fba.json_from_response(response)
+        "scores"    : network.json_from_response(response)
     })
 
 @router.get(config.get("base_url") + "/")

diff --git a/fba/commands.py b/fba/commands.py
index d85c1eb938e8071d24ea2fbe393c2ff47570f971..3f37fae795cbae1ce0d3332c616607ba7aa070ce 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -25,6 +25,7 @@ import atoma
 import bs4
 import markdown
 import reqto
+import requests
 import validators
 
 from fba import blacklist
@@ -105,8 +106,12 @@ def fetch_bkali(args: argparse.Namespace):
 
         print(f"INFO: Adding {len(domains)} new instances ...")
         for domain in domains:
-            print(f"INFO: Fetching instances from domain='{domain}' ...")
-            federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            try:
+                print(f"INFO: Fetching instances from domain='{domain}' ...")
+                federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            except requests.exceptions.Timeout as ex:
+                print(f"WARNING: Timeout during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, ex)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -312,8 +317,12 @@ def fetch_cs(args: argparse.Namespace):
                     blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level)
 
                 if not instances.is_registered(row["domain"]):
-                    print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
-                    federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
+                    try:
+                        print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
+                        federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
+                    except requests.exceptions.Timeout as ex:
+                        print(f"WARNING: Timeout during fetching instances from domain='{row['domain']}'")
+                        instances.update_last_error(row["domain"], ex)
 
         # DEBUG: print("DEBUG: Committing changes ...")
         fba.connection.commit()
@@ -356,8 +365,12 @@ def fetch_fba_rss(args: argparse.Namespace):
 
         print(f"INFO: Adding {len(domains)} new instances ...")
         for domain in domains:
-            print(f"INFO: Fetching instances from domain='{domain}' ...")
-            federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            try:
+                print(f"INFO: Fetching instances from domain='{domain}' ...")
+                federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            except requests.exceptions.Timeout as ex:
+                print(f"WARNING: Timeout during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, ex)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -405,8 +418,12 @@ def fetch_fbabot_atom(args: argparse.Namespace):
 
         print(f"INFO: Adding {len(domains)} new instances ...")
         for domain in domains:
-            print(f"INFO: Fetching instances from domain='{domain}' ...")
-            federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            try:
+                print(f"INFO: Fetching instances from domain='{domain}' ...")
+                federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            except requests.exceptions.Timeout as ex:
+                print(f"WARNING: Timeout during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, ex)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -415,7 +432,13 @@ def fetch_instances(args: argparse.Namespace):
     locking.acquire()
 
     # Initial fetch
-    federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
+    try:
+        print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
+        federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
+    except requests.exceptions.Timeout as ex:
+        print(f"WARNING: Timeout during fetching instances from args.domain='{args.domain}'")
+        instances.update_last_error(args.domain, ex)
+        return
 
     if args.single:
         # DEBUG: print("DEBUG: Not fetching more instances - EXIT!")
@@ -434,8 +457,12 @@ def fetch_instances(args: argparse.Namespace):
             print("WARNING: domain is blacklisted:", row[0])
             continue
 
-        print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
-        federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
+        try:
+            print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
+            federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
+        except requests.exceptions.Timeout as ex:
+            print(f"WARNING: Timeout during fetching instances from domain='{row[0]}'")
+            instances.update_last_error(row[0], ex)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -463,7 +490,11 @@ def fetch_federater(args: argparse.Namespace):
                 # DEBUG: print(f"DEBUG: domain='{row['#domain']}' is already registered - skipped!")
                 continue
 
-            print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
-            federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
+            try:
+                print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
+                federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
+            except requests.exceptions.Timeout as ex:
+                print(f"WARNING: Timeout during fetching instances from domain='{row['#domain']}'")
+                instances.update_last_error(row["#domain"], ex)
 
     # DEBUG: print("DEBUG: EXIT!")

diff --git a/fba/csrf.py b/fba/csrf.py
index 00ec0f32122693257cdd6d77119ce86e9e8b17e4..4602c284910c243294866bd0c8cff7316caf7305 100644 (file)
--- a/fba/csrf.py
+++ b/fba/csrf.py
@@ -21,7 +21,7 @@ from fba import config
 from fba import network
 
 def determine(domain: str, headers: dict) -> dict:
-    print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
@@ -33,26 +33,26 @@ def determine(domain: str, headers: dict) -> dict:
     reqheaders = headers
 
     # Fetch / to check for meta tag indicating csrf
-    print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...")
+    # DEBUG: print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...")
     response = reqto.get(
         f"https://{domain}/",
         headers=network.web_headers,
         timeout=(config.get("connection_timeout"), config.get("read_timeout"))
     )
 
-    print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}")
+    # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}")
     if response.ok and len(response.text) > 0:
         meta = bs4.BeautifulSoup(
             response.text,
             "html.parser"
         )
-        print(f"DEBUG: meta[]='{type(meta)}'")
+        # DEBUG: print(f"DEBUG: meta[]='{type(meta)}'")
         tag = meta.find("meta", attrs={"name": "csrf-token"})
 
-        print(f"DEBUG: tag={tag}")
+        # DEBUG: print(f"DEBUG: tag={tag}")
         if tag is not None:
-            print(f"DEBUG: Adding CSRF token='{tag['content']}' for domain='{domain}'")
+            # DEBUG: print(f"DEBUG: Adding CSRF token='{tag['content']}' for domain='{domain}'")
             reqheaders["X-CSRF-Token"] = tag["content"]
 
-    print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!")
+    # DEBUG: print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!")
     return reqheaders

diff --git a/fba/federation.py b/fba/federation.py
index 085c37b67202e2f82ba8dc327eb1b9242a2e47b2..4db6e5cdcab6f096c20e28c7e4c93bc106679580 100644 (file)
--- a/fba/federation.py
+++ b/fba/federation.py
@@ -122,8 +122,7 @@ def fetch_peers(domain: str, software: str) -> list:
         return peertube.fetch_peers(domain)
 
     # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...")
-    data = list()
-    response = network.get_json_api(
+    data = network.get_json_api(
         domain,
         "/api/v1/instance/peers",
         (config.get("connection_timeout"), config.get("read_timeout"))
@@ -141,16 +140,16 @@ def fetch_peers(domain: str, software: str) -> list:
         # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
         if "error_message" in data:
             print(f"WARNING: Could not reach any JSON API at domain='{domain}',status_code='{data['status_code']}',error_message='{data['error_message']}'")
-        elif "federated_instances" in data:
+        elif "federated_instances" in data["json"]:
             # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'")
-            peers = peers + add_peers(data["federated_instances"])
+            peers = peers + add_peers(data["json"]["federated_instances"])
             # DEBUG: print("DEBUG: Added instance(s) to peers")
         else:
             print("WARNING: JSON response does not contain 'federated_instances':", domain)
             instances.update_last_error(domain, response)
     else:
         # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
-        peers = data
+        peers = data["json"]
 
     # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
     instances.set_data("total_peers", domain, len(peers))
@@ -226,7 +225,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
     )
 
     if "error_message" not in data:
-        nodeinfo = data
+        nodeinfo = data["json"]
         # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain)
         if "links" in nodeinfo:
             # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"]))
@@ -242,7 +241,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
 
                     data = network.json_from_response(response)
                     # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code)
-                    if response.ok and isinstance(data, dict):
+                    if response.ok and len(data) > 0:
                         # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data))
                         instances.set_data("detection_mode", domain, "AUTO_DISCOVERY")
                         instances.set_data("nodeinfo_url"  , domain, link["href"])

diff --git a/fba/instances.py b/fba/instances.py
index 27a0821e3717843f0b9bba1f2efb019b13d9b083..c872c2b96832c47bf74ef85ac969ff27f0028983 100644 (file)
--- a/fba/instances.py
+++ b/fba/instances.py
@@ -201,8 +201,9 @@ def add(domain: str, origin: str, command: str, path: str = None):
 
     # DEBUG: print("DEBUG: domain,origin,command,path:", domain, origin, command, path)
     software = federation.determine_software(domain, path)
+
     # DEBUG: print("DEBUG: Determined software:", software)
-    if domain.find("/c/") > 0 and software == "lemmy":
+    if software == "lemmy" and domain.find("/c/") > 0:
         domain = domain.split("/c/")[0]
         if is_registered(domain):
             print(f"WARNING: domain='{domain}' already registered after cutting off user part. - EXIT!")
@@ -257,33 +258,33 @@ def update_last_nodeinfo(domain: str):
     # DEBUG: print("DEBUG: EXIT!")
 
 def update_last_error(domain: str, response: requests.models.Response):
-    # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
+    print("DEBUG: domain,response[]:", domain, type(response))
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
 
-    # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
+    print("DEBUG: BEFORE response[]:", type(response))
     if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
         response = f"response[{type(response)}]='{str(response)}'"
 
-    # DEBUG: print("DEBUG: AFTER response[]:", type(response))
+    print("DEBUG: AFTER response[]:", type(response))
     if isinstance(response, str):
-        # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'")
+        print(f"DEBUG: Setting last_error_details='{response}'")
         set_data("last_status_code"  , domain, 999)
         set_data("last_error_details", domain, response)
     else:
-        # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'")
+        print(f"DEBUG: Setting last_error_details='{response.reason}'")
         set_data("last_status_code"  , domain, response.status_code)
         set_data("last_error_details", domain, response.reason)
 
     # Running pending updated
-    # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...")
+    print(f"DEBUG: Invoking update_data({domain}) ...")
     update_data(domain)
 
     fba.log_error(domain, response)
 
-    # DEBUG: print("DEBUG: EXIT!")
+    print("DEBUG: EXIT!")
 
 def is_registered(domain: str) -> bool:
     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")

diff --git a/fba/network.py b/fba/network.py
index e77a658c234ba605e27a6648747f3eedcb803f43..9a30d5800d3b4b90651cf9fcfbdc7405ce0f93ad 100644 (file)
--- a/fba/network.py
+++ b/fba/network.py
@@ -51,7 +51,9 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict
     # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...")
     headers = csrf.determine(domain, {**api_headers, **headers})
 
-    json_reply = {}
+    json_reply = {
+        "status_code": 200,
+    }
 
     try:
         # DEBUG: print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',data='{data}',headers({len(headers)})={headers}")
@@ -62,6 +64,15 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict
             timeout=(config.get("connection_timeout"), config.get("read_timeout"))
         )
 
+        json_reply["json"] = json_from_response(response)
+
+        # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},json_reply[]='{type(json_reply)}'")
+        if not response.ok or response.status_code >= 400:
+            print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',data()={len(data)},response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'")
+            json_reply["status_code"]   = response.status_code
+            json_reply["error_message"] = response.text
+            instances.update_last_error(domain, response)
+
     except requests.exceptions.ConnectionError as exception:
         # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'")
         json_reply["status_code"]   = 999
@@ -69,15 +80,6 @@ def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict
         instances.update_last_error(domain, exception)
         raise exception
 
-    json_reply = json_from_response(response)
-
-    # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},json_reply[]='{type(json_reply)}'")
-    if not response.ok or response.status_code >= 400:
-        print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',data()={len(data)},response.status_code='{response.status_code}',json_reply[]='{type(json_reply)}'")
-        json_reply["status_code"]   = response.status_code
-        json_reply["error_message"] = response.text
-        instances.update_last_error(domain, response)
-
     # DEBUG: print(f"DEBUG: Returning json_reply({len(json_reply)})=[]:{type(json_reply)}")
     return json_reply
 
@@ -116,7 +118,7 @@ def get_json_api(domain: str, path: str, timeout: tuple) -> dict:
         instances.update_last_error(domain, exception)
         raise exception
 
-    json_reply = json_from_response(response)
+    json_reply["json"] = json_from_response(response)
 
     # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},json_reply[]='{type(json_reply)}'")
     if not response.ok or response.status_code >= 400:
@@ -209,7 +211,7 @@ def json_from_response(response: requests.models.Response) -> list:
     if not isinstance(response, requests.models.Response):
         raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'")
 
-    data = list()
+    data = dict()
     if response.text.strip() != "":
         # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...")
         try:

diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py
index f819fb0cd66da1dee06405d059a85f9d0ccc59d5..f2cc4e42f1595530a0bcace8f24236ac5aa68ba9 100644 (file)
--- a/fba/networks/lemmy.py
+++ b/fba/networks/lemmy.py
@@ -39,9 +39,9 @@ def fetch_peers(domain: str) -> list:
         if "error_message" in data:
             print("WARNING: Could not reach any JSON API:", domain)
             instances.update_last_error(domain, response)
-        elif "federated_instances" in data:
+        elif "federated_instances" in data["json"]:
             # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'")
-            peers = peers + federation.add_peers(data["federated_instances"])
+            peers = peers + federation.add_peers(data["json"]["federated_instances"])
             # DEBUG: print("DEBUG: Added instance(s) to peers")
         else:
             print("WARNING: JSON response does not contain 'federated_instances':", domain)

diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py
index 23f2536e41194ac400179afeab591d8ca35a5098..25d11f416595afee42a0238b6a7032e35565389b 100644 (file)
--- a/fba/networks/mastodon.py
+++ b/fba/networks/mastodon.py
@@ -140,16 +140,17 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str):
             }
 
             # DEBUG: print("DEBUG: Querying API domain_blocks:", domain)
-            blocklist = network.get_json_api(
+            data = network.get_json_api(
                 domain,
                 "/api/v1/instance/domain_blocks",
                 (config.get("connection_timeout"), config.get("read_timeout"))
             )
 
-            if "error_message" in blocklist:
+            if "error_message" in data:
                 print(f"WARNING: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'")
-                instances.update_last_error(domain, blocklist)
+                instances.update_last_error(domain, data)
 
+            blocklist = data["json"]
             print(f"INFO: Checking {len(blocklist)} entries from domain='{domain}',software='mastodon' ...")
             for block in blocklist:
                 entry = {

diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py
index 3a0207836f70bd3fc0a0abce964f0a71b0a7f0c2..83640da5eebc24d8714627008e432d9e7cad7098 100644 (file)
--- a/fba/networks/peertube.py
+++ b/fba/networks/peertube.py
@@ -39,10 +39,10 @@ def fetch_peers(domain: str) -> list:
 
             print(f"DEBUG: data['{type(data)}']='{data}'")
             if "error_message" not in data:
-                print("DEBUG: Success, data:", len(data))
-                if "data" in data:
+                print("DEBUG: Success, data[json]:", len(data["json"]))
+                if "data" in data["json"]:
                     print(f"DEBUG: Found {len(data['data'])} record(s).")
-                    for record in data["data"]:
+                    for record in data["json"]["data"]:
                         print(f"DEBUG: record()={len(record)}")
                         if mode in record and "host" in record[mode]:
                             print(f"DEBUG: Found host={record[mode]['host']}, adding ...")
@@ -50,7 +50,7 @@ def fetch_peers(domain: str) -> list:
                         else:
                             print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}")
 
-                    if len(data["data"]) < 100:
+                    if len(data["json"]["data"]) < 100:
                         print("DEBUG: Reached end of JSON response:", domain)
                         break