Continued:

- also update last_instance_fetch when software type isn't determined yet
- include command name in log messages

diff --git a/fba/commands.py b/fba/commands.py
index 1c0f74ec831a66ee62f9a5b9441effb14402bc23..74656c48712575d7abf0de189bf10b0329ace70e 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -118,7 +118,7 @@ def fetch_bkali(args: argparse.Namespace) -> int:
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
             except network.exceptions as exception:
-                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_bkali) from domain='{domain}'")
                 instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -342,7 +342,7 @@ def fetch_cs(args: argparse.Namespace):
                         print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
                         federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
                     except network.exceptions as exception:
-                        print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'")
+                        print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_cs) from domain='{row['domain']}'")
                         instances.update_last_error(row["domain"], exception)
 
         # DEBUG: print("DEBUG: Committing changes ...")
@@ -390,7 +390,7 @@ def fetch_fba_rss(args: argparse.Namespace):
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
             except network.exceptions as exception:
-                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fba_rss) from domain='{domain}'")
                 instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -443,7 +443,7 @@ def fetch_fbabot_atom(args: argparse.Namespace):
                 print(f"INFO: Fetching instances from domain='{domain}' ...")
                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
             except network.exceptions as exception:
-                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fbabot_atom) from domain='{domain}'")
                 instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -457,8 +457,9 @@ def fetch_instances(args: argparse.Namespace) -> int:
         print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
         federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
     except network.exceptions as exception:
-        print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'")
+        print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from args.domain='{args.domain}'")
         instances.update_last_error(args.domain, exception)
+
         return 100
 
     if args.single:
@@ -482,7 +483,7 @@ def fetch_instances(args: argparse.Namespace) -> int:
             print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
             federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
         except network.exceptions as exception:
-            print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'")
+            print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from domain='{row[0]}'")
             instances.update_last_error(row[0], exception)
 
     # DEBUG: print("DEBUG: EXIT!")
@@ -549,9 +550,8 @@ def fetch_oliphant(args: argparse.Namespace):
         # DEBUG: print(f"DEBUG: response[]='{type(response)}'")
         if response.ok and response.content != "":
             # DEBUG: print(f"DEBUG: Fetched {len(response.content)} Bytes, parsing CSV ...")
-            #print(f"DEBUG: response.content={response.content}")
             reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
-            #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate'
+
             # DEBUG: print(f"DEBUG: reader[]='{type(reader)}'")
             for row in reader:
                 if not validators.domain(row["#domain"]):
@@ -568,7 +568,7 @@ def fetch_oliphant(args: argparse.Namespace):
                     print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
                     federation.fetch_instances(row["#domain"], block["blocker"], None, inspect.currentframe().f_code.co_name)
                 except network.exceptions as exception:
-                    print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'")
+                    print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_oliphant) from domain='{row['#domain']}'")
                     instances.update_last_error(row["#domain"], exception)
 
     # DEBUG: print("DEBUG: EXIT!")

diff --git a/fba/federation.py b/fba/federation.py
index 5e633175811ee45c22c8f48cc1a5b21df6481d31..80d7a71e7a4aa424d942e384490778b4cbdb8a05 100644 (file)
--- a/fba/federation.py
+++ b/fba/federation.py
@@ -41,7 +41,7 @@ nodeinfo_identifier = [
     "http://nodeinfo.diaspora.software/ns/schema/1.0",
 ]
 
-def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None):
+def fetch_instances(domain: str, origin: str, software: str, command: str, path: str = None):
     # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
@@ -50,22 +50,25 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
     elif not isinstance(origin, str) and origin is not None:
         raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'")
     elif software is None:
+        # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
+        instances.update_last_instance_fetch(domain)
+
         # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...")
         software = determine_software(domain, path)
         # DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'")
     elif not isinstance(software, str):
         raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'")
-    elif not isinstance(script, str):
-        raise ValueError(f"Parameter script[]='{type(script)}' is not 'str'")
-    elif domain == "":
-        raise ValueError("Parameter 'domain' is empty")
+    elif not isinstance(command, str):
+        raise ValueError(f"Parameter command[]='{type(command)}' is not 'str'")
+    elif command == "":
+        raise ValueError("Parameter 'command' is empty")
 
     if domain.split(".")[-1] == "arpa":
         print(f"WARNING: domain='{domain}' is a reversed .arpa domain and should not be used generally.")
         return
     elif not instances.is_registered(domain):
         # DEBUG: print("DEBUG: Adding new domain:", domain, origin)
-        instances.add(domain, origin, script, path)
+        instances.add(domain, origin, command, path)
 
     # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software)
     peerlist = fetch_peers(domain, software)
@@ -107,7 +110,7 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
             continue
         elif not instances.is_registered(instance):
             # DEBUG: print("DEBUG: Adding new instance:", instance, domain)
-            instances.add(instance, domain, script)
+            instances.add(instance, domain, command)
 
     # DEBUG: print("DEBUG: EXIT!")
 
@@ -197,7 +200,7 @@ def fetch_nodeinfo(domain: str, path: str = None) -> dict:
     # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...")
     nodeinfo = fetch_wellknown_nodeinfo(domain)
 
-    # DEBUG: print(f"DEBUG: nodeinfo[{type(nodeinfo)}]()='{len(nodeinfo)}'")
+    # DEBUG: print(f"DEBUG: nodeinfo[{type(nodeinfo)}]({len(nodeinfo)}='{nodeinfo}'")
     if "error_message" not in nodeinfo and "json" in nodeinfo and len(nodeinfo["json"]) > 0:
         # DEBUG: print(f"DEBUG: Found nodeinfo[json]()={len(nodeinfo['json'])} - EXIT!")
         return nodeinfo["json"]
@@ -266,7 +269,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict:
         # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
         headers = csrf.determine(domain, dict())
     except network.exceptions as exception:
-        print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_wellknown,{__name__}) - EXIT!")
+        print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_wellknown_nodeinfo,{__name__}) - EXIT!")
         return {
             "status_code"  : 500,
             "error_message": type(exception),

diff --git a/fba/instances.py b/fba/instances.py
index 96a4cd6bed1162320a281a72ae76d8a564269052..5b5046f68341423fb20700a7c9689e3a62eec026 100644 (file)
--- a/fba/instances.py
+++ b/fba/instances.py
@@ -161,6 +161,7 @@ def update_last_instance_fetch(domain: str):
     # DEBUG: print("DEBUG: EXIT!")
 
 def update_last_blocked(domain: str):
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":

diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py
index 6fedb472dfafc02990281fd902f449cf32e28b6b..dc63120e72de2b44fe12e53928708e209536e8bd 100644 (file)
--- a/fba/networks/friendica.py
+++ b/fba/networks/friendica.py
@@ -47,8 +47,12 @@ def fetch_blocks(domain: str) -> dict:
 
         blocklist = doc.find(id="about_blocklist")
     except network.exceptions as exception:
-        print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+        print(f"WARNING: Exception '{type(exception)}' during fetching instances (friendica) from domain='{domain}'")
         instances.update_last_error(domain, exception)
+
+        if instances.has_pending(domain):
+            instances.update_data(domain)
+
         return dict()
 
     # Prevents exceptions:

diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py
index f8813d1a2b14f9cdecfbdc9cf0bac6450ebaaaab..c25cf8c57ce5fd02894b026264ea3804c96038e2 100644 (file)
--- a/fba/networks/peertube.py
+++ b/fba/networks/peertube.py
@@ -45,7 +45,7 @@ def fetch_peers(domain: str) -> list:
         while True:
             data = network.get_json_api(
                 domain,
-                "/api/v1/server/{mode}?start={start}&count=100",
+                f"/api/v1/server/{mode}?start={start}&count=100",
                 headers,
                 (config.get("connection_timeout"), config.get("read_timeout"))
             )
@@ -54,8 +54,10 @@ def fetch_peers(domain: str) -> list:
             if "error_message" not in data:
                 print("DEBUG: Success, data[json]:", len(data["json"]))
                 if "data" in data["json"]:
-                    print(f"DEBUG: Found {len(data['data'])} record(s).")
-                    for record in data["json"]["data"]:
+                    rows = data["json"]["data"]
+
+                    print(f"DEBUG: Found {len(rows)} record(s).")
+                    for record in rows:
                         print(f"DEBUG: record()={len(record)}")
                         if mode in record and "host" in record[mode]:
                             print(f"DEBUG: Found host={record[mode]['host']}, adding ...")
@@ -63,7 +65,7 @@ def fetch_peers(domain: str) -> list:
                         else:
                             print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}")
 
-                    if len(data["json"]["data"]) < 100:
+                    if len(rows) < 100:
                         print(f"DEBUG: Reached end of JSON response, domain='{domain}'")
                         break