Continued:

- try to update last error details and with a fake status_code

diff --git a/fba.py b/fba.py
index 5b27c3626409f194db8975c68b1be04bb65620d3..dcdcea4d655bfb68ca5cb1d312dce2d31459b7a6 100644 (file)
--- a/fba.py
+++ b/fba.py
@@ -20,6 +20,9 @@ blacklist = [
     "ngrok.io",
 ]
 
+pending_errors = {
+}
+
 nodeinfos = [
     "http://nodeinfo.diaspora.software/ns/schema/2.1",
     "http://nodeinfo.diaspora.software/ns/schema/2.0",
@@ -54,22 +57,36 @@ def update_last_blocked(domain: str):
             domain
         ])
 
-    except:
-        print("ERROR: failed SQL query:", domain)
+        if cursor.rowcount == 0:
+            print("WARNING: Did not update any rows:", domain)
+
+    except BaseException as e:
+        print("ERROR: failed SQL query:", domain, e)
         sys.exit(255)
 
 def update_last_error(domain: str, res: any):
     # NOISY-DEBUG: print("DEBUG: domain,res.status_code:", domain, res.status_code, res.reason)
     try:
-        cursor.execute("UPDATE instances SET last_status_code = ?, last_error_details = ?, last_updated = ? WHERE domain = ?", [
-            res.status_code,
-            res.reason,
-            time.time(),
-            domain
-        ])
+        if type(res) is str:
+            cursor.execute("UPDATE instances SET last_status_code = 999, last_error_details = ?, last_updated = ? WHERE domain = ?", [
+                res,
+                time.time(),
+                domain
+            ])
+        else:
+            cursor.execute("UPDATE instances SET last_status_code = ?, last_error_details = ?, last_updated = ? WHERE domain = ?", [
+                res.status_code,
+                res.reason,
+                time.time(),
+                domain
+            ])
 
-    except:
-        print("ERROR: failed SQL query:", domain)
+        if cursor.rowcount == 0:
+            # NOISY-DEBUG: print("DEBUG: Did not update any rows:", domain)
+            pending_errors[domain] = res
+
+    except BaseException as e:
+        print("ERROR: failed SQL query:", domain, e)
         sys.exit(255)
 
 def update_last_nodeinfo(domain: str):
@@ -81,8 +98,11 @@ def update_last_nodeinfo(domain: str):
             domain
         ])
 
-    except:
-        print("ERROR: failed SQL query:", domain)
+        if cursor.rowcount == 0:
+            print("WARNING: Did not update any rows:", domain)
+
+    except BaseException as e:
+        print("ERROR: failed SQL query:", domain, e)
         sys.exit(255)
 
     connection.commit()
@@ -159,7 +179,7 @@ def fetch_nodeinfo(domain: str) -> list:
                 json = res.json()
                 break
             elif not res.ok or res.status_code >= 400:
-                # NOISY-DEBUG: print("DEBUG: Failed fetching nodeinfo from domain:", domain)
+                print("WARNING: Failed fetching nodeinfo from domain:", domain)
                 update_last_error(domain, res)
                 continue
 
@@ -170,6 +190,7 @@ def fetch_nodeinfo(domain: str) -> list:
     # NOISY-DEBUG: print("DEBUG: json[]:", type(json))
     if json is None or len(json) == 0:
         print("WARNING: Failed fetching nodeinfo from domain:", domain)
+        update_last_error(domain, "Cannot fetch nodeinfo")
 
     # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json))
     return json
@@ -203,6 +224,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
 
     except:
         print("WARNING: Failed fetching .well-known info:", domain)
+        update_last_error(domain, "Cannot fetch .well-known")
         pass
 
     # NOISY-DEBUG: print("DEBUG: Returning json[]:", type(json))
@@ -275,6 +297,9 @@ def update_block_reason(reason: str, blocker: str, blocked: str, block_level: st
             ),
         )
 
+        if cursor.rowcount == 0:
+            print("WARNING: Did not update any rows:", domain)
+
     except:
         print("ERROR: failed SQL query:", reason, blocker, blocked, block_level)
         sys.exit(255)
@@ -292,6 +317,9 @@ def update_last_seen(blocker: str, blocked: str, block_level: str):
             )
         )
 
+        if cursor.rowcount == 0:
+            print("WARNING: Did not update any rows:", domain)
+
     except:
         print("ERROR: failed SQL query:", last_seen, blocker, blocked, block_level)
         sys.exit(255)
@@ -349,8 +377,13 @@ def add_instance(domain: str, origin: str, originator: str):
             ),
         )
 
-    except:
-        print("ERROR: failed SQL query:", domain)
+        if domain in pending_errors:
+            print("DEBUG: domain has pending error be updated:", domain)
+            update_last_error(domain, pending_errors[domain])
+            del pending_errors[domain]
+
+    except BaseException as e:
+        print("ERROR: failed SQL query:", domain, e)
         sys.exit(255)
     else:
         # NOISY-DEBUG: print("DEBUG: Updating nodeinfo for domain:", domain)
@@ -394,20 +427,20 @@ def get_mastodon_blocks(domain: str) -> dict:
     }
 
     translations = {
-        "Silenced instances": "Silenced servers",
-        "Suspended instances": "Suspended servers",
-        "Gesperrte Server": "Suspended servers",
-        "Gefilterte Medien": "Filtered media",
-        "Stummgeschaltete Server": "Silenced servers",
-        "停止済みのサーバー": "Suspended servers",
+        "Silenced instances"            : "Silenced servers",
+        "Suspended instances"           : "Suspended servers",
+        "Gesperrte Server"              : "Suspended servers",
+        "Gefilterte Medien"             : "Filtered media",
+        "Stummgeschaltete Server"       : "Silenced servers",
+        "停止済みのサーバー"            : "Suspended servers",
         "メディアを拒否しているサーバー": "Filtered media",
-        "サイレンス済みのサーバー": "Silenced servers",
-        "שרתים מושעים": "Suspended servers",
-        "מדיה מסוננת": "Filtered media",
-        "שרתים מוגבלים": "Silenced servers",
-        "Serveurs suspendus": "Suspended servers",
-        "Médias filtrés": "Filtered media",
-        "Serveurs limités": "Silenced servers",
+        "サイレンス済みのサーバー"      : "Silenced servers",
+        "שרתים מושעים"                  : "Suspended servers",
+        "מדיה מסוננת"                   : "Filtered media",
+        "שרתים מוגבלים"                 : "Silenced servers",
+        "Serveurs suspendus"            : "Suspended servers",
+        "Médias filtrés"                : "Filtered media",
+        "Serveurs limités"              : "Silenced servers",
     }
 
     try:
@@ -478,7 +511,7 @@ def get_misskey_blocks(domain: str) -> dict:
     # NOISY-DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain)
     blocks = {
         "suspended": [],
-        "blocked": []
+        "blocked"  : []
     }
 
     try:

diff --git a/fetch_instances.py b/fetch_instances.py
index 0fb8a452a8c6e51622cf394cbf7c1c4fd6135a0b..41d1d770547a57c6a867d8297ecce76bd3d9b349 100644 (file)
--- a/fetch_instances.py
+++ b/fetch_instances.py
@@ -5,13 +5,7 @@ import time
 import fba
 
 def fetch_instances(domain: str, origin: str):
-    # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, origin)
-    peerlist = fba.get_peers(domain)
-
-    if (peerlist is None):
-        print("ERROR: Cannot fetch peers:", domain)
-        return
-
+    # NOISY-DEBUG: print("DEBUG: domain,origin:", domain, origin)
     fba.cursor.execute(
         "SELECT domain FROM instances WHERE domain = ? LIMIT 1", [domain]
     )
@@ -20,6 +14,13 @@ def fetch_instances(domain: str, origin: str):
         # NOISY-DEBUG: print("DEBUG: Adding new domain:", domain, origin)
         fba.add_instance(domain, origin, sys.argv[0])
 
+    # NOISY-DEBUG: print("DEBUG: Fetching instances for domain:", domain, origin)
+    peerlist = fba.get_peers(domain)
+
+    if (peerlist is None):
+        print("ERROR: Cannot fetch peers:", domain)
+        return
+
     print(f"INFO: Checking {len(peerlist)} instances from {domain} ...")
     for instance in peerlist:
         instance = instance.lower()
@@ -58,12 +59,14 @@ fba.cursor.execute(
     "SELECT domain FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_nodeinfo IS NULL OR last_nodeinfo < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]]
 )
 
-for instance in fba.cursor.fetchall():
-    if fba.is_blacklisted(instance[0]):
-        # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", instance)
+for row in fba.cursor.fetchall():
+    domain = row[0]
+    # NOISY-DEBUG: print("DEBUG: domain:", domain)
+    if fba.is_blacklisted(domain):
+        # NOISY-DEBUG: print("DEBUG: domain is blacklisted:", domain)
         continue
 
-    print("INFO: Fetching instances for instance:", instance[0])
-    fetch_instances(instance[0], None)
+    print("INFO: Fetching instances for instance:", domain)
+    fetch_instances(domain, None)
 
 fba.connection.close()