From 1cdf928ac643b8a36afebe6256b2d6bbb52f43f6 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 13 Jun 2023 22:16:46 +0200 Subject: [PATCH] Continued: - re-introduced wrapper function around each column of the "model" `instances` - marked set_data() as private (now named _set_data()) - these wrappers do NOT invoke instances.update_data(), you have to do this from your code --- fba/commands.py | 14 ++-- fba/fba.py | 2 +- fba/federation.py | 29 ++++---- fba/models/instances.py | 141 +++++++++++++++++++++++++++++++------- fba/network.py | 10 +-- fba/networks/friendica.py | 2 +- fba/networks/lemmy.py | 8 +-- fba/networks/mastodon.py | 7 +- fba/networks/misskey.py | 18 ++--- fba/networks/peertube.py | 4 +- fba/networks/pleroma.py | 6 +- 11 files changed, 167 insertions(+), 74 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 2f31048..444f936 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -120,7 +120,7 @@ def fetch_bkali(args: argparse.Namespace) -> int: federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_bkali) from domain='{domain}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") return 0 @@ -169,7 +169,7 @@ def fetch_blocks(args: argparse.Namespace): continue # DEBUG: print(f"DEBUG: blocker='{blocker}'") - instances.set_data("last_blocked", blocker, time.time()) + instances.set_last_blocked(blocker) if software == "pleroma": print(f"INFO: blocker='{blocker}',software='{software}'") @@ -337,7 +337,7 @@ def fetch_cs(args: argparse.Namespace): federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_cs) from domain='{row['domain']}'") - instances.update_last_error(row["domain"], exception) + instances.set_last_error(row["domain"], exception) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() @@ -385,7 +385,7 @@ def fetch_fba_rss(args: argparse.Namespace): federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fba_rss) from domain='{domain}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -438,7 +438,7 @@ def fetch_fbabot_atom(args: argparse.Namespace): federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fbabot_atom) from domain='{domain}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -452,7 +452,7 @@ def fetch_instances(args: argparse.Namespace) -> int: federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from args.domain='{args.domain}'") - instances.update_last_error(args.domain, exception) + instances.set_last_error(args.domain, exception) return 100 @@ -478,7 +478,7 @@ def fetch_instances(args: argparse.Namespace) -> int: federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from domain='{row[0]}'") - instances.update_last_error(row[0], exception) + instances.set_last_error(row[0], exception) # DEBUG: print("DEBUG: EXIT!") return 0 diff --git a/fba/fba.py b/fba/fba.py index 3cfc925..1c7e973 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -127,7 +127,7 @@ def process_domain(domain: str, blocker: str, command: str) -> bool: processed = True except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_oliphant) from domain='{domain}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) # DEBUG: print(f"DEBUG: processed='{processed}' - EXIT!") return processed diff --git a/fba/federation.py b/fba/federation.py index 88e6236..d4959be 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -14,7 +14,6 @@ # along with this program. If not, see . import bs4 -import time import validators from fba import blacklist @@ -55,7 +54,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: raise ValueError(f"Parameter origin[]='{type(origin)}' is not 'str'") elif software is None: # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.set_data("last_instance_fetch", domain, time.time()) + instances.set_last_instance_fetch(domain) # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...") software = determine_software(domain, path) @@ -76,7 +75,7 @@ def fetch_instances(domain: str, origin: str, software: str, command: str, path: instances.add(domain, origin, command, path) # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.set_data("last_instance_fetch", domain, time.time()) + instances.set_last_instance_fetch(domain) # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) peerlist = fetch_peers(domain, software) @@ -185,13 +184,13 @@ def fetch_peers(domain: str, software: str) -> list: else: message = "JSON response does not contain 'federated_instances' or 'error_message'" print(f"WARNING: {message},domain='{domain}'") - instances.update_last_error(domain, message) + instances.set_last_error(domain, message) else: # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) peers = data["json"] # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) + instances.set_total_peers(domain, peers) # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers @@ -254,8 +253,8 @@ def fetch_nodeinfo(domain: str, path: str = None) -> dict: # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if "error_message" not in data: # DEBUG: print("DEBUG: Success:", request) - instances.set_data("detection_mode", domain, "STATIC_CHECK") - instances.set_data("nodeinfo_url" , domain, request) + instances.set_detection_mode(domain, "STATIC_CHECK") + instances.set_nodeinfo_url(domain, request) break print(f"WARNING: Failed fetching nodeinfo from domain='{domain}',status_code='{data['status_code']}',error_message='{data['error_message']}'") @@ -307,13 +306,13 @@ def fetch_wellknown_nodeinfo(domain: str) -> dict: ) # DEBUG: print("DEBUG: href,data[]:", link["href"], type(data)) - if "json" in data: + if not "error_message" in data and "json" in data: # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) - instances.set_data("detection_mode", domain, "AUTO_DISCOVERY") - instances.set_data("nodeinfo_url" , domain, link["href"]) + instances.set_detection_mode(domain, "AUTO_DISCOVERY") + instances.set_nodeinfo_url(domain, link["href"]) break else: - instances.update_last_error(domain, data) + instances.set_last_error(domain, data) else: print("WARNING: Unknown 'rel' value:", domain, link["rel"]) else: @@ -353,12 +352,12 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: # DEBUG: print("DEBUG: Found generator meta tag:", domain) software = tidyup.domain(generator.get("content")) print(f"INFO: domain='{domain}' is generated by '{software}'") - instances.set_data("detection_mode", domain, "GENERATOR") + instances.set_detection_mode(domain, "GENERATOR") elif isinstance(site_name, bs4.element.Tag): # DEBUG: print("DEBUG: Found property=og:site_name:", domain) sofware = tidyup.domain(site_name.get("content")) print(f"INFO: domain='{domain}' has og:site_name='{software}'") - instances.set_data("detection_mode", domain, "SITE_NAME") + instances.set_detection_mode(domain, "SITE_NAME") # DEBUG: print(f"DEBUG: software[]='{type(software)}'") if isinstance(software, str) and software == "": @@ -409,11 +408,11 @@ def determine_software(domain: str, path: str = None) -> str: return fetch_generator_from_path(domain) elif "status" in data and data["status"] == "error" and "message" in data: print("WARNING: JSON response is an error:", data["message"]) - instances.update_last_error(domain, data["message"]) + instances.set_last_error(domain, data["message"]) return fetch_generator_from_path(domain) elif "message" in data: print("WARNING: JSON response contains only a message:", data["message"]) - instances.update_last_error(domain, data["message"]) + instances.set_last_error(domain, data["message"]) return fetch_generator_from_path(domain) elif "software" not in data or "name" not in data["software"]: # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") diff --git a/fba/models/instances.py b/fba/models/instances.py index 154dbfa..7644566 100644 --- a/fba/models/instances.py +++ b/fba/models/instances.py @@ -56,7 +56,7 @@ _pending = { "last_error_details" : {}, } -def set_data(key: str, domain: str, value: any): +def _set_data(key: str, domain: str, value: any): # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!") if not isinstance(key, str): raise ValueError("Parameter key[]='{type(key)}' is not 'str'") @@ -100,7 +100,7 @@ def update_data(domain: str): elif domain == "": raise ValueError("Parameter 'domain' is empty") elif not has_pending(domain): - raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked") + raise Exception(f"domain='{domain}' has no pending instance data, but function invoked") # DEBUG: print(f"DEBUG: Updating instance data for domain='{domain}' ...") sql_string = "" @@ -112,12 +112,16 @@ def update_data(domain: str): fields.append(_pending[key][domain]) sql_string += f" {key} = ?," - fields.append(time.time()) - fields.append(domain) - + # DEBUG: print(f"DEBUG: sql_string()={len(sql_string)}") if sql_string == "": raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'") + # Set last_updated to current timestamp + fields.append(time.time()) + + # For WHERE statement + fields.append(domain) + # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}") sql_string = "UPDATE instances SET" + sql_string + " last_updated = ? WHERE domain = ? LIMIT 1" # DEBUG: print("DEBUG: sql_string:", sql_string) @@ -210,16 +214,13 @@ def add(domain: str, origin: str, command: str, path: str = None, software: str if has_pending(domain): # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...") - set_data("last_status_code" , domain, None) - set_data("last_error_details", domain, None) + _set_data("last_status_code" , domain, None) + _set_data("last_error_details", domain, None) update_data(domain) - # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}'") - update_last_nodeinfo(domain) - # DEBUG: print("DEBUG: EXIT!") -def update_last_nodeinfo(domain: str): +def set_last_nodeinfo(domain: str): # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") @@ -227,8 +228,8 @@ def update_last_nodeinfo(domain: str): raise ValueError("Parameter 'domain' is empty") # DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain) - set_data("last_nodeinfo", domain, time.time()) - set_data("last_updated" , domain, time.time()) + _set_data("last_nodeinfo", domain, time.time()) + _set_data("last_updated" , domain, time.time()) # Running pending updated # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...") @@ -236,7 +237,7 @@ def update_last_nodeinfo(domain: str): # DEBUG: print("DEBUG: EXIT!") -def update_last_error(domain: str, error: dict): +def set_last_error(domain: str, error: dict): # DEBUG: print("DEBUG: domain,error[]:", domain, type(error)) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") @@ -250,28 +251,27 @@ def update_last_error(domain: str, error: dict): if isinstance(error, str): # DEBUG: print(f"DEBUG: Setting last_error_details='{error}'") - set_data("last_status_code" , domain, 999) - set_data("last_error_details", domain, error if error != "" else None) + _set_data("last_status_code" , domain, 999) + _set_data("last_error_details", domain, error if error != "" else None) elif isinstance(error, requests.models.Response): # DEBUG: print(f"DEBUG: Setting last_error_details='{error.reason}'") - set_data("last_status_code" , domain, error.status_code) - set_data("last_error_details", domain, error.reason if error.reason != "" else None) + _set_data("last_status_code" , domain, error.status_code) + _set_data("last_error_details", domain, error.reason if error.reason != "" else None) elif not isinstance(error, dict): raise KeyError(f"Cannot handle keys in error[{type(error)}]='{error}'") elif "status_code" in error and "error_message" in error: # DEBUG: print(f"DEBUG: Setting last_error_details='{error['error_message']}'") - set_data("last_status_code" , domain, error["status_code"]) - set_data("last_error_details", domain, error["error_message"] if error["error_message"] != "" else None) + _set_data("last_status_code" , domain, error["status_code"]) + _set_data("last_error_details", domain, error["error_message"] if error["error_message"] != "" else None) elif "json" in error and "error" in error["json"]: - set_data("last_status_code" , domain, error["status_code"]) - set_data("last_error_details", domain, error["json"]["error"] if error["json"]["error"] != "" else None) + _set_data("last_status_code" , domain, error["status_code"]) + _set_data("last_error_details", domain, error["json"]["error"] if error["json"]["error"] != "" else None) # Running pending updated # DEBUG: print(f"DEBUG: Invoking update_data({domain}) ...") update_data(domain) error_log.add(domain, error) - # DEBUG: print("DEBUG: EXIT!") def is_registered(domain: str) -> bool: @@ -353,3 +353,98 @@ def deobscure(char: str, domain: str, blocked_hash: str = None) -> tuple: # DEBUG: print(f"DEBUG: row[]='{type(row)}' - EXIT!") return row + +def set_last_blocked (domain: str): + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + # Set timestamp + _set_data("last_blocked", domain, time.time()) + # DEBUG: print("DEBUG: EXIT!") + +def set_last_instance_fetch (domain: str): + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + + # Set timestamp + _set_data("last_instance_fetch", domain, time.time()) + # DEBUG: print("DEBUG: EXIT!") + +def set_total_peers (domain: str, peers: list): + # DEBUG: print(f"DEBUG: domain='{domain}',peers()={len(peers)} - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(peers, list): + raise ValueError("Parameter peers[]='{type(peers)}' is not 'list'") + + # Set timestamp + _set_data("total_peers", domain, len(peers)) + # DEBUG: print("DEBUG: EXIT!") + +def set_nodeinfo_url (domain: str, url: list): + # DEBUG: print(f"DEBUG: domain='{domain}',url='{url}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(url, str): + raise ValueError("Parameter url[]='{type(url)}' is not 'list'") + elif url == "": + raise ValueError("Parameter 'url' is empty") + + # Set timestamp + _set_data("nodeinfo_url", domain, url) + # DEBUG: print("DEBUG: EXIT!") + +def set_detection_mode (domain: str, url: list): + # DEBUG: print(f"DEBUG: domain='{domain}',url='{url}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(url, str): + raise ValueError("Parameter url[]='{type(url)}' is not 'list'") + elif url == "": + raise ValueError("Parameter 'url' is empty") + + # Set timestamp + _set_data("detection_mode", domain, url) + # DEBUG: print("DEBUG: EXIT!") + +def set_detection_mode (domain: str, url: list): + # DEBUG: print(f"DEBUG: domain='{domain}',url='{url}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(url, str): + raise ValueError("Parameter url[]='{type(url)}' is not 'list'") + elif url == "": + raise ValueError("Parameter 'url' is empty") + + # Set timestamp + _set_data("detection_mode", domain, url) + # DEBUG: print("DEBUG: EXIT!") + +def set_detection_mode (domain: str, mode: list): + # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}' - CALLED!") + if not isinstance(domain, str): + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") + elif domain == "": + raise ValueError("Parameter 'domain' is empty") + elif not isinstance(mode, str): + raise ValueError("Parameter mode[]='{type(mode)}' is not 'list'") + elif mode == "": + raise ValueError("Parameter 'mode' is empty") + + # Set timestamp + _set_data("detection_mode", domain, mode) + # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/network.py b/fba/network.py index ff43513..5a404a4 100644 --- a/fba/network.py +++ b/fba/network.py @@ -81,14 +81,14 @@ def post_json_api(domain: str, path: str, data: str = "", headers: dict = {}) -> json_reply["status_code"] = response.status_code json_reply["error_message"] = response.reason del json_reply["json"] - instances.update_last_error(domain, response) + instances.set_last_error(domain, response) except exceptions as exception: # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") json_reply["status_code"] = 999 json_reply["error_message"] = f"exception['{type(exception)}']='{str(exception)}'" json_reply["exception"] = exception - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) raise exception # DEBUG: print(f"DEBUG: Returning json_reply({len(json_reply)})=[]:{type(json_reply)}") @@ -162,7 +162,7 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict: json_reply["status_code"] = 999 json_reply["error_message"] = f"exception['{type(exception)}']='{str(exception)}'" json_reply["exception"] = exception - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) raise exception json_reply["json"] = json_from_response(response) @@ -173,7 +173,7 @@ def get_json_api(domain: str, path: str, headers: dict, timeout: tuple) -> dict: json_reply["status_code"] = response.status_code json_reply["error_message"] = response.reason del json_reply["json"] - instances.update_last_error(domain, response) + instances.set_last_error(domain, response) # DEBUG: print(f"DEBUG: Returning json_reply({len(json_reply)})=[]:{type(json_reply)}") return json_reply @@ -252,7 +252,7 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: tuple) -> req except exceptions as exception: # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) raise exception # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index aef3f23..0c881f7 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -49,7 +49,7 @@ def fetch_blocks(domain: str) -> dict: blocklist = doc.find(id="about_blocklist") except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances (friendica) from domain='{domain}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) return dict() # Prevents exceptions: diff --git a/fba/networks/lemmy.py b/fba/networks/lemmy.py index df6b6bc..038b4eb 100644 --- a/fba/networks/lemmy.py +++ b/fba/networks/lemmy.py @@ -49,23 +49,23 @@ def fetch_peers(domain: str) -> list: (config.get("connection_timeout"), config.get("read_timeout")) ) - # DEBUG: print(f"DEBUG: data['{type(data)}']='{data}'") + # DEBUG: print(f"DEBUG: data[]='{type(data)}'") if "error_message" in data: print("WARNING: Could not reach any JSON API:", domain) - instances.update_last_error(domain, data) + instances.set_last_error(domain, data) elif "federated_instances" in data["json"]: # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") peers = peers + federation.add_peers(data["json"]["federated_instances"]) # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) - instances.update_last_error(domain, data) + instances.set_last_error(domain, data) except BaseException as exception: print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) + instances.set_total_peers(domain, peers) # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index bcbfe9d..3f2de7c 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -84,7 +84,7 @@ def fetch_blocks_from_about(domain: str) -> dict: except BaseException as exception: print("ERROR: Cannot fetch from domain:", domain, exception) - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) break blocklist = { @@ -174,13 +174,14 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): (config.get("connection_timeout"), config.get("read_timeout")) ) + # DEBUG: print(f"DEBUG: data[]='{type(data)}'") if "error_message" in data: # DEBUG: print(f"DEBUG: Was not able to fetch domain_blocks from domain='{domain}': status_code='{data['status_code']}',error_message='{data['error_message']}'") - instances.update_last_error(domain, data) + instances.set_last_error(domain, data) return elif "json" in data and "error" in data["json"]: print(f"WARNING: JSON API returned error message: '{data['json']['error']}'") - instances.update_last_error(domain, data) + instances.set_last_error(domain, data) return else: # Getting blocklist diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index e1e7360..5cf1b56 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -71,11 +71,11 @@ def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") if "error_message" in fetched: print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") - instances.update_last_error(domain, fetched) + instances.set_last_error(domain, fetched) break elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - instances.update_last_error(domain, fetched["json"]["error"]["message"]) + instances.set_last_error(domain, fetched["json"]["error"]["message"]) break rows = fetched["json"] @@ -117,7 +117,7 @@ def fetch_peers(domain: str) -> list: break # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) + instances.set_total_peers(domain, peers) # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") return peers @@ -174,11 +174,11 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") if "error_message" in fetched: print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") - instances.update_last_error(domain, fetched) + instances.set_last_error(domain, fetched) break elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - instances.update_last_error(domain, fetched["json"]["error"]["message"]) + instances.set_last_error(domain, fetched["json"]["error"]["message"]) break rows = fetched["json"] @@ -213,7 +213,7 @@ def fetch_blocks(domain: str) -> dict: except network.exceptions as exception: print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) offset = 0 break @@ -241,11 +241,11 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") if "error_message" in fetched: print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") - instances.update_last_error(domain, fetched) + instances.set_last_error(domain, fetched) break elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - instances.update_last_error(domain, fetched["json"]["error"]["message"]) + instances.set_last_error(domain, fetched["json"]["error"]["message"]) break rows = fetched["json"] @@ -279,7 +279,7 @@ def fetch_blocks(domain: str) -> dict: except network.exceptions as exception: print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) offset = 0 break diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index 9949334..c15ec21 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -51,7 +51,7 @@ def fetch_peers(domain: str) -> list: (config.get("connection_timeout"), config.get("read_timeout")) ) - # DEBUG: print(f"DEBUG: data['{type(data)}']='{data}'") + # DEBUG: print(f"DEBUG: data[]='{type(data)}'") if "error_message" not in data: # DEBUG: print(f"DEBUG: Success, data[json]()={len(data['json'])}") if "data" in data["json"]: @@ -79,7 +79,7 @@ def fetch_peers(domain: str) -> list: break # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) + instances.set_total_peers(domain, peers) # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") return peers diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index 58c7301..05d8181 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -69,12 +69,10 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): print(f"WARNING: rows()={len(rows['metadata'])} does not have key 'federation', domain='{domain}'") return - # DEBUG: print("DEBUG: Updating nodeinfo:", domain) - instances.update_last_nodeinfo(domain) - data = rows["metadata"]["federation"] found = False + # DEBUG: print(f"DEBUG: data[]='{type(data)}'") if "mrf_simple" in data: # DEBUG: print("DEBUG: Found mrf_simple:", domain) found = True @@ -516,7 +514,7 @@ def fetch_blocks_from_about(domain: str) -> dict: except BaseException as exception: print("ERROR: Cannot fetch from domain:", domain, exception) - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) break blocklist = { -- 2.39.5