From ca57258e9e940dfc2f2bcae094f9cefb4c6de469 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Fri, 9 Jun 2023 20:42:53 +0200 Subject: [PATCH] Continued: - ops, double commented out - also peertube may need debugging later on --- fba/boot.py | 12 +-- fba/cache.py | 14 ++- fba/csrf.py | 16 ++-- fba/fba.py | 68 +++++++------- fba/federation.py | 188 +++++++++++++++++++------------------- fba/helpers/dicts.py | 8 +- fba/helpers/tidyup.py | 8 +- fba/locking.py | 16 ++-- fba/network.py | 32 +++---- fba/networks/friendica.py | 2 +- fba/networks/peertube.py | 24 ++--- 11 files changed, 199 insertions(+), 189 deletions(-) diff --git a/fba/boot.py b/fba/boot.py index 0ff07f3..cb3ccc9 100644 --- a/fba/boot.py +++ b/fba/boot.py @@ -23,10 +23,10 @@ from fba import locking _PARSER = None def init_parser(): - # DEBUG: # DEBUG: print("DEBUG: init_parser(): CALLED!") + # DEBUG: print("DEBUG: init_parser(): CALLED!") global _PARSER - # DEBUG: # DEBUG: print("DEBUG: Initializing parser ...") + # DEBUG: print("DEBUG: Initializing parser ...") _PARSER = argparse.ArgumentParser( description="Fetches block reasons from the fediverse", epilog="Please note that some commands have optional arguments, you may want to try fba.py --help to find them out.", @@ -99,14 +99,14 @@ def init_parser(): parser.add_argument("--single", action="store_true", help="Only fetch given instance.") parser.set_defaults(command=commands.fetch_instances) - # DEBUG: # DEBUG: print("DEBUG: init_parser(): EXIT!") + # DEBUG: print("DEBUG: init_parser(): EXIT!") def run_command(): - # DEBUG: # DEBUG: print("DEBUG: run_command(): CALLED!") + # DEBUG: print("DEBUG: run_command(): CALLED!") args = _PARSER.parse_args() - # DEBUG: # DEBUG: print(f"DEBUG: args[{type(args)}]={args}") + # DEBUG: print(f"DEBUG: args[{type(args)}]={args}") status = args.command(args) - # DEBUG: # DEBUG: print("DEBUG: status={status} - EXIT!") + # DEBUG: print("DEBUG: status={status} - EXIT!") return status if isinstance(status, int) else 0 def shutdown(): diff --git a/fba/cache.py b/fba/cache.py index 6fac6ce..8645781 100644 --- a/fba/cache.py +++ b/fba/cache.py @@ -32,8 +32,8 @@ def set_all(key: str, rows: list, value: any): for sub in rows: # DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}'") - if isinstance(sub, tuple): + # DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}',value[]='{type(value)}'") _cache[key][sub[0]] = value else: print(f"WARNING: Unsupported type sub[]='{type(sub)}'") @@ -41,23 +41,29 @@ def set_all(key: str, rows: list, value: any): # DEBUG: print("DEBUG: EXIT!") def set_sub_key(key: str, sub: str, value: any): + # DEBUG: print(f"DEBUG: key='{key}',sub='{sub}',value[]='{type(value)}' - CALLED!") if not isinstance(key, str): raise ValueError("Parameter key[]='{type(key)}' is not 'str'") elif not isinstance(sub, str): raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'") elif not key_exists(key): - print(f"WARNING: Bad method call, key='{key}' is not initialized yet.") raise Exception(f"Cache for key='{key}' is not initialized, but function invoked") + # DEBUG: print(f"DEBUG: Setting key='{key}',sub='{sub}',value[]='{type(value)}' ...") _cache[key][sub] = value + # DEBUG: print("DEBUG: EXIT!") + def sub_key_exists(key: str, sub: str) -> bool: + # DEBUG: print(f"DEBUG: key='{key}',sub='{sub}' - CALLED!") if not isinstance(key, str): raise ValueError("Parameter key[]='{type(key)}' is not 'str'") elif not isinstance(sub, str): raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'") elif not key_exists(key): - print(f"WARNING: Bad method call, key='{key}' is not initialized yet.") raise Exception(f"Cache for key='{key}' is not initialized, but function invoked") - return sub in _cache[key] + exists = sub in _cache[key] + + # DEBUG: print(f"DEBUG: exists='{exists}' - EXIT!") + return exists diff --git a/fba/csrf.py b/fba/csrf.py index fb0bec3..8a11dad 100644 --- a/fba/csrf.py +++ b/fba/csrf.py @@ -22,7 +22,7 @@ from fba import config from fba import network def determine(domain: str, headers: dict) -> dict: - print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -35,31 +35,31 @@ def determine(domain: str, headers: dict) -> dict: try: # Fetch / to check for meta tag indicating csrf - print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...") + # DEBUG: print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...") response = reqto.get( f"https://{domain}/", headers=network.web_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) - print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}") + # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}") if response.ok and len(response.text) > 0: meta = bs4.BeautifulSoup( response.text, "html.parser" ) - print(f"DEBUG: meta[]='{type(meta)}'") + # DEBUG: print(f"DEBUG: meta[]='{type(meta)}'") tag = meta.find("meta", attrs={"name": "csrf-token"}) - print(f"DEBUG: tag={tag}") + # DEBUG: print(f"DEBUG: tag={tag}") csrf = tag["content"] - print(f"DEBUG: Adding CSRF token='{csrf}' for domain='{domain}'") + # DEBUG: print(f"DEBUG: Adding CSRF token='{csrf}' for domain='{domain}'") reqheaders["X-CSRF-Token"] = csrf except BaseException as exception: - print(f"DEBUG: No CSRF token found, using normal headers: domain='{domain}',exception[{type(exception)}]={exception}") + # DEBUG: print(f"DEBUG: No CSRF token found, using normal headers: domain='{domain}',exception[{type(exception)}]={exception}") pass - print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!") + # DEBUG: print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!") return reqheaders diff --git a/fba/fba.py b/fba/fba.py index a89d20f..1dada8c 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -50,11 +50,11 @@ patterns = [ ##### Other functions ##### def is_primitive(var: any) -> bool: - print(f"DEBUG: var[]='{type(var)}' - CALLED!") + # DEBUG: print(f"DEBUG: var[]='{type(var)}' - CALLED!") return type(var) in {int, str, float, bool} or var is None def remove_version(software: str) -> str: - print(f"DEBUG: software='{software}' - CALLED!") + # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") if not "." in software and " " not in software: print(f"WARNING: software='{software}' does not contain a version number.") return software @@ -67,7 +67,7 @@ def remove_version(software: str) -> str: elif " - " in software: temp = software.split(" - ")[0] - print(f"DEBUG: software='{software}'") + # DEBUG: print(f"DEBUG: software='{software}'") version = None if " " in software: version = temp.split(" ")[-1] @@ -76,39 +76,39 @@ def remove_version(software: str) -> str: elif "-" in software: version = temp.split("-")[-1] else: - print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") + # DEBUG: print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'") return software match = None - print(f"DEBUG: Checking {len(patterns)} patterns ...") + # DEBUG: print(f"DEBUG: Checking {len(patterns)} patterns ...") for pattern in patterns: # Run match() match = pattern.match(version) - print(f"DEBUG: match[]={type(match)}") + # DEBUG: print(f"DEBUG: match[]={type(match)}") if isinstance(match, re.Match): - print(f"DEBUG: version='{version}' is matching pattern='{pattern}'") + # DEBUG: print(f"DEBUG: version='{version}' is matching pattern='{pattern}'") break - print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") + # DEBUG: print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'") if not isinstance(match, re.Match): print(f"WARNING: version='{version}' does not match regex, leaving software='{software}' untouched.") return software - print(f"DEBUG: Found valid version number: '{version}', removing it ...") + # DEBUG: print(f"DEBUG: Found valid version number: '{version}', removing it ...") end = len(temp) - len(version) - 1 - print(f"DEBUG: end[{type(end)}]={end}") + # DEBUG: print(f"DEBUG: end[{type(end)}]={end}") software = temp[0:end].strip() if " version" in software: - print(f"DEBUG: software='{software}' contains word ' version'") + # DEBUG: print(f"DEBUG: software='{software}' contains word ' version'") software = strip_until(software, " version") - print(f"DEBUG: software='{software}' - EXIT!") + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") return software def strip_powered_by(software: str) -> str: - print(f"DEBUG: software='{software}' - CALLED!") + # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": @@ -118,18 +118,18 @@ def strip_powered_by(software: str) -> str: return software start = software.find("powered by ") - print(f"DEBUG: start[{type(start)}]='{start}'") + # DEBUG: print(f"DEBUG: start[{type(start)}]='{start}'") software = software[start + 11:].strip() - print(f"DEBUG: software='{software}'") + # DEBUG: print(f"DEBUG: software='{software}'") software = strip_until(software, " - ") - print(f"DEBUG: software='{software}' - EXIT!") + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") return software def strip_hosted_on(software: str) -> str: - print(f"DEBUG: software='{software}' - CALLED!") + # DEBUG: print(f"DEBUG: software='{software}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": @@ -139,18 +139,18 @@ def strip_hosted_on(software: str) -> str: return software end = software.find("hosted on ") - print(f"DEBUG: end[{type(end)}]='{end}'") + # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") software = software[0, end].strip() - print(f"DEBUG: software='{software}'") + # DEBUG: print(f"DEBUG: software='{software}'") software = strip_until(software, " - ") - print(f"DEBUG: software='{software}' - EXIT!") + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") return software def strip_until(software: str, until: str) -> str: - print(f"DEBUG: software='{software}',until='{until}' - CALLED!") + # DEBUG: print(f"DEBUG: software='{software}',until='{until}' - CALLED!") if not isinstance(software, str): raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'") elif software == "": @@ -166,11 +166,11 @@ def strip_until(software: str, until: str) -> str: # Next, strip until part end = software.find(until) - print(f"DEBUG: end[{type(end)}]='{end}'") + # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'") if end > 0: software = software[0:end].strip() - print(f"DEBUG: software='{software}' - EXIT!") + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") return software def remove_pending_error(domain: str): @@ -186,7 +186,7 @@ def remove_pending_error(domain: str): except: pass - print("DEBUG: EXIT!") + # DEBUG: print("DEBUG: EXIT!") def get_hash(domain: str) -> str: if not isinstance(domain, str): @@ -197,21 +197,21 @@ def get_hash(domain: str) -> str: return hashlib.sha256(domain.encode("utf-8")).hexdigest() def log_error(domain: str, response: requests.models.Response): - print("DEBUG: domain,response[]:", domain, type(response)) + # DEBUG: print("DEBUG: domain,response[]:", domain, type(response)) if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") elif config.get("write_error_log").lower() != "true": - print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!") + # DEBUG: print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!") return try: - print("DEBUG: BEFORE response[]:", type(response)) + # DEBUG: print("DEBUG: BEFORE response[]:", type(response)) if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError): response = f"response[{type(response)}]='{str(response)}'" - print("DEBUG: AFTER response[]:", type(response)) + # DEBUG: print("DEBUG: AFTER response[]:", type(response)) if isinstance(response, str): cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[ domain, @@ -227,16 +227,16 @@ def log_error(domain: str, response: requests.models.Response): ]) # Cleanup old entries - print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") + # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})") cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")]) except BaseException as exception: print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") sys.exit(255) - print("DEBUG: EXIT!") + # DEBUG: print("DEBUG: EXIT!") def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Response: - print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!") + # DEBUG: print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!") if not isinstance(url, str): raise ValueError(f"Parameter url[]='{type(url)}' is not 'str'") elif url == "": @@ -246,15 +246,15 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon elif not isinstance(timeout, tuple): raise ValueError(f"Parameter timeout[]='{type(timeout)}' is not 'tuple'") - print(f"DEBUG: Parsing url='{url}'") + # DEBUG: print(f"DEBUG: Parsing url='{url}'") components = urlparse(url) # Invoke other function, avoid trailing ? - print(f"DEBUG: components[{type(components)}]={components}") + # DEBUG: print(f"DEBUG: components[{type(components)}]={components}") if components.query != "": response = network.fetch_response(components.hostname, f"{components.path}?{components.query}", headers, timeout) else: response = network.fetch_response(components.hostname, f"{components.path}", headers, timeout) - print(f"DEBUG: response[]='{type(response)}' - EXXIT!") + # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response diff --git a/fba/federation.py b/fba/federation.py index b7e86a2..acc58e2 100644 --- a/fba/federation.py +++ b/fba/federation.py @@ -43,7 +43,7 @@ nodeinfo_identifier = [ ] def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None): - print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -51,9 +51,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: elif not isinstance(origin, str) and origin is not None: raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'") elif software is None: - print(f"DEBUG: software for domain='{domain}' is not set, determining ...") + # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...") software = determine_software(domain, path) - print(f"DEBUG: Determined software='{software}' for domain='{domain}'") + # DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'") elif not isinstance(software, str): raise ValueError(f"Parameter software[]={type(software)} is not 'str'") elif not isinstance(script, str): @@ -62,17 +62,17 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: raise ValueError("Parameter 'domain' is empty") if not instances.is_registered(domain): - print("DEBUG: Adding new domain:", domain, origin) + # DEBUG: print("DEBUG: Adding new domain:", domain, origin) instances.add(domain, origin, script, path) - print("DEBUG: Fetching instances for domain:", domain, software) + # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software) peerlist = fetch_peers(domain, software) if peerlist is None: print("ERROR: Cannot fetch peers:", domain) return elif instances.has_pending_instance_data(domain): - print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") + # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...") instances.update_data(domain) print(f"INFO: Checking {len(peerlist)} instances from {domain} ...") @@ -81,9 +81,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: # Skip "None" types as tidup() cannot parse them continue - print(f"DEBUG: instance='{instance}' - BEFORE") + # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE") instance = tidyup.domain(instance) - print(f"DEBUG: instance='{instance}' - AFTER") + # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER") if instance == "": print("WARNING: Empty instance after tidyup.domain(), domain:", domain) @@ -92,22 +92,22 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path: print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'") continue elif blacklist.is_blacklisted(instance): - print("DEBUG: instance is blacklisted:", instance) + # DEBUG: print("DEBUG: instance is blacklisted:", instance) continue - print("DEBUG: Handling instance:", instance) + # DEBUG: print("DEBUG: Handling instance:", instance) try: if not instances.is_registered(instance): - print("DEBUG: Adding new instance:", instance, domain) + # DEBUG: print("DEBUG: Adding new instance:", instance, domain) instances.add(instance, domain, script) except BaseException as exception: print(f"ERROR: instance='{instance}',exception[{type(exception)}]:'{str(exception)}'") continue - print("DEBUG: EXIT!") + # DEBUG: print("DEBUG: EXIT!") def fetch_peers(domain: str, software: str) -> list: - print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") + # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -116,57 +116,57 @@ def fetch_peers(domain: str, software: str) -> list: raise ValueError(f"software[]={type(software)} is not 'str'") if software == "misskey": - print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") + # DEBUG: print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...") return misskey.fetch_peers(domain) elif software == "lemmy": - print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...") + # DEBUG: print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...") return lemmy.fetch_peers(domain) elif software == "peertube": - print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...") + # DEBUG: print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...") return peertube.fetch_peers(domain) - print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") + # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...") peers = list() response = network.fetch_response(domain, "/api/v1/instance/peers", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - print(f"DEBUG: response[]='{type(response)}'") + # DEBUG: print(f"DEBUG: response[]='{type(response)}'") data = network.json_from_response(response) - print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if not response.ok or response.status_code >= 400: - print("DEBUG: Was not able to fetch peers, trying alternative ...") + # DEBUG: print("DEBUG: Was not able to fetch peers, trying alternative ...") response = network.fetch_response(domain, "/api/v3/site", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = network.json_from_response(response) - print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if not response.ok or response.status_code >= 400: print("WARNING: Could not reach any JSON API:", domain) instances.update_last_error(domain, response) elif response.ok and isinstance(data, list): - print(f"DEBUG: domain='{domain}' returned a list: '{data}'") + # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'") sys.exit(255) elif "federated_instances" in data: - print(f"DEBUG: Found federated_instances for domain='{domain}'") + # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'") peers = peers + add_peers(data["federated_instances"]) - print("DEBUG: Added instance(s) to peers") + # DEBUG: print("DEBUG: Added instance(s) to peers") else: print("WARNING: JSON response does not contain 'federated_instances':", domain) instances.update_last_error(domain, response) else: - print("DEBUG: Querying API was successful:", domain, len(data)) + # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data)) peers = data - print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_data("total_peers", domain, len(peers)) - print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") instances.update_last_instance_fetch(domain) - print("DEBUG: Returning peers[]:", type(peers)) + # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) return peers def fetch_nodeinfo(domain: str, path: str = None) -> list: - print(f"DEBUG: domain='{domain}',path={path} - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -174,12 +174,12 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: elif not isinstance(path, str) and path is not None: raise ValueError(f"Parameter path[]={type(path)} is not 'str'") - print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") + # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...") nodeinfo = fetch_wellknown_nodeinfo(domain) - print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}") + # DEBUG: print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}") if len(nodeinfo) > 0: - print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!") + # DEBUG: print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!") return nodeinfo request_paths = [ @@ -193,16 +193,16 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: for request in request_paths: if path is not None and path != "" and path != request: - print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") + # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!") continue - print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...") + # DEBUG: print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...") response = network.fetch_response(domain, request, network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) data = network.json_from_response(response) - print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if response.ok and isinstance(data, dict): - print("DEBUG: Success:", request) + # DEBUG: print("DEBUG: Success:", request) instances.set_data("detection_mode", domain, "STATIC_CHECK") instances.set_data("nodeinfo_url" , domain, request) break @@ -214,36 +214,36 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: instances.update_last_error(domain, response) continue - print(f"DEBUG: data()={len(data)} - EXIT!") + # DEBUG: print(f"DEBUG: data()={len(data)} - EXIT!") return data def fetch_wellknown_nodeinfo(domain: str) -> list: - print(f"DEBUG: domain='{domain}' - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - print("DEBUG: Fetching .well-known info for domain:", domain) + # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain) response = network.fetch_response(domain, "/.well-known/nodeinfo", network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout"))) data = network.json_from_response(response) - print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) + # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data)) if response.ok and isinstance(data, dict): nodeinfo = data - print("DEBUG: Found entries:", len(nodeinfo), domain) + # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain) if "links" in nodeinfo: - print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) + # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"])) for link in nodeinfo["links"]: - print("DEBUG: rel,href:", link["rel"], link["href"]) + # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"]) if link["rel"] in nodeinfo_identifier: - print("DEBUG: Fetching nodeinfo from:", link["href"]) + # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"]) response = fba.fetch_url(link["href"], network.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))) data = network.json_from_response(response) - print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) + # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code) if response.ok and isinstance(data, dict): - print("DEBUG: Found JSON nodeinfo():", len(data)) + # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data)) instances.set_data("detection_mode", domain, "AUTO_DISCOVERY") instances.set_data("nodeinfo_url" , domain, link["href"]) break @@ -252,11 +252,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: else: print("WARNING: nodeinfo does not contain 'links':", domain) - print("DEBUG: Returning data[]:", type(data)) + # DEBUG: print("DEBUG: Returning data[]:", type(data)) return data def fetch_generator_from_path(domain: str, path: str = "/") -> str: - print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") + # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -266,62 +266,62 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: elif path == "": raise ValueError("Parameter 'path' is empty") - print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!") software = None - print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") + # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") response = network.fetch_response(domain, path, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) - print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) + # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text)) if response.ok and response.status_code < 300 and len(response.text) > 0: - print("DEBUG: Search for :", domain) + # DEBUG: print("DEBUG: Search for :", domain) doc = bs4.BeautifulSoup(response.text, "html.parser") - print("DEBUG: doc[]:", type(doc)) + # DEBUG: print("DEBUG: doc[]:", type(doc)) generator = doc.find("meta", {"name": "generator"}) site_name = doc.find("meta", {"property": "og:site_name"}) - print(f"DEBUG: generator='{generator}',site_name='{site_name}'") + # DEBUG: print(f"DEBUG: generator='{generator}',site_name='{site_name}'") if isinstance(generator, bs4.element.Tag): - print("DEBUG: Found generator meta tag:", domain) + # DEBUG: print("DEBUG: Found generator meta tag:", domain) software = tidyup.domain(generator.get("content")) print(f"INFO: domain='{domain}' is generated by '{software}'") instances.set_data("detection_mode", domain, "GENERATOR") fba.remove_pending_error(domain) elif isinstance(site_name, bs4.element.Tag): - print("DEBUG: Found property=og:site_name:", domain) + # DEBUG: print("DEBUG: Found property=og:site_name:", domain) sofware = tidyup.domain(site_name.get("content")) print(f"INFO: domain='{domain}' has og:site_name='{software}'") instances.set_data("detection_mode", domain, "SITE_NAME") fba.remove_pending_error(domain) - print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]={type(software)}") if isinstance(software, str) and software == "": - print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") + # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'") software = None elif isinstance(software, str) and ("." in software or " " in software): - print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") + # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") software = fba.remove_version(software) - print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]={type(software)}") if isinstance(software, str) and " powered by " in software: - print(f"DEBUG: software='{software}' has 'powered by' in it") + # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = fba.remove_version(fba.strip_powered_by(software)) elif isinstance(software, str) and " hosted on " in software: - print(f"DEBUG: software='{software}' has 'hosted on' in it") + # DEBUG: print(f"DEBUG: software='{software}' has 'hosted on' in it") software = fba.remove_version(fba.strip_hosted_on(software)) elif isinstance(software, str) and " by " in software: - print(f"DEBUG: software='{software}' has ' by ' in it") + # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") software = fba.strip_until(software, " by ") elif isinstance(software, str) and " see " in software: - print(f"DEBUG: software='{software}' has ' see ' in it") + # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") software = fba.strip_until(software, " see ") - print(f"DEBUG: software='{software}' - EXIT!") + # DEBUG: print(f"DEBUG: software='{software}' - EXIT!") return software def determine_software(domain: str, path: str = None) -> str: - print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") + # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -329,18 +329,18 @@ def determine_software(domain: str, path: str = None) -> str: elif not isinstance(path, str) and path is not None: raise ValueError(f"Parameter path[]={type(path)} is not 'str'") - print("DEBUG: Determining software for domain,path:", domain, path) + # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path) software = None - print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") + # DEBUG: print(f"DEBUG: Fetching nodeinfo from '{domain}' ...") data = fetch_nodeinfo(domain, path) - print("DEBUG: data[]:", type(data)) + # DEBUG: print("DEBUG: data[]:", type(data)) if not isinstance(data, dict) or len(data) == 0: - print("DEBUG: Could not determine software type:", domain) + # DEBUG: print("DEBUG: Could not determine software type:", domain) return fetch_generator_from_path(domain) - print("DEBUG: data():", len(data), data) + # DEBUG: print("DEBUG: data():", len(data), data) if "status" in data and data["status"] == "error" and "message" in data: print("WARNING: JSON response is an error:", data["message"]) instances.update_last_error(domain, data["message"]) @@ -350,23 +350,23 @@ def determine_software(domain: str, path: str = None) -> str: instances.update_last_error(domain, data["message"]) return fetch_generator_from_path(domain) elif "software" not in data or "name" not in data["software"]: - print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") + # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...") software = fetch_generator_from_path(domain) - print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") + # DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!") return software software = tidyup.domain(data["software"]["name"]) - print("DEBUG: sofware after tidyup.domain():", software) + # DEBUG: print("DEBUG: sofware after tidyup.domain():", software) if software in ["akkoma", "rebased"]: - print("DEBUG: Setting pleroma:", domain, software) + # DEBUG: print("DEBUG: Setting pleroma:", domain, software) software = "pleroma" elif software in ["hometown", "ecko"]: - print("DEBUG: Setting mastodon:", domain, software) + # DEBUG: print("DEBUG: Setting mastodon:", domain, software) software = "mastodon" elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]: - print("DEBUG: Setting misskey:", domain, software) + # DEBUG: print("DEBUG: Setting misskey:", domain, software) software = "misskey" elif software.find("/") > 0: print("WARNING: Spliting of slash:", software) @@ -375,38 +375,38 @@ def determine_software(domain: str, path: str = None) -> str: print("WARNING: Spliting of pipe:", software) software = tidyup.domain(software.split("|")[0]) elif "powered by" in software: - print(f"DEBUG: software='{software}' has 'powered by' in it") + # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = fba.strip_powered_by(software) elif isinstance(software, str) and " by " in software: - print(f"DEBUG: software='{software}' has ' by ' in it") + # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it") software = fba.strip_until(software, " by ") elif isinstance(software, str) and " see " in software: - print(f"DEBUG: software='{software}' has ' see ' in it") + # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it") software = fba.strip_until(software, " see ") - print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]={type(software)}") if software == "": print("WARNING: tidyup.domain() left no software name behind:", domain) software = None - print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]={type(software)}") if str(software) == "": - print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") + # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...") software = fetch_generator_from_path(domain) elif len(str(software)) > 0 and ("." in software or " " in software): - print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") + # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...") software = fba.remove_version(software) - print(f"DEBUG: software[]={type(software)}") + # DEBUG: print(f"DEBUG: software[]={type(software)}") if isinstance(software, str) and "powered by" in software: - print(f"DEBUG: software='{software}' has 'powered by' in it") + # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it") software = fba.remove_version(fba.strip_powered_by(software)) - print("DEBUG: Returning domain,software:", domain, software) + # DEBUG: print("DEBUG: Returning domain,software:", domain, software) return software def find_domains(tag: bs4.element.Tag) -> list: - print(f"DEBUG: tag[]={type(tag)} - CALLED!") + # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!") if not isinstance(tag, bs4.element.Tag): raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag") elif len(tag.select("tr")) == 0: @@ -414,21 +414,21 @@ def find_domains(tag: bs4.element.Tag) -> list: domains = list() for element in tag.select("tr"): - print(f"DEBUG: element[]={type(element)}") + # DEBUG: print(f"DEBUG: element[]={type(element)}") if not element.find("td"): - print("DEBUG: Skipping element, no found") + # DEBUG: print("DEBUG: Skipping element, no found") continue domain = tidyup.domain(element.find("td").text) reason = tidyup.reason(element.findAll("td")[1].text) - print(f"DEBUG: domain='{domain}',reason='{reason}'") + # DEBUG: print(f"DEBUG: domain='{domain}',reason='{reason}'") if blacklist.is_blacklisted(domain): print(f"WARNING: domain='{domain}' is blacklisted - skipped!") continue elif domain == "gab.com/.ai, develop.gab.com": - print("DEBUG: Multiple domains detected in one row") + # DEBUG: print("DEBUG: Multiple domains detected in one row") domains.append({ "domain": "gab.com", "reason": reason, @@ -446,13 +446,13 @@ def find_domains(tag: bs4.element.Tag) -> list: print(f"WARNING: domain='{domain}' is not a valid domain - skipped!") continue - print(f"DEBUG: Adding domain='{domain}' ...") + # DEBUG: print(f"DEBUG: Adding domain='{domain}' ...") domains.append({ "domain": domain, "reason": reason, }) - print(f"DEBUG: domains()={len(domains)} - EXIT!") + # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!") return domains def add_peers(rows: dict) -> list: diff --git a/fba/helpers/dicts.py b/fba/helpers/dicts.py index 51843d4..6820913 100644 --- a/fba/helpers/dicts.py +++ b/fba/helpers/dicts.py @@ -14,7 +14,7 @@ # along with this program. If not, see . def has_key(lists: list, key: str, value: any) -> bool: - print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!") + # DEBUG: print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!") if not isinstance(lists, list): raise ValueError(f"Parameter lists[]='{type(lists)}' is not 'list'") elif not isinstance(key, str): @@ -23,9 +23,9 @@ def has_key(lists: list, key: str, value: any) -> bool: raise ValueError("Parameter 'key' is empty") has = False - print(f"DEBUG: Checking lists()={len(lists)} ...") + # DEBUG: print(f"DEBUG: Checking lists()={len(lists)} ...") for row in lists: - print(f"DEBUG: row['{type(row)}']={row}") + # DEBUG: print(f"DEBUG: row['{type(row)}']={row}") if not isinstance(row, dict): raise ValueError(f"row[]='{type(row)}' is not 'dict'") elif not key in row: @@ -34,5 +34,5 @@ def has_key(lists: list, key: str, value: any) -> bool: has = True break - print(f"DEBUG: has={has} - EXIT!") + # DEBUG: print(f"DEBUG: has={has} - EXIT!") return has diff --git a/fba/helpers/tidyup.py b/fba/helpers/tidyup.py index c0de2c8..d2d322f 100644 --- a/fba/helpers/tidyup.py +++ b/fba/helpers/tidyup.py @@ -16,7 +16,7 @@ import re def reason(string: str) -> str: - print(f"DEBUG: string='{string}' - CALLED!") + # DEBUG: print(f"DEBUG: string='{string}' - CALLED!") if not isinstance(string, str): raise ValueError(f"Parameter string[]={type(string)} is not 'str'") @@ -26,11 +26,11 @@ def reason(string: str) -> str: # Replace â with " string = re.sub("â", "\"", string) - print(f"DEBUG: string='{string}' - EXIT!") + # DEBUG: print(f"DEBUG: string='{string}' - EXIT!") return string def domain(string: str) -> str: - print(f"DEBUG: string='{string}' - CALLED!") + # DEBUG: print(f"DEBUG: string='{string}' - CALLED!") if not isinstance(string, str): raise ValueError(f"Parameter string[]={type(string)} is not 'str'") @@ -56,5 +56,5 @@ def domain(string: str) -> str: elif string.find("/users/"): string = string.split("/users/")[0] - print(f"DEBUG: string='{string}' - EXIT!") + # DEBUG: print(f"DEBUG: string='{string}' - EXIT!") return string diff --git a/fba/locking.py b/fba/locking.py index 132a8ca..ddcbf7c 100644 --- a/fba/locking.py +++ b/fba/locking.py @@ -25,21 +25,25 @@ LOCK = None def acquire(): global LOCK + print("DEBUG: CALLED!") + try: - # DEBUG: print(f"DEBUG: Acquiring lock: '{lockfile}'") + print(f"DEBUG: Acquiring lock: '{lockfile}'") LOCK = zc.lockfile.LockFile(lockfile) - # DEBUG: print("DEBUG: Lock obtained.") + print("DEBUG: Lock obtained.") except zc.lockfile.LockError: print(f"ERROR: Cannot aquire lock: '{lockfile}'") sys.exit(100) + print("DEBUG: EXIT!") + def release(): - # DEBUG: print("DEBUG: CALLED!") + print("DEBUG: CALLED!") if LOCK is not None: - # DEBUG: print("DEBUG: Releasing lock ...") + print("DEBUG: Releasing lock ...") LOCK.close() - # DEBUG: print(f"DEBUG: Deleting lockfile='{lockfile}' ...") + print(f"DEBUG: Deleting lockfile='{lockfile}' ...") os.remove(lockfile) - # DEBUG: print("DEBUG: EXIT!") + print("DEBUG: EXIT!") diff --git a/fba/network.py b/fba/network.py index 7329979..261d266 100644 --- a/fba/network.py +++ b/fba/network.py @@ -34,7 +34,7 @@ api_headers = { } def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict: - print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": @@ -46,13 +46,13 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = elif not isinstance(parameter, str): raise ValueError(f"parameter[]={type(parameter)} is not 'str'") - print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...") + # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...") headers = csrf.determine(domain, {**api_headers, **extra_headers}) data = {} try: - print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',parameter='{parameter}',extra_headers({len(extra_headers)})={extra_headers}") + # DEBUG: print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',parameter='{parameter}',extra_headers({len(extra_headers)})={extra_headers}") response = reqto.post( f"https://{domain}{path}", data=parameter, @@ -61,7 +61,7 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = ) data = json_from_response(response) - print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") + # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'") if not response.ok or response.status_code >= 400: print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'") instances.update_last_error(domain, response) @@ -69,11 +69,11 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = except BaseException as exception: print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(exception)}]:'{str(exception)}'") - print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") + # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}") return data def send_bot_post(domain: str, blocklist: dict): - print(f"DEBUG: domain={domain},blocklist()={len(blocklist)} - CALLED!") + # DEBUG: print(f"DEBUG: domain={domain},blocklist()={len(blocklist)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -88,9 +88,9 @@ def send_bot_post(domain: str, blocklist: dict): truncated = True blocklist = blocklist[0 : 19] - print(f"DEBUG: blocklist()={len(blocklist)}") + # DEBUG: print(f"DEBUG: blocklist()={len(blocklist)}") for block in blocklist: - print(f"DEBUG: block['{type(block)}']={block}") + # DEBUG: print(f"DEBUG: block['{type(block)}']={block}") if block["reason"] is None or block["reason"] == '': message = message + block["blocked"] + " with unspecified reason\n" else: @@ -118,7 +118,7 @@ def send_bot_post(domain: str, blocklist: dict): return True def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response: - print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") + # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": @@ -128,11 +128,11 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requ elif path == "": raise ValueError("Parameter 'path' is empty") - print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}',headers()='{len(headers)}' ...") + # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}',headers()='{len(headers)}' ...") headers = csrf.determine(domain, headers) try: - print(f"DEBUG: Sending GET request to '{domain}{path}' ...") + # DEBUG: print(f"DEBUG: Sending GET request to '{domain}{path}' ...") response = reqto.get( f"https://{domain}{path}", headers=headers, @@ -140,25 +140,25 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requ ) except requests.exceptions.ConnectionError as exception: - print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") + # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) raise exception - print(f"DEBUG: response[]='{type(response)}' - EXXIT!") + # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!") return response def json_from_response(response: requests.models.Response) -> list: - print(f"DEBUG: response[]={type(response)} - CALLED!") + # DEBUG: print(f"DEBUG: response[]={type(response)} - CALLED!") if not isinstance(response, requests.models.Response): raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'") data = list() if response.text.strip() != "": - print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...") + # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...") try: data = response.json() except json.decoder.JSONDecodeError: pass - print(f"DEBUG: data[]={type(data)} - EXIT!") + # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!") return data diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py index ceba93d..2b88863 100644 --- a/fba/networks/friendica.py +++ b/fba/networks/friendica.py @@ -40,7 +40,7 @@ def fetch_blocks(domain: str) -> dict: ).text, "html.parser", ) - print(f"DEBUG: doc[]='{type(doc)}'") + # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'") blocklist = doc.find(id="about_blocklist") diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index da84baf..5529b97 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -19,17 +19,17 @@ from fba import instances from fba import network def fetch_peers(domain: str) -> list: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!") + print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") + print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") peers = list() start = 0 for mode in ["followers", "following"]: - # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") + print(f"DEBUG: domain='{domain}',mode='{mode}'") while True: try: response = network.fetch_response( @@ -40,21 +40,21 @@ def fetch_peers(domain: str) -> list: ) data = network.json_from_response(response) - # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") + print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'") if response.ok and isinstance(data, dict): - # DEBUG: print("DEBUG: Success, data:", len(data)) + print("DEBUG: Success, data:", len(data)) if "data" in data: - # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).") + print(f"DEBUG: Found {len(data['data'])} record(s).") for record in data["data"]: - # DEBUG: print(f"DEBUG: record()={len(record)}") + print(f"DEBUG: record()={len(record)}") if mode in record and "host" in record[mode]: - # DEBUG: print(f"DEBUG: Found host={record[mode]['host']}, adding ...") + print(f"DEBUG: Found host={record[mode]['host']}, adding ...") peers.append(record[mode]["host"]) else: print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") if len(data["data"]) < 100: - # DEBUG: print("DEBUG: Reached end of JSON response:", domain) + print("DEBUG: Reached end of JSON response:", domain) break # Continue with next row @@ -63,11 +63,11 @@ def fetch_peers(domain: str) -> list: except BaseException as exception: print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'") - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_data("total_peers", domain, len(peers)) - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") + print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") instances.update_last_instance_fetch(domain) - # DEBUG: print("DEBUG: Returning peers[]:", type(peers)) + print("DEBUG: Returning peers[]:", type(peers)) return peers -- 2.39.5