Continued:

- ops, double commented out
- also peertube may need debugging later on

diff --git a/fba/boot.py b/fba/boot.py
index 0ff07f34506bea523c2afbb6fe4d7af1e5b98a87..cb3ccc9420227201df03f40d26ac5c9758c57f8b 100644 (file)
--- a/fba/boot.py
+++ b/fba/boot.py
@@ -23,10 +23,10 @@ from fba import locking
 _PARSER = None
 
 def init_parser():
-    # DEBUG: # DEBUG: print("DEBUG: init_parser(): CALLED!")
+    # DEBUG: print("DEBUG: init_parser(): CALLED!")
     global _PARSER
 
-    # DEBUG: # DEBUG: print("DEBUG: Initializing parser ...")
+    # DEBUG: print("DEBUG: Initializing parser ...")
     _PARSER = argparse.ArgumentParser(
         description="Fetches block reasons from the fediverse",
         epilog="Please note that some commands have optional arguments, you may want to try fba.py <command> --help to find them out.",
@@ -99,14 +99,14 @@ def init_parser():
     parser.add_argument("--single", action="store_true", help="Only fetch given instance.")
     parser.set_defaults(command=commands.fetch_instances)
 
-    # DEBUG: # DEBUG: print("DEBUG: init_parser(): EXIT!")
+    # DEBUG: print("DEBUG: init_parser(): EXIT!")
 
 def run_command():
-    # DEBUG: # DEBUG: print("DEBUG: run_command(): CALLED!")
+    # DEBUG: print("DEBUG: run_command(): CALLED!")
     args = _PARSER.parse_args()
-    # DEBUG: # DEBUG: print(f"DEBUG: args[{type(args)}]={args}")
+    # DEBUG: print(f"DEBUG: args[{type(args)}]={args}")
     status = args.command(args)
-    # DEBUG: # DEBUG: print("DEBUG: status={status} - EXIT!")
+    # DEBUG: print("DEBUG: status={status} - EXIT!")
     return status if isinstance(status, int) else 0
 
 def shutdown():

diff --git a/fba/cache.py b/fba/cache.py
index 6fac6ce47a9717e1d881e20000d672b9789ad14e..8645781dd96f655c647e7ca1b33cdb9bed98122a 100644 (file)
--- a/fba/cache.py
+++ b/fba/cache.py
@@ -32,8 +32,8 @@ def set_all(key: str, rows: list, value: any):
 
     for sub in rows:
         # DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}'")
-
         if isinstance(sub, tuple):
+            # DEBUG: print(f"DEBUG: Setting key='{key}',sub[{type(sub)}]='{sub}',value[]='{type(value)}'")
             _cache[key][sub[0]] = value
         else:
             print(f"WARNING: Unsupported type sub[]='{type(sub)}'")
@@ -41,23 +41,29 @@ def set_all(key: str, rows: list, value: any):
     # DEBUG: print("DEBUG: EXIT!")
 
 def set_sub_key(key: str, sub: str, value: any):
+    # DEBUG: print(f"DEBUG: key='{key}',sub='{sub}',value[]='{type(value)}' - CALLED!")
     if not isinstance(key, str):
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
     elif not isinstance(sub, str):
         raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'")
     elif not key_exists(key):
-        print(f"WARNING: Bad method call, key='{key}' is not initialized yet.")
         raise Exception(f"Cache for key='{key}' is not initialized, but function invoked")
 
+    # DEBUG: print(f"DEBUG: Setting key='{key}',sub='{sub}',value[]='{type(value)}' ...")
     _cache[key][sub] = value
 
+    # DEBUG: print("DEBUG: EXIT!")
+
 def sub_key_exists(key: str, sub: str) -> bool:
+    # DEBUG: print(f"DEBUG: key='{key}',sub='{sub}' - CALLED!")
     if not isinstance(key, str):
         raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
     elif not isinstance(sub, str):
         raise ValueError("Parameter sub[]='{type(sub)}' is not 'str'")
     elif not key_exists(key):
-        print(f"WARNING: Bad method call, key='{key}' is not initialized yet.")
         raise Exception(f"Cache for key='{key}' is not initialized, but function invoked")
 
-    return sub in _cache[key]
+    exists = sub in _cache[key]
+
+    # DEBUG: print(f"DEBUG: exists='{exists}' - EXIT!")
+    return exists

diff --git a/fba/csrf.py b/fba/csrf.py
index fb0bec31d905fa78ba6c755383e69c92f74350f9..8a11dadd39d96bcca72c7f179f20427e6aca980d 100644 (file)
--- a/fba/csrf.py
+++ b/fba/csrf.py
@@ -22,7 +22,7 @@ from fba import config
 from fba import network
 
 def determine(domain: str, headers: dict) -> dict:
-    print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}',headers()={len(headers)} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
@@ -35,31 +35,31 @@ def determine(domain: str, headers: dict) -> dict:
 
     try:
         # Fetch / to check for meta tag indicating csrf
-        print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...")
+        # DEBUG: print(f"DEBUG: Fetching / from domain='{domain}' for CSRF check ...")
         response = reqto.get(
             f"https://{domain}/",
             headers=network.web_headers,
             timeout=(config.get("connection_timeout"), config.get("read_timeout"))
         )
 
-        print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}")
+        # DEBUG: print(f"DEBUG: response.ok='{response.ok}',response.status_code={response.status_code},response.text()={len(response.text)}")
         if response.ok and len(response.text) > 0:
             meta = bs4.BeautifulSoup(
                 response.text,
                 "html.parser"
             )
-            print(f"DEBUG: meta[]='{type(meta)}'")
+            # DEBUG: print(f"DEBUG: meta[]='{type(meta)}'")
 
             tag = meta.find("meta", attrs={"name": "csrf-token"})
-            print(f"DEBUG: tag={tag}")
+            # DEBUG: print(f"DEBUG: tag={tag}")
 
             csrf = tag["content"]
-            print(f"DEBUG: Adding CSRF token='{csrf}' for domain='{domain}'")
+            # DEBUG: print(f"DEBUG: Adding CSRF token='{csrf}' for domain='{domain}'")
             reqheaders["X-CSRF-Token"] = csrf
 
     except BaseException as exception:
-        print(f"DEBUG: No CSRF token found, using normal headers: domain='{domain}',exception[{type(exception)}]={exception}")
+        # DEBUG: print(f"DEBUG: No CSRF token found, using normal headers: domain='{domain}',exception[{type(exception)}]={exception}")
         pass
 
-    print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!")
+    # DEBUG: print(f"DEBUG: reqheaders()={len(reqheaders)} - EXIT!")
     return reqheaders

diff --git a/fba/fba.py b/fba/fba.py
index a89d20f77b4ab1ad5f08343ef1fe4fa711cef36c..1dada8cbf21ab8c0607b4eed5554b0b52b7e26a5 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -50,11 +50,11 @@ patterns = [
 ##### Other functions #####
 
 def is_primitive(var: any) -> bool:
-    print(f"DEBUG: var[]='{type(var)}' - CALLED!")
+    # DEBUG: print(f"DEBUG: var[]='{type(var)}' - CALLED!")
     return type(var) in {int, str, float, bool} or var is None
 
 def remove_version(software: str) -> str:
-    print(f"DEBUG: software='{software}' - CALLED!")
+    # DEBUG: print(f"DEBUG: software='{software}' - CALLED!")
     if not "." in software and " " not in software:
         print(f"WARNING: software='{software}' does not contain a version number.")
         return software
@@ -67,7 +67,7 @@ def remove_version(software: str) -> str:
     elif " - " in software:
         temp = software.split(" - ")[0]
 
-    print(f"DEBUG: software='{software}'")
+    # DEBUG: print(f"DEBUG: software='{software}'")
     version = None
     if " " in software:
         version = temp.split(" ")[-1]
@@ -76,39 +76,39 @@ def remove_version(software: str) -> str:
     elif "-" in software:
         version = temp.split("-")[-1]
     else:
-        print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'")
+        # DEBUG: print(f"DEBUG: Was not able to find common seperator, returning untouched software='{software}'")
         return software
 
     match = None
-    print(f"DEBUG: Checking {len(patterns)} patterns ...")
+    # DEBUG: print(f"DEBUG: Checking {len(patterns)} patterns ...")
     for pattern in patterns:
         # Run match()
         match = pattern.match(version)
 
-        print(f"DEBUG: match[]={type(match)}")
+        # DEBUG: print(f"DEBUG: match[]={type(match)}")
         if isinstance(match, re.Match):
-            print(f"DEBUG: version='{version}' is matching pattern='{pattern}'")
+            # DEBUG: print(f"DEBUG: version='{version}' is matching pattern='{pattern}'")
             break
 
-    print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'")
+    # DEBUG: print(f"DEBUG: version[{type(version)}]='{version}',match='{match}'")
     if not isinstance(match, re.Match):
         print(f"WARNING: version='{version}' does not match regex, leaving software='{software}' untouched.")
         return software
 
-    print(f"DEBUG: Found valid version number: '{version}', removing it ...")
+    # DEBUG: print(f"DEBUG: Found valid version number: '{version}', removing it ...")
     end = len(temp) - len(version) - 1
 
-    print(f"DEBUG: end[{type(end)}]={end}")
+    # DEBUG: print(f"DEBUG: end[{type(end)}]={end}")
     software = temp[0:end].strip()
     if " version" in software:
-        print(f"DEBUG: software='{software}' contains word ' version'")
+        # DEBUG: print(f"DEBUG: software='{software}' contains word ' version'")
         software = strip_until(software, " version")
 
-    print(f"DEBUG: software='{software}' - EXIT!")
+    # DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
     return software
 
 def strip_powered_by(software: str) -> str:
-    print(f"DEBUG: software='{software}' - CALLED!")
+    # DEBUG: print(f"DEBUG: software='{software}' - CALLED!")
     if not isinstance(software, str):
         raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'")
     elif software == "":
@@ -118,18 +118,18 @@ def strip_powered_by(software: str) -> str:
         return software
 
     start = software.find("powered by ")
-    print(f"DEBUG: start[{type(start)}]='{start}'")
+    # DEBUG: print(f"DEBUG: start[{type(start)}]='{start}'")
 
     software = software[start + 11:].strip()
-    print(f"DEBUG: software='{software}'")
+    # DEBUG: print(f"DEBUG: software='{software}'")
 
     software = strip_until(software, " - ")
 
-    print(f"DEBUG: software='{software}' - EXIT!")
+    # DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
     return software
 
 def strip_hosted_on(software: str) -> str:
-    print(f"DEBUG: software='{software}' - CALLED!")
+    # DEBUG: print(f"DEBUG: software='{software}' - CALLED!")
     if not isinstance(software, str):
         raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'")
     elif software == "":
@@ -139,18 +139,18 @@ def strip_hosted_on(software: str) -> str:
         return software
 
     end = software.find("hosted on ")
-    print(f"DEBUG: end[{type(end)}]='{end}'")
+    # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'")
 
     software = software[0, end].strip()
-    print(f"DEBUG: software='{software}'")
+    # DEBUG: print(f"DEBUG: software='{software}'")
 
     software = strip_until(software, " - ")
 
-    print(f"DEBUG: software='{software}' - EXIT!")
+    # DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
     return software
 
 def strip_until(software: str, until: str) -> str:
-    print(f"DEBUG: software='{software}',until='{until}' - CALLED!")
+    # DEBUG: print(f"DEBUG: software='{software}',until='{until}' - CALLED!")
     if not isinstance(software, str):
         raise ValueError(f"Parameter software[]='{type(software)}' is not 'str'")
     elif software == "":
@@ -166,11 +166,11 @@ def strip_until(software: str, until: str) -> str:
     # Next, strip until part
     end = software.find(until)
 
-    print(f"DEBUG: end[{type(end)}]='{end}'")
+    # DEBUG: print(f"DEBUG: end[{type(end)}]='{end}'")
     if end > 0:
         software = software[0:end].strip()
 
-    print(f"DEBUG: software='{software}' - EXIT!")
+    # DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
     return software
 
 def remove_pending_error(domain: str):
@@ -186,7 +186,7 @@ def remove_pending_error(domain: str):
     except:
         pass
 
-    print("DEBUG: EXIT!")
+    # DEBUG: print("DEBUG: EXIT!")
 
 def get_hash(domain: str) -> str:
     if not isinstance(domain, str):
@@ -197,21 +197,21 @@ def get_hash(domain: str) -> str:
     return hashlib.sha256(domain.encode("utf-8")).hexdigest()
 
 def log_error(domain: str, response: requests.models.Response):
-    print("DEBUG: domain,response[]:", domain, type(response))
+    # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
     elif config.get("write_error_log").lower() != "true":
-        print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!")
+        # DEBUG: print(f"DEBUG: Writing to error_log is disabled in configuruation file - EXIT!")
         return
 
     try:
-        print("DEBUG: BEFORE response[]:", type(response))
+        # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
         if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
             response = f"response[{type(response)}]='{str(response)}'"
 
-        print("DEBUG: AFTER response[]:", type(response))
+        # DEBUG: print("DEBUG: AFTER response[]:", type(response))
         if isinstance(response, str):
             cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[
                 domain,
@@ -227,16 +227,16 @@ def log_error(domain: str, response: requests.models.Response):
             ])
 
         # Cleanup old entries
-        print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})")
+        # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})")
         cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")])
     except BaseException as exception:
         print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'")
         sys.exit(255)
 
-    print("DEBUG: EXIT!")
+    # DEBUG: print("DEBUG: EXIT!")
 
 def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Response:
-    print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!")
+    # DEBUG: print(f"DEBUG: url='{url}',headers()={len(headers)},timeout={timeout} - CALLED!")
     if not isinstance(url, str):
         raise ValueError(f"Parameter url[]='{type(url)}' is not 'str'")
     elif url == "":
@@ -246,15 +246,15 @@ def fetch_url(url: str, headers: dict, timeout: tuple) -> requests.models.Respon
     elif not isinstance(timeout, tuple):
         raise ValueError(f"Parameter timeout[]='{type(timeout)}' is not 'tuple'")
 
-    print(f"DEBUG: Parsing url='{url}'")
+    # DEBUG: print(f"DEBUG: Parsing url='{url}'")
     components = urlparse(url)
 
     # Invoke other function, avoid trailing ?
-    print(f"DEBUG: components[{type(components)}]={components}")
+    # DEBUG: print(f"DEBUG: components[{type(components)}]={components}")
     if components.query != "":
         response = network.fetch_response(components.hostname, f"{components.path}?{components.query}", headers, timeout)
     else:
         response = network.fetch_response(components.hostname, f"{components.path}", headers, timeout)
 
-    print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
+    # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
     return response

diff --git a/fba/federation.py b/fba/federation.py
index b7e86a21dd5593fa5b8dc38ea6bc82e490c9a11e..acc58e2d189df2a4601ec777fec22b0951404edf 100644 (file)
--- a/fba/federation.py
+++ b/fba/federation.py
@@ -43,7 +43,7 @@ nodeinfo_identifier = [
 ]
 
 def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None):
-    print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}',origin='{origin}',software='{software}',path='{path}' - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
@@ -51,9 +51,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
     elif not isinstance(origin, str) and origin is not None:
         raise ValueError(f"Parameter origin[]={type(origin)} is not 'str'")
     elif software is None:
-        print(f"DEBUG: software for domain='{domain}' is not set, determining ...")
+        # DEBUG: print(f"DEBUG: software for domain='{domain}' is not set, determining ...")
         software = determine_software(domain, path)
-        print(f"DEBUG: Determined software='{software}' for domain='{domain}'")
+        # DEBUG: print(f"DEBUG: Determined software='{software}' for domain='{domain}'")
     elif not isinstance(software, str):
         raise ValueError(f"Parameter software[]={type(software)} is not 'str'")
     elif not isinstance(script, str):
@@ -62,17 +62,17 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
         raise ValueError("Parameter 'domain' is empty")
 
     if not instances.is_registered(domain):
-        print("DEBUG: Adding new domain:", domain, origin)
+        # DEBUG: print("DEBUG: Adding new domain:", domain, origin)
         instances.add(domain, origin, script, path)
 
-    print("DEBUG: Fetching instances for domain:", domain, software)
+    # DEBUG: print("DEBUG: Fetching instances for domain:", domain, software)
     peerlist = fetch_peers(domain, software)
 
     if peerlist is None:
         print("ERROR: Cannot fetch peers:", domain)
         return
     elif instances.has_pending_instance_data(domain):
-        print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...")
+        # DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...")
         instances.update_data(domain)
 
     print(f"INFO: Checking {len(peerlist)} instances from {domain} ...")
@@ -81,9 +81,9 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
             # Skip "None" types as tidup() cannot parse them
             continue
 
-        print(f"DEBUG: instance='{instance}' - BEFORE")
+        # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE")
         instance = tidyup.domain(instance)
-        print(f"DEBUG: instance='{instance}' - AFTER")
+        # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER")
 
         if instance == "":
             print("WARNING: Empty instance after tidyup.domain(), domain:", domain)
@@ -92,22 +92,22 @@ def fetch_instances(domain: str, origin: str, software: str, script: str, path:
             print(f"WARNING: Bad instance='{instance}' from domain='{domain}',origin='{origin}',software='{software}'")
             continue
         elif blacklist.is_blacklisted(instance):
-            print("DEBUG: instance is blacklisted:", instance)
+            # DEBUG: print("DEBUG: instance is blacklisted:", instance)
             continue
 
-        print("DEBUG: Handling instance:", instance)
+        # DEBUG: print("DEBUG: Handling instance:", instance)
         try:
             if not instances.is_registered(instance):
-                print("DEBUG: Adding new instance:", instance, domain)
+                # DEBUG: print("DEBUG: Adding new instance:", instance, domain)
                 instances.add(instance, domain, script)
         except BaseException as exception:
             print(f"ERROR: instance='{instance}',exception[{type(exception)}]:'{str(exception)}'")
             continue
 
-    print("DEBUG: EXIT!")
+    # DEBUG: print("DEBUG: EXIT!")
 
 def fetch_peers(domain: str, software: str) -> list:
-    print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software={software} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
@@ -116,57 +116,57 @@ def fetch_peers(domain: str, software: str) -> list:
         raise ValueError(f"software[]={type(software)} is not 'str'")
 
     if software == "misskey":
-        print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...")
+        # DEBUG: print(f"DEBUG: Invoking misskey.fetch_peers({domain}) ...")
         return misskey.fetch_peers(domain)
     elif software == "lemmy":
-        print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...")
+        # DEBUG: print(f"DEBUG: Invoking lemmy.fetch_peers({domain}) ...")
         return lemmy.fetch_peers(domain)
     elif software == "peertube":
-        print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...")
+        # DEBUG: print(f"DEBUG: Invoking peertube.fetch_peers({domain}) ...")
         return peertube.fetch_peers(domain)
 
-    print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...")
+    # DEBUG: print(f"DEBUG: Fetching peers from '{domain}',software='{software}' ...")
     peers = list()
     response = network.fetch_response(domain, "/api/v1/instance/peers", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
-    print(f"DEBUG: response[]='{type(response)}'")
+    # DEBUG: print(f"DEBUG: response[]='{type(response)}'")
 
     data = network.json_from_response(response)
-    print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+    # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
 
     if not response.ok or response.status_code >= 400:
-        print("DEBUG: Was not able to fetch peers, trying alternative ...")
+        # DEBUG: print("DEBUG: Was not able to fetch peers, trying alternative ...")
         response = network.fetch_response(domain, "/api/v3/site", network.api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
         data = network.json_from_response(response)
-        print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+        # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
         if not response.ok or response.status_code >= 400:
             print("WARNING: Could not reach any JSON API:", domain)
             instances.update_last_error(domain, response)
         elif response.ok and isinstance(data, list):
-            print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
+            # DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
             sys.exit(255)
         elif "federated_instances" in data:
-            print(f"DEBUG: Found federated_instances for domain='{domain}'")
+            # DEBUG: print(f"DEBUG: Found federated_instances for domain='{domain}'")
             peers = peers + add_peers(data["federated_instances"])
-            print("DEBUG: Added instance(s) to peers")
+            # DEBUG: print("DEBUG: Added instance(s) to peers")
         else:
             print("WARNING: JSON response does not contain 'federated_instances':", domain)
             instances.update_last_error(domain, response)
     else:
-        print("DEBUG: Querying API was successful:", domain, len(data))
+        # DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
         peers = data
 
-    print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
+    # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
     instances.set_data("total_peers", domain, len(peers))
 
-    print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
+    # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
     instances.update_last_instance_fetch(domain)
 
-    print("DEBUG: Returning peers[]:", type(peers))
+    # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
     return peers
 
 def fetch_nodeinfo(domain: str, path: str = None) -> list:
-    print(f"DEBUG: domain='{domain}',path={path} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}',path={path} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
@@ -174,12 +174,12 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list:
     elif not isinstance(path, str) and path is not None:
         raise ValueError(f"Parameter path[]={type(path)} is not 'str'")
 
-    print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...")
+    # DEBUG: print(f"DEBUG: Fetching nodeinfo from domain='{domain}' ...")
     nodeinfo = fetch_wellknown_nodeinfo(domain)
 
-    print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}")
+    # DEBUG: print(f"DEBUG: nodeinfo({len(nodeinfo)})={nodeinfo}")
     if len(nodeinfo) > 0:
-        print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!")
+        # DEBUG: print("DEBUG: nodeinfo()={len(nodeinfo))} - EXIT!")
         return nodeinfo
 
     request_paths = [
@@ -193,16 +193,16 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list:
 
     for request in request_paths:
         if path is not None and path != "" and path != request:
-            print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!")
+            # DEBUG: print(f"DEBUG: path='{path}' does not match request='{request}' - SKIPPED!")
             continue
 
-        print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...")
+        # DEBUG: print(f"DEBUG: Fetching request='{request}' from domain='{domain}' ...")
         response = network.fetch_response(domain, request, network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
 
         data = network.json_from_response(response)
-        print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+        # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
         if response.ok and isinstance(data, dict):
-            print("DEBUG: Success:", request)
+            # DEBUG: print("DEBUG: Success:", request)
             instances.set_data("detection_mode", domain, "STATIC_CHECK")
             instances.set_data("nodeinfo_url"  , domain, request)
             break
@@ -214,36 +214,36 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list:
             instances.update_last_error(domain, response)
             continue
 
-    print(f"DEBUG: data()={len(data)} - EXIT!")
+    # DEBUG: print(f"DEBUG: data()={len(data)} - EXIT!")
     return data
 
 def fetch_wellknown_nodeinfo(domain: str) -> list:
-    print(f"DEBUG: domain='{domain}' - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
 
-    print("DEBUG: Fetching .well-known info for domain:", domain)
+    # DEBUG: print("DEBUG: Fetching .well-known info for domain:", domain)
     response = network.fetch_response(domain, "/.well-known/nodeinfo", network.api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
 
     data = network.json_from_response(response)
-    print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data))
+    # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data))
     if response.ok and isinstance(data, dict):
         nodeinfo = data
-        print("DEBUG: Found entries:", len(nodeinfo), domain)
+        # DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain)
         if "links" in nodeinfo:
-            print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"]))
+            # DEBUG: print("DEBUG: Found links in nodeinfo():", len(nodeinfo["links"]))
             for link in nodeinfo["links"]:
-                print("DEBUG: rel,href:", link["rel"], link["href"])
+                # DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"])
                 if link["rel"] in nodeinfo_identifier:
-                    print("DEBUG: Fetching nodeinfo from:", link["href"])
+                    # DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"])
                     response = fba.fetch_url(link["href"], network.api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
                     data = network.json_from_response(response)
-                    print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code)
+                    # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code)
                     if response.ok and isinstance(data, dict):
-                        print("DEBUG: Found JSON nodeinfo():", len(data))
+                        # DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data))
                         instances.set_data("detection_mode", domain, "AUTO_DISCOVERY")
                         instances.set_data("nodeinfo_url"  , domain, link["href"])
                         break
@@ -252,11 +252,11 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
         else:
             print("WARNING: nodeinfo does not contain 'links':", domain)
 
-    print("DEBUG: Returning data[]:", type(data))
+    # DEBUG: print("DEBUG: Returning data[]:", type(data))
     return data
 
 def fetch_generator_from_path(domain: str, path: str = "/") -> str:
-    print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
@@ -266,62 +266,62 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
     elif path == "":
         raise ValueError("Parameter 'path' is empty")
 
-    print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}' - CALLED!")
     software = None
 
-    print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
+    # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
     response = network.fetch_response(domain, path, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
-    print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text))
+    # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text))
     if response.ok and response.status_code < 300 and len(response.text) > 0:
-        print("DEBUG: Search for <meta name='generator'>:", domain)
+        # DEBUG: print("DEBUG: Search for <meta name='generator'>:", domain)
         doc = bs4.BeautifulSoup(response.text, "html.parser")
 
-        print("DEBUG: doc[]:", type(doc))
+        # DEBUG: print("DEBUG: doc[]:", type(doc))
         generator = doc.find("meta", {"name": "generator"})
         site_name = doc.find("meta", {"property": "og:site_name"})
 
-        print(f"DEBUG: generator='{generator}',site_name='{site_name}'")
+        # DEBUG: print(f"DEBUG: generator='{generator}',site_name='{site_name}'")
         if isinstance(generator, bs4.element.Tag):
-            print("DEBUG: Found generator meta tag:", domain)
+            # DEBUG: print("DEBUG: Found generator meta tag:", domain)
             software = tidyup.domain(generator.get("content"))
             print(f"INFO: domain='{domain}' is generated by '{software}'")
             instances.set_data("detection_mode", domain, "GENERATOR")
             fba.remove_pending_error(domain)
         elif isinstance(site_name, bs4.element.Tag):
-            print("DEBUG: Found property=og:site_name:", domain)
+            # DEBUG: print("DEBUG: Found property=og:site_name:", domain)
             sofware = tidyup.domain(site_name.get("content"))
             print(f"INFO: domain='{domain}' has og:site_name='{software}'")
             instances.set_data("detection_mode", domain, "SITE_NAME")
             fba.remove_pending_error(domain)
 
-    print(f"DEBUG: software[]={type(software)}")
+    # DEBUG: print(f"DEBUG: software[]={type(software)}")
     if isinstance(software, str) and software == "":
-        print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'")
+        # DEBUG: print(f"DEBUG: Corrected empty string to None for software of domain='{domain}'")
         software = None
     elif isinstance(software, str) and ("." in software or " " in software):
-        print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...")
+        # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...")
         software = fba.remove_version(software)
 
-    print(f"DEBUG: software[]={type(software)}")
+    # DEBUG: print(f"DEBUG: software[]={type(software)}")
     if isinstance(software, str) and " powered by " in software:
-        print(f"DEBUG: software='{software}' has 'powered by' in it")
+        # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
         software = fba.remove_version(fba.strip_powered_by(software))
     elif isinstance(software, str) and " hosted on " in software:
-        print(f"DEBUG: software='{software}' has 'hosted on' in it")
+        # DEBUG: print(f"DEBUG: software='{software}' has 'hosted on' in it")
         software = fba.remove_version(fba.strip_hosted_on(software))
     elif isinstance(software, str) and " by " in software:
-        print(f"DEBUG: software='{software}' has ' by ' in it")
+        # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it")
         software = fba.strip_until(software, " by ")
     elif isinstance(software, str) and " see " in software:
-        print(f"DEBUG: software='{software}' has ' see ' in it")
+        # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it")
         software = fba.strip_until(software, " see ")
 
-    print(f"DEBUG: software='{software}' - EXIT!")
+    # DEBUG: print(f"DEBUG: software='{software}' - EXIT!")
     return software
 
 def determine_software(domain: str, path: str = None) -> str:
-    print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},path={path} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
@@ -329,18 +329,18 @@ def determine_software(domain: str, path: str = None) -> str:
     elif not isinstance(path, str) and path is not None:
         raise ValueError(f"Parameter path[]={type(path)} is not 'str'")
 
-    print("DEBUG: Determining software for domain,path:", domain, path)
+    # DEBUG: print("DEBUG: Determining software for domain,path:", domain, path)
     software = None
 
-    print(f"DEBUG: Fetching nodeinfo from '{domain}' ...")
+    # DEBUG: print(f"DEBUG: Fetching nodeinfo from '{domain}' ...")
     data = fetch_nodeinfo(domain, path)
 
-    print("DEBUG: data[]:", type(data))
+    # DEBUG: print("DEBUG: data[]:", type(data))
     if not isinstance(data, dict) or len(data) == 0:
-        print("DEBUG: Could not determine software type:", domain)
+        # DEBUG: print("DEBUG: Could not determine software type:", domain)
         return fetch_generator_from_path(domain)
 
-    print("DEBUG: data():", len(data), data)
+    # DEBUG: print("DEBUG: data():", len(data), data)
     if "status" in data and data["status"] == "error" and "message" in data:
         print("WARNING: JSON response is an error:", data["message"])
         instances.update_last_error(domain, data["message"])
@@ -350,23 +350,23 @@ def determine_software(domain: str, path: str = None) -> str:
         instances.update_last_error(domain, data["message"])
         return fetch_generator_from_path(domain)
     elif "software" not in data or "name" not in data["software"]:
-        print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...")
+        # DEBUG: print(f"DEBUG: JSON response from domain='{domain}' does not include [software][name], fetching / ...")
         software = fetch_generator_from_path(domain)
 
-        print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!")
+        # DEBUG: print(f"DEBUG: Generator for domain='{domain}' is: {software}, EXIT!")
         return software
 
     software = tidyup.domain(data["software"]["name"])
 
-    print("DEBUG: sofware after tidyup.domain():", software)
+    # DEBUG: print("DEBUG: sofware after tidyup.domain():", software)
     if software in ["akkoma", "rebased"]:
-        print("DEBUG: Setting pleroma:", domain, software)
+        # DEBUG: print("DEBUG: Setting pleroma:", domain, software)
         software = "pleroma"
     elif software in ["hometown", "ecko"]:
-        print("DEBUG: Setting mastodon:", domain, software)
+        # DEBUG: print("DEBUG: Setting mastodon:", domain, software)
         software = "mastodon"
     elif software in ["calckey", "groundpolis", "foundkey", "cherrypick", "meisskey"]:
-        print("DEBUG: Setting misskey:", domain, software)
+        # DEBUG: print("DEBUG: Setting misskey:", domain, software)
         software = "misskey"
     elif software.find("/") > 0:
         print("WARNING: Spliting of slash:", software)
@@ -375,38 +375,38 @@ def determine_software(domain: str, path: str = None) -> str:
         print("WARNING: Spliting of pipe:", software)
         software = tidyup.domain(software.split("|")[0])
     elif "powered by" in software:
-        print(f"DEBUG: software='{software}' has 'powered by' in it")
+        # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
         software = fba.strip_powered_by(software)
     elif isinstance(software, str) and " by " in software:
-        print(f"DEBUG: software='{software}' has ' by ' in it")
+        # DEBUG: print(f"DEBUG: software='{software}' has ' by ' in it")
         software = fba.strip_until(software, " by ")
     elif isinstance(software, str) and " see " in software:
-        print(f"DEBUG: software='{software}' has ' see ' in it")
+        # DEBUG: print(f"DEBUG: software='{software}' has ' see ' in it")
         software = fba.strip_until(software, " see ")
 
-    print(f"DEBUG: software[]={type(software)}")
+    # DEBUG: print(f"DEBUG: software[]={type(software)}")
     if software == "":
         print("WARNING: tidyup.domain() left no software name behind:", domain)
         software = None
 
-    print(f"DEBUG: software[]={type(software)}")
+    # DEBUG: print(f"DEBUG: software[]={type(software)}")
     if str(software) == "":
-        print(f"DEBUG: software for '{domain}' was not detected, trying generator ...")
+        # DEBUG: print(f"DEBUG: software for '{domain}' was not detected, trying generator ...")
         software = fetch_generator_from_path(domain)
     elif len(str(software)) > 0 and ("." in software or " " in software):
-        print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...")
+        # DEBUG: print(f"DEBUG: software='{software}' may contain a version number, domain='{domain}', removing it ...")
         software = fba.remove_version(software)
 
-    print(f"DEBUG: software[]={type(software)}")
+    # DEBUG: print(f"DEBUG: software[]={type(software)}")
     if isinstance(software, str) and "powered by" in software:
-        print(f"DEBUG: software='{software}' has 'powered by' in it")
+        # DEBUG: print(f"DEBUG: software='{software}' has 'powered by' in it")
         software = fba.remove_version(fba.strip_powered_by(software))
 
-    print("DEBUG: Returning domain,software:", domain, software)
+    # DEBUG: print("DEBUG: Returning domain,software:", domain, software)
     return software
 
 def find_domains(tag: bs4.element.Tag) -> list:
-    print(f"DEBUG: tag[]={type(tag)} - CALLED!")
+    # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!")
     if not isinstance(tag, bs4.element.Tag):
         raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag")
     elif len(tag.select("tr")) == 0:
@@ -414,21 +414,21 @@ def find_domains(tag: bs4.element.Tag) -> list:
 
     domains = list()
     for element in tag.select("tr"):
-        print(f"DEBUG: element[]={type(element)}")
+        # DEBUG: print(f"DEBUG: element[]={type(element)}")
         if not element.find("td"):
-            print("DEBUG: Skipping element, no <td> found")
+            # DEBUG: print("DEBUG: Skipping element, no <td> found")
             continue
 
         domain = tidyup.domain(element.find("td").text)
         reason = tidyup.reason(element.findAll("td")[1].text)
 
-        print(f"DEBUG: domain='{domain}',reason='{reason}'")
+        # DEBUG: print(f"DEBUG: domain='{domain}',reason='{reason}'")
 
         if blacklist.is_blacklisted(domain):
             print(f"WARNING: domain='{domain}' is blacklisted - skipped!")
             continue
         elif domain == "gab.com/.ai, develop.gab.com":
-            print("DEBUG: Multiple domains detected in one row")
+            # DEBUG: print("DEBUG: Multiple domains detected in one row")
             domains.append({
                 "domain": "gab.com",
                 "reason": reason,
@@ -446,13 +446,13 @@ def find_domains(tag: bs4.element.Tag) -> list:
             print(f"WARNING: domain='{domain}' is not a valid domain - skipped!")
             continue
 
-        print(f"DEBUG: Adding domain='{domain}' ...")
+        # DEBUG: print(f"DEBUG: Adding domain='{domain}' ...")
         domains.append({
             "domain": domain,
             "reason": reason,
         })
 
-    print(f"DEBUG: domains()={len(domains)} - EXIT!")
+    # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!")
     return domains
 
 def add_peers(rows: dict) -> list:

diff --git a/fba/helpers/dicts.py b/fba/helpers/dicts.py
index 51843d4b4a04daa94d5be638e008fbc36bfe63d0..68209139d8a1e5fef8befad4aa582a1ee19e77f1 100644 (file)
--- a/fba/helpers/dicts.py
+++ b/fba/helpers/dicts.py
@@ -14,7 +14,7 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 def has_key(lists: list, key: str, value: any) -> bool:
-    print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!")
+    # DEBUG: print(f"DEBUG: lists()={len(lists)},key='{key}',value[]='{type(value)}' - CALLED!")
     if not isinstance(lists, list):
         raise ValueError(f"Parameter lists[]='{type(lists)}' is not 'list'")
     elif not isinstance(key, str):
@@ -23,9 +23,9 @@ def has_key(lists: list, key: str, value: any) -> bool:
         raise ValueError("Parameter 'key' is empty")
 
     has = False
-    print(f"DEBUG: Checking lists()={len(lists)} ...")
+    # DEBUG: print(f"DEBUG: Checking lists()={len(lists)} ...")
     for row in lists:
-        print(f"DEBUG: row['{type(row)}']={row}")
+        # DEBUG: print(f"DEBUG: row['{type(row)}']={row}")
         if not isinstance(row, dict):
             raise ValueError(f"row[]='{type(row)}' is not 'dict'")
         elif not key in row:
@@ -34,5 +34,5 @@ def has_key(lists: list, key: str, value: any) -> bool:
             has = True
             break
 
-    print(f"DEBUG: has={has} - EXIT!")
+    # DEBUG: print(f"DEBUG: has={has} - EXIT!")
     return has

diff --git a/fba/helpers/tidyup.py b/fba/helpers/tidyup.py
index c0de2c859f2f108bcf58e1020fc187ef37624c2e..d2d322faaeae667f62723e79c6bdac522c12f312 100644 (file)
--- a/fba/helpers/tidyup.py
+++ b/fba/helpers/tidyup.py
@@ -16,7 +16,7 @@
 import re
 
 def reason(string: str) -> str:
-    print(f"DEBUG: string='{string}' - CALLED!")
+    # DEBUG: print(f"DEBUG: string='{string}' - CALLED!")
     if not isinstance(string, str):
         raise ValueError(f"Parameter string[]={type(string)} is not 'str'")
 
@@ -26,11 +26,11 @@ def reason(string: str) -> str:
     # Replace â with "
     string = re.sub("â", "\"", string)
 
-    print(f"DEBUG: string='{string}' - EXIT!")
+    # DEBUG: print(f"DEBUG: string='{string}' - EXIT!")
     return string
 
 def domain(string: str) -> str:
-    print(f"DEBUG: string='{string}' - CALLED!")
+    # DEBUG: print(f"DEBUG: string='{string}' - CALLED!")
     if not isinstance(string, str):
         raise ValueError(f"Parameter string[]={type(string)} is not 'str'")
 
@@ -56,5 +56,5 @@ def domain(string: str) -> str:
     elif string.find("/users/"):
         string = string.split("/users/")[0]
 
-    print(f"DEBUG: string='{string}' - EXIT!")
+    # DEBUG: print(f"DEBUG: string='{string}' - EXIT!")
     return string

diff --git a/fba/locking.py b/fba/locking.py
index 132a8ca48d8be29aa93d0b429343a7d1aaab3ebd..ddcbf7c2c88b289d7339bd130824780d916fc590 100644 (file)
--- a/fba/locking.py
+++ b/fba/locking.py
@@ -25,21 +25,25 @@ LOCK = None
 
 def acquire():
     global LOCK
+    print("DEBUG: CALLED!")
+
     try:
-        # DEBUG: print(f"DEBUG: Acquiring lock: '{lockfile}'")
+        print(f"DEBUG: Acquiring lock: '{lockfile}'")
         LOCK = zc.lockfile.LockFile(lockfile)
-        # DEBUG: print("DEBUG: Lock obtained.")
+        print("DEBUG: Lock obtained.")
 
     except zc.lockfile.LockError:
         print(f"ERROR: Cannot aquire lock: '{lockfile}'")
         sys.exit(100)
 
+    print("DEBUG: EXIT!")
+
 def release():
-    # DEBUG: print("DEBUG: CALLED!")
+    print("DEBUG: CALLED!")
     if LOCK is not None:
-        # DEBUG: print("DEBUG: Releasing lock ...")
+        print("DEBUG: Releasing lock ...")
         LOCK.close()
-        # DEBUG: print(f"DEBUG: Deleting lockfile='{lockfile}' ...")
+        print(f"DEBUG: Deleting lockfile='{lockfile}' ...")
         os.remove(lockfile)
 
-    # DEBUG: print("DEBUG: EXIT!")
+    print("DEBUG: EXIT!")

diff --git a/fba/network.py b/fba/network.py
index 7329979d658b0177afa63afa84bb4fe39b00fafa..261d266c85c564112ec0bef99fff230b3987f272 100644 (file)
--- a/fba/network.py
+++ b/fba/network.py
@@ -34,7 +34,7 @@ api_headers = {
 }
 
 def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = {}) -> dict:
-    print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',parameter='{parameter}',extra_headers()={len(extra_headers)} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
@@ -46,13 +46,13 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict =
     elif not isinstance(parameter, str):
         raise ValueError(f"parameter[]={type(parameter)} is not 'str'")
 
-    print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...")
+    # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}' ...")
     headers = csrf.determine(domain, {**api_headers, **extra_headers})
 
     data = {}
 
     try:
-        print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',parameter='{parameter}',extra_headers({len(extra_headers)})={extra_headers}")
+        # DEBUG: print(f"DEBUG: Sending POST to domain='{domain}',path='{path}',parameter='{parameter}',extra_headers({len(extra_headers)})={extra_headers}")
         response = reqto.post(
             f"https://{domain}{path}",
             data=parameter,
@@ -61,7 +61,7 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict =
         )
 
         data = json_from_response(response)
-        print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+        # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
         if not response.ok or response.status_code >= 400:
             print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'")
             instances.update_last_error(domain, response)
@@ -69,11 +69,11 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict =
     except BaseException as exception:
         print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(exception)}]:'{str(exception)}'")
 
-    print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}")
+    # DEBUG: print(f"DEBUG: Returning data({len(data)})=[]:{type(data)}")
     return data
 
 def send_bot_post(domain: str, blocklist: dict):
-    print(f"DEBUG: domain={domain},blocklist()={len(blocklist)} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain={domain},blocklist()={len(blocklist)} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
@@ -88,9 +88,9 @@ def send_bot_post(domain: str, blocklist: dict):
         truncated = True
         blocklist = blocklist[0 : 19]
 
-    print(f"DEBUG: blocklist()={len(blocklist)}")
+    # DEBUG: print(f"DEBUG: blocklist()={len(blocklist)}")
     for block in blocklist:
-        print(f"DEBUG: block['{type(block)}']={block}")
+        # DEBUG: print(f"DEBUG: block['{type(block)}']={block}")
         if block["reason"] is None or block["reason"] == '':
             message = message + block["blocked"] + " with unspecified reason\n"
         else:
@@ -118,7 +118,7 @@ def send_bot_post(domain: str, blocklist: dict):
     return True
 
 def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response:
-    print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!")
+    # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
     elif domain == "":
@@ -128,11 +128,11 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requ
     elif path == "":
         raise ValueError("Parameter 'path' is empty")
 
-    print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}',headers()='{len(headers)}' ...")
+    # DEBUG: print(f"DEBUG: Determining if CSRF header needs to be sent for domain='{domain}',headers()='{len(headers)}' ...")
     headers = csrf.determine(domain, headers)
 
     try:
-        print(f"DEBUG: Sending GET request to '{domain}{path}' ...")
+        # DEBUG: print(f"DEBUG: Sending GET request to '{domain}{path}' ...")
         response = reqto.get(
             f"https://{domain}{path}",
             headers=headers,
@@ -140,25 +140,25 @@ def fetch_response(domain: str, path: str, headers: dict, timeout: list) -> requ
         )
 
     except requests.exceptions.ConnectionError as exception:
-        print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'")
+        # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(exception)}]='{str(exception)}'")
         instances.update_last_error(domain, exception)
         raise exception
 
-    print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
+    # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
     return response
 
 def json_from_response(response: requests.models.Response) -> list:
-    print(f"DEBUG: response[]={type(response)} - CALLED!")
+    # DEBUG: print(f"DEBUG: response[]={type(response)} - CALLED!")
     if not isinstance(response, requests.models.Response):
         raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'")
 
     data = list()
     if response.text.strip() != "":
-        print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...")
+        # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...")
         try:
             data = response.json()
         except json.decoder.JSONDecodeError:
             pass
 
-    print(f"DEBUG: data[]={type(data)} - EXIT!")
+    # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!")
     return data

diff --git a/fba/networks/friendica.py b/fba/networks/friendica.py
index ceba93da31dafdafcb1132ea8da1fa3921693d0d..2b88863050eb8ae40b61c47f353a4bde1b344172 100644 (file)
--- a/fba/networks/friendica.py
+++ b/fba/networks/friendica.py
@@ -40,7 +40,7 @@ def fetch_blocks(domain: str) -> dict:
         ).text,
         "html.parser",
     )
-    print(f"DEBUG: doc[]='{type(doc)}'")
+    # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'")
 
     blocklist = doc.find(id="about_blocklist")
 

diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py
index da84bafac765f986fb2289c39490cb84f2c3efd7..5529b970ceb3fb2e783a464322ed752a72e8a502 100644 (file)
--- a/fba/networks/peertube.py
+++ b/fba/networks/peertube.py
@@ -19,17 +19,17 @@ from fba import instances
 from fba import network
 
 def fetch_peers(domain: str) -> list:
-    # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!")
+    print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!")
     if not isinstance(domain, str):
         raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
     elif domain == "":
         raise ValueError("Parameter 'domain' is empty")
 
-    # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...")
+    print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...")
     peers = list()
     start = 0
     for mode in ["followers", "following"]:
-        # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'")
+        print(f"DEBUG: domain='{domain}',mode='{mode}'")
         while True:
             try:
                 response = network.fetch_response(
@@ -40,21 +40,21 @@ def fetch_peers(domain: str) -> list:
                 )
 
                 data = network.json_from_response(response)
-                # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'")
+                print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'")
                 if response.ok and isinstance(data, dict):
-                    # DEBUG: print("DEBUG: Success, data:", len(data))
+                    print("DEBUG: Success, data:", len(data))
                     if "data" in data:
-                        # DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).")
+                        print(f"DEBUG: Found {len(data['data'])} record(s).")
                         for record in data["data"]:
-                            # DEBUG: print(f"DEBUG: record()={len(record)}")
+                            print(f"DEBUG: record()={len(record)}")
                             if mode in record and "host" in record[mode]:
-                                # DEBUG: print(f"DEBUG: Found host={record[mode]['host']}, adding ...")
+                                print(f"DEBUG: Found host={record[mode]['host']}, adding ...")
                                 peers.append(record[mode]["host"])
                             else:
                                 print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}")
 
                         if len(data["data"]) < 100:
-                            # DEBUG: print("DEBUG: Reached end of JSON response:", domain)
+                            print("DEBUG: Reached end of JSON response:", domain)
                             break
 
                     # Continue with next row
@@ -63,11 +63,11 @@ def fetch_peers(domain: str) -> list:
             except BaseException as exception:
                 print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(exception)}]:'{str(exception)}'")
 
-    # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
+    print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
     instances.set_data("total_peers", domain, len(peers))
 
-    # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
+    print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
     instances.update_last_instance_fetch(domain)
 
-    # DEBUG: print("DEBUG: Returning peers[]:", type(peers))
+    print("DEBUG: Returning peers[]:", type(peers))
     return peers