# along with this program. If not, see <https://www.gnu.org/licenses/>.
import bs4
-from fba import cache
import hashlib
import re
import reqto
+import requests
import json
import sqlite3
import sys
import time
import validators
-with open("config.json") as f:
- config = json.loads(f.read())
+from fba import cache
+from fba import config
+from fba import instances
# Don't check these, known trolls/flooders/testing/developing
blacklist = [
# HTTP headers for non-API requests
headers = {
- "User-Agent": config["useragent"],
+ "User-Agent": config.get("useragent"),
}
# HTTP headers for API requests
api_headers = {
- "User-Agent": config["useragent"],
+ "User-Agent": config.get("useragent"),
"Content-Type": "application/json",
}
-# Found info from node, such as nodeinfo URL, detection mode that needs to be
-# written to database. Both arrays must be filled at the same time or else
-# update_instance_data() will fail
-instance_data = {
- # Detection mode: 'AUTO_DISCOVERY', 'STATIC_CHECKS' or 'GENERATOR'
- # NULL means all detection methods have failed (maybe still reachable instance)
- "detection_mode" : {},
- # Found nodeinfo URL
- "nodeinfo_url" : {},
- # Found total peers
- "total_peers" : {},
- # Last fetched instances
- "last_instance_fetch": {},
- # Last updated
- "last_updated" : {},
- # Last blocked
- "last_blocked" : {},
- # Last nodeinfo (fetched)
- "last_nodeinfo" : {},
- # Last status code
- "last_status_code" : {},
- # Last error details
- "last_error_details" : {},
-}
-
language_mapping = {
# English -> English
"Silenced instances" : "Silenced servers",
return type(var) in {int, str, float, bool} or var == None
def fetch_instances(domain: str, origin: str, software: str, script: str, path: str = None):
+ # DEBUG: print(f"DEBUG: domain={domain},origin={origin},software={software},path={path} - CALLED!")
if type(domain) != str:
raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
elif domain == "":
elif domain == "":
raise ValueError(f"Parameter 'domain' cannot be empty")
- # DEBUG: print("DEBUG: domain,origin,software,path:", domain, origin, software, path)
if not is_instance_registered(domain):
# DEBUG: print("DEBUG: Adding new domain:", domain, origin)
add_instance(domain, origin, script, path)
if (peerlist is None):
print("ERROR: Cannot fetch peers:", domain)
return
- elif has_pending_instance_data(domain):
+ elif instances.has_pending_instance_data(domain):
# DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo data, flushing ...")
- update_instance_data(domain)
+ instances.update_instance_data(domain)
print(f"INFO: Checking {len(peerlist)} instances from {domain} ...")
for instance in peerlist:
# Skip "None" types as tidup() cannot parse them
continue
- # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - BEFORE")
+ # DEBUG: print(f"DEBUG: instance='{instance}' - BEFORE")
instance = tidyup_domain(instance)
- # DEBUG: print(f"DEBUG: instance[{type(instance}]={instance} - AFTER")
+ # DEBUG: print(f"DEBUG: instance='{instance}' - AFTER")
if instance == "":
print("WARNING: Empty instance after tidyup_domain(), domain:", domain)
# DEBUG: print("DEBUG: EXIT!")
-def set_instance_data(key: str, domain: str, value: any):
- # NOISY-DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',value[]='{type(value)}' - CALLED!")
- if type(key) != str:
- raise ValueError("Parameter key[]='{type(key)}' is not 'str'")
- elif key == "":
- raise ValueError(f"Parameter 'key' cannot be empty")
- elif type(domain) != str:
- raise ValueError("Parameter domain[]='{type(domain)}' is not 'str'")
- elif domain == "":
- raise ValueError(f"Parameter 'domain' cannot be empty")
- elif not key in instance_data:
- raise ValueError(f"key='{key}' not found in instance_data")
- elif not is_primitive(value):
- raise ValueError(f"value[]='{type(value)}' is not a primitive type")
-
- # Set it
- instance_data[key][domain] = value
-
- # DEBUG: print("DEBUG: EXIT!")
-
def add_peers(rows: dict) -> list:
# DEBUG: print(f"DEBUG: rows()={len(rows)} - CALLED!")
peers = list()
raise ValueError(f"Parameter 'domain' cannot be empty")
# DEBUG: print("DEBUG: Updating last_blocked for domain", domain)
- set_instance_data("last_blocked", domain, time.time())
+ instances.set("last_blocked", domain, time.time())
# Running pending updated
- # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...")
- update_instance_data(domain)
+ # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
+ instances.update_instance_data(domain)
# DEBUG: print("DEBUG: EXIT!")
-def has_pending_instance_data(domain: str) -> bool:
- # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+def log_error(domain: str, response: requests.models.Response):
+ # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
if type(domain) != str:
raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
elif domain == "":
raise ValueError(f"Parameter 'domain' cannot be empty")
- has_pending = False
- for key in instance_data:
- # DEBUG: print(f"DEBUG: key='{key}',domain='{domain}',instance_data[key]()='{len(instance_data[key])}'")
- if domain in instance_data[key]:
- has_pending = True
- break
-
- # DEBUG: print(f"DEBUG: has_pending='{has_pending}' - EXIT!")
- return has_pending
-
-def update_instance_data(domain: str):
- # DEBUG: print(f"DEBUG: domain={domain} - CALLED!")
- if type(domain) != str:
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
- elif domain == "":
- raise ValueError(f"Parameter 'domain' cannot be empty")
- elif not has_pending_instance_data(domain):
- raise Exception(f"Domain '{domain}' has no pending instance data, but function invoked")
-
- # DEBUG: print(f"DEBUG: Updating nodeinfo for domain='{domain}' ...")
- sql_string = ''
- fields = list()
- for key in instance_data:
- # DEBUG: print("DEBUG: key:", key)
- if domain in instance_data[key]:
- # DEBUG: print(f"DEBUG: Adding '{instance_data[key][domain]}' for key='{key}' ...")
- fields.append(instance_data[key][domain])
- sql_string += f" {key} = ?,"
-
- fields.append(domain)
-
- if sql_string == '':
- raise ValueError(f"No fields have been set, but method invoked, domain='{domain}'")
-
- # DEBUG: print(f"DEBUG: sql_string='{sql_string}',fields()={len(fields)}")
- sql_string = "UPDATE instances SET" + sql_string + " last_updated = TIME() WHERE domain = ? LIMIT 1"
- # DEBUG: print("DEBUG: sql_string:", sql_string)
-
try:
- # DEBUG: print("DEBUG: Executing SQL:", sql_string)
- cursor.execute(sql_string, fields)
-
- # DEBUG: print(f"DEBUG: Success! (rowcount={cursor.rowcount })")
- if cursor.rowcount == 0:
- print(f"WARNING: Did not update any rows: domain='{domain}',fields()={len(fields)} - EXIT!")
- return
-
- connection.commit()
-
- # DEBUG: print("DEBUG: Deleting instance_data for domain:", domain)
- for key in instance_data:
- try:
- # DEBUG: print("DEBUG: Deleting key:", key)
- del instance_data[key][domain]
- except:
- pass
-
- except BaseException as e:
- print(f"ERROR: failed SQL query: domain='{domain}',sql_string='{sql_string}',exception[{type(e)}]:'{str(e)}'")
- sys.exit(255)
+ # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
+ if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
+ response = str(response)
- # DEBUG: print("DEBUG: EXIT!")
-
-def log_error(domain: str, res: any):
- # DEBUG: print("DEBUG: domain,res[]:", domain, type(res))
- if type(domain) != str:
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
- elif domain == "":
- raise ValueError(f"Parameter 'domain' cannot be empty")
-
- try:
- # DEBUG: print("DEBUG: BEFORE res[]:", type(res))
- if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError):
- res = str(res)
-
- # DEBUG: print("DEBUG: AFTER res[]:", type(res))
- if type(res) is str:
+ # DEBUG: print("DEBUG: AFTER response[]:", type(response))
+ if type(response) is str:
cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, 999, ?, ?)",[
domain,
- res,
+ response,
time.time()
])
else:
cursor.execute("INSERT INTO error_log (domain, error_code, error_message, created) VALUES (?, ?, ?, ?)",[
domain,
- res.status_code,
- res.reason,
+ response.status_code,
+ response.reason,
time.time()
])
# Cleanup old entries
- # DEBUG: print(f"DEBUG: Purging old records (distance: {config['error_log_cleanup']})")
- cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config["error_log_cleanup"]])
+ # DEBUG: print(f"DEBUG: Purging old records (distance: {config.get('error_log_cleanup')})")
+ cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.get("error_log_cleanup")])
except BaseException as e:
print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
sys.exit(255)
# DEBUG: print("DEBUG: EXIT!")
-def update_last_error(domain: str, res: any):
- # DEBUG: print("DEBUG: domain,res[]:", domain, type(res))
+def update_last_error(domain: str, response: requests.models.Response):
+ # DEBUG: print("DEBUG: domain,response[]:", domain, type(response))
if type(domain) != str:
raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
elif domain == "":
raise ValueError(f"Parameter 'domain' cannot be empty")
- # DEBUG: print("DEBUG: BEFORE res[]:", type(res))
- if isinstance(res, BaseException) or isinstance(res, json.JSONDecodeError):
- res = str(res)
+ # DEBUG: print("DEBUG: BEFORE response[]:", type(response))
+ if isinstance(response, BaseException) or isinstance(response, json.decoder.JSONDecodeError):
+ response = f"{type}:str(response)"
- # DEBUG: print("DEBUG: AFTER res[]:", type(res))
- if type(res) is str:
- # DEBUG: print(f"DEBUG: Setting last_error_details='{res}'");
- set_instance_data("last_status_code" , domain, 999)
- set_instance_data("last_error_details", domain, res)
+ # DEBUG: print("DEBUG: AFTER response[]:", type(response))
+ if type(response) is str:
+ # DEBUG: print(f"DEBUG: Setting last_error_details='{response}'");
+ instances.set("last_status_code" , domain, 999)
+ instances.set("last_error_details", domain, response)
else:
- # DEBUG: print(f"DEBUG: Setting last_error_details='{res.reason}'");
- set_instance_data("last_status_code" , domain, res.status_code)
- set_instance_data("last_error_details", domain, res.reason)
+ # DEBUG: print(f"DEBUG: Setting last_error_details='{response.reason}'");
+ instances.set("last_status_code" , domain, response.status_code)
+ instances.set("last_error_details", domain, response.reason)
# Running pending updated
- # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...")
- update_instance_data(domain)
+ # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
+ instances.update_instance_data(domain)
- log_error(domain, res)
+ log_error(domain, response)
# DEBUG: print("DEBUG: EXIT!")
raise ValueError(f"Parameter 'domain' cannot be empty")
# DEBUG: print("DEBUG: Updating last_instance_fetch for domain:", domain)
- set_instance_data("last_instance_fetch", domain, time.time())
+ instances.set("last_instance_fetch", domain, time.time())
# Running pending updated
- # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...")
- update_instance_data(domain)
+ # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
+ instances.update_instance_data(domain)
# DEBUG: print("DEBUG: EXIT!")
raise ValueError(f"Parameter 'domain' cannot be empty")
# DEBUG: print("DEBUG: Updating last_nodeinfo for domain:", domain)
- set_instance_data("last_nodeinfo", domain, time.time())
- set_instance_data("last_updated" , domain, time.time())
+ instances.set("last_nodeinfo", domain, time.time())
+ instances.set("last_updated" , domain, time.time())
# Running pending updated
- # DEBUG: print(f"DEBUG: Invoking update_instance_data({domain}) ...")
- update_instance_data(domain)
+ # DEBUG: print(f"DEBUG: Invoking instances.update_instance_data({domain}) ...")
+ instances.update_instance_data(domain)
# DEBUG: print("DEBUG: EXIT!")
elif type(software) != str and software != None:
raise ValueError(f"software[]={type(software)} is not 'str'")
- # DEBUG: print(f"DEBUG: domain='{domain}',software='{software}' - CALLED!")
peers = list()
if software == "misskey":
# DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
offset = 0
- step = config["misskey_offset"]
+ step = config.get("misskey_limit")
# iterating through all "suspended" (follow-only in its terminology)
# instances page-by-page, since that troonware doesn't support
"sort" : "+pubAt",
"host" : None,
"limit": step
- }), {"Origin": domain})
+ }), {
+ "Origin": domain
+ })
else:
fetched = post_json_api(domain, "/api/federation/instances", json.dumps({
"sort" : "+pubAt",
"host" : None,
"limit" : step,
"offset": offset - 1
- }), {"Origin": domain})
+ }), {
+ "Origin": domain
+ })
- # DEBUG: print("DEBUG: fetched():", len(fetched))
+ # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
if len(fetched) == 0:
# DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
break
- elif len(fetched) != config["misskey_offset"]:
- # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
- offset = offset + (config["misskey_offset"] - len(fetched))
+ elif len(fetched) != config.get("misskey_limit"):
+ # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+ offset = offset + (config.get("misskey_limit") - len(fetched))
else:
# DEBUG: print("DEBUG: Raising offset by step:", step)
offset = offset + step
update_last_error(domain, fetched["error"]["message"])
break
+ already = 0
for row in fetched:
# DEBUG: print(f"DEBUG: row():{len(row)}")
if not "host" in row:
elif is_blacklisted(row["host"]):
# DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
continue
+ elif row["host"] in peers:
+ # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.")
+ already = already + 1
+ continue
# DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
peers.append(row["host"])
+ if already == len(fetched):
+ print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!")
+ break
+
# DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
- set_instance_data("total_peers", domain, len(peers))
+ instances.set("total_peers", domain, len(peers))
# DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
update_last_instance_fetch(domain)
elif software == "lemmy":
# DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...")
try:
- res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+ response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
- data = res.json()
- # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
- if not res.ok or res.status_code >= 400:
+ data = json_from_response(response)
+
+ # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'")
+ if not response.ok or response.status_code >= 400:
print("WARNING: Could not reach any JSON API:", domain)
- update_last_error(domain, res)
- elif res.ok and isinstance(data, list):
+ update_last_error(domain, response)
+ elif response.ok and isinstance(data, list):
# DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
sys.exit(255)
elif "federated_instances" in data:
# DEBUG: print("DEBUG: Added instance(s) to peers")
else:
print("WARNING: JSON response does not contain 'federated_instances':", domain)
- update_last_error(domain, res)
+ update_last_error(domain, response)
except BaseException as e:
print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
# DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
- set_instance_data("total_peers", domain, len(peers))
+ instances.set("total_peers", domain, len(peers))
# DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
update_last_instance_fetch(domain)
# DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'")
while True:
try:
- res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+ response = get_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout")))
- data = res.json()
- # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
- if res.ok and isinstance(data, dict):
+ data = json_from_response(response)
+ # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',data[]='{type(data)}'")
+ if response.ok and isinstance(data, dict):
# DEBUG: print("DEBUG: Success, data:", len(data))
if "data" in data:
# DEBUG: print(f"DEBUG: Found {len(data['data'])} record(s).")
print(f"WARNING: Exception during fetching JSON: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
# DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
- set_instance_data("total_peers", domain, len(peers))
+ instances.set("total_peers", domain, len(peers))
# DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
update_last_instance_fetch(domain)
# DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...")
try:
- res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+ response = get_response(domain, get_peers_url, api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+
+ data = json_from_response(response)
- data = res.json()
- # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
- if not res.ok or res.status_code >= 400:
+ # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+ if not response.ok or response.status_code >= 400:
# DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...")
- res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+ response = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
- data = res.json()
- # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
- if not res.ok or res.status_code >= 400:
+ data = json_from_response(response)
+ # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+ if not response.ok or response.status_code >= 400:
print("WARNING: Could not reach any JSON API:", domain)
- update_last_error(domain, res)
- elif res.ok and isinstance(data, list):
+ update_last_error(domain, response)
+ elif response.ok and isinstance(data, list):
# DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
sys.exit(255)
elif "federated_instances" in data:
# DEBUG: print("DEBUG: Added instance(s) to peers")
else:
print("WARNING: JSON response does not contain 'federated_instances':", domain)
- update_last_error(domain, res)
+ update_last_error(domain, response)
else:
# DEBUG: print("DEBUG: Querying API was successful:", domain, len(data))
peers = data
update_last_error(domain, e)
# DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
- set_instance_data("total_peers", domain, len(peers))
+ instances.set("total_peers", domain, len(peers))
# DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
update_last_instance_fetch(domain)
elif type(path) != str:
raise ValueError(f"path[]={type(path)} is not 'str'")
elif path == "":
- raise ValueError(f"path cannot be empty")
+ raise ValueError("Parameter 'path' cannot be empty")
elif type(parameter) != str:
raise ValueError(f"parameter[]={type(parameter)} is not 'str'")
# DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers)
data = {}
try:
- res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"]))
+ response = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
- data = res.json()
- # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
- if not res.ok or res.status_code >= 400:
- print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},res.status_code='{res.status_code}',data[]='{type(data)}'")
- update_last_error(domain, res)
+ data = json_from_response(response)
+ # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+ if not response.ok or response.status_code >= 400:
+ print(f"WARNING: Cannot query JSON API: domain='{domain}',path='{path}',parameter()={len(parameter)},response.status_code='{response.status_code}',data[]='{type(data)}'")
+ update_last_error(domain, response)
except BaseException as e:
print(f"WARNING: Some error during post(): domain='{domain}',path='{path}',parameter()={len(parameter)},exception[{type(e)}]:'{str(e)}'")
try:
# DEBUG: print("DEBUG: Fetching request:", request)
- res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+ response = reqto.get(request, headers=api_headers, timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
- data = res.json()
- # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
- if res.ok and isinstance(data, dict):
+ data = json_from_response(response)
+ # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code={response.status_code},data[]='{type(data)}'")
+ if response.ok and isinstance(data, dict):
# DEBUG: print("DEBUG: Success:", request)
- set_instance_data("detection_mode", domain, "STATIC_CHECK")
- set_instance_data("nodeinfo_url" , domain, request)
+ instances.set("detection_mode", domain, "STATIC_CHECK")
+ instances.set("nodeinfo_url" , domain, request)
break
- elif res.ok and isinstance(data, list):
+ elif response.ok and isinstance(data, list):
# DEBUG: print(f"DEBUG: domain='{domain}' returned a list: '{data}'")
sys.exit(255)
- elif not res.ok or res.status_code >= 400:
+ elif not response.ok or response.status_code >= 400:
print("WARNING: Failed fetching nodeinfo from domain:", domain)
- update_last_error(domain, res)
+ update_last_error(domain, response)
continue
except BaseException as e:
data = {}
try:
- res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+ response = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
- data = res.json()
- # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data))
- if res.ok and isinstance(data, dict):
+ data = json_from_response(response)
+ # DEBUG: print("DEBUG: domain,response.ok,data[]:", domain, response.ok, type(data))
+ if response.ok and isinstance(data, dict):
nodeinfo = data
# DEBUG: print("DEBUG: Found entries:", len(nodeinfo), domain)
if "links" in nodeinfo:
# DEBUG: print("DEBUG: rel,href:", link["rel"], link["href"])
if link["rel"] in nodeinfo_identifier:
# DEBUG: print("DEBUG: Fetching nodeinfo from:", link["href"])
- res = reqto.get(link["href"])
+ response = reqto.get(link["href"])
- data = res.json()
- # DEBUG: print("DEBUG: href,res.ok,res.status_code:", link["href"], res.ok, res.status_code)
- if res.ok and isinstance(data, dict):
+ data = json_from_response(response)
+ # DEBUG: print("DEBUG: href,response.ok,response.status_code:", link["href"], response.ok, response.status_code)
+ if response.ok and isinstance(data, dict):
# DEBUG: print("DEBUG: Found JSON nodeinfo():", len(data))
- set_instance_data("detection_mode", domain, "AUTO_DISCOVERY")
- set_instance_data("nodeinfo_url" , domain, link["href"])
+ instances.set("detection_mode", domain, "AUTO_DISCOVERY")
+ instances.set("nodeinfo_url" , domain, link["href"])
break
else:
print("WARNING: Unknown 'rel' value:", domain, link["rel"])
try:
# DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
- res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"]))
+ response = get_response(domain, path, headers, (config.get("connection_timeout"), config.get("read_timeout")))
- # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text))
- if res.ok and res.status_code < 300 and len(res.text) > 0:
+ # DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text))
+ if response.ok and response.status_code < 300 and len(response.text) > 0:
# DEBUG: print("DEBUG: Search for <meta name='generator'>:", domain)
- doc = bs4.BeautifulSoup(res.text, "html.parser")
+ doc = bs4.BeautifulSoup(response.text, "html.parser")
# DEBUG: print("DEBUG: doc[]:", type(doc))
generator = doc.find("meta", {"name": "generator"})
# DEBUG: print("DEBUG: Found generator meta tag:", domain)
software = tidyup_domain(generator.get("content"))
print(f"INFO: domain='{domain}' is generated by '{software}'")
- set_instance_data("detection_mode", domain, "GENERATOR")
+ instances.set("detection_mode", domain, "GENERATOR")
remove_pending_error(domain)
elif isinstance(site_name, bs4.element.Tag):
# DEBUG: print("DEBUG: Found property=og:site_name:", domain)
sofware = tidyup_domain(site_name.get("content"))
print(f"INFO: domain='{domain}' has og:site_name='{software}'")
- set_instance_data("detection_mode", domain, "SITE_NAME")
+ instances.set("detection_mode", domain, "SITE_NAME")
remove_pending_error(domain)
except BaseException as e:
elif is_blacklisted(blocked):
raise Exception(f"blocked='{blocked}' is blacklisted but function invoked")
+ if reason != None:
+ # Maybe needs cleaning
+ reason = tidyup_reason(reason)
+
print(f"INFO: New block: blocker='{blocker}',blocked='{blocked}', reason='{reason}', block_level='{block_level}'")
try:
cursor.execute(
time.time()
),
)
-
- connection.commit()
except BaseException as e:
print(f"ERROR: failed SQL query: blocker='{blocker}',blocked='{blocked}',reason='{reason}',block_level='{block_level}',exception[{type(e)}]:'{str(e)}'")
sys.exit(255)
elif domain == "":
raise ValueError(f"Parameter 'domain' cannot be empty")
- # NOISY-DEBUG: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
- if not cache.is_cache_initialized("is_registered"):
- # NOISY-DEBUG: # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
+ # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
+ if not cache.key_exists("is_registered"):
+ # DEBUG: print(f"DEBUG: Cache for 'is_registered' not initialized, fetching all rows ...")
try:
cursor.execute("SELECT domain FROM instances")
# Check Set all
- cache.set_all_cache_key("is_registered", cursor.fetchall(), True)
+ cache.set_all("is_registered", cursor.fetchall(), True)
except BaseException as e:
print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'")
sys.exit(255)
# Is cache found?
- registered = cache.is_cache_key_set("is_registered", domain)
+ registered = cache.sub_key_exists("is_registered", domain)
- # NOISY-DEBUG: # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
+ # DEBUG: print(f"DEBUG: registered='{registered}' - EXIT!")
return registered
def add_instance(domain: str, origin: str, originator: str, path: str = None):
),
)
- cache.set_cache_key("is_registered", domain, True)
+ cache.set_sub_key("is_registered", domain, True)
- if has_pending_instance_data(domain):
+ if instances.has_pending_instance_data(domain):
# DEBUG: print(f"DEBUG: domain='{domain}' has pending nodeinfo being updated ...")
- set_instance_data("last_status_code" , domain, None)
- set_instance_data("last_error_details", domain, None)
- update_instance_data(domain)
+ instances.set("last_status_code" , domain, None)
+ instances.set("last_error_details", domain, None)
+ instances.update_instance_data(domain)
remove_pending_error(domain)
if domain in pending_errors:
if truncated:
message = message + "(the list has been truncated to the first 20 entries)"
- botheaders = {**api_headers, **{"Authorization": "Bearer " + config["bot_token"]}}
+ botheaders = {**api_headers, **{"Authorization": "Bearer " + config.get("bot_token")}}
req = reqto.post(
- f"{config['bot_instance']}/api/v1/statuses",
+ f"{config.get('bot_instance')}/api/v1/statuses",
data={
"status" : message,
- "visibility" : config['bot_visibility'],
+ "visibility" : config.get('bot_visibility'),
"content_type": "text/plain"
},
headers=botheaders,
try:
doc = bs4.BeautifulSoup(
- reqto.get(f"https://{domain}/about", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text,
+ get_response(domain, "/about", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
"html.parser",
)
except BaseException as e:
try:
doc = bs4.BeautifulSoup(
- reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text,
+ get_response(domain, "/friendica", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
"html.parser",
)
except BaseException as e:
}
offset = 0
- step = config["misskey_offset"]
+ step = config.get("misskey_limit")
while True:
# iterating through all "suspended" (follow-only in its terminology)
# instances page-by-page, since that troonware doesn't support
"host" : None,
"suspended": True,
"limit" : step
- }), {"Origin": domain})
+ }), {
+ "Origin": domain
+ })
else:
# DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
fetched = post_json_api(domain, "/api/federation/instances", json.dumps({
"suspended": True,
"limit" : step,
"offset" : offset - 1
- }), {"Origin": domain})
+ }), {
+ "Origin": domain
+ })
# DEBUG: print("DEBUG: fetched():", len(fetched))
if len(fetched) == 0:
# DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
break
- elif len(fetched) != config["misskey_offset"]:
- # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
- offset = offset + (config["misskey_offset"] - len(fetched))
+ elif len(fetched) != config.get("misskey_limit"):
+ # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+ offset = offset + (config.get("misskey_limit") - len(fetched))
else:
# DEBUG: print("DEBUG: Raising offset by step:", step)
offset = offset + step
for instance in fetched:
# just in case
- if instance["isSuspended"]:
+ if instance["isSuspended"] and not has_element(blocks["suspended"], "domain", instance):
blocks["suspended"].append(
{
"domain": tidyup_domain(instance["host"]),
"host" : None,
"blocked": True,
"limit" : step
- }), {"Origin": domain})
+ }), {
+ "Origin": domain
+ })
else:
# DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
fetched = post_json_api(domain,"/api/federation/instances", json.dumps({
"host" : None,
"blocked": True,
"limit" : step,
- "offset" : offset-1
- }), {"Origin": domain})
+ "offset" : offset - 1
+ }), {
+ "Origin": domain
+ })
# DEBUG: print("DEBUG: fetched():", len(fetched))
if len(fetched) == 0:
# DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
break
- elif len(fetched) != config["misskey_offset"]:
- # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'")
- offset = offset + (config["misskey_offset"] - len(fetched))
+ elif len(fetched) != config.get("misskey_limit"):
+ # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
+ offset = offset + (config.get("misskey_limit") - len(fetched))
else:
# DEBUG: print("DEBUG: Raising offset by step:", step)
offset = offset + step
for instance in fetched:
- if instance["isBlocked"]:
+ if instance["isBlocked"] and not has_element(blocks["blocked"], "domain", instance):
blocks["blocked"].append({
"domain": tidyup_domain(instance["host"]),
"reason": None
"followers_only": blocks["suspended"]
}
+def tidyup_reason(reason: str) -> str:
+ # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!")
+ if type(reason) != str:
+ raise ValueError(f"Parameter reason[]={type(reason)} is not expected")
+
+ # Strip string
+ reason = reason.strip()
+
+ # Replace â with "
+ reason = re.sub("â", "\"", reason)
+
+ #print(f"DEBUG: reason='{reason}' - EXIT!")
+ return reason
+
def tidyup_domain(domain: str) -> str:
# DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
if type(domain) != str:
raise ValueError(f"Parameter domain[]={type(domain)} is not expected")
- # All lower-case and strip spaces out
- domain = domain.lower().strip()
+ # All lower-case and strip spaces out + last dot
+ domain = domain.lower().strip().rstrip(".")
# No port number
domain = re.sub("\:\d+$", "", domain)
- # No protocol, sometimes with the slashes
+ # No protocol, sometimes without the slashes
domain = re.sub("^https?\:(\/*)", "", domain)
# No trailing slash
# DEBUG: print(f"DEBUG: domain='{domain}' - EXIT!")
return domain
+
+def json_from_response(response: requests.models.Response) -> list:
+ # DEBUG: print(f"DEBUG: response[]={type(response)} - CALLED!")
+ if not isinstance(response, requests.models.Response):
+ raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'")
+
+ data = list()
+ if response.text.strip() != "":
+ # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...")
+ try:
+ data = response.json()
+ except json.decoder.JSONDecodeError:
+ pass
+
+ # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!")
+ return data
+
+def get_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response:
+ # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!")
+ if type(domain) != str:
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+ elif domain == "":
+ raise ValueError("Parameter 'domain' cannot be empty")
+ elif type(path) != str:
+ raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
+ elif path == "":
+ raise ValueError("Parameter 'path' cannot be empty")
+
+ try:
+ # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...")
+ response = reqto.get(f"https://{domain}{path}", headers=headers, timeout=timeout);
+ except requests.exceptions.ConnectionError as e:
+ # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(e)}]='{str(e)}'")
+ update_last_error(domain, e)
+ raise e
+
+ # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
+ return response
+
+def has_element(elements: list, key: str, value: any) -> bool:
+ # DEBUG: print(f"DEBUG: element()={len(element)},key='{key}',value[]='{type(value)}' - CALLED!")
+ if type(key) != str:
+ raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'")
+ elif key == "":
+ raise ValueError("Parameter 'key' cannot be empty")
+
+ has = False
+ # DEBUG: print(f"DEBUG: Checking elements()={len(elements)} ...")
+ for element in elements:
+ # DEBUG: print(f"DEBUG: element[]='{type(element)}'")
+ if type(element) != dict:
+ raise ValueError(f"element[]='{type(element)}' is not 'dict'")
+ elif not key in element:
+ raise KeyError(f"Cannot find key='{key}'")
+ elif element[key] == value:
+ has = True
+ break
+
+ # DEBUG: print(f"DEBUG: has={has} - EXIT!")
+ return has
+
+def find_domains(tag: bs4.element.Tag) -> list:
+ # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!")
+ if not isinstance(tag, bs4.element.Tag):
+ raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag")
+ elif not isinstance(tag, bs4.element.Tag):
+ raise KeyError("Cannot find table with instances!")
+ elif len(tag.select("tr")) == 0:
+ raise KeyError("No table rows found in table!")
+
+ domains = list()
+ for element in tag.select("tr"):
+ # DEBUG: print(f"DEBUG: element[]={type(element)}")
+ if not element.find("td"):
+ # DEBUG: print("DEBUG: Skipping element, no <td> found")
+ continue
+
+ domain = tidyup_domain(element.find("td").text)
+ reason = tidyup_reason(element.findAll("td")[1].text)
+
+ # DEBUG: print(f"DEBUG: domain='{domain}',reason='{reason}'")
+
+ if is_blacklisted(domain):
+ print(f"WARNING: domain='{domain}' is blacklisted - skipped!")
+ continue
+ elif domain == "gab.com/.ai, develop.gab.com":
+ print(f"DEBUG: Multiple domains detected in one row")
+ domains.append({
+ "domain": "gab.com",
+ "reason": reason,
+ })
+ domains.append({
+ "domain": "gab.ai",
+ "reason": reason,
+ })
+ domains.append({
+ "domain": "develop.gab.com",
+ "reason": reason,
+ })
+ continue
+ elif not validators.domain(domain):
+ print(f"WARNING: domain='{domain}' is not a valid domain - skipped!")
+ continue
+
+ # DEBUG: print(f"DEBUG: Adding domain='{domain}' ...")
+ domains.append({
+ "domain": domain,
+ "reason": reason,
+ })
+
+ # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!")
+ return domains
projects / fba.git / blobdiff