print(f"INFO: Fetching instances from domain='{domain}' ...")
federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
- print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_bkali) from domain='{domain}'")
instances.update_last_error(domain, exception)
# DEBUG: print("DEBUG: EXIT!")
continue
elif blocked.count("*") > 0:
# Some friendica servers also obscure domains without hash
- fba.cursor.execute(
- "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
- )
+ row = instances.deobscure("*", blocked)
- searchres = fba.cursor.fetchone()
-
- # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
- if searchres is None:
- print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
+ # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
+ if row is None:
+ print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocker='{blocker}',software='{software}' - SKIPPED!")
continue
- blocked = searchres[0]
- origin = searchres[1]
- nodeinfo_url = searchres[2]
+ blocked = row[0]
+ origin = row[1]
+ nodeinfo_url = row[2]
elif blocked.count("?") > 0:
# Some obscure them with question marks, not sure if that's dependent on version or not
- fba.cursor.execute(
- "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
- )
-
- searchres = fba.cursor.fetchone()
+ row = instances.deobscure("?", blocked)
- # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
- if searchres is None:
- print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
+ # DEBUG: print(f"DEBUG: row[]='{type(row)}'")
+ if row is None:
+ print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocker='{blocker}',software='{software}' - SKIPPED!")
continue
- blocked = searchres[0]
- origin = searchres[1]
- nodeinfo_url = searchres[2]
- elif not validators.domain(blocked):
- print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
- continue
+ blocked = row[0]
+ origin = row[1]
+ nodeinfo_url = row[2]
# DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
if not validators.domain(blocked):
- print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
- continue
- elif blocked.split(".")[-1] == "arpa":
- print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
+ print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - SKIPPED!")
continue
elif not instances.is_registered(blocked):
# DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
- print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'")
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_cs) from domain='{row['domain']}'")
instances.update_last_error(row["domain"], exception)
# DEBUG: print("DEBUG: Committing changes ...")
print(f"INFO: Fetching instances from domain='{domain}' ...")
federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
- print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fba_rss) from domain='{domain}'")
instances.update_last_error(domain, exception)
# DEBUG: print("DEBUG: EXIT!")
print(f"INFO: Fetching instances from domain='{domain}' ...")
federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
- print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fbabot_atom) from domain='{domain}'")
instances.update_last_error(domain, exception)
# DEBUG: print("DEBUG: EXIT!")
print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
- print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'")
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from args.domain='{args.domain}'")
instances.update_last_error(args.domain, exception)
+
return 100
if args.single:
print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
except network.exceptions as exception:
- print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'")
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from domain='{row[0]}'")
instances.update_last_error(row[0], exception)
# DEBUG: print("DEBUG: EXIT!")
}
)
+ domains = list()
for block in blocklists:
# Is domain given and not equal blocker?
if isinstance(args.domain, str) and args.domain != block["blocker"]:
# DEBUG: print(f"DEBUG: Skipping blocker='{block['blocker']}', not matching args.domain='{args.domain}'")
continue
+ elif domain in domains:
+ # DEBUG: print(f"DEBUG: domain='{domain}' already handled - SKIPPED!")
+ continue
# Fetch this URL
print(f"INFO: Fetching csv_url='{block['csv_url']}' for blocker='{block['blocker']}' ...")
# DEBUG: print(f"DEBUG: response[]='{type(response)}'")
if response.ok and response.content != "":
# DEBUG: print(f"DEBUG: Fetched {len(response.content)} Bytes, parsing CSV ...")
- #print(f"DEBUG: response.content={response.content}")
reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
- #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate'
+
# DEBUG: print(f"DEBUG: reader[]='{type(reader)}'")
for row in reader:
- if not validators.domain(row["#domain"]):
- print(f"WARNING: domain='{row['#domain']}' is not a valid domain - skipped!")
- continue
- elif blacklist.is_blacklisted(row["#domain"]):
- print(f"WARNING: domain='{row['#domain']}' is blacklisted - skipped!")
+ domain = None
+ if "#domain" in row:
+ domain = row["#domain"]
+ elif "domain" in row:
+ domain = row["domain"]
+ else:
+ # DEBUG: print(f"DEBUG: row='{row}' does not contain domain column")
continue
- elif instances.is_recent(row["#domain"]):
- # DEBUG: print(f"DEBUG: domain='{row['#domain']}' has been recently checked - skipped!")
+
+ # DEBUG: print(f"DEBUG: Marking domain='{domain}' as handled")
+ domains.append(domain)
+
+ # DEBUG: print(f"DEBUG: Processing domain='{domain}' ...")
+ processed = fba.process_domain(domain, block["blocker"], inspect.currentframe().f_code.co_name)
+
+ # DEBUG: print(f"DEBUG: processed='{processed}'")
+
+ # DEBUG: print("DEBUG: EXIT!")
+
+def fetch_txt(args: argparse.Namespace):
+ # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
+ locking.acquire()
+
+ # Static URLs
+ urls = (
+ "https://seirdy.one/pb/bsl.txt",
+ )
+
+ print(f"INFO: Checking {len(urls)} text file(s) ...")
+ for url in urls:
+ # DEBUG: print(f"DEBUG: Fetching url='{url}' ...")
+ response = fba.fetch_url(url, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+
+ # DEBUG: print(f"DEBUG: response[]='{type(response)}'")
+ if response.ok and response.text != "":
+ # DEBUG: print(f"DEBUG: Returned {len(response.text.strip())} Bytes for processing")
+ domains = response.text.split("\n")
+
+ print(f"INFO: Processing {len(domains)} domains ...")
+ for domain in domains:
+ if domain == "":
continue
- try:
- print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
- federation.fetch_instances(row["#domain"], block["blocker"], None, inspect.currentframe().f_code.co_name)
- except network.exceptions as exception:
- print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'")
- instances.update_last_error(row["#domain"], exception)
+ # DEBUG: print(f"DEBUG: domain='{domain}'")
+ processed = fba.process_domain(domain, 'seirdy.one', inspect.currentframe().f_code.co_name)
+
+ # DEBUG: print(f"DEBUG: processed='{processed}'")
+ if not processed:
+ # DEBUG: print(f"DEBUG: domain='{domain}' was not generically processed - SKIPPED!")
+ continue
# DEBUG: print("DEBUG: EXIT!")
projects / fba.git / blobdiff