X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Fcommands.py;h=da46e6c752f21a3cc091cc37feaa2c1d0df0c319;hb=b4b7a362cf34916fccecded268a717f6263c2d84;hp=1c0f74ec831a66ee62f9a5b9441effb14402bc23;hpb=92a6263c5be5992e094bcab62821a1eb437a5767;p=fba.git diff --git a/fba/commands.py b/fba/commands.py index 1c0f74e..da46e6c 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -118,7 +118,7 @@ def fetch_bkali(args: argparse.Namespace) -> int: print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_bkali) from domain='{domain}'") instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -210,46 +210,32 @@ def fetch_blocks(args: argparse.Namespace): continue elif blocked.count("*") > 0: # Some friendica servers also obscure domains without hash - fba.cursor.execute( - "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] - ) + row = instances.deobscure("*", blocked) - searchres = fba.cursor.fetchone() - - # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'") - if searchres is None: - print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + # DEBUG: print(f"DEBUG: row[]='{type(row)}'") + if row is None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocker='{blocker}',software='{software}' - SKIPPED!") continue - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] + blocked = row[0] + origin = row[1] + nodeinfo_url = row[2] elif blocked.count("?") > 0: # Some obscure them with question marks, not sure if that's dependent on version or not - fba.cursor.execute( - "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")] - ) - - searchres = fba.cursor.fetchone() + row = instances.deobscure("?", blocked) - # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'") - if searchres is None: - print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") + # DEBUG: print(f"DEBUG: row[]='{type(row)}'") + if row is None: + print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocker='{blocker}',software='{software}' - SKIPPED!") continue - blocked = searchres[0] - origin = searchres[1] - nodeinfo_url = searchres[2] - elif not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") - continue + blocked = row[0] + origin = row[1] + nodeinfo_url = row[2] # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) if not validators.domain(blocked): - print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") - continue - elif blocked.split(".")[-1] == "arpa": - print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.") + print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - SKIPPED!") continue elif not instances.is_registered(blocked): # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker) @@ -342,7 +328,7 @@ def fetch_cs(args: argparse.Namespace): print(f"INFO: Fetching instances from domain='{row['domain']}' ...") federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_cs) from domain='{row['domain']}'") instances.update_last_error(row["domain"], exception) # DEBUG: print("DEBUG: Committing changes ...") @@ -390,7 +376,7 @@ def fetch_fba_rss(args: argparse.Namespace): print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fba_rss) from domain='{domain}'") instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -443,7 +429,7 @@ def fetch_fbabot_atom(args: argparse.Namespace): print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_fbabot_atom) from domain='{domain}'") instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") @@ -457,8 +443,9 @@ def fetch_instances(args: argparse.Namespace) -> int: print(f"INFO: Fetching instances from args.domain='{args.domain}' ...") federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from args.domain='{args.domain}'") instances.update_last_error(args.domain, exception) + return 100 if args.single: @@ -482,7 +469,7 @@ def fetch_instances(args: argparse.Namespace) -> int: print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'") + print(f"WARNING: Exception '{type(exception)}' during fetching instances (fetch_instances) from domain='{row[0]}'") instances.update_last_error(row[0], exception) # DEBUG: print("DEBUG: EXIT!") @@ -536,11 +523,15 @@ def fetch_oliphant(args: argparse.Namespace): } ) + domains = list() for block in blocklists: # Is domain given and not equal blocker? if isinstance(args.domain, str) and args.domain != block["blocker"]: # DEBUG: print(f"DEBUG: Skipping blocker='{block['blocker']}', not matching args.domain='{args.domain}'") continue + elif domain in domains: + # DEBUG: print(f"DEBUG: domain='{domain}' already handled - SKIPPED!") + continue # Fetch this URL print(f"INFO: Fetching csv_url='{block['csv_url']}' for blocker='{block['blocker']}' ...") @@ -549,26 +540,59 @@ def fetch_oliphant(args: argparse.Namespace): # DEBUG: print(f"DEBUG: response[]='{type(response)}'") if response.ok and response.content != "": # DEBUG: print(f"DEBUG: Fetched {len(response.content)} Bytes, parsing CSV ...") - #print(f"DEBUG: response.content={response.content}") reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix") - #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate' + # DEBUG: print(f"DEBUG: reader[]='{type(reader)}'") for row in reader: - if not validators.domain(row["#domain"]): - print(f"WARNING: domain='{row['#domain']}' is not a valid domain - skipped!") - continue - elif blacklist.is_blacklisted(row["#domain"]): - print(f"WARNING: domain='{row['#domain']}' is blacklisted - skipped!") + domain = None + if "#domain" in row: + domain = row["#domain"] + elif "domain" in row: + domain = row["domain"] + else: + # DEBUG: print(f"DEBUG: row='{row}' does not contain domain column") continue - elif instances.is_recent(row["#domain"]): - # DEBUG: print(f"DEBUG: domain='{row['#domain']}' has been recently checked - skipped!") + + # DEBUG: print(f"DEBUG: Marking domain='{domain}' as handled") + domains.append(domain) + + # DEBUG: print(f"DEBUG: Processing domain='{domain}' ...") + processed = fba.process_domain(domain, block["blocker"], inspect.currentframe().f_code.co_name) + + # DEBUG: print(f"DEBUG: processed='{processed}'") + + # DEBUG: print("DEBUG: EXIT!") + +def fetch_txt(args: argparse.Namespace): + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") + locking.acquire() + + # Static URLs + urls = ( + "https://seirdy.one/pb/bsl.txt", + ) + + print(f"INFO: Checking {len(urls)} text file(s) ...") + for url in urls: + # DEBUG: print(f"DEBUG: Fetching url='{url}' ...") + response = fba.fetch_url(url, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) + + # DEBUG: print(f"DEBUG: response[]='{type(response)}'") + if response.ok and response.text != "": + # DEBUG: print(f"DEBUG: Returned {len(response.text.strip())} Bytes for processing") + domains = response.text.split("\n") + + print(f"INFO: Processing {len(domains)} domains ...") + for domain in domains: + if domain == "": continue - try: - print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") - federation.fetch_instances(row["#domain"], block["blocker"], None, inspect.currentframe().f_code.co_name) - except network.exceptions as exception: - print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'") - instances.update_last_error(row["#domain"], exception) + # DEBUG: print(f"DEBUG: domain='{domain}'") + processed = fba.process_domain(domain, 'seirdy.one', inspect.currentframe().f_code.co_name) + + # DEBUG: print(f"DEBUG: processed='{processed}'") + if not processed: + # DEBUG: print(f"DEBUG: domain='{domain}' was not generically processed - SKIPPED!") + continue # DEBUG: print("DEBUG: EXIT!")