X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Fcommands.py;h=e746deeb6cc2e7d35e00918c28842ad2395773c0;hb=96f9cee8f4bdd14be4e0f58636a9d8980c880293;hp=3f5b4354eb7e161e1330e8d23370e6cf3f7b6cd4;hpb=f2943f12ef3f5c396873e56c4f52eb31843086d2;p=fba.git diff --git a/fba/commands.py b/fba/commands.py index 3f5b435..e746dee 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -24,6 +24,8 @@ import argparse import atoma import bs4 import markdown +import reqto +import requests import validators from fba import blacklist @@ -61,7 +63,7 @@ def check_instance(args: argparse.Namespace) -> int: return status def fetch_bkali(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") domains = list() try: fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({ @@ -94,7 +96,7 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...") domains.append(entry["domain"]) - except BaseException as exception: + except network.exceptions as exception: print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'") sys.exit(255) @@ -104,13 +106,17 @@ def fetch_bkali(args: argparse.Namespace): print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: - print(f"INFO: Fetching instances from domain='{domain}' ...") - federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from domain='{domain}' ...") + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") def fetch_blocks(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") if args.domain is not None and args.domain != "": # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - checking ...") if not validators.domain(args.domain): @@ -162,13 +168,15 @@ def fetch_blocks(args: argparse.Namespace): mastodon.fetch_blocks(blocker, origin, nodeinfo_url) elif software == "friendica" or software == "misskey": print(f"INFO: blocker='{blocker}',software='{software}'") + + blocking = list() if software == "friendica": - blocks = friendica.fetch_blocks(blocker) + blocking = friendica.fetch_blocks(blocker) elif software == "misskey": - blocks = misskey.fetch_blocks(blocker) + blocking = misskey.fetch_blocks(blocker) - print(f"INFO: Checking {len(blocks.items())} entries from blocker='{blocker}',software='{software}' ...") - for block_level, blocklist in blocks.items(): + print(f"INFO: Checking {len(blocking.items())} entries from blocker='{blocker}',software='{software}' ...") + for block_level, blocklist in blocking.items(): # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist)) block_level = tidyup.domain(block_level) # DEBUG: print("DEBUG: AFTER-block_level:", block_level) @@ -198,12 +206,13 @@ def fetch_blocks(args: argparse.Namespace): searchres = fba.cursor.fetchone() + # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'") if searchres is None: print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue - blocked = searchres[0] - origin = searchres[1] + blocked = searchres[0] + origin = searchres[1] nodeinfo_url = searchres[2] elif blocked.count("?") > 0: # Some obscure them with question marks, not sure if that's dependent on version or not @@ -213,12 +222,13 @@ def fetch_blocks(args: argparse.Namespace): searchres = fba.cursor.fetchone() + # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'") if searchres is None: print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue - blocked = searchres[0] - origin = searchres[1] + blocked = searchres[0] + origin = searchres[1] nodeinfo_url = searchres[2] elif not validators.domain(blocked): print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!") @@ -256,7 +266,7 @@ def fetch_blocks(args: argparse.Namespace): # DEBUG: print("DEBUG: EXIT!") def fetch_cs(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") extensions = [ 'extra', 'abbr', @@ -283,17 +293,17 @@ def fetch_cs(args: argparse.Namespace): } raw = fba.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text - # DEBUG: print(f"DEBUG: raw()={len(raw)}[]={type(raw)}") + # DEBUG: print(f"DEBUG: raw()={len(raw)}[]='{type(raw)}'") doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser') - # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}") + # DEBUG: print(f"DEBUG: doc()={len(doc)}[]='{type(doc)}'") silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody") - # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}") + # DEBUG: print(f"DEBUG: silenced[]='{type(silenced)}'") domains["silenced"] = domains["silenced"] + federation.find_domains(silenced) blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody") - # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}") + # DEBUG: print(f"DEBUG: blocked[]='{type(blocked)}'") domains["reject"] = domains["reject"] + federation.find_domains(blocked) # DEBUG: print(f"DEBUG: domains()={len(domains)}") @@ -311,8 +321,12 @@ def fetch_cs(args: argparse.Namespace): blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level) if not instances.is_registered(row["domain"]): - print(f"INFO: Fetching instances from domain='{row['domain']}' ...") - federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from domain='{row['domain']}' ...") + federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'") + instances.update_last_error(row["domain"], exception) # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() @@ -320,7 +334,7 @@ def fetch_cs(args: argparse.Namespace): # DEBUG: print("DEBUG: EXIT!") def fetch_fba_rss(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") domains = list() print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...") @@ -331,7 +345,7 @@ def fetch_fba_rss(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Parsing RSS feed ({len(response.text)} Bytes) ...") rss = atoma.parse_rss_bytes(response.content) - # DEBUG: print(f"DEBUG: rss[]={type(rss)}") + # DEBUG: print(f"DEBUG: rss[]='{type(rss)}'") for item in rss.items: # DEBUG: print(f"DEBUG: item={item}") domain = item.link.split("=")[1] @@ -355,13 +369,17 @@ def fetch_fba_rss(args: argparse.Namespace): print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: - print(f"INFO: Fetching instances from domain='{domain}' ...") - federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from domain='{domain}' ...") + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") def fetch_fbabot_atom(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") feed = "https://ryona.agency/users/fba/feed.atom" domains = list() @@ -374,11 +392,11 @@ def fetch_fbabot_atom(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Parsing ATOM feed ({len(response.text)} Bytes) ...") atom = atoma.parse_atom_bytes(response.content) - # DEBUG: print(f"DEBUG: atom[]={type(atom)}") + # DEBUG: print(f"DEBUG: atom[]='{type(atom)}'") for entry in atom.entries: - # DEBUG: print(f"DEBUG: entry[]={type(entry)}") + # DEBUG: print(f"DEBUG: entry[]='{type(entry)}'") doc = bs4.BeautifulSoup(entry.content.value, "html.parser") - # DEBUG: print(f"DEBUG: doc[]={type(doc)}") + # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'") for element in doc.findAll("a"): for href in element["href"].split(","): # DEBUG: print(f"DEBUG: href[{type(href)}]={href}") @@ -404,17 +422,27 @@ def fetch_fbabot_atom(args: argparse.Namespace): print(f"INFO: Adding {len(domains)} new instances ...") for domain in domains: - print(f"INFO: Fetching instances from domain='{domain}' ...") - federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from domain='{domain}' ...") + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") + instances.update_last_error(domain, exception) # DEBUG: print("DEBUG: EXIT!") def fetch_instances(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") locking.acquire() # Initial fetch - federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances from args.domain='{args.domain}' ...") + federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'") + instances.update_last_error(args.domain, exception) + return if args.single: # DEBUG: print("DEBUG: Not fetching more instances - EXIT!") @@ -433,13 +461,17 @@ def fetch_instances(args: argparse.Namespace): print("WARNING: domain is blacklisted:", row[0]) continue - print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") - federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) + try: + print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") + federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'") + instances.update_last_error(row[0], exception) # DEBUG: print("DEBUG: EXIT!") def fetch_federater(args: argparse.Namespace): - # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!") + # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!") locking.acquire() # Fetch this URL @@ -450,7 +482,7 @@ def fetch_federater(args: argparse.Namespace): ## DEBUG: print(f"DEBUG: response.content={response.content}") reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect='unix') #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate' - # DEBUG: print(f"DEBUG: reader[]={type(reader)}") + # DEBUG: print(f"DEBUG: reader[]='{type(reader)}'") for row in reader: if not validators.domain(row["#domain"]): print(f"WARNING: domain='{row['#domain']}' is not a valid domain - skipped!") @@ -462,7 +494,11 @@ def fetch_federater(args: argparse.Namespace): # DEBUG: print(f"DEBUG: domain='{row['#domain']}' is already registered - skipped!") continue - print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") - federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) + try: + print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") + federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'") + instances.update_last_error(row["#domain"], exception) # DEBUG: print("DEBUG: EXIT!")