From: Roland Häder Date: Sun, 11 Jun 2023 02:48:43 +0000 (+0200) Subject: Continued: X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=dbffa1a7bd8b060822a9acb24fff2934752d4856;p=fba.git Continued: - forgot to copy-paste error_message check for "blocked" - also needed to take fetched['json'] for fetched rows --- diff --git a/fba/commands.py b/fba/commands.py index d8917de..401f6a4 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -96,7 +96,7 @@ def fetch_bkali(args: argparse.Namespace): # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...") domains.append(entry["domain"]) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'") sys.exit(255) @@ -109,7 +109,7 @@ def fetch_bkali(args: argparse.Namespace): try: print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") instances.update_last_error(domain, exception) @@ -206,6 +206,7 @@ def fetch_blocks(args: argparse.Namespace): searchres = fba.cursor.fetchone() + print(f"DEBUG: searchres[]='{type(searchres)}'") if searchres is None: print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue @@ -221,6 +222,7 @@ def fetch_blocks(args: argparse.Namespace): searchres = fba.cursor.fetchone() + print(f"DEBUG: searchres[]='{type(searchres)}'") if searchres is None: print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue @@ -322,7 +324,7 @@ def fetch_cs(args: argparse.Namespace): try: print(f"INFO: Fetching instances from domain='{row['domain']}' ...") federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'") instances.update_last_error(row["domain"], exception) @@ -370,7 +372,7 @@ def fetch_fba_rss(args: argparse.Namespace): try: print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") instances.update_last_error(domain, exception) @@ -423,7 +425,7 @@ def fetch_fbabot_atom(args: argparse.Namespace): try: print(f"INFO: Fetching instances from domain='{domain}' ...") federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'") instances.update_last_error(domain, exception) @@ -437,7 +439,7 @@ def fetch_instances(args: argparse.Namespace): try: print(f"INFO: Fetching instances from args.domain='{args.domain}' ...") federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'") instances.update_last_error(args.domain, exception) return @@ -462,7 +464,7 @@ def fetch_instances(args: argparse.Namespace): try: print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'") federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3]) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'") instances.update_last_error(row[0], exception) @@ -495,7 +497,7 @@ def fetch_federater(args: argparse.Namespace): try: print(f"INFO: Fetching instances for instane='{row['#domain']}' ...") federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name) - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'") instances.update_last_error(row["#domain"], exception) diff --git a/fba/network.py b/fba/network.py index 7c0bb8a..759e8e7 100644 --- a/fba/network.py +++ b/fba/network.py @@ -33,6 +33,14 @@ api_headers = { "Content-Type": "application/json", } +# Exceptions to always catch +exceptions = ( + requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.TooManyRedirects, + UnicodeEncodeError +) + def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',data='{data}',headers()={len(headers)} - CALLED!") if not isinstance(domain, str): diff --git a/fba/networks/mastodon.py b/fba/networks/mastodon.py index 4337bf4..424a023 100644 --- a/fba/networks/mastodon.py +++ b/fba/networks/mastodon.py @@ -27,6 +27,7 @@ from fba import csrf from fba import fba from fba import instances from fba import network + from fba.helpers import tidyup language_mapping = { @@ -135,7 +136,9 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): try: # json endpoint for newer mastodongs - blockdict = list() + found_blocks = list() + blocklist = list() + rows = { "reject" : [], "media_removal" : [], @@ -219,13 +222,14 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): ) searchres = fba.cursor.fetchone() + print(f"DEBUG: searchres[]='{type(searchres)}'") if searchres is None: print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!") continue # DEBUG: print("DEBUG: Updating domain: ", searchres[0]) - blocked = searchres[0] - origin = searchres[1] + blocked = searchres[0] + origin = searchres[1] nodeinfo_url = searchres[2] # DEBUG: print("DEBUG: Looking up instance by domain:", blocked) @@ -255,7 +259,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): blocks.add_instance(domain, blocking, reason, block_level) if block_level == "reject": - blockdict.append({ + found_blocks.append({ "blocked": blocked, "reason" : reason }) @@ -266,7 +270,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): # DEBUG: print("DEBUG: Committing changes ...") fba.connection.commit() - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"ERROR: domain='{domain}',software='mastodon',exception[{type(exception)}]:'{str(exception)}'") # DEBUG: print("DEBUG: EXIT!") diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 0327cfc..f3a1eb3 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -134,10 +134,9 @@ def fetch_blocks(domain: str) -> dict: step = config.get("misskey_limit") headers = csrf.determine(domain, {"Origin": domain}) + # iterating through all "suspended" (follow-only in its terminology) + # instances page-by-page since it doesn't support sending them all at once while True: - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once try: # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") if offset == 0: @@ -158,7 +157,6 @@ def fetch_blocks(domain: str) -> dict: "offset" : offset - 1 }), headers) - # Check records # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}") if "error_message" in fetched: print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") @@ -171,7 +169,7 @@ def fetch_blocks(domain: str) -> dict: rows = fetched["json"] - # DEBUG: print("DEBUG: rows():", len(rows)) + # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend") if len(rows) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break @@ -185,7 +183,8 @@ def fetch_blocks(domain: str) -> dict: count = 0 for instance in rows: # Is it there? - if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance): + # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend") + if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]): count = count + 1 blocklist["suspended"].append( { @@ -200,48 +199,61 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") break - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) offset = 0 break while True: - # same shit, different asshole ("blocked" aka full suspend) + # Fetch blocked (full suspended) instances try: if offset == 0: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", headers, json.dumps({ + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "blocked": True, "limit" : step - })) + }), headers) else: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", headers, json.dumps({ + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "blocked": True, "limit" : step, "offset" : offset - 1 - })) + }), headers) + + # DEBUG: print(f"DEBUG: fetched[]={type(fetched)}") + if "error_message" in fetched: + print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") + instances.update_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + instances.update_last_error(domain, fetched["json"]["error"]["message"]) + break + + rows = fetched["json"] - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: + # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked") + if len(rows) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) + elif len(rows) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(rows)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step count = 0 - for instance in fetched: + for instance in rows: # Is it there? - if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance): + # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked") + if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]): count = count + 1 blocklist["blocked"].append({ "domain": tidyup.domain(instance["host"]), @@ -253,7 +265,7 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") break - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError, UnicodeEncodeError) as exception: + except network.exceptions as exception: print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") instances.update_last_error(domain, exception) offset = 0 diff --git a/fba/networks/pleroma.py b/fba/networks/pleroma.py index b997dc6..86f69cd 100644 --- a/fba/networks/pleroma.py +++ b/fba/networks/pleroma.py @@ -94,8 +94,8 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")] ) searchres = fba.cursor.fetchone() - # DEBUG: print("DEBUG: searchres[]:", type(searchres)) + print(f"DEBUG: searchres[]='{type(searchres)}'") if searchres is None: print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue @@ -173,6 +173,7 @@ def fetch_blocks(domain: str, origin: str, nodeinfo_url: str): ) searchres = fba.cursor.fetchone() + print(f"DEBUG: searchres[]='{type(searchres)}'") if searchres is None: print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!") continue