X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;ds=sidebyside;f=fba%2Fnetworks%2Fmisskey.py;h=47a3de012440aadbdbdbd2fde0076bdc5a970c67;hb=c0ac309c7ca70790fba3715cdb61e75570c5950d;hp=7a32f8e738cb11ab763add8e461c0cd9eefc8753;hpb=f71083f2182908ee7bd6e8c24bdeeb3778cb9b41;p=fba.git diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 7a32f8e..47a3de0 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -16,27 +16,39 @@ import json -import requests - -from fba import blacklist -from fba import config -from fba import instances -from fba import network +from fba import csrf +from fba.helpers import blacklist +from fba.helpers import config from fba.helpers import dicts from fba.helpers import tidyup +from fba.http import network + +from fba.models import instances + def fetch_peers(domain: str) -> list: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!") + # DEBUG: print(f"DEBUG: domain({len(domain)})='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") - peers = list() - offset = 0 - step = config.get("misskey_limit") + peers = list() + offset = 0 + step = config.get("misskey_limit") + + # No CSRF by default, you don't have to add network.api_headers by yourself here + headers = tuple() + + try: + # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") + headers = csrf.determine(domain, dict()) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!") + instances.set_last_error(domain, exception) + return peers # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support @@ -48,49 +60,48 @@ def fetch_peers(domain: str) -> list: "sort" : "+pubAt", "host" : None, "limit": step - }), { - "Origin": domain - }) + }), headers) else: fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "limit" : step, "offset": offset - 1 - }), { - "Origin": domain - }) + }), headers) + + # Check records + # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") + if "error_message" in fetched: + print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") + instances.set_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + instances.set_last_error(domain, fetched["json"]["error"]["message"]) + break + + rows = fetched["json"] - # DEBUG: print(f"DEBUG: fetched()={len(fetched)}") - if len(fetched) == 0: + # DEBUG: print(f"DEBUG: rows()={len(rows)}") + if len(rows) == 0: # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'") break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) + elif len(rows) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(rows)) else: # DEBUG: print(f"DEBUG: Raising offset by step={step}") offset = offset + step - # Check records - # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}") - if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]: - print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - instances.update_last_error(domain, fetched["error"]["message"]) - break - elif "error_message" in fetched: - print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") - instances.update_last_error(domain, fetched) - break - already = 0 - for row in fetched["json"]: + # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'") + for row in rows: # DEBUG: print(f"DEBUG: row()={len(row)}") - if not "host" in row: + if "host" not in row: print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'") continue elif not isinstance(row["host"], str): - print(f"WARNING: row[host][]={type(row['host'])} is not 'str'") + print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'") continue elif blacklist.is_blacklisted(row["host"]): # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") @@ -103,15 +114,12 @@ def fetch_peers(domain: str) -> list: # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'") peers.append(row["host"]) - if already == len(fetched): - print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!") + if already == len(rows): + # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!") break # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) + instances.set_total_peers(domain, peers) # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") return peers @@ -119,22 +127,33 @@ def fetch_peers(domain: str) -> list: def fetch_blocks(domain: str) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") + raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain) + # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain='{domain}'") blocklist = { "suspended": [], "blocked" : [] } - offset = 0 - step = config.get("misskey_limit") + offset = 0 + step = config.get("misskey_limit") + + # No CSRF by default, you don't have to add network.api_headers by yourself here + headers = tuple() + + try: + # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") + headers = csrf.determine(domain, dict()) + except network.exceptions as exception: + print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_blocks,{__name__}) - EXIT!") + instances.set_last_error(domain, exception) + return blocklist + + # iterating through all "suspended" (follow-only in its terminology) + # instances page-by-page since it doesn't support sending them all at once while True: - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once try: # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") if offset == 0: @@ -144,9 +163,7 @@ def fetch_blocks(domain: str) -> dict: "host" : None, "suspended": True, "limit" : step - }), { - "Origin": domain - }) + }), headers) else: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ @@ -155,47 +172,56 @@ def fetch_blocks(domain: str) -> dict: "suspended": True, "limit" : step, "offset" : offset - 1 - }), { - "Origin": domain - }) + }), headers) + + # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") + if "error_message" in fetched: + print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") + instances.set_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + instances.set_last_error(domain, fetched["json"]["error"]["message"]) + break + + rows = fetched["json"] - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: + # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend") + if len(rows) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) + elif len(rows) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(rows)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step count = 0 - for instance in fetched: + for instance in rows: # Is it there? - if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance): + # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend") + if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]): count = count + 1 - blocklist["suspended"].append( - { - "domain": tidyup.domain(instance["host"]), - # no reason field, nothing - "reason": None - } - ) + blocklist["suspended"].append({ + "domain": tidyup.domain(instance["host"]), + # no reason field, nothing + "reason": None + }) # DEBUG: print(f"DEBUG: count={count}") if count == 0: # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") break - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: + except network.exceptions as exception: print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) offset = 0 break while True: - # same shit, different asshole ("blocked" aka full suspend) + # Fetch blocked (full suspended) instances try: if offset == 0: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) @@ -204,9 +230,7 @@ def fetch_blocks(domain: str) -> dict: "host" : None, "blocked": True, "limit" : step - }), { - "Origin": domain - }) + }), headers) else: # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ @@ -215,25 +239,36 @@ def fetch_blocks(domain: str) -> dict: "blocked": True, "limit" : step, "offset" : offset - 1 - }), { - "Origin": domain - }) + }), headers) + + # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'") + if "error_message" in fetched: + print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") + instances.set_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") + instances.set_last_error(domain, fetched["json"]["error"]["message"]) + break - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: + rows = fetched["json"] + + # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked") + if len(rows) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) + elif len(rows) != config.get("misskey_limit"): + # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'") + offset = offset + (config.get("misskey_limit") - len(rows)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step count = 0 - for instance in fetched: + for instance in rows: # Is it there? - if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance): + # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked") + if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]): count = count + 1 blocklist["blocked"].append({ "domain": tidyup.domain(instance["host"]), @@ -245,15 +280,12 @@ def fetch_blocks(domain: str) -> dict: # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") break - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: + except network.exceptions as exception: print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") - instances.update_last_error(domain, exception) + instances.set_last_error(domain, exception) offset = 0 break - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}") return { "reject" : blocklist["blocked"],