X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Fnetworks%2Fmisskey.py;h=e47c7dd9d04548c5d63171fec5dfb7665a83e69a;hb=1d06145d9e92de8458e3fd6d0a7e75e629466f16;hp=52c32e3e53aa15985c11f13365b7828e9bde7685;hpb=6bc99dfe2757e1e7275befeee91d7ac66ac7869a;p=fba.git diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index 52c32e3..e47c7dd 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -15,36 +15,47 @@ # along with this program. If not, see . import json +import logging -import requests - -from fba import blacklist -from fba import config from fba import csrf -from fba import instances -from fba import network +from fba import utils -from fba.helpers import dicts +from fba.helpers import config +from fba.helpers import domain as domain_helper from fba.helpers import tidyup +from fba.http import network + +from fba.models import instances + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + def fetch_peers(domain: str) -> list: - # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") - - # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") - peers = list() - offset = 0 - step = config.get("misskey_limit") - headers = csrf.determine(domain, {"Origin": domain}) + logger.debug("domain='%s' - CALLED!", domain) + domain_helper.raise_on(domain) + + logger.debug("domain='%s' is misskey, sending API POST request ...", domain) + peers = list() + offset = 0 + step = config.get("misskey_limit") + + # No CSRF by default, you don't have to add network.api_headers by yourself here + headers = tuple() + + try: + logger.debug("Checking CSRF for domain='%s'", domain) + headers = csrf.determine(domain, dict()) + except network.exceptions as exception: + logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s) - EXIT!", type(exception), __name__) + instances.set_last_error(domain, exception) + return list() # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support # sending them all at once while True: - # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") + logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain) if offset == 0: fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", @@ -59,86 +70,86 @@ def fetch_peers(domain: str) -> list: "offset": offset - 1 }), headers) - # DEBUG: print(f"DEBUG: fetched()={len(fetched)}") - if len(fetched) == 0: - # DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'") - break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) - else: - # DEBUG: print(f"DEBUG: Raising offset by step={step}") - offset = offset + step - # Check records - # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}") - if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]: - print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}") - instances.update_last_error(domain, fetched["error"]["message"]) + logger.debug("fetched[]='%s'", type(fetched)) + if "error_message" in fetched: + logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message']) + instances.set_last_error(domain, fetched) break - elif "error_message" in fetched: - print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}") - instances.update_last_error(domain, fetched) + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + logger.warning("post_json_api() returned error: '%s'", fetched['error']['message']) + instances.set_last_error(domain, fetched["json"]["error"]["message"]) break + rows = fetched["json"] + + logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) + if len(rows) == 0: + logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain) + break + elif len(rows) != config.get("misskey_limit"): + logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit')) + offset = offset + (config.get("misskey_limit") - len(rows)) + else: + logger.debug("Raising offset by step=%d", step) + offset = offset + step + already = 0 - for row in fetched["json"]: - # DEBUG: print(f"DEBUG: row()={len(row)}") - if not "host" in row: - print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'") + logger.debug("rows(%d))[]='%s'", len(rows), type(rows)) + for row in rows: + logger.debug("row()=%d", len(row)) + if "host" not in row: + logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain) continue elif not isinstance(row["host"], str): - print(f"WARNING: row[host][]={type(row['host'])} is not 'str'") + logger.warning("row[host][]='%s' is not 'str' - SKIPPED!", type(row['host'])) continue - elif blacklist.is_blacklisted(row["host"]): - # DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'") + elif not utils.is_domain_wanted(row["host"]): + logger.debug("row[host]='%s' is not wanted, domain='%s' - SKIPPED!", row['host'], domain) continue elif row["host"] in peers: - # DEBUG: print(f"DEBUG: Not adding row[host]='{row['host']}', already found.") + logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host']) already = already + 1 continue - # DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'") + logger.debug("Adding peer: row[host]='%s'", row['host']) peers.append(row["host"]) - if already == len(fetched): - # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!") + if already == len(rows): + logger.debug("Host returned same set of %d instance(s) - BREAK!", already) break - # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") - instances.set_data("total_peers", domain, len(peers)) - - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") + logger.debug("peers()=%d - EXIT!", len(peers)) return peers -def fetch_blocks(domain: str) -> dict: - # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!") - if not isinstance(domain, str): - raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'") - elif domain == "": - raise ValueError("Parameter 'domain' is empty") +def fetch_blocks(domain: str) -> list: + logger.debug("domain='%s' - CALLED!", domain) + domain_helper.raise_on(domain) - # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain={domain}") - blocklist = { - "suspended": [], - "blocked" : [] - } + logger.debug("Fetching misskey blocks from domain='%s'", domain) + blocklist = list() offset = 0 step = config.get("misskey_limit") - headers = csrf.determine(domain, {"Origin": domain}) + # No CSRF by default, you don't have to add network.api_headers by yourself here + headers = tuple() + + try: + logger.debug("Checking CSRF for domain='%s'", domain) + headers = csrf.determine(domain, dict()) + except network.exceptions as exception: + logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s) - EXIT!", type(exception), __name__) + instances.set_last_error(domain, exception) + return blocklist + + # iterating through all "suspended" (follow-only in its terminology) + # instances page-by-page since it doesn't support sending them all at once while True: - # iterating through all "suspended" (follow-only in its terminology) - # instances page-by-page, since that troonware doesn't support - # sending them all at once try: - # DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...") + logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain) if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, @@ -146,7 +157,7 @@ def fetch_blocks(domain: str) -> dict: "limit" : step }), headers) else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, @@ -155,99 +166,122 @@ def fetch_blocks(domain: str) -> dict: "offset" : offset - 1 }), headers) - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + logger.debug("fetched[]='%s'", type(fetched)) + if "error_message" in fetched: + logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message']) + instances.set_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + logger.warning("post_json_api() returned error: '%s'", fetched['error']['message']) + instances.set_last_error(domain, fetched["json"]["error"]["message"]) + break + + rows = fetched["json"] + + logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) + if len(rows) == 0: + logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain) break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) + elif len(rows) != config.get("misskey_limit"): + logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit')) + offset = offset + (config.get("misskey_limit") - len(rows)) else: - # DEBUG: print("DEBUG: Raising offset by step:", step) + logger.debug("Raising offset by step=%d", step) offset = offset + step count = 0 - for instance in fetched: + for instance in rows: # Is it there? - if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance): + logger.debug("instance[%s]='%s'", type(instance), instance) + if "isSuspended" in instance and instance["isSuspended"]: count = count + 1 - blocklist["suspended"].append( - { - "domain": tidyup.domain(instance["host"]), - # no reason field, nothing - "reason": None - } - ) - - # DEBUG: print(f"DEBUG: count={count}") + blocklist.append({ + "blocker" : domain, + "blocked" : tidyup.domain(instance["host"]), + "reason" : None, + "block_level": "suspended", + }) + + logger.debug("count=%d", count) if count == 0: - # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") + logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain) break - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: - print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") - instances.update_last_error(domain, exception) + except network.exceptions as exception: + logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception)) + instances.set_last_error(domain, exception) offset = 0 break while True: - # same shit, different asshole ("blocked" aka full suspend) + # Fetch blocked (full suspended) instances try: if offset == 0: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", headers, json.dumps({ + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "blocked": True, "limit" : step - })) + }), headers) else: - # DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset) - fetched = network.post_json_api(domain, "/api/federation/instances", headers, json.dumps({ + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) + fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, "blocked": True, "limit" : step, "offset" : offset - 1 - })) + }), headers) - # DEBUG: print("DEBUG: fetched():", len(fetched)) - if len(fetched) == 0: - # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) + logger.debug("fetched[]='%s'", type(fetched)) + if "error_message" in fetched: + logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message']) + instances.set_last_error(domain, fetched) + break + elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]: + logger.warning("post_json_api() returned error: '%s'", fetched['error']['message']) + instances.set_last_error(domain, fetched["json"]["error"]["message"]) break - elif len(fetched) != config.get("misskey_limit"): - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'") - offset = offset + (config.get("misskey_limit") - len(fetched)) + + rows = fetched["json"] + + logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) + if len(rows) == 0: + logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain) + break + elif len(rows) != config.get("misskey_limit"): + logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit')) + offset = offset + (config.get("misskey_limit") - len(rows)) else: - # DEBUG: print("DEBUG: Raising offset by step:", step) + logger.debug("Raising offset by step=%d", step) offset = offset + step count = 0 - for instance in fetched: + for instance in rows: # Is it there? - if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance): + logger.debug("instance[%s]='%s'", type(instance), instance) + if "isBlocked" in instance and instance["isBlocked"]: count = count + 1 - blocklist["blocked"].append({ - "domain": tidyup.domain(instance["host"]), - "reason": None + blocked = tidyup.domain(instance["host"]) + logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked) + blocklist.append({ + "blocker" : domain, + "blocked" : blocked, + "reason" : None, + "block_level": "reject", }) - # DEBUG: print(f"DEBUG: count={count}") + logger.debug("count=%d", count) if count == 0: - # DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!") + logger.debug("API is no more returning new instances, aborting loop!") break - except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception: - print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'") - instances.update_last_error(domain, exception) + except network.exceptions as exception: + logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception)) + instances.set_last_error(domain, exception) offset = 0 break - # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...") - instances.update_last_instance_fetch(domain) - - # DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}") - return { - "reject" : blocklist["blocked"], - "followers_only": blocklist["suspended"] - } + logger.debug("blocklist()=%d - EXIT!", len(blocklist)) + return blocklist