X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Fnetworks%2Fmisskey.py;h=045b5061c807fa4547802e26edcef8243e4b31b5;hb=f9d221393d508c052c56cbc8abf04aa411776454;hp=c18ef32c521ee17fc3a835e19edd2c76f4af6a4d;hpb=a96a4b8ca8e9296a2cd61e2ed15e8de51da13003;p=fba.git diff --git a/fba/networks/misskey.py b/fba/networks/misskey.py index c18ef32..045b506 100644 --- a/fba/networks/misskey.py +++ b/fba/networks/misskey.py @@ -17,14 +17,12 @@ import json import logging -from fba import csrf -from fba import utils - from fba.helpers import config -from fba.helpers import dicts +from fba.helpers import dicts as dict_helper from fba.helpers import domain as domain_helper from fba.helpers import tidyup +from fba.http import csrf from fba.http import network from fba.models import instances @@ -33,7 +31,7 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) def fetch_peers(domain: str) -> list: - logger.debug("domain(%d)='%s' - CALLED!", len(domain), domain) + logger.debug("domain='%s' - CALLED!", domain) domain_helper.raise_on(domain) logger.debug("domain='%s' is misskey, sending API POST request ...", domain) @@ -48,15 +46,17 @@ def fetch_peers(domain: str) -> list: logger.debug("Checking CSRF for domain='%s'", domain) headers = csrf.determine(domain, dict()) except network.exceptions as exception: - logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s) - EXIT!", type(exception), __name__) + logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__) instances.set_last_error(domain, exception) - return peers + + logger.debug("Returning empty list ... - EXIT!") + return list() # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support # sending them all at once while True: - logger.debug("Fetching offset='%d' from domain='%s' ...", offset, domain) + logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain) if offset == 0: fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", @@ -84,7 +84,7 @@ def fetch_peers(domain: str) -> list: rows = fetched["json"] - logger.debug("rows(%d)[]='%s'", len(rows), type(rows)) + logger.debug("rows(%d)[]='%s',step=%d", len(rows), type(rows), step) if len(rows) == 0: logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain) break @@ -92,52 +92,44 @@ def fetch_peers(domain: str) -> list: logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit')) offset = offset + (config.get("misskey_limit") - len(rows)) else: - logger.debug("Raising offset by step='%d'", step) + logger.debug("Raising offset by step=%d", step) offset = offset + step - already = 0 + added = 0 logger.debug("rows(%d))[]='%s'", len(rows), type(rows)) for row in rows: - logger.debug(f"row()={len(row)}") + logger.debug("row()=%d", len(row)) if "host" not in row: logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain) continue elif not isinstance(row["host"], str): - logger.warning("row[host][]='%s' is not 'str' - SKIPPED!", type(row['host'])) - continue - elif not utils.is_domain_wanted(row["host"]): - logger.debug("row[host]='%s' is not wanted, domain='{domain}' - SKIPPED!", row['host']) + logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host'])) continue elif row["host"] in peers: logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host']) - already = already + 1 + continue + elif not domain_helper.is_wanted(row["host"]): + logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"]) continue - logger.debug("Adding peer: '%s'", row['host']) + logger.debug("Adding peer: row[host]='%s'", row['host']) + added = added + 1 peers.append(row["host"]) - if already == len(rows): - logger.debug("Host returned same set of %d instance(s) - BREAK!", already) + logger.debug("added=%d,rows()=%d", added, len(rows)) + if added == 0: + logger.debug("Host returned already added (%d) peers - BREAK!", len(rows)) break - logger.debug("Adding %d for domain='%s'", len(peers), domain) - instances.set_total_peers(domain, peers) - logger.debug("peers()=%d - EXIT!", len(peers)) return peers -def fetch_blocks(domain: str) -> dict: - logger.debug("domain(%d)='%s' - CALLED!", len(domain), domain) +def fetch_blocks(domain: str) -> list: + logger.debug("domain='%s' - CALLED!", domain) domain_helper.raise_on(domain) - logger.debug("Fetching misskey blocks from domain='%s'", domain) - blocklist = { - "suspended": [], - "blocked" : [] - } - - offset = 0 - step = config.get("misskey_limit") + if not instances.is_registered(domain): + raise Exception(f"domain='{domain}' is not registered but function is invoked.") # No CSRF by default, you don't have to add network.api_headers by yourself here headers = tuple() @@ -146,17 +138,24 @@ def fetch_blocks(domain: str) -> dict: logger.debug("Checking CSRF for domain='%s'", domain) headers = csrf.determine(domain, dict()) except network.exceptions as exception: - logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s) - EXIT!", type(exception), __name__) + logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__) instances.set_last_error(domain, exception) - return blocklist + + logger.debug("Returning empty list ... - EXIT!") + return list() + + blocklist = list() + offset = 0 + step = config.get("misskey_limit") # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page since it doesn't support sending them all at once + logger.debug("Fetching misskey blocks from domain='%s'", domain) while True: try: - logger.debug("Fetching offset='%d' from domain='%s' ...", offset, domain) + logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain) if offset == 0: - logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset) + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, @@ -164,7 +163,7 @@ def fetch_blocks(domain: str) -> dict: "limit" : step }), headers) else: - logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset) + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, @@ -193,19 +192,38 @@ def fetch_blocks(domain: str) -> dict: logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit')) offset = offset + (config.get("misskey_limit") - len(rows)) else: - logger.debug("Raising offset by step='%d'", step) + logger.debug("Raising offset by step=%d", step) offset = offset + step count = 0 for instance in rows: # Is it there? - logger.debug("instance[%s]='%s'", type(instance), instance) - if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]): + logger.debug("instance[]='%s'", type(instance)) + if "host" not in instance: + logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance) + continue + elif instance["host"] is None or instance["host"] == "": + logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"]) + continue + + logger.debug("instance[host]='%s' - BEFORE!", instance["host"]) + blocked = tidyup.domain(instance["host"]) + + logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked) + if blocked is None or blocked == "": + logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"]) + continue + elif not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue + elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked): count = count + 1 - blocklist["suspended"].append({ - "domain": tidyup.domain(instance["host"]), - # no reason field, nothing - "reason": None + logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked) + blocklist.append({ + "blocker" : domain, + "blocked" : blocked, + "reason" : None, + "block_level": "suspended", }) logger.debug("count=%d", count) @@ -223,7 +241,7 @@ def fetch_blocks(domain: str) -> dict: # Fetch blocked (full suspended) instances try: if offset == 0: - logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset) + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, @@ -231,7 +249,7 @@ def fetch_blocks(domain: str) -> dict: "limit" : step }), headers) else: - logger.debug("Sending JSON API request to domain='%s',step='%d',offset='%d'", domain, step, offset) + logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset) fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({ "sort" : "+pubAt", "host" : None, @@ -257,21 +275,33 @@ def fetch_blocks(domain: str) -> dict: logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain) break elif len(rows) != config.get("misskey_limit"): - logger.debug("Fetched %d row(s) but expected: '{config.get('misskey_limit')}'", len(rows)) + logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit')) offset = offset + (config.get("misskey_limit") - len(rows)) else: - logger.debug("Raising offset by step='%d'", step) + logger.debug("Raising offset by step=%d", step) offset = offset + step count = 0 for instance in rows: # Is it there? - logger.debug("instance[%s]='%s'", type(instance), instance) - if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]): + logger.debug("instance[]='%s'", type(instance)) + blocked = tidyup.domain(instance["host"]) + + logger.debug("blocked='%s'", blocked) + if blocked is None or blocked == "": + logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"]) + continue + elif not domain_helper.is_wanted(blocked): + logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked) + continue + elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked): count = count + 1 - blocklist["blocked"].append({ - "domain": tidyup.domain(instance["host"]), - "reason": None + logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked) + blocklist.append({ + "blocker" : domain, + "blocked" : blocked, + "reason" : None, + "block_level": "reject", }) logger.debug("count=%d", count) @@ -285,8 +315,5 @@ def fetch_blocks(domain: str) -> dict: offset = 0 break - logger.debug("blocked()=%d,suspended()=%d - EXIT!", len(blocklist['blocked']), len(blocklist['suspended'])) - return { - "reject" : blocklist["blocked"], - "followers_only": blocklist["suspended"] - } + logger.debug("blocklist()=%d - EXIT!", len(blocklist)) + return blocklist