import json
import logging
+from fba.helpers import blacklist
from fba.helpers import config
from fba.helpers import dicts as dict_helper
from fba.helpers import domain as domain_helper
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
+#logger.setLevel(logging.DEBUG)
def fetch_peers(domain: str) -> list:
logger.debug("domain='%s' - CALLED!", domain)
domain_helper.raise_on(domain)
+ if blacklist.is_blacklisted(domain):
+ raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked")
+ elif not instances.is_registered(domain):
+ raise RuntimeError(f"domain='{domain}' is not registered but function was invoked")
+
logger.debug("domain='%s' is misskey, sending API POST request ...", domain)
- peers = list()
+ peers = []
offset = 0
step = config.get("misskey_limit")
# No CSRF by default, you don't have to add network.api_headers by yourself here
- headers = tuple()
+ headers = {}
try:
logger.debug("Checking CSRF for domain='%s'", domain)
- headers = csrf.determine(domain, dict())
+ headers = csrf.determine(domain, {})
except network.exceptions as exception:
logger.warning("Exception '%s' during checking CSRF (fetch_peers,%s)", type(exception), __name__)
instances.set_last_error(domain, exception)
logger.debug("Returning empty list ... - EXIT!")
- return list()
+ return []
# iterating through all "suspended" (follow-only in its terminology)
# instances page-by-page, since that troonware doesn't support
logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
if offset == 0:
fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
- "sort" : "+pubAt",
+ "sort" : "+pubSub",
"host" : None,
"limit": step
}), headers)
else:
fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
- "sort" : "+pubAt",
+ "sort" : "+pubSub",
"host" : None,
"limit" : step,
"offset": offset - 1
instances.set_last_error(domain, fetched)
break
elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
- logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
+ logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
instances.set_last_error(domain, fetched["json"]["error"]["message"])
break
if len(rows) == 0:
logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
break
- elif len(rows) != config.get("misskey_limit"):
- logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
- offset = offset + (config.get("misskey_limit") - len(rows))
+ elif len(rows) != step:
+ logger.debug("Fetched %d row(s) but expected: %d", len(rows), step)
+ offset = offset + (step - len(rows))
else:
logger.debug("Raising offset by step=%d", step)
offset = offset + step
logger.warning("row()=%d does not contain key 'host': row='%s',domain='%s' - SKIPPED!", len(row), row, domain)
continue
elif not isinstance(row["host"], str):
- logger.warning("row[host][]='%s' is not of type 'str' - SKIPPED!", type(row['host']))
+ logger.warning("row[host][]='%s' has not expected type 'str' - SKIPPED!", type(row["host"]))
+ continue
+ elif row["host"] == "":
+ logger.warning("row[host] is an empty string,domain='%s' - SKIPPED!", domain)
continue
elif row["host"] in peers:
- logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row['host'])
+ logger.debug("Not adding row[host]='%s', already found - SKIPPED!", row["host"])
continue
elif not domain_helper.is_wanted(row["host"]):
logger.debug("row[host]='%s' is not wanted - SKIPPED!", row["host"])
continue
- logger.debug("Adding peer: row[host]='%s'", row['host'])
+ logger.debug("Adding peer: row[host]='%s'", row["host"])
added = added + 1
peers.append(row["host"])
logger.debug("domain='%s' - CALLED!", domain)
domain_helper.raise_on(domain)
- if not instances.is_registered(domain):
- raise Exception(f"domain='{domain}' is not registered but function is invoked.")
+ if blacklist.is_blacklisted(domain):
+ raise RuntimeError(f"domain='{domain}' is blacklisted but function was invoked")
+ elif not instances.is_registered(domain):
+ raise RuntimeError(f"domain='{domain}' is not registered but function was invoked")
# No CSRF by default, you don't have to add network.api_headers by yourself here
- headers = tuple()
+ headers = {}
try:
- logger.debug("Checking CSRF for domain='%s'", domain)
- headers = csrf.determine(domain, dict())
+ logger.debug("Checking CSRF for domain='%s' ...", domain)
+ headers = csrf.determine(domain, {})
except network.exceptions as exception:
logger.warning("Exception '%s' during checking CSRF (fetch_blocks,%s)", type(exception), __name__)
instances.set_last_error(domain, exception)
logger.debug("Returning empty list ... - EXIT!")
- return list()
+ return []
- blocklist = list()
+ blocklist = []
offset = 0
step = config.get("misskey_limit")
# iterating through all "suspended" (follow-only in its terminology)
# instances page-by-page since it doesn't support sending them all at once
- logger.debug("Fetching misskey blocks from domain='%s'", domain)
+ logger.debug("Fetching misskey blocks from domain='%s' ...", domain)
while True:
+ logger.debug("offset=%d", offset)
try:
logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
if offset == 0:
- logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
+ logger.debug("Sending JSON API request to domain='%s',step=%d ...", domain, step)
fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
- "sort" : "+pubAt",
+ "sort" : "+pubSub",
"host" : None,
- "suspended": True,
"limit" : step
}), headers)
else:
- logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
+ logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d ...", domain, step, offset)
fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
- "sort" : "+pubAt",
+ "sort" : "+pubSub",
"host" : None,
- "suspended": True,
"limit" : step,
"offset" : offset - 1
}), headers)
instances.set_last_error(domain, fetched)
break
elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
- logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
+ logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
instances.set_last_error(domain, fetched["json"]["error"]["message"])
break
if len(rows) == 0:
logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
break
- elif len(rows) != config.get("misskey_limit"):
- logger.debug("Fetched %d row(s) but expected: %d", len(rows), config.get('misskey_limit'))
- offset = offset + (config.get("misskey_limit") - len(rows))
+ elif len(rows) != step:
+ logger.debug("Fetched %d row(s) but expected: %d", len(rows), step)
+ offset = offset + (step - len(rows))
else:
logger.debug("Raising offset by step=%d", step)
offset = offset + step
count = 0
+ logger.debug("Checking %d row(s) of instances ...", len(rows))
for instance in rows:
# Is it there?
logger.debug("instance[]='%s'", type(instance))
if "host" not in instance:
logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
continue
- elif instance["host"] is None or instance["host"] == "":
- logger.debug("instance[host]='%s' is None or empty - SKIPPED!", instance["host"])
+ elif not isinstance(instance["host"], str):
+ logger.warning("instance[host][]='%s' has not expected type 'str' - SKIPPED!", type(instance["host"]))
+ continue
+ elif instance["host"] == "":
+ logger.warning("instance[host] is an empty string,domain='%s' - SKIPPED!", domain)
continue
logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
blocked = tidyup.domain(instance["host"])
-
logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked)
- if blocked is None or blocked == "":
+
+ if blocked in [None, ""]:
logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
continue
elif not domain_helper.is_wanted(blocked):
continue
elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
count = count + 1
- logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended'", domain, blocked)
+ logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #1", domain, blocked)
blocklist.append({
"blocker" : domain,
"blocked" : blocked,
"reason" : None,
"block_level": "suspended",
})
-
- logger.debug("count=%d", count)
- if count == 0:
- logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
- break
-
- except network.exceptions as exception:
- logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
- instances.set_last_error(domain, exception)
- offset = 0
- break
-
- while True:
- # Fetch blocked (full suspended) instances
- try:
- if offset == 0:
- logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
- fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
- "sort" : "+pubAt",
- "host" : None,
- "blocked": True,
- "limit" : step
- }), headers)
- else:
- logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d", domain, step, offset)
- fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
- "sort" : "+pubAt",
- "host" : None,
- "blocked": True,
- "limit" : step,
- "offset" : offset - 1
- }), headers)
-
- logger.debug("fetched[]='%s'", type(fetched))
- if "error_message" in fetched:
- logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
- instances.set_last_error(domain, fetched)
- break
- elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
- logger.warning("post_json_api() returned error: '%s'", fetched['error']['message'])
- instances.set_last_error(domain, fetched["json"]["error"]["message"])
- break
-
- rows = fetched["json"]
-
- logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
- if len(rows) == 0:
- logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
- break
- elif len(rows) != config.get("misskey_limit"):
- logger.debug("Fetched %d row(s) but expected: %d'", len(rows), config.get('misskey_limit'))
- offset = offset + (config.get("misskey_limit") - len(rows))
- else:
- logger.debug("Raising offset by step=%d", step)
- offset = offset + step
-
- count = 0
- for instance in rows:
- # Is it there?
- logger.debug("instance[]='%s'", type(instance))
- blocked = tidyup.domain(instance["host"])
-
- logger.debug("blocked='%s'", blocked)
- if blocked is None or blocked == "":
- logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
- continue
- elif not domain_helper.is_wanted(blocked):
- logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
- continue
elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
count = count + 1
- logger.debug("Appending blocker='%s',blocked='%s',block_level='reject'", domain, blocked)
+ logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #2", domain, blocked)
+ blocklist.append({
+ "blocker" : domain,
+ "blocked" : blocked,
+ "reason" : None,
+ "block_level": "suspended",
+ })
+ elif "isSilenced" in instance and instance["isSilenced"] and not dict_helper.has_key(blocklist, "blocked", blocked):
+ count = count + 1
+ logger.debug("Appending blocker='%s',blocked='%s',block_level='silenced' ...", domain, blocked)
blocklist.append({
"blocker" : domain,
"blocked" : blocked,
"reason" : None,
- "block_level": "reject",
+ "block_level": "silenced",
})
+ else:
+ logger.debug("domain='%s',blocked='%s' is not marked suspended - SKIPPED!", domain, blocked)
+ continue
logger.debug("count=%d", count)
if count == 0:
- logger.debug("API is no more returning new instances, aborting loop!")
+ logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
break
except network.exceptions as exception:
projects / fba.git / blobdiff