(config.get("connection_timeout"), config.get("read_timeout"))
)
- logger.debug("JSON API returned %d elements", len(fetched))
+ logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
if "error_message" in fetched:
logger.warning("API returned error_message='%s' - EXIT!", fetched["error_message"])
return 101
if "domain" not in row:
logger.warning("row='%s' does not contain element 'domain' - SKIPPED!", row)
continue
- elif row["domain"] is None or row["domain"] == "":
+ elif row["domain"] in [None, ""]:
logger.debug("row[domain]='%s' is empty - SKIPPED!", row["domain"])
continue
if "domain" not in entry:
logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry))
continue
- elif entry["domain"] is None or entry["domain"] == "":
+ elif entry["domain"] in [None, ""]:
logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"])
continue
elif not domain_helper.is_wanted(entry["domain"]):
try:
logger.info("Fetching instances from domain='%s' ...", domain)
- federation.fetch_instances(domain, 'tak.teleyal.blog', None, inspect.currentframe().f_code.co_name)
+ federation.fetch_instances(domain, "tak.teleyal.blog", None, inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
logger.warning("Exception '%s' during fetching instances (fetch_bkali) from domain='%s'", type(exception), domain)
instances.set_last_error(domain, exception)
database.cursor.execute(
"SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software = ? AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_response_time ASC, last_updated ASC", [args.software]
)
- elif args.force:
- # Re-check all
- logger.debug("Re-checking all instances ...")
+ elif args.only_none:
+ # Check only entries with total_blocked=None
database.cursor.execute(
- "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey') AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_response_time ASC, last_updated ASC"
+ "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey') AND total_blocks IS NULL ORDER BY total_blocks DESC, last_response_time ASC, last_updated ASC"
)
else:
# Re-check after "timeout" (aka. minimum interval)
database.cursor.execute(
- "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey') AND (last_blocked IS NULL OR last_blocked < ?) AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_response_time ASC, last_updated ASC", [time.time() - config.get("recheck_block")]
+ "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey') AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_response_time ASC, last_updated ASC"
)
rows = database.cursor.fetchall()
if not domain_helper.is_wanted(blocker):
logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker)
continue
+ elif not args.force and instances.is_recent(blocker, "last_blocked"):
+ logger.debug("blocker='%s' has been recently accessed - SKIPPED!", blocker)
+ continue
logger.debug("Setting last_blocked,has_obfuscation=false for blocker='%s' ...", blocker)
instances.set_last_blocked(blocker)
block["reason"] = tidyup.reason(block["reason"]) if block["reason"] is not None and block["reason"] != "" else None
logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"])
- if block["blocked"] is None or block["blocked"] == "":
+ if block["blocked"] in [None, ""]:
logger.warning("block[blocked]='%s' is empty, blocker='%s'", block["blocked"], blocker)
continue
elif block["blocked"].endswith(".onion"):
logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"])
continue
- elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain"):
+ elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain") == "true":
logger.debug("blocked='%s' is an I2P .onion domain - SKIPPED", block["blocked"])
continue
elif block["blocked"].endswith(".arpa"):
nodeinfo_url = row["nodeinfo_url"]
logger.debug("Looking up instance by domain, blocked='%s'", block["blocked"])
- if block["blocked"] is None or block["blocked"] == "":
+ if block["blocked"] in [None, ""]:
logger.debug("block[blocked]='%s' is empty - SKIPPED!", block["blocked"])
continue
domain = tidyup.domain(domain) if domain not in [None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
domain = tidyup.domain(href) if href not in [None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
origin = row["origin"]
software = row["software"]
- if software_helper.is_relay(software):
- logger.warning("args.domain='%s' is of software type '%s' which is not supported by this command. Please invoke fetch_relays instead.", args.domain, software)
+ if software is None:
+ logger.warning("args.domain='%s' has no software detected. You can try to run ./fba.py update_nodeinfo --domain=%s --force to get it updated.", args.domain, args.domain)
return 102
+ elif software_helper.is_relay(software):
+ logger.warning("args.domain='%s' is of software type '%s' which is not supported by this command. Please invoke fetch_relays instead.", args.domain, software)
+ return 103
# Initial fetch
try:
logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain)
instances.set_last_error(args.domain, exception)
instances.update(args.domain)
- return 100
+ return 104
if args.single:
logger.debug("Not fetching more instances - EXIT!")
# Loop through some instances
database.cursor.execute(
- "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY total_peers DESC, last_response_time ASC, last_updated ASC", [time.time() - config.get("recheck_instance")]
+ "SELECT domain, origin, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY total_peers DESC, last_response_time ASC, last_updated ASC", [time.time() - config.get("recheck_instance")]
)
rows = database.cursor.fetchall()
continue
try:
- logger.info("Fetching instances for domain='%s',origin='%s',software='%s',nodeinfo_url='%s'", domain, row["origin"], row["software"], row["nodeinfo_url"])
- federation.fetch_instances(domain, row["origin"], row["software"], inspect.currentframe().f_code.co_name, row["nodeinfo_url"])
+ logger.info("Fetching instances for domain='%s',origin='%s',software='%s' ...", domain, row["origin"], row["software"])
+ federation.fetch_instances(domain, row["origin"], row["software"], inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
logger.warning("Exception '%s' during fetching instances (fetch_instances) from domain='%s'", type(exception), domain)
instances.set_last_error(domain, exception)
domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
elif not domain_helper.is_wanted(domain):
domain = tidyup.domain(row.contents[0]) if row.contents[0] not in [None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
if (args.force is None or not args.force) and args.domain is None and args.software is None and instances.is_recent(row["domain"], "last_blocked"):
logger.debug("row[domain]='%s' has been recently checked, args.force[]='%s' - SKIPPED!", row["domain"], type(args.force))
continue
+ elif blacklist.is_blacklisted(row["domain"]):
+ logger.warning("row[domain]='%s' is blacklisted - SKIPPED!", row["domain"])
+ continue
logger.debug("Invoking federation.fetch_blocks(%s) ...", row["domain"])
blocking = federation.fetch_blocks(row["domain"])
logger.debug("blocking()=%d", len(blocking))
if len(blocking) == 0:
+ logger.debug("Empty blocking list, trying individual fetch_blocks() for row[software]='%s' ...", row["software"])
if row["software"] == "pleroma":
logger.debug("domain='%s',software='%s'", row["domain"], row["software"])
blocking = pleroma.fetch_blocks(row["domain"])
# c.s isn't part of oliphant's "hidden" blocklists
logger.debug("row[domain]='%s'", row["domain"])
- if row["domain"] != "chaos.social" and not software_helper.is_relay(row["software"]) and not blocklists.has(row["domain"]):
+ if row["domain"] != "chaos.social" and row["software"] is not None and not software_helper.is_relay(row["software"]) and not blocklists.has(row["domain"]):
logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["domain"], len(blocking))
instances.set_last_blocked(row["domain"])
instances.set_total_blocks(row["domain"], blocking)
elif block["blocked"].endswith(".onion"):
logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
continue
- elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain"):
+ elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain") == "true":
logger.debug("blocked='%s' is an I2P onion domain name - SKIPPED!", block["blocked"])
continue
elif block["blocked"].endswith(".arpa"):
domain = tidyup.domain(row["hostname"]) if row["hostname"] not in [None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain(): row[hostname]='%s' - SKIPPED!", domain, row["hostname"])
continue
elif args.no_software:
logger.info("Fetching domains with no software type detected ...")
database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NULL ORDER BY last_updated ASC")
+ elif args.with_software:
+ logger.info("Fetching domains with any software type detected ...")
+ database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NOT NULL ORDER BY last_updated ASC")
elif args.no_auto:
logger.info("Fetching domains with other detection mode than AUTO_DISOVERY being set ...")
database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode IS NOT NULL AND detection_mode != 'AUTO_DISCOVERY' ORDER BY last_updated ASC")
if not args.force and instances.is_recent(row["domain"], "last_nodeinfo"):
logger.debug("row[domain]='%s' has been recently checked - SKIPPED!", row["domain"])
continue
+ elif blacklist.is_blacklisted(row["domain"]):
+ logger.debug("row[domain]='%s' is blacklisted - SKIPPED!", row["domain"])
+ continue
try:
logger.info("Checking nodeinfo for row[domain]='%s',row[software]='%s' (%s%%) ...", row["domain"], row["software"], "{:5.1f}".format(cnt / len(domains) * 100))
headers=headers,
timeout=(config.get("connection_timeout"), config.get("read_timeout"))
)
- logger.debug("fetched[]='%s'", type(fetched))
+ logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
if "error_message" in fetched:
logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"])
{},
(config.get("connection_timeout"), config.get("read_timeout"))
)
- logger.debug("fetched[]='%s'", type(fetched))
+ logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
if "error_message" in fetched:
logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"])
logger.info("Checking %d relays ...", len(rows))
for row in rows:
logger.debug("row[domain]='%s',row[software]='%s' ...", row["domain"], row["software"])
- peers = list()
if not args.force and instances.is_recent(row["domain"]):
logger.debug("row[domain]='%s' has been recently fetched - SKIPPED!", row["domain"])
continue
+ peers = list()
try:
if row["software"] == "pub-relay":
logger.info("Fetching row[nodeinfo_url]='%s' from relay row[domain]='%s',row[software]='%s' ...", row["nodeinfo_url"], row["domain"], row["software"])
domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() from origin='%s' - SKIPPED!", domain, row["domain"])
continue
elif domain not in peers:
domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() from origin='%s' - SKIPPED!", domain, row["domain"])
continue
elif domain not in peers:
domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain is None or domain == "":
+ if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() from origin='%s' - SKIPPED!", domain, row["domain"])
continue
elif domain not in peers: