def check_instance(args: argparse.Namespace) -> int:
logger.debug("args.domain='%s' - CALLED!", args.domain)
+
status = 0
if not validators.domain(args.domain):
logger.warning("args.domain='%s' is not valid", args.domain)
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
(config.get("connection_timeout"), config.get("read_timeout"))
)
- logger.debug("JSON API returned %d elements", len(fetched))
+ logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
if "error_message" in fetched:
logger.warning("API returned error_message='%s' - EXIT!", fetched["error_message"])
return 101
if "domain" not in row:
logger.warning("row='%s' does not contain element 'domain' - SKIPPED!", row)
continue
- elif row["domain"] == "":
- logger.debug("row[domain] is empty - SKIPPED!")
+ elif row["domain"] in [None, ""]:
+ logger.debug("row[domain]='%s' is empty - SKIPPED!", row["domain"])
continue
logger.debug("row[domain]='%s' - BEFORE!", row["domain"])
source_domain = "gql.api.bka.li"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
if "domain" not in entry:
logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry))
continue
- elif entry["domain"] == "":
- logger.debug("entry[domain] is empty - SKIPPED!")
+ elif entry["domain"] in [None, ""]:
+ logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"])
continue
elif not domain_helper.is_wanted(entry["domain"]):
logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"])
try:
logger.info("Fetching instances from domain='%s' ...", domain)
- federation.fetch_instances(domain, 'tak.teleyal.blog', None, inspect.currentframe().f_code.co_name)
+ federation.fetch_instances(domain, "tak.teleyal.blog", None, inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
logger.warning("Exception '%s' during fetching instances (fetch_bkali) from domain='%s'", type(exception), domain)
instances.set_last_error(domain, exception)
# Re-check single software
logger.debug("Querying database for args.software='%s' ...", args.software)
database.cursor.execute(
- "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software = ? AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_updated ASC", [args.software]
+ "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software = ? AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_response_time ASC, last_updated ASC", [args.software]
)
- elif args.force:
- # Re-check all
- logger.debug("Re-checking all instances ...")
+ elif args.only_none:
+ # Check only entries with total_blocked=None
database.cursor.execute(
- "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey') AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_updated ASC"
+ "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey') AND total_blocks IS NULL ORDER BY total_blocks DESC, last_response_time ASC, last_updated ASC"
)
else:
# Re-check after "timeout" (aka. minimum interval)
database.cursor.execute(
- "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey') AND (last_blocked IS NULL OR last_blocked < ?) AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_updated ASC", [time.time() - config.get("recheck_block")]
+ "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey') AND nodeinfo_url IS NOT NULL ORDER BY total_blocks DESC, last_response_time ASC, last_updated ASC"
)
rows = database.cursor.fetchall()
if not domain_helper.is_wanted(blocker):
logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker)
continue
+ elif not args.force and instances.is_recent(blocker, "last_blocked"):
+ logger.debug("blocker='%s' has been recently accessed - SKIPPED!", blocker)
+ continue
logger.debug("Setting last_blocked,has_obfuscation=false for blocker='%s' ...", blocker)
instances.set_last_blocked(blocker)
instances.set_has_obfuscation(blocker, False)
# c.s isn't part of oliphant's "hidden" blocklists
- if blocker == "chaos.social" or blocklists.has(blocker):
- logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs or fetch_oliphant instead!", blocker)
+ if blocker == "chaos.social" or software_helper.is_relay(software) or blocklists.has(blocker):
+ logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs, fetch_oliphant, fetch_csv instead!", blocker)
continue
logger.debug("Invoking federation.fetch_blocks(%s) ...", blocker)
blocking = federation.fetch_blocks(blocker)
- logger.debug("blocking()=%d,nodeinfo_url='%s'", len(blocking), nodeinfo_url)
+ logger.debug("blocker='%s',software='%s',blocking()=%d", blocker, software, len(blocking))
if len(blocking) == 0:
- logger.debug("blocker='%s',software='%s'", blocker, software)
+ logger.debug("blocker='%s',software='%s' - fetching blocklist ...", blocker, software)
if software == "pleroma":
- logger.info("blocker='%s',software='%s'", blocker, software)
- blocking = pleroma.fetch_blocks(blocker, nodeinfo_url)
+ blocking = pleroma.fetch_blocks(blocker)
logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
elif software == "mastodon":
- logger.info("blocker='%s',software='%s'", blocker, software)
- blocking = mastodon.fetch_blocks(blocker, nodeinfo_url)
+ blocking = mastodon.fetch_blocks(blocker)
logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
elif software == "lemmy":
- logger.info("blocker='%s',software='%s'", blocker, software)
- blocking = lemmy.fetch_blocks(blocker, nodeinfo_url)
+ blocking = lemmy.fetch_blocks(blocker)
logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
elif software == "friendica":
- logger.info("blocker='%s',software='%s'", blocker, software)
blocking = friendica.fetch_blocks(blocker)
logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
elif software == "misskey":
- logger.info("blocker='%s',software='%s'", blocker, software)
blocking = misskey.fetch_blocks(blocker)
logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software)
else:
logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", blocker, len(blocking))
instances.set_total_blocks(blocker, blocking)
- logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software)
blockdict = list()
+ deobfuscated = obfuscated = 0
+
+ logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software)
for block in blocking:
logger.debug("blocked='%s',block_level='%s',reason='%s'", block["blocked"], block["block_level"], block["reason"])
block["reason"] = tidyup.reason(block["reason"]) if block["reason"] is not None and block["reason"] != "" else None
logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"])
- if block["blocked"] == "":
- logger.warning("blocked is empty, blocker='%s'", blocker)
+ if block["blocked"] in [None, ""]:
+ logger.warning("block[blocked]='%s' is empty, blocker='%s'", block["blocked"], blocker)
continue
elif block["blocked"].endswith(".onion"):
logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"])
continue
+ elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain") == "true":
+ logger.debug("blocked='%s' is an I2P .onion domain - SKIPPED", block["blocked"])
+ continue
elif block["blocked"].endswith(".arpa"):
logger.debug("blocked='%s' is a reverse IP address - SKIPPED", block["blocked"])
continue
continue
elif block["blocked"].find("*") >= 0:
logger.debug("blocker='%s' uses obfuscated domains", blocker)
+ instances.set_has_obfuscation(blocker, True)
+ obfuscated = obfuscated + 1
# Some friendica servers also obscure domains without hash
- row = instances.deobfuscate("*", block["blocked"], block["hash"] if "hash" in block else None)
+ row = instances.deobfuscate("*", block["blocked"], block["digest"] if "digest" in block else None)
logger.debug("row[]='%s'", type(row))
if row is None:
logger.warning("Cannot deobfuscate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software)
- instances.set_has_obfuscation(blocker, True)
continue
+ deobfuscated = deobfuscated + 1
block["blocked"] = row["domain"]
origin = row["origin"]
nodeinfo_url = row["nodeinfo_url"]
elif block["blocked"].find("?") >= 0:
logger.debug("blocker='%s' uses obfuscated domains", blocker)
+ instances.set_has_obfuscation(blocker, True)
+ obfuscated = obfuscated + 1
# Some obscure them with question marks, not sure if that's dependent on version or not
- row = instances.deobfuscate("?", block["blocked"], block["hash"] if "hash" in block else None)
+ row = instances.deobfuscate("?", block["blocked"], block["digest"] if "digest" in block else None)
logger.debug("row[]='%s'", type(row))
if row is None:
logger.warning("Cannot deobfuscate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software)
- instances.set_has_obfuscation(blocker, True)
continue
+ deobfuscated = deobfuscated + 1
block["blocked"] = row["domain"]
origin = row["origin"]
nodeinfo_url = row["nodeinfo_url"]
- logger.debug("Looking up instance by domainm, blocked='%s'", block["blocked"])
- if block["blocked"] == "":
- logger.debug("block[blocked] is empty - SKIPPED!")
+ logger.debug("Looking up instance by domain, blocked='%s'", block["blocked"])
+ if block["blocked"] in [None, ""]:
+ logger.debug("block[blocked]='%s' is empty - SKIPPED!", block["blocked"])
continue
logger.debug("block[blocked]='%s' - BEFORE!", block["blocked"])
logger.debug("Invoking cookies.clear(%s) ...", block["blocked"])
cookies.clear(block["blocked"])
- logger.debug("Checking if blocker='%s' has pending updates ...", blocker)
- if instances.has_pending(blocker):
- logger.debug("Flushing updates for blocker='%s' ...", blocker)
- instances.update(blocker)
+ logger.info("blocker='%s' has %d obfuscated domain(s) and %d of them could be deobfuscated.", blocker, obfuscated, deobfuscated)
+ instances.set_obfuscated_blocks(blocker, obfuscated)
+
+ logger.debug("Flushing updates for blocker='%s' ...", blocker)
+ instances.update(blocker)
logger.debug("Invoking commit() ...")
database.connection.commit()
source_domain = "fediverse.observer"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
logger.info("Fetching %d different table data ...", len(types))
for software in types:
- logger.debug("software='%s' - BEFORE!", software)
+ logger.debug("software='%s'", software)
+
if args.software is not None and args.software != software:
logger.debug("args.software='%s' does not match software='%s' - SKIPPED!", args.software, software)
continue
for item in items:
logger.debug("item[]='%s'", type(item))
domain = item.decode_contents()
+ logger.debug("domain[%s]='%s'", type(domain), domain)
+ domain = tidyup.domain(domain) if domain not in [None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("domain is empty - SKIPPED!")
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
logger.debug("domain='%s' - BEFORE!", domain)
logger.debug("domain='%s' is already registered - SKIPPED!", domain)
continue
- software = software_helper.alias(software)
- logger.info("Fetching instances for domain='%s'", domain)
+ logger.info("Fetching instances for domain='%s' ...", domain)
federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
logger.debug("Success! - EXIT!")
source_domain = "wiki.todon.eu"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked)
instances.set_last_error(blocked, exception)
- if blocks.is_instance_blocked(blocker, blocked, block_level):
+ if not domain_helper.is_wanted(blocked):
+ logger.warning("blocked='%s' is not wanted - SKIPPED!", blocked)
+ continue
+ elif not domain_helper.is_wanted(blocker):
+ logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker)
+ continue
+ elif blocks.is_instance_blocked(blocker, blocked, block_level):
logger.debug("blocked='%s',block_level='%s' is already blocked - SKIPPED!", blocked, block_level)
continue
source_domain = "raw.githubusercontent.com"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
locking.acquire()
components = urlparse(args.feed)
+ domain = components.netloc.lower().split(":")[0]
- if sources.is_recent(components.netloc):
- logger.info("API from components.netloc='%s' has recently being accessed - EXIT!", components.netloc)
+ logger.debug("domain='%s'", domain)
+ if sources.is_recent(domain):
+ logger.info("API from domain='%s' has recently being accessed - EXIT!", domain)
return 0
else:
- logger.debug("components.netloc='%s' has not been recently used, marking ...", components.netloc)
- sources.update(components.netloc)
+ logger.debug("domain='%s' has not been recently used, marking ...", domain)
+ sources.update(domain)
logger.info("Fetch FBA-specific RSS args.feed='%s' ...", args.feed)
response = utils.fetch_url(args.feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
logger.debug("rss[]='%s'", type(rss))
for item in rss.items:
logger.debug("item[%s]='%s'", type(item), item)
- domain = tidyup.domain(item.link.split("=")[1])
+ domain = item.link.split("=")[1]
+ domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("domain is empty - SKIPPED!")
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
logger.debug("domain='%s' - BEFORE!", domain)
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
logger.debug("entry[]='%s'", type(entry))
doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
logger.debug("doc[]='%s'", type(doc))
- for element in doc.findAll("a"):
- logger.debug("element[]='%s'", type(element))
+ elements = doc.findAll("a")
+
+ logger.debug("Checking %d element(s) ...", len(elements))
+ for element in elements:
+ logger.debug("element[%s]='%s'", type(element), element)
for href in element["href"].split(","):
logger.debug("href[%s]='%s' - BEFORE!", type(href), href)
- domain = tidyup.domain(href)
+ domain = tidyup.domain(href) if href not in [None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("domain is empty - SKIPPED!")
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
logger.debug("domain='%s' - BEFORE!", domain)
origin = row["origin"]
software = row["software"]
+ logger.debug("software='%s'", software)
+ if software is None:
+ logger.warning("args.domain='%s' has no software detected. You can try to run ./fba.py update_nodeinfo --domain=%s --force to get it updated.", args.domain, args.domain)
+ return 102
+ elif software_helper.is_relay(software):
+ logger.warning("args.domain='%s' is of software type '%s' which is not supported by this command. Please invoke fetch_relays instead.", args.domain, software)
+ return 103
+
# Initial fetch
try:
logger.info("Fetching instances from args.domain='%s',origin='%s',software='%s' ...", domain, origin, software)
logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain)
instances.set_last_error(args.domain, exception)
instances.update(args.domain)
- return 100
+ return 104
if args.single:
logger.debug("Not fetching more instances - EXIT!")
# Loop through some instances
database.cursor.execute(
- "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY total_peers DESC, last_updated ASC", [time.time() - config.get("recheck_instance")]
+ "SELECT domain, origin, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY total_peers DESC, last_response_time ASC, last_updated ASC", [time.time() - config.get("recheck_instance")]
)
rows = database.cursor.fetchall()
logger.info("Checking %d entries ...", len(rows))
for row in rows:
- logger.debug("row[domain]='%s'", row["domain"])
- if row["domain"] == "":
- logger.debug("row[domain] is empty - SKIPPED!")
- continue
-
logger.debug("row[domain]='%s' - BEFORE!", row["domain"])
domain = row["domain"].encode("idna").decode("utf-8")
logger.debug("domain='%s' - AFTER!", domain)
continue
try:
- logger.info("Fetching instances for domain='%s',origin='%s',software='%s',nodeinfo_url='%s'", domain, row["origin"], row["software"], row["nodeinfo_url"])
- federation.fetch_instances(domain, row["origin"], row["software"], inspect.currentframe().f_code.co_name, row["nodeinfo_url"])
+ logger.info("Fetching instances for domain='%s',origin='%s',software='%s' ...", domain, row["origin"], row["software"])
+ federation.fetch_instances(domain, row["origin"], row["software"], inspect.currentframe().f_code.co_name)
except network.exceptions as exception:
logger.warning("Exception '%s' during fetching instances (fetch_instances) from domain='%s'", type(exception), domain)
instances.set_last_error(domain, exception)
logger.debug("Success - EXIT!")
return 0
+def fetch_csv(args: argparse.Namespace) -> int:
+ logger.debug("args[]='%s' - CALLED!", type(args))
+
+ logger.debug("Invoking locking.acquire() ...")
+ locking.acquire()
+
+ logger.info("Checking %d CSV files ...", len(blocklists.csv_files))
+ for block in blocklists.csv_files:
+ logger.debug("block[blocker]='%s',block[csv_url]='%s'", block["blocker"], block["csv_url"])
+
+ # Is domain given and not equal blocker?
+ if isinstance(args.domain, str) and args.domain != block["blocker"]:
+ logger.debug("Skipping blocker='%s', not matching args.domain='%s'", block["blocker"], args.domain)
+ continue
+
+ logger.debug("Invoking processing.csv_block(%s, %s, fetch_csv) ...", block["blocker"], block["csv_url"])
+ processing.csv_block(block["blocker"], block["csv_url"], inspect.currentframe().f_code.co_name)
+
+ logger.debug("Success - EXIT!")
+ return 0
+
def fetch_oliphant(args: argparse.Namespace) -> int:
logger.debug("args[]='%s' - CALLED!", type(args))
source_domain = "codeberg.org"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
# Base URL
base_url = f"https://{source_domain}/oliphant/blocklists/raw/branch/main/blocklists"
- domains = list()
-
logger.debug("Downloading %d files ...", len(blocklists.oliphant_blocklists))
for block in blocklists.oliphant_blocklists:
# Is domain given and not equal blocker?
+ logger.debug("block[blocker]='%s',block[csv_url]='%s'", block["blocker"], block["csv_url"])
if isinstance(args.domain, str) and args.domain != block["blocker"]:
logger.debug("Skipping blocker='%s', not matching args.domain='%s'", block["blocker"], args.domain)
continue
- elif args.domain in domains:
- logger.debug("args.domain='%s' already handled - SKIPPED!", args.domain)
- continue
-
- instances.set_last_blocked(block["blocker"])
-
- # Fetch this URL
- logger.info("Fetching csv_url='%s' for blocker='%s' ...", block["csv_url"], block["blocker"])
- response = utils.fetch_url(f"{base_url}/{block['csv_url']}", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
-
- logger.debug("response.ok='%s',response.status_code=%d,response.content()=%d", response.ok, response.status_code, len(response.content))
- if not response.ok or response.status_code > 200 or response.content == "":
- logger.warning("Could not fetch csv_url='%s' for blocker='%s' - SKIPPED!", block["csv_url"], block["blocker"])
- continue
-
- logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content))
- reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix")
-
- blockdict = list()
-
- cnt = 0
- for row in reader:
- logger.debug("row[%s]='%s'", type(row), row)
- domain = severity = None
- reject_media = reject_reports = False
-
- if "#domain" in row:
- domain = row["#domain"]
- elif "domain" in row:
- domain = row["domain"]
- else:
- logger.debug("row='%s' does not contain domain column", row)
- continue
-
- if "#severity" in row:
- severity = blocks.alias_block_level(row["#severity"])
- elif "severity" in row:
- severity = blocks.alias_block_level(row["severity"])
- else:
- logger.debug("row='%s' does not contain severity column", row)
- continue
-
- if "#reject_media" in row and row["#reject_media"].lower() == "true":
- reject_media = True
- elif "reject_media" in row and row["reject_media"].lower() == "true":
- reject_media = True
-
- if "#reject_reports" in row and row["#reject_reports"].lower() == "true":
- reject_reports = True
- elif "reject_reports" in row and row["reject_reports"].lower() == "true":
- reject_reports = True
-
- cnt = cnt + 1
- logger.debug("domain='%s',severity='%s',reject_media='%s',reject_reports='%s'", domain, severity, reject_media, reject_reports)
- if domain == "":
- logger.debug("domain is empty - SKIPPED!")
- continue
- elif domain.endswith(".onion"):
- logger.debug("domain='%s' is a TOR .onion domain - SKIPPED", domain)
- continue
- elif domain.endswith(".arpa"):
- logger.debug("domain='%s' is a reverse IP address - SKIPPED", domain)
- continue
- elif domain.endswith(".tld"):
- logger.debug("domain='%s' is a fake domain - SKIPPED", domain)
- continue
- elif domain.find("*") >= 0 or domain.find("?") >= 0:
- logger.debug("domain='%s' is obfuscated - Invoking utils.deobfuscate(%s, %s) ...", domain, domain, block["blocker"])
- domain = utils.deobfuscate(domain, block["blocker"])
- logger.debug("domain='%s' - AFTER!", domain)
- if not validators.domain(domain):
- logger.debug("domain='%s' is not a valid domain - SKIPPED!")
- continue
- elif blacklist.is_blacklisted(domain):
- logger.warning("domain='%s' is blacklisted - SKIPPED!", domain)
- continue
- elif blocks.is_instance_blocked(block["blocker"], domain, severity):
- logger.debug("block[blocker]='%s' has already blocked domain='%s' with severity='%s' - SKIPPED!", block["blocker"], domain, severity)
- continue
+ url = f"{base_url}/{block['csv_url']}"
- logger.debug("Marking domain='%s' as handled", domain)
- domains.append(domain)
-
- logger.debug("Processing domain='%s' ...", domain)
- processed = processing.domain(domain, block["blocker"], inspect.currentframe().f_code.co_name)
- logger.debug("processed='%s'", processed)
-
- if processing.block(block["blocker"], domain, None, severity) and config.get("bot_enabled"):
- logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", domain, block["block_level"], block["blocker"])
- blockdict.append({
- "blocked": domain,
- "reason" : block["reason"],
- })
-
- if reject_media:
- processing.block(block["blocker"], domain, None, "reject_media")
- if reject_reports:
- processing.block(block["blocker"], domain, None, "reject_reports")
-
- logger.debug("block[blocker]='%s'", block["blocker"])
- if not blocklists.has(block["blocker"]):
- logger.debug("Invoking instances.set_total_blocks(%s, domains()=%d) ...", block["blocker"], len(domains))
- instances.set_total_blocks(block["blocker"], domains)
-
- logger.debug("Checking if blocker='%s' has pending updates ...", block["blocker"])
- if instances.has_pending(block["blocker"]):
- logger.debug("Flushing updates for block[blocker]='%s' ...", block["blocker"])
- instances.update(block["blocker"])
-
- logger.debug("Invoking commit() ...")
- database.connection.commit()
-
- logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict))
- if config.get("bot_enabled") and len(blockdict) > 0:
- logger.info("Sending bot POST for blocker='%s',blockdict()=%d ...", block["blocker"], len(blockdict))
- network.send_bot_post(block["blocker"], blockdict)
+ logger.debug("Invoking processing.csv_block(%s, %s, fetch_oliphant) ...", block["blocker"], url)
+ processing.csv_block(block["blocker"], url, inspect.currentframe().f_code.co_name)
logger.debug("Success! - EXIT!")
return 0
logger.debug("Invoking locking.acquire() ...")
locking.acquire()
- # Static URLs
- urls = ({
- "blocker": "seirdy.one",
- "url" : "https://seirdy.one/pb/bsl.txt",
- },)
-
- logger.info("Checking %d text file(s) ...", len(urls))
- for row in urls:
+ logger.info("Checking %d text file(s) ...", len(blocklists.txt_files))
+ for row in blocklists.txt_files:
logger.debug("Fetching row[url]='%s' ...", row["url"])
response = utils.fetch_url(row["url"], network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
if response.ok and response.status_code == 200 and response.text != "":
logger.debug("Returned %d Bytes for processing", len(response.text.strip()))
- domains = response.text.split("\n")
+ domains = response.text.strip().split("\n")
logger.info("Processing %d domains ...", len(domains))
for domain in domains:
logger.debug("domain='%s' - BEFORE!", domain)
- domain = tidyup.domain(domain)
-
+ domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("domain is empty - SKIPPED!")
+
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
elif not domain_helper.is_wanted(domain):
logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
continue
- elif instances.is_recent(domain):
- logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
+ elif not args.force and instances.is_registered(domain):
+ logger.debug("domain='%s' is already registered - SKIPPED!", domain)
continue
- logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"])
- processed = processing.domain(domain, row["blocker"], inspect.currentframe().f_code.co_name)
-
+ logger.debug("Processing domain='%s',row[blocker]='%s' ...", domain, row["blocker"])
+ processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name, force=args.force)
logger.debug("processed='%s'", processed)
- if not processed:
- logger.debug("domain='%s' was not generically processed - SKIPPED!", domain)
- continue
logger.debug("Success! - EXIT!")
return 0
source_domain = "fedipact.online"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
logger.info("Checking %d row(s) ...", len(rows))
for row in rows:
logger.debug("row[]='%s'", type(row))
- domain = tidyup.domain(row.contents[0])
+ domain = tidyup.domain(row.contents[0]) if row.contents[0] not in [None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("domain is empty - SKIPPED!")
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
logger.debug("domain='%s' - BEFORE!", domain)
source_domain = "instances.joinmobilizon.org"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
source_domain = "instanceapp.misskey.page"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
logger.debug("Success! - EXIT!")
return 0
-def fetch_joinfediverse(args: argparse.Namespace) -> int:
- logger.debug("args[]='%s' - CALLED!", type(args))
-
- logger.debug("Invoking locking.acquire() ...")
- locking.acquire()
-
- source_domain = "joinfediverse.wiki"
- if sources.is_recent(source_domain):
- logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
- else:
- logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
- sources.update(source_domain)
-
- logger.info("Fetching /FediBlock wiki page from source_domain='%s' ...", source_domain)
- raw = utils.fetch_url(
- f"https://{source_domain}/FediBlock",
- network.web_headers,
- (config.get("connection_timeout"), config.get("read_timeout"))
- ).text
- logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
-
- doc = bs4.BeautifulSoup(raw, "html.parser")
- logger.debug("doc[]='%s'", type(doc))
-
- tables = doc.findAll("table", {"class": "wikitable"})
-
- logger.info("Analyzing %d table(s) ...", len(tables))
- blocklist = list()
- for table in tables:
- logger.debug("table[]='%s'", type(table))
-
- rows = table.findAll("tr")
- logger.info("Checking %d row(s) ...", len(rows))
- block_headers = dict()
- for row in rows:
- logger.debug("row[%s]='%s'", type(row), row)
-
- headers = row.findAll("th")
- logger.debug("Found headers()=%d header(s)", len(headers))
- if len(headers) > 1:
- block_headers = dict()
- cnt = 0
- for header in headers:
- cnt = cnt + 1
- logger.debug("header[]='%s',cnt=%d", type(header), cnt)
- text = header.contents[0]
-
- logger.debug("text[]='%s'", type(text))
- if not isinstance(text, str):
- logger.debug("text[]='%s' is not of type 'str' - SKIPPED!", type(text))
- continue
- elif validators.domain(text.strip()):
- logger.debug("text='%s' is a domain - SKIPPED!", text.strip())
- continue
-
- text = tidyup.domain(text.strip())
- logger.debug("text='%s' - AFTER!", text)
- if text in ["domain", "instance", "subdomain(s)", "block reason(s)"]:
- logger.debug("Found header: '%s'=%d", text, cnt)
- block_headers[cnt] = text
-
- elif len(block_headers) == 0:
- logger.debug("row is not scrapable - SKIPPED!")
- continue
- elif len(block_headers) > 0:
- logger.debug("Found a row with %d scrapable headers ...", len(block_headers))
- cnt = 0
- block = dict()
-
- for element in row.find_all(["th", "td"]):
- cnt = cnt + 1
- logger.debug("element[]='%s',cnt=%d", type(element), cnt)
- if cnt in block_headers:
- logger.debug("block_headers[%d]='%s'", cnt, block_headers[cnt])
-
- text = element.text.strip()
- key = block_headers[cnt] if block_headers[cnt] not in ["domain", "instance"] else "blocked"
-
- logger.debug("cnt=%d is wanted: key='%s',text[%s]='%s'", cnt, key, type(text), text)
- if key in ["domain", "instance"]:
- block[key] = text
- elif key == "reason":
- block[key] = tidyup.reason(text)
- elif key == "subdomain(s)":
- block[key] = list()
- if text != "":
- block[key] = text.split("/")
- else:
- logger.debug("key='%s'", key)
- block[key] = text
-
- logger.debug("block()=%d ...", len(block))
- if len(block) > 0:
- logger.debug("Appending block()=%d ...", len(block))
- blocklist.append(block)
-
- logger.debug("blocklist()=%d", len(blocklist))
-
- database.cursor.execute("SELECT domain FROM instances WHERE domain LIKE 'climatejustice.%'")
- domains = database.cursor.fetchall()
-
- logger.debug("domains(%d)[]='%s'", len(domains), type(domains))
- blocking = list()
- for block in blocklist:
- logger.debug("block='%s'", block)
- if "subdomain(s)" in block and len(block["subdomain(s)"]) > 0:
- origin = block["blocked"]
- logger.debug("origin='%s'", origin)
- for subdomain in block["subdomain(s)"]:
- block["blocked"] = subdomain + "." + origin
- logger.debug("block[blocked]='%s'", block["blocked"])
- blocking.append(block)
- else:
- blocking.append(block)
-
- logger.debug("blocking()=%d", blocking)
- for block in blocking:
- logger.debug("block[]='%s'", type(block))
- if "blocked" not in block:
- raise KeyError(f"block()={len(block)} does not have element 'blocked'")
-
- block["blocked"] = tidyup.domain(block["blocked"]).encode("idna").decode("utf-8")
- logger.debug("block[blocked]='%s' - AFTER!", block["blocked"])
-
- if block["blocked"] == "":
- logger.debug("block[blocked] is empty - SKIPPED!")
- continue
- elif not domain_helper.is_wanted(block["blocked"]):
- logger.debug("block[blocked]='%s' is not wanted - SKIPPED!", block["blocked"])
- continue
- elif instances.is_recent(block["blocked"]):
- logger.debug("block[blocked]='%s' has been recently checked - SKIPPED!", block["blocked"])
- continue
-
- logger.debug("Proccessing blocked='%s' ...", block["blocked"])
- processing.domain(block["blocked"], "climatejustice.social", inspect.currentframe().f_code.co_name)
-
- blockdict = list()
- for blocker in domains:
- blocker = blocker[0]
- logger.debug("blocker[%s]='%s'", type(blocker), blocker)
- instances.set_last_blocked(blocker)
-
- for block in blocking:
- logger.debug("block[blocked]='%s',block[block reason(s)]='%s' - BEFORE!", block["blocked"], block["block reason(s)"] if "block reason(s)" in block else None)
- block["reason"] = tidyup.reason(block["block reason(s)"]) if "block reason(s)" in block else None
-
- logger.debug("block[blocked]='%s',block[reason]='%s' - AFTER!", block["blocked"], block["reason"])
- if block["blocked"] == "":
- logger.debug("block[blocked] is empty - SKIPPED!")
- continue
- elif not domain_helper.is_wanted(block["blocked"]):
- logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
- continue
-
- logger.debug("blocked='%s',reason='%s'", block["blocked"], block["reason"])
- if processing.block(blocker, block["blocked"], block["reason"], "reject") and config.get("bot_enabled"):
- logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
- blockdict.append({
- "blocked": block["blocked"],
- "reason" : block["reason"],
- })
-
- if instances.has_pending(blocker):
- logger.debug("Flushing updates for blocker='%s' ...", blocker)
- instances.update(blocker)
-
- logger.debug("Invoking commit() ...")
- database.connection.commit()
-
- logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict))
- if config.get("bot_enabled") and len(blockdict) > 0:
- logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", blocker, len(blockdict))
- network.send_bot_post(blocker, blockdict)
-
- logger.debug("Success! - EXIT!")
- return 0
-
def recheck_obfuscation(args: argparse.Namespace) -> int:
logger.debug("args[]='%s' - CALLED!", type(args))
locking.acquire()
if isinstance(args.domain, str) and args.domain != "" and domain_helper.is_wanted(args.domain):
- database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 AND domain = ?", [args.domain])
+ database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND domain = ?", [args.domain])
elif isinstance(args.software, str) and args.software != "" and validators.domain(args.software) == args.software:
- database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 AND software = ?", [args.software])
+ database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND software = ?", [args.software])
else:
- database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1")
+ database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 OR has_obfuscation IS NULL")
rows = database.cursor.fetchall()
logger.info("Checking %d domains ...", len(rows))
if (args.force is None or not args.force) and args.domain is None and args.software is None and instances.is_recent(row["domain"], "last_blocked"):
logger.debug("row[domain]='%s' has been recently checked, args.force[]='%s' - SKIPPED!", row["domain"], type(args.force))
continue
+ elif blacklist.is_blacklisted(row["domain"]):
+ logger.warning("row[domain]='%s' is blacklisted - SKIPPED!", row["domain"])
+ continue
logger.debug("Invoking federation.fetch_blocks(%s) ...", row["domain"])
blocking = federation.fetch_blocks(row["domain"])
logger.debug("blocking()=%d", len(blocking))
if len(blocking) == 0:
+ logger.debug("Empty blocking list, trying individual fetch_blocks() for row[software]='%s' ...", row["software"])
if row["software"] == "pleroma":
logger.debug("domain='%s',software='%s'", row["domain"], row["software"])
- blocking = pleroma.fetch_blocks(row["domain"], row["nodeinfo_url"])
+ blocking = pleroma.fetch_blocks(row["domain"])
elif row["software"] == "mastodon":
logger.debug("domain='%s',software='%s'", row["domain"], row["software"])
- blocking = mastodon.fetch_blocks(row["domain"], row["nodeinfo_url"])
+ blocking = mastodon.fetch_blocks(row["domain"])
elif row["software"] == "lemmy":
logger.debug("domain='%s',software='%s'", row["domain"], row["software"])
- blocking = lemmy.fetch_blocks(row["domain"], row["nodeinfo_url"])
+ blocking = lemmy.fetch_blocks(row["domain"])
elif row["software"] == "friendica":
logger.debug("domain='%s',software='%s'", row["domain"], row["software"])
blocking = friendica.fetch_blocks(row["domain"])
# c.s isn't part of oliphant's "hidden" blocklists
logger.debug("row[domain]='%s'", row["domain"])
- if row["domain"] != "chaos.social" and not blocklists.has(row["domain"]):
+ if row["domain"] != "chaos.social" and row["software"] is not None and not software_helper.is_relay(row["software"]) and not blocklists.has(row["domain"]):
logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["domain"], len(blocking))
instances.set_last_blocked(row["domain"])
instances.set_total_blocks(row["domain"], blocking)
if block["blocked"] == "":
logger.debug("block[blocked] is empty - SKIPPED!")
continue
+ elif block["blocked"].endswith(".onion"):
+ logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
+ continue
+ elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain") == "true":
+ logger.debug("blocked='%s' is an I2P onion domain name - SKIPPED!", block["blocked"])
+ continue
elif block["blocked"].endswith(".arpa"):
logger.debug("blocked='%s' is a reversed IP address - SKIPPED!", block["blocked"])
continue
elif block["blocked"].endswith(".tld"):
logger.debug("blocked='%s' is a fake domain name - SKIPPED!", block["blocked"])
continue
- elif block["blocked"].endswith(".onion"):
- logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
- continue
elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0:
logger.debug("block='%s' is obfuscated.", block["blocked"])
obfuscated = obfuscated + 1
- blocked = utils.deobfuscate(block["blocked"], row["domain"], block["hash"] if "hash" in block else None)
+ blocked = utils.deobfuscate(block["blocked"], row["domain"], block["digest"] if "digest" in block else None)
elif not domain_helper.is_wanted(block["blocked"]):
logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
continue
logger.debug("blocked='%s' was deobfuscated to blocked='%s'", block["blocked"], blocked)
obfuscated = obfuscated - 1
- if blocks.is_instance_blocked(row["domain"], blocked):
- logger.debug("blocked='%s' is already blocked by domain='%s' - SKIPPED!", blocked, row["domain"])
- continue
- elif blacklist.is_blacklisted(blocked):
+ if blacklist.is_blacklisted(blocked):
logger.debug("blocked='%s' is blacklisted - SKIPPED!", blocked)
continue
+ elif blacklist.is_blacklisted(row["domain"]):
+ logger.debug("row[domain]='%s' is blacklisted - SKIPPED!", row["domain"])
+ continue
+ elif blocks.is_instance_blocked(row["domain"], blocked):
+ logger.debug("blocked='%s' is already blocked by domain='%s' - SKIPPED!", blocked, row["domain"])
+ continue
block["block_level"] = blocks.alias_block_level(block["block_level"])
"reason" : block["reason"],
})
- logger.debug("Settings obfuscated=%d for row[domain]='%s' ...", obfuscated, row["domain"])
+ logger.debug("Setting obfuscated=%d for row[domain]='%s' ...", obfuscated, row["domain"])
+ instances.set_has_obfuscation(row["domain"], (obfuscated > 0))
instances.set_obfuscated_blocks(row["domain"], obfuscated)
logger.info("domain='%s' has %d obfuscated domain(s)", row["domain"], obfuscated)
- if obfuscated == 0 and len(blocking) > 0:
- logger.info("Block list from domain='%s' has been fully deobfuscated.", row["domain"])
- instances.set_has_obfuscation(row["domain"], False)
-
if instances.has_pending(row["domain"]):
logger.debug("Flushing updates for blocker='%s' ...", row["domain"])
instances.update(row["domain"])
source_domain = "demo.fedilist.com"
if sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 1
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
continue
logger.debug("row[hostname]='%s' - BEFORE!", row["hostname"])
- domain = tidyup.domain(row["hostname"])
+ domain = tidyup.domain(row["hostname"]) if row["hostname"] not in [None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("domain is empty after tidyup: row[hostname]='%s' - SKIPPED!", row["hostname"])
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain(): row[hostname]='%s' - SKIPPED!", domain, row["hostname"])
continue
logger.debug("domain='%s' - BEFORE!", domain)
if args.domain is not None and args.domain != "":
logger.debug("Fetching args.domain='%s'", args.domain)
- database.cursor.execute("SELECT domain, software FROM instances WHERE domain = ?", [args.domain])
+ database.cursor.execute("SELECT domain, software FROM instances WHERE domain = ? LIMIT 1", [args.domain])
elif args.software is not None and args.software != "":
logger.info("Fetching domains for args.software='%s'", args.software)
- database.cursor.execute("SELECT domain, software FROM instances WHERE software = ? AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [args.software.lower(), time.time() - config.get("recheck_nodeinfo")])
+ database.cursor.execute("SELECT domain, software FROM instances WHERE software = ? ORDER BY last_updated ASC", [args.software])
elif args.mode is not None and args.mode != "":
logger.info("Fetching domains for args.mode='%s'", args.mode.upper())
- database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode = ? AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [args.mode.upper(), time.time() - config.get("recheck_nodeinfo")])
+ database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode = ? ORDER BY last_updated ASC", [args.mode])
elif args.no_software:
logger.info("Fetching domains with no software type detected ...")
- database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NULL AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [time.time() - config.get("recheck_nodeinfo")])
+ database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NULL ORDER BY last_updated ASC")
+ elif args.with_software:
+ logger.info("Fetching domains with any software type detected ...")
+ database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NOT NULL ORDER BY last_updated ASC")
+ elif args.no_auto:
+ logger.info("Fetching domains with other detection mode than AUTO_DISOVERY being set ...")
+ database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode IS NOT NULL AND detection_mode != 'AUTO_DISCOVERY' ORDER BY last_updated ASC")
+ elif args.no_detection:
+ logger.info("Fetching domains with no detection mode being set ...")
+ database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode IS NULL ORDER BY last_updated ASC")
else:
logger.info("Fetching domains for recently updated ...")
- database.cursor.execute("SELECT domain, software FROM instances WHERE last_nodeinfo < ? OR last_nodeinfo IS NULL", [time.time() - config.get("recheck_nodeinfo")])
+ database.cursor.execute("SELECT domain, software FROM instances ORDER BY last_updated ASC")
domains = database.cursor.fetchall()
if not args.force and instances.is_recent(row["domain"], "last_nodeinfo"):
logger.debug("row[domain]='%s' has been recently checked - SKIPPED!", row["domain"])
continue
+ elif blacklist.is_blacklisted(row["domain"]):
+ logger.debug("row[domain]='%s' is blacklisted - SKIPPED!", row["domain"])
+ continue
try:
logger.info("Checking nodeinfo for row[domain]='%s',row[software]='%s' (%s%%) ...", row["domain"], row["software"], "{:5.1f}".format(cnt / len(domains) * 100))
return 1
elif sources.is_recent(source_domain):
logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
- return 0
+ return 2
else:
logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
sources.update(source_domain)
fetched = network.get_json_api(
source_domain,
"/api/1.0/instances/list?count=0&sort_by=name",
- headers,
- (config.get("connection_timeout"), config.get("read_timeout"))
+ headers=headers,
+ timeout=(config.get("connection_timeout"), config.get("read_timeout"))
)
- logger.debug("fetched[]='%s'", type(fetched))
+ logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
if "error_message" in fetched:
logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"])
logger.info("Checking %d row(s) ...", len(rows))
for row in rows:
logger.debug("row[]='%s'", type(row))
- domain = tidyup.domain(row["name"])
+ domain = tidyup.domain(row["name"]) if row["name"] not in [None, ""] else None
+ logger.debug("domain='%s' - AFTER!", domain)
+
+ if domain is None and domain == "":
+ logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
+ continue
+
+ logger.debug("domain='%s' - BEFORE!", domain)
+ domain = domain.encode("idna").decode("utf-8")
+ logger.debug("domain='%s' - AFTER!", domain)
+
+ if not domain_helper.is_wanted(domain):
+ logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
+ continue
+ elif domain in domains:
+ logger.debug("domain='%s' is already added - SKIPPED!", domain)
+ continue
+ elif instances.is_registered(domain):
+ logger.debug("domain='%s' is already registered - SKIPPED!", domain)
+ continue
+ elif instances.is_recent(domain):
+ logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
+ continue
+
+ logger.info("Fetching instances from domain='%s'", domain)
+ federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+
+ logger.debug("Success! - EXIT!")
+ return 0
+
+def fetch_relaylist(args: argparse.Namespace) -> int:
+ logger.debug("args[]='%s' - CALLED!", type(args))
+
+ logger.debug("Invoking locking.acquire() ...")
+ locking.acquire()
+
+ source_domain = "api.relaylist.com"
+
+ if sources.is_recent(source_domain):
+ logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
+ return 1
+ else:
+ logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
+ sources.update(source_domain)
+
+ logger.info("Fetching list from source_domain='%s' ...", source_domain)
+ fetched = network.get_json_api(
+ source_domain,
+ "/relays",
+ {},
+ (config.get("connection_timeout"), config.get("read_timeout"))
+ )
+ logger.debug("fetched(%d)[]='%s'", len(fetched), type(fetched))
+
+ if "error_message" in fetched:
+ logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"])
+ return 2
+ elif "exception" in fetched:
+ logger.warning("Exception '%s' during fetching API result - EXIT!", type(fetched["exception"]))
+ return 3
+ elif "json" not in fetched:
+ logger.warning("fetched has no element 'json' - EXIT!")
+ return 4
+
+ domains = list()
+
+ logger.info("Checking %d row(s) ...", len(fetched["json"]))
+ for row in fetched["json"]:
+ logger.debug("row[]='%s'", type(row))
+ domain = urlparse(row["url"]).netloc.lower().split(":")[0]
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("domain is empty - SKIPPED!")
+ if domain is None and domain == "":
+ logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
logger.debug("domain='%s' - BEFORE!", domain)
locking.acquire()
if args.domain is not None and args.domain != "":
- database.cursor.execute("SELECT domain, software FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay') AND domain = ? LIMIT 1", [args.domain])
+ logger.debug("Fetching instances record for args.domain='%s' ...", args.domain)
+ database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay', 'pub-relay') AND domain = ? LIMIT 1", [args.domain])
+ elif args.software is not None and args.software != "":
+ logger.debug("Fetching instances records for args.software='%s' ...", args.software)
+ database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay', 'pub-relay') AND nodeinfo_url IS NOT NULL AND software = ? ORDER BY last_updated DESC", [args.software])
else:
- database.cursor.execute("SELECT domain, software FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay')")
+ logger.debug("Fetch all relay instances ...")
+ database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay', 'pub-relay') AND nodeinfo_url IS NOT NULL ORDER BY last_updated DESC")
domains = list()
rows = database.cursor.fetchall()
logger.info("Checking %d relays ...", len(rows))
for row in rows:
- logger.debug("row[domain]='%s',row[software]='%s' ...", row["domain"], row["software"])
- peers = list()
+ logger.debug("row[domain]='%s',row[software]='%s'", row["domain"], row["software"])
if not args.force and instances.is_recent(row["domain"]):
logger.debug("row[domain]='%s' has been recently fetched - SKIPPED!", row["domain"])
continue
+ elif row["nodeinfo_url"] is None:
+ logger.warning("row[domain]='%s' has empty nodeinfo_url but this is required - SKIPPED!", row["domain"])
+ continue
+ peers = list()
try:
- logger.info("Fetching / from relay row[domain]='%s',row[software]='%s' ...", row["domain"], row["software"])
- raw = utils.fetch_url(
- f"https://{row['domain']}",
- network.web_headers,
- (config.get("connection_timeout"), config.get("read_timeout"))
- ).text
- logger.debug("raw[%s]()=%d", type(raw), len(raw))
+ logger.debug("row[domain]='%s',row[software]='%s' - checking ....", row["domain"], row["software"])
+ if row["software"] == "pub-relay":
+ logger.info("Fetching row[nodeinfo_url]='%s' from relay row[domain]='%s',row[software]='%s' ...", row["nodeinfo_url"], row["domain"], row["software"])
+ raw = network.fetch_api_url(
+ row["nodeinfo_url"],
+ (config.get("connection_timeout"), config.get("read_timeout"))
+ )
+
+ logger.debug("raw[%s]()=%d", type(raw), len(raw))
+ if "exception" in raw:
+ logger.warning("row[domain]='%s' has caused an exception: '%s' - raising again ...", row["domain"], type(raw["exception"]))
+ raise raw["exception"]
+ elif "error_message" in raw:
+ logger.warning("row[domain]='%s' has caused error message: '%s' - SKIPPED!", row["domain"], raw["error_message"])
+ instances.set_last_error(row["domain"], raw)
+ instances.set_last_instance_fetch(row["domain"])
+ instances.update(row["domain"])
+ continue
+ elif "json" not in raw:
+ logger.warning("raw()=%d does not contain key 'json' in response - SKIPPED!", len(raw))
+ continue
+ elif not "metadata" in raw["json"]:
+ logger.warning("raw[json]()=%d does not contain key 'json' in response - SKIPPED!", len(raw["json"]))
+ continue
+ elif not "peers" in raw["json"]["metadata"]:
+ logger.warning("raw[json][metadata()=%d does not contain key 'json' in response - SKIPPED!", len(raw["json"]["metadata"]))
+ continue
+ else:
+ logger.info("Fetching / from relay row[domain]='%s',row[software]='%s' ...", row["domain"], row["software"])
+ raw = utils.fetch_url(
+ f"https://{row['domain']}",
+ network.web_headers,
+ (config.get("connection_timeout"), config.get("read_timeout"))
+ ).text
+ logger.debug("raw[%s]()=%d", type(raw), len(raw))
+
+ doc = bs4.BeautifulSoup(raw, features="html.parser")
+ logger.debug("doc[]='%s'", type(doc))
+
except network.exceptions as exception:
logger.warning("Exception '%s' during fetching from relay '%s': '%s'", type(exception), row["domain"], str(exception))
instances.set_last_error(row["domain"], exception)
instances.update(row["domain"])
continue
- doc = bs4.BeautifulSoup(raw, features="html.parser")
- logger.debug("doc[]='%s'", type(doc))
-
logger.debug("row[software]='%s'", row["software"])
if row["software"] == "activityrelay":
logger.debug("Checking row[domain]='%s' ...", row["domain"])
continue
logger.debug("domain='%s' - BEFORE!", domain)
- domain = tidyup.domain(domain)
+ domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("Empty domain after tidyup.domain() from origin='%s' - SKIPPED!", row["domain"])
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain() from origin='%s' - SKIPPED!", domain, row["domain"])
continue
elif domain not in peers:
logger.debug("Appending domain='%s' to peers list for relay='%s' ...", domain, row["domain"])
peers.append(domain)
+ logger.debug("domains()=%d,domain='%s'", len(domains), domain)
if dict_helper.has_key(domains, "domain", domain):
logger.debug("domain='%s' already added", domain)
continue
link = tag.find("a")
logger.debug("link[%s]='%s'", type(link), link)
- if link is None:
- logger.warning("tag='%s' has no a-tag ...", tag)
+ if not isinstance(link, bs4.element.Tag):
+ logger.warning("tag[%s]='%s' is not type of 'bs4.element.Tag' - SKIPPED!", type(tag), tag)
continue
- components = urlparse(link["href"])
- domain = components.netloc.lower()
+ components = urlparse(link.get("href"))
+ logger.debug("components(%d)='%s'", len(components), components)
+ domain = components.netloc.lower().split(":")[0]
- if not domain_helper.is_wanted(domain):
- logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
+ logger.debug("domain='%s' - BEFORE!", domain)
+ domain = tidyup.domain(domain) if domain not in[None, ""] else None
+ logger.debug("domain='%s' - AFTER!", domain)
+
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain() from origin='%s' - SKIPPED!", domain, row["domain"])
+ continue
+ elif domain not in peers:
+ logger.debug("Appending domain='%s' to peers list for relay='%s' ...", domain, row["domain"])
+ peers.append(domain)
+
+ logger.debug("domains()=%d,domain='%s'", len(domains), domain)
+ if dict_helper.has_key(domains, "domain", domain):
+ logger.debug("domain='%s' already added", domain)
continue
+ logger.debug("Appending domain='%s',origin='%s',software='%s'", domain, row["domain"], row["software"])
+ domains.append({
+ "domain": domain,
+ "origin": row["domain"],
+ })
+ elif row["software"] == "pub-relay":
+ logger.debug("Checking %d peer(s) row[domain]='%s' ...", len(raw["json"]["metadata"]["peers"]), row["domain"])
+ for domain in raw["json"]["metadata"]["peers"]:
logger.debug("domain='%s' - BEFORE!", domain)
- domain = tidyup.domain(domain)
+ domain = tidyup.domain(domain) if domain not in[None, ""] else None
logger.debug("domain='%s' - AFTER!", domain)
- if domain == "":
- logger.debug("Empty domain after tidyup.domain() from origin='%s' - SKIPPED!", row["domain"])
+ if domain in [None, ""]:
+ logger.debug("domain='%s' is empty after tidyup.domain() from origin='%s' - SKIPPED!", domain, row["domain"])
continue
elif domain not in peers:
logger.debug("Appending domain='%s' to peers list for relay='%s' ...", domain, row["domain"])
peers.append(domain)
+ logger.debug("domains()=%d,domain='%s'", len(domains), domain)
if dict_helper.has_key(domains, "domain", domain):
logger.debug("domain='%s' already added", domain)
continue
- logger.debug("Appending domain='%s',origin='%s',software='%s'", domain, row["domain"], row["software"])
+ logger.debug("Appending domain='%s',origin='%s',software='%s' ...", domain, row["domain"], row["software"])
domains.append({
"domain": domain,
"origin": row["domain"],
})
else:
logger.warning("row[domain]='%s',row[software]='%s' is not supported", row["domain"], row["software"])
+ continue
logger.debug("Updating last_instance_fetch for row[domain]='%s' ...", row["domain"])
instances.set_last_instance_fetch(row["domain"])
logger.info("Checking %d domains ...", len(domains))
for row in domains:
logger.debug("row[domain]='%s',row[origin]='%s'", row["domain"], row["origin"])
- if instances.is_registered(row["domain"]):
+ if not domain_helper.is_wanted(row["domain"]):
+ logger.debug("row[domain]='%s' is not wanted - SKIPPED!", row["domain"])
+ continue
+ elif instances.is_registered(row["domain"]):
logger.debug("row[domain]='%s' is already registered - SKIPPED!", row["domain"])
continue
projects / fba.git / blobdiff