elif block["blocked"].endswith(".onion"):
logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"])
continue
- elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain") == "true":
+ elif block["blocked"].endswith(".i2p") and not config.get("allow_i2p_domain"):
logger.debug("blocked='%s' is an I2P .onion domain - SKIPPED", block["blocked"])
continue
elif block["blocked"].endswith(".arpa"):
types = list()
if args.software is None:
logger.info("Fetching software list ...")
- raw = utils.fetch_url(
+ raw = network.fetch_url(
f"https://{source_domain}",
network.web_headers,
(config.get("connection_timeout"), config.get("read_timeout"))
}
logger.debug("Fetching domainblocks from source_domain='%s'", source_domain)
- raw = utils.fetch_url(
+ raw = network.fetch_url(
f"https://{source_domain}/todon/domainblocks",
network.web_headers,
(config.get("connection_timeout"), config.get("read_timeout"))
sources.update(source_domain)
logger.info("Fetching federation.md from source_domain='%s' ...", source_domain)
- raw = utils.fetch_url(
+ raw = network.fetch_url(
f"https://{source_domain}/chaossocial/meta/master/federation.md",
network.web_headers,
(config.get("connection_timeout"), config.get("read_timeout"))
sources.update(domain)
logger.info("Fetch FBA-specific RSS args.feed='%s' ...", args.feed)
- response = utils.fetch_url(args.feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+ response = network.fetch_url(args.feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
if response.ok and response.status_code == 200 and len(response.text) > 0:
domains = list()
logger.info("Fetching ATOM feed='%s' from FBA bot account ...", feed)
- response = utils.fetch_url(feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+ response = network.fetch_url(feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
if response.ok and response.status_code == 200 and len(response.text) > 0:
def fetch_instances(args: argparse.Namespace) -> int:
logger.debug("args[]='%s' - CALLED!", type(args))
- logger.debug("args.domain='%s' - checking ...", args.domain)
- if not validators.domain(args.domain):
- logger.warning("args.domain='%s' is not valid.", args.domain)
- return 100
- elif blacklist.is_blacklisted(args.domain):
- logger.warning("args.domain='%s' is blacklisted, won't check it!", args.domain)
- return 101
-
logger.debug("Invoking locking.acquire() ...")
locking.acquire()
- # Initialize values
- domain = tidyup.domain(args.domain)
- origin = software = None
+ # Init variables
+ rows = list()
- # Fetch record
- database.cursor.execute("SELECT origin, software FROM instances WHERE domain = ? LIMIT 1", [args.domain])
- row = database.cursor.fetchone()
- if row is not None:
- origin = row["origin"]
- software = row["software"]
+ # Is domain or software set?
+ if args.domain != "":
+ logger.debug("args.domain='%s' - checking ...", args.domain)
+ if not validators.domain(args.domain):
+ logger.warning("args.domain='%s' is not valid.", args.domain)
+ return 100
+ elif blacklist.is_blacklisted(args.domain):
+ logger.warning("args.domain='%s' is blacklisted, won't check it!", args.domain)
+ return 101
- logger.debug("software='%s'", software)
- if software is None:
- logger.warning("args.domain='%s' has no software detected. You can try to run ./fba.py update_nodeinfo --domain=%s --force to get it updated.", args.domain, args.domain)
- return 102
- elif software_helper.is_relay(software):
- logger.warning("args.domain='%s' is of software type '%s' which is not supported by this command. Please invoke fetch_relays instead.", args.domain, software)
- return 103
+ logger.debug("args.domain='%s' - BEFORE!", args.domain)
+ domain = tidyup.domain(args.domain)
+ logger.debug("domain='%s' - AFTER!", domain)
- # Initial fetch
- try:
- logger.info("Fetching instances from args.domain='%s',origin='%s',software='%s' ...", domain, origin, software)
- federation.fetch_instances(domain, origin, software, inspect.currentframe().f_code.co_name)
- except network.exceptions as exception:
- logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain)
- instances.set_last_error(args.domain, exception)
- instances.update(args.domain)
- return 104
+ # Fetch record
+ database.cursor.execute("SELECT domain, origin, software FROM instances WHERE domain = ? LIMIT 1", [domain])
+ rows = database.cursor.fetchall()
- if args.single:
- logger.debug("Not fetching more instances - EXIT!")
- return 0
+ logger.info("Checking %d entries ...", len(rows))
+ for row in rows:
+ logger.debug("row[domain]='%s',row[origin]='%s',row[software]='%s'", row["domain"], row["origin"], row["software"])
+ if row["software"] is None:
+ logger.warning("row[domain]='%s' has no software detected. You can try to run ./fba.py update_nodeinfo --domain=%s --force to get it updated - SKIPPED!", row["domain"], row["domain"])
+ continue
+ elif software_helper.is_relay(row["software"]):
+ logger.warning("row[domain]='%s' is of software type '%s' which is not supported by this command. Please invoke fetch_relays instead - SKIPPED!", row["domain"], row["software"])
+ continue
+
+ # Initial fetch
+ try:
+ logger.info("Fetching instances from row[domain]='%s',row[origin]='%s',row[software]='%s' ...", row["domain"], row["origin"], row["software"])
+ federation.fetch_instances(row["domain"], row["origin"], row["software"], inspect.currentframe().f_code.co_name)
+ except network.exceptions as exception:
+ logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[domain]='%s'", type(exception), row["domain"])
+ instances.set_last_error(row["domain"], exception)
+ instances.update(row["domain"])
+ raise exception
+
+ if args.single:
+ logger.debug("Not fetching more instances - BREAK!")
+ break
# Loop through some instances
database.cursor.execute(
- "SELECT domain, origin, software FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY total_peers DESC, last_response_time ASC, last_updated ASC", [time.time() - config.get("recheck_instance")]
+ "SELECT domain, origin, software \
+FROM instances \
+WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube', 'takahe', 'gotosocial', 'brighteon', 'wildebeest', 'bookwyrm', 'mitra', 'areionskey', 'mammuthus', 'neodb') \
+AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) \
+ORDER BY total_peers DESC, last_response_time ASC, last_updated ASC", [time.time() - config.get("recheck_instance")]
)
rows = database.cursor.fetchall()
logger.info("Checking %d text file(s) ...", len(blocklists.txt_files))
for row in blocklists.txt_files:
logger.debug("Fetching row[url]='%s' ...", row["url"])
- response = utils.fetch_url(row["url"], network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
+ response = network.fetch_url(row["url"], network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
if response.ok and response.status_code == 200 and response.text != "":
sources.update(source_domain)
logger.info("Fetching / from source_domain='%s' ...", source_domain)
- response = utils.fetch_url(
+ response = network.fetch_url(
f"https://{source_domain}",
network.web_headers,
(config.get("connection_timeout"), config.get("read_timeout"))
sources.update(source_domain)
logger.info("Fetching instances from source_domain='%s' ...", source_domain)
- raw = utils.fetch_url(
+ raw = network.fetch_url(
f"https://{source_domain}/api/v1/instances",
network.web_headers,
(config.get("connection_timeout"), config.get("read_timeout"))
sources.update(source_domain)
logger.info("Fetching instances.json from source_domain='%s' ...", source_domain)
- raw = utils.fetch_url(
+ raw = network.fetch_url(
f"https://{source_domain}/instances.json",
network.web_headers,
(config.get("connection_timeout"), config.get("read_timeout"))
locking.acquire()
if isinstance(args.domain, str) and args.domain != "" and domain_helper.is_wanted(args.domain):
+ logger.debug("Fetching record for args.domain='%s' ...", args.domain)
database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND domain = ?", [args.domain])
elif isinstance(args.software, str) and args.software != "" and validators.domain(args.software) == args.software:
+ logger.debug("Fetching records for args.software='%s' ...", args.software)
database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND software = ?", [args.software])
else:
+ logger.debug("Fetching records where domains have obfuscated block entries ...")
database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 OR has_obfuscation IS NULL")
rows = database.cursor.fetchall()
elif block["blocked"].endswith(".onion"):
logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"])
continue
- elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain") == "true":
+ elif block["blocked"].endswith(".i2p") and not config.get("allow_i2p_domain"):
logger.debug("blocked='%s' is an I2P onion domain name - SKIPPED!", block["blocked"])
continue
elif block["blocked"].endswith(".arpa"):
continue
else:
logger.info("Fetching / from relay row[domain]='%s',row[software]='%s' ...", row["domain"], row["software"])
- raw = utils.fetch_url(
+ raw = network.fetch_url(
f"https://{row['domain']}",
network.web_headers,
(config.get("connection_timeout"), config.get("read_timeout"))
projects / fba.git / blobdiff