X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Fcommands.py;h=7886d379a30c87f1c0dd0189560690a64ff9cba5;hb=d1aa7f3dfb18f62cea7881df31bfdd6029b89af6;hp=f62de8032ed81471e70b3c12625ff97a5b59d27f;hpb=3f72e05d2e3c87dfba70f0eaa862032ecf5b0c6f;p=fba.git diff --git a/fba/commands.py b/fba/commands.py index f62de80..7886d37 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -148,8 +148,8 @@ def fetch_pixelfed_api(args: argparse.Namespace) -> int: if "domain" not in row: logger.warning("row='%s' does not contain element 'domain' - SKIPPED!", row) continue - elif row["domain"] == "": - logger.debug("row[domain] is empty - SKIPPED!") + elif row["domain"] is None or row["domain"] == "": + logger.debug("row[domain]='%s' is empty - SKIPPED!", row["domain"]) continue logger.debug("row[domain]='%s' - BEFORE!", row["domain"]) @@ -224,8 +224,8 @@ def fetch_bkali(args: argparse.Namespace) -> int: if "domain" not in entry: logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry)) continue - elif entry["domain"] == "": - logger.debug("entry[domain] is empty - SKIPPED!") + elif entry["domain"] is None or entry["domain"] == "": + logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"]) continue elif not domain_helper.is_wanted(entry["domain"]): logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"]) @@ -319,7 +319,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: # c.s isn't part of oliphant's "hidden" blocklists if blocker == "chaos.social" or software_helper.is_relay(software) or blocklists.has(blocker): - logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs or fetch_oliphant instead!", blocker) + logger.debug("Skipping blocker='%s', run ./fba.py fetch_cs, fetch_oliphant, fetch_csv instead!", blocker) continue logger.debug("Invoking federation.fetch_blocks(%s) ...", blocker) @@ -330,19 +330,19 @@ def fetch_blocks(args: argparse.Namespace) -> int: logger.debug("blocker='%s',software='%s' - fetching blocklist ...", blocker, software) if software == "pleroma": blocking = pleroma.fetch_blocks(blocker) - logger.info("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) + logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) elif software == "mastodon": blocking = mastodon.fetch_blocks(blocker) - logger.info("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) + logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) elif software == "lemmy": blocking = lemmy.fetch_blocks(blocker) - logger.info("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) + logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) elif software == "friendica": blocking = friendica.fetch_blocks(blocker) - logger.info("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) + logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) elif software == "misskey": blocking = misskey.fetch_blocks(blocker) - logger.info("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) + logger.debug("blocker='%s' returned %d entries,software='%s'", blocker, len(blocking), software) else: logger.warning("Unknown software: blocker='%s',software='%s'", blocker, software) @@ -350,6 +350,7 @@ def fetch_blocks(args: argparse.Namespace) -> int: instances.set_total_blocks(blocker, blocking) blockdict = list() + deobfuscated = obfuscated = 0 logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software) for block in blocking: @@ -364,12 +365,15 @@ def fetch_blocks(args: argparse.Namespace) -> int: block["reason"] = tidyup.reason(block["reason"]) if block["reason"] is not None and block["reason"] != "" else None logger.debug("blocked='%s',reason='%s' - AFTER!", block["blocked"], block["reason"]) - if block["blocked"] == "": - logger.warning("blocked is empty, blocker='%s'", blocker) + if block["blocked"] is None or block["blocked"] == "": + logger.warning("block[blocked]='%s' is empty, blocker='%s'", block["blocked"], blocker) continue elif block["blocked"].endswith(".onion"): logger.debug("blocked='%s' is a TOR .onion domain - SKIPPED", block["blocked"]) continue + elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain") == "true": + logger.debug("blocked='%s' is an I2P .onion domain - SKIPPED", block["blocked"]) + continue elif block["blocked"].endswith(".arpa"): logger.debug("blocked='%s' is a reverse IP address - SKIPPED", block["blocked"]) continue @@ -378,38 +382,42 @@ def fetch_blocks(args: argparse.Namespace) -> int: continue elif block["blocked"].find("*") >= 0: logger.debug("blocker='%s' uses obfuscated domains", blocker) + instances.set_has_obfuscation(blocker, True) + obfuscated = obfuscated + 1 # Some friendica servers also obscure domains without hash - row = instances.deobfuscate("*", block["blocked"], block["hash"] if "hash" in block else None) + row = instances.deobfuscate("*", block["blocked"], block["digest"] if "digest" in block else None) logger.debug("row[]='%s'", type(row)) if row is None: logger.warning("Cannot deobfuscate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software) - instances.set_has_obfuscation(blocker, True) continue + deobfuscated = deobfuscated + 1 block["blocked"] = row["domain"] origin = row["origin"] nodeinfo_url = row["nodeinfo_url"] elif block["blocked"].find("?") >= 0: logger.debug("blocker='%s' uses obfuscated domains", blocker) + instances.set_has_obfuscation(blocker, True) + obfuscated = obfuscated + 1 # Some obscure them with question marks, not sure if that's dependent on version or not - row = instances.deobfuscate("?", block["blocked"], block["hash"] if "hash" in block else None) + row = instances.deobfuscate("?", block["blocked"], block["digest"] if "digest" in block else None) logger.debug("row[]='%s'", type(row)) if row is None: logger.warning("Cannot deobfuscate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block["blocked"], blocker, software) - instances.set_has_obfuscation(blocker, True) continue + deobfuscated = deobfuscated + 1 block["blocked"] = row["domain"] origin = row["origin"] nodeinfo_url = row["nodeinfo_url"] - logger.debug("Looking up instance by domainm, blocked='%s'", block["blocked"]) - if block["blocked"] == "": - logger.debug("block[blocked] is empty - SKIPPED!") + logger.debug("Looking up instance by domain, blocked='%s'", block["blocked"]) + if block["blocked"] is None or block["blocked"] == "": + logger.debug("block[blocked]='%s' is empty - SKIPPED!", block["blocked"]) continue logger.debug("block[blocked]='%s' - BEFORE!", block["blocked"]) @@ -438,10 +446,11 @@ def fetch_blocks(args: argparse.Namespace) -> int: logger.debug("Invoking cookies.clear(%s) ...", block["blocked"]) cookies.clear(block["blocked"]) - logger.debug("Checking if blocker='%s' has pending updates ...", blocker) - if instances.has_pending(blocker): - logger.debug("Flushing updates for blocker='%s' ...", blocker) - instances.update(blocker) + logger.info("blocker='%s' has %d obfuscated domain(s) and %d of them could be deobfuscated.", blocker, obfuscated, deobfuscated) + instances.set_obfuscated_blocks(blocker, obfuscated) + + logger.debug("Flushing updates for blocker='%s' ...", blocker) + instances.update(blocker) logger.debug("Invoking commit() ...") database.connection.commit() @@ -508,7 +517,8 @@ def fetch_observer(args: argparse.Namespace) -> int: logger.info("Fetching %d different table data ...", len(types)) for software in types: - logger.debug("software='%s' - BEFORE!", software) + logger.debug("software='%s'", software) + if args.software is not None and args.software != software: logger.debug("args.software='%s' does not match software='%s' - SKIPPED!", args.software, software) continue @@ -534,7 +544,8 @@ def fetch_observer(args: argparse.Namespace) -> int: for item in items: logger.debug("item[]='%s'", type(item)) domain = item.decode_contents() - domain = tidyup.domain(domain) if domain != None and domain != "" else None + logger.debug("domain[%s]='%s'", type(domain), domain) + domain = tidyup.domain(domain) if domain not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -552,7 +563,6 @@ def fetch_observer(args: argparse.Namespace) -> int: logger.debug("domain='%s' is already registered - SKIPPED!", domain) continue - software = software_helper.alias(software) logger.info("Fetching instances for domain='%s'", domain) federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) @@ -792,7 +802,7 @@ def fetch_fba_rss(args: argparse.Namespace) -> int: for item in rss.items: logger.debug("item[%s]='%s'", type(item), item) domain = item.link.split("=")[1] - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -872,11 +882,12 @@ def fetch_fbabot_atom(args: argparse.Namespace) -> int: logger.debug("entry[]='%s'", type(entry)) doc = bs4.BeautifulSoup(entry.content.value, "html.parser") logger.debug("doc[]='%s'", type(doc)) + for element in doc.findAll("a"): logger.debug("element[]='%s'", type(element)) for href in element["href"].split(","): logger.debug("href[%s]='%s' - BEFORE!", type(href), href) - domain = tidyup.domain(href) if href != None and href != "" else None + domain = tidyup.domain(href) if href not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1034,6 +1045,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int: logger.debug("Downloading %d files ...", len(blocklists.oliphant_blocklists)) for block in blocklists.oliphant_blocklists: # Is domain given and not equal blocker? + logger.debug("block[blocker]='%s',block[csv_url]='%s'", block["blocker"], block["csv_url"]) if isinstance(args.domain, str) and args.domain != block["blocker"]: logger.debug("Skipping blocker='%s', not matching args.domain='%s'", block["blocker"], args.domain) continue @@ -1052,14 +1064,8 @@ def fetch_txt(args: argparse.Namespace) -> int: logger.debug("Invoking locking.acquire() ...") locking.acquire() - # Static URLs - urls = ({ - "blocker": "seirdy.one", - "url" : "https://seirdy.one/pb/bsl.txt", - },) - - logger.info("Checking %d text file(s) ...", len(urls)) - for row in urls: + logger.info("Checking %d text file(s) ...", len(blocklists.txt_files)) + for row in blocklists.txt_files: logger.debug("Fetching row[url]='%s' ...", row["url"]) response = utils.fetch_url(row["url"], network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))) @@ -1071,7 +1077,7 @@ def fetch_txt(args: argparse.Namespace) -> int: logger.info("Processing %d domains ...", len(domains)) for domain in domains: logger.debug("domain='%s' - BEFORE!", domain) - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1127,7 +1133,7 @@ def fetch_fedipact(args: argparse.Namespace) -> int: logger.info("Checking %d row(s) ...", len(rows)) for row in rows: logger.debug("row[]='%s'", type(row)) - domain = tidyup.domain(row.contents[0]) if row.contents[0] != None and row.contents[0] != "" else None + domain = tidyup.domain(row.contents[0]) if row.contents[0] not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1250,185 +1256,6 @@ def fetch_joinmisskey(args: argparse.Namespace) -> int: logger.debug("Success! - EXIT!") return 0 -def fetch_joinfediverse(args: argparse.Namespace) -> int: - logger.debug("args[]='%s' - CALLED!", type(args)) - - logger.debug("Invoking locking.acquire() ...") - locking.acquire() - - source_domain = "joinfediverse.wiki" - if sources.is_recent(source_domain): - logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain) - return 1 - else: - logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain) - sources.update(source_domain) - - logger.info("Fetching /FediBlock wiki page from source_domain='%s' ...", source_domain) - raw = utils.fetch_url( - f"https://{source_domain}/FediBlock", - network.web_headers, - (config.get("connection_timeout"), config.get("read_timeout")) - ).text - logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw)) - - doc = bs4.BeautifulSoup(raw, "html.parser") - logger.debug("doc[]='%s'", type(doc)) - - tables = doc.findAll("table", {"class": "wikitable"}) - - logger.info("Analyzing %d table(s) ...", len(tables)) - blocklist = list() - for table in tables: - logger.debug("table[]='%s'", type(table)) - - rows = table.findAll("tr") - logger.info("Checking %d row(s) ...", len(rows)) - block_headers = dict() - for row in rows: - logger.debug("row[%s]='%s'", type(row), row) - - headers = row.findAll("th") - logger.debug("Found headers()=%d header(s)", len(headers)) - if len(headers) > 1: - block_headers = dict() - cnt = 0 - for header in headers: - cnt = cnt + 1 - logger.debug("header[]='%s',cnt=%d", type(header), cnt) - text = header.contents[0] - - logger.debug("text[]='%s'", type(text)) - if not isinstance(text, str): - logger.debug("text[]='%s' is not of type 'str' - SKIPPED!", type(text)) - continue - elif validators.domain(text.strip()): - logger.debug("text='%s' is a domain - SKIPPED!", text.strip()) - continue - - text = tidyup.domain(text.strip()) - logger.debug("text='%s' - AFTER!", text) - if text in ["domain", "instance", "subdomain(s)", "block reason(s)"]: - logger.debug("Found header: '%s'=%d", text, cnt) - block_headers[cnt] = text - - elif len(block_headers) == 0: - logger.debug("row is not scrapable - SKIPPED!") - continue - elif len(block_headers) > 0: - logger.debug("Found a row with %d scrapable headers ...", len(block_headers)) - cnt = 0 - block = dict() - - for element in row.find_all(["th", "td"]): - cnt = cnt + 1 - logger.debug("element[]='%s',cnt=%d", type(element), cnt) - if cnt in block_headers: - logger.debug("block_headers[%d]='%s'", cnt, block_headers[cnt]) - - text = element.text.strip() - key = block_headers[cnt] if block_headers[cnt] not in ["domain", "instance"] else "blocked" - - logger.debug("cnt=%d is wanted: key='%s',text[%s]='%s'", cnt, key, type(text), text) - if key in ["domain", "instance"]: - block[key] = text - elif key == "reason": - block[key] = tidyup.reason(text) - elif key == "subdomain(s)": - block[key] = list() - if text != "": - block[key] = text.split("/") - else: - logger.debug("key='%s'", key) - block[key] = text - - logger.debug("block()=%d ...", len(block)) - if len(block) > 0: - logger.debug("Appending block()=%d ...", len(block)) - blocklist.append(block) - - logger.debug("blocklist()=%d", len(blocklist)) - - database.cursor.execute("SELECT domain FROM instances WHERE domain LIKE 'climatejustice.%'") - domains = database.cursor.fetchall() - - logger.debug("domains(%d)[]='%s'", len(domains), type(domains)) - blocking = list() - for block in blocklist: - logger.debug("block='%s'", block) - if "subdomain(s)" in block and len(block["subdomain(s)"]) > 0: - origin = block["blocked"] - logger.debug("origin='%s'", origin) - for subdomain in block["subdomain(s)"]: - block["blocked"] = subdomain + "." + origin - logger.debug("block[blocked]='%s'", block["blocked"]) - blocking.append(block) - else: - blocking.append(block) - - logger.debug("blocking()=%d", blocking) - for block in blocking: - logger.debug("block[]='%s'", type(block)) - if "blocked" not in block: - raise KeyError(f"block()={len(block)} does not have element 'blocked'") - - block["blocked"] = tidyup.domain(block["blocked"]).encode("idna").decode("utf-8") - logger.debug("block[blocked]='%s' - AFTER!", block["blocked"]) - - if block["blocked"] == "": - logger.debug("block[blocked] is empty - SKIPPED!") - continue - elif not domain_helper.is_wanted(block["blocked"]): - logger.debug("block[blocked]='%s' is not wanted - SKIPPED!", block["blocked"]) - continue - elif instances.is_recent(block["blocked"]): - logger.debug("block[blocked]='%s' has been recently checked - SKIPPED!", block["blocked"]) - continue - - logger.debug("Proccessing blocked='%s' ...", block["blocked"]) - processing.instance(block["blocked"], "climatejustice.social", inspect.currentframe().f_code.co_name) - - blockdict = list() - for blocker in domains: - blocker = blocker[0] - logger.debug("blocker[%s]='%s'", type(blocker), blocker) - instances.set_last_blocked(blocker) - - for block in blocking: - logger.debug("block[blocked]='%s',block[block reason(s)]='%s' - BEFORE!", block["blocked"], block["block reason(s)"] if "block reason(s)" in block else None) - block["reason"] = tidyup.reason(block["block reason(s)"]) if "block reason(s)" in block else None - - logger.debug("block[blocked]='%s',block[reason]='%s' - AFTER!", block["blocked"], block["reason"]) - if block["blocked"] == "": - logger.debug("block[blocked] is empty - SKIPPED!") - continue - elif not domain_helper.is_wanted(block["blocked"]): - logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"]) - continue - - logger.debug("blocked='%s',reason='%s'", block["blocked"], block["reason"]) - if processing.block(blocker, block["blocked"], block["reason"], "reject") and config.get("bot_enabled"): - logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker) - blockdict.append({ - "blocked": block["blocked"], - "reason" : block["reason"], - }) - - if instances.has_pending(blocker): - logger.debug("Flushing updates for blocker='%s' ...", blocker) - instances.update(blocker) - - logger.debug("Invoking commit() ...") - database.connection.commit() - - logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict)) - if config.get("bot_enabled") and len(blockdict) > 0: - logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", blocker, len(blockdict)) - network.send_bot_post(blocker, blockdict) - - logger.debug("Success! - EXIT!") - return 0 - def recheck_obfuscation(args: argparse.Namespace) -> int: logger.debug("args[]='%s' - CALLED!", type(args)) @@ -1436,11 +1263,11 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: locking.acquire() if isinstance(args.domain, str) and args.domain != "" and domain_helper.is_wanted(args.domain): - database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 AND domain = ?", [args.domain]) + database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND domain = ?", [args.domain]) elif isinstance(args.software, str) and args.software != "" and validators.domain(args.software) == args.software: - database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 AND software = ?", [args.software]) + database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE (has_obfuscation = 1 OR has_obfuscation IS NULL) AND software = ?", [args.software]) else: - database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1") + database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE has_obfuscation = 1 OR has_obfuscation IS NULL") rows = database.cursor.fetchall() logger.info("Checking %d domains ...", len(rows)) @@ -1475,7 +1302,7 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: # c.s isn't part of oliphant's "hidden" blocklists logger.debug("row[domain]='%s'", row["domain"]) - if row["domain"] != "chaos.social" and not software_helper.is_relay(software) and not blocklists.has(row["domain"]): + if row["domain"] != "chaos.social" and not software_helper.is_relay(row["software"]) and not blocklists.has(row["domain"]): logger.debug("Invoking instances.set_total_blocks(%s, %d) ...", row["domain"], len(blocking)) instances.set_last_blocked(row["domain"]) instances.set_total_blocks(row["domain"], blocking) @@ -1491,19 +1318,22 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: if block["blocked"] == "": logger.debug("block[blocked] is empty - SKIPPED!") continue + elif block["blocked"].endswith(".onion"): + logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"]) + continue + elif block["blocked"].endswith(".i2p") and config.get("allow_i2p_domain") == "true": + logger.debug("blocked='%s' is an I2P onion domain name - SKIPPED!", block["blocked"]) + continue elif block["blocked"].endswith(".arpa"): logger.debug("blocked='%s' is a reversed IP address - SKIPPED!", block["blocked"]) continue elif block["blocked"].endswith(".tld"): logger.debug("blocked='%s' is a fake domain name - SKIPPED!", block["blocked"]) continue - elif block["blocked"].endswith(".onion"): - logger.debug("blocked='%s' is a TOR onion domain name - SKIPPED!", block["blocked"]) - continue elif block["blocked"].find("*") >= 0 or block["blocked"].find("?") >= 0: logger.debug("block='%s' is obfuscated.", block["blocked"]) obfuscated = obfuscated + 1 - blocked = utils.deobfuscate(block["blocked"], row["domain"], block["hash"] if "hash" in block else None) + blocked = utils.deobfuscate(block["blocked"], row["domain"], block["digest"] if "digest" in block else None) elif not domain_helper.is_wanted(block["blocked"]): logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"]) continue @@ -1536,7 +1366,8 @@ def recheck_obfuscation(args: argparse.Namespace) -> int: "reason" : block["reason"], }) - logger.debug("Settings obfuscated=%d for row[domain]='%s' ...", obfuscated, row["domain"]) + logger.debug("Setting obfuscated=%d for row[domain]='%s' ...", obfuscated, row["domain"]) + instances.set_has_obfuscation(row["domain"], (obfuscated > 0)) instances.set_obfuscated_blocks(row["domain"], obfuscated) logger.info("domain='%s' has %d obfuscated domain(s)", row["domain"], obfuscated) @@ -1604,7 +1435,7 @@ def fetch_fedilist(args: argparse.Namespace) -> int: continue logger.debug("row[hostname]='%s' - BEFORE!", row["hostname"]) - domain = tidyup.domain(row["hostname"]) if row["hostname"] != None and row["hostname"] != "" else None + domain = tidyup.domain(row["hostname"]) if row["hostname"] not in [None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1639,19 +1470,25 @@ def update_nodeinfo(args: argparse.Namespace) -> int: if args.domain is not None and args.domain != "": logger.debug("Fetching args.domain='%s'", args.domain) - database.cursor.execute("SELECT domain, software FROM instances WHERE domain = ?", [args.domain]) + database.cursor.execute("SELECT domain, software FROM instances WHERE domain = ? LIMIT 1", [args.domain]) elif args.software is not None and args.software != "": logger.info("Fetching domains for args.software='%s'", args.software) - database.cursor.execute("SELECT domain, software FROM instances WHERE software = ? AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [args.software.lower(), time.time() - config.get("recheck_nodeinfo")]) + database.cursor.execute("SELECT domain, software FROM instances WHERE software = ? ORDER BY last_updated ASC", [args.software]) elif args.mode is not None and args.mode != "": logger.info("Fetching domains for args.mode='%s'", args.mode.upper()) - database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode = ? AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [args.mode.upper(), time.time() - config.get("recheck_nodeinfo")]) + database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode = ? ORDER BY last_updated ASC", [args.mode]) elif args.no_software: logger.info("Fetching domains with no software type detected ...") - database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NULL AND (last_nodeinfo < ? OR last_nodeinfo IS NULL)", [time.time() - config.get("recheck_nodeinfo")]) + database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NULL ORDER BY last_updated ASC") + elif args.no_auto: + logger.info("Fetching domains with other detection mode than AUTO_DISOVERY being set ...") + database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode IS NOT NULL AND detection_mode != 'AUTO_DISCOVERY' ORDER BY last_updated ASC") + elif args.no_detection: + logger.info("Fetching domains with no detection mode being set ...") + database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode IS NULL ORDER BY last_updated ASC") else: logger.info("Fetching domains for recently updated ...") - database.cursor.execute("SELECT domain, software FROM instances WHERE last_nodeinfo < ? OR last_nodeinfo IS NULL", [time.time() - config.get("recheck_nodeinfo")]) + database.cursor.execute("SELECT domain, software FROM instances ORDER BY last_updated ASC") domains = database.cursor.fetchall() @@ -1717,8 +1554,8 @@ def fetch_instances_social(args: argparse.Namespace) -> int: fetched = network.get_json_api( source_domain, "/api/1.0/instances/list?count=0&sort_by=name", - headers, - (config.get("connection_timeout"), config.get("read_timeout")) + headers=headers, + timeout=(config.get("connection_timeout"), config.get("read_timeout")) ) logger.debug("fetched[]='%s'", type(fetched)) @@ -1741,7 +1578,76 @@ def fetch_instances_social(args: argparse.Namespace) -> int: logger.info("Checking %d row(s) ...", len(rows)) for row in rows: logger.debug("row[]='%s'", type(row)) - domain = tidyup.domain(row["name"]) if row["name"] != None and row["name"] != "" else None + domain = tidyup.domain(row["name"]) if row["name"] not in [None, ""] else None + logger.debug("domain='%s' - AFTER!", domain) + + if domain is None and domain == "": + logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain) + continue + + logger.debug("domain='%s' - BEFORE!", domain) + domain = domain.encode("idna").decode("utf-8") + logger.debug("domain='%s' - AFTER!", domain) + + if not domain_helper.is_wanted(domain): + logger.debug("domain='%s' is not wanted - SKIPPED!", domain) + continue + elif domain in domains: + logger.debug("domain='%s' is already added - SKIPPED!", domain) + continue + elif instances.is_registered(domain): + logger.debug("domain='%s' is already registered - SKIPPED!", domain) + continue + elif instances.is_recent(domain): + logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain) + continue + + logger.info("Fetching instances from domain='%s'", domain) + federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name) + + logger.debug("Success! - EXIT!") + return 0 + +def fetch_relaylist(args: argparse.Namespace) -> int: + logger.debug("args[]='%s' - CALLED!", type(args)) + + logger.debug("Invoking locking.acquire() ...") + locking.acquire() + + source_domain = "api.relaylist.com" + + if sources.is_recent(source_domain): + logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain) + return 1 + else: + logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain) + sources.update(source_domain) + + logger.info("Fetching list from source_domain='%s' ...", source_domain) + fetched = network.get_json_api( + source_domain, + "/relays", + {}, + (config.get("connection_timeout"), config.get("read_timeout")) + ) + logger.debug("fetched[]='%s'", type(fetched)) + + if "error_message" in fetched: + logger.warning("Error during fetching API result: '%s' - EXIT!", fetched["error_message"]) + return 2 + elif "exception" in fetched: + logger.warning("Exception '%s' during fetching API result - EXIT!", type(fetched["exception"])) + return 3 + elif "json" not in fetched: + logger.warning("fetched has no element 'json' - EXIT!") + return 4 + + domains = list() + + logger.info("Checking %d row(s) ...", len(fetched["json"])) + for row in fetched["json"]: + logger.debug("row[]='%s'", type(row)) + domain = urlparse(row["url"]).netloc.lower().split(":")[0] logger.debug("domain='%s' - AFTER!", domain) if domain is None and domain == "": @@ -1813,7 +1719,7 @@ def fetch_relays(args: argparse.Namespace) -> int: instances.set_last_instance_fetch(row["domain"]) instances.update(row["domain"]) continue - elif not "json" in raw: + elif "json" not in raw: logger.warning("raw()=%d does not contain key 'json' in response - SKIPPED!", len(raw)) continue elif not "metadata" in raw["json"]: @@ -1869,7 +1775,7 @@ def fetch_relays(args: argparse.Namespace) -> int: continue logger.debug("domain='%s' - BEFORE!", domain) - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1879,6 +1785,7 @@ def fetch_relays(args: argparse.Namespace) -> int: logger.debug("Appending domain='%s' to peers list for relay='%s' ...", domain, row["domain"]) peers.append(domain) + logger.debug("domains()=%d,domain='%s'", len(domains), domain) if dict_helper.has_key(domains, "domain", domain): logger.debug("domain='%s' already added", domain) continue @@ -1901,18 +1808,16 @@ def fetch_relays(args: argparse.Namespace) -> int: link = tag.find("a") logger.debug("link[%s]='%s'", type(link), link) - if link is None: - logger.warning("tag='%s' has no a-tag - SKIPPED!", tag) - continue - elif "href" not in link: - logger.warning("link()=%d has no key 'href' - SKIPPED!", len(link)) + if not isinstance(link, bs4.element.Tag): + logger.warning("tag[%s]='%s' is not type of 'bs4.element.Tag' - SKIPPED!", type(tag), tag) continue - components = urlparse(link["href"]) + components = urlparse(link.get("href")) + logger.debug("components(%d)='%s'", len(components), components) domain = components.netloc.lower().split(":")[0] logger.debug("domain='%s' - BEFORE!", domain) - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1922,6 +1827,7 @@ def fetch_relays(args: argparse.Namespace) -> int: logger.debug("Appending domain='%s' to peers list for relay='%s' ...", domain, row["domain"]) peers.append(domain) + logger.debug("domains()=%d,domain='%s'", len(domains), domain) if dict_helper.has_key(domains, "domain", domain): logger.debug("domain='%s' already added", domain) continue @@ -1935,7 +1841,7 @@ def fetch_relays(args: argparse.Namespace) -> int: logger.debug("Checking %d peer(s) row[domain]='%s' ...", len(raw["json"]["metadata"]["peers"]), row["domain"]) for domain in raw["json"]["metadata"]["peers"]: logger.debug("domain='%s' - BEFORE!", domain) - domain = tidyup.domain(domain) if domain != None and domain != "" else None + domain = tidyup.domain(domain) if domain not in[None, ""] else None logger.debug("domain='%s' - AFTER!", domain) if domain is None or domain == "": @@ -1945,11 +1851,12 @@ def fetch_relays(args: argparse.Namespace) -> int: logger.debug("Appending domain='%s' to peers list for relay='%s' ...", domain, row["domain"]) peers.append(domain) + logger.debug("domains()=%d,domain='%s'", len(domains), domain) if dict_helper.has_key(domains, "domain", domain): logger.debug("domain='%s' already added", domain) continue - logger.debug("Appending domain='%s',origin='%s',software='%s'", domain, row["domain"], row["software"]) + logger.debug("Appending domain='%s',origin='%s',software='%s' ...", domain, row["domain"], row["software"]) domains.append({ "domain": domain, "origin": row["domain"],