logger.debug("domain='%s' is already registered - SKIPPED!", domain)
continue
- logger.info("Fetching instances for domain='%s'", domain)
+ logger.info("Fetching instances for domain='%s' ...", domain)
federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
logger.debug("Success! - EXIT!")
logger.debug("entry[]='%s'", type(entry))
doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
logger.debug("doc[]='%s'", type(doc))
+ elements = doc.findAll("a")
- for element in doc.findAll("a"):
- logger.debug("element[]='%s'", type(element))
+ logger.debug("Checking %d element(s) ...", len(elements))
+ for element in elements:
+ logger.debug("element[%s]='%s'", type(element), element)
for href in element["href"].split(","):
logger.debug("href[%s]='%s' - BEFORE!", type(href), href)
domain = tidyup.domain(href) if href not in [None, ""] else None
origin = row["origin"]
software = row["software"]
+ logger.debug("software='%s'", software)
if software is None:
logger.warning("args.domain='%s' has no software detected. You can try to run ./fba.py update_nodeinfo --domain=%s --force to get it updated.", args.domain, args.domain)
return 102
rows = database.cursor.fetchall()
logger.info("Checking %d entries ...", len(rows))
for row in rows:
- logger.debug("row[domain]='%s'", row["domain"])
- if row["domain"] == "":
- logger.debug("row[domain] is empty - SKIPPED!")
- continue
-
logger.debug("row[domain]='%s' - BEFORE!", row["domain"])
domain = row["domain"].encode("idna").decode("utf-8")
logger.debug("domain='%s' - AFTER!", domain)
for domain in domains:
logger.debug("domain='%s' - BEFORE!", domain)
domain = tidyup.domain(domain) if domain not in[None, ""] else None
-
logger.debug("domain='%s' - AFTER!", domain)
+
if domain in [None, ""]:
logger.debug("domain='%s' is empty after tidyup.domain() - SKIPPED!", domain)
continue
elif not domain_helper.is_wanted(domain):
logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
continue
- elif instances.is_recent(domain):
- logger.debug("domain='%s' has been recently crawled - SKIPPED!", domain)
+ elif not args.force and instances.is_registered(domain):
+ logger.debug("domain='%s' is already registered - SKIPPED!", domain)
continue
- logger.debug("Processing domain='%s',row[blocker]='%s'", domain, row["blocker"])
- processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name)
-
+ logger.debug("Processing domain='%s',row[blocker]='%s' ...", domain, row["blocker"])
+ processed = processing.instance(domain, row["blocker"], inspect.currentframe().f_code.co_name, force=args.force)
logger.debug("processed='%s'", processed)
- if not processed:
- logger.debug("domain='%s' was not generically processed - SKIPPED!", domain)
- continue
logger.debug("Success! - EXIT!")
return 0
elif args.no_software:
logger.info("Fetching domains with no software type detected ...")
database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NULL ORDER BY last_updated ASC")
+ elif args.with_software:
+ logger.info("Fetching domains with any software type detected ...")
+ database.cursor.execute("SELECT domain, software FROM instances WHERE software IS NOT NULL ORDER BY last_updated ASC")
elif args.no_auto:
logger.info("Fetching domains with other detection mode than AUTO_DISOVERY being set ...")
database.cursor.execute("SELECT domain, software FROM instances WHERE detection_mode IS NOT NULL AND detection_mode != 'AUTO_DISCOVERY' ORDER BY last_updated ASC")
locking.acquire()
if args.domain is not None and args.domain != "":
+ logger.debug("Fetching instances record for args.domain='%s' ...", args.domain)
database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay', 'pub-relay') AND domain = ? LIMIT 1", [args.domain])
elif args.software is not None and args.software != "":
- database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay', 'pub-relay') AND software = ?", [args.software])
+ logger.debug("Fetching instances records for args.software='%s' ...", args.software)
+ database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay', 'pub-relay') AND nodeinfo_url IS NOT NULL AND software = ? ORDER BY last_updated DESC", [args.software])
else:
- database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay', 'pub-relay')")
+ logger.debug("Fetch all relay instances ...")
+ database.cursor.execute("SELECT domain, software, nodeinfo_url FROM instances WHERE software IN ('activityrelay', 'aoderelay', 'selective-relay', 'pub-relay') AND nodeinfo_url IS NOT NULL ORDER BY last_updated DESC")
domains = list()
rows = database.cursor.fetchall()
logger.info("Checking %d relays ...", len(rows))
for row in rows:
- logger.debug("row[domain]='%s',row[software]='%s' ...", row["domain"], row["software"])
- peers = list()
+ logger.debug("row[domain]='%s',row[software]='%s'", row["domain"], row["software"])
if not args.force and instances.is_recent(row["domain"]):
logger.debug("row[domain]='%s' has been recently fetched - SKIPPED!", row["domain"])
continue
+ elif row["nodeinfo_url"] is None:
+ logger.warning("row[domain]='%s' has empty nodeinfo_url but this is required - SKIPPED!", row["domain"])
+ continue
+ peers = list()
try:
+ logger.debug("row[domain]='%s',row[software]='%s' - checking ....", row["domain"], row["software"])
if row["software"] == "pub-relay":
logger.info("Fetching row[nodeinfo_url]='%s' from relay row[domain]='%s',row[software]='%s' ...", row["nodeinfo_url"], row["domain"], row["software"])
raw = network.fetch_api_url(
projects / fba.git / blobdiff