sources.update(source_domain)
domains = {}
- try:
- logger.info("Fetching domainlist from source_domain='%s' ...", source_domain)
- fetched = network.post_json_api(
- source_domain,
- "/v1/graphql",
- json.dumps({
- "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
- })
- )
- logger.debug("fetched[]='%s'", type(fetched))
- if "error_message" in fetched:
- logger.warning("post_json_api() for 'gql.sources.bka.li' returned error message='%s' - EXIT!", fetched["error_message"])
- return 100
- elif "json" not in fetched:
- logger.warning("post_json_api() returned fetched[]='%s' with missing 'json' element - EXIT!", type(fetched))
- return 101
- elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
- logger.warning("post_json_api() returned error: '%s' - EXIT!", fetched["json"]["error"]["message"])
- return 102
-
- rows = fetched["json"]
+ logger.info("Fetching domainlist from source_domain='%s' ...", source_domain)
+ fetched = network.post_json_api(
+ source_domain,
+ "/v1/graphql",
+ json.dumps({
+ "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
+ })
+ )
- logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
- if len(rows) == 0:
- raise RuntimeError("WARNING: Returned no records")
- elif "data" not in rows:
- raise KeyError(f"WARNING: rows()={len(rows)} does not contain key 'data'")
- elif "nodeinfo" not in rows["data"]:
- raise KeyError(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'")
-
- logger.info("Checking %d rows ...", len(rows["data"]["nodeinfo"]))
- for entry in rows["data"]["nodeinfo"]:
- logger.debug("entry[%s]='%s'", type(entry), entry)
- if "domain" not in entry:
- logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry))
- continue
- elif entry["domain"] in [None, ""]:
- logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"])
- continue
- elif not domain_helper.is_wanted(entry["domain"]):
- logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"])
- continue
- elif instances.is_registered(entry["domain"]):
- logger.debug("entry[domain]='%s' is already registered - SKIPPED!", entry["domain"])
- continue
+ logger.debug("fetched[]='%s'", type(fetched))
+ if "error_message" in fetched:
+ logger.warning("post_json_api() for '%s' returned error message='%s' - EXIT!", source_domain, fetched["error_message"])
+ return 100
+ elif "json" not in fetched:
+ logger.warning("post_json_api() returned fetched[]='%s' with missing 'json' element - EXIT!", type(fetched))
+ return 101
+ elif "exception" in fetched:
+ logger.warning("post_json_api() for '%s' has caused an exception='%s' - EXIT!", source_domain, fetched["exception"])
+ return 102
+ elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+ logger.warning("post_json_api() returned error: '%s' - EXIT!", fetched["json"]["error"]["message"])
+ return 103
- logger.debug("Adding domain='%s' ...", entry["domain"])
- domains.append(entry["domain"])
+ rows = fetched["json"]
+
+ logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
+ if len(rows) == 0:
+ raise RuntimeError("WARNING: Returned no records")
+ elif "data" not in rows:
+ raise KeyError(f"WARNING: rows()={len(rows)} does not contain key 'data'")
+ elif "nodeinfo" not in rows["data"]:
+ raise KeyError(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'")
+
+ logger.info("Checking %d rows ...", len(rows["data"]["nodeinfo"]))
+ for entry in rows["data"]["nodeinfo"]:
+ logger.debug("entry[%s]='%s'", type(entry), entry)
+ if "domain" not in entry:
+ logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry))
+ continue
+ elif entry["domain"] in [None, ""]:
+ logger.debug("entry[domain]='%s' is empty - SKIPPED!", entry["domain"])
+ continue
+ elif not domain_helper.is_wanted(entry["domain"]):
+ logger.debug("entry[domain]='%s' is not wanted - SKIPPED!", entry["domain"])
+ continue
+ elif instances.is_registered(entry["domain"]):
+ logger.debug("entry[domain]='%s' is already registered - SKIPPED!", entry["domain"])
+ continue
- except network.exceptions as exception:
- logger.warning("Cannot fetch graphql,exception[%s]:'%s' - EXIT!", type(exception), str(exception))
- return 102
+ logger.debug("Adding domain='%s' ...", entry["domain"])
+ domains.append(entry["domain"])
logger.debug("domains()=%d", len(domains))
if len(domains) > 0:
continue
nodes = []
- try:
- logger.debug("Fetching table data for software='%s' ...", software)
- raw = network.post_json_api(
- f"api.{source_domain}",
- "/",
- json.dumps({
- "query": "{nodes(softwarename:\"" + software + "\"){domain}}"
- })
- )
-
- logger.debug("raw[%s]()=%d", type(raw), len(raw))
- if "exception" in raw:
- logger.warning("source_domain='%s' has caused an exception: '%s' - raising again ...", raw["domain"], type(raw["exception"]))
- raise raw["exception"]
- elif "error_message" in raw:
- logger.warning("source_domain='%s' has caused error message: '%s' - SKIPPED!", raw["domain"], raw["error_message"])
- continue
- elif "data" not in raw["json"]:
- logger.warning("Cannot find key 'nodes' in raw[json]()=%d", len(raw["json"]))
- continue
- elif "nodes" not in raw["json"]["data"]:
- logger.warning("Cannot find key 'nodes' in raw[json][data]()=%d", len(raw["json"]["data"]))
- continue
- nodes = raw["json"]["data"]["nodes"]
- logger.debug("nodes()=%d", len(nodes))
+ logger.debug("Fetching table data for software='%s' ...", software)
+ raw = network.post_json_api(
+ f"api.{source_domain}",
+ "/",
+ json.dumps({
+ "query": "{nodes(softwarename:\"" + software + "\"){domain}}"
+ })
+ )
- except network.exceptions as exception:
- logger.warning("Cannot fetch software='%s' from source_domain='%s': '%s'", software, source_domain, type(exception))
+ logger.debug("raw[%s]()=%d", type(raw), len(raw))
+ if "exception" in raw:
+ logger.warning("source_domain='%s' has caused an exception: '%s' - raising again ...", raw["domain"], type(raw["exception"]))
+ raise raw["exception"]
+ elif "error_message" in raw:
+ logger.warning("source_domain='%s' has caused error message: '%s' - SKIPPED!", raw["domain"], raw["error_message"])
+ continue
+ elif "data" not in raw["json"]:
+ logger.warning("Cannot find key 'nodes' in raw[json]()=%d", len(raw["json"]))
continue
+ elif "nodes" not in raw["json"]["data"]:
+ logger.warning("Cannot find key 'nodes' in raw[json][data]()=%d", len(raw["json"]["data"]))
+ continue
+
+ nodes = raw["json"]["data"]["nodes"]
+ logger.debug("nodes()=%d", len(nodes))
logger.info("Checking %d nodes,software='%s' ...", len(nodes), software)
for node in nodes:
logger.debug("Invoking instances.set_last_error(%s,%s) ...", domain, exception)
instances.set_last_error(domain, exception)
- raise exception
logger.debug("Returning json_reply(%d)[]='%s' - EXIT!", len(json_reply), type(json_reply))
return json_reply
logger.debug("Returning empty list ... - EXIT!")
return []
- try:
- logger.debug("Fetching '/api/v3/site' from domain='%s' ...", domain)
- data = network.get_json_api(
- domain,
- "/api/v3/site",
- headers=headers,
- timeout=config.timeout
- )
-
- logger.debug("data[]='%s'", type(data))
- if "error_message" in data:
- logger.warning("Could not reach any JSON API: domain='%s',error_message='%s'", domain, data["error_message"])
- instances.set_last_error(domain, data)
- elif "federated_instances" in data["json"] and isinstance(data["json"]["federated_instances"], dict):
- logger.debug("Found federated_instances for domain='%s'", domain)
- peers = peers + federation.add_peers(data["json"]["federated_instances"])
- logger.debug("peers()=%d after adding", len(peers))
-
- logger.debug("Marking domain='%s' as successfully handled ...", domain)
- instances.set_success(domain)
-
- logger.debug("peers()=%d", len(peers))
- if len(peers) == 0:
- logger.debug("Fetching instances for domain='%s' from /instances ...", domain)
- peers = fetch_instances(domain, origin)
- logger.debug("peers()=%d after fetch_instances(%s, %s)", len(peers), domain, origin)
+ logger.debug("Fetching '/api/v3/site' from domain='%s' ...", domain)
+ data = network.get_json_api(
+ domain,
+ "/api/v3/site",
+ headers=headers,
+ timeout=config.timeout
+ )
+
+ logger.debug("data[]='%s'", type(data))
+ if "exception" in data:
+ logger.warning("Could not reach any JSON API: domain='%s',exception='%s'", domain, data["exception"])
+ instances.set_last_error(domain, data)
+ elif "error_message" in data:
+ logger.warning("Could not reach any JSON API: domain='%s',error_message='%s'", domain, data["error_message"])
+ instances.set_last_error(domain, data)
+ elif "federated_instances" in data["json"] and isinstance(data["json"]["federated_instances"], dict):
+ logger.debug("Found federated_instances for domain='%s'", domain)
+ peers = peers + federation.add_peers(data["json"]["federated_instances"])
+ logger.debug("peers()=%d after adding", len(peers))
+
+ logger.debug("Marking domain='%s' as successfully handled ...", domain)
+ instances.set_success(domain)
- except network.exceptions as exception:
- logger.warning("Exception during fetching JSON: domain='%s',exception[%s]:'%s'", domain, type(exception), str(exception))
- instances.set_last_error(domain, exception)
+ logger.debug("peers()=%d", len(peers))
+ if len(peers) == 0:
+ logger.debug("Fetching instances for domain='%s' from /instances ...", domain)
+ peers = fetch_instances(domain, origin)
+ logger.debug("peers()=%d after fetch_instances(%s, %s)", len(peers), domain, origin)
logger.debug("peers()=%d - EXIT!", len(peers))
return peers
# Check records
logger.debug("fetched[]='%s'", type(fetched))
- if "error_message" in fetched:
+ if "exception" in fetched:
+ logger.warning("post_json_api() for domain='%s' raised an exception: '%s'", domain, fetched['exception'])
+ instances.set_last_error(domain, fetched)
+ break
+ elif "error_message" in fetched:
logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
instances.set_last_error(domain, fetched)
break
# instances page-by-page since it doesn't support sending them all at once
logger.debug("Fetching misskey blocks from domain='%s' ...", domain)
while True:
- logger.debug("offset=%d", offset)
- try:
- logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
- if offset == 0:
- logger.debug("Sending JSON API request to domain='%s',step=%d ...", domain, step)
- fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
- "allowPartial": True,
- "sort" : "+pubSub",
- "host" : None,
- "limit" : step
- }), headers)
- else:
- logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d ...", domain, step, offset)
- fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
- "allowPartial": True,
- "sort" : "+pubSub",
- "host" : None,
- "limit" : step,
- "offset" : offset - 1
- }), headers)
-
- logger.debug("fetched[]='%s'", type(fetched))
- if "error_message" in fetched:
- logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
- instances.set_last_error(domain, fetched)
- break
- elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
- logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
- instances.set_last_error(domain, fetched["json"]["error"]["message"])
- break
-
- rows = fetched["json"]
-
- logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
- if len(rows) == 0:
- logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
- break
- elif len(rows) != step:
- logger.debug("Fetched %d row(s) but expected: %d", len(rows), step)
- offset = offset + (step - len(rows))
+ logger.debug("Fetching offset=%d from domain='%s' ...", offset, domain)
+ if offset == 0:
+ logger.debug("Sending JSON API request to domain='%s',step=%d ...", domain, step)
+ fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
+ "allowPartial": True,
+ "sort" : "+pubSub",
+ "host" : None,
+ "limit" : step
+ }), headers)
+ else:
+ logger.debug("Sending JSON API request to domain='%s',step=%d,offset=%d ...", domain, step, offset)
+ fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
+ "allowPartial": True,
+ "sort" : "+pubSub",
+ "host" : None,
+ "limit" : step,
+ "offset" : offset - 1
+ }), headers)
+
+ logger.debug("fetched[]='%s'", type(fetched))
+ if "exception" in fetched:
+ logger.warning("post_json_api() for domain='%s' caused an exception: '%s'", domain, fetched['exception'])
+ instances.set_last_error(domain, fetched)
+ break
+ elif "error_message" in fetched:
+ logger.warning("post_json_api() for domain='%s' returned error message: '%s'", domain, fetched['error_message'])
+ instances.set_last_error(domain, fetched)
+ break
+ elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+ logger.warning("post_json_api() returned error: '%s'", fetched["json"]["error"]["message"])
+ instances.set_last_error(domain, fetched["json"]["error"]["message"])
+ break
+
+ rows = fetched["json"]
+
+ logger.debug("rows(%d)[]='%s'", len(rows), type(rows))
+ if len(rows) == 0:
+ logger.debug("Returned zero bytes, domain='%s' - BREAK!", domain)
+ break
+ elif len(rows) != step:
+ logger.debug("Fetched %d row(s) but expected: %d", len(rows), step)
+ offset = offset + (step - len(rows))
+ else:
+ logger.debug("Raising offset by step=%d", step)
+ offset = offset + step
+
+ count = 0
+ logger.debug("Checking %d row(s) of instances ...", len(rows))
+ for instance in rows:
+ # Is it there?
+ logger.debug("instance[]='%s'", type(instance))
+ if "host" not in instance:
+ logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
+ continue
+ elif not isinstance(instance["host"], str):
+ logger.warning("instance[host][]='%s' is not of expected type 'str' - SKIPPED!", type(instance["host"]))
+ continue
+ elif instance["host"] == "":
+ logger.warning("instance[host] is an empty string,domain='%s' - SKIPPED!", domain)
+ continue
+
+ logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
+ blocked = tidyup.domain(instance["host"])
+ logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked)
+
+ if blocked in [None, ""]:
+ logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
+ continue
+ elif blocked.find("*") == -1 and blocked.find("?") == -1 and not domain_helper.is_wanted(blocked):
+ logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
+ continue
+ elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
+ count = count + 1
+ logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #1", domain, blocked)
+ blocklist.append({
+ "blocker" : domain,
+ "blocked" : blocked,
+ "reason" : None,
+ "block_level": "suspended",
+ })
+ elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
+ count = count + 1
+ logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #2", domain, blocked)
+ blocklist.append({
+ "blocker" : domain,
+ "blocked" : blocked,
+ "reason" : None,
+ "block_level": "suspended",
+ })
+ elif "isSilenced" in instance and instance["isSilenced"] and not dict_helper.has_key(blocklist, "blocked", blocked):
+ count = count + 1
+ logger.debug("Appending blocker='%s',blocked='%s',block_level='silenced' ...", domain, blocked)
+ blocklist.append({
+ "blocker" : domain,
+ "blocked" : blocked,
+ "reason" : None,
+ "block_level": "silenced",
+ })
else:
- logger.debug("Raising offset by step=%d", step)
- offset = offset + step
-
- count = 0
- logger.debug("Checking %d row(s) of instances ...", len(rows))
- for instance in rows:
- # Is it there?
- logger.debug("instance[]='%s'", type(instance))
- if "host" not in instance:
- logger.warning("instance(%d)='%s' has no key 'host' - SKIPPED!", len(instance), instance)
- continue
- elif not isinstance(instance["host"], str):
- logger.warning("instance[host][]='%s' is not of expected type 'str' - SKIPPED!", type(instance["host"]))
- continue
- elif instance["host"] == "":
- logger.warning("instance[host] is an empty string,domain='%s' - SKIPPED!", domain)
- continue
-
- logger.debug("instance[host]='%s' - BEFORE!", instance["host"])
- blocked = tidyup.domain(instance["host"])
- logger.debug("blocked[%s]='%s' - AFTER!", type(blocked), blocked)
-
- if blocked in [None, ""]:
- logger.warning("instance[host]='%s' is None or empty after tidyup.domain() - SKIPPED!", instance["host"])
- continue
- elif blocked.find("*") == -1 and blocked.find("?") == -1 and not domain_helper.is_wanted(blocked):
- logger.debug("blocked='%s' is not wanted - SKIPPED!", blocked)
- continue
- elif "isSuspended" in instance and instance["isSuspended"] and not dict_helper.has_key(blocklist, "blocked", blocked):
- count = count + 1
- logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #1", domain, blocked)
- blocklist.append({
- "blocker" : domain,
- "blocked" : blocked,
- "reason" : None,
- "block_level": "suspended",
- })
- elif "isBlocked" in instance and instance["isBlocked"] and not dict_helper.has_key(blocklist, "blocked", blocked):
- count = count + 1
- logger.debug("Appending blocker='%s',blocked='%s',block_level='suspended' ... #2", domain, blocked)
- blocklist.append({
- "blocker" : domain,
- "blocked" : blocked,
- "reason" : None,
- "block_level": "suspended",
- })
- elif "isSilenced" in instance and instance["isSilenced"] and not dict_helper.has_key(blocklist, "blocked", blocked):
- count = count + 1
- logger.debug("Appending blocker='%s',blocked='%s',block_level='silenced' ...", domain, blocked)
- blocklist.append({
- "blocker" : domain,
- "blocked" : blocked,
- "reason" : None,
- "block_level": "silenced",
- })
- else:
- count = count + 1
- logger.debug("domain='%s',blocked='%s' is not marked suspended, blocked or silenced - SKIPPED!", domain, blocked)
- continue
-
- logger.debug("count=%d", count)
- if count == 0:
- logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
- instances.set_last_offset(domain, 0)
- break
-
- except network.exceptions as exception:
- logger.warning("Caught error, exiting loop: domain='%s',exception[%s]='%s'", domain, type(exception), str(exception))
- instances.set_last_offset(domain, offset)
- instances.set_last_error(domain, exception)
+ count = count + 1
+ logger.debug("domain='%s',blocked='%s' is not marked suspended, blocked or silenced - SKIPPED!", domain, blocked)
+ continue
+
+ logger.debug("count=%d", count)
+ if count == 0:
+ logger.debug("API is no more returning new instances, aborting loop! domain='%s'", domain)
+ instances.set_last_offset(domain, 0)
break
sleep = default_rng.integers(low=config.get("low_sleep"), high=config.get("high_sleep"))
projects / fba.git / commitdiff