import json
-import requests
-
-from fba import blacklist
-from fba import config
-from fba import instances
-from fba import network
+from fba import csrf
+from fba.helpers import blacklist
+from fba.helpers import config
from fba.helpers import dicts
from fba.helpers import tidyup
+from fba.http import network
+
+from fba.models import instances
+
def fetch_peers(domain: str) -> list:
- # DEBUG: print(f"DEBUG: domain({len(domain)})={domain} - CALLED!")
+ # DEBUG: print(f"DEBUG: domain({len(domain)})='{domain}' - CALLED!")
if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
# DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...")
- peers = list()
- offset = 0
- step = config.get("misskey_limit")
+ peers = list()
+ offset = 0
+ step = config.get("misskey_limit")
+
+ # No CSRF by default, you don't have to add network.api_headers by yourself here
+ headers = tuple()
+
+ try:
+ # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
+ headers = csrf.determine(domain, dict())
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!")
+ instances.set_last_error(domain, exception)
+ return peers
# iterating through all "suspended" (follow-only in its terminology)
# instances page-by-page, since that troonware doesn't support
"sort" : "+pubAt",
"host" : None,
"limit": step
- }), {
- "Origin": domain
- })
+ }), headers)
else:
fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
"sort" : "+pubAt",
"host" : None,
"limit" : step,
"offset": offset - 1
- }), {
- "Origin": domain
- })
+ }), headers)
+
+ # Check records
+ # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
+ if "error_message" in fetched:
+ print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
+ instances.set_last_error(domain, fetched)
+ break
+ elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+ print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
+ instances.set_last_error(domain, fetched["json"]["error"]["message"])
+ break
+
+ rows = fetched["json"]
- # DEBUG: print(f"DEBUG: fetched()={len(fetched)}")
- if len(fetched) == 0:
+ # DEBUG: print(f"DEBUG: rows()={len(rows)}")
+ if len(rows) == 0:
# DEBUG: print(f"DEBUG: Returned zero bytes, exiting loop, domain='{domain}'")
break
- elif len(fetched) != config.get("misskey_limit"):
- # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
- offset = offset + (config.get("misskey_limit") - len(fetched))
+ elif len(rows) != config.get("misskey_limit"):
+ # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
+ offset = offset + (config.get("misskey_limit") - len(rows))
else:
# DEBUG: print(f"DEBUG: Raising offset by step={step}")
offset = offset + step
- # Check records
- # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]={type(fetched)}")
- if isinstance(fetched, dict) and "error" in fetched and "message" in fetched["error"]:
- print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
- instances.update_last_error(domain, fetched["error"]["message"])
- break
- elif "error_message" in fetched:
- print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
- instances.update_last_error(domain, fetched)
- break
-
already = 0
- for row in fetched["json"]:
+ # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
+ for row in rows:
# DEBUG: print(f"DEBUG: row()={len(row)}")
- if not "host" in row:
+ if "host" not in row:
print(f"WARNING: row()={len(row)} does not contain key 'host': {row},domain='{domain}'")
continue
elif not isinstance(row["host"], str):
- print(f"WARNING: row[host][]={type(row['host'])} is not 'str'")
+ print(f"WARNING: row[host][]='{type(row['host'])}' is not 'str'")
continue
elif blacklist.is_blacklisted(row["host"]):
# DEBUG: print(f"DEBUG: row[host]='{row['host']}' is blacklisted. domain='{domain}'")
# DEBUG: print(f"DEBUG: Adding peer: '{row['host']}'")
peers.append(row["host"])
- if already == len(fetched):
- print(f"WARNING: Host returned same set of '{already}' instances, aborting loop!")
+ if already == len(rows):
+ # DEBUG: print(f"DEBUG: Host returned same set of '{already}' instances, aborting loop!")
break
# DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'")
- instances.set_data("total_peers", domain, len(peers))
-
- # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
- instances.update_last_instance_fetch(domain)
+ instances.set_total_peers(domain, peers)
# DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'")
return peers
def fetch_blocks(domain: str) -> dict:
# DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
if not isinstance(domain, str):
- raise ValueError(f"Parameter domain[]={type(domain)} is not 'str'")
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
elif domain == "":
raise ValueError("Parameter 'domain' is empty")
- # DEBUG: print("DEBUG: Fetching misskey blocks from domain:", domain)
+ # DEBUG: print(f"DEBUG: Fetching misskey blocks from domain='{domain}'")
blocklist = {
"suspended": [],
"blocked" : []
}
- offset = 0
- step = config.get("misskey_limit")
+ offset = 0
+ step = config.get("misskey_limit")
+
+ # No CSRF by default, you don't have to add network.api_headers by yourself here
+ headers = tuple()
+
+ try:
+ # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'")
+ headers = csrf.determine(domain, dict())
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_blocks,{__name__}) - EXIT!")
+ instances.set_last_error(domain, exception)
+ return blocklist
+
+ # iterating through all "suspended" (follow-only in its terminology)
+ # instances page-by-page since it doesn't support sending them all at once
while True:
- # iterating through all "suspended" (follow-only in its terminology)
- # instances page-by-page, since that troonware doesn't support
- # sending them all at once
try:
# DEBUG: print(f"DEBUG: Fetching offset='{offset}' from '{domain}' ...")
if offset == 0:
"host" : None,
"suspended": True,
"limit" : step
- }), {
- "Origin": domain
- })
+ }), headers)
else:
# DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
"suspended": True,
"limit" : step,
"offset" : offset - 1
- }), {
- "Origin": domain
- })
+ }), headers)
+
+ # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
+ if "error_message" in fetched:
+ print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
+ instances.set_last_error(domain, fetched)
+ break
+ elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+ print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
+ instances.set_last_error(domain, fetched["json"]["error"]["message"])
+ break
+
+ rows = fetched["json"]
- # DEBUG: print("DEBUG: fetched():", len(fetched))
- if len(fetched) == 0:
+ # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - suspend")
+ if len(rows) == 0:
# DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
break
- elif len(fetched) != config.get("misskey_limit"):
- # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
- offset = offset + (config.get("misskey_limit") - len(fetched))
+ elif len(rows) != config.get("misskey_limit"):
+ # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
+ offset = offset + (config.get("misskey_limit") - len(rows))
else:
# DEBUG: print("DEBUG: Raising offset by step:", step)
offset = offset + step
count = 0
- for instance in fetched:
+ for instance in rows:
# Is it there?
- if instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance):
+ # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - suspend")
+ if "isSuspended" in instance and instance["isSuspended"] and not dicts.has_key(blocklist["suspended"], "domain", instance["host"]):
count = count + 1
- blocklist["suspended"].append(
- {
- "domain": tidyup.domain(instance["host"]),
- # no reason field, nothing
- "reason": None
- }
- )
+ blocklist["suspended"].append({
+ "domain": tidyup.domain(instance["host"]),
+ # no reason field, nothing
+ "reason": None
+ })
# DEBUG: print(f"DEBUG: count={count}")
if count == 0:
# DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
break
- except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
+ except network.exceptions as exception:
print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
- instances.update_last_error(domain, exception)
+ instances.set_last_error(domain, exception)
offset = 0
break
while True:
- # same shit, different asshole ("blocked" aka full suspend)
+ # Fetch blocked (full suspended) instances
try:
if offset == 0:
# DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
"host" : None,
"blocked": True,
"limit" : step
- }), {
- "Origin": domain
- })
+ }), headers)
else:
# DEBUG: print("DEBUG: Sending JSON API request to domain,step,offset:", domain, step, offset)
fetched = network.post_json_api(domain, "/api/federation/instances", json.dumps({
"blocked": True,
"limit" : step,
"offset" : offset - 1
- }), {
- "Origin": domain
- })
+ }), headers)
+
+ # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
+ if "error_message" in fetched:
+ print(f"WARNING: post_json_api() for domain='{domain}' returned error message: {fetched['error_message']}")
+ instances.set_last_error(domain, fetched)
+ break
+ elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
+ print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
+ instances.set_last_error(domain, fetched["json"]["error"]["message"])
+ break
- # DEBUG: print("DEBUG: fetched():", len(fetched))
- if len(fetched) == 0:
+ rows = fetched["json"]
+
+ # DEBUG: print(f"DEBUG: rows({len(rows)})={rows} - blocked")
+ if len(rows) == 0:
# DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain)
break
- elif len(fetched) != config.get("misskey_limit"):
- # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.get('misskey_limit')}'")
- offset = offset + (config.get("misskey_limit") - len(fetched))
+ elif len(rows) != config.get("misskey_limit"):
+ # DEBUG: print(f"DEBUG: Fetched '{len(rows)}' row(s) but expected: '{config.get('misskey_limit')}'")
+ offset = offset + (config.get("misskey_limit") - len(rows))
else:
# DEBUG: print("DEBUG: Raising offset by step:", step)
offset = offset + step
count = 0
- for instance in fetched:
+ for instance in rows:
# Is it there?
- if instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance):
+ # DEBUG: print(f"DEBUG: instance[{type(instance)}]='{instance}' - blocked")
+ if "isBlocked" in instance and instance["isBlocked"] and not dicts.has_key(blocklist["blocked"], "domain", instance["host"]):
count = count + 1
blocklist["blocked"].append({
"domain": tidyup.domain(instance["host"]),
# DEBUG: print("DEBUG: API is no more returning new instances, aborting loop!")
break
- except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as exception:
+ except network.exceptions as exception:
print(f"WARNING: Caught error, exiting loop: domain='{domain}',exception[{type(exception)}]='{str(exception)}'")
- instances.update_last_error(domain, exception)
+ instances.set_last_error(domain, exception)
offset = 0
break
- # DEBUG: print(f"DEBUG: Updating last_instance_fetch for domain='{domain}' ...")
- instances.update_last_instance_fetch(domain)
-
# DEBUG: print(f"DEBUG: Returning for domain='{domain}',blocked()={len(blocklist['blocked'])},suspended()={len(blocklist['suspended'])}")
return {
"reject" : blocklist["blocked"],
projects / fba.git / blobdiff