import atoma
import bs4
import markdown
+import reqto
+import requests
import validators
from fba import blacklist
from fba import blocks
-from fba import boot
from fba import config
+from fba import federation
from fba import fba
from fba import instances
+from fba import locking
from fba import network
-from fba.federation import *
+from fba.helpers import tidyup
+
+from fba.networks import friendica
+from fba.networks import mastodon
+from fba.networks import misskey
+from fba.networks import pleroma
def check_instance(args: argparse.Namespace) -> int:
# DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!")
return status
def fetch_bkali(args: argparse.Namespace):
- # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+ # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
domains = list()
try:
fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
# DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'")
if len(fetched) == 0:
raise Exception("WARNING: Returned no records")
- elif not "data" in fetched:
+ elif "data" not in fetched:
raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'")
- elif not "nodeinfo" in fetched["data"]:
+ elif "nodeinfo" not in fetched["data"]:
raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'")
for entry in fetched["data"]["nodeinfo"]:
# DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
domains.append(entry["domain"])
- except BaseException as exception:
+ except network.exceptions as exception:
print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'")
sys.exit(255)
# DEBUG: print(f"DEBUG: domains()={len(domains)}")
if len(domains) > 0:
- boot.acquire_lock()
+ locking.acquire()
print(f"INFO: Adding {len(domains)} new instances ...")
for domain in domains:
- print(f"INFO: Fetching instances from domain='{domain}' ...")
- fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+ try:
+ print(f"INFO: Fetching instances from domain='{domain}' ...")
+ federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+ instances.update_last_error(domain, exception)
# DEBUG: print("DEBUG: EXIT!")
def fetch_blocks(args: argparse.Namespace):
- # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+ # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
if args.domain is not None and args.domain != "":
# DEBUG: print(f"DEBUG: args.domain='{args.domain}' - checking ...")
if not validators.domain(args.domain):
print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.")
return
- boot.acquire_lock()
+ locking.acquire()
if args.domain is not None and args.domain != "":
# Re-check single domain
for blocker, software, origin, nodeinfo_url in rows:
# DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
blockdict = list()
- blocker = fba.tidyup_domain(blocker)
+ blocker = tidyup.domain(blocker)
# DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
if blocker == "":
mastodon.fetch_blocks(blocker, origin, nodeinfo_url)
elif software == "friendica" or software == "misskey":
print(f"INFO: blocker='{blocker}',software='{software}'")
+
+ blocking = list()
if software == "friendica":
- rows = friendica.fetch_blocks(blocker)
+ blocking = friendica.fetch_blocks(blocker)
elif software == "misskey":
- rows = misskey.fetch_blocks(blocker)
+ blocking = misskey.fetch_blocks(blocker)
- print(f"INFO: Checking {len(rows.items())} entries from blocker='{blocker}',software='{software}' ...")
- for block_level, blocklist in rows.items():
+ print(f"INFO: Checking {len(blocking.items())} entries from blocker='{blocker}',software='{software}' ...")
+ for block_level, blocklist in blocking.items():
# DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist))
- block_level = fba.tidyup_domain(block_level)
+ block_level = tidyup.domain(block_level)
# DEBUG: print("DEBUG: AFTER-block_level:", block_level)
if block_level == "":
print("WARNING: block_level is empty, blocker:", blocker)
for block in blocklist:
blocked, reason = block.values()
# DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!")
- blocked = fba.tidyup_domain(blocked)
- reason = fba.tidyup_reason(reason) if reason is not None and reason != "" else None
+ blocked = tidyup.domain(blocked)
+ reason = tidyup.reason(reason) if reason is not None and reason != "" else None
# DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
if blocked == "":
searchres = fba.cursor.fetchone()
+ # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
if searchres is None:
print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
continue
- blocked = searchres[0]
- origin = searchres[1]
+ blocked = searchres[0]
+ origin = searchres[1]
nodeinfo_url = searchres[2]
elif blocked.count("?") > 0:
# Some obscure them with question marks, not sure if that's dependent on version or not
searchres = fba.cursor.fetchone()
+ # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
if searchres is None:
print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
continue
- blocked = searchres[0]
- origin = searchres[1]
+ blocked = searchres[0]
+ origin = searchres[1]
nodeinfo_url = searchres[2]
elif not validators.domain(blocked):
print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
if config.get("bot_enabled") and len(blockdict) > 0:
network.send_bot_post(blocker, blockdict)
- blockdict = []
-
# DEBUG: print("DEBUG: EXIT!")
def fetch_cs(args: argparse.Namespace):
- # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+ # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
extensions = [
'extra',
'abbr',
}
raw = fba.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
- # DEBUG: print(f"DEBUG: raw()={len(raw)}[]={type(raw)}")
+ # DEBUG: print(f"DEBUG: raw()={len(raw)}[]='{type(raw)}'")
doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser')
- # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}")
+ # DEBUG: print(f"DEBUG: doc()={len(doc)}[]='{type(doc)}'")
silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody")
- # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}")
- domains["silenced"] = domains["silenced"] + fba.find_domains(silenced)
+ # DEBUG: print(f"DEBUG: silenced[]='{type(silenced)}'")
+ domains["silenced"] = domains["silenced"] + federation.find_domains(silenced)
blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody")
- # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}")
- domains["reject"] = domains["reject"] + fba.find_domains(blocked)
+ # DEBUG: print(f"DEBUG: blocked[]='{type(blocked)}'")
+ domains["reject"] = domains["reject"] + federation.find_domains(blocked)
# DEBUG: print(f"DEBUG: domains()={len(domains)}")
if len(domains) > 0:
- boot.acquire_lock()
+ locking.acquire()
print(f"INFO: Adding {len(domains)} new instances ...")
for block_level in domains:
blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level)
if not instances.is_registered(row["domain"]):
- print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
- fba.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
+ try:
+ print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
+ federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'")
+ instances.update_last_error(row["domain"], exception)
# DEBUG: print("DEBUG: Committing changes ...")
fba.connection.commit()
# DEBUG: print("DEBUG: EXIT!")
def fetch_fba_rss(args: argparse.Namespace):
- # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+ # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
domains = list()
print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...")
# DEBUG: print(f"DEBUG: Parsing RSS feed ({len(response.text)} Bytes) ...")
rss = atoma.parse_rss_bytes(response.content)
- # DEBUG: print(f"DEBUG: rss[]={type(rss)}")
+ # DEBUG: print(f"DEBUG: rss[]='{type(rss)}'")
for item in rss.items:
# DEBUG: print(f"DEBUG: item={item}")
domain = item.link.split("=")[1]
# DEBUG: print(f"DEBUG: domains()={len(domains)}")
if len(domains) > 0:
- boot.acquire_lock()
+ locking.acquire()
print(f"INFO: Adding {len(domains)} new instances ...")
for domain in domains:
- print(f"INFO: Fetching instances from domain='{domain}' ...")
- fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+ try:
+ print(f"INFO: Fetching instances from domain='{domain}' ...")
+ federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+ instances.update_last_error(domain, exception)
# DEBUG: print("DEBUG: EXIT!")
def fetch_fbabot_atom(args: argparse.Namespace):
- # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+ # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
feed = "https://ryona.agency/users/fba/feed.atom"
domains = list()
# DEBUG: print(f"DEBUG: Parsing ATOM feed ({len(response.text)} Bytes) ...")
atom = atoma.parse_atom_bytes(response.content)
- # DEBUG: print(f"DEBUG: atom[]={type(atom)}")
+ # DEBUG: print(f"DEBUG: atom[]='{type(atom)}'")
for entry in atom.entries:
- # DEBUG: print(f"DEBUG: entry[]={type(entry)}")
+ # DEBUG: print(f"DEBUG: entry[]='{type(entry)}'")
doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
- # DEBUG: print(f"DEBUG: doc[]={type(doc)}")
+ # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'")
for element in doc.findAll("a"):
for href in element["href"].split(","):
# DEBUG: print(f"DEBUG: href[{type(href)}]={href}")
- domain = fba.tidyup_domain(href)
+ domain = tidyup.domain(href)
# DEBUG: print(f"DEBUG: domain='{domain}'")
if blacklist.is_blacklisted(domain):
# DEBUG: print(f"DEBUG: domains({len(domains)})={domains}")
if len(domains) > 0:
- boot.acquire_lock()
+ locking.acquire()
print(f"INFO: Adding {len(domains)} new instances ...")
for domain in domains:
- print(f"INFO: Fetching instances from domain='{domain}' ...")
- fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+ try:
+ print(f"INFO: Fetching instances from domain='{domain}' ...")
+ federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+ instances.update_last_error(domain, exception)
# DEBUG: print("DEBUG: EXIT!")
def fetch_instances(args: argparse.Namespace):
- # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
- boot.acquire_lock()
+ # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
+ locking.acquire()
# Initial fetch
- fba.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
+ try:
+ print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
+ federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'")
+ instances.update_last_error(args.domain, exception)
+ return
if args.single:
# DEBUG: print("DEBUG: Not fetching more instances - EXIT!")
print("WARNING: domain is blacklisted:", row[0])
continue
- print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
- fba.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
+ try:
+ print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
+ federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'")
+ instances.update_last_error(row[0], exception)
# DEBUG: print("DEBUG: EXIT!")
def fetch_federater(args: argparse.Namespace):
- # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
- boot.acquire_lock()
+ # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
+ locking.acquire()
# Fetch this URL
response = fba.fetch_url("https://github.com/federater/blocks_recommended/raw/main/federater.csv", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
## DEBUG: print(f"DEBUG: response.content={response.content}")
reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect='unix')
#, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate'
- # DEBUG: print(f"DEBUG: reader[]={type(reader)}")
+ # DEBUG: print(f"DEBUG: reader[]='{type(reader)}'")
for row in reader:
if not validators.domain(row["#domain"]):
print(f"WARNING: domain='{row['#domain']}' is not a valid domain - skipped!")
# DEBUG: print(f"DEBUG: domain='{row['#domain']}' is already registered - skipped!")
continue
- print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
- fba.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
+ try:
+ print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
+ federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
+ except network.exceptions as exception:
+ print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'")
+ instances.update_last_error(row["#domain"], exception)
# DEBUG: print("DEBUG: EXIT!")
projects / fba.git / blobdiff