1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
31 from fba import blacklist
32 from fba import blocks
33 from fba import config
34 from fba import federation
36 from fba import instances
37 from fba import locking
38 from fba import network
40 from fba.helpers import tidyup
42 from fba.networks import friendica
43 from fba.networks import mastodon
44 from fba.networks import misskey
45 from fba.networks import pleroma
47 def check_instance(args: argparse.Namespace) -> int:
48 # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!")
50 if not validators.domain(args.domain):
51 print(f"WARNING: args.domain='{args.domain}' is not valid")
53 elif blacklist.is_blacklisted(args.domain):
54 print(f"WARNING: args.domain='{args.domain}' is blacklisted")
56 elif instances.is_registered(args.domain):
57 print(f"WARNING: args.domain='{args.domain}' is already registered")
60 print(f"INFO: args.domain='{args.domain}' is not known")
62 # DEBUG: print(f"DEBUG: status={status} - EXIT!")
65 def fetch_bkali(args: argparse.Namespace):
66 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
69 fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
70 "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
73 # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'")
75 raise Exception("WARNING: Returned no records")
76 elif "data" not in fetched:
77 raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'")
78 elif "nodeinfo" not in fetched["data"]:
79 raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'")
81 for entry in fetched["data"]["nodeinfo"]:
82 # DEBUG: print(f"DEBUG: entry['{type(entry)}']='{entry}'")
83 if not "domain" in entry:
84 print(f"WARNING: entry()={len(entry)} does not contain 'domain' - SKIPPED!")
86 elif not validators.domain(entry["domain"]):
87 print(f"WARNING: domain='{entry['domain']}' is not a valid domain - SKIPPED!")
89 elif blacklist.is_blacklisted(entry["domain"]):
90 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is blacklisted - SKIPPED!")
92 elif instances.is_registered(entry["domain"]):
93 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is already registered - SKIPPED!")
96 # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
97 domains.append(entry["domain"])
99 except network.exceptions as exception:
100 print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'")
103 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
107 print(f"INFO: Adding {len(domains)} new instances ...")
108 for domain in domains:
110 print(f"INFO: Fetching instances from domain='{domain}' ...")
111 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
112 except network.exceptions as exception:
113 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
114 instances.update_last_error(domain, exception)
116 # DEBUG: print("DEBUG: EXIT!")
118 def fetch_blocks(args: argparse.Namespace):
119 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
120 if args.domain is not None and args.domain != "":
121 # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - checking ...")
122 if not validators.domain(args.domain):
123 print(f"WARNING: domain='{args.domain}' is not valid.")
125 elif blacklist.is_blacklisted(args.domain):
126 print(f"WARNING: domain='{args.domain}' is blacklisted, won't check it!")
128 elif not instances.is_registered(args.domain):
129 print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.")
134 if args.domain is not None and args.domain != "":
135 # Re-check single domain
137 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'bookwyrm', 'takahe') AND domain = ?", [args.domain]
140 # Re-check after "timeout" (aka. minimum interval)
142 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
145 rows = fba.cursor.fetchall()
146 print(f"INFO: Checking {len(rows)} entries ...")
147 for blocker, software, origin, nodeinfo_url in rows:
148 # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
150 blocker = tidyup.domain(blocker)
151 # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
154 print("WARNING: blocker is now empty!")
156 elif blacklist.is_blacklisted(blocker):
157 print(f"WARNING: blocker='{blocker}' is blacklisted now!")
160 # DEBUG: print(f"DEBUG: blocker='{blocker}'")
161 instances.update_last_blocked(blocker)
163 if software == "pleroma":
164 print(f"INFO: blocker='{blocker}',software='{software}'")
165 pleroma.fetch_blocks(blocker, origin, nodeinfo_url)
166 elif software == "mastodon":
167 print(f"INFO: blocker='{blocker}',software='{software}'")
168 mastodon.fetch_blocks(blocker, origin, nodeinfo_url)
169 elif software == "friendica" or software == "misskey":
170 print(f"INFO: blocker='{blocker}',software='{software}'")
173 if software == "friendica":
174 blocking = friendica.fetch_blocks(blocker)
175 elif software == "misskey":
176 blocking = misskey.fetch_blocks(blocker)
178 print(f"INFO: Checking {len(blocking.items())} entries from blocker='{blocker}',software='{software}' ...")
179 for block_level, blocklist in blocking.items():
180 # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist))
181 block_level = tidyup.domain(block_level)
182 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
183 if block_level == "":
184 print("WARNING: block_level is empty, blocker:", blocker)
187 # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
188 for block in blocklist:
189 blocked, reason = block.values()
190 # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!")
191 blocked = tidyup.domain(blocked)
192 reason = tidyup.reason(reason) if reason is not None and reason != "" else None
193 # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
196 print("WARNING: blocked is empty:", blocker)
198 elif blacklist.is_blacklisted(blocked):
199 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
201 elif blocked.count("*") > 0:
202 # Some friendica servers also obscure domains without hash
204 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
207 searchres = fba.cursor.fetchone()
209 print(f"DEBUG: searchres[]='{type(searchres)}'")
210 if searchres is None:
211 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
214 blocked = searchres[0]
215 origin = searchres[1]
216 nodeinfo_url = searchres[2]
217 elif blocked.count("?") > 0:
218 # Some obscure them with question marks, not sure if that's dependent on version or not
220 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
223 searchres = fba.cursor.fetchone()
225 print(f"DEBUG: searchres[]='{type(searchres)}'")
226 if searchres is None:
227 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
230 blocked = searchres[0]
231 origin = searchres[1]
232 nodeinfo_url = searchres[2]
233 elif not validators.domain(blocked):
234 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
237 # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
238 if not validators.domain(blocked):
239 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
241 elif not instances.is_registered(blocked):
242 # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
243 instances.add(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
245 if not blocks.is_instance_blocked(blocker, blocked, block_level):
246 blocks.add_instance(blocker, blocked, reason, block_level)
248 if block_level == "reject":
254 # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
255 blocks.update_last_seen(blocker, blocked, block_level)
256 blocks.update_reason(reason, blocker, blocked, block_level)
258 # DEBUG: print("DEBUG: Committing changes ...")
259 fba.connection.commit()
261 print("WARNING: Unknown software:", blocker, software)
263 if config.get("bot_enabled") and len(blockdict) > 0:
264 network.send_bot_post(blocker, blockdict)
266 # DEBUG: print("DEBUG: EXIT!")
268 def fetch_cs(args: argparse.Namespace):
269 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
295 raw = fba.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
296 # DEBUG: print(f"DEBUG: raw()={len(raw)}[]={type(raw)}")
298 doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser')
300 # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}")
301 silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody")
302 # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}")
303 domains["silenced"] = domains["silenced"] + federation.find_domains(silenced)
305 blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody")
306 # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}")
307 domains["reject"] = domains["reject"] + federation.find_domains(blocked)
309 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
313 print(f"INFO: Adding {len(domains)} new instances ...")
314 for block_level in domains:
315 # DEBUG: print(f"DEBUG: block_level='{block_level}'")
317 for row in domains[block_level]:
318 # DEBUG: print(f"DEBUG: row='{row}'")
319 if not blocks.is_instance_blocked('chaos.social', row["domain"], block_level):
320 # DEBUG: print(f"DEBUG: domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...")
321 blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level)
323 if not instances.is_registered(row["domain"]):
325 print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
326 federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
327 except network.exceptions as exception:
328 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'")
329 instances.update_last_error(row["domain"], exception)
331 # DEBUG: print("DEBUG: Committing changes ...")
332 fba.connection.commit()
334 # DEBUG: print("DEBUG: EXIT!")
336 def fetch_fba_rss(args: argparse.Namespace):
337 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
340 print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...")
341 response = fba.fetch_url(args.feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
343 # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
344 if response.ok and response.status_code < 300 and len(response.text) > 0:
345 # DEBUG: print(f"DEBUG: Parsing RSS feed ({len(response.text)} Bytes) ...")
346 rss = atoma.parse_rss_bytes(response.content)
348 # DEBUG: print(f"DEBUG: rss[]={type(rss)}")
349 for item in rss.items:
350 # DEBUG: print(f"DEBUG: item={item}")
351 domain = item.link.split("=")[1]
353 if blacklist.is_blacklisted(domain):
354 # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
356 elif domain in domains:
357 # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
359 elif instances.is_registered(domain):
360 # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
363 # DEBUG: print(f"DEBUG: Adding domain='{domain}'")
364 domains.append(domain)
366 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
370 print(f"INFO: Adding {len(domains)} new instances ...")
371 for domain in domains:
373 print(f"INFO: Fetching instances from domain='{domain}' ...")
374 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
375 except network.exceptions as exception:
376 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
377 instances.update_last_error(domain, exception)
379 # DEBUG: print("DEBUG: EXIT!")
381 def fetch_fbabot_atom(args: argparse.Namespace):
382 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
383 feed = "https://ryona.agency/users/fba/feed.atom"
387 print(f"INFO: Fetching ATOM feed='{feed}' from FBA bot account ...")
388 response = fba.fetch_url(feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
390 # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
391 if response.ok and response.status_code < 300 and len(response.text) > 0:
392 # DEBUG: print(f"DEBUG: Parsing ATOM feed ({len(response.text)} Bytes) ...")
393 atom = atoma.parse_atom_bytes(response.content)
395 # DEBUG: print(f"DEBUG: atom[]={type(atom)}")
396 for entry in atom.entries:
397 # DEBUG: print(f"DEBUG: entry[]={type(entry)}")
398 doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
399 # DEBUG: print(f"DEBUG: doc[]={type(doc)}")
400 for element in doc.findAll("a"):
401 for href in element["href"].split(","):
402 # DEBUG: print(f"DEBUG: href[{type(href)}]={href}")
403 domain = tidyup.domain(href)
405 # DEBUG: print(f"DEBUG: domain='{domain}'")
406 if blacklist.is_blacklisted(domain):
407 # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
409 elif domain in domains:
410 # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
412 elif instances.is_registered(domain):
413 # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
416 # DEBUG: print(f"DEBUG: Adding domain='{domain}',domains()={len(domains)}")
417 domains.append(domain)
419 # DEBUG: print(f"DEBUG: domains({len(domains)})={domains}")
423 print(f"INFO: Adding {len(domains)} new instances ...")
424 for domain in domains:
426 print(f"INFO: Fetching instances from domain='{domain}' ...")
427 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
428 except network.exceptions as exception:
429 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
430 instances.update_last_error(domain, exception)
432 # DEBUG: print("DEBUG: EXIT!")
434 def fetch_instances(args: argparse.Namespace):
435 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
440 print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
441 federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
442 except network.exceptions as exception:
443 print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'")
444 instances.update_last_error(args.domain, exception)
448 # DEBUG: print("DEBUG: Not fetching more instances - EXIT!")
451 # Loop through some instances
453 "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
456 rows = fba.cursor.fetchall()
457 print(f"INFO: Checking {len(rows)} entries ...")
459 # DEBUG: print(f"DEBUG: domain='{row[0]}'")
460 if blacklist.is_blacklisted(row[0]):
461 print("WARNING: domain is blacklisted:", row[0])
465 print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
466 federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
467 except network.exceptions as exception:
468 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'")
469 instances.update_last_error(row[0], exception)
471 # DEBUG: print("DEBUG: EXIT!")
473 def fetch_federater(args: argparse.Namespace):
474 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
478 response = fba.fetch_url("https://github.com/federater/blocks_recommended/raw/main/federater.csv", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
479 # DEBUG: print(f"DEBUG: response[]='{type(response)}'")
480 if response.ok and response.content != "":
481 # DEBUG: print(f"DEBUG: Fetched {len(response.content)} Bytes, parsing CSV ...")
482 ## DEBUG: print(f"DEBUG: response.content={response.content}")
483 reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect='unix')
484 #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate'
485 # DEBUG: print(f"DEBUG: reader[]={type(reader)}")
487 if not validators.domain(row["#domain"]):
488 print(f"WARNING: domain='{row['#domain']}' is not a valid domain - skipped!")
490 elif blacklist.is_blacklisted(row["#domain"]):
491 print(f"WARNING: domain='{row['#domain']}' is blacklisted - skipped!")
493 elif instances.is_registered(row["#domain"]):
494 # DEBUG: print(f"DEBUG: domain='{row['#domain']}' is already registered - skipped!")
498 print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
499 federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
500 except network.exceptions as exception:
501 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'")
502 instances.update_last_error(row["#domain"], exception)
504 # DEBUG: print("DEBUG: EXIT!")