1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
29 from fba import blacklist
30 from fba import blocks
32 from fba import config
34 from fba import instances
36 from fba.federation import *
38 def check_instance(args: argparse.Namespace) -> int:
39 # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!")
41 if not validators.domain(args.domain):
42 print(f"WARNING: args.domain='{args.domain}' is not valid")
44 elif blacklist.is_blacklisted(args.domain):
45 print(f"WARNING: args.domain='{args.domain}' is blacklisted")
47 elif fba.is_instance_registered(args.domain):
48 print(f"WARNING: args.domain='{args.domain}' is already registered")
51 print(f"INFO: args.domain='{args.domain}' is not known")
53 # DEBUG: print(f"DEBUG: status={status} - EXIT!")
56 def fetch_bkali(args: argparse.Namespace):
57 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
60 fetched = fba.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
61 "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
64 # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'")
66 raise Exception("WARNING: Returned no records")
67 elif not "data" in fetched:
68 raise Exception(f"WARNING: fetched()={len(fetched)} does not contain key 'data'")
69 elif not "nodeinfo" in fetched["data"]:
70 raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain key 'nodeinfo'")
72 for entry in fetched["data"]["nodeinfo"]:
73 # DEBUG: print(f"DEBUG: entry['{type(entry)}']='{entry}'")
74 if not "domain" in entry:
75 print(f"WARNING: entry does not contain 'domain' - SKIPPED!")
77 elif not validators.domain(entry["domain"]):
78 print(f"WARNING: domain='{entry['domain']}' is not a valid domain - SKIPPED!")
80 elif blacklist.is_blacklisted(entry["domain"]):
81 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is blacklisted - SKIPPED!")
83 elif fba.is_instance_registered(entry["domain"]):
84 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is already registered - SKIPPED!")
87 # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
88 domains.append(entry["domain"])
90 except BaseException as e:
91 print(f"ERROR: Cannot fetch graphql,exception[{type(e)}]:'{str(e)}'")
94 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
98 print(f"INFO: Adding {len(domains)} new instances ...")
99 for domain in domains:
100 print(f"INFO: Fetching instances from domain='{domain}' ...")
101 fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
103 # DEBUG: print("DEBUG: EXIT!")
105 def fetch_blocks(args: argparse.Namespace):
106 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
107 if args.domain != None and args.domain != "":
108 if not validators.domain(args.domain):
109 print(f"WARNING: domain='{args.domain}' is not valid.")
111 elif blacklist.is_blacklisted(args.domain):
112 print(f"WARNING: domain='{args.domain}' is blacklisted, won't check it!")
114 elif not fba.is_instance_registered(args.domain):
115 print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.")
120 if args.domain != None and args.domain != "":
122 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND domain = ?", [args.domain]
126 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
129 rows = fba.cursor.fetchall()
130 print(f"INFO: Checking {len(rows)} entries ...")
131 for blocker, software, origin, nodeinfo_url in rows:
132 # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
134 blocker = fba.tidyup_domain(blocker)
135 # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
138 print("WARNING: blocker is now empty!")
140 elif blacklist.is_blacklisted(blocker):
141 print(f"WARNING: blocker='{blocker}' is blacklisted now!")
144 # DEBUG: print(f"DEBUG: blocker='{blocker}'")
145 instances.update_last_blocked(blocker)
147 if software == "pleroma":
148 print(f"INFO: blocker='{blocker}',software='{software}'")
149 pleroma.fetch_blocks(blocker, origin, nodeinfo_url)
150 elif software == "mastodon":
151 print(f"INFO: blocker='{blocker}',software='{software}'")
152 mastodon.fetch_blocks(blocker, origin, nodeinfo_url)
153 elif software == "friendica" or software == "misskey":
154 print(f"INFO: blocker='{blocker}',software='{software}'")
156 if software == "friendica":
157 json = fba.fetch_friendica_blocks(blocker)
158 elif software == "misskey":
159 json = fba.fetch_misskey_blocks(blocker)
161 print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
162 for block_level, blocklist in json.items():
163 # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist))
164 block_level = fba.tidyup_domain(block_level)
165 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
166 if block_level == "":
167 print("WARNING: block_level is empty, blocker:", blocker)
170 # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
171 for block in blocklist:
172 blocked, reason = block.values()
173 # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
174 blocked = fba.tidyup_domain(blocked)
175 # DEBUG: print("DEBUG: AFTER blocked:", blocked)
178 print("WARNING: blocked is empty:", blocker)
180 elif blacklist.is_blacklisted(blocked):
181 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
183 elif blocked.count("*") > 0:
184 # Some friendica servers also obscure domains without hash
186 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
189 searchres = fba.cursor.fetchone()
191 if searchres == None:
192 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
195 blocked = searchres[0]
196 origin = searchres[1]
197 nodeinfo_url = searchres[2]
198 elif blocked.count("?") > 0:
199 # Some obscure them with question marks, not sure if that's dependent on version or not
201 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
204 searchres = fba.cursor.fetchone()
206 if searchres == None:
207 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
210 blocked = searchres[0]
211 origin = searchres[1]
212 nodeinfo_url = searchres[2]
213 elif not validators.domain(blocked):
214 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
217 # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
218 if not validators.domain(blocked):
219 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
221 elif not fba.is_instance_registered(blocked):
222 # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
223 fba.add_instance(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
225 if not blocks.is_instance_blocked(blocker, blocked, block_level):
226 blocks.add_instance(blocker, blocked, reason, block_level)
228 if block_level == "reject":
234 # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
235 blocks.update_last_seen(blocker, blocked, block_level)
236 blocks.update_reason(reason, blocker, blocked, block_level)
238 # DEBUG: print("DEBUG: Committing changes ...")
239 fba.connection.commit()
240 except Exception as e:
241 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
242 elif software == "gotosocial":
243 print(f"INFO: blocker='{blocker}',software='{software}'")
244 gotosocial.fetch_blocks(blocker, origin, nodeinfo_url)
246 print("WARNING: Unknown software:", blocker, software)
248 if config.get("bot_enabled") and len(blockdict) > 0:
249 send_bot_post(blocker, blockdict)
253 # DEBUG: print("DEBUG: EXIT!")
255 def fetch_cs(args: argparse.Namespace):
256 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
263 doc = bs4.BeautifulSoup(
264 fba.get_response("meta.chaos.social", "/federation", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
267 # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}")
268 silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table")
270 # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}")
271 domains["silenced"] = domains["silenced"] + fba.find_domains(silenced)
272 blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table")
274 # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}")
275 domains["blocked"] = domains["blocked"] + fba.find_domains(blocked)
277 except BaseException as e:
278 print(f"ERROR: Cannot fetch from meta.chaos.social,exception[{type(e)}]:'{str(e)}'")
281 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
285 print(f"INFO: Adding {len(domains)} new instances ...")
286 for block_level in domains:
287 # DEBUG: print(f"DEBUG: block_level='{block_level}'")
289 for row in domains[block_level]:
290 # DEBUG: print(f"DEBUG: row='{row}'")
291 if not fba.is_instance_registered(row["domain"]):
292 print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
293 fba.fetch_instances(row["domain"], None, None, inspect.currentframe().f_code.co_name)
295 if not blocks.is_instance_blocked('chaos.social', row["domain"], block_level):
296 # DEBUG: print(f"DEBUG: domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...")
297 blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level)
299 # DEBUG: print("DEBUG: Committing changes ...")
300 fba.connection.commit()
302 # DEBUG: print("DEBUG: EXIT!")
304 def fetch_fba_rss(args: argparse.Namespace):
305 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
309 print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...")
310 response = fba.get_url(args.feed, fba.headers, (config.get("connection_timeout"), config.get("read_timeout")))
312 # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
313 if response.ok and response.status_code < 300 and len(response.text) > 0:
314 # DEBUG: print(f"DEBUG: Parsing RSS feed ...")
315 rss = atoma.parse_rss_bytes(response.content)
317 # DEBUG: print(f"DEBUG: rss[]={type(rss)}")
318 for item in rss.items:
319 # DEBUG: print(f"DEBUG: item={item}")
320 domain = item.link.split("=")[1]
322 if blacklist.is_blacklisted(domain):
323 # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
325 elif domain in domains:
326 # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
328 elif fba.is_instance_registered(domain):
329 # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
332 # DEBUG: print(f"DEBUG: Adding domain='{domain}'")
333 domains.append(domain)
335 except BaseException as e:
336 print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
339 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
343 print(f"INFO: Adding {len(domains)} new instances ...")
344 for domain in domains:
345 print(f"INFO: Fetching instances from domain='{domain}' ...")
346 fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
348 # DEBUG: print("DEBUG: EXIT!")
350 def fetch_fbabot_atom(args: argparse.Namespace):
351 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
352 feed = "https://ryona.agency/users/fba/feed.atom"
356 print(f"INFO: Fetching ATOM feed='{feed}' from FBA bot account ...")
357 response = fba.get_url(feed, fba.headers, (config.get("connection_timeout"), config.get("read_timeout")))
359 # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
360 if response.ok and response.status_code < 300 and len(response.text) > 0:
361 # DEBUG: print(f"DEBUG: Parsing ATOM feed ...")
362 atom = atoma.parse_atom_bytes(response.content)
364 # DEBUG: print(f"DEBUG: atom[]={type(atom)}")
365 for entry in atom.entries:
366 # DEBUG: print(f"DEBUG: entry[]={type(entry)}")
367 doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
368 # DEBUG: print(f"DEBUG: doc[]={type(doc)}")
369 for element in doc.findAll("a"):
370 for href in element["href"].split(","):
371 # DEBUG: print(f"DEBUG: href[{type(href)}]={href}")
372 domain = fba.tidyup_domain(href)
374 # DEBUG: print(f"DEBUG: domain='{domain}'")
375 if blacklist.is_blacklisted(domain):
376 # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
378 elif domain in domains:
379 # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
381 elif fba.is_instance_registered(domain):
382 # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
385 # DEBUG: print(f"DEBUG: Adding domain='{domain}',domains()={len(domains)}")
386 domains.append(domain)
388 except BaseException as e:
389 print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
392 # DEBUG: print(f"DEBUG: domains({len(domains)})={domains}")
396 print(f"INFO: Adding {len(domains)} new instances ...")
397 for domain in domains:
398 print(f"INFO: Fetching instances from domain='{domain}' ...")
399 fba.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
401 # DEBUG: print("DEBUG: EXIT!")
403 def fetch_instances(args: argparse.Namespace):
404 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
408 fba.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
411 # DEBUG: print(f"DEBUG: Not fetching more instances - EXIT!")
414 # Loop through some instances
416 "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
419 rows = fba.cursor.fetchall()
420 print(f"INFO: Checking {len(rows)} entries ...")
422 # DEBUG: print("DEBUG: domain:", row[0])
423 if blacklist.is_blacklisted(row[0]):
424 print("WARNING: domain is blacklisted:", row[0])
427 print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
428 fba.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
430 # DEBUG: print("DEBUG: EXIT!")