1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
29 from fba import config
32 def check_instance(args: argparse.Namespace) -> int:
33 # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!")
35 if not validators.domain(args.domain):
36 print(f"WARNING: args.domain='{args.domain}' is not valid")
38 elif fba.is_blacklisted(args.domain):
39 print(f"WARNING: args.domain='{args.domain}' is blacklisted")
41 elif fba.is_instance_registered(args.domain):
42 print(f"WARNING: args.domain='{args.domain}' is already registered")
45 print(f"INFO: args.domain='{args.domain}' is not known")
47 # DEBUG: print(f"DEBUG: status={status} - EXIT!")
50 def fetch_bkali(args: argparse.Namespace):
51 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
54 fetched = fba.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
55 "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
58 # DEBUG: print(f"DEBUG: fetched({len(fetched)})[]='{type(fetched)}'")
60 raise Exception("WARNING: Returned no records")
61 elif not "data" in fetched:
62 raise Exception(f"WARNING: fetched()={len(fetched)} does not contain element 'data'")
63 elif not "nodeinfo" in fetched["data"]:
64 raise Exception(f"WARNING: fetched()={len(fetched['data'])} does not contain element 'nodeinfo'")
66 for entry in fetched["data"]["nodeinfo"]:
67 # DEBUG: print(f"DEBUG: entry['{type(entry)}']='{entry}'")
68 if not "domain" in entry:
69 print(f"WARNING: entry does not contain 'domain' - SKIPPED!")
71 elif not validators.domain(entry["domain"]):
72 print(f"WARNING: domain='{entry['domain']}' is not a valid domain - SKIPPED!")
74 elif fba.is_blacklisted(entry["domain"]):
75 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is blacklisted - SKIPPED!")
77 elif fba.is_instance_registered(entry["domain"]):
78 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is already registered - SKIPPED!")
81 # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
82 domains.append(entry["domain"])
84 except BaseException as e:
85 print(f"ERROR: Cannot fetch graphql,exception[{type(e)}]:'{str(e)}'")
88 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
92 print(f"INFO: Adding {len(domains)} new instances ...")
93 for domain in domains:
94 print(f"INFO: Fetching instances from domain='{domain}' ...")
95 fba.fetch_instances(domain, None, None, sys.argv[0])
97 # DEBUG: print("DEBUG: EXIT!")
99 def fetch_blocks(args: argparse.Namespace):
100 print(f"DEBUG: args[]={type(args)} - CALLED!")
101 if args.domain != None and args.domain != "":
102 if not validators.domain(args.domain):
103 print(f"WARNING: domain='{args.domain}' is not valid.")
105 elif fba.is_blacklisted(args.domain):
106 print(f"WARNING: domain='{args.domain}' is blacklisted, won't check it!")
108 elif not fba.is_instance_registered(args.domain):
109 print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.")
114 if args.domain != None and args.domain != "":
116 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND domain = ?", [args.domain]
120 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
123 rows = fba.cursor.fetchall()
124 print(f"INFO: Checking {len(rows)} entries ...")
125 for blocker, software, origin, nodeinfo_url in rows:
126 # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
128 blocker = fba.tidyup_domain(blocker)
129 # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
132 print("WARNING: blocker is now empty!")
134 elif fba.is_blacklisted(blocker):
135 print(f"WARNING: blocker='{blocker}' is blacklisted now!")
138 # DEBUG: print(f"DEBUG: blocker='{blocker}'")
139 fba.update_last_blocked(blocker)
141 if software == "pleroma":
142 print("INFO: blocker:", blocker)
145 json = fba.fetch_nodeinfo(blocker, nodeinfo_url)
147 print("WARNING: Could not fetch nodeinfo from blocker:", blocker)
149 elif not "metadata" in json:
150 print(f"WARNING: json()={len(json)} does not have key 'metadata', blocker='{blocker}'")
152 elif not "federation" in json["metadata"]:
153 print(f"WARNING: json()={len(json['metadata'])} does not have key 'federation', blocker='{blocker}'")
156 # DEBUG: print("DEBUG: Updating nodeinfo:", blocker)
157 fba.update_last_nodeinfo(blocker)
159 federation = json["metadata"]["federation"]
161 if "enabled" in federation:
162 # DEBUG: print("DEBUG: Instance has no block list to analyze:", blocker)
165 if "mrf_simple" in federation:
166 for block_level, blocks in (
167 {**federation["mrf_simple"],
168 **{"quarantined_instances": federation["quarantined_instances"]}}
170 # DEBUG: print("DEBUG: block_level, blocks():", block_level, len(blocks))
171 block_level = fba.tidyup_domain(block_level)
172 # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
174 if block_level == "":
175 print("WARNING: block_level is now empty!")
178 # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
179 for blocked in blocks:
180 # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
181 blocked = fba.tidyup_domain(blocked)
182 # DEBUG: print("DEBUG: AFTER blocked:", blocked)
185 print("WARNING: blocked is empty after fba.tidyup_domain():", blocker, block_level)
187 elif fba.is_blacklisted(blocked):
188 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
190 elif blocked.count("*") > 1:
191 # -ACK!-oma also started obscuring domains without hash
193 "SELECT domain, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
195 searchres = fba.cursor.fetchone()
196 # DEBUG: print("DEBUG: searchres[]:", type(searchres))
198 if searchres == None:
199 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
202 blocked = searchres[0]
203 nodeinfo_url = searchres[1]
204 # DEBUG: print("DEBUG: Looked up domain:", blocked)
205 elif not validators.domain(blocked):
206 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
209 # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
210 if not validators.domain(blocked):
211 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
213 elif not fba.is_instance_registered(blocked):
214 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
215 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
217 if not fba.is_instance_blocked(blocker, blocked, block_level):
218 # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
219 fba.block_instance(blocker, blocked, "unknown", block_level)
221 if block_level == "reject":
222 # DEBUG: print("DEBUG: Adding to blockdict:", blocked)
229 # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
230 fba.update_last_seen(blocker, blocked, block_level)
232 # DEBUG: print("DEBUG: Committing changes ...")
233 fba.connection.commit()
236 if "mrf_simple_info" in federation:
237 # DEBUG: print("DEBUG: Found mrf_simple_info:", blocker)
238 for block_level, info in (
239 {**federation["mrf_simple_info"],
240 **(federation["quarantined_instances_info"]
241 if "quarantined_instances_info" in federation
244 # DEBUG: print("DEBUG: block_level, info.items():", block_level, len(info.items()))
245 block_level = fba.tidyup_domain(block_level)
246 # DEBUG: print("DEBUG: BEFORE block_level:", block_level)
248 if block_level == "":
249 print("WARNING: block_level is now empty!")
252 # DEBUG: print(f"DEBUG: Checking {len(info.items())} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
253 for blocked, reason in info.items():
254 # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
255 blocked = fba.tidyup_domain(blocked)
256 # DEBUG: print("DEBUG: AFTER blocked:", blocked)
259 print("WARNING: blocked is empty after fba.tidyup_domain():", blocker, block_level)
261 elif fba.is_blacklisted(blocked):
262 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
264 elif blocked.count("*") > 1:
265 # same domain guess as above, but for reasons field
267 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
269 searchres = fba.cursor.fetchone()
271 if searchres == None:
272 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
275 blocked = searchres[0]
276 origin = searchres[1]
277 nodeinfo_url = searchres[2]
278 elif not validators.domain(blocked):
279 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
282 # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
283 if not validators.domain(blocked):
284 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
286 elif not fba.is_instance_registered(blocked):
287 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
288 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
290 # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, reason["reason"])
291 fba.update_block_reason(reason["reason"], blocker, blocked, block_level)
293 for entry in blockdict:
294 if entry["blocked"] == blocked:
295 # DEBUG: print("DEBUG: Updating entry reason:", blocked)
296 entry["reason"] = reason["reason"]
298 fba.connection.commit()
299 except Exception as e:
300 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
301 elif software == "mastodon":
302 print("INFO: blocker:", blocker)
304 # json endpoint for newer mastodongs
308 "media_removal" : [],
309 "followers_only": [],
313 # handling CSRF, I've saw at least one server requiring it to access the endpoint
314 # DEBUG: print("DEBUG: Fetching meta:", blocker)
315 meta = bs4.BeautifulSoup(
316 fba.get_response(blocker, "/", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
320 csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
321 # DEBUG: print("DEBUG: Adding CSRF token:", blocker, csrf)
322 reqheaders = {**fba.api_headers, **{"X-CSRF-Token": csrf}}
323 except BaseException as e:
324 # DEBUG: print("DEBUG: No CSRF token found, using normal headers:", blocker, e)
325 reqheaders = fba.api_headers
327 # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker)
328 blocks = fba.get_response(blocker, "/api/v1/instance/domain_blocks", reqheaders, (config.get("connection_timeout"), config.get("read_timeout"))).json()
330 print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...")
333 'domain': block['domain'],
334 'hash' : block['digest'],
335 'reason': block['comment']
338 # DEBUG: print("DEBUG: severity,domain,hash,comment:", block['severity'], block['domain'], block['digest'], block['comment'])
339 if block['severity'] == 'suspend':
340 # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
341 json['reject'].append(entry)
342 elif block['severity'] == 'silence':
343 # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
344 json['followers_only'].append(entry)
345 elif block['severity'] == 'reject_media':
346 # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
347 json['media_removal'].append(entry)
348 elif block['severity'] == 'reject_reports':
349 # DEBUG: print(f"DEBUG: Adding entry='{entry}' with severity='{block['severity']}' ...")
350 json['report_removal'].append(entry)
352 print("WARNING: Unknown severity:", block['severity'], block['domain'])
353 except BaseException as e:
354 # DEBUG: print(f"DEBUG: Failed, trying mastodon-specific fetches: blocker='{blocker}',exception[{type(e)}]={str(e)}")
355 json = fba.get_mastodon_blocks(blocker)
357 print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
358 for block_level, blocks in json.items():
359 # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
360 block_level = fba.tidyup_domain(block_level)
361 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
362 if block_level == "":
363 print("WARNING: block_level is empty, blocker:", blocker)
366 # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
368 blocked, blocked_hash, reason = block.values()
369 # DEBUG: print("DEBUG: blocked,hash,reason:", blocked, blocked_hash, reason)
370 blocked = fba.tidyup_domain(blocked)
371 # DEBUG: print("DEBUG: AFTER-blocked:", blocked)
374 print("WARNING: blocked is empty:", blocker)
376 elif fba.is_blacklisted(blocked):
377 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
379 elif blocked.count("*") > 0:
380 # Doing the hash search for instance names as well to tidy up DB
382 "SELECT domain, origin, nodeinfo_url FROM instances WHERE hash = ? LIMIT 1", [blocked_hash]
384 searchres = fba.cursor.fetchone()
386 if searchres == None:
387 print(f"WARNING: Cannot deobsfucate blocked='{blocked}',blocked_hash='{blocked_hash}' - SKIPPED!")
390 # DEBUG: print("DEBUG: Updating domain: ", searchres[0])
391 blocked = searchres[0]
392 origin = searchres[1]
393 nodeinfo_url = searchres[2]
395 # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
396 if not validators.domain(blocked):
397 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
399 elif not fba.is_instance_registered(blocked):
400 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
401 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
402 elif not validators.domain(blocked):
403 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
406 # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
407 if not validators.domain(blocked):
408 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
410 elif not fba.is_instance_registered(blocked):
411 # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
412 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
414 blocking = blocked if blocked.count("*") <= 1 else blocked_hash
415 # DEBUG: print(f"DEBUG: blocking='{blocking}',blocked='{blocked}',blocked_hash='{blocked_hash}'")
417 if not fba.is_instance_blocked(blocker, blocked, block_level):
418 # DEBUG: print("DEBUG: Blocking:", blocker, blocked, block_level)
419 fba.block_instance(blocker, blocking, reason, block_level)
421 if block_level == "reject":
427 # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocking='{blocking}' ...")
428 fba.update_last_seen(blocker, blocking, block_level)
429 fba.update_block_reason(reason, blocker, blocking, block_level)
431 # DEBUG: print("DEBUG: Committing changes ...")
432 fba.connection.commit()
433 except Exception as e:
434 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
435 elif software == "friendica" or software == "misskey" or software == "bookwyrm" or software == "takahe":
436 print("INFO: blocker:", blocker)
438 if software == "friendica":
439 json = fba.get_friendica_blocks(blocker)
440 elif software == "misskey":
441 json = fba.get_misskey_blocks(blocker)
442 elif software == "bookwyrm":
443 print("WARNING: bookwyrm is not fully supported for fetching blacklist!", blocker)
444 #json = fba.get_bookwyrm_blocks(blocker)
446 elif software == "takahe":
447 print("WARNING: takahe is not fully supported for fetching blacklist!", blocker)
448 #json = fba.get_takahe_blocks(blocker)
451 print(f"INFO: Checking {len(json.items())} entries from blocker='{blocker}',software='{software}' ...")
452 for block_level, blocks in json.items():
453 # DEBUG: print("DEBUG: blocker,block_level,blocks():", blocker, block_level, len(blocks))
454 block_level = fba.tidyup_domain(block_level)
455 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
456 if block_level == "":
457 print("WARNING: block_level is empty, blocker:", blocker)
460 # DEBUG: print(f"DEBUG: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
462 blocked, reason = block.values()
463 # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
464 blocked = fba.tidyup_domain(blocked)
465 # DEBUG: print("DEBUG: AFTER blocked:", blocked)
468 print("WARNING: blocked is empty:", blocker)
470 elif fba.is_blacklisted(blocked):
471 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
473 elif blocked.count("*") > 0:
474 # Some friendica servers also obscure domains without hash
476 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
479 searchres = fba.cursor.fetchone()
481 if searchres == None:
482 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
485 blocked = searchres[0]
486 origin = searchres[1]
487 nodeinfo_url = searchres[2]
488 elif blocked.count("?") > 0:
489 # Some obscure them with question marks, not sure if that's dependent on version or not
491 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
494 searchres = fba.cursor.fetchone()
496 if searchres == None:
497 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
500 blocked = searchres[0]
501 origin = searchres[1]
502 nodeinfo_url = searchres[2]
503 elif not validators.domain(blocked):
504 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
507 # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
508 if not validators.domain(blocked):
509 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
511 elif not fba.is_instance_registered(blocked):
512 # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
513 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
515 if not fba.is_instance_blocked(blocker, blocked, block_level):
516 fba.block_instance(blocker, blocked, reason, block_level)
518 if block_level == "reject":
524 # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
525 fba.update_last_seen(blocker, blocked, block_level)
526 fba.update_block_reason(reason, blocker, blocked, block_level)
528 # DEBUG: print("DEBUG: Committing changes ...")
529 fba.connection.commit()
530 except Exception as e:
531 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
532 elif software == "gotosocial":
533 print("INFO: blocker:", blocker)
536 federation = fba.get_response(blocker, "{fba.get_peers_url}?filter=suspended", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))).json()
538 if (federation == None):
539 print("WARNING: No valid response:", blocker);
540 elif "error" in federation:
541 print("WARNING: API returned error:", federation["error"])
543 print(f"INFO: Checking {len(federation)} entries from blocker='{blocker}',software='{software}' ...")
544 for peer in federation:
545 blocked = peer["domain"].lower()
546 # DEBUG: print("DEBUG: BEFORE blocked:", blocked)
547 blocked = fba.tidyup_domain(blocked)
548 # DEBUG: print("DEBUG: AFTER blocked:", blocked)
551 print("WARNING: blocked is empty:", blocker)
553 elif fba.is_blacklisted(blocked):
554 # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
556 elif blocked.count("*") > 0:
557 # GTS does not have hashes for obscured domains, so we have to guess it
559 "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
561 searchres = fba.cursor.fetchone()
563 if searchres == None:
564 print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
567 blocked = searchres[0]
568 origin = searchres[1]
569 nodeinfo_url = searchres[2]
570 elif not validators.domain(blocked):
571 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
574 # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
575 if not validators.domain(blocked):
576 print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
578 elif not fba.is_instance_registered(blocked):
579 # DEBUG: print(f"DEBUG: Domain blocked='{blocked}' wasn't found, adding ..., blocker='{blocker}',origin='{origin}',nodeinfo_url='{nodeinfo_url}'")
580 fba.add_instance(blocked, blocker, sys.argv[0], nodeinfo_url)
582 if not fba.is_instance_blocked(blocker, blocked, "reject"):
583 # DEBUG: print(f"DEBUG: blocker='{blocker}' is blocking '{blocked}' for unknown reason at this point")
584 fba.block_instance(blocker, blocked, "unknown", "reject")
591 # DEBUG: print(f"DEBUG: Updating block last seen for blocker='{blocker}',blocked='{blocked}' ...")
592 fba.update_last_seen(blocker, blocked, "reject")
594 if "public_comment" in peer:
595 # DEBUG: print("DEBUG: Updating block reason:", blocker, blocked, peer["public_comment"])
596 fba.update_block_reason(peer["public_comment"], blocker, blocked, "reject")
598 for entry in blockdict:
599 if entry["blocked"] == blocked:
600 # DEBUG: print(f"DEBUG: Setting block reason for blocked='{blocked}':'{peer['public_comment']}'")
601 entry["reason"] = peer["public_comment"]
603 # DEBUG: print("DEBUG: Committing changes ...")
604 fba.connection.commit()
605 except Exception as e:
606 print(f"ERROR: blocker='{blocker}',software='{software}',exception[{type(e)}]:'{str(e)}'")
608 print("WARNING: Unknown software:", blocker, software)
610 if config.get("bot_enabled") and len(blockdict) > 0:
611 send_bot_post(blocker, blockdict)
615 # DEBUG: print("DEBUG: EXIT!")
617 def fetch_cs(args: argparse.Namespace):
618 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
625 doc = bs4.BeautifulSoup(
626 reqto.get("https://meta.chaos.social/federation", headers=fba.headers, timeout=(config.get("connection_timeout"), config.get("read_timeout"))).text,
629 # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}")
630 silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table")
632 # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}")
633 domains["silenced"] = domains["silenced"] + find_domains(silenced)
634 blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table")
636 # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}")
637 domains["blocked"] = domains["blocked"] + find_domains(blocked)
639 except BaseException as e:
640 print(f"ERROR: Cannot fetch from meta.chaos.social,exception[{type(e)}]:'{str(e)}'")
643 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
647 print(f"INFO: Adding {len(domains)} new instances ...")
648 for block_level in domains:
649 # DEBUG: print(f"DEBUG: block_level='{block_level}'")
651 for row in domains[block_level]:
652 # DEBUG: print(f"DEBUG: row='{row}'")
653 if not fba.is_instance_registered(row["domain"]):
654 print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
655 fba.fetch_instances(row["domain"], None, None, sys.argv[0])
657 if not fba.is_instance_blocked('chaos.social', row["domain"], block_level):
658 # DEBUG: print(f"DEBUG: domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...")
659 fba.block_instance('chaos.social', row["domain"], row["reason"], block_level)
661 # DEBUG: print("DEBUG: Committing changes ...")
662 fba.connection.commit()
664 # DEBUG: print("DEBUG: EXIT!")
666 def fetch_fba_rss(args: argparse.Namespace):
667 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
671 print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...")
672 response = reqto.get(args.feed, headers=fba.headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
674 # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
675 if response.ok and response.status_code < 300 and len(response.text) > 0:
676 # DEBUG: print(f"DEBUG: Parsing RSS feed ...")
677 rss = atoma.parse_rss_bytes(response.content)
679 # DEBUG: print(f"DEBUG: rss[]={type(rss)}")
680 for item in rss.items:
681 # DEBUG: print(f"DEBUG: item={item}")
682 domain = item.link.split("=")[1]
684 if fba.is_blacklisted(domain):
685 # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
687 elif domain in domains:
688 # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
690 elif fba.is_instance_registered(domain):
691 # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
694 # DEBUG: print(f"DEBUG: domain='{domain}'")
695 domains.append(domain)
697 except BaseException as e:
698 print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
701 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
705 print(f"INFO: Adding {len(domains)} new instances ...")
706 for domain in domains:
707 print(f"INFO: Fetching instances from domain='{domain}' ...")
708 fba.fetch_instances(domain, None, None, sys.argv[0])
710 # DEBUG: print("DEBUG: EXIT!")
712 def fetch_fbabot_atom(args: argparse.Namespace):
713 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
714 feed = "https://ryona.agency/users/fba/feed.atom"
718 print(f"INFO: Fetching ATOM feed='{feed}' from FBA bot account ...")
719 response = reqto.get(feed, headers=fba.headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
721 # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
722 if response.ok and response.status_code < 300 and len(response.text) > 0:
723 # DEBUG: print(f"DEBUG: Parsing ATOM feed ...")
724 atom = atoma.parse_atom_bytes(response.content)
726 # DEBUG: print(f"DEBUG: atom[]={type(atom)}")
727 for entry in atom.entries:
728 # DEBUG: print(f"DEBUG: entry[]={type(entry)}")
729 doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
730 # DEBUG: print(f"DEBUG: doc[]={type(doc)}")
731 for element in doc.findAll("a"):
732 # DEBUG: print(f"DEBUG: element[{type(element)}]={element}")
733 domain = fba.tidyup_domain(element["href"])
735 # DEBUG: print(f"DEBUG: domain='{domain}'")
736 if fba.is_blacklisted(domain):
737 # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
739 elif domain in domains:
740 # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
742 elif fba.is_instance_registered(domain):
743 # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
746 # DEBUG: print(f"DEBUG: domain='{domain}'")
747 domains.append(domain)
749 except BaseException as e:
750 print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
753 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
757 print(f"INFO: Adding {len(domains)} new instances ...")
758 for domain in domains:
759 print(f"INFO: Fetching instances from domain='{domain}' ...")
760 fba.fetch_instances(domain, None, None, sys.argv[0])
762 # DEBUG: print("DEBUG: EXIT!")
764 def fetch_instances(args: argparse.Namespace):
765 # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
769 fba.fetch_instances(args.domain, None, None, sys.argv[0])
772 print(f"DEBUG: Not fetching more instances - EXIT!")
775 # Loop through some instances
777 "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
780 rows = fba.cursor.fetchall()
781 print(f"INFO: Checking {len(rows)} entries ...")
783 # DEBUG: print("DEBUG: domain:", row[0])
784 if fba.is_blacklisted(row[0]):
785 print("WARNING: domain is blacklisted:", row[0])
788 print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
789 fba.fetch_instances(row[0], row[1], row[2], sys.argv[0], row[3])
791 # DEBUG: print("DEBUG: EXIT!")