1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
30 from fba import database
33 from fba.helpers import blacklist
34 from fba.helpers import config
35 from fba.helpers import cookies
36 from fba.helpers import locking
37 from fba.helpers import tidyup
39 from fba.http import federation
40 from fba.http import network
42 from fba.models import blocks
43 from fba.models import instances
45 from fba.networks import friendica
46 from fba.networks import lemmy
47 from fba.networks import mastodon
48 from fba.networks import misskey
49 from fba.networks import pleroma
51 logging.basicConfig(level=logging.INFO)
52 logger = logging.getLogger(__name__)
53 #logger.setLevel(logging.DEBUG)
55 def check_instance(args: argparse.Namespace) -> int:
56 logger.debug("args.domain='%s' - CALLED!", args.domain)
58 if not validators.domain(args.domain):
59 logger.warning("args.domain='%s' is not valid", args.domain)
61 elif blacklist.is_blacklisted(args.domain):
62 logger.warning("args.domain='%s' is blacklisted", args.domain)
64 elif instances.is_registered(args.domain):
65 logger.warning("args.domain='%s' is already registered", args.domain)
68 logger.info("args.domain='%s' is not known", args.domain)
70 logger.debug(f"status={status} - EXIT!")
73 def fetch_bkali(args: argparse.Namespace) -> int:
74 logger.debug("args[]='%s' - CALLED!", type(args))
77 fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
78 "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
81 logger.debug("fetched[]='%s'", type(fetched))
82 if "error_message" in fetched:
83 logger.warning("post_json_api() for 'gql.api.bka.li' returned error message='%s", fetched['error_message'])
85 elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
86 logger.warning("post_json_api() returned error: '%s", fetched['error']['message'])
89 rows = fetched["json"]
91 logger.debug(f"rows({len(rows)})[]='{type(rows)}'")
93 raise Exception("WARNING: Returned no records")
94 elif "data" not in rows:
95 raise Exception(f"WARNING: rows()={len(rows)} does not contain key 'data'")
96 elif "nodeinfo" not in rows["data"]:
97 raise Exception(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'")
99 for entry in rows["data"]["nodeinfo"]:
100 logger.debug(f"entry['{type(entry)}']='{entry}'")
101 if "domain" not in entry:
102 logger.warning("entry()=%d does not contain 'domain' - SKIPPED!", len(entry))
104 elif not utils.is_domain_wanted(entry["domain"]):
105 logger.debug("entry[domain]='%s' is not wanted - SKIPPED!")
107 elif instances.is_registered(entry["domain"]):
108 logger.debug("domain='%s' is already registered - SKIPPED!", entry['domain'])
111 logger.debug(f"Adding domain='{entry['domain']}' ...")
112 domains.append(entry["domain"])
114 except network.exceptions as exception:
115 logger.error(f"Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}' - EXIT!")
118 logger.debug(f"domains()={len(domains)}")
122 logger.info("Adding %d new instances ...", len(domains))
123 for domain in domains:
125 logger.info("Fetching instances from domain='%s' ...", domain)
126 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
128 logger.debug("Invoking cookies.clear(%s) ...", domain)
129 cookies.clear(domain)
130 except network.exceptions as exception:
131 logger.warning("Exception '%s' during fetching instances (fetch_bkali) from domain='%s'", type(exception), domain)
132 instances.set_last_error(domain, exception)
134 logger.debug("Success - EXIT!")
137 def fetch_blocks(args: argparse.Namespace):
138 logger.debug("args[]='%s' - CALLED!", type(args))
139 if args.domain is not None and args.domain != "":
140 logger.debug(f"args.domain='{args.domain}' - checking ...")
141 if not validators.domain(args.domain):
142 logger.warning("args.domain='%s' is not valid.", args.domain)
144 elif blacklist.is_blacklisted(args.domain):
145 logger.warning("args.domain='%s' is blacklisted, won't check it!", args.domain)
147 elif not instances.is_registered(args.domain):
148 logger.warning("args.domain='%s' is not registered, please run ./utils.py fetch_instances '%s' first.", args.domain, args.domain)
153 if args.domain is not None and args.domain != "":
154 # Re-check single domain
155 logger.debug("Querying database for single args.domain='%s' ...", args.domain)
156 database.cursor.execute(
157 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE domain = ?", [args.domain]
159 elif args.software is not None and args.software != "":
160 # Re-check single software
161 logger.debug("Querying database for args.software='%s' ...", args.software)
162 database.cursor.execute(
163 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software = ?", [args.software]
166 # Re-check after "timeout" (aka. minimum interval)
167 database.cursor.execute(
168 "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'lemmy', 'friendica', 'misskey', 'peertube') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
171 rows = database.cursor.fetchall()
172 logger.info("Checking %d entries ...", len(rows))
173 for blocker, software, origin, nodeinfo_url in rows:
174 logger.debug("blocker='%s',software='%s',origin='%s',nodeinfo_url='%s'", blocker, software, origin, nodeinfo_url)
176 blocker = tidyup.domain(blocker)
177 logger.debug("blocker='%s' - AFTER!", blocker)
180 logger.warning("blocker is now empty!")
182 elif nodeinfo_url is None or nodeinfo_url == "":
183 logger.debug("blocker='%s',software='%s' has empty nodeinfo_url", blocker, software)
185 elif not utils.is_domain_wanted(blocker):
186 logger.warning("blocker='%s' is not wanted - SKIPPED!", blocker)
189 logger.debug("blocker='%s'", blocker)
190 instances.set_last_blocked(blocker)
191 instances.set_has_obfucation(blocker, False)
194 if software == "pleroma":
195 logger.info("blocker='%s',software='%s'", blocker, software)
196 blocking = pleroma.fetch_blocks(blocker, nodeinfo_url)
197 elif software == "mastodon":
198 logger.info("blocker='%s',software='%s'", blocker, software)
199 blocking = mastodon.fetch_blocks(blocker, nodeinfo_url)
200 elif software == "lemmy":
201 logger.info("blocker='%s',software='%s'", blocker, software)
202 blocking = lemmy.fetch_blocks(blocker, nodeinfo_url)
203 elif software == "friendica":
204 logger.info("blocker='%s',software='%s'", blocker, software)
205 blocking = friendica.fetch_blocks(blocker)
206 elif software == "misskey":
207 logger.info("blocker='%s',software='%s'", blocker, software)
208 blocking = misskey.fetch_blocks(blocker)
210 logger.warning("Unknown software: blocker='%s',software='%s'", blocker, software)
212 logger.info("Checking %d entries from blocker='%s',software='%s' ...", len(blocking), blocker, software)
213 for block in blocking:
214 logger.debug("blocked='%s',block_level='%s',reason='%s'", block['blocked'], block['block_level'], block['reason'])
216 if block['block_level'] == "":
217 logger.warning("block_level is empty, blocker='%s',blocked='%s'", block['blocker'], block['blocked'])
220 logger.debug("blocked='%s',reason='%s' - BEFORE!", block['blocked'], block['reason'])
221 block['blocked'] = tidyup.domain(block['blocked'])
222 block['reason'] = tidyup.reason(block['reason']) if block['reason'] is not None and block['reason'] != "" else None
223 logger.debug("blocked='%s',reason='%s' - AFTER!", block['blocked'], block['reason'])
225 if block['blocked'] == "":
226 logger.warning("blocked is empty, blocker='%s'", blocker)
228 elif block['blocked'].count("*") > 0:
229 logger.debug("blocker='%s' uses obfucated domains, marking ...", blocker)
230 instances.set_has_obfucation(blocker, True)
232 # Some friendica servers also obscure domains without hash
233 row = instances.deobfucate("*", block['blocked'])
235 logger.debug("row[]='%s'", type(row))
237 logger.warning("Cannot deobfucate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block['blocked'], blocker, software)
240 block['blocked'] = row[0]
242 nodeinfo_url = row[2]
243 elif block['blocked'].count("?") > 0:
244 logger.debug("blocker='%s' uses obfucated domains, marking ...", blocker)
245 instances.set_has_obfucation(blocker, True)
247 # Some obscure them with question marks, not sure if that's dependent on version or not
248 row = instances.deobfucate("?", block['blocked'])
250 logger.debug("row[]='%s'", type(row))
252 logger.warning("Cannot deobfucate blocked='%s',blocker='%s',software='%s' - SKIPPED!", block['blocked'], blocker, software)
255 block['blocked'] = row[0]
257 nodeinfo_url = row[2]
259 logger.debug("Looking up instance by domainm, blocked='%s'", block['blocked'])
260 if not utils.is_domain_wanted(block['blocked']):
261 logger.debug("blocked='%s' is not wanted - SKIPPED!", block['blocked'])
263 elif not instances.is_registered(block['blocked']):
264 logger.debug("Hash wasn't found, adding: blocked='%s',blocker='%s'", block['blocked'], blocker)
266 instances.add(block['blocked'], blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
267 except network.exceptions as exception:
268 logger.warning("Exception during adding blocked='%s',blocker='%s': '%s'", block['blocked'], blocker, type(exception))
271 if not blocks.is_instance_blocked(blocker, block['blocked'], block['block_level']):
272 blocks.add_instance(blocker, block['blocked'], block['reason'], block['block_level'])
274 if block['block_level'] == "reject":
276 "blocked": block['blocked'],
277 "reason" : block['reason'],
280 logger.debug("Updating block last seen and reason for blocker='%s',blocked='%s' ...", blocker, block['blocked'])
281 blocks.update_last_seen(blocker, block['blocked'], block['block_level'])
282 blocks.update_reason(block['reason'], blocker, block['blocked'], block['block_level'])
284 logger.debug("Invoking cookies.clear(%s) ...", block['blocked'])
285 cookies.clear(block['blocked'])
287 if instances.has_pending(blocker):
288 logger.debug("Invoking instances.update_data(%s) ...", blocker)
289 instances.update_data(blocker)
291 logger.debug("Invoking commit() ...")
292 database.connection.commit()
294 if config.get("bot_enabled") and len(blockdict) > 0:
295 network.send_bot_post(blocker, blockdict)
297 logger.debug(f"Invoking cookies.clear({blocker}) ...")
298 cookies.clear(blocker)
300 logger.debug("EXIT!")
302 def fetch_observer(args: argparse.Namespace):
303 logger.debug("args[]='%s' - CALLED!", type(args))
339 logger.info("Fetching %d different table data ...", len(types))
340 for software in types:
344 logger.debug(f"Fetching table data for software='{software}' ...")
345 raw = utils.fetch_url(f"https://fediverse.observer/app/views/tabledata.php?software={software}", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
346 logger.debug("raw[%s]()=%d", type(raw), len(raw))
348 doc = bs4.BeautifulSoup(raw, features='html.parser')
349 logger.debug("doc[]='%s'", type(doc))
350 except network.exceptions as exception:
351 logger.warning("Cannot fetch software='%s' from fediverse.observer: '%s'", software, type(exception))
354 items = doc.findAll("a", {"class": "url"})
355 logger.info("Checking %d items,software='%s' ...", len(items), software)
357 logger.debug("item[]='%s'", type(item))
358 domain = item.decode_contents()
360 logger.debug("domain='%s'", domain)
361 if not utils.is_domain_wanted(domain):
362 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
364 elif instances.is_registered(domain):
365 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
368 logger.info("Fetching instances for domain='%s',software='%s'", domain, software)
369 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
371 logger.debug("Invoking cookies.clear(%s) ...", domain)
372 cookies.clear(domain)
374 logger.debug("EXIT!")
376 def fetch_todon_wiki(args: argparse.Namespace):
377 logger.debug("args[]='%s' - CALLED!", type(args))
385 raw = utils.fetch_url("https://wiki.todon.eu/todon/domainblocks", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
386 logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
388 doc = bs4.BeautifulSoup(raw, "html.parser")
389 logger.debug("doc[]='%s'", type(doc))
391 silenced = doc.find("h3", {"id": "silencedlimited_servers"}).find_next("ul").findAll("li")
392 logger.info("Checking %d silenced/limited entries ...", len(silenced))
393 blocklist["silenced"] = utils.find_domains(silenced, "div")
395 suspended = doc.find("h3", {"id": "suspended_servers"}).find_next("ul").findAll("li")
396 logger.info("Checking %d suspended entries ...", len(suspended))
397 blocklist["reject"] = utils.find_domains(suspended, "div")
399 for block_level in blocklist:
400 blockers = blocklist[block_level]
402 logger.debug("block_level='%s',blockers()=%d'", block_level, len(blockers))
403 for blocked in blockers:
404 logger.debug("blocked='%s'", blocked)
406 if not instances.is_registered(blocked):
408 logger.info("Fetching instances from domain='%s' ...", blocked)
409 federation.fetch_instances(blocked, 'chaos.social', None, inspect.currentframe().f_code.co_name)
411 logger.debug("Invoking cookies.clear(%s) ...", blocked)
412 cookies.clear(blocked)
413 except network.exceptions as exception:
414 logger.warning("Exception '%s' during fetching instances (fetch_cs) from blocked='%s'", type(exception), blocked)
415 instances.set_last_error(blocked, exception)
417 if blocks.is_instance_blocked("todon.eu", blocked, block_level):
418 logger.debug("blocked='%s',block_level='%s' is already blocked - SKIPPED!", blocked, block_level)
421 logger.info("Adding new block: blocked='%s',block_level='%s'", blocked, block_level)
422 blocks.add_instance("todon.eu", blocked, None, block_level)
424 logger.debug("Invoking commit() ...")
425 database.connection.commit()
427 logger.debug("EXIT!")
429 def fetch_cs(args: argparse.Namespace):
430 logger.debug("args[]='%s' - CALLED!", type(args))
456 raw = utils.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
457 logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
459 doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser')
460 logger.debug(f"doc()={len(doc)}[]='{type(doc)}'")
462 silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody")
463 logger.debug("silenced[%s]()=%d", type(silenced), len(silenced))
464 domains["silenced"] = federation.find_domains(silenced)
466 blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody")
467 logger.debug("blocked[%s]()=%d", type(blocked), len(blocked))
468 domains["reject"] = federation.find_domains(blocked)
470 logger.debug("domains[silenced]()=%d,domains[reject]()=%d", len(domains["silenced"]), len(domains["reject"]))
474 for block_level in domains:
475 logger.info("block_level='%s' has %d row(s)", block_level, len(domains[block_level]))
477 for row in domains[block_level]:
478 logger.debug(f"row='{row}'")
479 if not instances.is_registered(row["domain"]):
481 logger.info("Fetching instances from domain='%s' ...", row["domain"])
482 federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
484 logger.debug("Invoking cookies.clear(%s) ...", row["domain"])
485 cookies.clear(row["domain"])
486 except network.exceptions as exception:
487 logger.warning("Exception '%s' during fetching instances (fetch_cs) from row[domain]='%s'", type(exception), row["domain"])
488 instances.set_last_error(row["domain"], exception)
490 if not blocks.is_instance_blocked('chaos.social', row["domain"], block_level):
491 logger.debug("domain='%s',block_level='%s' blocked by chaos.social, adding ...", row["domain"], block_level)
492 blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level)
494 logger.debug("Invoking commit() ...")
495 database.connection.commit()
497 logger.debug("EXIT!")
499 def fetch_fba_rss(args: argparse.Namespace):
500 logger.debug("args[]='%s' - CALLED!", type(args))
503 logger.info("Fetch FBA-specific RSS args.feed='%s' ...", args.feed)
504 response = utils.fetch_url(args.feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
506 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
507 if response.ok and response.status_code < 300 and len(response.text) > 0:
508 logger.debug(f"Parsing RSS feed ({len(response.text)} Bytes) ...")
509 rss = atoma.parse_rss_bytes(response.content)
511 logger.debug(f"rss[]='{type(rss)}'")
512 for item in rss.items:
513 logger.debug(f"item={item}")
514 domain = item.link.split("=")[1]
516 if blacklist.is_blacklisted(domain):
517 logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
519 elif domain in domains:
520 logger.debug("domain='%s' is already added - SKIPPED!", domain)
522 elif instances.is_registered(domain):
523 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
526 logger.debug(f"Adding domain='{domain}'")
527 domains.append(domain)
529 logger.debug(f"domains()={len(domains)}")
533 logger.info("Adding %d new instances ...", len(domains))
534 for domain in domains:
536 logger.info("Fetching instances from domain='%s' ...", domain)
537 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
539 logger.debug("Invoking cookies.clear(%s) ...", domain)
540 cookies.clear(domain)
541 except network.exceptions as exception:
542 logger.warning("Exception '%s' during fetching instances (fetch_fba_rss) from domain='%s'", type(exception), domain)
543 instances.set_last_error(domain, exception)
545 logger.debug("EXIT!")
547 def fetch_fbabot_atom(args: argparse.Namespace):
548 logger.debug("args[]='%s' - CALLED!", type(args))
549 feed = "https://ryona.agency/users/fba/feed.atom"
553 logger.info("Fetching ATOM feed='%s' from FBA bot account ...", feed)
554 response = utils.fetch_url(feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
556 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
557 if response.ok and response.status_code < 300 and len(response.text) > 0:
558 logger.debug(f"Parsing ATOM feed ({len(response.text)} Bytes) ...")
559 atom = atoma.parse_atom_bytes(response.content)
561 logger.debug(f"atom[]='{type(atom)}'")
562 for entry in atom.entries:
563 logger.debug(f"entry[]='{type(entry)}'")
564 doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
565 logger.debug("doc[]='%s'", type(doc))
566 for element in doc.findAll("a"):
567 for href in element["href"].split(","):
568 logger.debug("href[%s]='%s", type(href), href)
569 domain = tidyup.domain(href)
571 logger.debug("domain='%s'", domain)
572 if blacklist.is_blacklisted(domain):
573 logger.debug("domain='%s' is blacklisted - SKIPPED!", domain)
575 elif domain in domains:
576 logger.debug("domain='%s' is already added - SKIPPED!", domain)
578 elif instances.is_registered(domain):
579 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
582 logger.debug(f"Adding domain='{domain}',domains()={len(domains)}")
583 domains.append(domain)
585 logger.debug(f"domains({len(domains)})={domains}")
589 logger.info("Adding %d new instances ...", len(domains))
590 for domain in domains:
592 logger.info("Fetching instances from domain='%s' ...", domain)
593 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
595 logger.debug("Invoking cookies.clear(%s) ...", domain)
596 cookies.clear(domain)
597 except network.exceptions as exception:
598 logger.warning("Exception '%s' during fetching instances (fetch_fbabot_atom) from domain='%s'", type(exception), domain)
599 instances.set_last_error(domain, exception)
601 logger.debug("EXIT!")
603 def fetch_instances(args: argparse.Namespace) -> int:
604 logger.debug("args[]='%s' - CALLED!", type(args))
609 logger.info("Fetching instances from args.domain='%s' ...", args.domain)
610 federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
612 logger.debug(f"Invoking cookies.clear({args.domain}) ...")
613 cookies.clear(args.domain)
614 except network.exceptions as exception:
615 logger.warning("Exception '%s' during fetching instances (fetch_instances) from args.domain='%s'", type(exception), args.domain)
616 instances.set_last_error(args.domain, exception)
620 logger.debug("Not fetching more instances - EXIT!")
623 # Loop through some instances
624 database.cursor.execute(
625 "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'lemmy', 'peertube') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
628 rows = database.cursor.fetchall()
629 logger.info("Checking %d entries ...", len(rows))
631 logger.debug("domain='%s'", row[0])
632 if blacklist.is_blacklisted(row[0]):
633 logger.warning("domain is blacklisted: row[0]='%s'", row[0])
637 logger.info("Fetching instances for instance domain='%s',software='%s',origin='%s',nodeinfo_url='%s'", row[0], row[2], row[1], row[3])
638 federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
640 logger.debug(f"Invoking cookies.clear({row[0]}) ...")
641 cookies.clear(row[0])
642 except network.exceptions as exception:
643 logger.warning("Exception '%s' during fetching instances (fetch_instances) from row[0]='%s'", type(exception), row[0])
644 instances.set_last_error(row[0], exception)
646 logger.debug("Success - EXIT!")
649 def fetch_oliphant(args: argparse.Namespace):
650 logger.debug("args[]='%s' - CALLED!", type(args))
654 base_url = "https://codeberg.org/oliphant/blocklists/raw/branch/main/blocklists"
659 "blocker": "artisan.chat",
660 "csv_url": "mastodon/artisan.chat.csv",
662 "blocker": "mastodon.art",
663 "csv_url": "mastodon/mastodon.art.csv",
665 "blocker": "pleroma.envs.net",
666 "csv_url": "mastodon/pleroma.envs.net.csv",
668 "blocker": "oliphant.social",
669 "csv_url": "mastodon/_unified_tier3_blocklist.csv",
671 "blocker": "mastodon.online",
672 "csv_url": "mastodon/mastodon.online.csv",
674 "blocker": "mastodon.social",
675 "csv_url": "mastodon/mastodon.social.csv",
677 "blocker": "mastodon.social",
678 "csv_url": "other/missing-tier0-mastodon.social.csv",
680 "blocker": "rage.love",
681 "csv_url": "mastodon/rage.love.csv",
683 "blocker": "sunny.garden",
684 "csv_url": "mastodon/sunny.garden.csv",
686 "blocker": "solarpunk.moe",
687 "csv_url": "mastodon/solarpunk.moe.csv",
689 "blocker": "toot.wales",
690 "csv_url": "mastodon/toot.wales.csv",
692 "blocker": "union.place",
693 "csv_url": "mastodon/union.place.csv",
699 logger.debug("Downloading %d files ...", len(blocklists))
700 for block in blocklists:
701 # Is domain given and not equal blocker?
702 if isinstance(args.domain, str) and args.domain != block["blocker"]:
703 logger.debug("Skipping blocker='%s', not matching args.domain='%s'", block['blocker'], args.domain)
705 elif args.domain in domains:
706 logger.debug("args.domain='%s' already handled - SKIPPED!", args.domain)
710 logger.info("Fetching csv_url='%s' for blocker='%s' ...", block['csv_url'], block['blocker'])
711 response = utils.fetch_url(f"{base_url}/{block['csv_url']}", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
713 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
714 if response.ok and response.content != "":
715 logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content))
716 reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
718 logger.debug("reader[]='%s'", type(reader))
722 domain = row["#domain"]
723 elif "domain" in row:
724 domain = row["domain"]
726 logger.debug(f"row='{row}' does not contain domain column")
729 logger.debug("domain='%s'", domain)
730 if not utils.is_domain_wanted(domain):
731 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
734 logger.debug("Marking domain='%s' as handled", domain)
735 domains.append(domain)
737 logger.debug("Processing domain='%s' ...", domain)
738 processed = utils.process_domain(domain, block["blocker"], inspect.currentframe().f_code.co_name)
740 logger.debug("processed='%s'", processed)
742 logger.debug("EXIT!")
744 def fetch_txt(args: argparse.Namespace):
745 logger.debug("args[]='%s' - CALLED!", type(args))
750 "https://seirdy.one/pb/bsl.txt",
753 logger.info("Checking %d text file(s) ...", len(urls))
755 logger.debug("Fetching url='%s' ...", url)
756 response = utils.fetch_url(url, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
758 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
759 if response.ok and response.status_code < 300 and response.text != "":
760 logger.debug("Returned %d Bytes for processing", len(response.text.strip()))
761 domains = response.text.split("\n")
763 logger.info("Processing %d domains ...", len(domains))
764 for domain in domains:
765 logger.debug("domain='%s'", domain)
767 logger.debug("domain is empty - SKIPPED!")
769 elif not utils.is_domain_wanted(domain):
770 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
773 logger.debug("domain='%s'", domain)
774 processed = utils.process_domain(domain, 'seirdy.one', inspect.currentframe().f_code.co_name)
776 logger.debug("processed='%s'", processed)
778 logger.debug(f"domain='{domain}' was not generically processed - SKIPPED!")
781 logger.debug("EXIT!")
783 def fetch_fedipact(args: argparse.Namespace):
784 logger.debug("args[]='%s' - CALLED!", type(args))
787 response = utils.fetch_url("https://fedipact.online", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
789 logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
790 if response.ok and response.status_code < 300 and response.text != "":
791 logger.debug("Parsing %d Bytes ...", len(response.text))
793 doc = bs4.BeautifulSoup(response.text, "html.parser")
794 logger.debug("doc[]='%s'", type(doc))
796 rows = doc.findAll("li")
797 logger.info("Checking %d row(s) ...", len(rows))
799 logger.debug("row[]='%s'", type(row))
800 domain = tidyup.domain(row.contents[0])
802 logger.debug("domain='%s'", domain)
804 logger.debug("domain is empty - SKIPPED!")
806 elif not utils.is_domain_wanted(domain):
807 logger.debug("domain='%s' is not wanted - SKIPPED!", domain)
809 elif instances.is_registered(domain):
810 logger.debug("domain='%s' is already registered - SKIPPED!", domain)
813 logger.info("Fetching domain='%s' ...", domain)
814 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
816 logger.debug("EXIT!")