1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
2 # Copyright (C) 2023 Free Software Foundation
4 # This program is free software: you can redistribute it and/or modify
5 # it under the terms of the GNU Affero General Public License as published
6 # by the Free Software Foundation, either version 3 of the License, or
7 # (at your option) any later version.
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU Affero General Public License for more details.
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <https://www.gnu.org/licenses/>.
17 def fetch_joinfediverse(args: argparse.Namespace) -> int:
18 logger.debug("args[]='%s' - CALLED!", type(args))
20 logger.debug("Invoking locking.acquire() ...")
23 source_domain = "joinfediverse.wiki"
24 if sources.is_recent(source_domain):
25 logger.info("API from source_domain='%s' has recently being accessed - EXIT!", source_domain)
28 logger.debug("source_domain='%s' has not been recently used, marking ...", source_domain)
29 sources.update(source_domain)
31 logger.info("Fetching /FediBlock wiki page from source_domain='%s' ...", source_domain)
32 raw = utils.fetch_url(
33 f"https://{source_domain}/FediBlock",
35 (config.get("connection_timeout"), config.get("read_timeout"))
37 logger.debug("raw()=%d,raw[]='%s'", len(raw), type(raw))
39 doc = bs4.BeautifulSoup(raw, "html.parser")
40 logger.debug("doc[]='%s'", type(doc))
42 tables = doc.findAll("table", {"class": "wikitable"})
44 logger.info("Analyzing %d table(s) ...", len(tables))
47 logger.debug("table[]='%s'", type(table))
49 rows = table.findAll("tr")
50 logger.info("Checking %d row(s) ...", len(rows))
51 block_headers = dict()
53 logger.debug("row[%s]='%s'", type(row), row)
55 headers = row.findAll("th")
56 logger.debug("Found headers()=%d header(s)", len(headers))
58 block_headers = dict()
60 for header in headers:
62 logger.debug("header[]='%s',cnt=%d", type(header), cnt)
63 text = header.contents[0]
65 logger.debug("text[]='%s'", type(text))
66 if not isinstance(text, str):
67 logger.debug("text[]='%s' is not of type 'str' - SKIPPED!", type(text))
69 elif validators.domain(text.strip()):
70 logger.debug("text='%s' is a domain - SKIPPED!", text.strip())
73 text = tidyup.domain(text.strip())
74 logger.debug("text='%s' - AFTER!", text)
75 if text in ["domain", "instance", "subdomain(s)", "block reason(s)"]:
76 logger.debug("Found header: '%s'=%d", text, cnt)
77 block_headers[cnt] = text
79 elif len(block_headers) == 0:
80 logger.debug("row is not scrapable - SKIPPED!")
82 elif len(block_headers) > 0:
83 logger.debug("Found a row with %d scrapable headers ...", len(block_headers))
87 for element in row.find_all(["th", "td"]):
89 logger.debug("element[]='%s',cnt=%d", type(element), cnt)
90 if cnt in block_headers:
91 logger.debug("block_headers[%d]='%s'", cnt, block_headers[cnt])
93 text = element.text.strip()
94 key = block_headers[cnt] if block_headers[cnt] not in ["domain", "instance"] else "blocked"
96 logger.debug("cnt=%d is wanted: key='%s',text[%s]='%s'", cnt, key, type(text), text)
97 if key in ["domain", "instance"]:
100 block[key] = tidyup.reason(text)
101 elif key == "subdomain(s)":
104 block[key] = text.split("/")
106 logger.debug("key='%s'", key)
109 logger.debug("block()=%d ...", len(block))
111 logger.debug("Appending block()=%d ...", len(block))
112 blocklist.append(block)
114 logger.debug("blocklist()=%d", len(blocklist))
116 database.cursor.execute("SELECT domain FROM instances WHERE domain LIKE 'climatejustice.%'")
117 domains = database.cursor.fetchall()
119 logger.debug("domains(%d)[]='%s'", len(domains), type(domains))
121 for block in blocklist:
122 logger.debug("block='%s'", block)
123 if "subdomain(s)" in block and len(block["subdomain(s)"]) > 0:
124 origin = block["blocked"]
125 logger.debug("origin='%s'", origin)
126 for subdomain in block["subdomain(s)"]:
127 block["blocked"] = subdomain + "." + origin
128 logger.debug("block[blocked]='%s'", block["blocked"])
129 blocking.append(block)
131 blocking.append(block)
133 logger.debug("blocking()=%d", blocking)
134 for block in blocking:
135 logger.debug("block[]='%s'", type(block))
136 if "blocked" not in block:
137 raise KeyError(f"block()={len(block)} does not have element 'blocked'")
139 block["blocked"] = tidyup.domain(block["blocked"]).encode("idna").decode("utf-8")
140 logger.debug("block[blocked]='%s' - AFTER!", block["blocked"])
142 if block["blocked"] == "":
143 logger.debug("block[blocked] is empty - SKIPPED!")
145 elif not domain_helper.is_wanted(block["blocked"]):
146 logger.debug("block[blocked]='%s' is not wanted - SKIPPED!", block["blocked"])
148 elif instances.is_recent(block["blocked"]):
149 logger.debug("block[blocked]='%s' has been recently checked - SKIPPED!", block["blocked"])
152 logger.debug("Proccessing blocked='%s' ...", block["blocked"])
153 processing.instance(block["blocked"], "climatejustice.social", inspect.currentframe().f_code.co_name)
156 for blocker in domains:
158 logger.debug("blocker[%s]='%s'", type(blocker), blocker)
159 instances.set_last_blocked(blocker)
161 for block in blocking:
162 logger.debug("block[blocked]='%s',block[block reason(s)]='%s' - BEFORE!", block["blocked"], block["block reason(s)"] if "block reason(s)" in block else None)
163 block["reason"] = tidyup.reason(block["block reason(s)"]) if "block reason(s)" in block else None
165 logger.debug("block[blocked]='%s',block[reason]='%s' - AFTER!", block["blocked"], block["reason"])
166 if block["blocked"] == "":
167 logger.debug("block[blocked] is empty - SKIPPED!")
169 elif not domain_helper.is_wanted(block["blocked"]):
170 logger.debug("blocked='%s' is not wanted - SKIPPED!", block["blocked"])
173 logger.debug("blocked='%s',reason='%s'", block["blocked"], block["reason"])
174 if processing.block(blocker, block["blocked"], block["reason"], "reject") and config.get("bot_enabled"):
175 logger.debug("Appending blocked='%s',reason='%s' for blocker='%s' ...", block["blocked"], block["block_level"], blocker)
177 "blocked": block["blocked"],
178 "reason" : block["reason"],
181 if instances.has_pending(blocker):
182 logger.debug("Flushing updates for blocker='%s' ...", blocker)
183 instances.update(blocker)
185 logger.debug("Invoking commit() ...")
186 database.connection.commit()
188 logger.debug("config.get(bot_enabled)='%s',blockdict()=%d", config.get("bot_enabled"), len(blockdict))
189 if config.get("bot_enabled") and len(blockdict) > 0:
190 logger.info("Sending bot POST for blocker='%s,blockdict()=%d ...", blocker, len(blockdict))
191 network.send_bot_post(blocker, blockdict)
193 logger.debug("Success! - EXIT!")