fetch_blocks.py

   1 from requests import get
   2 from hashlib import sha256
   3 import sqlite3
   4 from bs4 import BeautifulSoup
   5
   6 headers = {
   7     "user-agent": "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0"
   8 }
   9
  10
  11 def get_mastodon_blocks(domain: str) -> dict:
  12     blocks = {
  13         "Suspended servers": [],
  14         "Filtered media": [],
  15         "Limited servers": [],
  16         "Silenced servers": [],
  17     }
  18
  19     translations = {
  20         "Gesperrte Server": "Suspended servers",
  21         "Gefilterte Medien": "Filtered media",
  22         "Stummgeschaltete Server": "Silenced servers",
  23         "停止済みのサーバー": "Suspended servers",
  24         "メディアを拒否しているサーバー": "Filtered media",
  25         "サイレンス済みのサーバー": "Silenced servers",
  26         "Serveurs suspendus": "Suspended servers",
  27         "Médias filtrés": "Filtered media",
  28         "Serveurs limités": "Silenced servers",
  29     }
  30
  31     try:
  32         doc = BeautifulSoup(
  33             get(f"https://{domain}/about/more", headers=headers, timeout=5).text,
  34             "html.parser",
  35         )
  36     except:
  37         return {}
  38
  39     for header in doc.find_all("h3"):
  40         for line in header.find_next_siblings("table")[0].find_all("tr")[1:]:
  41             header_text = header.text
  42             if header_text in translations:
  43                     header_text = translations[header_text]
  44             if header_text in blocks:
  45                 blocks[header_text].append(
  46                     {
  47                         "domain": line.find("span").text,
  48                         "hash": line.find("span")["title"][9:],
  49                         "reason": line.find_all("td")[1].text.strip(),
  50                     }
  51                 )
  52     return {
  53         "reject": blocks["Suspended servers"],
  54         "media_removal": blocks["Filtered media"],
  55         "federated_timeline_removal": blocks["Limited servers"]
  56         + blocks["Silenced servers"],
  57     }
  58
  59 def get_friendica_blocks(domain: str) -> dict:
  60     blocks = []
  61
  62     try:
  63         doc = BeautifulSoup(
  64             get(f"https://{domain}/friendica", headers=headers, timeout=5).text,
  65             "html.parser",
  66         )
  67     except:
  68         return {}
  69
  70     blocklist = doc.find(id="about_blocklist")
  71     for line in blocklist.find("table").find_all("tr")[1:]:
  72             blocks.append(
  73                 {
  74                     "domain": line.find_all("td")[0].text.strip(),
  75                     "reason": line.find_all("td")[1].text.strip()
  76                 }
  77             )
  78
  79     return {
  80         "reject": blocks
  81     }
  82
  83 def get_hash(domain: str) -> str:
  84     return sha256(domain.encode("utf-8")).hexdigest()
  85
  86
  87 def get_type(domain: str) -> str:
  88     try:
  89         res = get(f"https://{domain}/nodeinfo/2.1.json", headers=headers, timeout=5)
  90         if res.status_code == 404:
  91             res = get(f"https://{domain}/nodeinfo/2.0", headers=headers, timeout=5)
  92         if res.status_code == 404:
  93             res = get(f"https://{domain}/nodeinfo/2.0.json", headers=headers, timeout=5)
  94         if res.ok and "text/html" in res.headers["content-type"]:
  95             res = get(f"https://{domain}/nodeinfo/2.1", headers=headers, timeout=5)
  96         if res.ok:
  97             if res.json()["software"]["name"] == "akkoma":
  98                 return "pleroma"
  99             elif res.json()["software"]["name"] == "hometown":
 100                 return "mastodon"
 101             else:
 102                 return res.json()["software"]["name"]
 103         elif res.status_code == 404:
 104             res = get(f"https://{domain}/api/v1/instance", headers=headers, timeout=5)
 105         if res.ok:
 106             return "mastodon"
 107     except:
 108         return None
 109
 110
 111 conn = sqlite3.connect("blocks.db")
 112 c = conn.cursor()
 113
 114 c.execute(
 115     "select domain, software from instances where software in ('pleroma', 'mastodon')"
 116 )
 117
 118 for blocker, software in c.fetchall():
 119     if software == "pleroma":
 120         print(blocker)
 121         try:
 122             # Blocks
 123             federation = get(
 124                 f"https://{blocker}/nodeinfo/2.1.json", headers=headers, timeout=5
 125             ).json()["metadata"]["federation"]
 126             if "mrf_simple" in federation:
 127                 for block_level, blocks in (
 128                     {**federation["mrf_simple"],
 129                     **{"quarantined_instances": federation["quarantined_instances"]}}
 130                 ).items():
 131                     for blocked in blocks:
 132                         if blocked == "":
 133                             continue
 134                         blocked == blocked.lower()
 135                         blocker == blocker.lower()
 136                         c.execute(
 137                             "select domain from instances where domain = ?", (blocked,)
 138                         )
 139                         if c.fetchone() == None:
 140                             c.execute(
 141                                 "insert into instances select ?, ?, ?",
 142                                 (blocked, get_hash(blocked), get_type(blocked)),
 143                             )
 144                         c.execute(
 145                             "select * from blocks where blocker = ? and blocked = ? and block_level = ?",
 146                             (blocker, blocked, block_level),
 147                         )
 148                         if c.fetchone() == None:
 149                             c.execute(
 150                                 "insert into blocks select ?, ?, '', ?",
 151                                 (blocker, blocked, block_level),
 152                             )
 153             conn.commit()
 154             # Reasons
 155             if "mrf_simple_info" in federation:
 156                 for block_level, info in (
 157                     {**federation["mrf_simple_info"],
 158                     **(federation["quarantined_instances_info"]
 159                     if "quarantined_instances_info" in federation
 160                     else {})}
 161                 ).items():
 162                     for blocked, reason in info.items():
 163                         blocker == blocker.lower()
 164                         blocked == blocked.lower()
 165                         c.execute(
 166                             "update blocks set reason = ? where blocker = ? and blocked = ? and block_level = ?",
 167                             (reason["reason"], blocker, blocked, block_level),
 168                         )
 169             conn.commit()
 170         except Exception as e:
 171             print("error:", e, blocker)
 172     elif software == "mastodon":
 173         print(blocker)
 174         try:
 175             json = get_mastodon_blocks(blocker)
 176             for block_level, blocks in json.items():
 177                 for instance in blocks:
 178                     blocked, blocked_hash, reason = instance.values()
 179                     blocked == blocked.lower()
 180                     blocker == blocker.lower()
 181                     if blocked.count("*") <= 1:
 182                         c.execute(
 183                             "select hash from instances where hash = ?", (blocked_hash,)
 184                         )
 185                         if c.fetchone() == None:
 186                             c.execute(
 187                                 "insert into instances select ?, ?, ?",
 188                                 (blocked, get_hash(blocked), get_type(blocked)),
 189                             )
 190                     c.execute(
 191                         "select * from blocks where blocker = ? and blocked = ? and block_level = ?",
 192                         (blocker, blocked if blocked.count("*") <= 1 else blocked_hash, block_level),
 193                     )
 194                     if c.fetchone() == None:
 195                         c.execute(
 196                             "insert into blocks select ?, ?, ?, ?",
 197                             (
 198                                 blocker,
 199                                 blocked if blocked.count("*") <= 1 else blocked_hash,
 200                                 reason,
 201                                 block_level,
 202                             ),
 203                         )
 204             conn.commit()
 205         except Exception as e:
 206             print("error:", e, blocker)
 207     elif software == "friendica"
 208         print(blocker)
 209         try:
 210             json = get_friendica_blocks(blocker)
 211             for blocks in json.items():
 212                 for instance in blocks:
 213                     blocked, reason = instance.values()
 214                     blocked == blocked.lower()
 215                     blocker == blocker.lower()
 216                     c.execute(
 217                         "select domain from instances where domain = ?", (blocked,)
 218                     )
 219                     if c.fetchone() == None:
 220                         c.execute(
 221                             "insert into instances select ?, ?, ?",
 222                             (blocked, get_hash(blocked), get_type(blocked)),
 223                         )
 224                     c.execute(
 225                         "select * from blocks where blocker = ? and blocked = ?",
 226                         (blocker, blocked),
 227                     )
 228                     if c.fetchone() == None:
 229                         c.execute(
 230                             "insert into blocks select ?, ?, ?, ?",
 231                             (
 232                                 blocker,
 233                                 blocked,
 234                                 reason,
 235                                 "reject",
 236                             ),
 237                         )
 238             conn.commit()
 239         except Exception as e:
 240             print("error:", e, blocker)
 241 conn.close()