Handle CSRF tokens on masto endpoint

diff --git a/fetch_blocks.py b/fetch_blocks.py
index 5e076e7f1788ea50d94903dc50aada00e54c51e7..187ac094455167155dd956a873175e69f905a234 100644 (file)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -297,8 +297,20 @@ for blocker, software in c.fetchall():
                     "followers_only": [],
                     "report_removal": []
                 }
+
+                # handling CSRF, I've saw at least one server requiring it to access the endpoint
+                meta = BeautifulSoup(
+                    get(f"https://{blocker}/about", headers=headers, timeout=5).text,
+                    "html.parser",
+                )
+                try:
+                    csrf = meta.find("meta", attrs={"name": "csrf-token"})["content"]
+                    reqheaders = {**headers, **{"x-csrf-token": csrf}}
+                except:
+                    reqheaders = headers
+
                 blocks = get(
-                    f"https://{blocker}/api/v1/instance/domain_blocks", headers=headers, timeout=5
+                    f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=5
                 ).json()
                 for block in blocks:
                     entry = {'domain': block['domain'], 'hash': block['digest'], 'reason': block['comment']}