Continued:

- introduced fba.get_response()

diff --git a/fba/fba.py b/fba/fba.py
index a94558e0dde8d96b7a5f165f7452dbdaf596d5bd..e81e478a80b1167254c7c6ad6ad617088a844844 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -561,7 +561,7 @@ def get_peers(domain: str, software: str) -> list:
     elif software == "lemmy":
         # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...")
         try:
-            res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
+            res = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
             data = json_from_response(res)
 
@@ -599,7 +599,7 @@ def get_peers(domain: str, software: str) -> list:
             # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'")
             while True:
                 try:
-                    res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
+                    res = get_response(domain, "/api/v1/server/{mode}?start={start}&count=100", headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
                     data = json_from_response(res)
                     # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'")
@@ -636,14 +636,14 @@ def get_peers(domain: str, software: str) -> list:
 
     # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...")
     try:
-        res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
+        res = get_response(domain, get_peers_url, api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
         data = json_from_response(res)
 
         # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
         if not res.ok or res.status_code >= 400:
             # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...")
-            res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
+            res = get_response(domain, "/api/v3/site", api_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
             data = json_from_response(res)
             # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'")
@@ -776,7 +776,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list:
     data = {}
 
     try:
-        res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
+        res = get_response(domain, "/.well-known/nodeinfo", api_headers, (config.get("nodeinfo_connection_timeout"), config.get("nodeinfo_read_timeout")))
 
         data = json_from_response(res)
         # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data))
@@ -827,7 +827,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str:
 
     try:
         # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
-        res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config.get("connection_timeout"), config.get("read_timeout")))
+        response = get_response(domain, path, headers, (config.get("connection_timeout"), config.get("read_timeout")))
 
         # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text))
         if res.ok and res.status_code < 300 and len(res.text) > 0:
@@ -1231,7 +1231,7 @@ def get_mastodon_blocks(domain: str) -> dict:
 
     try:
         doc = bs4.BeautifulSoup(
-            reqto.get(f"https://{domain}/about", headers=headers, timeout=(config.get("connection_timeout"), config.get("read_timeout"))).text,
+            get_response(domain, "/about", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
             "html.parser",
         )
     except BaseException as e:
@@ -1276,7 +1276,7 @@ def get_friendica_blocks(domain: str) -> dict:
 
     try:
         doc = bs4.BeautifulSoup(
-            reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config.get("connection_timeout"), config.get("read_timeout"))).text,
+            get_response(domain, "/friendica", headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
             "html.parser",
         )
     except BaseException as e:
@@ -1473,3 +1473,16 @@ def json_from_response(response: requests.models.Response) -> list:
 
     # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!")
     return data
+
+def get_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response:
+    # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!")
+    if type(domain) != str:
+        raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+    elif type(path) != str:
+        raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
+
+    # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...")
+    response = reqto.get(f"https://{domain}{path}", headers=headers, timeout=timeout);
+
+    # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
+    return response

diff --git a/fetch_blocks.py b/fetch_blocks.py
index e399d7081a238de9045a5900d02f8e99177620b7..424f61ac0af8c7341559f3fc8c4da3b978153497 100755 (executable)
--- a/fetch_blocks.py
+++ b/fetch_blocks.py
@@ -225,7 +225,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                 # handling CSRF, I've saw at least one server requiring it to access the endpoint
                 # DEBUG: print("DEBUG: Fetching meta:", blocker)
                 meta = bs4.BeautifulSoup(
-                    reqto.get(f"https://{blocker}/", headers=fba.headers, timeout=(config.get("connection_timeout"), config.get("read_timeout"))).text,
+                    fba.get_response(blocker, "/", fba.headers, (config.get("connection_timeout"), config.get("read_timeout"))).text,
                     "html.parser",
                 )
                 try:
@@ -237,7 +237,7 @@ for blocker, software, origin, nodeinfo_url in rows:
                     reqheaders = fba.api_headers
 
                 # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker)
-                blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(config.get("connection_timeout"), config.get("read_timeout"))).json()
+                blocks = fba.get_response(blocker, "/api/v1/instance/domain_blocks", reqheaders, (config.get("connection_timeout"), config.get("read_timeout"))).json()
 
                 print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...")
                 for block in blocks:
@@ -447,7 +447,7 @@ for blocker, software, origin, nodeinfo_url in rows:
         print("INFO: blocker:", blocker)
         try:
             # Blocks
-            federation = reqto.get(f"https://{blocker}{fba.get_peers_url}?filter=suspended", headers=fba.api_headers, timeout=(config.get("connection_timeout"), config.get("read_timeout"))).json()
+            federation = fba.get_response(blocker, "{fba.get_peers_url}?filter=suspended", fba.api_headers, (config.get("connection_timeout"), config.get("read_timeout"))).json()
 
             if (federation == None):
                 print("WARNING: No valid response:", blocker);