+
+def json_from_response(response: requests.models.Response) -> list:
+ # DEBUG: print(f"DEBUG: response[]={type(response)} - CALLED!")
+ if not isinstance(response, requests.models.Response):
+ raise ValueError(f"Parameter response[]='{type(response)}' is not type of 'Response'")
+
+ data = list()
+ if response.text.strip() != "":
+ # DEBUG: print(f"DEBUG: response.text()={len(response.text)} is not empty, invoking response.json() ...")
+ try:
+ data = response.json()
+ except json.decoder.JSONDecodeError:
+ pass
+
+ # DEBUG: print(f"DEBUG: data[]={type(data)} - EXIT!")
+ return data
+
+def get_response(domain: str, path: str, headers: dict, timeout: list) -> requests.models.Response:
+ # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',headers()={len(headers)},timeout={timeout} - CALLED!")
+ if type(domain) != str:
+ raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'")
+ elif domain == "":
+ raise ValueError("Parameter 'domain' cannot be empty")
+ elif type(path) != str:
+ raise ValueError(f"Parameter path[]='{type(path)}' is not 'str'")
+ elif path == "":
+ raise ValueError("Parameter 'path' cannot be empty")
+
+ try:
+ # DEBUG: print(f"DEBUG: Sending request to '{domain}{path}' ...")
+ response = reqto.get(f"https://{domain}{path}", headers=headers, timeout=timeout);
+ except requests.exceptions.ConnectionError as e:
+ # DEBUG: print(f"DEBUG: Fetching '{path}' from '{domain}' failed. exception[{type(e)}]='{str(e)}'")
+ update_last_error(domain, e)
+ raise e
+
+ # DEBUG: print(f"DEBUG: response[]='{type(response)}' - EXXIT!")
+ return response
+
+def has_element(elements: list, key: str, value: any) -> bool:
+ # DEBUG: print(f"DEBUG: element()={len(element)},key='{key}',value[]='{type(value)}' - CALLED!")
+ if type(key) != str:
+ raise ValueError(f"Parameter key[]='{type(key)}' is not 'str'")
+ elif key == "":
+ raise ValueError("Parameter 'key' cannot be empty")
+
+ has = False
+ # DEBUG: print(f"DEBUG: Checking elements()={len(elements)} ...")
+ for element in elements:
+ # DEBUG: print(f"DEBUG: element[]='{type(element)}'")
+ if type(element) != dict:
+ raise ValueError(f"element[]='{type(element)}' is not 'dict'")
+ elif not key in element:
+ raise KeyError(f"Cannot find key='{key}'")
+ elif element[key] == value:
+ has = True
+ break
+
+ # DEBUG: print(f"DEBUG: has={has} - EXIT!")
+ return has
+
+def find_domains(tag: bs4.element.Tag) -> list:
+ # DEBUG: print(f"DEBUG: tag[]={type(tag)} - CALLED!")
+ if not isinstance(tag, bs4.element.Tag):
+ raise ValueError(f"Parameter tag[]={type(tag)} is not type of bs4.element.Tag")
+ elif not isinstance(tag, bs4.element.Tag):
+ raise KeyError("Cannot find table with instances!")
+ elif len(tag.select("tr")) == 0:
+ raise KeyError("No table rows found in table!")
+
+ domains = list()
+ for element in tag.select("tr"):
+ # DEBUG: print(f"DEBUG: element[]={type(element)}")
+ if not element.find("td"):
+ # DEBUG: print("DEBUG: Skipping element, no <td> found")
+ continue
+
+ domain = tidyup_domain(element.find("td").text)
+ reason = tidyup_reason(element.findAll("td")[1].text)
+
+ # DEBUG: print(f"DEBUG: domain='{domain}',reason='{reason}'")
+
+ if is_blacklisted(domain):
+ print(f"WARNING: domain='{domain}' is blacklisted - skipped!")
+ continue
+ elif domain == "gab.com/.ai, develop.gab.com":
+ print(f"DEBUG: Multiple domains detected in one row")
+ domains.append({
+ "domain": "gab.com",
+ "reason": reason,
+ })
+ domains.append({
+ "domain": "gab.ai",
+ "reason": reason,
+ })
+ domains.append({
+ "domain": "develop.gab.com",
+ "reason": reason,
+ })
+ continue
+ elif not validators.domain(domain):
+ print(f"WARNING: domain='{domain}' is not a valid domain - skipped!")
+ continue
+
+ # DEBUG: print(f"DEBUG: Adding domain='{domain}' ...")
+ domains.append({
+ "domain": domain,
+ "reason": reason,
+ })
+
+ # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!")
+ return domains