From baf882870687c37fa78bef72d8e1e9f65ec69976 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 13 Jun 2023 18:26:43 +0200 Subject: [PATCH] Continued: - 'data' is optional, e.g. empty POST bodies are okay - fixed handling of JSON replies from peertube instances: - the request /api/v1/server/followers is correct, then always the JSON has "follow" and "following" as elements --- fba/commands.py | 4 ++-- fba/network.py | 2 +- fba/networks/peertube.py | 37 +++++++++++++++++++++---------------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/fba/commands.py b/fba/commands.py index 7a64081..2f31048 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -537,8 +537,8 @@ def fetch_oliphant(args: argparse.Namespace): if isinstance(args.domain, str) and args.domain != block["blocker"]: # DEBUG: print(f"DEBUG: Skipping blocker='{block['blocker']}', not matching args.domain='{args.domain}'") continue - elif domain in domains: - # DEBUG: print(f"DEBUG: domain='{domain}' already handled - SKIPPED!") + elif args.domain in domains: + # DEBUG: print(f"DEBUG: args.domain='{args.domain}' already handled - SKIPPED!") continue # Fetch this URL diff --git a/fba/network.py b/fba/network.py index a86d6d4..ff43513 100644 --- a/fba/network.py +++ b/fba/network.py @@ -43,7 +43,7 @@ exceptions = ( UnicodeEncodeError ) -def post_json_api(domain: str, path: str, data: str, headers: dict = {}) -> dict: +def post_json_api(domain: str, path: str, data: str = "", headers: dict = {}) -> dict: # DEBUG: print(f"DEBUG: domain='{domain}',path='{path}',data='{data}',headers()={len(headers)} - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") diff --git a/fba/networks/peertube.py b/fba/networks/peertube.py index b6115d5..9949334 100644 --- a/fba/networks/peertube.py +++ b/fba/networks/peertube.py @@ -21,13 +21,13 @@ from fba import network from fba.models import instances def fetch_peers(domain: str) -> list: - print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!") + # DEBUG: print(f"DEBUG: domain({len(domain)})={domain},software='peertube' - CALLED!") if not isinstance(domain, str): raise ValueError(f"Parameter domain[]='{type(domain)}' is not 'str'") elif domain == "": raise ValueError("Parameter 'domain' is empty") - print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") + # DEBUG: print(f"DEBUG: domain='{domain}' is a PeerTube, fetching JSON ...") peers = list() start = 0 @@ -35,14 +35,14 @@ def fetch_peers(domain: str) -> list: headers = tuple() try: - print(f"DEBUG: Checking CSRF for domain='{domain}'") + # DEBUG: print(f"DEBUG: Checking CSRF for domain='{domain}'") headers = csrf.determine(domain, dict()) except network.exceptions as exception: print(f"WARNING: Exception '{type(exception)}' during checking CSRF (fetch_peers,{__name__}) - EXIT!") return peers for mode in ["followers", "following"]: - print(f"DEBUG: domain='{domain}',mode='{mode}'") + # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") while True: data = network.get_json_api( domain, @@ -51,30 +51,35 @@ def fetch_peers(domain: str) -> list: (config.get("connection_timeout"), config.get("read_timeout")) ) - print(f"DEBUG: data['{type(data)}']='{data}'") + # DEBUG: print(f"DEBUG: data['{type(data)}']='{data}'") if "error_message" not in data: - print(f"DEBUG: Success, data[json]()={len(data['json'])}") + # DEBUG: print(f"DEBUG: Success, data[json]()={len(data['json'])}") if "data" in data["json"]: rows = data["json"]["data"] - print(f"DEBUG: Found {len(rows)} record(s).") + # DEBUG: print(f"DEBUG: Found {len(rows)} record(s).") for record in rows: - print(f"DEBUG: record()={len(record)}") - if mode in record and "host" in record[mode]: - print(f"DEBUG: Found host={record[mode]['host']}, adding ...") - peers.append(record[mode]["host"]) - else: - print(f"WARNING: record from '{domain}' has no '{mode}' or 'host' record: {record}") + # DEBUG: print(f"DEBUG: record()={len(record)}") + for mode2 in ["follower", "following" ]: + # DEBUG: print(f"DEBUG: mode2='{mode2}'") + if mode2 in record and "host" in record[mode2]: + # DEBUG: print(f"DEBUG: Found host='{record[mode2]['host']}', adding ...") + peers.append(record[mode2]["host"]) + else: + print(f"WARNING: record from '{domain}' has no '{mode2}' or 'host' record: {record}") if len(rows) < 100: - print(f"DEBUG: Reached end of JSON response, domain='{domain}'") + # DEBUG: print(f"DEBUG: Reached end of JSON response, domain='{domain}'") break # Continue with next row start = start + 100 + else: + print(f"WARNING: domain='{domain}' causes error during API query: '{data['error_message']}' - SKIPPED!") + break - print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") + # DEBUG: print(f"DEBUG: Adding '{len(peers)}' for domain='{domain}'") instances.set_data("total_peers", domain, len(peers)) - print(f"DEBUG: Returning peers[]='{type(peers)}'") + # DEBUG: print(f"DEBUG: Returning peers[]='{type(peers)}'") return peers -- 2.39.5