From 73771207aaf42639e2ab445ce2e2feb85d3125cc Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Tue, 4 Jul 2023 10:39:50 +0200 Subject: [PATCH] Continued: - renamed column apis.hostname to apis.api_domain - added apis.fetch() - check if fetching an URL from fedilist.com was successful --- blocks_empty.db | Bin 40960 -> 40960 bytes fba/commands.py | 10 +++++++--- fba/models/apis.py | 21 ++++++++++++++------- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/blocks_empty.db b/blocks_empty.db index 5e62f7136fdf8970606fda5d5324a40579723296..44712f5b4a21426374320fdf3a2ec2018f391aa8 100644 GIT binary patch delta 100 zcmZoTz|?SnX@a!iUj_ySE+B>hhlx7IjDII4NHbgT?%gaX@R@CMA-f9;W822YCPuDC oCUJIgQBlUm^vQ0#YFvp0nei$4xrv#1lS_DQk+_?8^14p~01U7ky#N3J delta 95 zcmZoTz|?SnX@a!i9|i^nE+B>h`-wWnjDIF3NHg2=?%gaX@R@CMA-f9;WB10!c1F%R ladvT0QO3sj$xgg#92xn=C3%UtsgsL%ZQ-2FJ9*tF0RWKv8%+QJ diff --git a/fba/commands.py b/fba/commands.py index cbfc97a..45bc929 100644 --- a/fba/commands.py +++ b/fba/commands.py @@ -953,7 +953,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int: continue logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content)) - reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix") + reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix") blockdict = list() @@ -1436,7 +1436,7 @@ def fetch_fedilist(args: argparse.Namespace) -> int: locking.acquire() - logger.info("Fetching url='%s' from fedilist.com ...", url) + logger.info("Fetching url='%s' ...", url) response = reqto.get( url, headers=network.web_headers, @@ -1445,7 +1445,11 @@ def fetch_fedilist(args: argparse.Namespace) -> int: ) logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text)) - reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix") + if not response.ok or response.status_code >= 300 or len(response.content) == 0: + logger.warning("Failed fetching url='%s': response.ok='%s',response.status_code=%d,response.content()=%d - EXIT!", response.ok, response.status_code, len(response.text)) + return 1 + + reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix") logger.debug("reader[]='%s'", type(reader)) blockdict = list() diff --git a/fba/models/apis.py b/fba/models/apis.py index 8725e9a..97ff453 100644 --- a/fba/models/apis.py +++ b/fba/models/apis.py @@ -26,14 +26,23 @@ from fba.helpers import domain as domain_helper logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) +def fetch (api_domain: str) -> dict: + logger.debug("api_domain='%s' - CALLED!", api_domain) + domain_helper.raise_on(api_domain) + + database.cursor.execute("SELECT * FROM apis WHERE api_domain = ? LIMIT 1", [api_domain]) + + row = database.cursor.fetchone() + logger.debug("row[]='%s' - EXIT!", type(row)) + return row + def is_recent(api_domain: str) -> bool: logger.debug("api_domain='%s' - CALLED!", api_domain) domain_helper.raise_on(api_domain) is_recent = False - database.cursor.execute("SELECT last_accessed FROM apis WHERE hostname = ? LIMIT 1", [api_domain]) - row = database.cursor.fetchone() + row = fetch(api_domain) logger.debug("row[]='%s'", type(row)) if row is not None: logger.debug("api_domain='%s',row[last_accessed]=%d", api_domain, row["last_accessed"]) @@ -46,19 +55,17 @@ def update (api_domain: str): logger.debug("api_domain='%s' - CALLED!", api_domain) domain_helper.raise_on(api_domain) - database.cursor.execute("SELECT * FROM apis WHERE hostname = ? LIMIT 1", [api_domain]) - - row = database.cursor.fetchone() + row = fetch(api_domain) logger.debug("row[]='%s'", type(row)) if row is None: # Add instance - database.cursor.execute("INSERT INTO apis (hostname, last_accessed) VALUES (?, ?)", [ + database.cursor.execute("INSERT INTO apis (api_domain, last_accessed) VALUES (?, ?)", [ api_domain, time.time() ]) else: # Update last_accessed - database.cursor.execute("UPDATE apis SET last_accessed = ? WHERE hostname = ? LIMIT 1", [ + database.cursor.execute("UPDATE apis SET last_accessed = ? WHERE api_domain = ? LIMIT 1", [ time.time(), api_domain ]) -- 2.39.5