From: Roland Häder <roland@mxchange.org>
Date: Tue, 4 Jul 2023 08:39:50 +0000 (+0200)
Subject: Continued:
X-Git-Url: https://git.mxchange.org/?a=commitdiff_plain;h=73771207aaf42639e2ab445ce2e2feb85d3125cc;p=fba.git

Continued:
- renamed column apis.hostname to apis.api_domain
- added apis.fetch()
- check if fetching an URL from fedilist.com was successful
---

diff --git a/blocks_empty.db b/blocks_empty.db
index 5e62f71..44712f5 100644
Binary files a/blocks_empty.db and b/blocks_empty.db differ
diff --git a/fba/commands.py b/fba/commands.py
index cbfc97a..45bc929 100644
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -953,7 +953,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
             continue
 
         logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content))
-        reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
+        reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix")
 
         blockdict = list()
 
@@ -1436,7 +1436,7 @@ def fetch_fedilist(args: argparse.Namespace) -> int:
 
     locking.acquire()
 
-    logger.info("Fetching url='%s' from fedilist.com ...", url)
+    logger.info("Fetching url='%s' ...", url)
     response = reqto.get(
         url,
         headers=network.web_headers,
@@ -1445,7 +1445,11 @@ def fetch_fedilist(args: argparse.Namespace) -> int:
     )
 
     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
-    reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
+    if not response.ok or response.status_code >= 300 or len(response.content) == 0:
+        logger.warning("Failed fetching url='%s': response.ok='%s',response.status_code=%d,response.content()=%d - EXIT!", response.ok, response.status_code, len(response.text))
+        return 1
+
+    reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix")
 
     logger.debug("reader[]='%s'", type(reader))
     blockdict = list()
diff --git a/fba/models/apis.py b/fba/models/apis.py
index 8725e9a..97ff453 100644
--- a/fba/models/apis.py
+++ b/fba/models/apis.py
@@ -26,14 +26,23 @@ from fba.helpers import domain as domain_helper
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+def fetch (api_domain: str) -> dict:
+    logger.debug("api_domain='%s' - CALLED!", api_domain)
+    domain_helper.raise_on(api_domain)
+
+    database.cursor.execute("SELECT * FROM apis WHERE api_domain = ? LIMIT 1", [api_domain])
+
+    row = database.cursor.fetchone()
+    logger.debug("row[]='%s' - EXIT!", type(row))
+    return row
+
 def is_recent(api_domain: str) -> bool:
     logger.debug("api_domain='%s' - CALLED!", api_domain)
     domain_helper.raise_on(api_domain)
 
     is_recent = False
-    database.cursor.execute("SELECT last_accessed FROM apis WHERE hostname = ? LIMIT 1", [api_domain])
 
-    row = database.cursor.fetchone()
+    row = fetch(api_domain)
     logger.debug("row[]='%s'", type(row))
     if row is not None:
         logger.debug("api_domain='%s',row[last_accessed]=%d", api_domain, row["last_accessed"])
@@ -46,19 +55,17 @@ def update (api_domain: str):
     logger.debug("api_domain='%s' - CALLED!", api_domain)
     domain_helper.raise_on(api_domain)
 
-    database.cursor.execute("SELECT * FROM apis WHERE hostname = ? LIMIT 1", [api_domain])
-
-    row = database.cursor.fetchone()
+    row = fetch(api_domain)
     logger.debug("row[]='%s'", type(row))
     if row is None:
         # Add instance
-        database.cursor.execute("INSERT INTO apis (hostname, last_accessed) VALUES (?, ?)", [
+        database.cursor.execute("INSERT INTO apis (api_domain, last_accessed) VALUES (?, ?)", [
             api_domain,
             time.time()
         ])
     else:
         # Update last_accessed
-        database.cursor.execute("UPDATE apis SET last_accessed = ? WHERE hostname = ? LIMIT 1", [
+        database.cursor.execute("UPDATE apis SET last_accessed = ? WHERE api_domain = ? LIMIT 1", [
             time.time(),
             api_domain
         ])