Continued:

- renamed column apis.hostname to apis.api_domain
- added apis.fetch()
- check if fetching an URL from fedilist.com was successful

diff --git a/blocks_empty.db b/blocks_empty.db
index 5e62f7136fdf8970606fda5d5324a40579723296..44712f5b4a21426374320fdf3a2ec2018f391aa8 100644 (file)
Binary files a/blocks_empty.db and b/blocks_empty.db differ

diff --git a/fba/commands.py b/fba/commands.py
index cbfc97ad35904ba553007c96eb383a965b479b73..45bc929aa8bb27fae8defb80fd2175fc3c597ef0 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -953,7 +953,7 @@ def fetch_oliphant(args: argparse.Namespace) -> int:
             continue
 
         logger.debug("Fetched %d Bytes, parsing CSV ...", len(response.content))
-        reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
+        reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix")
 
         blockdict = list()
 
@@ -1436,7 +1436,7 @@ def fetch_fedilist(args: argparse.Namespace) -> int:
 
     locking.acquire()
 
-    logger.info("Fetching url='%s' from fedilist.com ...", url)
+    logger.info("Fetching url='%s' ...", url)
     response = reqto.get(
         url,
         headers=network.web_headers,
@@ -1445,7 +1445,11 @@ def fetch_fedilist(args: argparse.Namespace) -> int:
     )
 
     logger.debug("response.ok='%s',response.status_code=%d,response.text()=%d", response.ok, response.status_code, len(response.text))
-    reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect="unix")
+    if not response.ok or response.status_code >= 300 or len(response.content) == 0:
+        logger.warning("Failed fetching url='%s': response.ok='%s',response.status_code=%d,response.content()=%d - EXIT!", response.ok, response.status_code, len(response.text))
+        return 1
+
+    reader = csv.DictReader(response.content.decode("utf-8").splitlines(), dialect="unix")
 
     logger.debug("reader[]='%s'", type(reader))
     blockdict = list()

diff --git a/fba/models/apis.py b/fba/models/apis.py
index 8725e9a125730136e94d5b05247043dcf0f0fd83..97ff4539d28e5f939f54a3ad71eaf25bf9085b8a 100644 (file)
--- a/fba/models/apis.py
+++ b/fba/models/apis.py
@@ -26,14 +26,23 @@ from fba.helpers import domain as domain_helper
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
+def fetch (api_domain: str) -> dict:
+    logger.debug("api_domain='%s' - CALLED!", api_domain)
+    domain_helper.raise_on(api_domain)
+
+    database.cursor.execute("SELECT * FROM apis WHERE api_domain = ? LIMIT 1", [api_domain])
+
+    row = database.cursor.fetchone()
+    logger.debug("row[]='%s' - EXIT!", type(row))
+    return row
+
 def is_recent(api_domain: str) -> bool:
     logger.debug("api_domain='%s' - CALLED!", api_domain)
     domain_helper.raise_on(api_domain)
 
     is_recent = False
-    database.cursor.execute("SELECT last_accessed FROM apis WHERE hostname = ? LIMIT 1", [api_domain])
 
-    row = database.cursor.fetchone()
+    row = fetch(api_domain)
     logger.debug("row[]='%s'", type(row))
     if row is not None:
         logger.debug("api_domain='%s',row[last_accessed]=%d", api_domain, row["last_accessed"])
@@ -46,19 +55,17 @@ def update (api_domain: str):
     logger.debug("api_domain='%s' - CALLED!", api_domain)
     domain_helper.raise_on(api_domain)
 
-    database.cursor.execute("SELECT * FROM apis WHERE hostname = ? LIMIT 1", [api_domain])
-
-    row = database.cursor.fetchone()
+    row = fetch(api_domain)
     logger.debug("row[]='%s'", type(row))
     if row is None:
         # Add instance
-        database.cursor.execute("INSERT INTO apis (hostname, last_accessed) VALUES (?, ?)", [
+        database.cursor.execute("INSERT INTO apis (api_domain, last_accessed) VALUES (?, ?)", [
             api_domain,
             time.time()
         ])
     else:
         # Update last_accessed
-        database.cursor.execute("UPDATE apis SET last_accessed = ? WHERE hostname = ? LIMIT 1", [
+        database.cursor.execute("UPDATE apis SET last_accessed = ? WHERE api_domain = ? LIMIT 1", [
             time.time(),
             api_domain
         ])