Continued:

- moved acquiring lock to a place when a domain is supposed to be
  added
- added missing fba.tidyup_reason()

diff --git a/fba/fba.py b/fba/fba.py
index 5350a0e6df19be181e7da26d6e477d62372dc857..a7b1b38c88659c12e23f9b73de83d4ab2716ad38 100644 (file)
--- a/fba/fba.py
+++ b/fba/fba.py
@@ -1432,6 +1432,20 @@ def get_misskey_blocks(domain: str) -> dict:
         "followers_only": blocks["suspended"]
     }
 
+def tidyup_reason(reason: str) -> str:
+    # DEBUG: print(f"DEBUG: reason='{reason}' - CALLED!")
+    if type(reason) != str:
+        raise ValueError(f"Parameter reason[]={type(reason)} is not expected")
+
+    # Strip string
+    reason = reason.strip()
+
+    # Replace â with "
+    reason = re.sub("â", "\"", reason)
+
+    #print(f"DEBUG: reason='{reason}' - EXIT!")
+    return reason
+
 def tidyup_domain(domain: str) -> str:
     # DEBUG: print(f"DEBUG: domain='{domain}' - CALLED!")
     if type(domain) != str:

diff --git a/fetch_bkali.py b/fetch_bkali.py
index 0dbefc0d6f905b350617b3185ae9241c4fa0b96e..1a1aafcee5aa99ea942d704ede3be2d781452c81 100755 (executable)
--- a/fetch_bkali.py
+++ b/fetch_bkali.py
@@ -22,8 +22,6 @@ import sys
 import validators
 from fba import *
 
-boot.acquire_lock()
-
 domains = list()
 try:
     fetched = fba.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
@@ -60,9 +58,10 @@ except BaseException as e:
     print(f"ERROR: Cannot fetch graphql,exception[{type(e)}]:'{str(e)}'")
     sys.exit(255)
 
-# Show domains
 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 if len(domains) > 0:
+    boot.acquire_lock()
+
     print(f"INFO: Adding {len(domains)} new instances ...")
     for domain in domains:
         print(f"INFO: Fetching instances from domain='{domain}' ...")

diff --git a/fetch_cs.py b/fetch_cs.py
index 20d0c563efb31286e38b35a83f2df1f42288f06e..ec7b6de180d90ed80404dea321b652155ffdb019 100755 (executable)
--- a/fetch_cs.py
+++ b/fetch_cs.py
@@ -76,8 +76,6 @@ def find_domains(tag: bs4.element.Tag) -> list:
     # DEBUG: print(f"DEBUG: domains()={len(domains)} - EXIT!")
     return domains
 
-boot.acquire_lock()
-
 domains = {
     "silenced": list(),
     "blocked": list(),
@@ -102,9 +100,10 @@ except BaseException as e:
     print(f"ERROR: Cannot fetch from meta.chaos.social,exception[{type(e)}]:'{str(e)}'")
     sys.exit(255)
 
-# Show domains
 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 if len(domains) > 0:
+    boot.acquire_lock()
+
     print(f"INFO: Adding {len(domains)} new instances ...")
     for block_level in domains:
         # DEBUG: print(f"DEBUG: block_level='{block_level}'")

diff --git a/fetch_fba_rss.py b/fetch_fba_rss.py
index c7c28e6d0f6102cfd1cfed1917ac0e636bd8d5e9..614b10e885510ddcd1ce078578fdbbd7ec2c2050 100755 (executable)
--- a/fetch_fba_rss.py
+++ b/fetch_fba_rss.py
@@ -18,12 +18,10 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 import reqto
-import rss_parser
+import atoma
 import sys
 from fba import *
 
-boot.acquire_lock()
-
 feed = sys.argv[1]
 
 domains = list()
@@ -34,9 +32,11 @@ try:
     # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
     if response.ok and response.status_code < 300 and len(response.text) > 0:
         # DEBUG: print(f"DEBUG: Parsing RSS feed ...")
-        rss = rss_parser.Parser.parse(response.text)
-        for item in rss.channel.items:
-            # DEBUG: print(f"DEBUG: item.link={item.link}")
+        rss = atoma.parse_rss_bytes(response.content)
+
+        # DEBUG: print(f"DEBUG: rss[]={type(rss)}")
+        for item in rss.items:
+            # DEBUG: print(f"DEBUG: item={item}")
             domain = item.link.split("=")[1]
 
             if fba.is_blacklisted(domain):
@@ -55,9 +55,10 @@ try:
 except BaseException as e:
     print(f"ERROR: Cannot fetch feed='{feed}',exception[{type(e)}]:'{str(e)}'")
 
-# Show domains
 # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 if len(domains) > 0:
+    boot.acquire_lock()
+
     print(f"INFO: Adding {len(domains)} new instances ...")
     for domain in domains:
         print(f"INFO: Fetching instances from domain='{domain}' ...")

diff --git a/requirements.txt b/requirements.txt
index 8e233a0de7f8323f0339b63e3e9f303bce11cc36..a02a02ac25248bd92c1d17fbdbdca306621beafd 100644 (file)
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+atoma
 beautifulsoup4
 fastapi
 uvicorn
@@ -5,6 +6,5 @@ requests
 jinja2
 eventlet
 reqto
-rss-parser
 validators
 zc.lockfile