]> git.mxchange.org Git - fba.git/blobdiff - fba/commands.py
projects / fba.git / blobdiff
? search:
summary | shortlog | log | commit | commitdiff | tree
raw | inline | side by side
Continued:
[fba.git] / fba / commands.py
diff --git a/fba/commands.py b/fba/commands.py
index 3f5b4354eb7e161e1330e8d23370e6cf3f7b6cd4..e746deeb6cc2e7d35e00918c28842ad2395773c0 100644 (file)
--- a/fba/commands.py
+++ b/fba/commands.py
@@ -24,6 +24,8 @@ import argparse
 import atoma
 import bs4
 import markdown
+import reqto
+import requests
 import validators
 
 from fba import blacklist
@@ -61,7 +63,7 @@ def check_instance(args: argparse.Namespace) -> int:
     return status
 
 def fetch_bkali(args: argparse.Namespace):
-    # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+    # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
     domains = list()
     try:
         fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
@@ -94,7 +96,7 @@ def fetch_bkali(args: argparse.Namespace):
             # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
             domains.append(entry["domain"])
 
-    except BaseException as exception:
+    except network.exceptions as exception:
         print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}'")
         sys.exit(255)
 
@@ -104,13 +106,17 @@ def fetch_bkali(args: argparse.Namespace):
 
         print(f"INFO: Adding {len(domains)} new instances ...")
         for domain in domains:
-            print(f"INFO: Fetching instances from domain='{domain}' ...")
-            federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            try:
+                print(f"INFO: Fetching instances from domain='{domain}' ...")
+                federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            except network.exceptions as exception:
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
 
 def fetch_blocks(args: argparse.Namespace):
-    # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+    # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
     if args.domain is not None and args.domain != "":
         # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - checking ...")
         if not validators.domain(args.domain):
@@ -162,13 +168,15 @@ def fetch_blocks(args: argparse.Namespace):
             mastodon.fetch_blocks(blocker, origin, nodeinfo_url)
         elif software == "friendica" or software == "misskey":
             print(f"INFO: blocker='{blocker}',software='{software}'")
+
+            blocking = list()
             if software == "friendica":
-                blocks = friendica.fetch_blocks(blocker)
+                blocking = friendica.fetch_blocks(blocker)
             elif software == "misskey":
-                blocks = misskey.fetch_blocks(blocker)
+                blocking = misskey.fetch_blocks(blocker)
 
-            print(f"INFO: Checking {len(blocks.items())} entries from blocker='{blocker}',software='{software}' ...")
-            for block_level, blocklist in blocks.items():
+            print(f"INFO: Checking {len(blocking.items())} entries from blocker='{blocker}',software='{software}' ...")
+            for block_level, blocklist in blocking.items():
                 # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist))
                 block_level = tidyup.domain(block_level)
                 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
@@ -198,12 +206,13 @@ def fetch_blocks(args: argparse.Namespace):
 
                         searchres = fba.cursor.fetchone()
 
+                        # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
                         if searchres is None:
                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
                             continue
 
-                        blocked = searchres[0]
-                        origin = searchres[1]
+                        blocked      = searchres[0]
+                        origin       = searchres[1]
                         nodeinfo_url = searchres[2]
                     elif blocked.count("?") > 0:
                         # Some obscure them with question marks, not sure if that's dependent on version or not
@@ -213,12 +222,13 @@ def fetch_blocks(args: argparse.Namespace):
 
                         searchres = fba.cursor.fetchone()
 
+                        # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
                         if searchres is None:
                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
                             continue
 
-                        blocked = searchres[0]
-                        origin = searchres[1]
+                        blocked      = searchres[0]
+                        origin       = searchres[1]
                         nodeinfo_url = searchres[2]
                     elif not validators.domain(blocked):
                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
@@ -256,7 +266,7 @@ def fetch_blocks(args: argparse.Namespace):
     # DEBUG: print("DEBUG: EXIT!")
 
 def fetch_cs(args: argparse.Namespace):
-    # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+    # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
     extensions = [
         'extra',
         'abbr',
@@ -283,17 +293,17 @@ def fetch_cs(args: argparse.Namespace):
     }
 
     raw = fba.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
-    # DEBUG: print(f"DEBUG: raw()={len(raw)}[]={type(raw)}")
+    # DEBUG: print(f"DEBUG: raw()={len(raw)}[]='{type(raw)}'")
 
     doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser')
 
-    # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}")
+    # DEBUG: print(f"DEBUG: doc()={len(doc)}[]='{type(doc)}'")
     silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody")
-    # DEBUG: print(f"DEBUG: silenced[]={type(silenced)}")
+    # DEBUG: print(f"DEBUG: silenced[]='{type(silenced)}'")
     domains["silenced"] = domains["silenced"] + federation.find_domains(silenced)
 
     blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody")
-    # DEBUG: print(f"DEBUG: blocked[]={type(blocked)}")
+    # DEBUG: print(f"DEBUG: blocked[]='{type(blocked)}'")
     domains["reject"] = domains["reject"] + federation.find_domains(blocked)
 
     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
@@ -311,8 +321,12 @@ def fetch_cs(args: argparse.Namespace):
                     blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level)
 
                 if not instances.is_registered(row["domain"]):
-                    print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
-                    federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
+                    try:
+                        print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
+                        federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
+                    except network.exceptions as exception:
+                        print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'")
+                        instances.update_last_error(row["domain"], exception)
 
         # DEBUG: print("DEBUG: Committing changes ...")
         fba.connection.commit()
@@ -320,7 +334,7 @@ def fetch_cs(args: argparse.Namespace):
     # DEBUG: print("DEBUG: EXIT!")
 
 def fetch_fba_rss(args: argparse.Namespace):
-    # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+    # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
     domains = list()
 
     print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...")
@@ -331,7 +345,7 @@ def fetch_fba_rss(args: argparse.Namespace):
         # DEBUG: print(f"DEBUG: Parsing RSS feed ({len(response.text)} Bytes) ...")
         rss = atoma.parse_rss_bytes(response.content)
 
-        # DEBUG: print(f"DEBUG: rss[]={type(rss)}")
+        # DEBUG: print(f"DEBUG: rss[]='{type(rss)}'")
         for item in rss.items:
             # DEBUG: print(f"DEBUG: item={item}")
             domain = item.link.split("=")[1]
@@ -355,13 +369,17 @@ def fetch_fba_rss(args: argparse.Namespace):
 
         print(f"INFO: Adding {len(domains)} new instances ...")
         for domain in domains:
-            print(f"INFO: Fetching instances from domain='{domain}' ...")
-            federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            try:
+                print(f"INFO: Fetching instances from domain='{domain}' ...")
+                federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            except network.exceptions as exception:
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
 
 def fetch_fbabot_atom(args: argparse.Namespace):
-    # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+    # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
     feed = "https://ryona.agency/users/fba/feed.atom"
 
     domains = list()
@@ -374,11 +392,11 @@ def fetch_fbabot_atom(args: argparse.Namespace):
         # DEBUG: print(f"DEBUG: Parsing ATOM feed ({len(response.text)} Bytes) ...")
         atom = atoma.parse_atom_bytes(response.content)
 
-        # DEBUG: print(f"DEBUG: atom[]={type(atom)}")
+        # DEBUG: print(f"DEBUG: atom[]='{type(atom)}'")
         for entry in atom.entries:
-            # DEBUG: print(f"DEBUG: entry[]={type(entry)}")
+            # DEBUG: print(f"DEBUG: entry[]='{type(entry)}'")
             doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
-            # DEBUG: print(f"DEBUG: doc[]={type(doc)}")
+            # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'")
             for element in doc.findAll("a"):
                 for href in element["href"].split(","):
                     # DEBUG: print(f"DEBUG: href[{type(href)}]={href}")
@@ -404,17 +422,27 @@ def fetch_fbabot_atom(args: argparse.Namespace):
 
         print(f"INFO: Adding {len(domains)} new instances ...")
         for domain in domains:
-            print(f"INFO: Fetching instances from domain='{domain}' ...")
-            federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            try:
+                print(f"INFO: Fetching instances from domain='{domain}' ...")
+                federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
+            except network.exceptions as exception:
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
+                instances.update_last_error(domain, exception)
 
     # DEBUG: print("DEBUG: EXIT!")
 
 def fetch_instances(args: argparse.Namespace):
-    # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+    # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
     locking.acquire()
 
     # Initial fetch
-    federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
+    try:
+        print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
+        federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
+    except network.exceptions as exception:
+        print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'")
+        instances.update_last_error(args.domain, exception)
+        return
 
     if args.single:
         # DEBUG: print("DEBUG: Not fetching more instances - EXIT!")
@@ -433,13 +461,17 @@ def fetch_instances(args: argparse.Namespace):
             print("WARNING: domain is blacklisted:", row[0])
             continue
 
-        print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
-        federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
+        try:
+            print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
+            federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
+        except network.exceptions as exception:
+            print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'")
+            instances.update_last_error(row[0], exception)
 
     # DEBUG: print("DEBUG: EXIT!")
 
 def fetch_federater(args: argparse.Namespace):
-    # DEBUG: print(f"DEBUG: args[]={type(args)} - CALLED!")
+    # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
     locking.acquire()
 
     # Fetch this URL
@@ -450,7 +482,7 @@ def fetch_federater(args: argparse.Namespace):
         ## DEBUG: print(f"DEBUG: response.content={response.content}")
         reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect='unix')
         #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate'
-        # DEBUG: print(f"DEBUG: reader[]={type(reader)}")
+        # DEBUG: print(f"DEBUG: reader[]='{type(reader)}'")
         for row in reader:
             if not validators.domain(row["#domain"]):
                 print(f"WARNING: domain='{row['#domain']}' is not a valid domain - skipped!")
@@ -462,7 +494,11 @@ def fetch_federater(args: argparse.Namespace):
                 # DEBUG: print(f"DEBUG: domain='{row['#domain']}' is already registered - skipped!")
                 continue
 
-            print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
-            federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
+            try:
+                print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
+                federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
+            except network.exceptions as exception:
+                print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'")
+                instances.update_last_error(row["#domain"], exception)
 
     # DEBUG: print("DEBUG: EXIT!")
Personal clone of "fedi-block-api", if you like to use the command fetch_instances_social you need to get an API key from https://instances.social .