X-Git-Url: https://git.mxchange.org/?a=blobdiff_plain;f=fba%2Fboot.py;h=2ffdf16a4b44e2a4f0dfd72f4dce0382d5ab9a99;hb=4c84e7a6e350f7ed3ef676ba054f44dbf26ec3ed;hp=ed80085c1b2f581bb28b0e372e559267f350c156;hpb=e3ff9507f9037f860d83cf7d00310efb18b64fae;p=fba.git

diff --git a/fba/boot.py b/fba/boot.py
index ed80085..2ffdf16 100644
--- a/fba/boot.py
+++ b/fba/boot.py
@@ -14,24 +14,41 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import logging
+
 import argparse
 
 from fba import commands
-from fba import fba
+from fba import database
 
 from fba.helpers import locking
 
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Argument parser
 _PARSER = None
 
 def init_parser():
-    # DEBUG: print("DEBUG: init_parser(): CALLED!")
+    logger.debug("CALLED!")
     global _PARSER
 
-    # DEBUG: print("DEBUG: Initializing parser ...")
+    logger.debug("Initializing parser ...")
     _PARSER = argparse.ArgumentParser(
         description="Fetches block reasons from the fediverse",
-        epilog="Please note that some commands have optional arguments, you may want to try fba.py <command> --help to find them out.",
+        epilog="Please note that some commands have optional arguments, you may want to try fba.py <command> --help to find them out. Please DO NOT overdose requests that are not limited by themselves. Typically parameters like --domain, --software and --force are unlimited. \"Unlimited\" here means that there is no \"is recently accessed?\" limitation.",
     )
+
+    # Generic:
+    _PARSER.add_argument(
+        "--debug",
+        action="store_const",
+        dest="log_level",
+        const=logging.DEBUG,
+        help="Full debug output"
+    )
+
+    # Commands:
     subparser_command = _PARSER.add_subparsers(
         dest="command",
         title="Commands to execute",
@@ -44,8 +61,8 @@ def init_parser():
         "check_instance",
         help="Checks given instance if it exists and returns proper exit code"
     )
-    parser.add_argument("--domain", required=True, help="Instance name (aka. domain) to check")
     parser.set_defaults(command=commands.check_instance)
+    parser.add_argument("--domain", required=True, help="Instance name (aka. domain) to check")
 
     ### Fetch from bka.li ###
     parser = subparser_command.add_parser(
@@ -54,14 +71,26 @@ def init_parser():
     )
     parser.set_defaults(command=commands.fetch_bkali)
 
+    ### Recheck instance's obfuscated peers ###
+    parser = subparser_command.add_parser(
+        "recheck_obfuscation",
+        help="Checks all instance's obfuscated peers if they can be de-obfuscated now.",
+    )
+    parser.set_defaults(command=commands.recheck_obfuscation)
+    parser.add_argument("--domain", help="Instance name (aka. domain)")
+    parser.add_argument("--software", help="Name of software, e.g. 'lemmy'")
+    parser.add_argument("--force", action="store_true", help="Include also already existing instances, otherwise only new are checked")
+
     ### Fetch blocks from registered instances or given ###
     parser = subparser_command.add_parser(
         "fetch_blocks",
         help="Fetches blocks from registered instances (run command fetch_instances first!).",
     )
+    parser.set_defaults(command=commands.fetch_blocks)
     parser.add_argument("--domain", help="Instance name (aka. domain)")
     parser.add_argument("--software", help="Name of software, e.g. 'lemmy'")
-    parser.set_defaults(command=commands.fetch_blocks)
+    parser.add_argument("--only-none", action="store_true", help="Checks only entries which has never been checked.")
+    parser.add_argument("--force", action="store_true", help="Forces update of data, no matter what.")
 
     ### Fetch blocks from chaos.social ###
     parser = subparser_command.add_parser(
@@ -70,37 +99,53 @@ def init_parser():
     )
     parser.set_defaults(command=commands.fetch_cs)
 
+    ### Fetch blocks from todon.eu wiki ###
+    parser = subparser_command.add_parser(
+        "fetch_todon_wiki",
+        help="Fetches blocks from todon.eu's wiki.",
+    )
+    parser.set_defaults(command=commands.fetch_todon_wiki)
+
     ### Fetch blocks from a FBA-specific RSS feed  ###
     parser = subparser_command.add_parser(
         "fetch_fba_rss",
         help="Fetches domains from a FBA-specific RSS feed.",
     )
-    parser.add_argument("--feed", required=True, help="RSS feed to fetch domains from (e.g. https://fba.ryoma.agency/rss?domain=foo.bar).")
     parser.set_defaults(command=commands.fetch_fba_rss)
+    parser.add_argument("--feed", required=True, help="RSS feed to fetch domains from (e.g. https://fba.ryoma.agency/rss?domain=foo.bar).")
 
     ### Fetch blocks from FBA's bot account ###
     parser = subparser_command.add_parser(
         "fetch_fbabot_atom",
-        help="Fetches ATOM feed with domains from FBA's bot account. You may wish to re-run this command several times (at least 3 with big instances) to have a decent amount of valid instances.",
+        help="Fetches ATOM feed with domains from FBA's bot account.",
     )
     parser.set_defaults(command=commands.fetch_fbabot_atom)
+    parser.add_argument("--feed", required=True, help="RSS feed to fetch domains from (e.g. https://fba.ryoma.agency/rss?domain=foo.bar).")
 
     ### Fetch blocks from oliphant's GIT repository ###
     parser = subparser_command.add_parser(
         "fetch_oliphant",
+        help="Fetches CSV files from GIT generated by Oliphant 'member instances'.",
+    )
+    parser.set_defaults(command=commands.fetch_oliphant)
+    parser.add_argument("--domain", help="Instance name (aka. domain) to check")
+
+    ### Fetch blocks from other CSV files
+    parser = subparser_command.add_parser(
+        "fetch_csv",
         help="Fetches CSV files (block recommendations) for more possible instances to disover",
     )
+    parser.set_defaults(command=commands.fetch_csv)
     parser.add_argument("--domain", help="Instance name (aka. domain) to check")
-    parser.set_defaults(command=commands.fetch_oliphant)
 
     ### Fetch instances from given initial instance ###
     parser = subparser_command.add_parser(
         "fetch_instances",
-        help="Fetches instances (aka. \"domains\") from an initial instance.",
+        help="Fetches instances (aka. \"domains\") from an initial instance. You may want to re-run this command several times (at least 3 with big instances) to have a decent amount of valid instances.",
     )
+    parser.set_defaults(command=commands.fetch_instances)
     parser.add_argument("--domain", required=True, help="Instance name (aka. domain) to fetch further instances from. Start with a large instance, e.g. mastodon.social .")
     parser.add_argument("--single", action="store_true", help="Only fetch given instance.")
-    parser.set_defaults(command=commands.fetch_instances)
 
     ### Fetch blocks from static text file(s) ###
     parser = subparser_command.add_parser(
@@ -109,25 +154,137 @@ def init_parser():
     )
     parser.set_defaults(command=commands.fetch_txt)
 
+    ### Fetch blocks from joinfediverse.wiki ###
+    #parser = subparser_command.add_parser(
+    #    "fetch_joinfediverse",
+    #    help="Fetches FediBlock page from joinfediverse.wiki",
+    #)
+    #parser.set_defaults(command=commands.fetch_joinfediverse)
+
+    ### Fetch instances JSON from instances.joinmobilizon.org
+    parser = subparser_command.add_parser(
+        "fetch_joinmobilizon",
+        help="Fetches instances from joinmobilizon",
+    )
+    parser.set_defaults(command=commands.fetch_joinmobilizon)
+
+    ### Fetch blocks from misskey.page ###
+    parser = subparser_command.add_parser(
+        "fetch_joinmisskey",
+        help="Fetches instances.json from misskey.page",
+    )
+    parser.set_defaults(command=commands.fetch_joinmisskey)
+
     ### Fetch blocks from fediverse.observer ###
     parser = subparser_command.add_parser(
         "fetch_observer",
         help="Fetches blocks from fediverse.observer.",
     )
     parser.set_defaults(command=commands.fetch_observer)
+    parser.add_argument("--software", help="Name of software, e.g. 'lemmy'")
 
-    # DEBUG: print("DEBUG: init_parser(): EXIT!")
+    ### Fetch instances from fedipact.online ###
+    parser = subparser_command.add_parser(
+        "fetch_fedipact",
+        help="Fetches blocks from fedipact.online.",
+    )
+    parser.set_defaults(command=commands.fetch_fedipact)
+
+    ### Fetch from pixelfed.org's API ###
+    parser = subparser_command.add_parser(
+        "fetch_pixelfed_api",
+        help="Fetches domain names from pixelfed.org's API",
+    )
+    parser.set_defaults(command=commands.fetch_pixelfed_api)
+
+    ### Check nodeinfo ###
+    parser = subparser_command.add_parser(
+        "check_nodeinfo",
+        help="Checks if domain is part of nodeinfo.",
+    )
+    parser.set_defaults(command=commands.check_nodeinfo)
+
+    ### Fetch CSV from fedilist.com ###
+    parser = subparser_command.add_parser(
+        "fetch_fedilist",
+        help="Fetches CSV from fedilist.com",
+    )
+    parser.set_defaults(command=commands.fetch_fedilist)
+    parser.add_argument("--software", help="Name of software, e.g. 'lemmy'")
+    parser.add_argument("--force", action="store_true", help="Include also already existing instances, otherwise only new are checked")
+
+    ### Update nodeinfo ###
+    parser = subparser_command.add_parser(
+        "update_nodeinfo",
+        help="Updates nodeinfo for all instances",
+    )
+    parser.set_defaults(command=commands.update_nodeinfo)
+    parser.add_argument("--domain", help="Instance name (aka. domain)")
+    parser.add_argument("--software", help="Name of software, e.g. 'lemmy'")
+    parser.add_argument("--mode", help="Name of detection mode, e.g. 'auto_discovery'")
+    parser.add_argument("--force", action="store_true", help="Forces update of data, no matter what.")
+    parser.add_argument("--no-software", action="store_true", help="Checks only entries with no software type detected.")
+    parser.add_argument("--no-auto", action="store_true", help="Checks only entries with other than AUTO_DISCOVERY as detection mode.")
+    parser.add_argument("--no-detection", action="store_true", help="Checks only entries with no detection mode set.")
+    parser.add_argument("--with-software", action="store_true", help="Checks only entries with any software type detected.")
+
+    ### Fetch instances from instances.social ###
+    parser = subparser_command.add_parser(
+        "fetch_instances_social",
+        help="Fetch instances from instances.social, you need an API key to access the API. Please consider donating to them when you want to more frequent use their API!",
+    )
+    parser.set_defaults(command=commands.fetch_instances_social)
+
+    ### Convert international domain names to punycode domains ###
+    parser = subparser_command.add_parser(
+        "convert_idna",
+        help="Convertes UTF-8 encoded international domain names into punycode (IDNA) domain names.",
+    )
+    parser.set_defaults(command=commands.convert_idna)
+
+    ### Fetch instances from ActivityPub relays ###
+    parser = subparser_command.add_parser(
+        "fetch_relays",
+        help="Fetches instances from ActivityPub relays",
+    )
+    parser.set_defaults(command=commands.fetch_relays)
+    parser.add_argument("--domain", help="Instance name (aka. 'relay')")
+    parser.add_argument("--software", help="Name of software, e.g. 'lemmy'")
+    parser.add_argument("--force", action="store_true", help="Forces update of data, no matter what.")
+
+    ### Fetches relay list from relaylist.com
+    parser = subparser_command.add_parser(
+        "fetch_relaylist",
+        help="Fetches relay list from relaylist.com",
+    )
+    parser.set_defaults(command=commands.fetch_relaylist)
+
+    ### Remove invalid domains ###
+    parser = subparser_command.add_parser(
+        "remove_invalid",
+        help="Removes invalid domains.",
+    )
+    parser.set_defaults(command=commands.remove_invalid)
+
+    logger.debug("EXIT!")
 
 def run_command():
-    # DEBUG: print("DEBUG: run_command(): CALLED!")
+    logger.debug("CALLED!")
     args = _PARSER.parse_args()
-    # DEBUG: print(f"DEBUG: args[{type(args)}]={args}")
+
+    if args.log_level is not None:
+        loggers = [logging.getLogger(name) for name in logging.root.manager.loggerDict]
+        for _logger in loggers:
+            _logger.setLevel(args.log_level)
+
+    logger.debug("args[%s]='%s'", type(args), args)
     status = args.command(args)
-    # DEBUG: print("DEBUG: status={status} - EXIT!")
-    return status if isinstance(status, int) else 0
+
+    logger.debug("status=%d - EXIT!", status)
+    return status
 
 def shutdown():
-    # DEBUG: print("DEBUG: Closing database connection ...")
-    fba.connection.close()
+    logger.debug("Closing database connection ...")
+    database.connection.close()
     locking.release()
-    # DEBUG: print("DEBUG: Shutdown completed.")
+    logger.debug("Shutdown completed.")