From 8b62df9449878dc64f591faa23ee73d3eda7ac17 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Roland=20H=C3=A4der?= Date: Sun, 4 Jun 2023 06:32:58 +0200 Subject: [PATCH] Continued: - move config-related stuff to new module 'config' --- api.py | 46 +++++++++++++++++------------------ check_instance.py | 1 - fba/__init__.py | 2 +- fba/config.py | 20 ++++++++++++++++ fba/fba.py | 60 ++++++++++++++++++++++------------------------ fetch_blocks.py | 10 ++++---- fetch_cs.py | 2 +- fetch_fba_rss.py | 2 +- fetch_instances.py | 2 +- 9 files changed, 81 insertions(+), 64 deletions(-) create mode 100644 fba/config.py diff --git a/api.py b/api.py index d89cf3f..129fdbe 100644 --- a/api.py +++ b/api.py @@ -27,10 +27,10 @@ import requests import re from fba import * -router = fastapi.FastAPI(docs_url=fba.config["base_url"] + "/docs", redoc_url=fba.config["base_url"] + "/redoc") +router = fastapi.FastAPI(docs_url=config.config["base_url"] + "/docs", redoc_url=config.config["base_url"] + "/redoc") templates = Jinja2Templates(directory="templates") -@router.get(fba.config["base_url"] + "/api/info.json", response_class=JSONResponse) +@router.get(config.config["base_url"] + "/api/info.json", response_class=JSONResponse) def info(): fba.cursor.execute("SELECT (SELECT COUNT(domain) FROM instances), (SELECT COUNT(domain) FROM instances WHERE software IN ('pleroma', 'mastodon', 'misskey', 'gotosocial', 'friendica', 'bookwyrm', 'takahe', 'peertube')), (SELECT COUNT(blocker) FROM blocks), (SELECT COUNT(domain) FROM instances WHERE last_status_code IS NOT NULL)") known, indexed, blocks, errorous = fba.cursor.fetchone() @@ -40,10 +40,10 @@ def info(): "indexed_instances" : indexed, "blocks_recorded" : blocks, "errorous_instances": errorous, - "slogan" : fba.config["slogan"] + "slogan" : config.config["slogan"] } -@router.get(fba.config["base_url"] + "/api/top.json", response_class=JSONResponse) +@router.get(config.config["base_url"] + "/api/top.json", response_class=JSONResponse) def top(blocked: int = None, blockers: int = None, reference: int = None, software: int = None, originator: int = None): if blocked != None: if blocked > 500: @@ -80,7 +80,7 @@ def top(blocked: int = None, blockers: int = None, reference: int = None, softwa return scoreboard -@router.get(fba.config["base_url"] + "/api/index.json", response_class=JSONResponse) +@router.get(config.config["base_url"] + "/api/index.json", response_class=JSONResponse) def blocked(domain: str = None, reason: str = None, reverse: str = None): if domain == None and reason == None and reverse == None: raise HTTPException(status_code=400, detail="No filter specified") @@ -118,7 +118,7 @@ def blocked(domain: str = None, reason: str = None, reverse: str = None): return result -@router.get(fba.config["base_url"] + "/api/mutual.json", response_class=JSONResponse) +@router.get(config.config["base_url"] + "/api/mutual.json", response_class=JSONResponse) def mutual(domains: list[str] = Query()): """Return 200 if federation is open between the two, 4xx otherwise""" fba.cursor.execute( @@ -142,20 +142,20 @@ def mutual(domains: list[str] = Query()): # No known blocks return JSONResponse(status_code=200, content={}) -@router.get(fba.config["base_url"] + "/scoreboard") +@router.get(config.config["base_url"] + "/scoreboard") def index(request: Request, blockers: int = None, blocked: int = None, reference: int = None, software: int = None, originator: int = None): scores = None if blockers != None and blockers > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?blockers={blockers}") + res = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/top.json?blockers={blockers}") elif blocked != None and blocked > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?blocked={blocked}") + res = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/top.json?blocked={blocked}") elif reference != None and reference > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?reference={reference}") + res = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/top.json?reference={reference}") elif software != None and software > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?software={software}") + res = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/top.json?software={software}") elif originator != None and originator > 0: - res = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/top.json?originator={originator}") + res = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/top.json?originator={originator}") else: raise HTTPException(status_code=400, detail="No filter specified") @@ -165,8 +165,8 @@ def index(request: Request, blockers: int = None, blocked: int = None, reference raise HTTPException(status_code=res.status_code, detail=res.text) return templates.TemplateResponse("scoreboard.html", { - "base_url" : fba.config["base_url"], - "slogan" : fba.config["slogan"], + "base_url" : config.config["base_url"], + "slogan" : config.config["slogan"], "request" : request, "scoreboard": True, "blockers" : blockers, @@ -177,7 +177,7 @@ def index(request: Request, blockers: int = None, blocked: int = None, reference "scores" : res.json() }) -@router.get(fba.config["base_url"] + "/") +@router.get(config.config["base_url"] + "/") def index(request: Request, domain: str = None, reason: str = None, reverse: str = None): if domain == "" or reason == "" or reverse == "": return fastapi.responses.RedirectResponse("/") @@ -186,18 +186,18 @@ def index(request: Request, domain: str = None, reason: str = None, reverse: str blocks = None if domain == None and reason == None and reverse == None: - info = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/info.json") + info = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/info.json") if not info.ok: raise HTTPException(status_code=info.status_code, detail=info.text) info = info.json() elif domain != None: - blocks = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/index.json?domain={domain}") + blocks = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/index.json?domain={domain}") elif reason != None: - blocks = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/index.json?reason={reason}") + blocks = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/index.json?reason={reason}") elif reverse != None: - blocks = requests.get(f"http://{fba.config['host']}:{fba.config['port']}{fba.config['base_url']}/api/index.json?reverse={reverse}") + blocks = requests.get(f"http://{config.config['host']}:{config.config['port']}{config.config['base_url']}/api/index.json?reverse={reverse}") if blocks != None: if not blocks.ok: @@ -217,7 +217,7 @@ def index(request: Request, domain: str = None, reason: str = None, reverse: str "info" : info }) -@router.get(fba.config["base_url"] + "/rss") +@router.get(config.config["base_url"] + "/rss") def rss(request: Request, domain: str = None): if domain != None: wildchar = "*." + ".".join(domain.split(".")[-domain.count("."):]) @@ -256,12 +256,12 @@ def rss(request: Request, domain: str = None): "Content-Type": "routerlication/rss+xml" }) -@router.get(fba.config["base_url"] + "/robots.txt", response_class=PlainTextResponse) +@router.get(config.config["base_url"] + "/robots.txt", response_class=PlainTextResponse) def robots(request: Request): return templates.TemplateResponse("robots.txt", { "request" : request, - "base_url": fba.config["base_url"] + "base_url": config.config["base_url"] }) if __name__ == "__main__": - uvicorn.run("api:router", host=fba.config["host"], port=fba.config["port"], log_level=fba.config["log_level"]) + uvicorn.run("api:router", host=config.config["host"], port=config.config["port"], log_level=config.config["log_level"]) diff --git a/check_instance.py b/check_instance.py index d7711e1..cea34c8 100755 --- a/check_instance.py +++ b/check_instance.py @@ -17,7 +17,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import json import sys import validators from fba import * diff --git a/fba/__init__.py b/fba/__init__.py index 638446a..2df28f7 100644 --- a/fba/__init__.py +++ b/fba/__init__.py @@ -1 +1 @@ -__all__ = ['boot', 'cache', 'fba', 'instances'] +__all__ = ['boot', 'cache', 'config', 'fba', 'instances'] diff --git a/fba/config.py b/fba/config.py new file mode 100644 index 0000000..7473b8f --- /dev/null +++ b/fba/config.py @@ -0,0 +1,20 @@ +# Fedi API Block - An aggregator for fetching blocking data from fediverse nodes +# Copyright (C) 2023 Free Software Foundation +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import json + +with open("config.json") as f: + config = json.loads(f.read()) diff --git a/fba/fba.py b/fba/fba.py index 14d500b..7e1a124 100644 --- a/fba/fba.py +++ b/fba/fba.py @@ -25,11 +25,9 @@ import time import validators from fba import cache +from fba import config from fba import instances -with open("config.json") as f: - config = json.loads(f.read()) - # Don't check these, known trolls/flooders/testing/developing blacklist = [ # Floods network with fake nodes as "research" project @@ -66,12 +64,12 @@ nodeinfo_identifier = [ # HTTP headers for non-API requests headers = { - "User-Agent": config["useragent"], + "User-Agent": config.config["useragent"], } # HTTP headers for API requests api_headers = { - "User-Agent": config["useragent"], + "User-Agent": config.config["useragent"], "Content-Type": "application/json", } @@ -400,8 +398,8 @@ def log_error(domain: str, res: any): ]) # Cleanup old entries - # DEBUG: print(f"DEBUG: Purging old records (distance: {config['error_log_cleanup']})") - cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config["error_log_cleanup"]]) + # DEBUG: print(f"DEBUG: Purging old records (distance: {config.config['error_log_cleanup']})") + cursor.execute("DELETE FROM error_log WHERE created < ?", [time.time() - config.config["error_log_cleanup"]]) except BaseException as e: print(f"ERROR: failed SQL query: domain='{domain}',exception[{type(e)}]:'{str(e)}'") sys.exit(255) @@ -485,7 +483,7 @@ def get_peers(domain: str, software: str) -> list: if software == "misskey": # DEBUG: print(f"DEBUG: domain='{domain}' is misskey, sending API POST request ...") offset = 0 - step = config["misskey_offset"] + step = config.config["misskey_offset"] # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support @@ -510,9 +508,9 @@ def get_peers(domain: str, software: str) -> list: if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config["misskey_offset"]: - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") - offset = offset + (config["misskey_offset"] - len(fetched)) + elif len(fetched) != config.config["misskey_offset"]: + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.config['misskey_offset']}'") + offset = offset + (config.config["misskey_offset"] - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step @@ -550,7 +548,7 @@ def get_peers(domain: str, software: str) -> list: elif software == "lemmy": # DEBUG: print(f"DEBUG: domain='{domain}' is Lemmy, fetching JSON ...") try: - res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) data = res.json() # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") @@ -587,7 +585,7 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: domain='{domain}',mode='{mode}'") while True: try: - res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.get(f"https://{domain}/api/v1/server/{mode}?start={start}&count=100", headers=headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) data = res.json() # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',data[]='{type(data)}'") @@ -624,13 +622,13 @@ def get_peers(domain: str, software: str) -> list: # DEBUG: print(f"DEBUG: Fetching get_peers_url='{get_peers_url}' from '{domain}' ...") try: - res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.get(f"https://{domain}{get_peers_url}", headers=api_headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) data = res.json() # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") if not res.ok or res.status_code >= 400: # DEBUG: print(f"DEBUG: Was not able to fetch '{get_peers_url}', trying alternative ...") - res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.get(f"https://{domain}/api/v3/site", headers=api_headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) data = res.json() # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") @@ -679,7 +677,7 @@ def post_json_api(domain: str, path: str, parameter: str, extra_headers: dict = # DEBUG: print("DEBUG: Sending POST to domain,path,parameter:", domain, path, parameter, extra_headers) data = {} try: - res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.post(f"https://{domain}{path}", data=parameter, headers={**api_headers, **extra_headers}, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) data = res.json() # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") @@ -727,7 +725,7 @@ def fetch_nodeinfo(domain: str, path: str = None) -> list: try: # DEBUG: print("DEBUG: Fetching request:", request) - res = reqto.get(request, headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.get(request, headers=api_headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) data = res.json() # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code={res.status_code},data[]='{type(data)}'") @@ -763,7 +761,7 @@ def fetch_wellknown_nodeinfo(domain: str) -> list: data = {} try: - res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.get(f"https://{domain}/.well-known/nodeinfo", headers=api_headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) data = res.json() # DEBUG: print("DEBUG: domain,res.ok,data[]:", domain, res.ok, type(data)) @@ -814,7 +812,7 @@ def fetch_generator_from_path(domain: str, path: str = "/") -> str: try: # DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...") - res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])) + res = reqto.get(f"https://{domain}{path}", headers=headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) # DEBUG: print("DEBUG: domain,res.ok,res.status_code,res.text[]:", domain, res.ok, res.status_code, type(res.text)) if res.ok and res.status_code < 300 and len(res.text) > 0: @@ -1186,13 +1184,13 @@ def send_bot_post(instance: str, blocks: dict): if truncated: message = message + "(the list has been truncated to the first 20 entries)" - botheaders = {**api_headers, **{"Authorization": "Bearer " + config["bot_token"]}} + botheaders = {**api_headers, **{"Authorization": "Bearer " + config.config["bot_token"]}} req = reqto.post( - f"{config['bot_instance']}/api/v1/statuses", + f"{config.config['bot_instance']}/api/v1/statuses", data={ "status" : message, - "visibility" : config['bot_visibility'], + "visibility" : config.config['bot_visibility'], "content_type": "text/plain" }, headers=botheaders, @@ -1218,7 +1216,7 @@ def get_mastodon_blocks(domain: str) -> dict: try: doc = bs4.BeautifulSoup( - reqto.get(f"https://{domain}/about", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text, + reqto.get(f"https://{domain}/about", headers=headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])).text, "html.parser", ) except BaseException as e: @@ -1263,7 +1261,7 @@ def get_friendica_blocks(domain: str) -> dict: try: doc = bs4.BeautifulSoup( - reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config["connection_timeout"], config["read_timeout"])).text, + reqto.get(f"https://{domain}/friendica", headers=headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])).text, "html.parser", ) except BaseException as e: @@ -1304,7 +1302,7 @@ def get_misskey_blocks(domain: str) -> dict: } offset = 0 - step = config["misskey_offset"] + step = config.config["misskey_offset"] while True: # iterating through all "suspended" (follow-only in its terminology) # instances page-by-page, since that troonware doesn't support @@ -1333,9 +1331,9 @@ def get_misskey_blocks(domain: str) -> dict: if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config["misskey_offset"]: - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") - offset = offset + (config["misskey_offset"] - len(fetched)) + elif len(fetched) != config.config["misskey_offset"]: + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.config['misskey_offset']}'") + offset = offset + (config.config["misskey_offset"] - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step @@ -1382,9 +1380,9 @@ def get_misskey_blocks(domain: str) -> dict: if len(fetched) == 0: # DEBUG: print("DEBUG: Returned zero bytes, exiting loop:", domain) break - elif len(fetched) != config["misskey_offset"]: - # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config['misskey_offset']}'") - offset = offset + (config["misskey_offset"] - len(fetched)) + elif len(fetched) != config.config["misskey_offset"]: + # DEBUG: print(f"DEBUG: Fetched '{len(fetched)}' row(s) but expected: '{config.config['misskey_offset']}'") + offset = offset + (config.config["misskey_offset"] - len(fetched)) else: # DEBUG: print("DEBUG: Raising offset by step:", step) offset = offset + step diff --git a/fetch_blocks.py b/fetch_blocks.py index 260945f..c114aad 100755 --- a/fetch_blocks.py +++ b/fetch_blocks.py @@ -29,7 +29,7 @@ from fba import * boot.acquire_lock() fba.cursor.execute( - "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_block"]] + "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - config.config["recheck_block"]] ) rows = fba.cursor.fetchall() @@ -225,7 +225,7 @@ for blocker, software, origin, nodeinfo_url in rows: # handling CSRF, I've saw at least one server requiring it to access the endpoint # DEBUG: print("DEBUG: Fetching meta:", blocker) meta = bs4.BeautifulSoup( - reqto.get(f"https://{blocker}/", headers=fba.headers, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).text, + reqto.get(f"https://{blocker}/", headers=fba.headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])).text, "html.parser", ) try: @@ -237,7 +237,7 @@ for blocker, software, origin, nodeinfo_url in rows: reqheaders = fba.api_headers # DEBUG: print("DEBUG: Querying API domain_blocks:", blocker) - blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).json() + blocks = reqto.get(f"https://{blocker}/api/v1/instance/domain_blocks", headers=reqheaders, timeout=(config.config["connection_timeout"], config.config["read_timeout"])).json() print(f"INFO: Checking {len(blocks)} entries from blocker='{blocker}',software='{software}' ...") for block in blocks: @@ -447,7 +447,7 @@ for blocker, software, origin, nodeinfo_url in rows: print("INFO: blocker:", blocker) try: # Blocks - federation = reqto.get(f"https://{blocker}{fba.get_peers_url}?filter=suspended", headers=fba.api_headers, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).json() + federation = reqto.get(f"https://{blocker}{fba.get_peers_url}?filter=suspended", headers=fba.api_headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])).json() if (federation == None): print("WARNING: No valid response:", blocker); @@ -522,7 +522,7 @@ for blocker, software, origin, nodeinfo_url in rows: else: print("WARNING: Unknown software:", blocker, software) - if fba.config["bot_enabled"] and len(blockdict) > 0: + if config.config["bot_enabled"] and len(blockdict) > 0: send_bot_post(blocker, blockdict) blockdict = [] diff --git a/fetch_cs.py b/fetch_cs.py index e65738d..0b69100 100755 --- a/fetch_cs.py +++ b/fetch_cs.py @@ -85,7 +85,7 @@ domains = { try: doc = bs4.BeautifulSoup( - reqto.get("https://meta.chaos.social/federation", headers=fba.headers, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])).text, + reqto.get("https://meta.chaos.social/federation", headers=fba.headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])).text, "html.parser", ) # DEBUG: print(f"DEBUG: doc()={len(doc)}[]={type(doc)}") diff --git a/fetch_fba_rss.py b/fetch_fba_rss.py index 4db5434..a204ea5 100755 --- a/fetch_fba_rss.py +++ b/fetch_fba_rss.py @@ -29,7 +29,7 @@ feed = sys.argv[1] domains = list() try: print(f"INFO: Fetch FBA-specific RSS feed='{feed}' ...") - res = reqto.get(feed, headers=fba.headers, timeout=(fba.config["connection_timeout"], fba.config["read_timeout"])) + res = reqto.get(feed, headers=fba.headers, timeout=(config.config["connection_timeout"], config.config["read_timeout"])) # DEBUG: print(f"DEBUG: res.ok={res.ok},res.status_code='{res.status_code}',res.text()={len(res.text)}") if res.ok and res.status_code < 300 and len(res.text) > 0: diff --git a/fetch_instances.py b/fetch_instances.py index 08b49e8..7b4b76f 100755 --- a/fetch_instances.py +++ b/fetch_instances.py @@ -33,7 +33,7 @@ fba.fetch_instances(instance, None, None, sys.argv[0]) # Loop through some instances fba.cursor.execute( - "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - fba.config["recheck_instance"]] + "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'gotosocial', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.config["recheck_instance"]] ) rows = fba.cursor.fetchall() -- 2.39.5