fba/commands.py

   1 # Fedi API Block - An aggregator for fetching blocking data from fediverse nodes
   2 # Copyright (C) 2023 Free Software Foundation
   3 #
   4 # This program is free software: you can redistribute it and/or modify
   5 # it under the terms of the GNU Affero General Public License as published
   6 # by the Free Software Foundation, either version 3 of the License, or
   7 # (at your option) any later version.
   8 #
   9 # This program is distributed in the hope that it will be useful,
  10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12 # GNU Affero General Public License for more details.
  13 #
  14 # You should have received a copy of the GNU Affero General Public License
  15 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
  16
  17 import csv
  18 import inspect
  19 import json
  20 import time
  21
  22 import argparse
  23 import atoma
  24 import bs4
  25 import markdown
  26 import reqto
  27 import validators
  28
  29 from fba import blacklist
  30 from fba import blocks
  31 from fba import config
  32 from fba import federation
  33 from fba import fba
  34 from fba import instances
  35 from fba import locking
  36 from fba import network
  37
  38 from fba.helpers import tidyup
  39
  40 from fba.networks import friendica
  41 from fba.networks import mastodon
  42 from fba.networks import misskey
  43 from fba.networks import pleroma
  44
  45 def check_instance(args: argparse.Namespace) -> int:
  46     # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - CALLED!")
  47     status = 0
  48     if not validators.domain(args.domain):
  49         print(f"WARNING: args.domain='{args.domain}' is not valid")
  50         status = 100
  51     elif blacklist.is_blacklisted(args.domain):
  52         print(f"WARNING: args.domain='{args.domain}' is blacklisted")
  53         status = 101
  54     elif instances.is_registered(args.domain):
  55         print(f"WARNING: args.domain='{args.domain}' is already registered")
  56         status = 102
  57     else:
  58         print(f"INFO: args.domain='{args.domain}' is not known")
  59
  60     # DEBUG: print(f"DEBUG: status={status} - EXIT!")
  61     return status
  62
  63 def fetch_bkali(args: argparse.Namespace) -> int:
  64     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
  65     domains = list()
  66     try:
  67         fetched = network.post_json_api("gql.api.bka.li", "/v1/graphql", json.dumps({
  68             "query": "query domainlist {nodeinfo(order_by: {domain: asc}) {domain}}"
  69         }))
  70
  71         # DEBUG: print(f"DEBUG: fetched[]='{type(fetched)}'")
  72         if "error_message" in fetched:
  73             print(f"WARNING: post_json_api() for 'gql.api.bka.li' returned error message: {fetched['error_message']}")
  74             return 100
  75         elif isinstance(fetched["json"], dict) and "error" in fetched["json"] and "message" in fetched["json"]["error"]:
  76             print(f"WARNING: post_json_api() returned error: {fetched['error']['message']}")
  77             return 101
  78
  79         rows = fetched["json"]
  80
  81         # DEBUG: print(f"DEBUG: rows({len(rows)})[]='{type(rows)}'")
  82         if len(rows) == 0:
  83             raise Exception("WARNING: Returned no records")
  84         elif "data" not in rows:
  85             raise Exception(f"WARNING: rows()={len(rows)} does not contain key 'data'")
  86         elif "nodeinfo" not in rows["data"]:
  87             raise Exception(f"WARNING: rows()={len(rows['data'])} does not contain key 'nodeinfo'")
  88
  89         for entry in rows["data"]["nodeinfo"]:
  90             # DEBUG: print(f"DEBUG: entry['{type(entry)}']='{entry}'")
  91             if not "domain" in entry:
  92                 print(f"WARNING: entry()={len(entry)} does not contain 'domain' - SKIPPED!")
  93                 continue
  94             elif not validators.domain(entry["domain"]):
  95                 print(f"WARNING: domain='{entry['domain']}' is not a valid domain - SKIPPED!")
  96                 continue
  97             elif blacklist.is_blacklisted(entry["domain"]):
  98                 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is blacklisted - SKIPPED!")
  99                 continue
 100             elif instances.is_registered(entry["domain"]):
 101                 # DEBUG: print(f"DEBUG: domain='{entry['domain']}' is already registered - SKIPPED!")
 102                 continue
 103
 104             # DEBUG: print(f"DEBUG: Adding domain='{entry['domain']}' ...")
 105             domains.append(entry["domain"])
 106
 107     except network.exceptions as exception:
 108         print(f"ERROR: Cannot fetch graphql,exception[{type(exception)}]:'{str(exception)}' - EXIT!")
 109         return 102
 110
 111     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 112     if len(domains) > 0:
 113         locking.acquire()
 114
 115         print(f"INFO: Adding {len(domains)} new instances ...")
 116         for domain in domains:
 117             try:
 118                 print(f"INFO: Fetching instances from domain='{domain}' ...")
 119                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 120             except network.exceptions as exception:
 121                 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
 122                 instances.update_last_error(domain, exception)
 123
 124     # DEBUG: print("DEBUG: EXIT!")
 125     return 0
 126
 127 def fetch_blocks(args: argparse.Namespace):
 128     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
 129     if args.domain is not None and args.domain != "":
 130         # DEBUG: print(f"DEBUG: args.domain='{args.domain}' - checking ...")
 131         if not validators.domain(args.domain):
 132             print(f"WARNING: domain='{args.domain}' is not valid.")
 133             return
 134         elif blacklist.is_blacklisted(args.domain):
 135             print(f"WARNING: domain='{args.domain}' is blacklisted, won't check it!")
 136             return
 137         elif not instances.is_registered(args.domain):
 138             print(f"WARNING: domain='{args.domain}' is not registered, please run ./fba.py fetch_instances {args.domain} first.")
 139             return
 140
 141     locking.acquire()
 142
 143     if args.domain is not None and args.domain != "":
 144         # Re-check single domain
 145         fba.cursor.execute(
 146             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'bookwyrm', 'takahe') AND domain = ?", [args.domain]
 147         )
 148     else:
 149         # Re-check after "timeout" (aka. minimum interval)
 150         fba.cursor.execute(
 151             "SELECT domain, software, origin, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'bookwyrm', 'takahe') AND (last_blocked IS NULL OR last_blocked < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_block")]
 152         )
 153
 154     rows = fba.cursor.fetchall()
 155     print(f"INFO: Checking {len(rows)} entries ...")
 156     for blocker, software, origin, nodeinfo_url in rows:
 157         # DEBUG: print("DEBUG: BEFORE blocker,software,origin,nodeinfo_url:", blocker, software, origin, nodeinfo_url)
 158         blockdict = list()
 159         blocker = tidyup.domain(blocker)
 160         # DEBUG: print("DEBUG: AFTER blocker,software:", blocker, software)
 161
 162         if blocker == "":
 163             print("WARNING: blocker is now empty!")
 164             continue
 165         elif blacklist.is_blacklisted(blocker):
 166             print(f"WARNING: blocker='{blocker}' is blacklisted now!")
 167             continue
 168
 169         # DEBUG: print(f"DEBUG: blocker='{blocker}'")
 170         instances.update_last_blocked(blocker)
 171
 172         if software == "pleroma":
 173             print(f"INFO: blocker='{blocker}',software='{software}'")
 174             pleroma.fetch_blocks(blocker, origin, nodeinfo_url)
 175         elif software == "mastodon":
 176             print(f"INFO: blocker='{blocker}',software='{software}'")
 177             mastodon.fetch_blocks(blocker, origin, nodeinfo_url)
 178         elif software == "friendica" or software == "misskey":
 179             print(f"INFO: blocker='{blocker}',software='{software}'")
 180
 181             blocking = list()
 182             if software == "friendica":
 183                 blocking = friendica.fetch_blocks(blocker)
 184             elif software == "misskey":
 185                 blocking = misskey.fetch_blocks(blocker)
 186
 187             print(f"INFO: Checking {len(blocking.items())} entries from blocker='{blocker}',software='{software}' ...")
 188             for block_level, blocklist in blocking.items():
 189                 # DEBUG: print("DEBUG: blocker,block_level,blocklist():", blocker, block_level, len(blocklist))
 190                 block_level = tidyup.domain(block_level)
 191                 # DEBUG: print("DEBUG: AFTER-block_level:", block_level)
 192                 if block_level == "":
 193                     print("WARNING: block_level is empty, blocker:", blocker)
 194                     continue
 195
 196                 # DEBUG: print(f"DEBUG: Checking {len(blocklist)} entries from blocker='{blocker}',software='{software}',block_level='{block_level}' ...")
 197                 for block in blocklist:
 198                     blocked, reason = block.values()
 199                     # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - BEFORE!")
 200                     blocked = tidyup.domain(blocked)
 201                     reason  = tidyup.reason(reason) if reason is not None and reason != "" else None
 202                     # DEBUG: print(f"DEBUG: blocked='{blocked}',reason='{reason}' - AFTER!")
 203
 204                     if blocked == "":
 205                         print("WARNING: blocked is empty:", blocker)
 206                         continue
 207                     elif blacklist.is_blacklisted(blocked):
 208                         # DEBUG: print(f"DEBUG: blocked='{blocked}' is blacklisted - skipping!")
 209                         continue
 210                     elif blocked.count("*") > 0:
 211                         # Some friendica servers also obscure domains without hash
 212                         fba.cursor.execute(
 213                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("*", "_")]
 214                         )
 215
 216                         searchres = fba.cursor.fetchone()
 217
 218                         # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
 219                         if searchres is None:
 220                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 221                             continue
 222
 223                         blocked      = searchres[0]
 224                         origin       = searchres[1]
 225                         nodeinfo_url = searchres[2]
 226                     elif blocked.count("?") > 0:
 227                         # Some obscure them with question marks, not sure if that's dependent on version or not
 228                         fba.cursor.execute(
 229                             "SELECT domain, origin, nodeinfo_url FROM instances WHERE domain LIKE ? ORDER BY rowid LIMIT 1", [blocked.replace("?", "_")]
 230                         )
 231
 232                         searchres = fba.cursor.fetchone()
 233
 234                         # DEBUG: print(f"DEBUG: searchres[]='{type(searchres)}'")
 235                         if searchres is None:
 236                             print(f"WARNING: Cannot deobsfucate blocked='{blocked}' - SKIPPED!")
 237                             continue
 238
 239                         blocked      = searchres[0]
 240                         origin       = searchres[1]
 241                         nodeinfo_url = searchres[2]
 242                     elif not validators.domain(blocked):
 243                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 244                         continue
 245
 246                     # DEBUG: print("DEBUG: Looking up instance by domain:", blocked)
 247                     if not validators.domain(blocked):
 248                         print(f"WARNING: blocked='{blocked}',software='{software}' is not a valid domain name - skipped!")
 249                         continue
 250                     elif blocked.split(".")[-1] == "arpa":
 251                         print(f"WARNING: blocked='{blocked}' is a reversed .arpa domain and should not be used generally.")
 252                         continue
 253                     elif not instances.is_registered(blocked):
 254                         # DEBUG: print("DEBUG: Hash wasn't found, adding:", blocked, blocker)
 255                         try:
 256                             instances.add(blocked, blocker, inspect.currentframe().f_code.co_name, nodeinfo_url)
 257                         except network.exceptions as exception:
 258                             print(f"Exception during adding blocked='{blocked}',blocker='{blocker}': '{type(exception)}'")
 259                             continue
 260
 261                     if not blocks.is_instance_blocked(blocker, blocked, block_level):
 262                         blocks.add_instance(blocker, blocked, reason, block_level)
 263
 264                         if block_level == "reject":
 265                             blockdict.append({
 266                                 "blocked": blocked,
 267                                 "reason" : reason
 268                             })
 269                     else:
 270                         # DEBUG: print(f"DEBUG: Updating block last seen and reason for blocker='{blocker}',blocked='{blocked}' ...")
 271                         blocks.update_last_seen(blocker, blocked, block_level)
 272                         blocks.update_reason(reason, blocker, blocked, block_level)
 273
 274             # DEBUG: print("DEBUG: Committing changes ...")
 275             fba.connection.commit()
 276         else:
 277             print("WARNING: Unknown software:", blocker, software)
 278
 279         if config.get("bot_enabled") and len(blockdict) > 0:
 280             network.send_bot_post(blocker, blockdict)
 281
 282     # DEBUG: print("DEBUG: EXIT!")
 283
 284 def fetch_cs(args: argparse.Namespace):
 285     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
 286     extensions = [
 287         'extra',
 288         'abbr',
 289         'attr_list',
 290         'def_list',
 291         'fenced_code',
 292         'footnotes',
 293         'md_in_html',
 294         'admonition',
 295         'codehilite',
 296         'legacy_attrs',
 297         'legacy_em',
 298         'meta',
 299         'nl2br',
 300         'sane_lists',
 301         'smarty',
 302         'toc',
 303         'wikilinks'
 304     ]
 305
 306     domains = {
 307         "silenced": list(),
 308         "reject"  : list(),
 309     }
 310
 311     raw = fba.fetch_url("https://raw.githubusercontent.com/chaossocial/meta/master/federation.md", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout"))).text
 312     # DEBUG: print(f"DEBUG: raw()={len(raw)}[]='{type(raw)}'")
 313
 314     doc = bs4.BeautifulSoup(markdown.markdown(raw, extensions=extensions), features='html.parser')
 315
 316     # DEBUG: print(f"DEBUG: doc()={len(doc)}[]='{type(doc)}'")
 317     silenced = doc.find("h2", {"id": "silenced-instances"}).findNext("table").find("tbody")
 318     # DEBUG: print(f"DEBUG: silenced[]='{type(silenced)}'")
 319     domains["silenced"] = domains["silenced"] + federation.find_domains(silenced)
 320
 321     blocked = doc.find("h2", {"id": "blocked-instances"}).findNext("table").find("tbody")
 322     # DEBUG: print(f"DEBUG: blocked[]='{type(blocked)}'")
 323     domains["reject"] = domains["reject"] + federation.find_domains(blocked)
 324
 325     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 326     if len(domains) > 0:
 327         locking.acquire()
 328
 329         print(f"INFO: Adding {len(domains)} new instances ...")
 330         for block_level in domains:
 331             # DEBUG: print(f"DEBUG: block_level='{block_level}'")
 332
 333             for row in domains[block_level]:
 334                 # DEBUG: print(f"DEBUG: row='{row}'")
 335                 if not blocks.is_instance_blocked('chaos.social', row["domain"], block_level):
 336                     # DEBUG: print(f"DEBUG: domain='{row['domain']}',block_level='{block_level}' blocked by chaos.social, adding ...")
 337                     blocks.add_instance('chaos.social', row["domain"], row["reason"], block_level)
 338
 339                 if not instances.is_registered(row["domain"]):
 340                     try:
 341                         print(f"INFO: Fetching instances from domain='{row['domain']}' ...")
 342                         federation.fetch_instances(row["domain"], 'chaos.social', None, inspect.currentframe().f_code.co_name)
 343                     except network.exceptions as exception:
 344                         print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['domain']}'")
 345                         instances.update_last_error(row["domain"], exception)
 346
 347         # DEBUG: print("DEBUG: Committing changes ...")
 348         fba.connection.commit()
 349
 350     # DEBUG: print("DEBUG: EXIT!")
 351
 352 def fetch_fba_rss(args: argparse.Namespace):
 353     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
 354     domains = list()
 355
 356     print(f"INFO: Fetch FBA-specific RSS args.feed='{args.feed}' ...")
 357     response = fba.fetch_url(args.feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 358
 359     # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
 360     if response.ok and response.status_code < 300 and len(response.text) > 0:
 361         # DEBUG: print(f"DEBUG: Parsing RSS feed ({len(response.text)} Bytes) ...")
 362         rss = atoma.parse_rss_bytes(response.content)
 363
 364         # DEBUG: print(f"DEBUG: rss[]='{type(rss)}'")
 365         for item in rss.items:
 366             # DEBUG: print(f"DEBUG: item={item}")
 367             domain = item.link.split("=")[1]
 368
 369             if blacklist.is_blacklisted(domain):
 370                 # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
 371                 continue
 372             elif domain in domains:
 373                 # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
 374                 continue
 375             elif instances.is_registered(domain):
 376                 # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
 377                 continue
 378
 379             # DEBUG: print(f"DEBUG: Adding domain='{domain}'")
 380             domains.append(domain)
 381
 382     # DEBUG: print(f"DEBUG: domains()={len(domains)}")
 383     if len(domains) > 0:
 384         locking.acquire()
 385
 386         print(f"INFO: Adding {len(domains)} new instances ...")
 387         for domain in domains:
 388             try:
 389                 print(f"INFO: Fetching instances from domain='{domain}' ...")
 390                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 391             except network.exceptions as exception:
 392                 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
 393                 instances.update_last_error(domain, exception)
 394
 395     # DEBUG: print("DEBUG: EXIT!")
 396
 397 def fetch_fbabot_atom(args: argparse.Namespace):
 398     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
 399     feed = "https://ryona.agency/users/fba/feed.atom"
 400
 401     domains = list()
 402
 403     print(f"INFO: Fetching ATOM feed='{feed}' from FBA bot account ...")
 404     response = fba.fetch_url(feed, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 405
 406     # DEBUG: print(f"DEBUG: response.ok={response.ok},response.status_code='{response.status_code}',response.text()={len(response.text)}")
 407     if response.ok and response.status_code < 300 and len(response.text) > 0:
 408         # DEBUG: print(f"DEBUG: Parsing ATOM feed ({len(response.text)} Bytes) ...")
 409         atom = atoma.parse_atom_bytes(response.content)
 410
 411         # DEBUG: print(f"DEBUG: atom[]='{type(atom)}'")
 412         for entry in atom.entries:
 413             # DEBUG: print(f"DEBUG: entry[]='{type(entry)}'")
 414             doc = bs4.BeautifulSoup(entry.content.value, "html.parser")
 415             # DEBUG: print(f"DEBUG: doc[]='{type(doc)}'")
 416             for element in doc.findAll("a"):
 417                 for href in element["href"].split(","):
 418                     # DEBUG: print(f"DEBUG: href[{type(href)}]={href}")
 419                     domain = tidyup.domain(href)
 420
 421                     # DEBUG: print(f"DEBUG: domain='{domain}'")
 422                     if blacklist.is_blacklisted(domain):
 423                         # DEBUG: print(f"DEBUG: domain='{domain}' is blacklisted - SKIPPED!")
 424                         continue
 425                     elif domain in domains:
 426                         # DEBUG: print(f"DEBUG: domain='{domain}' is already added - SKIPPED!")
 427                         continue
 428                     elif instances.is_registered(domain):
 429                         # DEBUG: print(f"DEBUG: domain='{domain}' is already registered - SKIPPED!")
 430                         continue
 431
 432                     # DEBUG: print(f"DEBUG: Adding domain='{domain}',domains()={len(domains)}")
 433                     domains.append(domain)
 434
 435     # DEBUG: print(f"DEBUG: domains({len(domains)})={domains}")
 436     if len(domains) > 0:
 437         locking.acquire()
 438
 439         print(f"INFO: Adding {len(domains)} new instances ...")
 440         for domain in domains:
 441             try:
 442                 print(f"INFO: Fetching instances from domain='{domain}' ...")
 443                 federation.fetch_instances(domain, None, None, inspect.currentframe().f_code.co_name)
 444             except network.exceptions as exception:
 445                 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{domain}'")
 446                 instances.update_last_error(domain, exception)
 447
 448     # DEBUG: print("DEBUG: EXIT!")
 449
 450 def fetch_instances(args: argparse.Namespace) -> int:
 451     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
 452     locking.acquire()
 453
 454     # Initial fetch
 455     try:
 456         print(f"INFO: Fetching instances from args.domain='{args.domain}' ...")
 457         federation.fetch_instances(args.domain, None, None, inspect.currentframe().f_code.co_name)
 458     except network.exceptions as exception:
 459         print(f"WARNING: Exception '{type(exception)}' during fetching instances from args.domain='{args.domain}'")
 460         instances.update_last_error(args.domain, exception)
 461         return 100
 462
 463     if args.single:
 464         # DEBUG: print("DEBUG: Not fetching more instances - EXIT!")
 465         return 0
 466
 467     # Loop through some instances
 468     fba.cursor.execute(
 469         "SELECT domain, origin, software, nodeinfo_url FROM instances WHERE software IN ('pleroma', 'mastodon', 'friendica', 'misskey', 'bookwyrm', 'takahe', 'lemmy') AND (last_instance_fetch IS NULL OR last_instance_fetch < ?) ORDER BY rowid DESC", [time.time() - config.get("recheck_instance")]
 470     )
 471
 472     rows = fba.cursor.fetchall()
 473     print(f"INFO: Checking {len(rows)} entries ...")
 474     for row in rows:
 475         # DEBUG: print(f"DEBUG: domain='{row[0]}'")
 476         if blacklist.is_blacklisted(row[0]):
 477             print("WARNING: domain is blacklisted:", row[0])
 478             continue
 479
 480         try:
 481             print(f"INFO: Fetching instances for instance '{row[0]}' ('{row[2]}') of origin='{row[1]}',nodeinfo_url='{row[3]}'")
 482             federation.fetch_instances(row[0], row[1], row[2], inspect.currentframe().f_code.co_name, row[3])
 483         except network.exceptions as exception:
 484             print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row[0]}'")
 485             instances.update_last_error(row[0], exception)
 486
 487     # DEBUG: print("DEBUG: EXIT!")
 488     return 0
 489
 490 def fetch_federater(args: argparse.Namespace):
 491     # DEBUG: print(f"DEBUG: args[]='{type(args)}' - CALLED!")
 492     locking.acquire()
 493
 494     # Fetch this URL
 495     response = fba.fetch_url("https://github.com/federater/blocks_recommended/raw/main/federater.csv", network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
 496     # DEBUG: print(f"DEBUG: response[]='{type(response)}'")
 497     if response.ok and response.content != "":
 498         # DEBUG: print(f"DEBUG: Fetched {len(response.content)} Bytes, parsing CSV ...")
 499         ## DEBUG: print(f"DEBUG: response.content={response.content}")
 500         reader = csv.DictReader(response.content.decode('utf-8').splitlines(), dialect='unix')
 501         #, fieldnames='domain,severity,reject_media,reject_reports,public_comment,obfuscate'
 502         # DEBUG: print(f"DEBUG: reader[]='{type(reader)}'")
 503         for row in reader:
 504             if not validators.domain(row["#domain"]):
 505                 print(f"WARNING: domain='{row['#domain']}' is not a valid domain - skipped!")
 506                 continue
 507             elif blacklist.is_blacklisted(row["#domain"]):
 508                 print(f"WARNING: domain='{row['#domain']}' is blacklisted - skipped!")
 509                 continue
 510             elif instances.is_registered(row["#domain"]):
 511                 # DEBUG: print(f"DEBUG: domain='{row['#domain']}' is already registered - skipped!")
 512                 continue
 513
 514             try:
 515                 print(f"INFO: Fetching instances for instane='{row['#domain']}' ...")
 516                 federation.fetch_instances(row["#domain"], None, None, inspect.currentframe().f_code.co_name)
 517             except network.exceptions as exception:
 518                 print(f"WARNING: Exception '{type(exception)}' during fetching instances from domain='{row['#domain']}'")
 519                 instances.update_last_error(row["#domain"], exception)
 520
 521     # DEBUG: print("DEBUG: EXIT!")