# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
-import bs4
import hashlib
import re
-import requests
import json
import sqlite3
import sys
import time
+
+import bs4
+import requests
import validators
from urllib.parse import urlparse
from fba import blacklist
-from fba import cache
from fba import config
from fba import instances
from fba import network
try:
# DEBUG: print(f"DEBUG: Fetching path='{path}' from '{domain}' ...")
- response = network.fetch_response(domain, path, network.headers, (config.get("connection_timeout"), config.get("read_timeout")))
+ response = network.fetch_response(domain, path, network.web_headers, (config.get("connection_timeout"), config.get("read_timeout")))
# DEBUG: print("DEBUG: domain,response.ok,response.status_code,response.text[]:", domain, response.ok, response.status_code, type(response.text))
if response.ok and response.status_code < 300 and len(response.text) > 0: