Прокси для SEO-софта

# pip install requests[socks]
import csv, random, re, time, requests
from concurrent.futures import ThreadPoolExecutor

PROXY_LIST_URL = "PROXY_LIST_URL_FROM_DASHBOARD"
REFRESH_SEC = 600
THREADS = 1200
HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; SeoAudit/1.0)", "Accept-Language": "en-US,en;q=0.9"}

_pool, _loaded = [], 0

def pool():
    global _pool, _loaded
    if time.time() - _loaded > REFRESH_SEC:
        text = requests.get(PROXY_LIST_URL, timeout=15).text
        _pool = [line.strip() for line in text.splitlines() if ":" in line]
        _loaded = time.time()
    return _pool

def first(pattern, html):
    match = re.search(pattern, html, re.S | re.I)
    return re.sub(r"\s+", " ", re.sub(r"<[^>]+>", " ", match.group(1))).strip() if match else ""

def audit(url, tries=3):
    for _ in range(tries):
        proxy = random.choice(pool())
        try:
            r = requests.get(url, headers=HEADERS, timeout=20, allow_redirects=False,
                             proxies={"http": f"socks5h://{proxy}", "https": f"socks5h://{proxy}"})
        except requests.RequestException:
            continue
        html = r.text if "html" in r.headers.get("Content-Type", "") else ""
        return {
            "url": url,
            "status": r.status_code,
            "redirect": r.headers.get("Location", ""),
            "title": first(r"<title[^>]*>(.*?)</title>", html),
            "h1": first(r"<h1[^>]*>(.*?)</h1>", html),
            "description": first(r'<meta[^>]+name="description"[^>]+content="([^"]*)"', html),
            "canonical": first(r'<link[^>]+rel="canonical"[^>]+href="([^"]*)"', html),
            "robots": first(r'<meta[^>]+name="robots"[^>]+content="([^"]*)"', html),
        }
    return {"url": url, "status": 0, "redirect": "", "title": "", "h1": "",
            "description": "", "canonical": "", "robots": ""}

urls = [line.strip() for line in open("urls.txt") if line.strip()]
fields = ["url", "status", "redirect", "title", "h1", "description", "canonical", "robots"]
with open("audit.csv", "w", newline="", encoding="utf-8") as out:
    writer = csv.DictWriter(out, fieldnames=fields, delimiter=";")
    writer.writeheader()
    with ThreadPoolExecutor(THREADS) as executor:
        for row in executor.map(audit, urls):
            writer.writerow(row)
            print(row["status"], row["url"], row["title"][:60])