Proxies for price monitoring

# pip install requests[socks]
import json, random, re, time, requests

PROXY_LIST_URL = "PROXY_LIST_URL_FROM_DASHBOARD"
REFRESH_SEC = 600
HEADERS = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
           "Accept-Language": "en-US,en;q=0.9"}

_pool, _loaded = [], 0

def pool():
    global _pool, _loaded
    if time.time() - _loaded > REFRESH_SEC:
        text = requests.get(PROXY_LIST_URL, timeout=15).text
        _pool = [line.strip() for line in text.splitlines() if ":" in line]
        _loaded = time.time()
    return _pool

def fetch(url, tries=3):
    for _ in range(tries):
        proxy = random.choice(pool())
        try:
            r = requests.get(url, headers=HEADERS, timeout=15,
                             proxies={"http": f"socks5h://{proxy}", "https": f"socks5h://{proxy}"})
            if r.status_code == 200:
                return r.text
        except requests.RequestException:
            pass
        time.sleep(2)
    return ""

def offers(node, currency=""):
    if isinstance(node, dict):
        currency = node.get("priceCurrency", currency)
        if "price" in node:
            yield node["price"], currency
        for value in node.values():
            yield from offers(value, currency)
    elif isinstance(node, list):
        for item in node:
            yield from offers(item, currency)

def price(html):
    for block in re.findall(r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', html, re.S):
        try:
            data = json.loads(block)
        except ValueError:
            continue
        for value, currency in offers(data):
            return str(value), currency
    return "", ""

with open("prices.csv", "a") as out:
    for url in open("products.txt"):
        url = url.strip()
        if url:
            value, currency = price(fetch(url))
            out.write(f"{int(time.time())};{url};{value};{currency}\n")
            print(url, value, currency)