diff --git a/harvester/proxy.py b/harvester/proxy.py index f5a2407..be52075 100644 --- a/harvester/proxy.py +++ b/harvester/proxy.py @@ -1,5 +1,6 @@ """Proxy harvester module""" import concurrent.futures +import logging import re import requests @@ -9,10 +10,12 @@ def fetch_list(url): response = requests.get(url) response.raise_for_status() except requests.RequestException: + logging.warning(f'Error fetching proxies from {url}') return [] proxy_regex = r"(?:\b(?:[\S]+:)?(?:[\S]+)?@\b)?(?:\d{1,3}\.){3}\d{1,3}:\d+" proxies = re.findall(proxy_regex, response.text) + logging.info(f'Fetched {len(proxies)} proxies from {url}') return proxies @@ -30,6 +33,7 @@ def fetch_all(urls, max_workers=8): except: pass + logging.info(f'Fetched {len(proxies)} proxies in total') return proxies