agatha/harvester
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

Fetching functions log info and warnings

Browse Source
This commit is contained in:
agatha 2023-11-06 16:47:12 -05:00
parent 87108ef446
commit fe02eaea6f
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
harvester/proxy.py
View File

@ -1,5 +1,6 @@
"""Proxy harvester module""" """Proxy harvester module"""
import concurrent.futures import concurrent.futures
import logging
import re import re
import requests import requests
@ -9,10 +10,12 @@ def fetch_list(url):
response = requests.get(url) response = requests.get(url)
response.raise_for_status() response.raise_for_status()
except requests.RequestException: except requests.RequestException:
logging.warning(f'Error fetching proxies from {url}')
return [] return []
proxy_regex = r"(?:\b(?:[\S]+:)?(?:[\S]+)?@\b)?(?:\d{1,3}\.){3}\d{1,3}:\d+" proxy_regex = r"(?:\b(?:[\S]+:)?(?:[\S]+)?@\b)?(?:\d{1,3}\.){3}\d{1,3}:\d+"
proxies = re.findall(proxy_regex, response.text) proxies = re.findall(proxy_regex, response.text)
logging.info(f'Fetched {len(proxies)} proxies from {url}')
return proxies return proxies
@ -30,6 +33,7 @@ def fetch_all(urls, max_workers=8):
except: except:
pass pass
logging.info(f'Fetched {len(proxies)} proxies in total')
return proxies return proxies