agatha/harvester
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

Add docstrings for fetch_list function

Browse Source
This commit is contained in:
agatha 2023-11-06 17:07:08 -05:00
parent e813199501
commit c7a9931561
1 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
harvester/proxy.py
View File

@ -6,6 +6,22 @@ import requests
def fetch_list(url): def fetch_list(url):
"""Fetches proxy list from the given URL.
The HTTP response text will be searched for `ip:port` and `username:password@ip:port`
patterns to accommodate different source list formats.
If an error occurs while fetching the list, it will be logged with WARNING
and an empty list will be returned.
Args:
url (str): The URL to fetch proxy list from.
Returns:
list: A list of proxy server addresses fetched from the URL.
If an error occurs while fetching the list, it will be logged
with WARNING and an empty list will be returned.
"""
try: try:
response = requests.get(url) response = requests.get(url)
response.raise_for_status() response.raise_for_status()
@ -14,9 +30,9 @@ def fetch_list(url):
return [] return []
proxy_regex = r"(?:\b(?:[\S]+:)?(?:[\S]+)?@\b)?(?:\d{1,3}\.){3}\d{1,3}:\d+" proxy_regex = r"(?:\b(?:[\S]+:)?(?:[\S]+)?@\b)?(?:\d{1,3}\.){3}\d{1,3}:\d+"
proxies = re.findall(proxy_regex, response.text) proxies = set(re.findall(proxy_regex, response.text))
logging.info(f'Fetched {len(proxies)} proxies from {url}') logging.info(f'Fetched {len(proxies)} proxies from {url}')
return proxies return list(proxies)
def fetch_all(urls, max_workers=8): def fetch_all(urls, max_workers=8):