agatha/harvester
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

Add docstrings for fetch_all function

Browse Source
This commit is contained in:
agatha 2023-11-06 17:08:15 -05:00
parent 104eb8d949
commit cf152e0ca7
1 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
harvester/proxy.py
View File

@ -36,6 +36,17 @@ def fetch_list(url):
def fetch_all(urls, max_workers=8): def fetch_all(urls, max_workers=8):
"""Fetches proxy server addresses from multiple URLs concurrently.
Args:
urls (list): A list of URLs to fetch proxy server addresses from.
max_workers (int, optional): The maximum number of worker threads to use.
Defaults to 8.
Returns:
list: A list of unique proxy server addresses fetched from all the URLs.
"""
proxies = [] proxies = []
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_url = {executor.submit(fetch_list, url): url for url in urls} future_to_url = {executor.submit(fetch_list, url): url for url in urls}