diff --git a/harvester/proxy.py b/harvester/proxy.py index 979a611..ed5e5ce 100644 --- a/harvester/proxy.py +++ b/harvester/proxy.py @@ -36,6 +36,17 @@ def fetch_list(url): def fetch_all(urls, max_workers=8): + """Fetches proxy server addresses from multiple URLs concurrently. + + Args: + urls (list): A list of URLs to fetch proxy server addresses from. + max_workers (int, optional): The maximum number of worker threads to use. + Defaults to 8. + + Returns: + list: A list of unique proxy server addresses fetched from all the URLs. + + """ proxies = [] with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_url = {executor.submit(fetch_list, url): url for url in urls}