From cf152e0ca7fec8ce908d9da8821160e1f2dbecdd Mon Sep 17 00:00:00 2001 From: agatha Date: Mon, 6 Nov 2023 17:08:15 -0500 Subject: [PATCH] Add docstrings for fetch_all function --- harvester/proxy.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/harvester/proxy.py b/harvester/proxy.py index 979a611..ed5e5ce 100644 --- a/harvester/proxy.py +++ b/harvester/proxy.py @@ -36,6 +36,17 @@ def fetch_list(url): def fetch_all(urls, max_workers=8): + """Fetches proxy server addresses from multiple URLs concurrently. + + Args: + urls (list): A list of URLs to fetch proxy server addresses from. + max_workers (int, optional): The maximum number of worker threads to use. + Defaults to 8. + + Returns: + list: A list of unique proxy server addresses fetched from all the URLs. + + """ proxies = [] with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: future_to_url = {executor.submit(fetch_list, url): url for url in urls}