harvester/proxy.py

@@ -6,22 +6,6 @@ import requests
 
 
 def fetch_list(url):
-    """Fetches proxy list from the given URL.
-
-    The HTTP response text will be searched for `ip:port` and `username:password@ip:port`
-    patterns to accommodate different source list formats.
-
-    If an error occurs while fetching the list, it will be logged with WARNING
-    and an empty list will be returned.
-
-    Args:
-        url (str): The URL to fetch proxy list from.
-
-    Returns:
-        list: A list of proxy server addresses fetched from the URL.
-            If an error occurs while fetching the list, it will be logged
-            with WARNING and an empty list will be returned.
-    """
     try:
         response = requests.get(url)
         response.raise_for_status()
@@ -30,9 +14,9 @@ def fetch_list(url):
         return []
 
     proxy_regex = r"(?:\b(?:[\S]+:)?(?:[\S]+)?@\b)?(?:\d{1,3}\.){3}\d{1,3}:\d+"
-    proxies = set(re.findall(proxy_regex, response.text))
+    proxies = re.findall(proxy_regex, response.text)
     logging.info(f'Fetched {len(proxies)} proxies from {url}')
-    return list(proxies)
+    return proxies
 
 
 def fetch_all(urls, max_workers=8):

tests/test_harvester.py

@@ -17,8 +17,7 @@ def start_web_server():
 def test_fetch_list():
     expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
     result = fetch_list('http://localhost:8888/proxies1.txt')
-    for proxy in expected:
-        assert proxy in result
+    assert result == expected
 
 
 def test_fetch_list_fail():
@@ -30,8 +29,7 @@ def test_fetch_list_fail():
 def test_fetch_list_only_valid():
     expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
     result = fetch_list('http://localhost:8888/proxies1.txt')
-    for proxy in expected:
-        assert proxy in result
+    assert result == expected
 
 
 def test_fetch_all():