harvester/proxy.py

@@ -51,6 +51,7 @@ def fetch_all(urls, max_workers=8):
     with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
         future_to_url = {executor.submit(fetch_list, url): url for url in urls}
         for future in concurrent.futures.as_completed(future_to_url):
+            url = future_to_url[future]
             try:
                 proxy_list = future.result()
                 for proxy in proxy_list:

tests/test_harvester.py

@@ -17,7 +17,8 @@ def start_web_server():
 def test_fetch_list():
     expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
     result = fetch_list('http://localhost:8888/proxies1.txt')
-    assert set(expected).issubset(result)
+    for proxy in expected:
+        assert proxy in result
 
 
 def test_fetch_list_fail():
@@ -29,10 +30,12 @@ def test_fetch_list_fail():
 def test_fetch_list_only_valid():
     expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
     result = fetch_list('http://localhost:8888/proxies1.txt')
-    assert set(expected).issubset(result)
+    for proxy in expected:
+        assert proxy in result
 
 
 def test_fetch_all():
     expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002', '127.0.0.1:9999']
     result = fetch_all(['http://localhost:8888/proxies1.txt', 'http://localhost:8888/proxies2.txt'])
-    assert set(expected).issubset(result)
+    for proxy in expected:
+        assert proxy in result