agatha/harvester
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: agatha:893e3000a9a40b7340879d53c2d1343c9d493105
Branches Tags
agatha:master
agatha:db
agatha:proxychains-conf
..
compare: agatha:cf152e0ca7fec8ce908d9da8821160e1f2dbecdd
Branches Tags
agatha:db
agatha:master
agatha:proxychains-conf

No commits in common. "893e3000a9a40b7340879d53c2d1343c9d493105" and "cf152e0ca7fec8ce908d9da8821160e1f2dbecdd" have entirely different histories.
893e3000a9 ... cf152e0ca7

2 changed files with 7 additions and 3 deletions
Show Stats Expand all files Collapse all files

1
harvester/proxy.py
View File

@ -51,6 +51,7 @@ def fetch_all(urls, max_workers=8):
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_url = {executor.submit(fetch_list, url): url for url in urls} future_to_url = {executor.submit(fetch_list, url): url for url in urls}
for future in concurrent.futures.as_completed(future_to_url): for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future]
try: try:
proxy_list = future.result() proxy_list = future.result()
for proxy in proxy_list: for proxy in proxy_list:

9
tests/test_harvester.py
View File

@ -17,7 +17,8 @@ def start_web_server():
def test_fetch_list(): def test_fetch_list():
expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002'] expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
result = fetch_list('http://localhost:8888/proxies1.txt') result = fetch_list('http://localhost:8888/proxies1.txt')
assert set(expected).issubset(result) for proxy in expected:
assert proxy in result
def test_fetch_list_fail(): def test_fetch_list_fail():
@ -29,10 +30,12 @@ def test_fetch_list_fail():
def test_fetch_list_only_valid(): def test_fetch_list_only_valid():
expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002'] expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
result = fetch_list('http://localhost:8888/proxies1.txt') result = fetch_list('http://localhost:8888/proxies1.txt')
assert set(expected).issubset(result) for proxy in expected:
assert proxy in result
def test_fetch_all(): def test_fetch_all():
expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002', '127.0.0.1:9999'] expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002', '127.0.0.1:9999']
result = fetch_all(['http://localhost:8888/proxies1.txt', 'http://localhost:8888/proxies2.txt']) result = fetch_all(['http://localhost:8888/proxies1.txt', 'http://localhost:8888/proxies2.txt'])
assert set(expected).issubset(result) for proxy in expected:
assert proxy in result