agatha/harvester
1
0
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: agatha:cf152e0ca7fec8ce908d9da8821160e1f2dbecdd
Branches Tags
agatha:master
agatha:db
agatha:proxychains-conf
...
compare: agatha:893e3000a9a40b7340879d53c2d1343c9d493105
Branches Tags
agatha:db
agatha:master
agatha:proxychains-conf

2 Commits
cf152e0ca7 ... 893e3000a9

Author SHA1 Message Date
agatha
893e3000a9 Update unit tests to use set and issubset 2023-11-06 17:23:21 -05:00
agatha
da219575a1 Remove unneeded lin 2023-11-06 17:23:04 -05:00
2 changed files with 3 additions and 7 deletions
Show Stats Expand all files Collapse all files

1
harvester/proxy.py
View File

@ -51,7 +51,6 @@ def fetch_all(urls, max_workers=8):
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
future_to_url = {executor.submit(fetch_list, url): url for url in urls} future_to_url = {executor.submit(fetch_list, url): url for url in urls}
for future in concurrent.futures.as_completed(future_to_url): for future in concurrent.futures.as_completed(future_to_url):
url = future_to_url[future]
try: try:
proxy_list = future.result() proxy_list = future.result()
for proxy in proxy_list: for proxy in proxy_list:

9
tests/test_harvester.py
View File

@ -17,8 +17,7 @@ def start_web_server():
def test_fetch_list(): def test_fetch_list():
expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002'] expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
result = fetch_list('http://localhost:8888/proxies1.txt') result = fetch_list('http://localhost:8888/proxies1.txt')
for proxy in expected: assert set(expected).issubset(result)
assert proxy in result
def test_fetch_list_fail(): def test_fetch_list_fail():
@ -30,12 +29,10 @@ def test_fetch_list_fail():
def test_fetch_list_only_valid(): def test_fetch_list_only_valid():
expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002'] expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
result = fetch_list('http://localhost:8888/proxies1.txt') result = fetch_list('http://localhost:8888/proxies1.txt')
for proxy in expected: assert set(expected).issubset(result)
assert proxy in result
def test_fetch_all(): def test_fetch_all():
expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002', '127.0.0.1:9999'] expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002', '127.0.0.1:9999']
result = fetch_all(['http://localhost:8888/proxies1.txt', 'http://localhost:8888/proxies2.txt']) result = fetch_all(['http://localhost:8888/proxies1.txt', 'http://localhost:8888/proxies2.txt'])
for proxy in expected: assert set(expected).issubset(result)
assert proxy in result