import subprocess import time import pytest from harvester.proxy import fetch_list, fetch_all @pytest.fixture(scope='session', autouse=True) def start_web_server(): """Start a local webserver to serve unit test data.""" server_process = subprocess.Popen(['python', '-m', 'http.server', '8888'], cwd='tests/data') time.sleep(1) yield server_process.terminate() def test_fetch_list(): """Proxies should be returned in the form of [username:password]@ip:port.""" expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002'] result = fetch_list('http://localhost:8888/proxies1.txt') assert set(expected).issubset(result) def test_fetch_list_fail(): """Failures are logged to console and an empty list is returned.""" expected = [] result = fetch_list('http://localhost:12345/proxies1.txt') assert result == expected def test_fetch_all(): expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002', '127.0.0.1:9999'] result = fetch_all(['http://localhost:8888/proxies1.txt', 'http://localhost:8888/proxies2.txt']) assert set(expected).issubset(result)