harvester/tests/test_harvester.py

import subprocess
import time
import pytest
from harvester.proxy import fetch_list, fetch_all


@pytest.fixture(scope='session', autouse=True)
def start_web_server():
    """Start a local webserver to serve unit test data."""
    server_process = subprocess.Popen(['python', '-m', 'http.server', '8888'], cwd='tests/data')
    time.sleep(1)

    yield

    server_process.terminate()


def test_fetch_list():
    """Proxies should be returned in the form of [username:password]@ip:port."""
    expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
    result = fetch_list('http://localhost:8888/proxies1.txt')
    assert set(expected).issubset(result)


def test_fetch_list_fail():
    """Failures are logged to console and an empty list is returned."""
    expected = []
    result = fetch_list('http://localhost:12345/proxies1.txt')
    assert result == expected


def test_fetch_all():
    expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002', '127.0.0.1:9999']
    result = fetch_all(['http://localhost:8888/proxies1.txt', 'http://localhost:8888/proxies2.txt'])
    assert set(expected).issubset(result)