harvester/tests/test_harvester.py

import subprocess
import time
import pytest
from harvester.proxy import fetch_list, fetch_all


@pytest.fixture(scope='session', autouse=True)
def start_web_server():
    server_process = subprocess.Popen(['python', '-m', 'http.server', '8888'], cwd='tests/data')
    time.sleep(1)

    yield

    server_process.terminate()


def test_fetch_list():
    expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
    result = fetch_list('http://localhost:8888/proxies1.txt')
    for proxy in expected:
        assert proxy in result


def test_fetch_list_fail():
    expected = []
    result = fetch_list('http://localhost:12345/proxies1.txt')
    assert result == expected


def test_fetch_list_only_valid():
    expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002']
    result = fetch_list('http://localhost:8888/proxies1.txt')
    for proxy in expected:
        assert proxy in result


def test_fetch_all():
    expected = ['127.0.0.1:9000', '127.0.0.1:9001', 'username:pa$$@word@127.0.0.1:9002', '127.0.0.1:9999']
    result = fetch_all(['http://localhost:8888/proxies1.txt', 'http://localhost:8888/proxies2.txt'])
    for proxy in expected:
        assert proxy in result