harvester/main.py

"""Harvester: Proxy collection tool
Inspired by https://github.com/acidvegas/proxytools
"""
from harvester.proxy import fetch_all


URLS = [
    'https://api.openproxylist.xyz/socks4.txt',
    'https://api.openproxylist.xyz/socks5.txt',
    'https://api.proxyscrape.com/?request=displayproxies&proxytype=socks4',
]

def main():
    """Main entry point."""
    proxies = fetch_all(URLS)
    print(proxies)
    print(len(proxies))


if __name__ == '__main__':
    main()