.gitignore

@@ -3,6 +3,3 @@ venv/
 
 __pycache__/
 *.py[cod]
-
-# proxies dev results
-proxies/

README.md

@@ -1,16 +1,6 @@
 # Harvester
 Python package for harvesting commonly available data, such as free proxy servers.
 
-## Running the Demo
-If you just want proxies, just run the demo code in [main.py](main.py):
-
-```shell
-git clone https://git.juggalol.com/agatha/harvester
-pip install -r requirements.txt
-mkdir proxies
-python main.py
-```
-
 ## Modules
 ### Proxy
 #### fetch_list
@@ -20,11 +10,56 @@ It functions by running a regular expression against the HTTP response, looking
 strings that match a `username:password@ip:port` pattern where username and password
 are optional.
 
+```python
+from harvester.proxy import fetch_list
+
+
+URLS = [
+    'https://api.openproxylist.xyz/socks4.txt',
+    'https://api.openproxylist.xyz/socks5.txt',
+    'https://api.proxyscrape.com/?request=displayproxies&proxytype=socks4',
+]
+
+
+def main():
+    """Main entry point."""
+    for url in URLS:
+        proxies = fetch_list(url)
+        print(proxies)
+
+
+if __name__ == '__main__':
+    main()
+
+```
+
 #### fetch_all
 Proxies can be fetched from multiple source URLs by using the `fetch_all` function.
 
 It takes a list of URLs and an optional `max_workers` parameter. Proxies will be fetched from
-the source URLs concurrently using a `ThreadPoolExecutor`.
+the source URLs concurrently using a `ThreadPoolExecutor`:
+
+```python
+from harvester.proxy import fetch_all
+
+
+URLS = [
+    'https://api.openproxylist.xyz/socks4.txt',
+    'https://api.openproxylist.xyz/socks5.txt',
+    'https://api.proxyscrape.com/?request=displayproxies&proxytype=socks4',
+]
+
+
+def main():
+    """Main entry point."""
+    proxies = fetch_all(URLS)
+    print(proxies)
+
+
+if __name__ == '__main__':
+    main()
+
+```
 
 #### validate_socks
 SOCKS5 proxies can be tested with the `validate_socks` method. The method takes a proxy
@@ -34,14 +69,8 @@ with no issues, otherwise it will raise an exception and the caller can decide h
 For an example implementation, see [main.py](main.py). 
 
 ## Testing
-I was trying to get into the habit of writing unit tests, but god damn I hate them. There are
-a few, but I don't plan on continuing any time soon.
 ```
 pip install -r requirements.txt
 pip install -r requirement-dev.txt
 pytest -v
 ```
-
-## Greets
-Shoutouts to [acidvegas](https://git.supernets.org/acidvegas/). This project was inspired by
-[proxytools](https://git.supernets.org/acidvegas/proxytools)

main.py

@@ -1,7 +1,6 @@
 """Harvester: Proxy collection tool
 Inspired by https://github.com/acidvegas/proxytools
 """
-import time
 import concurrent.futures
 import logging
 from harvester.proxy import fetch_all, validate_socks
@@ -14,11 +13,6 @@ def load_urls(path):
     return urls
 
 
-def write_file(path, data):
-    with open(path, 'w', encoding='utf-8') as file:
-        file.write(data)
-
-
 def main():
     """Main entry point."""
     logging.basicConfig(level=logging.WARN)
@@ -39,7 +33,7 @@ def main():
                 response = future.result()
                 response.raise_for_status()
             except Exception as exception:
-                # TODO: Handle exceptions differently. See https://git.juggalol.com/agatha/harvester/issues/1.
+                # TODO: Handle exceptions differently. See issues.
                 logging.info(str(exception))
                 continue
 
@@ -47,11 +41,9 @@ def main():
             valid.append(proxy)
             print(f'{proxy} -> {ip}')
 
-    # Write to file with timestamp
+    with open('data/valid-socks.txt', 'w', encoding='utf-8') as file:
-    write_file(
+        file.write('\n'.join(valid))
-        path=f'proxies/valid-socks-{time.strftime("%Y%m%d%H%M%S")}.txt',
+
-        data='\n'.join(valid)
-    )
     for proxy in valid:
         print(proxy)