feat: write proxies to database

harvester/db/session.py

@@ -1,5 +1,3 @@
-# harvester/db/session.py
-
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
@@ -25,4 +23,4 @@ def get_session_factory(engine_url):
 
 
 def get_session(engine_url):
-    return get_session_factory(engine_url).create_session()
+    return get_session_factory(engine_url).create_session()

main.py

@@ -5,8 +5,17 @@ import time
 import concurrent.futures
 import logging
 import os
+from datetime import datetime
+from harvester.db.models import Proxy
+from harvester.db.schema import init_db
+from harvester.db.session import SessionFactory
 from harvester.proxy import fetch_all, validate_socks
 
+DATABASE_URL = os.environ.get('DATABASE_URL', 'sqlite:///proxies.db')
+
+init_db(DATABASE_URL)
+session_factory = SessionFactory(DATABASE_URL)
+
 
 def read_file(path):
     with open(path, 'r', encoding='utf-8') as file:
@@ -48,6 +57,18 @@ def main():
             valid.append(proxy)
             print(f'{proxy} -> {ip}')
 
+            # Save to DB
+            session = session_factory.create_session()
+            proxy = Proxy(
+                host=proxy.split(':')[0],
+                port=int(proxy.split(':')[1]),
+                egress_ip=ip,
+                date_added=datetime.now(),
+                date_validated=datetime.now(),
+            )
+            session.add(proxy)
+            session.commit()
+
     # Create output directory if it does not exist
     if not os.path.exists('proxies'):
         os.makedirs('proxies')