feat: write proxies to database

.gitignore

@@ -6,3 +6,6 @@ __pycache__/
 
 # proxies dev results
 proxies/
+
+# sqlite database
+*.db

harvester/db/models.py

@@ -1,9 +1,5 @@
-# harvester/db/models.py
-
 from sqlalchemy import Column, Integer, String, DateTime
-from sqlalchemy.ext.declarative import declarative_base
+from .schema import Base
-
-Base = declarative_base()
 
 
 class Proxy(Base):
@@ -21,5 +17,5 @@ class Proxy(Base):
     def __repr__(self):
         return (
             f'Proxy(id={self.id}, host={self.host}, port={self.port}, '
-            f'egress={self.egress}, created_at={self.created_at})'
+            f'egress_ip={self.egress_ip}, date_added={self.created_at})'
         )

harvester/db/schema.py

@@ -0,0 +1,10 @@
+from sqlalchemy import MetaData, create_engine
+from sqlalchemy.ext.declarative import declarative_base
+
+metadata = MetaData()
+Base = declarative_base(metadata=metadata)
+
+
+def init_db(engine_url):
+    engine = create_engine(engine_url)
+    Base.metadata.create_all(engine)

harvester/db/session.py

@@ -1,5 +1,3 @@
-# harvester/db/session.py
-
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 

main.py

@@ -5,8 +5,17 @@ import time
 import concurrent.futures
 import logging
 import os
+from datetime import datetime
+from harvester.db.models import Proxy
+from harvester.db.schema import init_db
+from harvester.db.session import SessionFactory
 from harvester.proxy import fetch_all, validate_socks
 
+DATABASE_URL = os.environ.get('DATABASE_URL', 'sqlite:///proxies.db')
+
+init_db(DATABASE_URL)
+session_factory = SessionFactory(DATABASE_URL)
+
 
 def read_file(path):
     with open(path, 'r', encoding='utf-8') as file:
@@ -48,6 +57,18 @@ def main():
             valid.append(proxy)
             print(f'{proxy} -> {ip}')
 
+            # Save to DB
+            session = session_factory.create_session()
+            proxy = Proxy(
+                host=proxy.split(':')[0],
+                port=int(proxy.split(':')[1]),
+                egress_ip=ip,
+                date_added=datetime.now(),
+                date_validated=datetime.now(),
+            )
+            session.add(proxy)
+            session.commit()
+
     # Create output directory if it does not exist
     if not os.path.exists('proxies'):
         os.makedirs('proxies')