.gitignore

@@ -6,6 +6,3 @@ __pycache__/
 
 # proxies dev results
 proxies/
-
-# sqlite database
-*.db

harvester/db/models.py

@@ -1,5 +1,9 @@
+# harvester/db/models.py
+
 from sqlalchemy import Column, Integer, String, DateTime
-from .schema import Base
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
 
 
 class Proxy(Base):
@@ -17,5 +21,5 @@ class Proxy(Base):
     def __repr__(self):
         return (
             f'Proxy(id={self.id}, host={self.host}, port={self.port}, '
-            f'egress_ip={self.egress_ip}, date_added={self.created_at})'
+            f'egress={self.egress}, created_at={self.created_at})'
         )

harvester/db/schema.py

@@ -1,10 +0,0 @@
-from sqlalchemy import MetaData, create_engine
-from sqlalchemy.ext.declarative import declarative_base
-
-metadata = MetaData()
-Base = declarative_base(metadata=metadata)
-
-
-def init_db(engine_url):
-    engine = create_engine(engine_url)
-    Base.metadata.create_all(engine)

harvester/db/session.py

@@ -1,3 +1,5 @@
+# harvester/db/session.py
+
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 

main.py

@@ -5,17 +5,8 @@ import time
 import concurrent.futures
 import logging
 import os
-from datetime import datetime
-from harvester.db.models import Proxy
-from harvester.db.schema import init_db
-from harvester.db.session import SessionFactory
 from harvester.proxy import fetch_all, validate_socks
 
-DATABASE_URL = os.environ.get('DATABASE_URL', 'sqlite:///proxies.db')
-
-init_db(DATABASE_URL)
-session_factory = SessionFactory(DATABASE_URL)
-
 
 def read_file(path):
     with open(path, 'r', encoding='utf-8') as file:
@@ -57,18 +48,6 @@ def main():
             valid.append(proxy)
             print(f'{proxy} -> {ip}')
 
-            # Save to DB
-            session = session_factory.create_session()
-            proxy = Proxy(
-                host=proxy.split(':')[0],
-                port=int(proxy.split(':')[1]),
-                egress_ip=ip,
-                date_added=datetime.now(),
-                date_validated=datetime.now(),
-            )
-            session.add(proxy)
-            session.commit()
-
     # Create output directory if it does not exist
     if not os.path.exists('proxies'):
         os.makedirs('proxies')