Compare commits

..

No commits in common. "5ac63e1fb5ba469bbeaf528bf54d2ed18785ec9e" and "d710dc723e9d3219dc7e062d6913d9747a3c23ff" have entirely different histories.

5 changed files with 9 additions and 37 deletions

3
.gitignore vendored
View File

@ -6,6 +6,3 @@ __pycache__/
# proxies dev results
proxies/
# sqlite database
*.db

View File

@ -1,5 +1,9 @@
# harvester/db/models.py
from sqlalchemy import Column, Integer, String, DateTime
from .schema import Base
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class Proxy(Base):
@ -17,5 +21,5 @@ class Proxy(Base):
def __repr__(self):
return (
f'Proxy(id={self.id}, host={self.host}, port={self.port}, '
f'egress_ip={self.egress_ip}, date_added={self.created_at})'
f'egress={self.egress}, created_at={self.created_at})'
)

View File

@ -1,10 +0,0 @@
from sqlalchemy import MetaData, create_engine
from sqlalchemy.ext.declarative import declarative_base
metadata = MetaData()
Base = declarative_base(metadata=metadata)
def init_db(engine_url):
engine = create_engine(engine_url)
Base.metadata.create_all(engine)

View File

@ -1,3 +1,5 @@
# harvester/db/session.py
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker

21
main.py
View File

@ -5,17 +5,8 @@ import time
import concurrent.futures
import logging
import os
from datetime import datetime
from harvester.db.models import Proxy
from harvester.db.schema import init_db
from harvester.db.session import SessionFactory
from harvester.proxy import fetch_all, validate_socks
DATABASE_URL = os.environ.get('DATABASE_URL', 'sqlite:///proxies.db')
init_db(DATABASE_URL)
session_factory = SessionFactory(DATABASE_URL)
def read_file(path):
with open(path, 'r', encoding='utf-8') as file:
@ -57,18 +48,6 @@ def main():
valid.append(proxy)
print(f'{proxy} -> {ip}')
# Save to DB
session = session_factory.create_session()
proxy = Proxy(
host=proxy.split(':')[0],
port=int(proxy.split(':')[1]),
egress_ip=ip,
date_added=datetime.now(),
date_validated=datetime.now(),
)
session.add(proxy)
session.commit()
# Create output directory if it does not exist
if not os.path.exists('proxies'):
os.makedirs('proxies')