From 5ac63e1fb5ba469bbeaf528bf54d2ed18785ec9e Mon Sep 17 00:00:00 2001 From: agatha Date: Sun, 22 Sep 2024 14:15:01 -0400 Subject: [PATCH] feat: write proxies to database --- harvester/db/session.py | 4 +--- main.py | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/harvester/db/session.py b/harvester/db/session.py index 34b5c1d..8316ee2 100644 --- a/harvester/db/session.py +++ b/harvester/db/session.py @@ -1,5 +1,3 @@ -# harvester/db/session.py - from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker @@ -25,4 +23,4 @@ def get_session_factory(engine_url): def get_session(engine_url): - return get_session_factory(engine_url).create_session() \ No newline at end of file + return get_session_factory(engine_url).create_session() diff --git a/main.py b/main.py index a3dff0c..0a227c0 100644 --- a/main.py +++ b/main.py @@ -5,8 +5,17 @@ import time import concurrent.futures import logging import os +from datetime import datetime +from harvester.db.models import Proxy +from harvester.db.schema import init_db +from harvester.db.session import SessionFactory from harvester.proxy import fetch_all, validate_socks +DATABASE_URL = os.environ.get('DATABASE_URL', 'sqlite:///proxies.db') + +init_db(DATABASE_URL) +session_factory = SessionFactory(DATABASE_URL) + def read_file(path): with open(path, 'r', encoding='utf-8') as file: @@ -48,6 +57,18 @@ def main(): valid.append(proxy) print(f'{proxy} -> {ip}') + # Save to DB + session = session_factory.create_session() + proxy = Proxy( + host=proxy.split(':')[0], + port=int(proxy.split(':')[1]), + egress_ip=ip, + date_added=datetime.now(), + date_validated=datetime.now(), + ) + session.add(proxy) + session.commit() + # Create output directory if it does not exist if not os.path.exists('proxies'): os.makedirs('proxies')