diff --git a/src/proxy_pool/worker/settings.py b/src/proxy_pool/worker/settings.py new file mode 100644 index 0000000..726c196 --- /dev/null +++ b/src/proxy_pool/worker/settings.py @@ -0,0 +1,45 @@ +from arq import cron +from arq.connections import RedisSettings + +from proxy_pool.config import get_settings +from proxy_pool.worker.context import shutdown, startup +from proxy_pool.worker.tasks_cleanup import ( + expire_leases, + prune_dead_proxies, + prune_old_checks, +) +from proxy_pool.worker.tasks_scrape import scrape_all, scrape_source +from proxy_pool.worker.tasks_validate import ( + revalidate_sweep, + validate_proxy, +) + +settings = get_settings() + + +class WorkerSettings: + functions = [ + scrape_source, + scrape_all, + validate_proxy, + revalidate_sweep, + prune_dead_proxies, + prune_old_checks, + expire_leases, + ] + + cron_jobs = [ + cron(scrape_all, minute={0, 30}), + cron(revalidate_sweep, minute={10, 25, 40, 55}), + cron(prune_dead_proxies, hour={3}, minute={0}), + cron(prune_old_checks, hour={4}, minute={0}), + cron(expire_leases, minute=set(range(60))), + ] + + on_startup = startup + on_shutdown = shutdown + + redis_settings = RedisSettings.from_dsn(settings.redis.url) + max_jobs = 50 + job_timeout = 300 + keep_result = 3600