agatha/pastebin-client
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: agatha:dev-refactor
Branches Tags
agatha:master
agatha:dev-refactor
..
compare: agatha:master
Branches Tags
agatha:dev-refactor
agatha:master

No commits in common. "dev-refactor" and "master" have entirely different histories.
dev-refact ... master

2 changed files with 14 additions and 24 deletions
Show Stats Expand all files Collapse all files

16
src/main.py
View File

@ -1,23 +1,23 @@
import time import time
from pastebin.client import PastebinClient from pastebin.client import PastebinClient
from pastebin.util import search_by_category from pastebin.util import search_by_language
def main(): def main():
pastebin = PastebinClient(debug=True) pastebin = PastebinClient()
filters = ['Gaming'] lang_filters = ['Email', 'Python']
while True: while True:
print('Updating pastes...')
new_pastes = pastebin.update_paste_list() new_pastes = pastebin.update_paste_list()
print('Updating pastes...')
for filter_ in filters: for lang in lang_filters:
pastes = search_by_category(new_pastes, filter_) pastes = search_by_language(new_pastes, lang)
for paste in pastes: for paste in pastes:
print(f'New {filter_} paste: {paste}') print(f'New {lang} paste: {paste}')
time.sleep(300) time.sleep(60)
if __name__ == '__main__': if __name__ == '__main__':

22
src/pastebin/client.py
View File

@ -63,7 +63,7 @@ class PastebinAPI:
return response return response
def get_paste_list(self): def get_public_paste_list(self):
endpoint = '/archive' endpoint = '/archive'
try: try:
@ -77,14 +77,9 @@ class PastebinAPI:
pastes = [self._parse_paste(paste_html) for paste_html in pastes_table] pastes = [self._parse_paste(paste_html) for paste_html in pastes_table]
# Sketchy way of getting the metadata each time
# TODO: This should be using ThreadPoolExecutor here
for i, paste in enumerate(pastes):
pastes[i] = self.get_paste_meta(paste)
return pastes return pastes
def get_paste_meta(self, paste): def get_paste(self, paste):
endpoint = f'/{paste.href}' endpoint = f'/{paste.href}'
try: try:
@ -98,15 +93,10 @@ class PastebinAPI:
paste = self._parse_paste_metadata(soup, paste) paste = self._parse_paste_metadata(soup, paste)
# Get paste text # Get paste text
# paste = self._fetch_paste_text(paste) paste = self._fetch_paste_text(paste)
return paste return paste
def download_paste(self, paste):
data = self._fetch_paste_text(paste)
with open(paste.href, 'w') as f:
f.write(data)
def _parse_paste(self, paste_html): def _parse_paste(self, paste_html):
paste_title = paste_html.find('a').text paste_title = paste_html.find('a').text
paste_link = paste_html.find('a')['href'].lstrip('/') paste_link = paste_html.find('a')['href'].lstrip('/')
@ -140,7 +130,7 @@ class PastebinClient(PastebinAPI):
self.pastes = [] self.pastes = []
def update_paste_list(self): def update_paste_list(self):
fetched_pastes = set(self.get_paste_list()) fetched_pastes = set(self.get_public_paste_list())
existing_pastes = set(self.pastes) existing_pastes = set(self.pastes)
new_pastes = fetched_pastes - existing_pastes new_pastes = fetched_pastes - existing_pastes
@ -151,9 +141,9 @@ class PastebinClient(PastebinAPI):
def fetch_paste(self, paste_index): def fetch_paste(self, paste_index):
if paste_index >= len(self.pastes): if paste_index >= len(self.pastes):
return None return None
return self.get_paste_meta(self.pastes[paste_index]) return self.get_paste(self.pastes[paste_index])
def paste_list(self): def get_paste_list(self):
return self.pastes return self.pastes
def total_pastes(self): def total_pastes(self):