diff --git a/.gitignore b/.gitignore index 66bca12..41028c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .env venv/ +RSS \ No newline at end of file diff --git a/DiscordBot.py b/DiscordBot.py index 5a40d4f..44bf4ab 100644 --- a/DiscordBot.py +++ b/DiscordBot.py @@ -5,6 +5,8 @@ import re import requests from bs4 import BeautifulSoup import logging +import rss_gen +from urlencode import urlencode ########################################## #Logging @@ -33,9 +35,6 @@ intents.messages = True intents.message_content = True bot = discord.Bot(intents=intents) -def urlencode(inp: str): - return "".join([i if i in ["~", "-", "_", "."] or i.isalnum() else "%"+str(hex(ord(i))[2:].upper()) for i in inp]) - def pixelget(url: str): response = requests.get(url) soup = BeautifulSoup(response.text, 'html.parser') @@ -138,6 +137,9 @@ async def on_message(message): os.remove(i) # noqa: E501 logger.info(f"Posted link for {i}") + df = rss_gen.create_df() + rss_gen.create_48h_latest(df) + rss_gen.create_25_recent_feed(df) return def main(): diff --git a/requirements.txt b/requirements.txt index 70f09f1..1c81488 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ py-cord python-dotenv requests bs4 +pandas \ No newline at end of file diff --git a/rss_gen.py b/rss_gen.py new file mode 100644 index 0000000..b9d61eb --- /dev/null +++ b/rss_gen.py @@ -0,0 +1,70 @@ +import requests +from bs4 import BeautifulSoup +from datetime import datetime +import pandas as pd +import argparse +import os +from urlencode import urlencode + +XML_BEGIN_RECENT = """Torfiles Last 25 + https://torfiles.fieryeagle.org + Torfiles RSS Feed + en + 5 + """ + +XML_BEGIN_LAST = """Torfiles Last 48h + https://torfiles.fieryeagle.org + Torfiles RSS Feed + en + 5 + """ + +XML_END = """""" + +def make_torrent_entry(name, timestamp, category): + if category: + category = "DODI Repack" + else: + category = "Non_Repack" + return f""" + <![CDATA[{name}]]> + {timestamp} +0000 + {category} + + + """ + +def write_file(path, data): + if not os.path.exists("RSS"): + os.mkdir("RSS") + with open(os.path.realpath(path), "wt") as f: + f.write(data) + +def create_25_recent_feed(df): + most_recent_25 = df.sort_values("timestamp", ascending=False).head(25) + latest_25_xml = "\n".join([make_torrent_entry(i[0], i[1], i[3]) for i in most_recent_25.values]) + write_file("RSS/recent_25/index", XML_BEGIN_RECENT+latest_25_xml+XML_END) + +def create_48h_latest(df): + last_48_hours = df[df['timestamp'] > datetime.now() - pd.Timedelta(days=2)] + last_48_xml = "\n".join([make_torrent_entry(i[0], i[1], i[3]) for i in last_48_hours.values]) + write_file("RSS/latest_48/index", XML_BEGIN_LAST+last_48_xml+XML_END) + +def create_df(): + torfiles_scrape = requests.get("https://upload.fieryeagle.org/torfiles", auth=("",os.getenv("copyparty"))) + + soup = BeautifulSoup(torfiles_scrape.content, "html.parser") + torrent_files_names = [i.find_all("td")[1].text for i in soup.find_all("tbody")[0].find_all("tr")] + torrent_files_timestamps = [i.find_all("td")[7].text for i in soup.find_all("tbody")[0].find_all("tr")] + torrent_files_timestamps = [datetime.strptime(i, "%Y-%m-%d %H:%M:%S") for i in torrent_files_timestamps] + torrent_files_upload_ip = [i.find_all("td")[4].text for i in soup.find_all("tbody")[0].find_all("tr")] + + df = pd.DataFrame({ + "name": torrent_files_names, + "timestamp": torrent_files_timestamps, + "upload_ip": torrent_files_upload_ip + }) + + df['is_repack'] = df['name'].str.contains('DODI', case=False) + return df[df["upload_ip"] == "49.12.185.202"] \ No newline at end of file diff --git a/urlencode.py b/urlencode.py new file mode 100644 index 0000000..5f643a7 --- /dev/null +++ b/urlencode.py @@ -0,0 +1,2 @@ +def urlencode(inp: str): + return "".join([i if i in ["~", "-", "_", "."] or i.isalnum() else "%"+str(hex(ord(i))[2:].upper()) for i in inp]) \ No newline at end of file