diff --git a/archive.py b/archive.py index d3989a9..5035d0f 100644 --- a/archive.py +++ b/archive.py @@ -1,32 +1,36 @@ +# archive.py import logging -from waybackpy import WaybackMachineSaveAPI -import time import random +import time +from typing import List + +from waybackpy import WaybackMachineSaveAPI from config import ARCHIVE_URLS, USER_AGENT -# Configure logging -logging.basicConfig( - level=logging.INFO, - format='%(asctime)s - %(levelname)s - %(message)s' -) logger = logging.getLogger(__name__) -def archive_url(url): + +def archive_url(url: str): logger.info(f"π Archiving {url} ...") try: save_api = WaybackMachineSaveAPI(url, user_agent=USER_AGENT) save_api.save() logger.info(f"β Archived {url}") except Exception as e: - logger.error(f"β οΈ Exception archiving {url}: {e}") + logger.error(f"β οΈ Exception archiving {url}: {e}", exc_info=True) + def archive_all_urls(): + logger.info("--- Starting archival process for all URLs ---") for url in ARCHIVE_URLS: delay = 10 + random.uniform(-3, 3) + logger.info(f"Waiting {delay:.2f} seconds before next archive...") time.sleep(delay) archive_url(url) + logger.info("--- Archival process finished ---") + def test_archive(): test_url = "https://httpbin.org/anything/foo/bar" - archive_url(test_url) + archive_url(test_url) \ No newline at end of file diff --git a/config.py b/config.py index bd17e4c..9ccd2c5 100644 --- a/config.py +++ b/config.py @@ -11,6 +11,7 @@ CSV_FILENAME = "artists.csv" XLSX_FILENAME = "artists.xlsx" exclude_names = { + "πΉWorst Comps & Edits" "K4$H K4$$!n0", "K4HKn0", "AI Models", diff --git a/diff.py b/diff.py index ceb2937..2850e7d 100644 --- a/diff.py +++ b/diff.py @@ -1,22 +1,37 @@ +# diff.py import csv +import logging +from typing import Dict, List -def read_csv_to_dict(filename): - d = {} - with open(filename, newline='', encoding='utf-8') as f: - reader = csv.DictReader(f) - for row in reader: - d[row["Artist Name"]] = row - return d +logger = logging.getLogger(__name__) -def detect_changes(old_data, new_data): + +def read_csv_to_dict(filename: str) -> Dict[str, Dict[str, str]]: + data = {} + try: + with open(filename, newline="", encoding="utf-8") as f: + reader = csv.DictReader(f) + for row in reader: + if "Artist Name" in row and row["Artist Name"]: + data[row["Artist Name"]] = row + except FileNotFoundError: + logger.warning(f"CSV file not found: {filename}") + except Exception as e: + logger.error(f"Error reading CSV file {filename}: {e}", exc_info=True) + return data + + +def detect_changes( + old_data: Dict[str, Dict[str, str]], new_data: Dict[str, Dict[str, str]] +) -> List[str]: changes = [] old_keys = set(old_data.keys()) new_keys = set(new_data.keys()) - removed = old_keys - new_keys - added = new_keys - old_keys - common = old_keys & new_keys + removed = sorted(list(old_keys - new_keys)) + added = sorted(list(new_keys - old_keys)) + common = sorted(list(old_keys & new_keys)) for artist in removed: changes.append(f"β Removed: **{artist}**") @@ -28,15 +43,15 @@ def detect_changes(old_data, new_data): old_row = old_data[artist] new_row = new_data[artist] - if old_row["URL"] != new_row["URL"]: + if old_row.get("URL") != new_row.get("URL"): changes.append(f"π Link changed for **{artist}**") - if old_row["Credit"] != new_row["Credit"]: + if old_row.get("Credit") != new_row.get("Credit"): changes.append(f"βοΈ Credit changed for **{artist}**") - if old_row["Links Work"] != new_row["Links Work"]: + if old_row.get("Links Work") != new_row.get("Links Work"): changes.append(f"π Links Work status changed for **{artist}**") - if old_row["Updated"] != new_row["Updated"]: + if old_row.get("Updated") != new_row.get("Updated"): changes.append(f"π Updated date changed for **{artist}**") - if old_row["Best"] != new_row["Best"]: + if old_row.get("Best") != new_row.get("Best"): changes.append(f"β Best flag changed for **{artist}**") - return changes + return changes \ No newline at end of file diff --git a/downloader.py b/downloader.py index 94f1470..36da269 100644 --- a/downloader.py +++ b/downloader.py @@ -1,36 +1,42 @@ -import requests, zipfile +# downloader.py +import logging +import zipfile + +import requests + +from config import HTML_FILENAME, XLSX_FILENAME, XLSX_URL, ZIP_FILENAME, ZIP_URL + +logger = logging.getLogger(__name__) + + +def _download_file(url: str, filename: str, timeout: int = 30) -> bool: + logger.info(f"π Downloading {filename}...") + try: + with requests.get(url, timeout=timeout) as r: + r.raise_for_status() + with open(filename, "wb") as f: + f.write(r.content) + logger.info(f"β Saved {filename}") + return True + except requests.RequestException as e: + logger.error(f"β Failed to download {filename}: {e}") + return False -from config import ZIP_URL, ZIP_FILENAME, HTML_FILENAME, XLSX_URL, XLSX_FILENAME def download_zip_and_extract_html(): - print("π Downloading ZIP...") - try: - with requests.get(ZIP_URL, timeout=30) as r: - r.raise_for_status() - with open(ZIP_FILENAME, "wb") as f: - f.write(r.content) - print(f"β Saved ZIP as {ZIP_FILENAME}") - except requests.RequestException as e: - print(f"β Failed to download ZIP: {e}") + if not _download_file(ZIP_URL, ZIP_FILENAME): return + logger.info(f"π¦ Extracting {HTML_FILENAME} from {ZIP_FILENAME}...") try: with zipfile.ZipFile(ZIP_FILENAME, "r") as z: - with z.open(HTML_FILENAME) as html_file: - html_content = html_file.read() - with open(HTML_FILENAME, "wb") as f: - f.write(html_content) - print(f"β Extracted {HTML_FILENAME}") - except (zipfile.BadZipFile, KeyError) as e: - print(f"β Failed to extract {HTML_FILENAME}: {e}") + html_content = z.read(HTML_FILENAME) + with open(HTML_FILENAME, "wb") as f: + f.write(html_content) + logger.info(f"β Extracted {HTML_FILENAME}") + except (zipfile.BadZipFile, KeyError, FileNotFoundError) as e: + logger.error(f"β Failed to extract {HTML_FILENAME}: {e}") + def download_xlsx(): - print("π Downloading XLSX...") - try: - with requests.get(XLSX_URL, timeout=30) as r: - r.raise_for_status() - with open(XLSX_FILENAME, "wb") as f: - f.write(r.content) - print(f"β Saved XLSX as {XLSX_FILENAME}") - except requests.RequestException as e: - print(f"β Failed to download XLSX: {e}") + _download_file(XLSX_URL, XLSX_FILENAME) \ No newline at end of file diff --git a/frontend/artistgrid-sheets-frontend/.gitignore b/frontend/artistgrid-sheets-frontend/.gitignore new file mode 100644 index 0000000..5ef6a52 --- /dev/null +++ b/frontend/artistgrid-sheets-frontend/.gitignore @@ -0,0 +1,41 @@ +# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. + +# dependencies +/node_modules +/.pnp +.pnp.* +.yarn/* +!.yarn/patches +!.yarn/plugins +!.yarn/releases +!.yarn/versions + +# testing +/coverage + +# next.js +/.next/ +/out/ + +# production +/build + +# misc +.DS_Store +*.pem + +# debug +npm-debug.log* +yarn-debug.log* +yarn-error.log* +.pnpm-debug.log* + +# env files (can opt-in for committing if needed) +.env* + +# vercel +.vercel + +# typescript +*.tsbuildinfo +next-env.d.ts diff --git a/frontend/artistgrid-sheets-frontend/README.md b/frontend/artistgrid-sheets-frontend/README.md new file mode 100644 index 0000000..e215bc4 --- /dev/null +++ b/frontend/artistgrid-sheets-frontend/README.md @@ -0,0 +1,36 @@ +This is a [Next.js](https://nextjs.org) project bootstrapped with [`create-next-app`](https://nextjs.org/docs/app/api-reference/cli/create-next-app). + +## Getting Started + +First, run the development server: + +```bash +npm run dev +# or +yarn dev +# or +pnpm dev +# or +bun dev +``` + +Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. + +You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file. + +This project uses [`next/font`](https://nextjs.org/docs/app/building-your-application/optimizing/fonts) to automatically optimize and load [Geist](https://vercel.com/font), a new font family for Vercel. + +## Learn More + +To learn more about Next.js, take a look at the following resources: + +- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. +- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. + +You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js) - your feedback and contributions are welcome! + +## Deploy on Vercel + +The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. + +Check out our [Next.js deployment documentation](https://nextjs.org/docs/app/building-your-application/deploying) for more details. diff --git a/frontend/artistgrid-sheets-frontend/app/favicon.ico b/frontend/artistgrid-sheets-frontend/app/favicon.ico new file mode 100644 index 0000000..718d6fe Binary files /dev/null and b/frontend/artistgrid-sheets-frontend/app/favicon.ico differ diff --git a/frontend/artistgrid-sheets-frontend/app/globals.css b/frontend/artistgrid-sheets-frontend/app/globals.css new file mode 100644 index 0000000..dc98be7 --- /dev/null +++ b/frontend/artistgrid-sheets-frontend/app/globals.css @@ -0,0 +1,122 @@ +@import "tailwindcss"; +@import "tw-animate-css"; + +@custom-variant dark (&:is(.dark *)); + +@theme inline { + --color-background: var(--background); + --color-foreground: var(--foreground); + --font-sans: var(--font-geist-sans); + --font-mono: var(--font-geist-mono); + --color-sidebar-ring: var(--sidebar-ring); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar: var(--sidebar); + --color-chart-5: var(--chart-5); + --color-chart-4: var(--chart-4); + --color-chart-3: var(--chart-3); + --color-chart-2: var(--chart-2); + --color-chart-1: var(--chart-1); + --color-ring: var(--ring); + --color-input: var(--input); + --color-border: var(--border); + --color-destructive: var(--destructive); + --color-accent-foreground: var(--accent-foreground); + --color-accent: var(--accent); + --color-muted-foreground: var(--muted-foreground); + --color-muted: var(--muted); + --color-secondary-foreground: var(--secondary-foreground); + --color-secondary: var(--secondary); + --color-primary-foreground: var(--primary-foreground); + --color-primary: var(--primary); + --color-popover-foreground: var(--popover-foreground); + --color-popover: var(--popover); + --color-card-foreground: var(--card-foreground); + --color-card: var(--card); + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); +} + +:root { + --radius: 0.625rem; + --background: oklch(1 0 0); + --foreground: oklch(0.145 0 0); + --card: oklch(1 0 0); + --card-foreground: oklch(0.145 0 0); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.145 0 0); + --primary: oklch(0.205 0 0); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.97 0 0); + --secondary-foreground: oklch(0.205 0 0); + --muted: oklch(0.97 0 0); + --muted-foreground: oklch(0.556 0 0); + --accent: oklch(0.97 0 0); + --accent-foreground: oklch(0.205 0 0); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.922 0 0); + --input: oklch(0.922 0 0); + --ring: oklch(0.708 0 0); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.145 0 0); + --sidebar-primary: oklch(0.205 0 0); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.97 0 0); + --sidebar-accent-foreground: oklch(0.205 0 0); + --sidebar-border: oklch(0.922 0 0); + --sidebar-ring: oklch(0.708 0 0); +} + +.dark { + --background: oklch(0.145 0 0); + --foreground: oklch(0.985 0 0); + --card: oklch(0.205 0 0); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.205 0 0); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.922 0 0); + --primary-foreground: oklch(0.205 0 0); + --secondary: oklch(0.269 0 0); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.269 0 0); + --muted-foreground: oklch(0.708 0 0); + --accent: oklch(0.269 0 0); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.556 0 0); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.205 0 0); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.269 0 0); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.556 0 0); +} + +@layer base { + * { + @apply border-border outline-ring/50; + } + body { + @apply bg-background text-foreground; + } +} diff --git a/frontend/artistgrid-sheets-frontend/app/layout.tsx b/frontend/artistgrid-sheets-frontend/app/layout.tsx new file mode 100644 index 0000000..408f5cc --- /dev/null +++ b/frontend/artistgrid-sheets-frontend/app/layout.tsx @@ -0,0 +1,37 @@ +import type { Metadata } from "next"; +import { Geist, Geist_Mono } from "next/font/google"; +import "./globals.css"; + +const geistSans = Geist({ + variable: "--font-geist-sans", + subsets: ["latin"], +}); + +const geistMono = Geist_Mono({ + variable: "--font-geist-mono", + subsets: ["latin"], +}); + +export const metadata: Metadata = { + title: "ArtistGrid Sheets", + description: "We pull from TrackerHub and parse it into a CSV file. Still a work in progress.", + icons: { + icon: "/favicon.png", // make sure favicon.png is in /public + }, +}; + +export default function RootLayout({ + children, +}: Readonly<{ + children: React.ReactNode; +}>) { + return ( + +
+ {children} + + + ); +} diff --git a/frontend/artistgrid-sheets-frontend/app/page.tsx b/frontend/artistgrid-sheets-frontend/app/page.tsx new file mode 100644 index 0000000..08a5d72 --- /dev/null +++ b/frontend/artistgrid-sheets-frontend/app/page.tsx @@ -0,0 +1,68 @@ +"use client"; + +import { Button } from "@/components/ui/button"; +import { Github, FileDown, FileText, FileSpreadsheet } from "lucide-react"; + +const buttonData = [ + { + name: "View on GitHub", + href: "https://github.com/ArtistGrid/Sheets", + icon: Github, + isExternal: true, + }, + { + name: "Download CSV", + href: "https://sheets.artistgrid.cx/artists.csv", + icon: FileDown, + downloadName: "artists.csv", + }, + { + name: "View HTML", + href: "https://sheets.artistgrid.cx/artists.html", + icon: FileText, + isExternal: true, + }, + { + name: "Download XLSX", + href: "https://sheets.artistgrid.cx/artists.xlsx", + icon: FileSpreadsheet, + downloadName: "ArtistGrid.xlsx", + }, +]; + +export default function ArtistGridSheets() { + return ( ++ We pull from TrackerHub and parse it into a CSV file. Still a work in + progress. +
+Last Updated: {data.get('last_updated')}
-Status info not available.
", 404 + data = get_status_data() + if not data: + return "Status info not available.
", 404 + + files_info = data.get("files", {}) + html_info = files_info.get(HTML_FILENAME, {}) + csv_info = files_info.get(CSV_FILENAME, {}) + xlsx_info = files_info.get(XLSX_FILENAME, {}) + + return f""" + + + + +Last Updated: {data.get('last_updated', 'N/A')}
+