import { parseHTML } from "linkedom"; import { jsonToCsv, ndjsonToJson, prettyStringify, requireBasicAuth, tripleBool, type Change, type Entry, } from "./lib"; import * as fs from "fs/promises"; import { existsSync } from "fs"; import { $, type BodyInit } from "bun"; interface ChangeWithMessage { changes: Change[]; webhook: unknown; } let changelist_ids: Record = {}; if (existsSync("./changelist_ids.json")) { changelist_ids = JSON.parse( (await fs.readFile("./changelist_ids.json")).toString("utf8"), ); } async function updateChangelistIds() { await fs.writeFile("./changelist_ids.json", JSON.stringify(changelist_ids)); } const tracker_page = "https://docs.google.com/spreadsheets/u/0/d/1Z8aANbxXbnUGoZPRvJfWL3gz6jrzPPrwVt3d0c1iJ_4/htmlview/sheet?headers=false&gid=1884837542"; async function getTH() { const req = await fetch(tracker_page); const txt = await req.text(); const { document } = parseHTML(txt); const table_body = document.querySelector(".waffle > tbody"); if (!table_body) throw new Error("Missing table body.."); const rows = [...table_body.children]; let ndjson = ""; for (let i = 4; i < rows.length; i++) { if (!rows[i].children[1]) { break; } let trackerName = rows[i].children[1].innerText; if (!trackerName) continue; const urlElement = rows[i].children[1].querySelector("a"); if (!urlElement) continue; const trackerUrl = new URL(urlElement.href).searchParams.get("q"); const credit = rows[i].children[2].innerText; const updated = tripleBool(rows[i].children[3].innerText); const links_work = tripleBool(rows[i].children[4].innerText); const best = trackerName.startsWith("⭐"); trackerName = trackerName .replace(/\p{Extended_Pictographic}/gu, "") .replace(/[\uFE00-\uFE0F]/g, "") .replace(/\u200D/g, "") .trim(); ndjson += JSON.stringify({ name: trackerName, url: trackerUrl, credit, updated, links_work, best, }) + "\n"; } return ndjson; } async function runComparison() { const data = await getTH(); const old = (await fs.readFile("./th_artists.ndjson")).toString("utf8"); if (Bun.hash(old) !== Bun.hash(data)) { const oldJson: Entry[] = ndjsonToJson(old) as Entry[]; const newJson: Entry[] = ndjsonToJson(data) as Entry[]; const oldMap = Object.fromEntries(oldJson.map((item) => [item.name, item])); const newMap = Object.fromEntries(newJson.map((item) => [item.name, item])); const changes: Change[] = []; for (const name in oldMap) { const oldItem = oldMap[name]!; const newItem = newMap[name]; if (!newItem) { changes.push({ op: "delete", name }); continue; } const delta: Partial = {}; if (oldItem.url !== newItem.url) delta.url = newItem.url; if (oldItem.credit !== newItem.credit) delta.credit = newItem.credit; if (oldItem.links_work !== newItem.links_work) delta.links_work = newItem.links_work; if (oldItem.updated !== newItem.updated) delta.updated = newItem.updated; if (oldItem.best !== newItem.best) delta.best = newItem.best; if (Object.keys(delta).length > 0) { changes.push({ op: "update", name, changes: delta }); } } for (const name in newMap) { if (!oldMap[name]) { changes.push({ op: "create", name, item: newMap[name]! }); } } let message = ""; for (const c of changes) { if (c.op === "delete") { message += `**DELETED**: \`${c.name}\`\n`; } else if (c.op === "create") { message += `**NEW**: \`${c.name}\`\n`; } else if (c.op === "update") { for (const k in c.changes) { message += `**CHANGED ${k.toUpperCase()}** for \`${c.name}\`\n`; } } } if (message) { const id = crypto.randomUUID().split("-")[0]!; const z = await fetch( process.env.WEBHOOK_URL! + "?wait=true&with_components=true", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ components: [ { type: 1, components: [ { type: 2, style: 5, label: "Accept", emoji: { name: "✅", }, url: `${process.env.PUBLIC}/merge-th/${id}`, }, { type: 2, style: 5, url: `${process.env.PUBLIC}/ignore-th/${id}`, label: "Deny", emoji: { name: "❌", }, }, ], }, ], embeds: [ { title: "TH Tracker Changes", description: message, }, ], }), }, ); const json = await z.json(); changelist_ids[id] = { changes, webhook: json, }; await updateChangelistIds(); } await fs.writeFile("./th_artists.ndjson", data); } } export class ClientResponse extends Response { constructor(url: URL, body?: BodyInit, init?: ResponseInit) { super(body, init); const origins = process.env.ORIGIN!.split(","); if (origins.includes(url.origin)) { this.headers.set("Access-Control-Allow-Origin", url.origin); this.headers.set( "Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS", ); } } static json_c(body: unknown, url: URL, init?: ResponseInit): Response { const res = new Response(JSON.stringify(body), { ...init, headers: { "Content-Type": "application/json", ...init?.headers, }, }); const origins = process.env.ORIGIN!.split(","); if (origins.includes(url.origin)) { res.headers.set("Access-Control-Allow-Origin", url.origin); res.headers.set( "Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS", ); return res; } else { return res; } } } Bun.serve({ routes: { "/": (req) => new ClientResponse(new URL(req.url), "Sheets v2"), "/artists.json": async (req) => ClientResponse.json_c( ndjsonToJson((await fs.readFile("artists.ndjson")).toString("utf8")), new URL(req.url), ), "/artists.csv": async (req) => new ClientResponse( new URL(req.url), jsonToCsv( ndjsonToJson((await fs.readFile("artists.ndjson")).toString("utf8")), ), { headers: { "Content-Type": "text/csv", }, }, ), "/artists.ndjson": async (req) => new ClientResponse(new URL(req.url), await fs.readFile("artists.ndjson")), "/th_artists.ndjson": async (req) => new ClientResponse( new URL(req.url), await fs.readFile("th_artists.ndjson"), ), "/ignore-th/:id": async (req) => { const authFail = requireBasicAuth(req); if (authFail) return authFail; const id = req.params.id; const changes = changelist_ids[id]; if (!changes) { return new ClientResponse(new URL(req.url), "Id is invalid.", { status: 404, }); } const embed = (changes.webhook as { embeds: any[] }).embeds[0]; embed.title = embed.title + " - Denied."; await fetch( process.env.WEBHOOK_URL! + "/messages/" + (changes.webhook as { id: string }).id + "?with_components=true", { method: "PATCH", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ embeds: [embed], components: [], }), }, ); delete changelist_ids[id]; await updateChangelistIds(); return new ClientResponse(new URL(req.url), "Ignored successfully."); }, "/merge-th/:id": async (req) => { const authFail = requireBasicAuth(req); if (authFail) return authFail; const id = req.params.id; const changes = changelist_ids[id]; if (!changes) { return new ClientResponse(new URL(req.url), "Id is invalid.", { status: 404, }); } const artistsRaw = await fs.readFile("./artists.ndjson", "utf8"); const artists = ndjsonToJson(artistsRaw) as Entry[]; const map = new Map(artists.map((a) => [a.name, a])); for (const change of changes.changes) { if (change.op === "delete") { map.delete(change.name); } if (change.op === "create") { map.set(change.name, change.item); } if (change.op === "update") { const item = map.get(change.name); if (!item) continue; Object.assign(item, change.changes); } } const merged = [...map.values()].map((v) => prettyStringify(v)).join("\n") + "\n"; await fs.writeFile("./artists.ndjson", merged); await fs.writeFile( "./th_artists.ndjson", await fs.readFile("./th_artists.ndjson"), ); const embed = (changes.webhook as { embeds: any[] }).embeds[0]; embed.title = embed.title + " - Accepted!"; await fetch( process.env.WEBHOOK_URL! + "/messages/" + (changes.webhook as { id: string }).id + "?with_components=true", { method: "PATCH", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ embeds: [embed], components: [], }), }, ); delete changelist_ids[id]; await updateChangelistIds(); try { await $`git add artists.ndjson`; await $`git commit -m "Auto-merge."`; const push = await $`git push origin main`.nothrow(); const out = `${push.stdout}${push.stderr}`; if (out.includes("its remote counterpart")) { await $`git pull --rebase`; const retry = await $`git push origin main`.nothrow(); const retryOut = `${retry.stdout}${retry.stderr}`; if (!retryOut.includes("main -> main")) { throw new Error("Push failed after rebase"); } } else if (!out.includes("main -> main")) { throw new Error("Push failed"); } } catch (err) { console.error("Error:", err); return new ClientResponse( new URL(req.url), "Failed to automerge to git. Error: " + err, { status: 400, }, ); } return new ClientResponse(new URL(req.url), "Merged successfully."); }, }, fetch(req) { return new ClientResponse(new URL(req.url), "Unmatched route"); }, hostname: process.env.HOST || "127.0.0.1", }); if (!existsSync("./th_artists.ndjson")) { console.log("Assuming first run. Downloading TH info and waiting 50s."); await fs.writeFile("./th_artists.ndjson", await getTH()); } else { await runComparison(); } setInterval(async () => { await runComparison(); }, 50000);