import { parseHTML } from "linkedom" import { ndjsonToJson, prettyStringify, requireBasicAuth, tripleBool, type Change, type Entry } from "./lib"; import * as fs from "fs/promises" import { existsSync } from "fs"; interface ChangeWithMessage { changes: Change[], webhook: unknown } let changelist_ids: Record = {} if(existsSync("./changelist_ids.json")) { changelist_ids = JSON.parse((await fs.readFile("./changelist_ids.json")).toString("utf8")) } async function updateChangelistIds() { await fs.writeFile("./changelist_ids.json", JSON.stringify(changelist_ids)); } const tracker_page = "https://docs.google.com/spreadsheets/u/0/d/1Z8aANbxXbnUGoZPRvJfWL3gz6jrzPPrwVt3d0c1iJ_4/htmlview/sheet?headers=false&gid=1884837542" async function getTH() { const req = await fetch(tracker_page); const txt = await req.text() const { document } = parseHTML(txt); const table_body = document.querySelector(".waffle > tbody"); if(!table_body) throw new Error("Missing table body..") //@ts-expect-error .children can be spread-operator'd const rows = [...table_body.children] let ndjson = ""; for (let i = 4; i < rows.length; i++) { if(!rows[i].children[1]) { break; } let trackerName = rows[i].children[1].innerText; if (!trackerName) continue; const urlElement = rows[i].children[1].querySelector("a"); if (!urlElement) continue; const trackerUrl = new URL(urlElement.href).searchParams.get("q"); const credits = rows[i].children[2].innerText; const updated = tripleBool(rows[i].children[3].innerText); const links_work = tripleBool(rows[i].children[4].innerText); const best = trackerName.startsWith("⭐"); trackerName = trackerName .replace(/\p{Extended_Pictographic}/gu, '') .replace(/[\uFE00-\uFE0F]/g, '') .replace(/\u200D/g, '') .trim(); ndjson += JSON.stringify({ name: trackerName, url: trackerUrl, credits, updated, links_work, best }) + "\n"; } return ndjson; } async function runComparison() { const data = await getTH(); const old = (await fs.readFile("./th_artists.ndjson")).toString("utf8"); if(Bun.hash(old) !== Bun.hash(data)) { const oldJson: Entry[] = ndjsonToJson(old) as Entry[]; const newJson: Entry[] = ndjsonToJson(data) as Entry[]; const oldMap = Object.fromEntries(oldJson.map(item => [item.name, item])); const newMap = Object.fromEntries(newJson.map(item => [item.name, item])); const changes: Change[] = []; for (const name in oldMap) { const oldItem = oldMap[name]!; const newItem = newMap[name]; if (!newItem) { changes.push({ op: "delete", name }); continue; } const delta: Partial = {}; if (oldItem.url !== newItem.url) delta.url = newItem.url; if (oldItem.credits !== newItem.credits) delta.credits = newItem.credits; if (oldItem.links_work !== newItem.links_work) delta.links_work = newItem.links_work; if (oldItem.updated !== newItem.updated) delta.updated = newItem.updated; if (oldItem.best !== newItem.best) delta.best = newItem.best; if (Object.keys(delta).length > 0) { changes.push({ op: "update", name, changes: delta }); } } for (const name in newMap) { if (!oldMap[name]) { changes.push({ op: "create", name, item: newMap[name]! }); } } let message = ""; for (const c of changes) { if (c.op === "delete") { message += `**DELETED**: \`${c.name}\`\n`; } else if (c.op === "create") { message += `**NEW**: \`${c.name}\`\n`; } else if (c.op === "update") { for (const k in c.changes) { message += `**CHANGED ${k.toUpperCase()}** for \`${c.name}\`\n`; } } } if (message) { const id = crypto.randomUUID().split("-")[0]!; const z = await fetch(process.env.WEBHOOK_URL! + "?wait=true&with_components=true", { method: 'POST', headers: { "Content-Type": "application/json" }, body: JSON.stringify({ "components": [ { "type": 1, "components": [ { "type": 2, "style": 5, "label": "Accept", "emoji": { "name": "✅" }, "url": `${process.env.PUBLIC}/merge-th/${id}`, }, { "type": 2, "style": 5, "url": `${process.env.PUBLIC}/ignore-th/${id}`, "label": "Deny", "emoji": { "name": "❌" } } ] } ], "embeds": [{ "title": "TH Tracker Changes", "description": message, }] }) }); const json = await z.json() changelist_ids[id] = { changes, webhook: json } await updateChangelistIds(); } await fs.writeFile("./th_artists.ndjson", data); } } export class ClientResponse extends Response { constructor(body?: BodyInit, init?: ResponseInit) { super(body, init); this.headers.set("Access-Control-Allow-Origin", process.env.ORIGIN!); this.headers.set("Access-Control-Allow-Methods", "OPTIONS, GET, POST, PUT, DELETE"); } static override json(body: unknown, init?: ResponseInit): Response { return new Response(JSON.stringify(body), { ...init, headers: { "Content-Type": "application/json", "Access-Control-Allow-Origin": process.env.ORIGIN!, "Access-Control-Allow-Methods": "OPTIONS, GET, POST, PUT, DELETE", ...init?.headers, }, }); } } Bun.serve({ routes: { "/": () => new ClientResponse("Sheets v2"), "/artists.ndjson": async () => new ClientResponse(await fs.readFile("artists.ndjson")), "/th_artists.ndjson": async () => new ClientResponse(await fs.readFile("th_artists.ndjson")), "/ignore-th/:id": async (req) => { const authFail = requireBasicAuth(req); if (authFail) return authFail; const id = req.params.id; const changes = changelist_ids[id]; if (!changes) { return new ClientResponse("Id is invalid.", { status: 404 }); } const embed = changes.webhook.embeds[0]; embed.title = embed.title + " - Denied." await fetch(process.env.WEBHOOK_URL! + "/messages/" + changes.webhook.id + "?with_components=true", { method: 'PATCH', headers: { "Content-Type": "application/json" }, body: JSON.stringify({ embeds: [embed], components: [] }) }); delete changelist_ids[id]; await updateChangelistIds(); return new ClientResponse("Ignored successfully."); }, "/merge-th/:id": async (req) => { const authFail = requireBasicAuth(req); if (authFail) return authFail; const id = req.params.id; const changes = changelist_ids[id]; if (!changes) { return new ClientResponse("Id is invalid.", { status: 404 }); } const artistsRaw = await fs.readFile("./artists.ndjson", "utf8"); const artists = ndjsonToJson(artistsRaw) as Entry[]; const map = new Map( artists.map(a => [a.name, a]) ); for (const change of changes.changes) { if (change.op === "delete") { map.delete(change.name); } if (change.op === "create") { map.set(change.name, change.item); } if (change.op === "update") { const item = map.get(change.name); if (!item) continue; Object.assign(item, change.changes); } } const merged = [...map.values()] .map(v => prettyStringify(v)) .join("\n") + "\n"; await fs.writeFile("./artists.ndjson", merged); await fs.writeFile( "./th_artists.ndjson", await fs.readFile("./th_artists.ndjson") ); const embed = changes.webhook.embeds[0]; embed.title = embed.title + " - Accepted!" await fetch(process.env.WEBHOOK_URL! + "/messages/" + changes.webhook.id + "?with_components=true", { method: 'PATCH', headers: { "Content-Type": "application/json" }, body: JSON.stringify({ embeds: [embed], components: [] }) }); delete changelist_ids[id]; await updateChangelistIds(); return new ClientResponse("Merged successfully."); } }, fetch() { return new ClientResponse("Unmatched route"); }, hostname: process.env.HOST || "127.0.0.1", }); if (!existsSync("./th_artists.ndjson")) { console.log("Assuming first run. Downloading TH info and waiting 50s.") await fs.writeFile("./th_artists.ndjson", await getTH()); } else { await runComparison(); } setInterval(async () => { await runComparison(); }, 50000)