import { parseHTML } from "linkedom";
import {
jsonToCsv,
ndjsonToJson,
prettyStringify,
requireBasicAuth,
tripleBool,
type Change,
type Entry,
} from "./lib";
import * as fs from "fs/promises";
import { existsSync } from "fs";
import { $, type BodyInit } from "bun";
interface ChangeWithMessage {
changes: Change[];
webhook: unknown;
}
let changelist_ids: Record = {};
if (existsSync("./changelist_ids.json")) {
changelist_ids = JSON.parse(
(await fs.readFile("./changelist_ids.json")).toString("utf8"),
);
}
async function updateChangelistIds() {
await fs.writeFile("./changelist_ids.json", JSON.stringify(changelist_ids));
}
const tracker_page =
"https://docs.google.com/spreadsheets/u/0/d/1Z8aANbxXbnUGoZPRvJfWL3gz6jrzPPrwVt3d0c1iJ_4/htmlview/sheet?headers=false&gid=1884837542";
async function getTH() {
const req = await fetch(tracker_page);
const txt = await req.text();
const { document } = parseHTML(txt);
const table_body = document.querySelector(".waffle > tbody");
if (!table_body) throw new Error("Missing table body..");
const rows = [...table_body.children];
let ndjson = "";
for (let i = 4; i < rows.length; i++) {
if (!rows[i].children[1]) {
break;
}
let trackerName = rows[i].children[1].innerText;
if (!trackerName) continue;
const urlElement = rows[i].children[1].querySelector("a");
if (!urlElement) continue;
const trackerUrl = new URL(urlElement.href).searchParams.get("q");
const credit = rows[i].children[2].innerText;
const updated = tripleBool(rows[i].children[3].innerText);
const links_work = tripleBool(rows[i].children[4].innerText);
const best = trackerName.startsWith("⭐");
trackerName = trackerName
.replace(/\p{Extended_Pictographic}/gu, "")
.replace(/[\uFE00-\uFE0F]/g, "")
.replace(/\u200D/g, "")
.trim();
ndjson +=
JSON.stringify({
name: trackerName,
url: trackerUrl,
credit,
updated,
links_work,
best,
}) + "\n";
}
return ndjson;
}
async function runComparison() {
const data = await getTH();
const old = (await fs.readFile("./th_artists.ndjson")).toString("utf8");
if (Bun.hash(old) !== Bun.hash(data)) {
const oldJson: Entry[] = ndjsonToJson(old) as Entry[];
const newJson: Entry[] = ndjsonToJson(data) as Entry[];
const oldMap = Object.fromEntries(oldJson.map((item) => [item.name, item]));
const newMap = Object.fromEntries(newJson.map((item) => [item.name, item]));
const changes: Change[] = [];
for (const name in oldMap) {
const oldItem = oldMap[name]!;
const newItem = newMap[name];
if (!newItem) {
changes.push({ op: "delete", name });
continue;
}
const delta: Partial = {};
if (oldItem.url !== newItem.url) delta.url = newItem.url;
if (oldItem.credit !== newItem.credit) delta.credit = newItem.credit;
if (oldItem.links_work !== newItem.links_work)
delta.links_work = newItem.links_work;
if (oldItem.updated !== newItem.updated) delta.updated = newItem.updated;
if (oldItem.best !== newItem.best) delta.best = newItem.best;
if (Object.keys(delta).length > 0) {
changes.push({ op: "update", name, changes: delta });
}
}
for (const name in newMap) {
if (!oldMap[name]) {
changes.push({ op: "create", name, item: newMap[name]! });
}
}
let message = "";
for (const c of changes) {
if (c.op === "delete") {
message += `**DELETED**: \`${c.name}\`\n`;
} else if (c.op === "create") {
message += `**NEW**: \`${c.name}\`\n`;
} else if (c.op === "update") {
for (const k in c.changes) {
message += `**CHANGED ${k.toUpperCase()}** for \`${c.name}\`\n`;
}
}
}
if (message) {
const id = crypto.randomUUID().split("-")[0]!;
const z = await fetch(
process.env.WEBHOOK_URL! + "?wait=true&with_components=true",
{
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
components: [
{
type: 1,
components: [
{
type: 2,
style: 5,
label: "Accept",
emoji: {
name: "✅",
},
url: `${process.env.PUBLIC}/merge-th/${id}`,
},
{
type: 2,
style: 5,
url: `${process.env.PUBLIC}/ignore-th/${id}`,
label: "Deny",
emoji: {
name: "❌",
},
},
],
},
],
embeds: [
{
title: "TH Tracker Changes",
description: message,
},
],
}),
},
);
const json = await z.json();
changelist_ids[id] = {
changes,
webhook: json,
};
await updateChangelistIds();
}
await fs.writeFile("./th_artists.ndjson", data);
}
}
export class ClientResponse extends Response {
constructor(url: URL, body?: BodyInit, init?: ResponseInit) {
super(body, init);
const origins = process.env.ORIGIN!.split(",");
if (origins.includes(url.origin)) {
this.headers.set("Access-Control-Allow-Origin", url.origin);
this.headers.set(
"Access-Control-Allow-Methods",
"GET, POST, PUT, DELETE, OPTIONS",
);
}
}
static json_c(body: unknown, url: URL, init?: ResponseInit): Response {
const res = new Response(JSON.stringify(body), {
...init,
headers: {
"Content-Type": "application/json",
...init?.headers,
},
});
const origins = process.env.ORIGIN!.split(",");
if (origins.includes(url.origin)) {
res.headers.set("Access-Control-Allow-Origin", url.origin);
res.headers.set(
"Access-Control-Allow-Methods",
"GET, POST, PUT, DELETE, OPTIONS",
);
return res;
} else {
return res;
}
}
}
Bun.serve({
routes: {
"/": (req) => new ClientResponse(new URL(req.url), "Sheets v2"),
"/artists.json": async (req) =>
ClientResponse.json_c(
ndjsonToJson((await fs.readFile("artists.ndjson")).toString("utf8")),
new URL(req.url),
),
"/artists.csv": async (req) =>
new ClientResponse(
new URL(req.url),
jsonToCsv(
ndjsonToJson((await fs.readFile("artists.ndjson")).toString("utf8")),
),
{
headers: {
"Content-Type": "text/csv",
},
},
),
"/artists.ndjson": async (req) =>
new ClientResponse(new URL(req.url), await fs.readFile("artists.ndjson")),
"/th_artists.ndjson": async (req) =>
new ClientResponse(
new URL(req.url),
await fs.readFile("th_artists.ndjson"),
),
"/ignore-th/:id": async (req) => {
const authFail = requireBasicAuth(req);
if (authFail) return authFail;
const id = req.params.id;
const changes = changelist_ids[id];
if (!changes) {
return new ClientResponse(new URL(req.url), "Id is invalid.", {
status: 404,
});
}
const embed = (changes.webhook as { embeds: any[] }).embeds[0];
embed.title = embed.title + " - Denied.";
await fetch(
process.env.WEBHOOK_URL! +
"/messages/" +
(changes.webhook as { id: string }).id +
"?with_components=true",
{
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
embeds: [embed],
components: [],
}),
},
);
delete changelist_ids[id];
await updateChangelistIds();
return new ClientResponse(new URL(req.url), "Ignored successfully.");
},
"/merge-th/:id": async (req) => {
const authFail = requireBasicAuth(req);
if (authFail) return authFail;
const id = req.params.id;
const changes = changelist_ids[id];
if (!changes) {
return new ClientResponse(new URL(req.url), "Id is invalid.", {
status: 404,
});
}
const artistsRaw = await fs.readFile("./artists.ndjson", "utf8");
const artists = ndjsonToJson(artistsRaw) as Entry[];
const map = new Map(artists.map((a) => [a.name, a]));
for (const change of changes.changes) {
if (change.op === "delete") {
map.delete(change.name);
}
if (change.op === "create") {
map.set(change.name, change.item);
}
if (change.op === "update") {
const item = map.get(change.name);
if (!item) continue;
Object.assign(item, change.changes);
}
}
const merged =
[...map.values()].map((v) => prettyStringify(v)).join("\n") + "\n";
await fs.writeFile("./artists.ndjson", merged);
await fs.writeFile(
"./th_artists.ndjson",
await fs.readFile("./th_artists.ndjson"),
);
const embed = (changes.webhook as { embeds: any[] }).embeds[0];
embed.title = embed.title + " - Accepted!";
await fetch(
process.env.WEBHOOK_URL! +
"/messages/" +
(changes.webhook as { id: string }).id +
"?with_components=true",
{
method: "PATCH",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
embeds: [embed],
components: [],
}),
},
);
delete changelist_ids[id];
await updateChangelistIds();
try {
await $`git add artists.ndjson`;
await $`git commit -m "Auto-merge."`;
const push = await $`git push origin main`.nothrow();
const out = `${push.stdout}${push.stderr}`;
if (out.includes("its remote counterpart")) {
await $`git pull --rebase`;
const retry = await $`git push origin main`.nothrow();
const retryOut = `${retry.stdout}${retry.stderr}`;
if (!retryOut.includes("main -> main")) {
throw new Error("Push failed after rebase");
}
} else if (!out.includes("main -> main")) {
throw new Error("Push failed");
}
} catch (err) {
console.error("Error:", err);
return new ClientResponse(
new URL(req.url),
"Failed to automerge to git. Error: " + err,
{
status: 400,
},
);
}
return new ClientResponse(new URL(req.url), "Merged successfully.");
},
},
fetch(req) {
return new ClientResponse(new URL(req.url), "Unmatched route");
},
hostname: process.env.HOST || "127.0.0.1",
});
if (!existsSync("./th_artists.ndjson")) {
console.log("Assuming first run. Downloading TH info and waiting 50s.");
await fs.writeFile("./th_artists.ndjson", await getTH());
} else {
await runComparison();
}
setInterval(async () => {
await runComparison();
}, 50000);