server.ts

import Koa from "koa";
import bodyParser from "koa-bodyparser";
import compression from "koa-compress";
import morgan from "koa-morgan";
import Router from "koa-router";
import { inspect } from "util";
import "dotenv/config";
import { Client, auth } from "twitter-api-sdk";

const port = 7860;

const app = new Koa();

app.use(morgan("dev"));
app.use(compression());
app.use(bodyParser());

const { API_KEY, API_SECRET, BEARER_TOKEN, CLIENT_ID, CLIENT_SECRET, COOKIE } = process.env;

const router = new Router();

app.use(router.routes());
app.use(router.allowedMethods());

// Initialize auth client first
const authClient = new auth.OAuth2User({
	client_id: CLIENT_ID as string,
	client_secret: CLIENT_SECRET as string,
	callback: "https://huggingface-projects-twitter-image-alt-bot.hf.space/callback",
	scopes: ["tweet.read", "users.read", "offline.access"],
});

// Pass auth credentials to the library client
const twitterClient = new Client(authClient);

const BOT_NAME = "AltImageBot1";
const BOT_ID = "1612094318906417152";

function debug(stuff: any) {
	console.log(inspect(stuff, { depth: 20 }));
}

interface TweetMentions {
	data: Array<{ id: string; text: string }>;
	meta: {
		result_count: number;
		newest_id: string;
		oldest_id: string;
	};
}

interface TweetLookups {
	data: Array<{
		id: string;
		conversation_id: "string";
		text: string;
		attachments?: { media_keys: string[] };
	}>;
	includes: { media: Array<{ media_key: string; url: string }> };
}

async function ff(url: string) {
	const resp = await fetch(`https://api.twitter.com/2/${url}`, {
		headers: { Authorization: `Bearer ${BEARER_TOKEN}` },
	});

	if (resp.status !== 200) {
		throw new Error("invalid status: " + resp.status + "- " + (await resp.text()));
	}

	return await resp.json();
}

async function lookupTweets() {
	const data: TweetMentions = await ff(`users/${BOT_ID}/mentions`);

	const lookups: TweetLookups = await ff(
		`tweets?ids=${data.data
			.map((t) => t.id)
			.join(
				","
			)}&expansions=attachments.media_keys&media.fields=duration_ms,height,media_key,preview_image_url,public_metrics,type,url,width,alt_text`
	);

	const tweetWithImage = lookups.data.find((tweet) => tweet.attachments?.media_keys);
	const imageUrl = lookups.includes.media.find(
		(media) => media.media_key === tweetWithImage?.attachments!.media_keys[0]
	)?.url!;

	console.log("imageUrl", imageUrl);

	const imageResp = await fetch(imageUrl);
	const contentType = imageResp.headers.get("Content-Type");
	const image = await imageResp.arrayBuffer();

	console.log(contentType, image);

	const altText = await fetch("https://olivierdehaene-git-large-coco.hf.space/run/predict", {
		method: "POST",
		headers: { "Content-Type": "application/json" },
		body: JSON.stringify({
			data: [`data:${contentType};base64,${Buffer.from(image).toString("base64")}`],
		}),
	})
		.then((r) => r.json())
		.then((r) => r.data);

	console.log(altText);

	await twitterClient.tweets.createTweet({ reply: { in_reply_to_tweet_id: tweetWithImage!.id }, text: altText });
}

async function listen() {
	try {
		const promise = new Promise<void>((resolve, reject) => {
			app.listen(port, "localhost", () => resolve());
			app.once("error", (err) => reject(err));
		});

		await promise;

		console.log("app started on port", port);

		process.send?.("ready");
	} catch (err) {
		console.error(err);
	}
}

listen();

process.on("unhandledRejection", async (err) => {
	console.error("unhandled rejection", err);
});

lookupTweets();