gemini vision

This commit is contained in:
derpystuff 2023-12-13 21:57:47 +01:00
parent 9254eb1abf
commit ebe629feae
4 changed files with 87 additions and 2 deletions

View file

@ -0,0 +1,80 @@
const { geminiVision } = require("../../../labscore/api/obelisk");
const { getRecentImage } = require("../../../labscore/utils/attachment");
const { createEmbed } = require("../../../labscore/utils/embed");
const { editOrReply } = require("../../../labscore/utils/message");
const { getUser } = require("../../../labscore/utils/users");
const { Permissions } = require("detritus-client/lib/constants");
const superagent = require('superagent');
const { STATIC_ICONS } = require("../../../labscore/utils/statics");
const { stringwrap } = require("../../../labscore/utils/markdown");
const { canUseLimitedTestCommands } = require("../../../labscore/utils/testing");
module.exports = {
name: 'gemini-vision',
label: 'text',
aliases: ['gv'],
metadata: {
description: 'Gemini Vision.',
description_short: 'Pride overlay',
examples: ['gv Which show is this image from?'],
category: 'limited',
usage: 'gemini-vision <attachment> <prompt>'
},
permissionsClient: [Permissions.EMBED_LINKS, Permissions.SEND_MESSAGES, Permissions.USE_EXTERNAL_EMOJIS, Permissions.ATTACH_FILES, Permissions.READ_MESSAGE_HISTORY],
run: async (context, args) => {
context.triggerTyping();
if(!canUseLimitedTestCommands(context)) return;
context.triggerTyping();
let image = await getRecentImage(context, 50)
if (!image) return editOrReply(context, createEmbed("warning", context, "No images found."))
if(!args.text) return editOrReply(context, createEmbed("warning", context, `Missing Parameter (text).`))
let input = args.text;
try{
await editOrReply(context, createEmbed("ai_custom", context, STATIC_ICONS.ai_gemini))
let res = await geminiVision(context, input, image)
let description = []
let files = [];
if(res.response.body.message) return editOrReply(context, createEmbed("error", context, e.response.body.message))
console.log(res.response.body)
let output = res.response.body.gemini?.candidates[0]?.content?.parts[0]?.text
if(!output) return editOrReply(context, createEmbed("error", context, `PaLM 2 returned an error. Try again later.`))
if(output.length <= 4000) description.push(output)
else {
files.push({
filename: `gemini.${Date.now().toString(36)}.txt`,
value: Buffer.from(output)
})
}
return editOrReply(context, {
embeds:[createEmbed("defaultNoFooter", context, {
author: {
name: stringwrap(input, 50, false),
iconUrl: STATIC_ICONS.ai_gemini
},
description: description.join('\n'),
footer: {
text: `Gemini Pro • Generative AI is experimental. Response may be factually incorrect or biased.`
}
})],
files
})
} catch(e){
console.log(e)
if(e.response?.body?.message) return editOrReply(context, createEmbed("error", context, e.response.body.message))
return editOrReply(context, createEmbed("error", context, `Unable to generate response.`))
}
}
};

View file

@ -7,7 +7,7 @@ const ObeliskApi = Object.freeze({
HOST: ObeliskHosts.prod,
GOOGLE_BARD: "/parrot/v1/google:bard",
GEMINI_PRO_VISION: "/parrot/v1/google:geminivision",
GEMINI_PRO_VISION: "/parrot/v1/google:geminiVision",
SUMMARIZE_WEBPAGES: "/flamingo/v1/web:summarize"
})

View file

@ -48,7 +48,7 @@ module.exports.bard = async function(context, input){
}
module.exports.geminiVision = async function(context, input, url){
return await request(ObeliskApi.GOOGLE_BARD, "POST", {}, {
return await request(ObeliskApi.GEMINI_PRO_VISION, "POST", {}, {
input,
url
})

View file

@ -152,6 +152,10 @@ const Statics = Object.freeze({
file: "brands/_clyde/clyde.png",
revision: 0
},
ai_gemini: {
file: "icons/aiv2/gemini_spark.png",
revision: 0
},
ai_palm_idle: {
file: "icons/core/ico_notice_palm_idle.png",
revision: 0
@ -209,6 +213,7 @@ module.exports.STATIC_ICONS = Object.freeze({
ai_bard_idle: staticAsset(Statics.icons.ai_bard_idle),
ai_clyde: staticAsset(Statics.icons.ai_clyde),
ai_clyde_idle: staticAsset(Statics.icons.ai_clyde_idle),
ai_gemini: staticAsset(Statics.icons.ai_gemini),
ai_palm_idle: staticAsset(Statics.icons.ai_palm_idle),
ai_summary: staticAsset(Statics.icons.ai_summary),
warning: staticAsset(Statics.icons.warning)