gemini vision

commands/message/genai/gemini-vision.js

@@ -0,0 +1,80 @@
+const { geminiVision } = require("../../../labscore/api/obelisk");
+const { getRecentImage } = require("../../../labscore/utils/attachment");
+const { createEmbed } = require("../../../labscore/utils/embed");
+const { editOrReply } = require("../../../labscore/utils/message");
+const { getUser } = require("../../../labscore/utils/users");
+
+const { Permissions } = require("detritus-client/lib/constants");
+
+const superagent = require('superagent');
+const { STATIC_ICONS } = require("../../../labscore/utils/statics");
+const { stringwrap } = require("../../../labscore/utils/markdown");
+const { canUseLimitedTestCommands } = require("../../../labscore/utils/testing");
+module.exports = {
+  name: 'gemini-vision',
+  label: 'text',
+  aliases: ['gv'],
+  metadata: {
+    description: 'Gemini Vision.',
+    description_short: 'Pride overlay',
+    examples: ['gv Which show is this image from?'],
+    category: 'limited',
+    usage: 'gemini-vision <attachment> <prompt>'
+  },
+  permissionsClient: [Permissions.EMBED_LINKS, Permissions.SEND_MESSAGES, Permissions.USE_EXTERNAL_EMOJIS, Permissions.ATTACH_FILES, Permissions.READ_MESSAGE_HISTORY],
+  run: async (context, args) => {
+    context.triggerTyping();
+    if(!canUseLimitedTestCommands(context)) return;
+    context.triggerTyping();
+
+    let image = await getRecentImage(context, 50)
+    if (!image) return editOrReply(context, createEmbed("warning", context, "No images found."))
+
+    if(!args.text) return editOrReply(context, createEmbed("warning", context, `Missing Parameter (text).`))
+
+    let input = args.text;
+
+    try{
+      await editOrReply(context, createEmbed("ai_custom", context, STATIC_ICONS.ai_gemini))
+
+      let res = await geminiVision(context, input, image)
+
+      let description = []
+      let files = [];
+      
+      if(res.response.body.message) return editOrReply(context, createEmbed("error", context, e.response.body.message))
+
+      console.log(res.response.body)
+
+      let output = res.response.body.gemini?.candidates[0]?.content?.parts[0]?.text
+      if(!output) return editOrReply(context, createEmbed("error", context, `PaLM 2 returned an error. Try again later.`)) 
+
+
+      if(output.length <= 4000) description.push(output)
+      else {
+        files.push({
+          filename: `gemini.${Date.now().toString(36)}.txt`,
+          value: Buffer.from(output)
+        })
+      }
+
+      return editOrReply(context, {
+        embeds:[createEmbed("defaultNoFooter", context, {
+          author: {
+            name: stringwrap(input, 50, false),
+            iconUrl: STATIC_ICONS.ai_gemini
+          },
+          description: description.join('\n'),
+          footer: {
+            text: `Gemini Pro • Generative AI is experimental. Response may be factually incorrect or biased.`
+          }
+        })],
+        files
+      })
+    } catch(e){
+      console.log(e)
+      if(e.response?.body?.message) return editOrReply(context, createEmbed("error", context, e.response.body.message))
+      return editOrReply(context, createEmbed("error", context, `Unable to generate response.`))
+    }
+  }
+};

labscore/api/obelisk/endpoints.js

@@ -7,7 +7,7 @@ const ObeliskApi = Object.freeze({
   HOST: ObeliskHosts.prod,
 
   GOOGLE_BARD: "/parrot/v1/google:bard",
-  GEMINI_PRO_VISION: "/parrot/v1/google:geminivision",
+  GEMINI_PRO_VISION: "/parrot/v1/google:geminiVision",
 
   SUMMARIZE_WEBPAGES: "/flamingo/v1/web:summarize"
 })

labscore/api/obelisk/index.js

@@ -48,7 +48,7 @@ module.exports.bard = async function(context, input){
 }
 
 module.exports.geminiVision = async function(context, input, url){
-  return await request(ObeliskApi.GOOGLE_BARD, "POST", {}, {
+  return await request(ObeliskApi.GEMINI_PRO_VISION, "POST", {}, {
     input,
     url
   })

labscore/utils/statics.js

@@ -152,6 +152,10 @@ const Statics = Object.freeze({
       file: "brands/_clyde/clyde.png",
       revision: 0
     },
+    ai_gemini: {
+      file: "icons/aiv2/gemini_spark.png",
+      revision: 0
+    },
     ai_palm_idle: {
       file: "icons/core/ico_notice_palm_idle.png",
       revision: 0
@@ -209,6 +213,7 @@ module.exports.STATIC_ICONS = Object.freeze({
   ai_bard_idle: staticAsset(Statics.icons.ai_bard_idle),
   ai_clyde: staticAsset(Statics.icons.ai_clyde),
   ai_clyde_idle: staticAsset(Statics.icons.ai_clyde_idle),
+  ai_gemini: staticAsset(Statics.icons.ai_gemini),
   ai_palm_idle: staticAsset(Statics.icons.ai_palm_idle),
   ai_summary: staticAsset(Statics.icons.ai_summary),
   warning: staticAsset(Statics.icons.warning)