diff --git a/src/bidara.js b/src/bidara.js index 4c8fbaa..67a34e4 100644 --- a/src/bidara.js +++ b/src/bidara.js @@ -4,6 +4,9 @@ export const BIDARA_SYS = `You are BIDARA, a biomimetic designer and research as Your goal is to help the user work in a step by step way through the Biomimicry Design Process (https://toolbox.biomimicry.org/methods/process/) to propose biomimetic solutions to a challenge. Cite peer reviewed sources for your information. Stop often (at a minimum after every step) to ask the user for feedback or clarification. +You are provided several functions to assist you in your efforts including searching for academic papers and generating images. +Part of this process is about the user's understanding of the problem, and visual components can help with that. At any time, you are able to generate images for the user for them to further their visual understanding of the biological process/structure you are discussing. + 1. Define - The first step in any design process is to define the problem or opportunity that you want your design to address. Prompt the user to think through the next four steps to define their challenge. Don't try to answer these for the user. You may offer suggestions if asked to. a. Frame your challenge: Give a simple explanation of the impact you want to have. (Hint: This is not what you want to make, but want you want to your design to achieve or do.) b. Consider context: Describe some of the contextual factors that are important to the challenge. (Hint: This could include stakeholders, location conditions, resource availability, etc.) @@ -39,6 +42,8 @@ Stating the strategy this way makes it easier to translate it into a design appl 5. Emulate Nature's Lessons - Once you have found a number of biological strategies and analyzed them for the design strategies you can extract, you are ready to begin the creative part—dreaming up nature-inspired solutions. Here we’ll guide you through the key activities of the Emulate step. Look for patterns and relationships among the strategies you found and hone in on the the key lessons that should inform your solution. Develop design concepts based on these strategies. Emulation is the heart of biomimicry; learning from living things and then applying those insights to the challenges humans want to solve. More than a rote copying of nature’s strategies, emulation is an exploratory process that strives to capture a “recipe” or “blueprint” in nature’s example that can be modeled in our own designs. During this part of the process you must reconcile what you have learned in the last four steps of the Design Spiral into a coherent, life-friendly design concept. It’s important to remain open-minded at this stage and let go of any preconceived notions you have about what your solution might be. +At this step, it is particularly important for the user to have a visual understanding of the problem and solution, so generating images is strongly recommended. +ALWAYS USE YOUR "generate_image_from_description" function whenever possible. Do not use code to create an image. As you examine your bio-inspired design strategies, try these techniques to help you uncover potentially valuable patterns and insights. List each of your inspiring organisms along with notes about their strategies, functions, and key features. (Hint: Think about contextual factors). Create categories that group the strategies by shared features, such as context, constraints, or key mechanisms. Do you see any patterns? What additional questions emerge as you consider these groups? If you are struggling, consider two different organisms and try to identify something they have in common, even if it seems superficial. As you practice, your groupings will likely become more meaningful or nuanced. @@ -66,7 +71,12 @@ Nature runs on information. Nature uses chemistry and materials that are safe for living beings. Nature builds using abundant resources, incorporating rare resources only sparingly. Nature is locally attuned and responsive. -Nature uses shape to determine functionality.`; +Nature uses shape to determine functionality. + +Functions you have access to: +- Searching for academic papers with 'get_graph_paper_relevance_search' +- Generating images with 'generate_image_from_description' +`; export const PAPER_SEARCH_FUNC = { "name": "get_graph_paper_relevance_search", @@ -128,6 +138,21 @@ export const PAPER_SEARCH_FUNC = { } } +const GEN_IMAGE_FUNC = { + "name": "generate_image_from_description", + "description": "Generate an image from a description.", + "parameters": { + "type": "object", + "properties": { + "description_of_image": { + "type": "string", + "image_description": "The description of the image provided by the user." + } + }, + "required": [] + } +} + export const BIDARA_CONFIG = { model: "gpt-4-0125-preview", name: "BIDARAv"+BIDARA_VERSION, @@ -138,6 +163,10 @@ export const BIDARA_CONFIG = { { type: "function", function: PAPER_SEARCH_FUNC + }, + { + type: "function", + function: GEN_IMAGE_FUNC } ] -} \ No newline at end of file +} diff --git a/src/bidaraFunctions.js b/src/bidaraFunctions.js index 6333725..9fff983 100644 --- a/src/bidaraFunctions.js +++ b/src/bidaraFunctions.js @@ -1,3 +1,5 @@ +import { getDalleImageGeneration } from "./openaiUtils"; + export function getCurrentWeather(location) { location = location.toLowerCase(); if (location.includes('tokyo')) { @@ -39,11 +41,43 @@ export async function ssSearch(params) { return JSON.stringify(papers); } +async function genImage(params) { + + var imageParams = JSON.parse(params); + + if ("parameters" in imageParams) { + imageParams = imageParams.parameters; + } + + var imageDescription = JSON.stringify(imageParams.description) + " Realistic depiction of the object and its environment. Stay true to science, engineering, and biology. DO NOT INCLUDE ANY WORDS OR BRANDING." + + const res = await getDalleImageGeneration(imageDescription); + + if (!res) { + return "We are having trouble generating images at this time."; + } + + const imageURL = res.data[0].url; + + const imageWidth = 256; + const imageHeight = 256; + const imageTag = `Image of: ${imageDescription}. Generated by AI.`; + const message = {html: imageTag, role: "ai"}; + + const deepChatRef = document.getElementById('chat-element'); + deepChatRef._addMessage(message); + + return "The image has been inserted into the chat, respond by tieing this image back into the Biomimicry Design Process with a short question or response. DO NOT RESPOND WITH AN IMAGE, URL, OR MARKDOWN."; +} + export async function callFunc(functionDetails) { let tmp = ''; if(functionDetails.name == "get_graph_paper_relevance_search") { tmp = await ssSearch(functionDetails.arguments); } + else if(functionDetails.name == "generate_image_from_description") { + tmp = await genImage(functionDetails.arguments); + } else if(functionDetails.name == "get_weather") { tmp = getCurrentWeather(functionDetails.arguments); } @@ -56,4 +90,4 @@ export async function callFunc(functionDetails) { export async function funcCalling(functionsDetails) { let tmp = await Promise.all(functionsDetails.map(callFunc)); return tmp; -} \ No newline at end of file +} diff --git a/src/openaiUtils.js b/src/openaiUtils.js index 8088d83..2ba40ae 100644 --- a/src/openaiUtils.js +++ b/src/openaiUtils.js @@ -177,4 +177,46 @@ export async function getKeyAndAsst() { let asst = await getAsst(); return [key, asst] -} \ No newline at end of file +} + +export async function getDalleImageGeneration(prompt, image_size = null, image_quality = null, num_images = null) { + if (!image_size) image_size = "1024x1024"; + if (!image_quality) image_quality = "standard"; + if (!num_images) num_images = 1; + + try { + if (!openaiKey) { + return null; + } + + const requestURL = "https://api.openai.com/v1/images/generations"; + + const request = { + method: "POST", + headers: { + 'Authorization': 'Bearer ' + openaiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: "dall-e-3", + prompt: prompt, + n: num_images, + size: image_size, + quality: image_quality + }) + } + + const response = await fetch(requestURL, request); + + const r = await response.json(); + if (r.error && r.error.type === 'invalid_request_error') { + return null; + } + + return r; + + } catch (e) { + + return null; + } +}