new actions for file, video, image, excel, powerpoint, fix twitter and updates

Author: cxbxmxcx

.gitignore

@@ -130,6 +130,7 @@ env.bak/
 venv.bak/
 
 assistant_outputs/
+assistants_working_folder/
 
 logs.txt
 

logs/logs.txt

@@ -2,6 +2,7 @@
 import re
 import threading
 from contextlib import contextmanager
+import os
 
 import gradio as gr
 
@@ -12,6 +13,7 @@
 from playground.environment_manager import EnvironmentManager
 from playground.logging import Logger
 from playground.semantic_manager import SemanticManager
+from playground.constants import ASSISTANTS_WORKING_FOLDER
 
 thread = api.create_thread()  # create a new thread everytime this is run
 actions_manager = ActionsManager()
@@ -78,6 +80,27 @@ def dummy_stream(*args, **kwargs):
     yield ["streaming data"]
 
 
+def extract_file_paths(text):
+    # Regular expression pattern to match file paths
+    # This pattern matches typical file paths in Windows and Unix-like systems
+    pattern = r"(?:[a-zA-Z]:\\)?(?:[a-zA-Z0-9_-]+\\)*[a-zA-Z0-9_-]+\.[a-zA-Z0-9]+|(?:\/[a-zA-Z0-9_-]+)+\/?"
+
+    # Find all matching file paths in the text
+    file_paths = re.findall(pattern, text)
+
+    unique_file_paths = list(set(file_paths))
+
+    return unique_file_paths
+
+
+def get_file_path(file):
+    if os.path.isabs(file):
+        return file
+
+    file_path = os.path.join(ASSISTANTS_WORKING_FOLDER, file)
+    return file_path
+
+
 def run(history, assistant_id):
     assistant = api.retrieve_assistant(assistant_id)
     output_queue = queue.Queue()
@@ -117,6 +140,14 @@ def stream_worker(assistant_id, thread_id, event_handler):
             pass
     # history[-1][1] = wrap_latex_with_markdown(history[-1][1])
     yield history
+
+    files = extract_file_paths(history[-1][1])
+    for file in files:
+        file_path = get_file_path(file)
+        if os.path.exists(file_path):
+            history.append((None, (file_path,)))
+        yield history
+
     # Final flush of images
     while len(eh.images) > 0:
         history.append((None, (eh.images.pop(),)))
@@ -171,6 +202,18 @@ def stream_worker(assistant_id, thread_id, event_handler):
 #actions { 
     color: #000000; 
  }
+ 
+video {
+    width: 300px;  /* initial width */
+    height: 200px; /* initial height */
+    transition: width 0.5s ease, height 0.5s ease;
+    cursor: pointer;
+}
+video:hover {
+    width: auto;
+    height: auto;
+    max-width: 100%; /* ensures it doesn’t exceed the container's width */
+}
 """
 
 # theme = gr.themes.Default()

main.py

@@ -0,0 +1,64 @@
+import pandas as pd
+import io
+import os
+
+from playground.actions_manager import agent_action
+from playground.constants import ASSISTANTS_WORKING_FOLDER
+
+
+@agent_action
+def excel_to_csv(filename, skiprows):
+    """
+    Reads an Excel file, skips the specified number of rows, and returns the content as a CSV-formatted text string.
+
+    Parameters:
+    filename (str): The path to the Excel file.
+    skiprows (int or list of int): The number of rows to skip at the start or a list of row indices to skip.
+
+    Returns:
+    str: The CSV-formatted text string.
+    """
+    file_path = os.path.join(ASSISTANTS_WORKING_FOLDER, filename)
+    # Read the Excel file and skip the specified rows
+    df = pd.read_excel(file_path, skiprows=skiprows)
+
+    # Create a buffer to store the CSV content
+    buffer = io.StringIO()
+
+    # Write the DataFrame to the buffer as CSV
+    df.to_csv(buffer, index=False)
+
+    # Get the CSV content from the buffer
+    csv_content = buffer.getvalue()
+
+    # Close the buffer
+    buffer.close()
+
+    return csv_content
+
+
+@agent_action
+def csv_to_excel(csv_content, start_row, filename):
+    """
+    Reads a CSV-formatted text string, skips the specified number of rows, and saves the content as an Excel file.
+
+    Parameters:
+    csv_content (str): The CSV-formatted text string.
+    start_row (int): The row number to start to add content to.
+    filename (str): The path to the output Excel file.
+
+    Returns:
+    None
+    """
+    # Create a buffer from the CSV content
+    buffer = io.StringIO(csv_content)
+
+    # Read the CSV content into a DataFrame, skipping the specified rows
+    df = pd.read_csv(buffer)
+
+    # Save the DataFrame to an Excel file
+    file_path = os.path.join(ASSISTANTS_WORKING_FOLDER, filename)
+    df.to_excel(file_path, index=False, startrow=start_row)
+
+    # Close the buffer
+    buffer.close()

playground/assistant_actions/excel_actions.py

@@ -1,8 +1,7 @@
 import os
 
 from playground.actions_manager import agent_action
-
-OUTPUT_FOLDER = "assistant_outputs"
+from playground.constants import ASSISTANTS_WORKING_FOLDER
 
 
 @agent_action
@@ -13,10 +12,10 @@ def save_file(filename, content):
     :param filename: The name of the file including extension.
     :param content: The content to save in the file.
     """
-    file_path = os.path.join(OUTPUT_FOLDER, filename)
+    file_path = os.path.join(ASSISTANTS_WORKING_FOLDER, filename)
     with open(file_path, "w", encoding="utf-8") as file:
         file.write(content)
-    print(f"File '{filename}' saved successfully.")
+    return f"File '{filename}' saved successfully."
 
 
 @agent_action
@@ -27,7 +26,7 @@ def load_file(filename):
     :param filename: The name of the file including extension.
     :return: The content of the file.
     """
-    file_path = os.path.join(OUTPUT_FOLDER, filename)
+    file_path = os.path.join(ASSISTANTS_WORKING_FOLDER, filename)
     if not os.path.exists(file_path):
         print(f"File '{filename}' does not exist.")
         return None
@@ -45,12 +44,12 @@ def delete_file(filename):
 
     :param filename: The name of the file including extension.
     """
-    file_path = os.path.join(OUTPUT_FOLDER, filename)
+    file_path = os.path.join(ASSISTANTS_WORKING_FOLDER, filename)
     if os.path.exists(file_path):
         os.remove(file_path)
-        print(f"File '{filename}' deleted successfully.")
+        return f"File '{filename}' deleted successfully."
     else:
-        print(f"File '{filename}' does not exist.")
+        return f"File '{filename}' does not exist."
 
 
 @agent_action
@@ -60,12 +59,23 @@ def create_folder(foldername):
 
     :param foldername: The name of the folder to create.
     """
-    folder_path = os.path.join(OUTPUT_FOLDER, foldername)
+    folder_path = os.path.join(ASSISTANTS_WORKING_FOLDER, foldername)
     if not os.path.exists(folder_path):
         os.makedirs(folder_path)
-        print(f"Folder '{foldername}' created successfully.")
+        return f"Folder '{foldername}' created successfully."
     else:
-        print(f"Folder '{foldername}' already exists.")
+        return f"Folder '{foldername}' already exists."
+
+
+@agent_action
+def list_files():
+    """
+    List all files in the working folder.
+
+    :return: A list of file names.
+    """
+    files = os.listdir(ASSISTANTS_WORKING_FOLDER)
+    return files
 
 
 # # Example usage:

playground/assistant_actions/file_actions.py

@@ -0,0 +1,116 @@
+import os
+import requests
+from openai import OpenAI
+from dotenv import load_dotenv
+import base64
+
+from playground.actions_manager import agent_action
+from playground.constants import ASSISTANTS_WORKING_FOLDER
+
+load_dotenv()
+
+client = OpenAI()
+
+
+@agent_action
+def create_image(prompt, model="dall-e-3", size="1024x1024", quality="standard", n=1):
+    """
+    Generate an image based on the provided prompt using the DALL-E model.
+    Args:
+        prompt (str): The prompt used to generate the image.
+        model (str, optional): The model used to generate the image. Defaults to "dall-e-3".
+        size (str, optional): The size of the image in pixels. Defaults to "1024x1024". Options are 1024x1024, 1024x1792 or 1792x1024.
+        quality (str, optional): The quality of the generation. Defaults to "standard". Options are "standard" or "hd" for high definition.
+        n (int, optional): The number of generations. Defaults to 1. Always 1 for now.
+    Returns:
+        str: The path to the generated image file.
+    """
+
+    try:
+        response = client.images.generate(
+            model=model, prompt=prompt, size=size, quality=quality, n=n
+        )
+        image_url = response.data[0].url
+    except Exception as e:
+        return str(e)
+
+    # Download the image
+    image_response = requests.get(image_url)
+    if image_response.status_code == 200:
+        local_filename = f"{prompt.replace(' ', '_').replace(',','')[:50]}.png"
+        local_path = os.path.join(ASSISTANTS_WORKING_FOLDER, local_filename)
+        with open(local_path, "wb") as f:
+            f.write(image_response.content)
+        return local_filename
+    else:
+        raise Exception(
+            f"Failed to download image, status code: {image_response.status_code}"
+        )
+
+
+def encode_image(image_filename):
+    """
+    Encode the image to base64 format.
+
+    Args:
+        image_path (str): The path to the image file.
+
+    Returns:
+        str: The base64 encoded string of the image.
+    """
+    local_path = os.path.join(ASSISTANTS_WORKING_FOLDER, image_filename)
+    if not os.path.exists(local_path):
+        return f"File not found: {image_filename}"
+    with open(local_path, "rb") as image_file:
+        return base64.b64encode(image_file.read()).decode("utf-8")
+
+
+@agent_action
+def describe_image(
+    image_filename,
+    model="gpt-4o",
+    prompt="""
+    Describe exactly what is in the image and locate the objects in the image relative to its position.
+    e.g. "There is a cat sitting on the table in the bottom right of the image."
+    """,
+    max_tokens=512,
+):
+    """
+    Describe the content of an image using the OpenAI API.
+
+    Args:
+        image_path (str): The path to the image file.
+        model (str, optional): The model to use for the OpenAI API. Defaults to "gpt-4o".
+        prompt (str, optional): The prompt to describe the image. Defaults to 'Describe exactly what is in the image and locate the objects in the image relative to its position. e.g. "There is a cat sitting on the table in the bottom right of the image.'.
+        max_tokens (int, optional): The maximum number of tokens for the response. Defaults to 512.
+
+    Returns:
+        dict: The response from the OpenAI API.
+    """
+    # Getting the base64 string
+    base64_image = encode_image(image_filename)
+    api_key = os.getenv("OPENAI_API_KEY")
+    headers = {"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}
+
+    payload = {
+        "model": model,
+        "messages": [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": prompt},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"},
+                    },
+                ],
+            }
+        ],
+        "max_tokens": max_tokens,
+    }
+
+    response = requests.post(
+        "https://api.openai.com/v1/chat/completions", headers=headers, json=payload
+    )
+
+    return response.json()