Skip to content

Add doc strings to all computer API functions for the LLM to use in the skill library #1011

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 14, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions interpreter/core/computer/browser/browser.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ def __init__(self, computer):
self.computer = computer

def search(self, query):
"""
Searches the web for the specified query and returns the results.
"""
response = requests.get(
f'{self.computer.api_base.strip("/")}/browser/search', params={"q": query}
)
Expand Down
9 changes: 9 additions & 0 deletions interpreter/core/computer/clipboard/clipboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,22 @@ def __init__(self, computer):
self.modifier_key = "command"

def view(self):
"""
Returns the current content of on the clipboard.
"""
return pyperclip.paste()

def copy(self, text=None):
"""
Copies the given text to the clipboard.
"""
if text is not None:
pyperclip.copy(text)
else:
self.computer.keyboard.hotkey(self.modifier_key, "c")

def paste(self):
"""
Pastes the current content of the clipboard.
"""
self.computer.keyboard.hotkey(self.modifier_key, "v")
2 changes: 1 addition & 1 deletion interpreter/core/computer/computer.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ def run(self, *args, **kwargs):

def exec(self, code):
"""
It has hallucinated this.
Shortcut for computer.terminal.run("shell", code)
It has hallucinated this.
"""
return self.terminal.run("shell", code)

Expand Down
22 changes: 19 additions & 3 deletions interpreter/core/computer/display/display.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,15 @@ def __init__(self, computer):
pass

def size(self):
"""
Returns the current screen size as a tuple (width, height).
"""
return pyautogui.size()

def center(self):
"""
Calculates and returns the center point of the screen as a tuple (x, y).
"""
return self.width // 2, self.height // 2

def view(self, show=True, quadrant=None):
Expand All @@ -48,6 +54,9 @@ def view(self, show=True, quadrant=None):
# return get_active_window()

def screenshot(self, show=True, quadrant=None, active_app_only=False):
"""
Shows you what's on the screen by taking a screenshot of the entire screen or a specified quadrant. Returns a `pil_image` `in case you need it (rarely). **You almost always want to do this first!**
"""
time.sleep(2)
if not self.computer.emit_images:
text = self.get_text_as_list_of_lists()
Expand Down Expand Up @@ -110,7 +119,9 @@ def screenshot(self, show=True, quadrant=None, active_app_only=False):
return screenshot

def find_text(self, text, screenshot=None):
# Take a screenshot
"""
Searches for specified text within a screenshot or the current screen if no screenshot is provided.
"""
if screenshot == None:
screenshot = self.screenshot(show=False)

Expand Down Expand Up @@ -140,7 +151,9 @@ def find_text(self, text, screenshot=None):
] # Have it deliver the text properly soon.

def get_text_as_list_of_lists(self, screenshot=None):
# Take a screenshot
"""
Extracts and returns text from a screenshot or the current screen as a list of lists, each representing a line of text.
"""
if screenshot == None:
screenshot = self.screenshot(show=False)

Expand Down Expand Up @@ -172,6 +185,9 @@ def get_text_as_list_of_lists(self, screenshot=None):

# locate text should be moved here as well!
def find_icon(self, query, screenshot=None):
"""
Locates an icon on the screen and returns its coordinates.
"""
message = format_to_recipient(
"Locating this icon will take ~30 seconds. We're working on speeding this up.",
recipient="user",
Expand Down Expand Up @@ -201,5 +217,5 @@ def find_icon(self, query, screenshot=None):
except Exception as e:
raise Exception(
str(e)
+ "\n\nIcon locating API not avaliable, or we were unable to find the icon. Please try another method to find this icon."
+ "\n\nIcon locating API not available, or we were unable to find the icon. Please try another method to find this icon."
)
8 changes: 7 additions & 1 deletion interpreter/core/computer/files/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,14 @@ def __init__(self, computer):

def search(self, *args, **kwargs):
"""
AI Filesystem Search
Search the filesystem for the given query.
"""
return search(*args, **kwargs)

def edit(self, path, original_text, replacement_text):
"""
Edits a file on the filesystem, replacing the original text with the replacement text.
"""
with open(path, "r") as file:
filedata = file.read()

Expand All @@ -32,6 +35,9 @@ def edit(self, path, original_text, replacement_text):


def get_close_matches_in_text(original_text, filedata, n=3):
"""
Returns the closest matches to the original text in the content of the file.
"""
words = filedata.split()
original_words = original_text.split()
len_original = len(original_words)
Expand Down
24 changes: 3 additions & 21 deletions interpreter/core/computer/keyboard/keyboard.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,7 @@ def __init__(self, computer):

def write(self, text, interval=None, **kwargs):
"""
Type out a string of characters.

Args:
text (str): The string to be typed out.
interval (int or float, optional): The delay between pressing each character key. Defaults to 0.1.
Type out a string of characters.
"""
time.sleep(0.15)

Expand Down Expand Up @@ -68,11 +64,6 @@ def press(self, *args, presses=1, interval=0.1):

If keys is a string, it is treated as a single key and is pressed the number of times specified by presses.
If keys is a list, each key in the list is pressed once.

Args:
keys (str or list): The key(s) to be pressed.
presses (int, optional): The number of times to press the key. Defaults to 1.
interval (float, optional): The delay between each key press. Defaults to 0.1.
"""
time.sleep(0.15)
pyautogui.press(keys, presses=presses, interval=interval)
Expand All @@ -81,9 +72,6 @@ def press(self, *args, presses=1, interval=0.1):
def hotkey(self, *args, interval=0.1):
"""
Press a sequence of keys in the order they are provided, and then release them in reverse order.

Args:
*args: The keys to be pressed.
"""
time.sleep(0.15)
modifiers = ["command", "option", "alt", "ctrl", "shift"]
Expand Down Expand Up @@ -121,21 +109,15 @@ def hotkey(self, *args, interval=0.1):

def down(self, key):
"""
Simulate the pressing down of a key.

Args:
key (str): The key to be pressed down.
Press down a key.
"""
time.sleep(0.15)
pyautogui.keyDown(key)
time.sleep(0.15)

def up(self, key):
"""
Simulate the releasing of a key.

Args:
key (str): The key to be released.
Release a key.
"""
time.sleep(0.15)
pyautogui.keyUp(key)
Expand Down
25 changes: 25 additions & 0 deletions interpreter/core/computer/mouse/mouse.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ def __init__(self, computer):
self.computer = computer

def scroll(self, clicks):
"""
Scrolls the mouse wheel up or down the specified number of clicks.
"""
pyautogui.scroll(clicks)

def position(self):
Expand All @@ -36,6 +39,10 @@ def position(self):
)

def move(self, *args, x=None, y=None, icon=None, text=None, screenshot=None):
"""
Moves the mouse to specified coordinates, an icon, or text.
"""
screenshot = None
if len(args) > 1:
raise ValueError(
"Too many positional arguments provided. To move/click specific coordinates, use kwargs (x=x, y=y).\n\nPlease take a screenshot with computer.display.view() to find text/icons to click, then use computer.mouse.click(text) or computer.mouse.click(icon=description_of_icon) if at all possible. This is **significantly** more accurate than using coordinates. Specifying (x=x, y=y) is highly likely to fail. Specifying ('text to click') is highly likely to succeed."
Expand Down Expand Up @@ -216,29 +223,47 @@ def move(self, *args, x=None, y=None, icon=None, text=None, screenshot=None):
smooth_move_to(x, y)

def click(self, *args, button="left", clicks=1, interval=0.1, **kwargs):
"""
Clicks the mouse at the specified coordinates, icon, or text.
"""
if args or kwargs:
self.move(*args, **kwargs)
pyautogui.click(button=button, clicks=clicks, interval=interval)

def double_click(self, *args, button="left", interval=0.1, **kwargs):
"""
Double-clicks the mouse at the specified coordinates, icon, or text.
"""
if args or kwargs:
self.move(*args, **kwargs)
pyautogui.doubleClick(button=button, interval=interval)

def triple_click(self, *args, button="left", interval=0.1, **kwargs):
"""
Triple-clicks the mouse at the specified coordinates, icon, or text.
"""
if args or kwargs:
self.move(*args, **kwargs)
pyautogui.tripleClick(button=button, interval=interval)

def right_click(self, *args, **kwargs):
"""
Right-clicks the mouse at the specified coordinates, icon, or text.
"""
if args or kwargs:
self.move(*args, **kwargs)
pyautogui.rightClick()

def down(self):
"""
Presses the mouse button down.
"""
pyautogui.mouseDown()

def up(self):
"""
Releases the mouse button.
"""
pyautogui.mouseUp()


Expand Down
6 changes: 6 additions & 0 deletions interpreter/core/computer/os/os.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ def __init__(self, computer):
self.computer = computer

def get_selected_text(self):
"""
Returns the currently selected text.
"""
# Store the current clipboard content
current_clipboard = self.computer.clipboard.view()
# Copy the selected text to clipboard
Expand All @@ -18,6 +21,9 @@ def get_selected_text(self):
return selected_text

def notify(self, text):
"""
Displays a notification on the computer.
"""
try:
title = "Open Interpreter"

Expand Down