chore: 添加虚拟环境到仓库

- 添加 backend_service/venv 虚拟环境
- 包含所有Python依赖包
- 注意：虚拟环境约393MB，包含12655个文件

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/__init__.py

@@ -0,0 +1,44 @@
+# -*- coding: utf-8 -*-
+"""The evaluation module in AgentScope."""
+
+from ._evaluator import (
+    EvaluatorBase,
+    RayEvaluator,
+    GeneralEvaluator,
+)
+from ._metric_base import (
+    MetricBase,
+    MetricResult,
+    MetricType,
+)
+from ._task import Task
+from ._solution import SolutionOutput
+from ._benchmark_base import BenchmarkBase
+from ._evaluator_storage import (
+    EvaluatorStorageBase,
+    FileEvaluatorStorage,
+)
+from ._ace_benchmark import (
+    ACEBenchmark,
+    ACEAccuracy,
+    ACEProcessAccuracy,
+    ACEPhone,
+)
+
+__all__ = [
+    "BenchmarkBase",
+    "EvaluatorBase",
+    "RayEvaluator",
+    "GeneralEvaluator",
+    "MetricBase",
+    "MetricResult",
+    "MetricType",
+    "EvaluatorStorageBase",
+    "FileEvaluatorStorage",
+    "Task",
+    "SolutionOutput",
+    "ACEBenchmark",
+    "ACEAccuracy",
+    "ACEProcessAccuracy",
+    "ACEPhone",
+]

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/__init__.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+"""The ACE benchmark related implementations in AgentScope."""
+
+from ._ace_benchmark import ACEBenchmark
+from ._ace_metric import (
+    ACEAccuracy,
+    ACEProcessAccuracy,
+)
+from ._ace_tools_zh import ACEPhone
+
+__all__ = [
+    "ACEBenchmark",
+    "ACEPhone",
+    "ACEAccuracy",
+    "ACEProcessAccuracy",
+]

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_benchmark.py

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+"""The ACE benchmark class in agentscope. The code is implemented with
+reference to the `ACEBench <https://github.com/ACEBench/ACEBench>`_
+under the MIT license."""
+import json
+import os
+from typing import Generator
+
+import json5
+import requests
+from tqdm import tqdm
+
+from ._ace_metric import ACEAccuracy, ACEProcessAccuracy
+from ._ace_tools_zh import ACEPhone
+from .._benchmark_base import BenchmarkBase
+from .._task import Task
+
+
+class ACEBenchmark(BenchmarkBase):
+    """The ACE benchmark for evaluating AI agents."""
+
+    data_dir_url: str = (
+        "https://raw.githubusercontent.com/ACEBench/ACEBench/main/data_all"
+    )
+    """The URL to the data dir"""
+
+    data_subdir: list[str] = [
+        # "data_en",  # TODO: enable English version
+        "data_zh",
+    ]
+
+    ground_truth_dir: str = "possible_answer"
+
+    data_files: list[str] = [
+        "data_agent_multi_step.json",
+        "data_agent_multi_turn.json",
+        # "data_normal_atom_bool.json",
+        # "data_normal_atom_enum.json",
+        # "data_normal_atom_list.json",
+        # "data_normal_atom_number.json",
+        # "data_normal_atom_object_deep.json",
+        # "data_normal_atom_object_short.json",
+        #
+        # "data_normal_multi_turn_user_adjust.json",
+        # "data_normal_multi_turn_user_switch.json",
+        #
+        # "data_normal_preference.json",
+        # "data_normal_similar_api.json",
+        # "data_normal_single_turn_parallel_function.json",
+        # "data_normal_single_turn_single_function.json",
+        #
+        # "data_special_error_param.json",
+        # "data_special_incomplete.json",
+        # "data_special_irrelevant.json",
+    ]
+    """The data filenames"""
+
+    def __init__(
+        self,
+        data_dir: str,
+    ) -> None:
+        """Initialize the ACEBenchmark
+
+        Args:
+            data_dir (`str`):
+                The directory where the dataset is downloaded and saved.
+        """
+        super().__init__(
+            name="ACEBench",
+            description="The ACE benchmark for evaluating AI agents.",
+        )
+
+        self.data_dir = os.path.abspath(data_dir)
+
+        if os.path.exists(data_dir) and not os.path.isdir(data_dir):
+            raise RuntimeError(
+                f"The data_dir `{data_dir}` is not a valid directory path.",
+            )
+
+        os.makedirs(data_dir, exist_ok=True)
+
+        if not self._verify_data():
+            self._download_data()
+
+        self.dataset = self._load_data()
+
+    def _load_data(self) -> list[dict]:
+        """Load the dataset from the data directory."""
+        dataset = []
+        for subdir in self.data_subdir:
+            for filename in self.data_files:
+                file_path = os.path.join(self.data_dir, subdir, filename)
+
+                gt_path = os.path.join(
+                    self.data_dir,
+                    subdir,
+                    self.ground_truth_dir,
+                    filename,
+                )
+                gt_dataset = {}
+                with open(gt_path, "r", encoding="utf-8") as gt_file:
+                    for line in gt_file:
+                        gt_data = json5.loads(line)
+                        gt_dataset[gt_data["id"]] = gt_data
+
+                with open(file_path, "r", encoding="utf-8") as f:
+                    for line in f:
+                        data = json5.loads(line)
+                        gt = gt_dataset[data["id"]]
+                        gt.pop("id", None)
+                        data["ground_truth"] = gt["ground_truth"]
+                        data["mile_stone"] = gt["mile_stone"]
+                        data["language"] = subdir.rsplit(
+                            "_",
+                            maxsplit=1,
+                        )[-1]
+                        data["tags"] = {
+                            "language": data["language"],
+                            "category": filename.split(
+                                ".",
+                                maxsplit=1,
+                            )[0].removeprefix(
+                                "data_",
+                            ),
+                        }
+                        dataset.append(data)
+
+        return dataset
+
+    def _verify_data(self) -> bool:
+        """Verify the data completeness and integrity."""
+        for subdir in self.data_subdir:
+            for filename in self.data_files:
+                file_path = os.path.join(self.data_dir, subdir, filename)
+                if not os.path.exists(file_path):
+                    return False
+
+                gt_path = os.path.join(
+                    self.data_dir,
+                    subdir,
+                    self.ground_truth_dir,
+                    filename,
+                )
+                if not os.path.exists(gt_path):
+                    return False
+
+        return True
+
+    def _download_data(self) -> None:
+        """Download the data from the URL"""
+        for subdir in self.data_subdir:
+            subdir_path = os.path.join(self.data_dir, subdir)
+            subdir_gt_path = os.path.join(subdir_path, self.ground_truth_dir)
+            os.makedirs(subdir_path, exist_ok=True)
+            os.makedirs(subdir_gt_path, exist_ok=True)
+            for filename in tqdm(
+                self.data_files,
+                desc=f"Downloading {subdir}",
+            ):
+                response = requests.get(
+                    f"{self.data_dir_url}/{subdir}/{filename}",
+                )
+                response.raise_for_status()
+                with open(os.path.join(subdir_path, filename), "wb") as f:
+                    f.write(response.content)
+
+                gt_response = requests.get(
+                    f"{self.data_dir_url}/{subdir}/"
+                    f"{self.ground_truth_dir}/{filename}",
+                )
+                gt_response.raise_for_status()
+                with open(os.path.join(subdir_gt_path, filename), "wb") as f:
+                    f.write(gt_response.content)
+
+    @staticmethod
+    def _data_to_task(item: dict) -> Task:
+        """Convert a dataset item to a Task object."""
+        # Start the simulated phone and load initial configuration
+        ace_phone = ACEPhone()
+        ace_phone.load_initial_config(item["initial_config"])
+
+        # Obtain tool functions
+        tools: list[tuple] = []
+        for function_schema in item["function"]:
+            name = function_schema["name"]
+
+            # Handle the schema differences
+            formatted_schema = json.loads(
+                json.dumps(
+                    function_schema,
+                ).replace(
+                    '"type": "dict"',
+                    '"type": "object"',
+                ),
+            )
+
+            tool_function = ace_phone.get_tool_function(name)
+            tools.append(
+                (
+                    tool_function,
+                    {
+                        "type": "function",
+                        "function": formatted_schema,
+                    },
+                ),
+            )
+
+        return Task(
+            id=item["id"],
+            input=item["question"],
+            ground_truth={
+                "state": item["ground_truth"],
+                "mile_stone": item.get("mile_stone", []),
+            },
+            tags=item.get("tags", {}),
+            metrics=[
+                ACEAccuracy(item["ground_truth"]),
+                ACEProcessAccuracy(item["mile_stone"]),
+            ],
+            metadata={
+                # The phone is used to extract the final state after finishing
+                # the task.
+                "phone": ace_phone,
+                # The provided tools for this task, used to equip the agent
+                "tools": tools,
+            },
+        )
+
+    def __iter__(self) -> Generator[Task, None, None]:
+        """Iterate over the benchmark."""
+        for item in self.dataset:
+            yield self._data_to_task(item)
+
+    def __getitem__(self, index: int) -> Task:
+        """Get a task by index."""
+        return self._data_to_task(self.dataset[index])
+
+    def __len__(self) -> int:
+        """Get the length of the benchmark."""
+        return len(self.dataset)

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_metric.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+"""The ACE benchmark metric implementations in AgentScope."""
+
+from .._solution import SolutionOutput
+from .._metric_base import MetricBase, MetricResult, MetricType
+
+
+class ACEProcessAccuracy(MetricBase):
+    """The ace benchmark process accuracy metric."""
+
+    def __init__(
+        self,
+        mile_stone: list[str],
+    ) -> None:
+        """Initialize the AceBench process accuracy metric."""
+        super().__init__(
+            name="process_accuracy",
+            metric_type=MetricType.NUMERICAL,
+            description="The AceBench Agent eval process accuracy metric.",
+        )
+        self.mile_stone = mile_stone
+
+    async def __call__(
+        self,
+        solution: SolutionOutput,
+    ) -> MetricResult:
+        """Calculate the metric result."""
+
+        # Turn the tool use block sequence into ACEBench format
+        # e.g. func(arg1='dfd', arg2=44)
+        gathered_trajectory = []
+        for tool_call in solution.trajectory:
+            if tool_call.get("type") == "tool_use":
+                function_name = tool_call.get("name")
+                kwargs = tool_call.get("input")
+
+                gathered_kwargs = []
+                for key, value in kwargs.items():
+                    if isinstance(value, str):
+                        gathered_kwargs.append(
+                            f"{key}='{value}'",
+                        )
+
+                    else:
+                        gathered_kwargs.append(
+                            f"{key}={value}",
+                        )
+
+                kwargs_str = ", ".join(gathered_kwargs)
+                gathered_trajectory.append(
+                    f"[{function_name}({kwargs_str})]",
+                )
+
+        for stone in self.mile_stone:
+            if stone not in gathered_trajectory:
+                return MetricResult(
+                    name=self.name,
+                    result=0,
+                    message=f"Error: Missing milestone '{stone}' in "
+                    "the given trajectory.",
+                )
+
+        return MetricResult(
+            name=self.name,
+            result=1,
+            message="Success",
+        )
+
+
+class ACEAccuracy(MetricBase):
+    """The ace benchmark metric"""
+
+    def __init__(
+        self,
+        state: list[dict],
+    ) -> None:
+        """Initialize the _metric object."""
+        super().__init__(
+            "accuracy",
+            MetricType.NUMERICAL,
+            "The AceBench Agent eval accuracy metric.",
+        )
+        self.state = state
+
+    async def __call__(
+        self,
+        solution: SolutionOutput,
+    ) -> MetricResult:
+        """Calculate the metric result."""
+        # Check if the solution matches the ground truth
+        if not isinstance(solution.output, list):
+            raise ValueError("Ground truth state must be a list.")
+
+        # Handle the typos in ACEBench dataset
+        gathered_state = {}
+        for item in self.state:
+            for key, value in item.items():
+                if key.endswith("API"):
+                    key = key.replace("API", "Api")
+                elif key.endswith("rpi"):
+                    key = key.replace("pi", "Api")
+                gathered_state[key] = value
+
+        gathered_output = {}
+        for item in solution.output:
+            for key, value in item.items():
+                gathered_output[key] = value
+
+        if not set(gathered_state.keys()).issubset(gathered_output.keys()):
+            raise ValueError(
+                "Missing keys in solution output compared to state, "
+                f"ground truth keys: {gathered_state.keys()}, "
+                f"solution keys: {gathered_output.keys()}",
+            )
+
+        for key, value in gathered_state.items():
+            if value != gathered_output.get(key):
+                return MetricResult(
+                    name=self.name,
+                    result=0,
+                    message=(
+                        f"Error: Mismatch in key '{key}':"
+                        f"\n{value}\n{gathered_output.get(key)}"
+                    ),
+                )
+
+        return MetricResult(
+            name=self.name,
+            result=1,
+            message="Success: All keys match",
+        )

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_tools_api/__init__.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+"""The ACEBench simulation tools in AgentScope."""
+
+from ._message_api import MessageApi
+from ._travel_api import TravelApi
+from ._reminder_api import ReminderApi
+from ._food_platform_api import FoodPlatformApi
+
+__all__ = [
+    "MessageApi",
+    "TravelApi",
+    "ReminderApi",
+    "FoodPlatformApi",
+]

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_tools_api/_food_platform_api.py

@@ -0,0 +1,302 @@
+# -*- coding: utf-8 -*-
+"""The food platform API in the ACEBench evaluation."""
+
+from ._shared_state import SharedState
+
+
+class FoodPlatformApi(SharedState):
+    """The food platform Api in the ACEBench evaluation."""
+
+    tool_functions: list[str] = [
+        "login_food_platform",
+        "view_logged_in_users",
+        "check_balance",
+        "add_food_delivery_order",
+        "get_products",
+        "view_orders",
+        "search_orders",
+    ]
+
+    def __init__(self, shared_state: dict) -> None:
+        super().__init__(shared_state)
+
+        # 设置用户和初始金额
+        self.users: dict = {
+            "Eve": {
+                "user_id": "U100",
+                "password": "password123",
+                "balance": 500.0,
+            },
+            "Frank": {
+                "user_id": "U101",
+                "password": "password456",
+                "balance": 300.0,
+            },
+            "Grace": {
+                "user_id": "U102",
+                "password": "password789",
+                "balance": 150.0,
+            },
+            "Helen": {
+                "user_id": "U103",
+                "password": "password321",
+                "balance": 800.0,
+            },
+            "Isaac": {
+                "user_id": "U104",
+                "password": "password654",
+                "balance": 400.0,
+            },
+            "Jack": {
+                "user_id": "U105",
+                "password": "password654",
+                "balance": 120.0,
+            },
+        }
+
+        # 设置六个商家及其菜单
+        self.merchant_list: dict[str, dict] = {
+            "达美乐": {
+                "merchant_id": "M100",
+                "service_type": "Pizza",
+                "menu": [
+                    {"product": "玛格丽特披萨", "price": 68.0},
+                    {"product": "超级至尊披萨", "price": 88.0},
+                ],
+            },
+            "米村拌饭": {
+                "merchant_id": "M101",
+                "service_type": "Bibimbap",
+                "menu": [
+                    {"product": "石锅拌饭", "price": 35.0},
+                    {"product": "韩式牛肉拌饭", "price": 45.0},
+                ],
+            },
+            "海底捞": {
+                "merchant_id": "M102",
+                "service_type": "Hotpot",
+                "menu": [
+                    {"product": "牛肉卷", "price": 68.0},
+                    {"product": "海鲜拼盘", "price": 88.0},
+                ],
+            },
+            "喜茶": {
+                "merchant_id": "M103",
+                "service_type": "Milk Tea",
+                "menu": [
+                    {"product": "芝士奶茶", "price": 25.0},
+                    {"product": "四季春奶茶", "price": 22.0},
+                ],
+            },
+            "盒马生鲜": {
+                "merchant_id": "M104",
+                "service_type": "Fresh Grocery",
+                "menu": [
+                    {"product": "有机蔬菜包", "price": 15.0},
+                    {"product": "生鲜大礼包", "price": 99.0},
+                ],
+            },
+            "九田家烤肉": {
+                "merchant_id": "M105",
+                "service_type": "BBQ",
+                "menu": [
+                    {"product": "韩式烤牛肉", "price": 128.0},
+                    {"product": "烤五花肉", "price": 78.0},
+                ],
+            },
+        }
+
+        # 设置已登录用户列表
+        self.logged_in_users: list[str] = []
+        # 订单列表
+        self.orders: list = []
+
+    def get_state_dict(self) -> dict:
+        """Get the current state dict of the FoodPlatformApi."""
+        return {
+            "FoodPlatform": {
+                "logged_in_users": self.logged_in_users,
+                "orders": self.orders,
+                "users": self.users,
+            },
+        }
+
+    def login_food_platform(
+        self,
+        username: str,
+        password: str,
+    ) -> dict[str, bool | str]:
+        """使用用户名和密码登录外卖平台。
+
+        Args:
+            username (`str`):
+                用户的用户名。
+            password (`str`):
+                用户的密码。
+        """
+        if not self.wifi:
+            return {"status": False, "message": "wifi未打开，无法登录"}
+        if username not in self.users:
+            return {"status": False, "message": "用户不存在"}
+        if self.users[username]["password"] != password:
+            return {"status": False, "message": "密码错误"}
+
+        # 检查是否已经有用户登录
+        if username in self.logged_in_users:
+            return {"status": False, "message": f"{username} 已经登录"}
+
+        # 记录已登录用户
+        self.logged_in_users.append(username)
+        return {"status": True, "message": f"用户{username}登陆成功！"}
+
+    def view_logged_in_users(self) -> dict:
+        """查看当前所有登录的用户。"""
+        if not self.logged_in_users:
+            return {
+                "status": False,
+                "message": "当前没有登录food platform",
+            }
+
+        return {"status": True, "logged_in_users": self.logged_in_users}
+
+    def check_balance(self, user_name: str) -> float:
+        """查询指定用户的余额。
+
+        Args:
+            user_name (`str`):
+                用户的用户名。
+        """
+        if user_name in self.users:
+            return self.users[user_name]["balance"]
+        else:
+            return 0.0
+
+    def add_food_delivery_order(
+        self,
+        username: str,
+        merchant_name: str,
+        items: list[dict[str, str | int]],
+    ) -> dict[str, bool | str]:
+        """订外卖
+
+        Args:
+            username (`str`):
+                下订单的用户姓名。
+            merchant_name (`str`):
+                下订单的商家名称。
+            items (`list[dict[str, str | int]]`):
+                订单中商品的列表，每个商品包含名称和数量。
+        """
+        if username not in self.logged_in_users:
+            return {
+                "status": False,
+                "message": f"用户 {username} 未登录food platform",
+            }
+
+        if merchant_name not in self.merchant_list:
+            return {"status": False, "message": "商家不存在"}
+
+        total_price = 0.0
+        order_items = []
+
+        for item in items:
+            product_name = item.get("product")
+            quantity = item.get("quantity", 1)
+
+            if not isinstance(quantity, int) or quantity <= 0:
+                return {
+                    "status": False,
+                    "message": f"无效的数量 {quantity} 对于商品 {product_name}",
+                }
+
+            # 查找商品价格
+            product_found = False
+            for product in self.merchant_list[merchant_name]["menu"]:
+                if product["product"] == product_name:
+                    total_price += product["price"] * quantity
+                    order_items.append(
+                        {
+                            "product": product_name,
+                            "quantity": quantity,
+                            "price_per_unit": product["price"],
+                        },
+                    )
+                    product_found = True
+                    break
+            if not product_found:
+                return {
+                    "status": False,
+                    "message": f"商品 {product_name} 不存在于 "
+                    f"{merchant_name} 的菜单中",
+                }
+
+        # 检查余额是否足够
+        if total_price >= self.users[username]["balance"]:
+            return {"status": False, "message": "余额不足，无法下单"}
+
+        # 扣除余额并创建订单
+        self.users[username]["balance"] -= total_price
+        order = {
+            "user_name": username,
+            "merchant_name": merchant_name,
+            "items": order_items,
+            "total_price": total_price,
+        }
+        self.orders.append(order)
+        return {
+            "status": True,
+            "message": f"外卖订单成功下单给 {merchant_name}，" f"总金额为 {total_price} 元",
+        }
+
+    def get_products(
+        self,
+        merchant_name: str,
+    ) -> list[dict[str, str | float]] | dict[str, bool | str]:
+        """获取特定商家的商品列表。
+
+        Args:
+            merchant_name (`str`):
+                要获取商品的商家名称。
+        """
+        merchant = self.merchant_list.get(merchant_name)
+        if merchant:
+            return merchant["menu"]
+        else:
+            return {
+                "status": False,
+                "message": f"商家 '{merchant_name}' 不存在",
+            }
+
+    def view_orders(
+        self,
+        user_name: str,
+    ) -> dict[str, bool | str | list[dict[str, str | int | float]]]:
+        """查看用户的所有订单"""
+        user_orders = [
+            order for order in self.orders if order["user_name"] == user_name
+        ]
+
+        if not user_orders:
+            return {"status": False, "message": "用户没有订单记录"}
+
+        return {"status": True, "orders": user_orders}
+
+    def search_orders(
+        self,
+        keyword: str,
+    ) -> dict[str, bool | str | list[dict[str, str | float]]]:
+        """根据关键字搜索订单。"""
+        matched_orders = [
+            order
+            for order in self.orders
+            if keyword.lower() in order["merchant_name"].lower()
+            or any(
+                keyword.lower() in item.lower()
+                for item in order.get("items", [])
+            )
+        ]
+
+        if not matched_orders:
+            return {"status": False, "message": "没有找到匹配的订单"}
+
+        return {"status": True, "orders": matched_orders}

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_tools_api/_message_api.py

@@ -0,0 +1,340 @@
+# -*- coding: utf-8 -*-
+"""The Message API in the ACEBench evaluation."""
+from datetime import datetime
+
+from ._shared_state import SharedState
+
+
+class MessageApi(SharedState):
+    """The message Api in the ACEBench evaluation."""
+
+    tool_functions: list[str] = [
+        "send_message",
+        "delete_message",
+        "view_messages_between_users",
+        "search_messages",
+        "get_all_message_times_with_ids",
+        "get_latest_message_id",
+        "get_earliest_message_id",
+    ]
+
+    def __init__(self, share_state: dict) -> None:
+        """Initialize the MessageApi with shared state."""
+        super().__init__(share_state)
+
+        # 设置六个用户
+        self.max_capacity = 6
+        self.user_list: dict[str, dict[str, str | int]] = {
+            "Eve": {
+                "user_id": "USR100",
+                "phone_number": "123-456-7890",
+                "occupation": "Software Engineer",
+            },
+            "Frank": {
+                "user_id": "USR101",
+                "phone_number": "234-567-8901",
+                "occupation": "Data Scientist",
+            },
+            "Grace": {
+                "user_id": "USR102",
+                "phone_number": "345-678-9012",
+                "occupation": "Product Manager",
+            },
+            "Helen": {
+                "user_id": "USR103",
+                "phone_number": "456-789-0123",
+                "occupation": "UX Designer",
+            },
+            "Isaac": {
+                "user_id": "USR104",
+                "phone_number": "567-890-1234",
+                "occupation": "DevOps Engineer",
+            },
+            "Jack": {
+                "user_id": "USR105",
+                "phone_number": "678-901-2345",
+                "occupation": "Marketing Specialist",
+            },
+        }
+
+        # 设置六个用户之间的短信记录
+        # 信息1和reminder配合  信息2和food配合
+        self.inbox: dict[int, dict[str, str | int]] = {
+            1: {
+                "sender_id": "USR100",
+                "receiver_id": "USR101",
+                "message": "Hey Frank, don't forget about our meeting on "
+                "2024-06-11 at 4 PM in Conference Room 1.",
+                "time": "2024-06-09",
+            },
+            2: {
+                "sender_id": "USR101",
+                "receiver_id": "USR102",
+                "message": """你能帮我点一个\"玛格丽特披萨\"的外卖吗,商家是达美乐。""",
+                "time": "2024-03-09",
+            },
+            3: {
+                "sender_id": "USR102",
+                "receiver_id": "USR103",
+                "message": "帮我查一些喜茶有哪些奶茶外卖，买一杯便宜些的奶茶。"
+                "买完以后记得回复我,回复的内容是（已经买好了）",
+                "time": "2023-12-05",
+            },
+            4: {
+                "sender_id": "USR103",
+                "receiver_id": "USR102",
+                "message": "No problem Helen, I can assist you.",
+                "time": "2024-09-09",
+            },
+            5: {
+                "sender_id": "USR104",
+                "receiver_id": "USR105",
+                "message": "Isaac, are you available for a call?",
+                "time": "2024-06-06",
+            },
+            6: {
+                "sender_id": "USR105",
+                "receiver_id": "USR104",
+                "message": "Yes Jack, let's do it in 30 minutes.",
+                "time": "2024-01-15",
+            },
+        }
+
+        self.message_id_counter: int = 6
+
+    def get_state_dict(self) -> dict:
+        """Get the current state dict of the MessageApi."""
+
+        # To avoid the error in ACEBench dataset
+        inbox_state = {}
+        for key, value in self.inbox.items():
+            inbox_state[str(key)] = value
+
+        return {
+            "MessageApi": {
+                "inbox": inbox_state,
+            },
+        }
+
+    def send_message(
+        self,
+        sender_name: str,
+        receiver_name: str,
+        message: str,
+    ) -> dict[str, bool | str]:
+        """将一条消息从一个用户发送给另一个用户。
+
+        Args:
+            sender_name (`str`):
+                发送消息的用户姓名。
+            receiver_name (`str`):
+                接收消息的用户姓名。
+            message (`str`):
+                要发送的消息内容。
+        """
+        if not self.logged_in:
+            return {"status": False, "message": "device未登录，无法发送短信"}
+
+        if not self.wifi:
+            return {"status": False, "message": "wifi关闭，此时不能发送信息"}
+
+        if len(self.inbox) >= self.max_capacity:
+            return {
+                "status": False,
+                "message": "内存容量不够了，你需要询问user删除哪一条短信。",
+            }
+
+        # 验证发送者和接收者是否存在
+        if (
+            sender_name not in self.user_list
+            or receiver_name not in self.user_list
+        ):
+            return {"status": False, "message": "发送者或接收者不存在"}
+
+        sender_id = self.user_list[sender_name]["user_id"]
+        receiver_id = self.user_list[receiver_name]["user_id"]
+
+        # 将短信添加到inbox
+        self.message_id_counter += 1
+        self.inbox[self.message_id_counter] = {
+            "sender_id": sender_id,
+            "receiver_id": receiver_id,
+            "message": message,
+        }
+
+        return {"status": True, "message": f"短信成功发送给{receiver_name}。"}
+
+    def delete_message(self, message_id: int) -> dict[str, bool | str]:
+        """根据消息 ID 删除一条消息。
+
+        Args:
+            message_id (`int`):
+                要删除的消息的 ID。
+        """
+        if not self.logged_in:
+            return {"status": False, "message": "device未登录，无法删除短信"}
+        if message_id not in self.inbox:
+            return {"status": False, "message": "短信ID不存在"}
+
+        del self.inbox[message_id]
+        return {"status": True, "message": f"短信ID {message_id} 已成功删除。"}
+
+    def view_messages_between_users(
+        self,
+        sender_name: str,
+        receiver_name: str,
+    ) -> dict:
+        """获取特定用户发送给另一个用户的所有消息。
+
+        Args:
+            sender_name (`str`):
+                发送消息的用户姓名。
+            receiver_name (`str`):
+                接收消息的用户姓名。
+        """
+        if not self.logged_in:
+            return {
+                "status": False,
+                "message": "device未登录，无法查看短信信息",
+            }
+
+        if sender_name not in self.user_list:
+            return {"status": False, "message": "发送者不存在"}
+
+        if receiver_name not in self.user_list:
+            return {"status": False, "message": "接收者不存在"}
+
+        sender_id = self.user_list[sender_name]["user_id"]
+        receiver_id = self.user_list[receiver_name]["user_id"]
+        messages_between_users = []
+
+        # 遍历 inbox，找出 sender_id 发送给 receiver_id 的短信
+        for msg_id, msg_data in self.inbox.items():
+            if (
+                msg_data["sender_id"] == sender_id
+                and msg_data["receiver_id"] == receiver_id
+            ):
+                messages_between_users.append(
+                    {
+                        "id": msg_id,
+                        "sender": sender_name,
+                        "receiver": receiver_name,
+                        "message": msg_data["message"],
+                    },
+                )
+
+        if not messages_between_users:
+            return {"status": False, "message": "没有找到相关的短信记录"}
+
+        return {"status": True, "messages": messages_between_users}
+
+    def search_messages(
+        self,
+        user_name: str,
+        keyword: str,
+    ) -> dict:
+        """搜索特定用户消息中包含特定关键字的消息。
+
+        Args:
+            user_name (`str`):
+                要搜索消息的用户姓名。
+            keyword (`str`):
+                要在消息中搜索的关键字。
+        """
+        if user_name not in self.user_list:
+            return {"status": False, "message": "用户不存在"}
+
+        user_id = self.user_list[user_name]["user_id"]
+        matched_messages = []
+
+        # 遍历 inbox，找到发送或接收中包含关键词的消息
+        for msg_id, msg_data in self.inbox.items():
+            if (
+                user_id in (msg_data["sender_id"], msg_data["receiver_id"])
+                and keyword.lower() in msg_data["message"].lower()
+            ):
+                matched_messages.append(
+                    {
+                        "id": msg_id,
+                        "sender_id": msg_data["sender_id"],
+                        "receiver_id": msg_data["receiver_id"],
+                        "message": msg_data["message"],
+                    },
+                )
+
+        if not matched_messages:
+            return {"status": False, "message": "没有找到包含关键词的短信"}
+
+        return {"status": True, "messages": matched_messages}
+
+    def get_all_message_times_with_ids(
+        self,
+    ) -> dict:
+        """获取所有短信的时间以及对应的短信编号。"""
+        if not self.logged_in:
+            return {
+                "status": False,
+                "message": "device未登录，获取所有短信的时间以及对应的短信编号。",
+            }
+        message_times_with_ids = {
+            msg_id: msg_data["time"] for msg_id, msg_data in self.inbox.items()
+        }
+        return message_times_with_ids
+
+    def get_latest_message_id(self) -> dict:
+        """获取最近发送的消息的 ID。"""
+        if not self.logged_in:
+            return {
+                "status": False,
+                "message": "device未登录，无法获取最新发送的短信ID。",
+            }
+        if not self.inbox:
+            return {"status": False, "message": "短信记录为空"}
+
+        # 遍历所有短信，找出时间最新的短信
+        latest_message_id = None
+        latest_time = None
+
+        for message_id, message_data in self.inbox.items():
+            message_time = datetime.strptime(
+                str(message_data["time"]),
+                "%Y-%m-%d",
+            )
+            if latest_time is None or message_time > latest_time:
+                latest_time = message_time
+                latest_message_id = message_id
+
+        return {
+            "status": True,
+            "message": f"最新的短信ID是 {latest_message_id}",
+            "message_id": latest_message_id,
+        }
+
+    def get_earliest_message_id(self) -> dict:
+        """获取最早发送的消息的 ID。"""
+        if not self.logged_in:
+            return {
+                "status": False,
+                "message": "device未登录，无法获取最早发送的短信ID",
+            }
+        if not self.inbox:
+            return {"status": False, "message": "短信记录为空"}
+
+        # 遍历所有短信，找出时间最早的短信
+        earliest_message_id = None
+        earliest_time = None
+
+        for message_id, message_data in self.inbox.items():
+            message_time = datetime.strptime(
+                str(message_data["time"]),
+                "%Y-%m-%d",
+            )
+            if earliest_time is None or message_time < earliest_time:
+                earliest_time = message_time
+                earliest_message_id = message_id
+
+        return {
+            "status": True,
+            "message": f"最早的短信ID是 {earliest_message_id}",
+            "message_id": earliest_message_id,
+        }

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_tools_api/_reminder_api.py

@@ -0,0 +1,214 @@
+# -*- coding: utf-8 -*-
+"""The reminder API in ACEBench simulation tools."""
+from datetime import datetime
+
+from ._shared_state import SharedState
+
+
+class ReminderApi(SharedState):
+    """The reminder Api in the ACEBench evaluation."""
+
+    tool_functions: list[str] = [
+        "view_reminder_by_title",
+        "add_reminder",
+        "delete_reminder",
+        "view_all_reminders",
+        "mark_as_notified",
+        "search_reminders",
+    ]
+
+    def __init__(self, share_state: dict) -> None:
+        """Initialize the Reminder Api in the ACEBench evaluation."""
+        super().__init__(share_state)
+
+        self.max_capacity = 6
+        self.reminder_list: dict[
+            int,
+            dict,
+        ] = {
+            1: {
+                "reminder_id": 1001,
+                "title": "Doctor's Appointment",
+                "description": "Visit Dr. Smith for a checkup.",
+                "time": "2024-07-15 09:30",
+                "notified": False,
+            },
+            2: {
+                "reminder_id": 1002,
+                "title": "Team Meeting",
+                "description": "Monthly project review with the team.",
+                "time": "2024-07-17 11:00",
+                "notified": False,
+            },
+            3: {
+                "reminder_id": 1003,
+                "title": "To-do list",
+                "description": '首先帮Frank在"盒马生鲜"点外卖，'
+                '需要定两个"生鲜大礼包"，再发短信告诉Frank：'
+                '"购买商品的价格是()元"。要把括号换成实际金额，'
+                "保留一位小数。",
+                "time": "2024-07-16 11:00",
+                "notified": False,
+            },
+        }
+        self.reminder_id_counter: int = 3
+
+    def get_state_dict(self) -> dict:
+        """Get the current state dict of the ReminderApi."""
+        return {
+            "ReminderApi": {
+                "reminder_list": self.reminder_list,
+            },
+        }
+
+    def _check_capacity(self) -> bool:
+        """检查备忘录容量是否已满。"""
+        return len(self.reminder_list) >= self.max_capacity
+
+    def view_reminder_by_title(
+        self,
+        title: str,
+    ) -> dict[str, str | bool | dict[str, str | bool | datetime]]:
+        """根据提醒的标题查看特定的提醒。
+
+        Args:
+            title (str): 提醒的标题。
+
+        Returns:
+            dict[str, str | bool | dict[str, str | bool | datetime]]:
+                包含查找状态和提醒详情的字典。
+        """
+        if not self.logged_in:
+            return {"status": False, "message": "device未登录，无法查看提醒"}
+        for reminder in self.reminder_list.values():
+            if reminder["title"] == title:
+                return {"status": True, "reminder": reminder}
+
+        return {"status": False, "message": f"没有找到标题为 '{title}' 的提醒"}
+
+    def add_reminder(
+        self,
+        title: str,
+        description: str,
+        time: datetime,
+    ) -> dict[str, bool | str]:
+        """添加一个新的提醒。
+
+        Args:
+            title (str): 提醒标题。
+            description (str): 提醒描述。
+            time (datetime): 提醒时间, 一定遵循格式"YYYY-MM-DD HH:MM"。
+
+        Returns:
+            dict[str, bool | str]: 包含添加状态和结果的字典。
+        """
+        if not self.logged_in:
+            return {
+                "status": False,
+                "message": "device未登录，无法添加一个新的提醒",
+            }
+        if self._check_capacity():
+            return {"status": False, "message": "提醒容量已满，无法添加新的提醒"}
+
+        self.reminder_id_counter += 1
+        reminder_id = self.reminder_id_counter
+        self.reminder_list[reminder_id] = {
+            "reminder_id": reminder_id,
+            "title": title,
+            "description": description,
+            "time": time,
+            "notified": False,
+        }
+        return {"status": True, "message": f"提醒 '{title}' 已成功添加"}
+
+    def delete_reminder(self, reminder_id: int) -> dict[str, bool | str]:
+        """删除指定的提醒。
+
+        Args:
+            reminder_id (int): 要删除的提醒ID。
+
+        Returns:
+            dict[str, bool | str]: 包含删除状态和结果的字典。
+        """
+        if not self.logged_in:
+            return {"status": False, "message": "device未登录，无法删除指定的提醒"}
+        if reminder_id not in self.reminder_list:
+            return {"status": False, "message": "提醒ID不存在"}
+
+        del self.reminder_list[reminder_id]
+        return {"status": True, "message": f"提醒ID {reminder_id} 已成功删除"}
+
+    def view_all_reminders(
+        self,
+    ) -> dict:
+        """查看所有的提醒。
+
+        Returns:
+            dict:
+                包含所有提醒的字典列表。
+        """
+        if not self.reminder_list:
+            return {"status": False, "message": "没有任何提醒"}
+
+        reminders = []
+        for reminder in self.reminder_list.values():
+            reminders.append(
+                {
+                    "title": reminder["title"],
+                    "description": reminder["description"],
+                    "time": reminder["time"],
+                    "notified": reminder["notified"],
+                },
+            )
+        return {"status": True, "reminders": reminders}
+
+    def mark_as_notified(
+        self,
+        reminder_id: int,
+    ) -> dict[str, bool | str]:
+        """标记提醒为已通知。
+
+        Args:
+            reminder_id (int): 要标记为已通知的提醒ID。
+
+        Returns:
+            dict[str, bool | str]:: 包含操作结果的字典。
+        """
+        if reminder_id not in self.reminder_list:
+            return {"status": False, "message": "提醒ID不存在"}
+
+        self.reminder_list[reminder_id]["notified"] = True
+        return {"status": True, "message": f"提醒ID {reminder_id} 已标记为已通知"}
+
+    def search_reminders(
+        self,
+        keyword: str,
+    ) -> dict:
+        """根据关键词搜索提醒。
+
+        Args:
+            keyword (str): 搜索关键词。
+
+        Returns:
+            `dict`:
+                包含匹配提醒的字典列表。
+        """
+        matched_reminders = []
+
+        for reminder in self.reminder_list.values():
+            if (
+                keyword.lower() in reminder["title"].lower()
+                or keyword.lower() in reminder["description"].lower()
+            ):
+                matched_reminders.append(
+                    {
+                        "title": reminder["title"],
+                        "description": reminder["description"],
+                        "time": reminder["time"].strftime("%Y-%m-%d %H:%M"),
+                    },
+                )
+
+        if not matched_reminders:
+            return {"status": False, "message": "没有找到包含该关键词的提醒"}
+
+        return {"status": True, "reminders": matched_reminders}

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_tools_api/_shared_state.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+"""The shared state class for ACEBench simulation tools."""
+
+
+class SharedState:
+    """The sharing state class for ACEBench simulation tools."""
+
+    def __init__(self, shared_state: dict) -> None:
+        """Initialize the shared state"""
+        self._shared_state = shared_state
+
+    @property
+    def wifi(self) -> bool:
+        """The WI-FI state"""
+        return self._shared_state["wifi"]
+
+    @property
+    def logged_in(self) -> bool:
+        """The logged in state"""
+        return self._shared_state["logged_in"]

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_tools_api/_travel_api.py

@@ -0,0 +1,834 @@
+# -*- coding: utf-8 -*-
+# type: ignore
+# pylint: disable=too-many-lines
+# pylint: disable=too-many-statements
+# pylint: disable=too-many-branches
+# pylint: disable=too-many-statements
+# pylint: disable=too-many-return-statements
+"""The travel API for the ACEBench simulation tools in AgentScope."""
+
+from datetime import datetime, timedelta
+
+
+class TravelApi:
+    """旅行预订系统类。
+
+    提供航班查询、用户认证、预订管理等功能的旅行系统。
+    支持直飞和中转航班查询、航班预订、预订修改和取消等功能。
+    """
+
+    tool_functions: list[str] = [
+        "get_user_details",
+        "get_flight_details",
+        "get_reservation_details",
+        "reserve_flight",
+        "cancel_reservation",
+        "modify_flight",
+    ]
+
+    def __init__(self) -> None:
+        """初始化旅行系统。
+
+        设置用户档案和航班信息，包含用户信息、航班数据和预订记录。
+        """
+        # 初始化用户信息
+        self.users = {
+            "user1": {
+                "user_name": "Eve",
+                "password": "password123",
+                "cash_balance": 2000.0,
+                "bank_balance": 50000.0,
+                "membership_level": "regular",
+            },
+            "user2": {
+                "user_name": "Frank",
+                "password": "password456",
+                "cash_balance": 8000.0,
+                "bank_balance": 8000.0,
+                "membership_level": "silver",
+            },
+            "user3": {
+                "user_name": "Grace",
+                "password": "password789",
+                "cash_balance": 1000.0,
+                "bank_balance": 5000.0,
+                "membership_level": "gold",
+            },
+        }
+
+        # 初始化航班信息
+        self.flights = [
+            {
+                "flight_no": "CA1234",
+                "origin": "北京",
+                "destination": "上海",
+                "depart_time": "2024-07-15 08:00:00",
+                "arrival_time": "2024-07-15 10:30:00",
+                "status": "available",
+                "seats_available": 5,
+                "economy_price": 1200,
+                "business_price": 3000,
+            },
+            {
+                "flight_no": "MU5678",
+                "origin": "上海",
+                "destination": "北京",
+                "depart_time": "2024-07-16 09:00:00",
+                "arrival_time": "2024-07-16 11:30:00",
+                "status": "available",
+                "seats_available": 3,
+                "economy_price": 1900,
+                "business_price": 3000,
+            },
+            {
+                "flight_no": "CZ4321",
+                "origin": "上海",
+                "destination": "北京",
+                "depart_time": "2024-07-16 20:00:00",
+                "arrival_time": "2024-07-16 22:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 2500,
+                "business_price": 4000,
+            },
+            {
+                "flight_no": "CZ4352",
+                "origin": "上海",
+                "destination": "北京",
+                "depart_time": "2024-07-17 20:00:00",
+                "arrival_time": "2024-07-17 22:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 1600,
+                "business_price": 2500,
+            },
+            {
+                "flight_no": "MU3561",
+                "origin": "北京",
+                "destination": "南京",
+                "depart_time": "2024-07-18 08:00:00",
+                "arrival_time": "2024-07-18 10:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 1500,
+                "business_price": 4000,
+            },
+            {
+                "flight_no": "MU1566",
+                "origin": "北京",
+                "destination": "南京",
+                "depart_time": "2024-07-18 20:00:00",
+                "arrival_time": "2024-07-18 22:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 1500,
+                "business_price": 4000,
+            },
+            {
+                "flight_no": "CZ1765",
+                "origin": "南京",
+                "destination": "深圳",
+                "depart_time": "2024-07-17 20:30:00",
+                "arrival_time": "2024-07-17 22:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 1500,
+                "business_price": 2500,
+            },
+            {
+                "flight_no": "CZ1765",
+                "origin": "南京",
+                "destination": "深圳",
+                "depart_time": "2024-07-18 12:30:00",
+                "arrival_time": "2024-07-18 15:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 1500,
+                "business_price": 2500,
+            },
+            {
+                "flight_no": "MH1765",
+                "origin": "厦门",
+                "destination": "成都",
+                "depart_time": "2024-07-17 12:30:00",
+                "arrival_time": "2024-07-17 15:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 1500,
+                "business_price": 2500,
+            },
+            {
+                "flight_no": "MH2616",
+                "origin": "成都",
+                "destination": "厦门",
+                "depart_time": "2024-07-18 18:30:00",
+                "arrival_time": "2024-07-18 21:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 1500,
+                "business_price": 2500,
+            },
+            {
+                "flight_no": "MH2616",
+                "origin": "成都",
+                "destination": "福州",
+                "depart_time": "2024-07-16 18:30:00",
+                "arrival_time": "2024-07-16 21:00:00",
+                "status": "available",
+                "seats_available": 8,
+                "economy_price": 1500,
+                "business_price": 2500,
+            },
+        ]
+
+        # 初始化预订列表
+        self.reservations = [
+            {
+                "reservation_id": "res_1",
+                "user_id": "user1",
+                "flight_no": "CA1234",
+                "payment_method": "bank",
+                "cabin": "经济舱",
+                "baggage": 1,
+                "origin": "北京",
+                "destination": "上海",
+            },
+            {
+                "reservation_id": "res_2",
+                "user_id": "user1",
+                "flight_no": "MU5678",
+                "payment_method": "bank",
+                "cabin": "商务舱",
+                "baggage": 1,
+                "origin": "上海",
+                "destination": "北京",
+            },
+            {
+                "reservation_id": "res_3",
+                "user_id": "user2",
+                "flight_no": "MH1765",
+                "payment_method": "bank",
+                "cabin": "商务舱",
+                "baggage": 1,
+                "origin": "厦门",
+                "destination": "成都",
+            },
+            {
+                "reservation_id": "res_4",
+                "user_id": "user2",
+                "flight_no": "MU2616",
+                "payment_method": "bank",
+                "cabin": "商务舱",
+                "baggage": 1,
+                "origin": "成都",
+                "destination": "厦门",
+            },
+        ]
+
+    def get_state_dict(self) -> dict:
+        """Get the current state dict of the TravelApi."""
+        return {
+            "Travel": {
+                "users": self.users,
+                "reservations": self.reservations,
+            },
+        }
+
+    # 根据出发地和到达地查询航班
+
+    def get_flight_details(
+        self,
+        origin: str = None,
+        destination: str = None,
+    ) -> list[dict] | str:
+        """根据出发地和到达地查询航班的基本信息。
+
+        Args:
+            origin (str, optional): 出发地城市名称。默认为None。
+            destination (str, optional): 目的地城市名称。默认为None。
+
+        Returns:
+            list[dict] | str: 符合条件的航班列表或无航班的提示信息。
+        """
+        flights = self.flights
+
+        # 过滤出发地
+        if origin:
+            flights = [
+                flight for flight in flights if flight["origin"] == origin
+            ]
+
+        # 过滤到达地
+        if destination:
+            flights = [
+                flight
+                for flight in flights
+                if flight["destination"] == destination
+            ]
+        if len(flights) == 0:
+            return "没有符合条件的直达航班"
+        # 返回查询结果
+        return [
+            {
+                "flight_no": flight["flight_no"],
+                "origin": flight["origin"],
+                "destination": flight["destination"],
+                "depart_time": flight["depart_time"],
+                "arrival_time": flight["arrival_time"],
+                "status": flight["status"],
+                "seats_available": flight["seats_available"],
+                "economy_price": flight["economy_price"],
+                "business_price": flight["business_price"],
+            }
+            for flight in flights
+        ]
+
+    def get_user_details(self, user_id: str, password: str) -> dict:
+        """根据用户名和密码查询用户信息。
+
+        Args:
+            user_id (str): 用户ID。
+            password (str): 用户密码。
+
+        Returns:
+            dict: 用户信息字典（不包含密码）或错误信息。
+        """
+        user = self.users.get(user_id)
+        if user and user["password"] == password:
+            return {
+                key: value for key, value in user.items() if key != "password"
+            }
+        return {"status": "error", "message": "用户名或密码不正确"}
+
+    def get_reservation_details(
+        self,
+        reservation_id: str = None,
+        user_id: str = None,
+    ) -> list[dict] | dict:
+        """根据预订ID或用户ID查询预订信息，包括对应航班的基本信息。
+
+        Args:
+            reservation_id (str, optional): 预订ID。默认为None。
+            user_id (str, optional): 用户ID。默认为None。
+
+        Returns:
+            `list[dict] | dict`:
+                详细预订信息列表或错误信息字典。
+        """
+        # 根据预订ID或用户ID筛选预订信息
+        if reservation_id:
+            reservations = [
+                reservation
+                for reservation in self.reservations
+                if reservation["reservation_id"] == reservation_id
+            ]
+        elif user_id:
+            reservations = [
+                reservation
+                for reservation in self.reservations
+                if reservation["user_id"] == user_id
+            ]
+        else:
+            return {"status": "error", "message": "请提供有效的预订ID或用户ID"}
+
+        # 对每个预订，附加航班信息
+        detailed_reservations = []
+        for reservation in reservations:
+            flight_info = next(
+                (
+                    flight
+                    for flight in self.flights
+                    if flight["flight_no"] == reservation["flight_no"]
+                ),
+                None,
+            )
+            detailed_reservation = {**reservation, "flight_info": flight_info}
+            detailed_reservations.append(detailed_reservation)
+
+        return detailed_reservations
+
+    def authenticate_user(self, user_id: str, password: str) -> dict:
+        """验证用户身份。
+
+        Args:
+            user_id (str): 用户ID。
+            password (str): 用户密码。
+
+        Returns:
+            `dict`:
+                用户信息字典或错误信息字典。
+        """
+        user = self.users.get(user_id)
+        if user and user["password"] == password:
+            return user
+        return {"status": "error", "message": "用户名或密码不正确"}
+
+    def get_baggage_allowance(
+        self,
+        membership_level: str,
+        cabin_class: str,
+    ) -> int:
+        """获取用户基于会员等级和舱位的免费托运行李限额。
+
+        Args:
+            membership_level (str): 会员等级 ("regular", "silver", "gold")。
+            cabin_class (str): 舱位 ("基础经济舱", "经济舱", "商务舱")。
+
+        Returns:
+            int: 免费托运行李数量。
+        """
+        allowance = {
+            "regular": {"经济舱": 1, "商务舱": 2},
+            "silver": {"经济舱": 2, "商务舱": 3},
+            "gold": {"经济舱": 3, "商务舱": 3},
+        }
+        return allowance.get(membership_level, {}).get(cabin_class, 0)
+
+    def find_transfer_flights(
+        self,
+        origin_city: str,
+        transfer_city: str,
+        destination_city: str,
+    ) -> list[dict] | str:
+        """查找从出发城市到目的地城市的中转航班。
+
+        确保第一班航班降落时间早于第二班航班起飞时间。
+
+        Args:
+            origin_city (str): 出发城市。
+            transfer_city (str): 中转城市。
+            destination_city (str): 到达城市。
+
+        Returns:
+            list[dict] | str:
+                满足条件的中转航班列表，每个航班包含两段航程的信息，或无航班提示。
+        """
+        # 获取从出发城市到中转城市的航班
+        first_leg_flights: list[dict] = [
+            flight
+            for flight in self.flights
+            if flight["origin"] == origin_city
+            and flight["destination"] == transfer_city
+            and flight["status"] == "available"
+        ]
+
+        # 获取从中转城市到目的地城市的航班
+        second_leg_flights = [
+            flight
+            for flight in self.flights
+            if flight["origin"] == transfer_city
+            and flight["destination"] == destination_city
+            and flight["status"] == "available"
+        ]
+
+        # 存储符合条件的中转航班
+        transfer_flights = []
+
+        # 遍历第一段航班和第二段航班，查找符合时间条件的组合
+        for first_flight in first_leg_flights:
+            first_arrival = datetime.strptime(
+                first_flight["arrival_time"],
+                "%Y-%m-%d %H:%M:%S",
+            )
+
+            for second_flight in second_leg_flights:
+                second_departure = datetime.strptime(
+                    str(second_flight["depart_time"]),
+                    "%Y-%m-%d %H:%M:%S",
+                )
+
+                # 检查第一班航班降落时间早于第二班航班起飞时间
+                if first_arrival < second_departure:
+                    transfer_flights.append(
+                        {
+                            "first_leg": first_flight,
+                            "second_leg": second_flight,
+                        },
+                    )
+
+        # 返回符合条件的中转航班列表
+        if transfer_flights:
+            return transfer_flights
+        else:
+            return "未找到符合条件的中转航班。"
+
+    def calculate_baggage_fee(
+        self,
+        membership_level: str,
+        cabin_class: str,
+        baggage_count: int,
+    ) -> float:
+        """计算行李费用。
+
+        Args:
+            membership_level (str): 会员等级。
+            cabin_class (str): 舱位等级。
+            baggage_count (int): 行李数量。
+
+        Returns:
+            float: 额外行李费用。
+        """
+        free_baggage = {
+            "regular": {"经济舱": 1, "商务舱": 2},
+            "silver": {"经济舱": 2, "商务舱": 3},
+            "gold": {"经济舱": 3, "商务舱": 3},
+        }
+        free_limit = free_baggage[membership_level][cabin_class]
+        additional_baggage = max(baggage_count - free_limit, 0)
+        return additional_baggage * 50
+
+    def update_balance(
+        self,
+        user: dict,
+        payment_method: str,
+        amount: float,
+    ) -> bool:
+        """更新用户的余额。
+
+        Args:
+            user (dict): 用户信息字典。
+            payment_method (str): 支付方式（"cash" 或 "bank"）。
+            amount (float): 更新金额（正数表示增加，负数表示减少）。
+
+        Returns:
+            bool: 如果余额充足且更新成功，返回 True，否则返回 False。
+        """
+        if payment_method == "cash":
+            if user["cash_balance"] + amount < 0:
+                return False  # 余额不足
+            user["cash_balance"] += amount
+        elif payment_method == "bank":
+            if user["bank_balance"] + amount < 0:
+                return False  # 余额不足
+            user["bank_balance"] += amount
+        return True
+
+    def reserve_flight(
+        self,
+        user_id: str,
+        password: str,
+        flight_no: str,
+        cabin: str,
+        payment_method: str,
+        baggage_count: int,
+    ) -> str:
+        """预订航班。
+
+        Args:
+            user_id (str): 用户ID。
+            password (str): 用户密码。
+            flight_no (str): 航班号。
+            cabin (str): 舱位等级。
+            payment_method (str): 支付方式。
+            baggage_count (int): 行李数量。
+
+        Returns:
+            str: 预订结果信息。
+        """
+        user = self.authenticate_user(user_id, password)
+        if not user:
+            return "认证失败，请检查用户ID和密码。"
+
+        # 检查航班和座位
+        flight = next(
+            (
+                f
+                for f in self.flights
+                if f["flight_no"] == flight_no and f["status"] == "available"
+            ),
+            None,
+        )
+
+        # 计算航班价格
+        price: int = (
+            flight["economy_price"]
+            if cabin == "经济舱"
+            else flight["business_price"]
+        )
+        total_cost = price
+
+        # 计算行李费用
+        baggage_fee = self.calculate_baggage_fee(
+            user["membership_level"],
+            cabin,
+            baggage_count,
+        )
+        total_cost += baggage_fee
+
+        # 检查支付方式
+        if payment_method not in ["cash", "bank"]:
+            return "支付方式无效"
+
+        # 更新预定后的余额
+        if payment_method == "cash":
+            if total_cost > self.users.get(user_id)["cash_balance"]:
+                return "cash余额不足，请考虑换一种支付方式"
+            self.users.get(user_id)["cash_balance"] -= total_cost
+        else:
+            if total_cost > self.users.get(user_id)["bank_balance"]:
+                return "bank余额不足，请考虑换一种支付方式"
+            self.users.get(user_id)["bank_balance"] -= total_cost
+
+        # 更新航班信息并生成预订
+        flight["seats_available"] -= 1
+        reservation_id = f"res_{len(self.reservations) + 1}"
+        reservation = {
+            "reservation_id": reservation_id,
+            "user_id": user_id,
+            "flight_no": flight_no,
+            "payment_method": payment_method,
+            "cabin": cabin,
+            "baggage": baggage_count,
+        }
+        self.reservations.append(reservation)
+
+        return f"预订成功，预订号：{reservation_id}，" f"总费用：{total_cost}元（包含行李费用）。"
+
+    def modify_flight(
+        self,
+        user_id: str,
+        reservation_id: str,
+        new_flight_no: str = None,
+        new_cabin: str = None,
+        add_baggage: int = 0,
+        new_payment_method: str = None,
+    ) -> str:
+        """修改航班预订，包括更改航班、舱位和行李。
+
+        Args:
+            user_id (str): 用户ID。
+            reservation_id (str): 预订ID。
+            new_flight_no (str, optional): 新的航班号。默认为None。
+            new_cabin (str, optional): 新的舱位。默认为None。
+            add_baggage (int, optional): 新增托运行李的数量。默认为0。
+            new_payment_method (str, optional): 新的付款方式。默认为None。
+
+        Returns:
+            str: 修改结果信息。
+        """
+        # 获取对应的预订
+        reservation = next(
+            (
+                r
+                for r in self.reservations
+                if r["reservation_id"] == reservation_id
+                and r["user_id"] == user_id
+            ),
+            None,
+        )
+        if not reservation:
+            return "预订未找到或用户ID不匹配。"
+
+        # 检查当前预订的航班信息
+        current_flight = next(
+            (
+                f
+                for f in self.flights
+                if f["flight_no"] == reservation["flight_no"]
+            ),
+            None,
+        )
+        if not current_flight:
+            return "航班信息未找到。"
+
+        # 获取原始支付方式或新提供的支付方式
+        payment_method = (
+            new_payment_method
+            if new_payment_method
+            else reservation["payment_method"]
+        )
+        user = self.users[user_id]
+        if not user:
+            return "用户信息未找到。"
+
+        # 存储处理结果
+        result_messages = []
+
+        if new_flight_no and new_flight_no != reservation["flight_no"]:
+            # 更新航班号（若提供）但必须匹配出发地和目的地
+            new_flight = next(
+                (f for f in self.flights if f["flight_no"] == new_flight_no),
+                None,
+            )
+            if (
+                new_flight
+                and new_flight["origin"] == current_flight["origin"]
+                and new_flight["destination"] == current_flight["destination"]
+            ):
+                reservation["flight_no"] = new_flight_no
+                result_messages.append("航班号已更改。")
+            else:
+                return "航班更改失败：新的航班号无效或目的地不匹配。"
+
+        # 更新舱位（若提供）并计算价格差价
+        if new_cabin and new_cabin != reservation.get("cabin"):
+            price_difference = self.calculate_price_difference(
+                current_flight,
+                reservation["cabin"],
+                new_cabin,
+            )
+            reservation["cabin"] = new_cabin
+            if price_difference > 0:
+                # 扣除差价
+                if self.update_balance(
+                    user,
+                    payment_method,
+                    -price_difference,
+                ):
+                    result_messages.append(
+                        f"舱位更改成功。已支付差价: {price_difference}。",
+                    )
+                else:
+                    result_messages.append("余额不足，无法支付舱位差价。")
+            elif price_difference < 0:
+                # 退款
+                self.update_balance(user, payment_method, -price_difference)
+                result_messages.append(f"舱位更改成功。已退款差价: {-price_difference}。")
+
+        # 增加托运行李，检查免费限额和计算费用
+        if add_baggage > 0:
+            membership = user["membership_level"]
+            max_free_baggage = self.get_baggage_allowance(
+                membership,
+                reservation["cabin"],
+            )
+            current_baggage = reservation.get("baggage", 0)
+            total_baggage = current_baggage + add_baggage
+            extra_baggage = max(0, total_baggage - max_free_baggage)
+            baggage_cost = extra_baggage * 50
+            if baggage_cost > 0:
+                # 扣除行李费用
+                if self.update_balance(user, payment_method, -baggage_cost):
+                    result_messages.append(
+                        f"行李已增加。需支付额外费用: {baggage_cost}。",
+                    )
+                else:
+                    result_messages.append("余额不足，无法支付额外行李费用。")
+            reservation["baggage"] = total_baggage
+
+        # 返回最终结果
+        if not result_messages:
+            result_messages.append("修改完成，无需额外费用。")
+        return " ".join(result_messages)
+
+    def cancel_reservation(
+        self,
+        user_id: str,
+        reservation_id: str,
+        reason: str,
+    ) -> str:
+        """取消预订。
+
+        Args:
+            user_id (str): 用户ID。
+            reservation_id (str): 预订ID。
+            reason (str): 取消原因。
+
+        Returns:
+            str: 取消结果信息。
+        """
+        # 设置默认当前时间为 2024年7月14日早上6点
+        current_time = datetime(2024, 7, 14, 6, 0, 0)
+
+        # 验证用户和预订是否存在
+        user = self.users.get(user_id, None)
+        if not user:
+            return "用户ID无效。"
+
+        reservation = next(
+            (
+                r
+                for r in self.reservations
+                if r["reservation_id"] == reservation_id
+                and r["user_id"] == user_id
+            ),
+            None,
+        )
+        if not reservation:
+            return "预订ID无效或与该用户无关。"
+
+        # 检查航班信息是否存在
+        flight = next(
+            (
+                f
+                for f in self.flights
+                if f["flight_no"] == reservation["flight_no"]
+            ),
+            None,
+        )
+        if not flight:
+            return "航班信息无效。"
+
+        # 检查航班是否已起飞
+        depart_time = datetime.strptime(
+            flight["depart_time"],
+            "%Y-%m-%d %H:%M:%S",
+        )
+        if current_time > depart_time:
+            return "航段已使用，无法取消。"
+
+        # 计算距离出发时间
+        time_until_departure = depart_time - current_time
+        cancel_fee = 0
+        refund_amount = 0
+
+        # 获取航班价格
+        flight_price = (
+            flight["economy_price"]
+            if reservation["cabin"] == "经济舱"
+            else flight["business_price"]
+        )
+
+        # 取消政策及退款计算
+        if reason == "航空公司取消航班":
+            # 航空公司取消航班，全额退款
+            refund_amount = flight_price
+            self.process_refund(user, refund_amount)
+            return f"航班已取消，您的预订将被免费取消，已退款{refund_amount}元。"
+
+        elif time_until_departure > timedelta(days=1):
+            # 离出发时间超过24小时免费取消
+            refund_amount = flight_price
+            self.process_refund(user, refund_amount)
+            return f"距离出发时间超过24小时，免费取消成功，已退款{refund_amount}元。"
+
+        else:
+            # 若不符合免费取消条件，可根据需求设置取消费
+            cancel_fee = flight_price * 0.1  # 假设取消费为票价的10%
+            refund_amount = flight_price - cancel_fee
+            self.process_refund(user, refund_amount)
+            return f"距离出发时间不足24小时，已扣除取消费{cancel_fee}元，退款{refund_amount}元。"
+
+    def process_refund(self, user: dict, amount: float) -> str:
+        """将退款金额添加到用户的现金余额中。
+
+        Args:
+            user (dict): 用户信息字典。
+            amount (float): 退款金额。
+        """
+        user["cash_balance"] += amount
+        return f"已成功处理退款，{user['user_name']}的现金余额增加了{amount}元。"
+
+    def calculate_price_difference(
+        self,
+        flight: dict,
+        old_cabin: str,
+        new_cabin: str,
+    ) -> float:
+        """计算舱位价格差异。
+
+        Args:
+            flight (dict): 航班信息字典。
+            old_cabin (str): 原舱位等级。
+            new_cabin (str): 新舱位等级。
+
+        Returns:
+            float: 价格差异（正数表示需支付差价，负数表示退款）。
+        """
+        cabin_prices = {
+            "经济舱": flight["economy_price"],
+            "商务舱": flight["business_price"],
+        }
+        old_price = cabin_prices.get(old_cabin, 0)
+        new_price = cabin_prices.get(new_cabin, 0)
+        return new_price - old_price

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_ace_benchmark/_ace_tools_zh.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+"""The Chinese tools for ACEBench evaluation."""
+from functools import wraps
+from typing import Callable, Any
+
+from ._ace_tools_api import (
+    ReminderApi,
+    FoodPlatformApi,
+    TravelApi,
+    MessageApi,
+)
+from ...message import TextBlock
+from ...tool import ToolResponse
+
+
+def _tool_function_wrapper(get_tool_function: Callable) -> Callable:
+    """Wrap the tool function result to be ToolResponse."""
+
+    @wraps(get_tool_function)
+    def wrapper(self: "ACEPhone", name: str) -> Callable:
+        """Wrap the tool function to return ToolResponse."""
+        tool_function = get_tool_function(self, name)
+
+        @wraps(tool_function)
+        def wrapper_tool_function(*args: Any, **kwargs: Any) -> ToolResponse:
+            """The wrapped tool function"""
+            res = tool_function(*args, **kwargs)
+            return ToolResponse(
+                content=[
+                    TextBlock(
+                        type="text",
+                        text=str(res),
+                    ),
+                ],
+            )
+
+        return wrapper_tool_function
+
+    return wrapper
+
+
+class ACEPhone:
+    """Simulate a user phone with various apps and functionalities in
+    ACEBench. The code is implemented with reference to the
+    `ACEBench <https://github.com/ACEBench/ACEBench>`_.
+    """
+
+    def __init__(self) -> None:
+        """Initialize the shared state and apps for the ACEPhone."""
+        self._state = {
+            "wifi": False,
+            "logged_in": False,
+        }
+        self._message_app = MessageApi(self._state)
+        self._reminder_app = ReminderApi(self._state)
+        self._food_platform_app = FoodPlatformApi(self._state)
+        self._travel = TravelApi()
+
+    def turn_on_wifi(self) -> dict[str, bool | str]:
+        """开启WiFi连接。"""
+        self._state["wifi"] = True
+        return {"status": True, "message": "wifi已经打开"}
+
+    def login_device(self) -> dict[str, bool | str]:
+        """登录设备。"""
+        self._state["logged_in"] = True
+        return {"status": True, "message": "设备已经登录"}
+
+    def load_initial_config(self, initial_config: dict) -> None:
+        """Load the initial config from the application configuration."""
+        # Empty initial config
+        if len(initial_config) == 0:
+            return
+
+        # Fix the typo in ACEBench by renaming "Baspi" to "BaseApi"
+        if "Baspi" in initial_config:
+            initial_config["BaseApi"] = initial_config.pop("Baspi")
+
+        # Verify state
+        assert (
+            "BaseApi" in initial_config
+            and "wifi" in initial_config["BaseApi"]
+            and "logged_in" in initial_config["BaseApi"]
+        ), f"Invalid initial config: {initial_config}"
+
+        self._state["wifi"] = initial_config["BaseApi"]["wifi"]
+        self._state["logged_in"] = initial_config["BaseApi"]["logged_in"]
+
+    def get_current_state(self) -> list[dict]:
+        """Follow ACEBench to get the current state of the ACEPhone."""
+        return [
+            {"BaseApi": self._state},
+            self._message_app.get_state_dict(),
+            self._reminder_app.get_state_dict(),
+            self._food_platform_app.get_state_dict(),
+            self._travel.get_state_dict(),
+        ]
+
+    @_tool_function_wrapper
+    def get_tool_function(self, name: str) -> Callable:
+        """Get a tool function by name."""
+        if name in [
+            "turn_on_wifi",
+            "login_device",
+        ]:
+            return getattr(self, name)
+
+        if name in self._message_app.tool_functions:
+            return getattr(self._message_app, name)
+
+        if name in self._food_platform_app.tool_functions:
+            return getattr(self._food_platform_app, name)
+
+        if name in self._reminder_app.tool_functions:
+            return getattr(self._reminder_app, name)
+
+        if name in self._travel.tool_functions:
+            return getattr(self._travel, name)
+
+        raise ValueError(
+            f"Tool function '{name}' not found in ACEPhone.",
+        )

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_benchmark_base.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+"""The base class for benchmark evaluation."""
+from abc import ABC, abstractmethod
+from typing import Generator
+
+from ._task import Task
+
+
+class BenchmarkBase(ABC):
+    """The base class for benchmark evaluation."""
+
+    name: str
+    """The name of the benchmark."""
+
+    description: str
+    """The description of the benchmark."""
+
+    def __init__(self, name: str, description: str) -> None:
+        """Initialize the benchmark.
+
+        Args:
+            name (`str`):
+                The name of the benchmark.
+            description (`str`):
+                A brief description of the benchmark.
+        """
+        self.name = name
+        self.description = description
+
+    @abstractmethod
+    def __iter__(self) -> Generator[Task, None, None]:
+        """Iterate over the benchmark."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @abstractmethod
+    def __len__(self) -> int:
+        """Get the length of the benchmark."""
+        raise NotImplementedError("Subclasses must implement this method.")
+
+    @abstractmethod
+    def __getitem__(self, index: int) -> Task:
+        """Get the task at the given index."""
+        raise NotImplementedError("Subclasses must implement this method.")

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_evaluator/__init__.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+"""The evaluator module in AgentScope."""
+
+from ._evaluator_base import EvaluatorBase
+from ._ray_evaluator import RayEvaluator
+from ._general_evaluator import GeneralEvaluator
+
+__all__ = [
+    "EvaluatorBase",
+    "RayEvaluator",
+    "GeneralEvaluator",
+]

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_evaluator/_evaluator_base.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+"""The base class for evaluator in evaluation."""
+import collections
+import json
+from abc import abstractmethod
+from typing import Callable, Coroutine, Any
+
+from .._solution import SolutionOutput
+from .._task import Task
+from .._benchmark_base import BenchmarkBase
+from .._evaluator_storage import EvaluatorStorageBase
+from .._metric_base import MetricType
+from ..._utils._common import _get_timestamp
+
+
+class EvaluatorBase:
+    """The class that runs the evaluation process."""
+
+    def __init__(
+        self,
+        name: str,
+        benchmark: BenchmarkBase,
+        n_repeat: int,
+        storage: EvaluatorStorageBase,
+    ) -> None:
+        """Initialize the evaluator.
+
+        Args:
+            name (`str`):
+                The name of this evaluator.
+            benchmark: (`BenchmarkBase`):
+                A benchmark instance inheriting from `BenchmarkBase` that
+                defines the evaluation dataset.
+            n_repeat (`int`):
+                How many times to repeat the evaluation for each task.
+            storage (`EvaluatorStorageBase`):
+                A instance inheriting from the child class of
+                `EvaluatorStorageBase` that supports storing and loading
+                solution output and evaluation results.
+        """
+        self.name = name
+        self.benchmark = benchmark
+        self.n_repeat = n_repeat
+        self.storage = storage
+
+    @abstractmethod
+    async def run(
+        self,
+        solution: Callable[
+            [Task, Callable],
+            Coroutine[Any, Any, SolutionOutput],
+        ],
+    ) -> None:
+        """Run the evaluation and return the results.
+
+        Args:
+            solution (`Callable[[Task, Callable], Coroutine[Any, Any, \
+            SolutionOutput]]`):
+                A async function that takes a `Task` instance and a pre-hook
+                as input and returns a `SolutionOutput` instance.
+        """
+
+    async def _save_evaluation_meta(self) -> None:
+        """Save the evaluation meta information."""
+        self.storage.save_evaluation_meta(
+            {
+                "evaluation_name": self.name,
+                "created_at": _get_timestamp(),
+                "total_repeats": self.n_repeat,
+                "benchmark": {
+                    "name": self.benchmark.name,
+                    "description": self.benchmark.description,
+                    "total_tasks": len(self.benchmark),
+                },
+                "schema_version": 1,
+            },
+        )
+
+    async def aggregate(self) -> None:  # pylint: disable=too-many-branches
+        """Aggregate the evaluation results and save an overall result."""
+        meta_info: dict = {
+            "total_tasks": len(self.benchmark),
+            "total_repeats": self.n_repeat,
+            "repeats": {},
+            "schema_version": 1,
+        }
+
+        for repeat_index in range(self.n_repeat):
+            repeat_id = str(repeat_index)
+            current_repeat: dict = {
+                "completed_tasks": 0,
+                "incomplete_tasks": 0,
+                "metrics": {},
+                "completed_ids": [],
+                "incomplete_ids": [],
+            }
+            for task in self.benchmark:
+                for metric in task.metrics:
+                    # Create a new dict in aggregated_result
+                    if metric.name not in current_repeat["metrics"]:
+                        current_repeat["metrics"][metric.name] = {
+                            "type": metric.metric_type,
+                            "involved_tasks": 0,
+                            "completed_tasks": 0,
+                            "incomplete_tasks": 0,
+                            "aggregation": {},
+                            "distribution": collections.defaultdict(list),
+                        }
+
+                    # Record the submitted task
+                    current_repeat["metrics"][metric.name][
+                        "involved_tasks"
+                    ] += 1
+
+                    # Not finished
+                    if not self.storage.evaluation_result_exists(
+                        task.id,
+                        repeat_id,
+                        metric.name,
+                    ):
+                        if task.id not in current_repeat["incomplete_ids"]:
+                            current_repeat["incomplete_tasks"] += 1
+                            current_repeat["incomplete_ids"].append(task.id)
+                        current_repeat["metrics"][metric.name][
+                            "incomplete_tasks"
+                        ] += 1
+                        continue
+
+                    if task.id not in current_repeat["completed_ids"]:
+                        current_repeat["completed_tasks"] += 1
+                        current_repeat["completed_ids"].append(task.id)
+                    current_repeat["metrics"][metric.name][
+                        "completed_tasks"
+                    ] += 1
+
+                    # Get the evaluation result
+                    eval_result = self.storage.get_evaluation_result(
+                        task.id,
+                        repeat_id,
+                        metric.name,
+                    )
+
+                    # Record the metric result
+                    if metric.metric_type == MetricType.CATEGORY:
+                        current_repeat["metrics"][metric.name]["distribution"][
+                            eval_result.result
+                        ].append(
+                            task.id,
+                        )
+
+                    elif metric.metric_type == MetricType.NUMERICAL:
+                        current_repeat["metrics"][metric.name]["distribution"][
+                            task.id
+                        ] = eval_result.result
+
+            print("Repeat ID:", repeat_id)
+
+            for metric, value in current_repeat["metrics"].items():
+                print("\tMetric:", metric)
+                print("\t\tType:", value["type"])
+                print("\t\tInvolved tasks:", value["involved_tasks"])
+                print("\t\tCompleted tasks:", value["completed_tasks"])
+                print("\t\tIncomplete tasks:", value["incomplete_tasks"])
+
+                if value["type"] == MetricType.CATEGORY:
+                    # Count the distribution
+                    for category, task_ids in value["distribution"].items():
+                        value["aggregation"][category] = (
+                            len(task_ids) * 1.0 / value["involved_tasks"]
+                        )
+
+                elif value["type"] == MetricType.NUMERICAL:
+                    scores = list(value["distribution"].values())
+                    value["aggregation"] = {
+                        "mean": sum(scores) / value["involved_tasks"],
+                        "max": max(scores),
+                        "min": min(scores),
+                    }
+
+                print(
+                    "\t\tAggregation:",
+                    json.dumps(
+                        value["aggregation"],
+                        indent=4,
+                        ensure_ascii=False,
+                    ).replace("\n", "\n\t\t"),
+                )
+
+            meta_info["repeats"][repeat_id] = current_repeat
+
+        # save
+        self.storage.save_aggregation_result(meta_info)

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_evaluator/_general_evaluator.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+"""General evaluator implementation in AgentScope, which is easy to debug
+compared to the RayEvaluator."""
+from typing import Callable, Awaitable, Coroutine, Any
+
+from ._evaluator_base import EvaluatorBase
+from .._evaluator_storage import EvaluatorStorageBase
+from .._task import Task
+from .._solution import SolutionOutput
+from .._benchmark_base import BenchmarkBase
+
+
+class GeneralEvaluator(EvaluatorBase):
+    """The general evaluator that support users to debug their evaluation"""
+
+    def __init__(
+        self,
+        name: str,
+        benchmark: BenchmarkBase,
+        n_repeat: int,
+        storage: EvaluatorStorageBase,
+        n_workers: int,
+    ) -> None:
+        """Initialize the evaluator."""
+        super().__init__(
+            name=name,
+            benchmark=benchmark,
+            n_repeat=n_repeat,
+            storage=storage,
+        )
+
+        assert isinstance(benchmark, BenchmarkBase)
+
+        assert n_repeat >= 1, "n_repeat must be at least 1"
+
+        assert n_workers >= 1, "n_workers must be at least 1"
+
+        self.benchmark = benchmark
+        self.n_repeat = n_repeat
+        self.n_workers = n_workers
+
+    async def run_evaluation(
+        self,
+        task: Task,
+        repeat_id: str,
+        solution_output: SolutionOutput,
+    ) -> None:
+        """Run the evaluation for a task and solution result."""
+        evaluation_results = await task.evaluate(solution_output)
+        # store the evaluation result
+        for result in evaluation_results:
+            self.storage.save_evaluation_result(
+                task_id=task.id,
+                repeat_id=repeat_id,
+                evaluation=result,
+            )
+
+    async def run_solution(
+        self,
+        repeat_id: str,
+        task: Task,
+        solution: Callable[[Task, Callable], Awaitable[SolutionOutput]],
+    ) -> None:
+        """Generate a solution to a task and evaluate."""
+        if self.storage.solution_result_exists(task.id, repeat_id):
+            # Obtain from storage
+            solution_result = self.storage.get_solution_result(
+                task.id,
+                repeat_id,
+            )
+
+        else:
+            # Run the solution
+            solution_result = await solution(
+                task,
+                self.storage.get_agent_pre_print_hook(
+                    task.id,
+                    repeat_id,
+                ),
+            )
+            self.storage.save_solution_result(
+                task.id,
+                repeat_id,
+                solution_result,
+            )
+
+        # Evaluate the solution with the
+        for metric in task.metrics:
+            if not self.storage.evaluation_result_exists(
+                task.id,
+                repeat_id,
+                metric.name,
+            ):
+                await self.run_evaluation(
+                    task,
+                    repeat_id,
+                    solution_result,
+                )
+
+    async def run(
+        self,
+        solution: Callable[
+            [Task, Callable],
+            Coroutine[Any, Any, SolutionOutput],
+        ],
+    ) -> None:
+        """Run the ray-based distributed and parallel evaluation, and get the
+        results.
+
+        Args:
+            solution (`Callable[[Task, Callable], Coroutine[Any, Any, \
+            SolutionOutput]]`):
+                A async function that takes a `Task` instance and a pre-print
+                hook function as input, returns a `SolutionOutput` instance.
+        """
+
+        await self._save_evaluation_meta()
+
+        for repeat_id in range(self.n_repeat):
+            for task in self.benchmark:
+                await self.run_solution(
+                    str(repeat_id),
+                    task,
+                    solution,
+                )
+
+        await self.aggregate()

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_evaluator/_ray_evaluator.py

@@ -0,0 +1,211 @@
+# -*- coding: utf-8 -*-
+"""The evaluator base class in agentscope."""
+import asyncio
+from typing import Callable, Awaitable, Coroutine, Any
+
+from .._benchmark_base import BenchmarkBase
+from .._evaluator._evaluator_base import EvaluatorBase
+from .._solution import SolutionOutput
+from .._task import Task
+from .._evaluator_storage import EvaluatorStorageBase
+
+
+def _check_ray_available() -> None:
+    """Check if ray is available and raise ImportError if not."""
+    try:
+        import ray  # noqa  # pylint: disable=unused-import
+    except ImportError as e:
+        raise ImportError(
+            "Ray is not installed. Please install it with `pip install ray` "
+            "to use the RayEvaluator.",
+        ) from e
+
+
+# Create a conditional decorator for ray.remote
+def _ray_remote_decorator(cls: Any) -> Any:
+    """
+    Conditional ray.remote decorator that only applies when ray is available.
+    """
+    try:
+        import ray
+
+        return ray.remote(cls)
+    except ImportError:
+        return cls
+
+
+@_ray_remote_decorator
+class RayEvaluationActor:
+    """
+    Actor class for running evaluation with ray remote.
+    """
+
+    @staticmethod
+    async def run(
+        storage: EvaluatorStorageBase,
+        task: Task,
+        repeat_id: str,
+        solution_output: SolutionOutput,
+    ) -> None:
+        """
+        Run the evaluation for a task and solution result.
+
+        Args:
+            storage (EvaluatorStorageBase): Evaluator storage.
+            task (Task): Task to be evaluated.
+            repeat_id (str): Repeat ID
+            solution_output (SolutionOutput): output data after execute agents.
+        """
+        evaluation_results = await task.evaluate(solution_output)
+        # store the evaluation result
+        for result in evaluation_results:
+            storage.save_evaluation_result(
+                task_id=task.id,
+                repeat_id=repeat_id,
+                evaluation=result,
+            )
+
+
+@_ray_remote_decorator
+class RaySolutionActor:
+    """
+    Actor class for running agent solutions with ray remote.
+    """
+
+    def __init__(self, n_workers: int = 1):
+        self.eval_actor = RayEvaluationActor.options(
+            max_concurrency=n_workers,
+        ).remote()
+
+    async def run(
+        self,
+        storage: EvaluatorStorageBase,
+        repeat_id: str,
+        task: Task,
+        solution: Callable[
+            [Task, Callable],
+            Coroutine[Any, Any, SolutionOutput],
+        ],
+    ) -> None:
+        """Generate a solution to a task and evaluate.
+
+        Args:
+            storage (EvaluatorStorageBase): Evaluator storage.
+            repeat_id (str): Repeat ID.
+            task (Task): Task to be evaluated.
+            solution
+                (Callable[[Task, Callable], Awaitable[SolutionOutput, Any]]):
+                callable function to execute agents and generate results.
+        """
+        if storage.solution_result_exists(task.id, repeat_id):
+            # Obtain from storage
+            solution_result = storage.get_solution_result(
+                task.id,
+                repeat_id,
+            )
+
+        else:
+            # Run the solution
+            solution_result = await solution(
+                task,
+                storage.get_agent_pre_print_hook(
+                    task.id,
+                    repeat_id,
+                ),
+            )
+
+            storage.save_solution_result(
+                task.id,
+                repeat_id,
+                solution_result,
+            )
+
+        # Evaluate the solution with the
+        futures = []
+        for metric in task.metrics:
+            if not storage.evaluation_result_exists(
+                task.id,
+                repeat_id,
+                metric.name,
+            ):
+                futures.append(
+                    self.eval_actor.run.remote(
+                        storage,
+                        task,
+                        repeat_id,
+                        solution_result,
+                    ),
+                )
+        if futures:
+            await asyncio.gather(*futures)
+
+
+class RayEvaluator(EvaluatorBase):
+    """The ray-based evaluator that supports distributed and parallel
+    evaluation."""
+
+    def __init__(
+        self,
+        name: str,
+        benchmark: BenchmarkBase,
+        n_repeat: int,
+        storage: EvaluatorStorageBase,
+        n_workers: int,
+    ) -> None:
+        """Initialize the evaluator."""
+        super().__init__(
+            name=name,
+            benchmark=benchmark,
+            n_repeat=n_repeat,
+            storage=storage,
+        )
+
+        # Check ray availability early
+        _check_ray_available()
+
+        assert isinstance(benchmark, BenchmarkBase)
+
+        assert n_repeat >= 1, "n_repeat must be at least 1"
+
+        assert n_workers >= 1, "n_workers must be at least 1"
+
+        self.benchmark = benchmark
+        self.n_repeat = n_repeat
+        self.n_workers = n_workers
+
+    async def run(
+        self,
+        solution: Callable[
+            [Task, Callable],
+            Awaitable[SolutionOutput] | SolutionOutput,
+        ],
+    ) -> None:
+        """Run the ray-based distributed and parallel evaluation, and get the
+        results.
+
+        Args:
+            solution (`Callable[[Task], SolutionOutput]`):
+                A sync or async function that takes a `Task` instance as input
+                and returns a `SolutionOutput` instance.
+        """
+
+        await self._save_evaluation_meta()
+
+        futures = []
+        solution_actor = RaySolutionActor.options(
+            max_concurrency=self.n_workers,
+        ).remote(n_workers=self.n_workers)
+        for repeat_id in range(self.n_repeat):
+            for task in self.benchmark:
+                futures.append(
+                    solution_actor.run.remote(
+                        self.storage,
+                        str(repeat_id),
+                        task,
+                        solution,
+                    ),
+                )
+        if futures:
+            await asyncio.gather(*futures)
+
+        await self.aggregate()

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_evaluator_storage/__init__.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+"""The evaluator storage module in AgentScope."""
+
+from ._evaluator_storage_base import EvaluatorStorageBase
+from ._file_evaluator_storage import FileEvaluatorStorage
+
+__all__ = [
+    "EvaluatorStorageBase",
+    "FileEvaluatorStorage",
+]

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_evaluator_storage/_evaluator_storage_base.py

@@ -0,0 +1,195 @@
+# -*- coding: utf-8 -*-
+"""The evaluator storage base class for storing solution and evaluation
+results."""
+from abc import abstractmethod
+from typing import Any, Callable
+
+from .._metric_base import MetricResult
+from .._solution import SolutionOutput
+from ...agent import AgentBase
+
+
+class EvaluatorStorageBase:
+    """Used to store the solution results and evaluation results to support
+    resuming the evaluation process"""
+
+    @abstractmethod
+    def save_solution_result(
+        self,
+        task_id: str,
+        repeat_id: str,
+        output: SolutionOutput,
+        **kwargs: Any,
+    ) -> None:
+        """Save the solution result.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+            output (`SolutionOutput`):
+                The solution output to be saved.
+        """
+
+    @abstractmethod
+    def get_evaluation_result(
+        self,
+        task_id: str,
+        repeat_id: str,
+        metric_name: str,
+    ) -> MetricResult:
+        """Get the evaluation result by the given task id and repeat id
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+            metric_name (`str`):
+                The metric name.
+
+        Returns:
+            `MetricResult`:
+                The evaluation result for the given task and repeat ID.
+        """
+
+    @abstractmethod
+    def save_evaluation_result(
+        self,
+        task_id: str,
+        repeat_id: str,
+        evaluation: MetricResult,
+        **kwargs: Any,
+    ) -> None:
+        """Save the evaluation result.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+            evaluation (`MetricResult`):
+                The evaluation result to be saved.
+        """
+
+    @abstractmethod
+    def get_solution_result(
+        self,
+        task_id: str,
+        repeat_id: str,
+        **kwargs: Any,
+    ) -> SolutionOutput:
+        """Get the solution result for the given task and repeat id.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+
+        Returns:
+            `SolutionOutput`:
+                The solution output for the given task and repeat ID.
+        """
+
+    @abstractmethod
+    def solution_result_exists(self, task_id: str, repeat_id: str) -> bool:
+        """Check if the solution for the given task and repeat is finished.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+
+        Returns:
+            `bool`:
+                True if the solution result file exists, False otherwise.
+        """
+
+    @abstractmethod
+    def evaluation_result_exists(
+        self,
+        task_id: str,
+        repeat_id: str,
+        metric_name: str,
+    ) -> bool:
+        """Check if the evaluation result for the given solution and metric
+        is finished.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+            metric_name (`str`):
+                The name of the metric.
+
+        Returns:
+            `bool`:
+                True if the evaluation result file exists, False otherwise.
+        """
+
+    @abstractmethod
+    def save_aggregation_result(
+        self,
+        aggregation_result: dict,
+        **kwargs: Any,
+    ) -> None:
+        """Save the aggregation result.
+
+        Args:
+            aggregation_result (`dict`):
+                A dictionary containing the aggregation result.
+        """
+
+    @abstractmethod
+    def aggregation_result_exists(
+        self,
+        **kwargs: Any,
+    ) -> bool:
+        """Check if the aggregation result exists
+
+        Returns:
+            `bool`:
+                `True` if the aggregation result file exists.
+        """
+
+    @abstractmethod
+    def save_evaluation_meta(self, meta_info: dict) -> None:
+        """Save the evaluation meta information.
+
+        Args:
+            meta_info (`dict`):
+                A dictionary containing the meta information.
+        """
+
+    @abstractmethod
+    def get_agent_pre_print_hook(
+        self,
+        task_id: str,
+        repeat_id: str,
+    ) -> Callable[[AgentBase, dict], None]:
+        """Get a pre-print hook function for the agent to save the agent
+        printing in the evaluation storage.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+
+        Returns:
+            `Callable[[AgentBase, dict], None]`:
+                A hook function that takes an `AgentBase` instance and a
+                keyword arguments dictionary as input, saving the agent's
+                printing Msg into the evaluation storage.
+        """

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_evaluator_storage/_file_evaluator_storage.py

@@ -0,0 +1,345 @@
+# -*- coding: utf-8 -*-
+"""A file system based evaluator storage."""
+import json
+import os
+from json import JSONDecodeError
+from typing import Any, Callable
+
+from ._evaluator_storage_base import EvaluatorStorageBase
+from .._solution import SolutionOutput
+from .._metric_base import MetricResult
+from ...agent import AgentBase
+from ...message import Msg
+
+
+class FileEvaluatorStorage(EvaluatorStorageBase):
+    """File system based evaluator storage, providing methods to save and
+    retrieve evaluation results. So that the evaluation process can be resumed
+    from the last saved state.
+
+    The files are organized in a directory structure:
+    - save_dir/
+        - evaluation_result.json
+        - evaluation_meta.json
+        - {task_id}/
+            - {repeat_id}/
+                - solution.json
+                - evaluation/
+                    - {metric_name}.json
+    """
+
+    SOLUTION_FILE_NAME = "solution.json"
+    EVALUATION_DIR_NAME = "evaluation"
+    EVALUATION_RESULT_FILE = "evaluation_result.json"
+    EVALUATION_META_FILE = "evaluation_meta.json"
+    AGENT_PRINTING_LOG = "logging.txt"
+
+    def __init__(self, save_dir: str) -> None:
+        """Initialize the file evaluator storage."""
+        self.save_dir = save_dir
+
+    def _get_save_path(self, task_id: str, repeat_id: str, *args: str) -> str:
+        """Get the save path for a given task and repeat ID."""
+        return os.path.join(self.save_dir, repeat_id, task_id, *args)
+
+    def save_solution_result(
+        self,
+        task_id: str,
+        repeat_id: str,
+        output: SolutionOutput,
+        **kwargs: Any,
+    ) -> None:
+        """Save the solution result.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+            output (`SolutionOutput`):
+                The solution output to be saved.
+        """
+        path_file = self._get_save_path(
+            task_id,
+            repeat_id,
+            self.SOLUTION_FILE_NAME,
+        )
+        os.makedirs(os.path.dirname(path_file), exist_ok=True)
+        with open(path_file, "w", encoding="utf-8") as f:
+            json.dump(output, f, ensure_ascii=False, indent=4)
+
+    def save_evaluation_result(
+        self,
+        task_id: str,
+        repeat_id: str,
+        evaluation: MetricResult,
+        **kwargs: Any,
+    ) -> None:
+        """Save the evaluation result.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+            evaluation (`MetricResult`):
+                The evaluation result to be saved.
+        """
+        path_file = self._get_save_path(
+            task_id,
+            repeat_id,
+            self.EVALUATION_DIR_NAME,
+            f"{evaluation.name}.json",
+        )
+        os.makedirs(os.path.dirname(path_file), exist_ok=True)
+        with open(path_file, "w", encoding="utf-8") as f:
+            json.dump(evaluation, f, ensure_ascii=False, indent=4)
+
+    def get_evaluation_result(
+        self,
+        task_id: str,
+        repeat_id: str,
+        metric_name: str,
+    ) -> MetricResult:
+        """Get the evaluation result by the given task id and repeat id
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+            metric_name (`str`):
+                The metric name.
+
+        Returns:
+            `MetricResult`:
+                The evaluation result for the given task and repeat ID.
+        """
+        path_file = self._get_save_path(
+            task_id,
+            repeat_id,
+            self.EVALUATION_DIR_NAME,
+            f"{metric_name}.json",
+        )
+        if not os.path.exists(path_file):
+            raise FileNotFoundError(path_file)
+        with open(path_file, "r", encoding="utf-8") as f:
+            evaluation = json.load(f)
+        return MetricResult(**evaluation)
+
+    def get_solution_result(
+        self,
+        task_id: str,
+        repeat_id: str,
+        **kwargs: Any,
+    ) -> SolutionOutput:
+        """Get the solution result for the given task and repeat id from the
+        file system.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+
+        Raises:
+            `FileNotFoundError`:
+                If the solution result file does not exist for the given task
+                and repeat ID.
+
+        Returns:
+            `SolutionOutput`:
+                The solution output for the given task and repeat ID.
+        """
+        path_file = self._get_save_path(
+            task_id,
+            repeat_id,
+            self.SOLUTION_FILE_NAME,
+        )
+        if not os.path.exists(path_file):
+            raise FileNotFoundError(
+                f"Solution result for task {task_id} and repeat {repeat_id} "
+                "not found.",
+            )
+
+        try:
+            with open(path_file, "r", encoding="utf-8") as f:
+                solution_data = json.load(f)
+        except JSONDecodeError as e:
+            raise JSONDecodeError(
+                f"Failed to load JSON from {path_file}: {e.msg}",
+                e.doc,
+                e.pos,
+            ) from e
+
+        return SolutionOutput(**solution_data)
+
+    def solution_result_exists(self, task_id: str, repeat_id: str) -> bool:
+        """Check if the solution for the given task and repeat is finished.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+
+        Returns:
+            `bool`:
+                True if the solution result file exists, False otherwise.
+        """
+        path_file = self._get_save_path(
+            task_id,
+            repeat_id,
+            self.SOLUTION_FILE_NAME,
+        )
+
+        return os.path.exists(path_file) and os.path.getsize(path_file) > 0
+
+    def evaluation_result_exists(
+        self,
+        task_id: str,
+        repeat_id: str,
+        metric_name: str,
+    ) -> bool:
+        """Check if the evaluation result for the given solution and metric
+        is finished.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+            metric_name (`str`):
+                The name of the metric.
+
+        Returns:
+            `bool`:
+                True if the evaluation result file exists, False otherwise.
+        """
+        path_file = self._get_save_path(
+            task_id,
+            repeat_id,
+            self.EVALUATION_DIR_NAME,
+            f"{metric_name}.json",
+        )
+        return os.path.exists(path_file) and os.path.getsize(path_file) > 0
+
+    def save_aggregation_result(
+        self,
+        aggregation_result: dict,
+        **kwargs: Any,
+    ) -> None:
+        """Save the aggregation result.
+
+        Args:
+            aggregation_result (`dict`):
+                A dictionary containing the aggregation result.
+        """
+        path_file = os.path.join(
+            self.save_dir,
+            self.EVALUATION_RESULT_FILE,
+        )
+        os.makedirs(os.path.dirname(path_file), exist_ok=True)
+        with open(path_file, "w", encoding="utf-8") as f:
+            json.dump(aggregation_result, f, ensure_ascii=False, indent=4)
+
+    def aggregation_result_exists(
+        self,
+        **kwargs: Any,
+    ) -> bool:
+        """Check if the aggregation result exists
+
+        Returns:
+            `bool`:
+                `True` if the aggregation result file exists.
+        """
+        path_file = os.path.join(
+            self.save_dir,
+            self.EVALUATION_RESULT_FILE,
+        )
+        return os.path.exists(path_file) and os.path.getsize(path_file) > 0
+
+    def save_evaluation_meta(self, meta_info: dict) -> None:
+        """Save the evaluation meta information.
+
+        Args:
+            meta_info (`dict`):
+                A dictionary containing the meta information.
+        """
+        path_file = os.path.join(
+            self.save_dir,
+            self.EVALUATION_META_FILE,
+        )
+        os.makedirs(os.path.dirname(path_file), exist_ok=True)
+        with open(path_file, "w", encoding="utf-8") as f:
+            json.dump(meta_info, f, ensure_ascii=False, indent=4)
+
+    def get_agent_pre_print_hook(
+        self,
+        task_id: str,
+        repeat_id: str,
+    ) -> Callable[[AgentBase, dict], None]:
+        """Get a pre-print hook function for the agent to save the agent
+        printing in the evaluation storage.
+
+        Args:
+            task_id (`str`):
+                The task ID.
+            repeat_id (`str`):
+                The repeat ID for the task, usually the index of the repeat
+                evaluation.
+
+        Returns:
+            `Callable[[AgentBase, dict], None]`:
+                A hook function that takes an `AgentBase` instance and a
+                keyword arguments dictionary as input, saving the agent's
+                printing Msg into the evaluation storage.
+        """
+
+        def pre_print_hook(_agent: AgentBase, kwargs: dict) -> None:
+            """Hook function to save agent's printing."""
+            msg: Msg | None = kwargs.get("msg", None)
+            last: bool = kwargs.get("last", False)
+
+            if msg is None or not last:
+                return
+
+            # Only save the last message
+            printing_str = []
+            for block in msg.get_content_blocks():
+                match block["type"]:
+                    case "text":
+                        printing_str.append(
+                            f"{msg.name}: {block['text']}",
+                        )
+                    case "thinking":
+                        printing_str.append(
+                            f"{msg.name} (thinking): {block['text']}",
+                        )
+                    case _:
+                        block_str = json.dumps(
+                            block,
+                            ensure_ascii=False,
+                            indent=4,
+                        )
+                        if printing_str:
+                            printing_str.append(block_str)
+                        else:
+                            printing_str.append(f"{msg.name}: {block_str}")
+
+            path_file = self._get_save_path(
+                task_id,
+                repeat_id,
+                self.AGENT_PRINTING_LOG,
+            )
+            os.makedirs(os.path.dirname(path_file), exist_ok=True)
+            with open(path_file, "a", encoding="utf-8") as f:
+                f.write("\n".join(printing_str) + "\n")
+
+        return pre_print_hook

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_metric_base.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+"""The base class for _metric in evaluation."""
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from enum import Enum
+from typing import Any
+
+from .._utils._common import _get_timestamp
+from .._utils._mixin import DictMixin
+from ..types import JSONSerializableObject
+
+
+@dataclass
+class MetricResult(DictMixin):
+    """The result of a _metric."""
+
+    name: str
+    """The metric name."""
+
+    result: str | float | int
+    """The metric result."""
+
+    created_at: str = field(default_factory=_get_timestamp)
+    """The timestamp when the metric result was created."""
+
+    message: str | None = field(default_factory=lambda: None)
+    """An optional message for the metric result, can be used to provide
+    additional information or context about the result."""
+
+    metadata: dict[str, JSONSerializableObject] | None = field(default=None)
+    """Optional metadata for the metric result, can be used to store
+    additional information related to the metric result."""
+
+
+class MetricType(str, Enum):
+    """The metric type enum."""
+
+    CATEGORY = "category"
+    """The metric result is a category, e.g. "pass" or "fail"."""
+
+    NUMERICAL = "numerical"
+    """The metric result is a numerical value, e.g. 0.95 or 100."""
+
+
+class MetricBase(ABC):
+    """The base class for _metric in evaluation."""
+
+    def __init__(
+        self,
+        name: str,
+        metric_type: MetricType,
+        description: str | None = None,
+        categories: list[str] | None = None,
+    ) -> None:
+        """Initialize the _metric object.
+
+        Args:
+            name (`str`):
+                The name of the metric.
+            metric_type (`MetricType`):
+                The type of the metric, can be either "category" or
+                "numerical", which will determine how to display the result.
+            description (`str`):
+                The description of the metric.
+            categories (`list[str] | None`, optional):
+                The candidate categories. If `metric_type` is "category", the
+                categories must be provided, otherwise it should be `None`.
+        """
+        self.name = name
+        self.metric_type = metric_type
+        self.description = description
+
+        if metric_type == MetricType.CATEGORY and categories is None:
+            raise ValueError(
+                "Categories must be provided for category metrics.",
+            )
+
+        self.categories = categories
+
+    @abstractmethod
+    async def __call__(
+        self,
+        *args: Any,
+        **kwargs: Any,
+    ) -> MetricResult:
+        """The call function to calculate the _metric result"""

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_solution.py

@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+"""Solution class for evaluation tasks."""
+
+from dataclasses import dataclass, field
+from typing import Any
+
+from ..message import (
+    ToolResultBlock,
+    ToolUseBlock,
+    TextBlock,
+)
+from ..types._json import JSONSerializableObject
+from .._utils._mixin import DictMixin
+
+
+@dataclass
+class SolutionOutput(DictMixin):
+    """The output of a solution in evaluation task"""
+
+    success: bool
+    """Indicates whether the solution is executed successfully. When the
+    solution raise exception, this should be set to False."""
+    output: JSONSerializableObject
+    """The final output of the solution."""
+    trajectory: list[ToolUseBlock | ToolResultBlock | TextBlock]
+    """The tool calls and results trajectory"""
+    meta: dict[str, Any] | None = field(default_factory=lambda: None)
+    """Additional metadata for the solution"""
+
+    def __getstate__(self) -> dict[str, Any]:
+        """Custom pickling to handle dataclass + DictMixin inheritance."""
+        return self.__dict__.copy()
+
+    def __setstate__(self, state: dict[str, Any]) -> None:
+        """Custom unpickling to handle dataclass + DictMixin inheritance."""
+        self.__dict__.update(state)

backend_service/venv/lib/python3.13/site-packages/agentscope/evaluate/_task.py

@@ -0,0 +1,53 @@
+# -*- coding: utf-8 -*-
+"""The base class for task in evaluation."""
+from dataclasses import dataclass, field
+from typing import Any
+
+from ._solution import SolutionOutput
+from ._metric_base import MetricBase, MetricResult
+from ..types._json import JSONSerializableObject
+
+
+@dataclass
+class Task:
+    """The base class for task in evaluation."""
+
+    id: str
+    """The unique identifier for the task."""
+
+    input: JSONSerializableObject
+    """The task input, which should be a JSON serializable object."""
+
+    ground_truth: JSONSerializableObject
+    """The task ground truth if exists, which should be a JSON serializable
+    object."""
+
+    metrics: list[MetricBase]
+    """The metrics to evaluate the task, which should be a list of
+    `MetricBase` objects."""
+
+    tags: dict[str, str] | None = field(default_factory=lambda: None)
+    """Tags to categorize the task, e.g. `{"difficulty": "easy",
+    "cate": "math"}`."""
+
+    metadata: dict[str, Any] | None = field(
+        default_factory=lambda: None,
+    )
+    """Additional metadata for the task."""
+
+    async def evaluate(self, solution: SolutionOutput) -> list[MetricResult]:
+        """Evaluate the task with the given solution.
+
+        Args:
+            solution (`SolutionOutput`):
+                The solution to evaluate the task with.
+
+        Returns:
+            `MetricResult`:
+                The result of the evaluation.
+        """
+        evaluations = []
+        for metric in self.metrics:
+            result = await metric(solution)
+            evaluations.append(result)
+        return evaluations