chore: 添加虚拟环境到仓库

- 添加 backend_service/venv 虚拟环境
- 包含所有Python依赖包
- 注意:虚拟环境约393MB,包含12655个文件
This commit is contained in:
2025-12-03 10:19:25 +08:00
parent a6c2027caa
commit c4f851d387
12655 changed files with 3009376 additions and 0 deletions

View File

@@ -0,0 +1,45 @@
# Copyright (c) Alibaba, Inc. and its affiliates.
import os
from typing import List
from dashscope.common.error import UnsupportedModel
from dashscope.tokenizers.qwen_tokenizer import QwenTokenizer
from .tokenizer_base import Tokenizer
QWEN_SERIALS = ['qwen-7b-chat', 'qwen-turbo', 'qwen-plus', 'qwen-max']
current_path = os.path.dirname(os.path.abspath(__file__))
root_path = os.path.dirname(current_path)
def get_tokenizer(model: str) -> Tokenizer:
"""Get a tokenizer based on model name.
Args:
model (str): The model name.
Raises:
UnsupportedModel: Not support model
Returns:
Tokenizer: The `Tokenizer` of the model.
"""
if model in QWEN_SERIALS:
return QwenTokenizer(
os.path.join(root_path, 'resources', 'qwen.tiktoken'))
elif model.startswith('qwen'):
return QwenTokenizer(
os.path.join(root_path, 'resources', 'qwen.tiktoken'))
else:
raise UnsupportedModel(
f'Not support model: {model}, currently only support qwen models.')
def list_tokenizers() -> List[str]:
"""List support models
Returns:
List[str]: The model list.
"""
return QWEN_SERIALS