chore: 添加虚拟环境到仓库
- 添加 backend_service/venv 虚拟环境 - 包含所有Python依赖包 - 注意:虚拟环境约393MB,包含12655个文件
This commit is contained in:
@@ -0,0 +1,45 @@
|
||||
# Copyright (c) Alibaba, Inc. and its affiliates.
|
||||
|
||||
import os
|
||||
from typing import List
|
||||
|
||||
from dashscope.common.error import UnsupportedModel
|
||||
from dashscope.tokenizers.qwen_tokenizer import QwenTokenizer
|
||||
|
||||
from .tokenizer_base import Tokenizer
|
||||
|
||||
QWEN_SERIALS = ['qwen-7b-chat', 'qwen-turbo', 'qwen-plus', 'qwen-max']
|
||||
current_path = os.path.dirname(os.path.abspath(__file__))
|
||||
root_path = os.path.dirname(current_path)
|
||||
|
||||
|
||||
def get_tokenizer(model: str) -> Tokenizer:
|
||||
"""Get a tokenizer based on model name.
|
||||
|
||||
Args:
|
||||
model (str): The model name.
|
||||
|
||||
Raises:
|
||||
UnsupportedModel: Not support model
|
||||
|
||||
Returns:
|
||||
Tokenizer: The `Tokenizer` of the model.
|
||||
"""
|
||||
if model in QWEN_SERIALS:
|
||||
return QwenTokenizer(
|
||||
os.path.join(root_path, 'resources', 'qwen.tiktoken'))
|
||||
elif model.startswith('qwen'):
|
||||
return QwenTokenizer(
|
||||
os.path.join(root_path, 'resources', 'qwen.tiktoken'))
|
||||
else:
|
||||
raise UnsupportedModel(
|
||||
f'Not support model: {model}, currently only support qwen models.')
|
||||
|
||||
|
||||
def list_tokenizers() -> List[str]:
|
||||
"""List support models
|
||||
|
||||
Returns:
|
||||
List[str]: The model list.
|
||||
"""
|
||||
return QWEN_SERIALS
|
||||
Reference in New Issue
Block a user