diff --git a/frontend/src/database/models/user.ts b/frontend/src/database/models/user.ts index 6de2e6af..f9ed94d1 100644 --- a/frontend/src/database/models/user.ts +++ b/frontend/src/database/models/user.ts @@ -18,7 +18,7 @@ class _UserModel extends BaseModel { const list = (await this.table.toArray()) as (DB_User & { id: number })[]; - return list[0]; + returnList[0]; }; create = async (user: DB_User) => { diff --git a/model-providers/model_providers.yaml b/model-providers/model_providers.yaml index d88736b3..908883c7 100644 --- a/model-providers/model_providers.yaml +++ b/model-providers/model_providers.yaml @@ -27,3 +27,7 @@ xinference: model_uid: 'chatglm3-6b' +zhipuai: + + provider_credential: + api_key: 'd4fa0690b6dfa205204cae2e12aa6fb6.1' \ No newline at end of file diff --git a/model-providers/model_providers/__main__.py b/model-providers/model_providers/__main__.py index ecde3e4a..3f9918fe 100644 --- a/model-providers/model_providers/__main__.py +++ b/model-providers/model_providers/__main__.py @@ -16,7 +16,7 @@ if __name__ == "__main__": parser.add_argument( "--model-providers", type=str, - default="D:\\project\\Langchain-Chatchat\\model-providers\\model_providers.yaml", + default="/mnt/d/project/Langchain-Chatchat/model-providers/model_providers.yaml", help="run model_providers servers", dest="model_providers", ) diff --git a/model-providers/model_providers/bootstrap_web/entities/model_provider_entities.py b/model-providers/model_providers/bootstrap_web/entities/model_provider_entities.py index e7899a0c..77819c8a 100644 --- a/model-providers/model_providers/bootstrap_web/entities/model_provider_entities.py +++ b/model-providers/model_providers/bootstrap_web/entities/model_provider_entities.py @@ -50,7 +50,7 @@ class SystemConfigurationResponse(BaseModel): enabled: bool current_quota_type: Optional[ProviderQuotaType] = None - quota_configurations: list[QuotaConfiguration] = [] + quota_configurations: List[QuotaConfiguration] = [] class ProviderResponse(BaseModel): @@ -65,8 +65,8 @@ class ProviderResponse(BaseModel): icon_large: Optional[I18nObject] = None background: Optional[str] = None help: Optional[ProviderHelpEntity] = None - supported_model_types: list[ModelType] - configurate_methods: list[ConfigurateMethod] + supported_model_types: List[ModelType] + configurate_methods: List[ConfigurateMethod] provider_credential_schema: Optional[ProviderCredentialSchema] = None model_credential_schema: Optional[ModelCredentialSchema] = None preferred_provider_type: ProviderType @@ -114,7 +114,7 @@ class ProviderWithModelsResponse(BaseModel): icon_small: Optional[I18nObject] = None icon_large: Optional[I18nObject] = None status: CustomConfigurationStatus - models: list[ModelResponse] + models: List[ModelResponse] def __init__(self, **data) -> None: super().__init__(**data) diff --git a/model-providers/model_providers/bootstrap_web/openai_bootstrap_web.py b/model-providers/model_providers/bootstrap_web/openai_bootstrap_web.py index 3c3e91af..e7784a83 100644 --- a/model-providers/model_providers/bootstrap_web/openai_bootstrap_web.py +++ b/model-providers/model_providers/bootstrap_web/openai_bootstrap_web.py @@ -159,7 +159,7 @@ class RESTFulOpenAIBootstrapBaseWeb(OpenAIBootstrapBaseWeb): async def list_models(self, provider: str, request: Request): logger.info(f"Received list_models request for provider: {provider}") # 返回ModelType所有的枚举 - llm_models: list[AIModelEntity] = [] + llm_models: List[AIModelEntity] = [] for model_type in ModelType.__members__.values(): try: provider_model_bundle = ( @@ -176,7 +176,7 @@ class RESTFulOpenAIBootstrapBaseWeb(OpenAIBootstrapBaseWeb): ) logger.error(e) - # models list[AIModelEntity]转换称List[ModelCard] + # modelsList[AIModelEntity]转换称List[ModelCard] models_list = [ ModelCard(id=model.model, object=model.model_type.to_origin_model_type()) diff --git a/model-providers/model_providers/core/bootstrap/openai_protocol.py b/model-providers/model_providers/core/bootstrap/openai_protocol.py index 2945c0ba..2bd364f3 100644 --- a/model-providers/model_providers/core/bootstrap/openai_protocol.py +++ b/model-providers/model_providers/core/bootstrap/openai_protocol.py @@ -145,7 +145,7 @@ class ChatCompletionRequest(BaseModel): top_k: Optional[float] = None n: int = 1 max_tokens: Optional[int] = 256 - stop: Optional[list[str]] = None + stop: Optional[List[str]] = None stream: Optional[bool] = False def to_model_parameters_dict(self, *args, **kwargs): diff --git a/model-providers/model_providers/core/bootstrap/providers_wapper.py b/model-providers/model_providers/core/bootstrap/providers_wapper.py index d958a999..9d1858ed 100644 --- a/model-providers/model_providers/core/bootstrap/providers_wapper.py +++ b/model-providers/model_providers/core/bootstrap/providers_wapper.py @@ -112,7 +112,7 @@ class ProvidersWrapper: provider_models[model.provider.provider].append(model) # convert to ProviderWithModelsResponse list - providers_with_models: list[ProviderWithModelsResponse] = [] + providers_with_models: List[ProviderWithModelsResponse] = [] for provider, models in provider_models.items(): if not models: continue diff --git a/model-providers/model_providers/core/entities/application_entities.py b/model-providers/model_providers/core/entities/application_entities.py index 9a5e0ff4..263693cc 100644 --- a/model-providers/model_providers/core/entities/application_entities.py +++ b/model-providers/model_providers/core/entities/application_entities.py @@ -21,9 +21,9 @@ class ModelConfigEntity(BaseModel): model_schema: AIModelEntity mode: str provider_model_bundle: ProviderModelBundle - credentials: dict[str, Any] = {} - parameters: dict[str, Any] = {} - stop: list[str] = [] + credentials: Dict[str, Any] = {} + parameters: Dict[str, Any] = {} + stop: List[str] = [] class AdvancedChatMessageEntity(BaseModel): @@ -40,7 +40,7 @@ class AdvancedChatPromptTemplateEntity(BaseModel): Advanced Chat Prompt Template Entity. """ - messages: list[AdvancedChatMessageEntity] + messages: List[AdvancedChatMessageEntity] class AdvancedCompletionPromptTemplateEntity(BaseModel): @@ -102,7 +102,7 @@ class ExternalDataVariableEntity(BaseModel): variable: str type: str - config: dict[str, Any] = {} + config: Dict[str, Any] = {} class DatasetRetrieveConfigEntity(BaseModel): @@ -146,7 +146,7 @@ class DatasetEntity(BaseModel): Dataset Config Entity. """ - dataset_ids: list[str] + dataset_ids: List[str] retrieve_config: DatasetRetrieveConfigEntity @@ -156,7 +156,7 @@ class SensitiveWordAvoidanceEntity(BaseModel): """ type: str - config: dict[str, Any] = {} + config: Dict[str, Any] = {} class TextToSpeechEntity(BaseModel): @@ -185,7 +185,7 @@ class AgentToolEntity(BaseModel): provider_type: Literal["builtin", "api"] provider_id: str tool_name: str - tool_parameters: dict[str, Any] = {} + tool_parameters: Dict[str, Any] = {} class AgentPromptEntity(BaseModel): @@ -234,7 +234,7 @@ class AgentEntity(BaseModel): model: str strategy: Strategy prompt: Optional[AgentPromptEntity] = None - tools: list[AgentToolEntity] = None + tools: List[AgentToolEntity] = None max_iteration: int = 5 @@ -245,7 +245,7 @@ class AppOrchestrationConfigEntity(BaseModel): model_config: ModelConfigEntity prompt_template: PromptTemplateEntity - external_data_variables: list[ExternalDataVariableEntity] = [] + external_data_variables: List[ExternalDataVariableEntity] = [] agent: Optional[AgentEntity] = None # features @@ -319,13 +319,13 @@ class ApplicationGenerateEntity(BaseModel): app_orchestration_config_entity: AppOrchestrationConfigEntity conversation_id: Optional[str] = None - inputs: dict[str, str] + inputs: Dict[str, str] query: Optional[str] = None - files: list[FileObj] = [] + files: List[FileObj] = [] user_id: str # extras stream: bool invoke_from: InvokeFrom # extra parameters, like: auto_generate_conversation_name - extras: dict[str, Any] = {} + extras: Dict[str, Any] = {} diff --git a/model-providers/model_providers/core/entities/message_entities.py b/model-providers/model_providers/core/entities/message_entities.py index b7ad8172..52aa3fa0 100644 --- a/model-providers/model_providers/core/entities/message_entities.py +++ b/model-providers/model_providers/core/entities/message_entities.py @@ -47,12 +47,12 @@ class ImagePromptMessageFile(PromptMessageFile): class LCHumanMessageWithFiles(HumanMessage): - # content: Union[str, list[Union[str, Dict]]] + # content: Union[str,List[Union[str, Dict]]] content: str - files: list[PromptMessageFile] + files: List[PromptMessageFile] -def lc_messages_to_prompt_messages(messages: list[BaseMessage]) -> list[PromptMessage]: +def lc_messages_to_prompt_messages(messages: List[BaseMessage]) -> List[PromptMessage]: prompt_messages = [] for message in messages: if isinstance(message, HumanMessage): @@ -109,8 +109,8 @@ def lc_messages_to_prompt_messages(messages: list[BaseMessage]) -> list[PromptMe def prompt_messages_to_lc_messages( - prompt_messages: list[PromptMessage], -) -> list[BaseMessage]: + prompt_messages: List[PromptMessage], +) -> List[BaseMessage]: messages = [] for prompt_message in prompt_messages: if isinstance(prompt_message, UserPromptMessage): diff --git a/model-providers/model_providers/core/entities/model_entities.py b/model-providers/model_providers/core/entities/model_entities.py index 20e5dbc9..cfaf6b82 100644 --- a/model-providers/model_providers/core/entities/model_entities.py +++ b/model-providers/model_providers/core/entities/model_entities.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional +from typing import List, Optional from pydantic import BaseModel @@ -31,7 +31,7 @@ class SimpleModelProviderEntity(BaseModel): label: I18nObject icon_small: Optional[I18nObject] = None icon_large: Optional[I18nObject] = None - supported_model_types: list[ModelType] + supported_model_types: List[ModelType] def __init__(self, provider_entity: ProviderEntity) -> None: """ @@ -66,7 +66,7 @@ class DefaultModelProviderEntity(BaseModel): label: I18nObject icon_small: Optional[I18nObject] = None icon_large: Optional[I18nObject] = None - supported_model_types: list[ModelType] + supported_model_types: List[ModelType] class DefaultModelEntity(BaseModel): diff --git a/model-providers/model_providers/core/entities/provider_configuration.py b/model-providers/model_providers/core/entities/provider_configuration.py index 22d42587..a068bd9e 100644 --- a/model-providers/model_providers/core/entities/provider_configuration.py +++ b/model-providers/model_providers/core/entities/provider_configuration.py @@ -1,9 +1,8 @@ import datetime import json import logging -from collections.abc import Iterator from json import JSONDecodeError -from typing import Optional +from typing import Dict, Iterator, List, Optional from pydantic import BaseModel @@ -162,7 +161,7 @@ class ProviderConfiguration(BaseModel): def get_provider_models( self, model_type: Optional[ModelType] = None, only_active: bool = False - ) -> list[ModelWithProviderEntity]: + ) -> List[ModelWithProviderEntity]: """ Get provider models. :param model_type: model type @@ -189,8 +188,8 @@ class ProviderConfiguration(BaseModel): return sorted(provider_models, key=lambda x: x.model_type.value) def _get_custom_provider_models( - self, model_types: list[ModelType], provider_instance: ModelProvider - ) -> list[ModelWithProviderEntity]: + self, model_types: List[ModelType], provider_instance: ModelProvider + ) -> List[ModelWithProviderEntity]: """ Get custom provider models. @@ -266,7 +265,7 @@ class ProviderConfigurations(BaseModel): Model class for provider configuration dict. """ - configurations: dict[str, ProviderConfiguration] = {} + configurations: Dict[str, ProviderConfiguration] = {} def __init__(self): super().__init__() @@ -276,7 +275,7 @@ class ProviderConfigurations(BaseModel): provider: Optional[str] = None, model_type: Optional[ModelType] = None, only_active: bool = False, - ) -> list[ModelWithProviderEntity]: + ) -> List[ModelWithProviderEntity]: """ Get available models. @@ -317,7 +316,7 @@ class ProviderConfigurations(BaseModel): return all_models - def to_list(self) -> list[ProviderConfiguration]: + def to_list(self) -> List[ProviderConfiguration]: """ Convert to list. diff --git a/model-providers/model_providers/core/entities/provider_entities.py b/model-providers/model_providers/core/entities/provider_entities.py index ba4a3fb1..7b0705db 100644 --- a/model-providers/model_providers/core/entities/provider_entities.py +++ b/model-providers/model_providers/core/entities/provider_entities.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional +from typing import List, Optional from pydantic import BaseModel @@ -68,7 +68,7 @@ class QuotaConfiguration(BaseModel): quota_limit: int quota_used: int is_valid: bool - restrict_models: list[RestrictModel] = [] + restrict_models: List[RestrictModel] = [] class SystemConfiguration(BaseModel): @@ -78,7 +78,7 @@ class SystemConfiguration(BaseModel): enabled: bool current_quota_type: Optional[ProviderQuotaType] = None - quota_configurations: list[QuotaConfiguration] = [] + quota_configurations: List[QuotaConfiguration] = [] credentials: Optional[dict] = None @@ -106,4 +106,4 @@ class CustomConfiguration(BaseModel): """ provider: Optional[CustomProviderConfiguration] = None - models: list[CustomModelConfiguration] = [] + models: List[CustomModelConfiguration] = [] diff --git a/model-providers/model_providers/core/entities/queue_entities.py b/model-providers/model_providers/core/entities/queue_entities.py index 7ba21aa6..f72080cb 100644 --- a/model-providers/model_providers/core/entities/queue_entities.py +++ b/model-providers/model_providers/core/entities/queue_entities.py @@ -68,7 +68,7 @@ class QueueRetrieverResourcesEvent(AppQueueEvent): """ event = QueueEvent.RETRIEVER_RESOURCES - retriever_resources: list[dict] + retriever_resources: List[dict] class AnnotationReplyEvent(AppQueueEvent): diff --git a/model-providers/model_providers/core/model_manager.py b/model-providers/model_providers/core/model_manager.py index af896423..fc045509 100644 --- a/model-providers/model_providers/core/model_manager.py +++ b/model-providers/model_providers/core/model_manager.py @@ -1,5 +1,4 @@ -from collections.abc import Generator -from typing import IO, Optional, Union, cast +from typing import IO, Generator, List, Optional, Union, cast from model_providers.core.entities.provider_configuration import ProviderModelBundle from model_providers.core.model_runtime.callbacks.base_callback import Callback @@ -68,13 +67,13 @@ class ModelInstance: def invoke_llm( self, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: Optional[dict] = None, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> Union[LLMResult, Generator]: """ Invoke large language model @@ -105,7 +104,7 @@ class ModelInstance: ) def invoke_text_embedding( - self, texts: list[str], user: Optional[str] = None + self, texts: List[str], user: Optional[str] = None ) -> TextEmbeddingResult: """ Invoke large language model @@ -125,7 +124,7 @@ class ModelInstance: def invoke_rerank( self, query: str, - docs: list[str], + docs: List[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None, diff --git a/model-providers/model_providers/core/model_runtime/README_CN.md b/model-providers/model_providers/core/model_runtime/README_CN.md index 3664fa2c..de984853 100644 --- a/model-providers/model_providers/core/model_runtime/README_CN.md +++ b/model-providers/model_providers/core/model_runtime/README_CN.md @@ -69,9 +69,9 @@ Model Runtime 分三层： 在这里我们需要先区分模型参数与模型凭据。 - - 模型参数(**在本层定义**)：这是一类经常需要变动，随时调整的参数，如 LLM 的 **max_tokens**、**temperature** 等，这些参数是由用户在前端页面上进行调整的，因此需要在后端定义参数的规则，以便前端页面进行展示和调整。在DifyRuntime中，他们的参数名一般为**model_parameters: dict[str, any]**。 + - 模型参数(**在本层定义**)：这是一类经常需要变动，随时调整的参数，如 LLM 的 **max_tokens**、**temperature** 等，这些参数是由用户在前端页面上进行调整的，因此需要在后端定义参数的规则，以便前端页面进行展示和调整。在DifyRuntime中，他们的参数名一般为**model_parameters: Dict[str, any]**。 - - 模型凭据(**在供应商层定义**)：这是一类不经常变动，一般在配置好后就不会再变动的参数，如 **api_key**、**server_url** 等。在DifyRuntime中，他们的参数名一般为**credentials: dict[str, any]**，Provider层的credentials会直接被传递到这一层，不需要再单独定义。 + - 模型凭据(**在供应商层定义**)：这是一类不经常变动，一般在配置好后就不会再变动的参数，如 **api_key**、**server_url** 等。在DifyRuntime中，他们的参数名一般为**credentials: Dict[str, any]**，Provider层的credentials会直接被传递到这一层，不需要再单独定义。 ## 下一步 diff --git a/model-providers/model_providers/core/model_runtime/callbacks/base_callback.py b/model-providers/model_providers/core/model_runtime/callbacks/base_callback.py index f7b8c3e5..c7a51cd1 100644 --- a/model-providers/model_providers/core/model_runtime/callbacks/base_callback.py +++ b/model-providers/model_providers/core/model_runtime/callbacks/base_callback.py @@ -1,5 +1,5 @@ from abc import ABC -from typing import Optional +from typing import List, Optional from model_providers.core.model_runtime.entities.llm_entities import ( LLMResult, @@ -33,10 +33,10 @@ class Callback(ABC): llm_instance: AIModel, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> None: @@ -61,10 +61,10 @@ class Callback(ABC): chunk: LLMResultChunk, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ): @@ -90,10 +90,10 @@ class Callback(ABC): result: LLMResult, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> None: @@ -119,10 +119,10 @@ class Callback(ABC): ex: Exception, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> None: diff --git a/model-providers/model_providers/core/model_runtime/callbacks/logging_callback.py b/model-providers/model_providers/core/model_runtime/callbacks/logging_callback.py index be78e354..287a9f67 100644 --- a/model-providers/model_providers/core/model_runtime/callbacks/logging_callback.py +++ b/model-providers/model_providers/core/model_runtime/callbacks/logging_callback.py @@ -1,7 +1,7 @@ import json import logging import sys -from typing import Optional +from typing import List, Optional from model_providers.core.model_runtime.callbacks.base_callback import Callback from model_providers.core.model_runtime.entities.llm_entities import ( @@ -23,10 +23,10 @@ class LoggingCallback(Callback): llm_instance: AIModel, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> None: @@ -79,10 +79,10 @@ class LoggingCallback(Callback): chunk: LLMResultChunk, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ): @@ -109,10 +109,10 @@ class LoggingCallback(Callback): result: LLMResult, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> None: @@ -154,10 +154,10 @@ class LoggingCallback(Callback): ex: Exception, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> None: diff --git a/model-providers/model_providers/core/model_runtime/docs/en_US/interfaces.md b/model-providers/model_providers/core/model_runtime/docs/en_US/interfaces.md index dc70bfad..0f3cfdd0 100644 --- a/model-providers/model_providers/core/model_runtime/docs/en_US/interfaces.md +++ b/model-providers/model_providers/core/model_runtime/docs/en_US/interfaces.md @@ -71,7 +71,7 @@ All models need to uniformly implement the following 2 methods: ```python @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[type[InvokeError],List[type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller @@ -95,8 +95,8 @@ Inherit the `__base.large_language_model.LargeLanguageModel` base class and impl ```python def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[List[str]] = None, + prompt_messages:List[PromptMessage], model_parameters: dict, + tools: Optional[List[PromptMessageTool]] = None, stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None) \ -> Union[LLMResult, Generator]: """ @@ -157,8 +157,8 @@ Inherit the `__base.large_language_model.LargeLanguageModel` base class and impl If the model does not provide a pre-calculated tokens interface, you can directly return 0. ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: + def get_num_tokens(self, model: str, credentials: dict, prompt_messages:List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None) -> int: """ Get number of tokens for given prompt messages @@ -196,7 +196,7 @@ Inherit the `__base.text_embedding_model.TextEmbeddingModel` base class and impl ```python def _invoke(self, model: str, credentials: dict, - texts: list[str], user: Optional[str] = None) \ + texts:List[str], user: Optional[str] = None) \ -> TextEmbeddingResult: """ Invoke large language model @@ -230,7 +230,7 @@ Inherit the `__base.text_embedding_model.TextEmbeddingModel` base class and impl - Pre-calculating Tokens ```python - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts:List[str]) -> int: """ Get number of tokens for given prompt messages @@ -251,7 +251,7 @@ Inherit the `__base.rerank_model.RerankModel` base class and implement the follo ```python def _invoke(self, model: str, credentials: dict, - query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, + query: str, docs:List[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None) \ -> RerankResult: """ @@ -498,7 +498,7 @@ class PromptMessage(ABC, BaseModel): Model class for prompt message. """ role: PromptMessageRole - content: Optional[str | list[PromptMessageContent]] = None # Supports two types: string and content list. The content list is designed to meet the needs of multimodal inputs. For more details, see the PromptMessageContent explanation. + content: Optional[str |List[PromptMessageContent]] = None # Supports two types: string and content list. The content list is designed to meet the needs of multimodal inputs. For more details, see the PromptMessageContent explanation. name: Optional[str] = None ``` @@ -539,7 +539,7 @@ class AssistantPromptMessage(PromptMessage): function: ToolCallFunction # tool call information role: PromptMessageRole = PromptMessageRole.ASSISTANT - tool_calls: list[ToolCall] = [] # The result of tool invocation in response from the model (returned only when tools are input and the model deems it necessary to invoke a tool). + tool_calls:List[ToolCall] = [] # The result of tool invocation in response from the model (returned only when tools are input and the model deems it necessary to invoke a tool). ``` Where `tool_calls` are the list of `tool calls` returned by the model after invoking the model with the `tools` input. @@ -593,7 +593,7 @@ class LLMResult(BaseModel): Model class for llm result. """ model: str # Actual used modele - prompt_messages: list[PromptMessage] # prompt messages + prompt_messages:List[PromptMessage] # prompt messages message: AssistantPromptMessage # response message usage: LLMUsage # usage info system_fingerprint: Optional[str] = None # request fingerprint, refer to OpenAI definition @@ -624,7 +624,7 @@ class LLMResultChunk(BaseModel): Model class for llm result chunk. """ model: str # Actual used modele - prompt_messages: list[PromptMessage] # prompt messages + prompt_messages:List[PromptMessage] # prompt messages system_fingerprint: Optional[str] = None # request fingerprint, refer to OpenAI definition delta: LLMResultChunkDelta ``` @@ -660,7 +660,7 @@ class TextEmbeddingResult(BaseModel): Model class for text embedding result. """ model: str # Actual model used - embeddings: list[list[float]] # List of embedding vectors, corresponding to the input texts list + embeddings:List[List[float]] # List of embedding vectors, corresponding to the input texts list usage: EmbeddingUsage # Usage information ``` @@ -690,7 +690,7 @@ class RerankResult(BaseModel): Model class for rerank result. """ model: str # Actual model used - docs: list[RerankDocument] # Reranked document list + docs:List[RerankDocument] # Reranked document list ``` ### RerankDocument diff --git a/model-providers/model_providers/core/model_runtime/docs/en_US/provider_scale_out.md b/model-providers/model_providers/core/model_runtime/docs/en_US/provider_scale_out.md index ba356c5c..9a1ba736 100644 --- a/model-providers/model_providers/core/model_runtime/docs/en_US/provider_scale_out.md +++ b/model-providers/model_providers/core/model_runtime/docs/en_US/provider_scale_out.md @@ -160,8 +160,8 @@ In `llm.py`, create an Anthropic LLM class, which we name `AnthropicLargeLanguag ```python def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + prompt_messages:List[PromptMessage], model_parameters: dict, + tools: Optional[List[PromptMessageTool]] = None, stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None) \ -> Union[LLMResult, Generator]: """ @@ -184,8 +184,8 @@ In `llm.py`, create an Anthropic LLM class, which we name `AnthropicLargeLanguag If the model does not provide a pre-calculated tokens interface, you can directly return 0. ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: + def get_num_tokens(self, model: str, credentials: dict, prompt_messages:List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None) -> int: """ Get number of tokens for given prompt messages @@ -226,7 +226,7 @@ In `llm.py`, create an Anthropic LLM class, which we name `AnthropicLargeLanguag ```python @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[type[InvokeError],List[type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md b/model-providers/model_providers/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md index 7b3a8edb..8945d0f0 100644 --- a/model-providers/model_providers/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md +++ b/model-providers/model_providers/core/model_runtime/docs/zh_Hans/customizable_model_scale_out.md @@ -126,8 +126,8 @@ provider_credential_schema: ```python def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + prompt_messages:List[PromptMessage], model_parameters: dict, + tools: Optional[List[PromptMessageTool]] = None, stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None) \ -> Union[LLMResult, Generator]: """ @@ -166,8 +166,8 @@ provider_credential_schema: 若模型未提供预计算 tokens 接口，可直接返回 0。 ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: + def get_num_tokens(self, model: str, credentials: dict, prompt_messages:List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None) -> int: """ Get number of tokens for given prompt messages @@ -283,7 +283,7 @@ provider_credential_schema: ```python @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[type[InvokeError],List[type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/docs/zh_Hans/interfaces.md b/model-providers/model_providers/core/model_runtime/docs/zh_Hans/interfaces.md index 743e575d..da921012 100644 --- a/model-providers/model_providers/core/model_runtime/docs/zh_Hans/interfaces.md +++ b/model-providers/model_providers/core/model_runtime/docs/zh_Hans/interfaces.md @@ -80,7 +80,7 @@ class XinferenceProvider(Provider): ```python @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[type[InvokeError],List[type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller @@ -95,7 +95,7 @@ class XinferenceProvider(Provider): ```python @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[type[InvokeError],List[type[Exception]]]: return { InvokeConnectionError: [ InvokeConnectionError @@ -127,8 +127,8 @@ class XinferenceProvider(Provider): ```python def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + prompt_messages:List[PromptMessage], model_parameters: dict, + tools: Optional[List[PromptMessageTool]] = None, stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None) \ -> Union[LLMResult, Generator]: """ @@ -189,8 +189,8 @@ class XinferenceProvider(Provider): 若模型未提供预计算 tokens 接口，可直接返回 0。 ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: + def get_num_tokens(self, model: str, credentials: dict, prompt_messages:List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None) -> int: """ Get number of tokens for given prompt messages @@ -232,7 +232,7 @@ class XinferenceProvider(Provider): ```python def _invoke(self, model: str, credentials: dict, - texts: list[str], user: Optional[str] = None) \ + texts:List[str], user: Optional[str] = None) \ -> TextEmbeddingResult: """ Invoke large language model @@ -266,7 +266,7 @@ class XinferenceProvider(Provider): - 预计算 tokens ```python - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts:List[str]) -> int: """ Get number of tokens for given prompt messages @@ -289,7 +289,7 @@ class XinferenceProvider(Provider): ```python def _invoke(self, model: str, credentials: dict, - query: str, docs: list[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, + query: str, docs:List[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None) \ -> RerankResult: """ @@ -538,7 +538,7 @@ class PromptMessage(ABC, BaseModel): Model class for prompt message. """ role: PromptMessageRole # 消息角色 - content: Optional[str | list[PromptMessageContent]] = None # 支持两种类型，字符串和内容列表，内容列表是为了满足多模态的需要，可详见 PromptMessageContent 说明。 + content: Optional[str |List[PromptMessageContent]] = None # 支持两种类型，字符串和内容列表，内容列表是为了满足多模态的需要，可详见 PromptMessageContent 说明。 name: Optional[str] = None # 名称，可选。 ``` @@ -579,7 +579,7 @@ class AssistantPromptMessage(PromptMessage): function: ToolCallFunction # 工具调用信息 role: PromptMessageRole = PromptMessageRole.ASSISTANT - tool_calls: list[ToolCall] = [] # 模型回复的工具调用结果（仅当传入 tools，并且模型认为需要调用工具时返回） + tool_calls:List[ToolCall] = [] # 模型回复的工具调用结果（仅当传入 tools，并且模型认为需要调用工具时返回） ``` 其中 `tool_calls` 为调用模型传入 `tools` 后，由模型返回的 `tool call` 列表。 @@ -633,7 +633,7 @@ class LLMResult(BaseModel): Model class for llm result. """ model: str # 实际使用模型 - prompt_messages: list[PromptMessage] # prompt 消息列表 + prompt_messages:List[PromptMessage] # prompt 消息列表 message: AssistantPromptMessage # 回复消息 usage: LLMUsage # 使用的 tokens 及费用信息 system_fingerprint: Optional[str] = None # 请求指纹，可参考 OpenAI 该参数定义 @@ -664,7 +664,7 @@ class LLMResultChunk(BaseModel): Model class for llm result chunk. """ model: str # 实际使用模型 - prompt_messages: list[PromptMessage] # prompt 消息列表 + prompt_messages:List[PromptMessage] # prompt 消息列表 system_fingerprint: Optional[str] = None # 请求指纹，可参考 OpenAI 该参数定义 delta: LLMResultChunkDelta # 每个迭代存在变化的内容 ``` @@ -700,7 +700,7 @@ class TextEmbeddingResult(BaseModel): Model class for text embedding result. """ model: str # 实际使用模型 - embeddings: list[list[float]] # embedding 向量列表，对应传入的 texts 列表 + embeddings:List[List[float]] # embedding 向量列表，对应传入的 texts 列表 usage: EmbeddingUsage # 使用信息 ``` @@ -730,7 +730,7 @@ class RerankResult(BaseModel): Model class for rerank result. """ model: str # 实际使用模型 - docs: list[RerankDocument] # 重排后的分段列表 + docs:List[RerankDocument] # 重排后的分段列表 ``` ### RerankDocument diff --git a/model-providers/model_providers/core/model_runtime/docs/zh_Hans/predefined_model_scale_out.md b/model-providers/model_providers/core/model_runtime/docs/zh_Hans/predefined_model_scale_out.md index 56f379a9..ed569340 100644 --- a/model-providers/model_providers/core/model_runtime/docs/zh_Hans/predefined_model_scale_out.md +++ b/model-providers/model_providers/core/model_runtime/docs/zh_Hans/predefined_model_scale_out.md @@ -76,8 +76,8 @@ pricing: # 价格信息 ```python def _invoke(self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, stop: Optional[list[str]] = None, + prompt_messages:List[PromptMessage], model_parameters: dict, + tools: Optional[List[PromptMessageTool]] = None, stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None) \ -> Union[LLMResult, Generator]: """ @@ -116,8 +116,8 @@ pricing: # 价格信息 若模型未提供预计算 tokens 接口，可直接返回 0。 ```python - def get_num_tokens(self, model: str, credentials: dict, prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None) -> int: + def get_num_tokens(self, model: str, credentials: dict, prompt_messages:List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None) -> int: """ Get number of tokens for given prompt messages @@ -158,7 +158,7 @@ pricing: # 价格信息 ```python @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[type[InvokeError],List[type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/entities/defaults.py b/model-providers/model_providers/core/model_runtime/entities/defaults.py index 98719aac..a9092830 100644 --- a/model-providers/model_providers/core/model_runtime/entities/defaults.py +++ b/model-providers/model_providers/core/model_runtime/entities/defaults.py @@ -1,8 +1,10 @@ +from typing import Dict + from model_providers.core.model_runtime.entities.model_entities import ( DefaultParameterName, ) -PARAMETER_RULE_TEMPLATE: dict[DefaultParameterName, dict] = { +PARAMETER_RULE_TEMPLATE: Dict[DefaultParameterName, dict] = { DefaultParameterName.TEMPERATURE: { "label": { "en_US": "Temperature", diff --git a/model-providers/model_providers/core/model_runtime/entities/llm_entities.py b/model-providers/model_providers/core/model_runtime/entities/llm_entities.py index eafdfb2b..8976ff8c 100644 --- a/model-providers/model_providers/core/model_runtime/entities/llm_entities.py +++ b/model-providers/model_providers/core/model_runtime/entities/llm_entities.py @@ -1,6 +1,6 @@ from decimal import Decimal from enum import Enum -from typing import Optional +from typing import List, Optional from pydantic import BaseModel @@ -78,7 +78,7 @@ class LLMResult(BaseModel): """ model: str - prompt_messages: list[PromptMessage] + prompt_messages: List[PromptMessage] message: AssistantPromptMessage usage: LLMUsage system_fingerprint: Optional[str] = None @@ -101,7 +101,7 @@ class LLMResultChunk(BaseModel): """ model: str - prompt_messages: list[PromptMessage] + prompt_messages: List[PromptMessage] system_fingerprint: Optional[str] = None delta: LLMResultChunkDelta diff --git a/model-providers/model_providers/core/model_runtime/entities/message_entities.py b/model-providers/model_providers/core/model_runtime/entities/message_entities.py index a66294ad..0fd0ed17 100644 --- a/model-providers/model_providers/core/model_runtime/entities/message_entities.py +++ b/model-providers/model_providers/core/model_runtime/entities/message_entities.py @@ -1,6 +1,6 @@ from abc import ABC from enum import Enum -from typing import Optional +from typing import List, Optional, Union from pydantic import BaseModel @@ -110,7 +110,7 @@ class PromptMessage(ABC, BaseModel): """ role: PromptMessageRole - content: Optional[str | list[PromptMessageContent]] = None + content: Optional[Union[str, List[PromptMessageContent]]] = None name: Optional[str] = None @@ -145,7 +145,7 @@ class AssistantPromptMessage(PromptMessage): function: ToolCallFunction role: PromptMessageRole = PromptMessageRole.ASSISTANT - tool_calls: list[ToolCall] = [] + tool_calls: List[ToolCall] = [] class SystemPromptMessage(PromptMessage): diff --git a/model-providers/model_providers/core/model_runtime/entities/model_entities.py b/model-providers/model_providers/core/model_runtime/entities/model_entities.py index 0fea8c1d..5cc6e80b 100644 --- a/model-providers/model_providers/core/model_runtime/entities/model_entities.py +++ b/model-providers/model_providers/core/model_runtime/entities/model_entities.py @@ -1,6 +1,6 @@ from decimal import Decimal from enum import Enum -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional from pydantic import BaseModel @@ -158,9 +158,9 @@ class ProviderModel(BaseModel): model: str label: I18nObject model_type: ModelType - features: Optional[list[ModelFeature]] = None + features: Optional[List[ModelFeature]] = None fetch_from: FetchFrom - model_properties: dict[ModelPropertyKey, Any] + model_properties: Dict[ModelPropertyKey, Any] deprecated: bool = False class Config: @@ -182,7 +182,7 @@ class ParameterRule(BaseModel): min: Optional[float] = None max: Optional[float] = None precision: Optional[int] = None - options: list[str] = [] + options: List[str] = [] class PriceConfig(BaseModel): @@ -201,7 +201,7 @@ class AIModelEntity(ProviderModel): Model class for AI model. """ - parameter_rules: list[ParameterRule] = [] + parameter_rules: List[ParameterRule] = [] pricing: Optional[PriceConfig] = None diff --git a/model-providers/model_providers/core/model_runtime/entities/provider_entities.py b/model-providers/model_providers/core/model_runtime/entities/provider_entities.py index 21b610ad..2bfee500 100644 --- a/model-providers/model_providers/core/model_runtime/entities/provider_entities.py +++ b/model-providers/model_providers/core/model_runtime/entities/provider_entities.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Optional +from typing import List, Optional from pydantic import BaseModel @@ -48,7 +48,7 @@ class FormOption(BaseModel): label: I18nObject value: str - show_on: list[FormShowOnObject] = [] + show_on: List[FormShowOnObject] = [] def __init__(self, **data): super().__init__(**data) @@ -66,10 +66,10 @@ class CredentialFormSchema(BaseModel): type: FormType required: bool = True default: Optional[str] = None - options: Optional[list[FormOption]] = None + options: Optional[List[FormOption]] = None placeholder: Optional[I18nObject] = None max_length: int = 0 - show_on: list[FormShowOnObject] = [] + show_on: List[FormShowOnObject] = [] class ProviderCredentialSchema(BaseModel): @@ -77,7 +77,7 @@ class ProviderCredentialSchema(BaseModel): Model class for provider credential schema. """ - credential_form_schemas: list[CredentialFormSchema] + credential_form_schemas: List[CredentialFormSchema] class FieldModelSchema(BaseModel): @@ -91,7 +91,7 @@ class ModelCredentialSchema(BaseModel): """ model: FieldModelSchema - credential_form_schemas: list[CredentialFormSchema] + credential_form_schemas: List[CredentialFormSchema] class SimpleProviderEntity(BaseModel): @@ -103,8 +103,8 @@ class SimpleProviderEntity(BaseModel): label: I18nObject icon_small: Optional[I18nObject] = None icon_large: Optional[I18nObject] = None - supported_model_types: list[ModelType] - models: list[AIModelEntity] = [] + supported_model_types: List[ModelType] + models: List[AIModelEntity] = [] class ProviderHelpEntity(BaseModel): @@ -128,9 +128,9 @@ class ProviderEntity(BaseModel): icon_large: Optional[I18nObject] = None background: Optional[str] = None help: Optional[ProviderHelpEntity] = None - supported_model_types: list[ModelType] - configurate_methods: list[ConfigurateMethod] - models: list[ProviderModel] = [] + supported_model_types: List[ModelType] + configurate_methods: List[ConfigurateMethod] + models: List[ProviderModel] = [] provider_credential_schema: Optional[ProviderCredentialSchema] = None model_credential_schema: Optional[ModelCredentialSchema] = None diff --git a/model-providers/model_providers/core/model_runtime/entities/rerank_entities.py b/model-providers/model_providers/core/model_runtime/entities/rerank_entities.py index 99709e1b..034a7286 100644 --- a/model-providers/model_providers/core/model_runtime/entities/rerank_entities.py +++ b/model-providers/model_providers/core/model_runtime/entities/rerank_entities.py @@ -1,3 +1,5 @@ +from typing import List + from pydantic import BaseModel @@ -17,4 +19,4 @@ class RerankResult(BaseModel): """ model: str - docs: list[RerankDocument] + docs: List[RerankDocument] diff --git a/model-providers/model_providers/core/model_runtime/entities/text_embedding_entities.py b/model-providers/model_providers/core/model_runtime/entities/text_embedding_entities.py index fa2172a0..454e41ee 100644 --- a/model-providers/model_providers/core/model_runtime/entities/text_embedding_entities.py +++ b/model-providers/model_providers/core/model_runtime/entities/text_embedding_entities.py @@ -1,4 +1,5 @@ from decimal import Decimal +from typing import List from pydantic import BaseModel @@ -25,5 +26,5 @@ class TextEmbeddingResult(BaseModel): """ model: str - embeddings: list[list[float]] + embeddings: List[List[float]] usage: EmbeddingUsage diff --git a/model-providers/model_providers/core/model_runtime/model_providers/__base/ai_model.py b/model-providers/model_providers/core/model_runtime/model_providers/__base/ai_model.py index 2c3233d4..6d34f88a 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/__base/ai_model.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/__base/ai_model.py @@ -1,7 +1,7 @@ import decimal import os from abc import ABC, abstractmethod -from typing import Optional +from typing import Dict, List, Optional, Type import yaml @@ -35,7 +35,7 @@ class AIModel(ABC): """ model_type: ModelType - model_schemas: list[AIModelEntity] = None + model_schemas: List[AIModelEntity] = None started_at: float = 0 @abstractmethod @@ -51,7 +51,7 @@ class AIModel(ABC): @property @abstractmethod - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller @@ -133,7 +133,7 @@ class AIModel(ABC): currency=price_config.currency, ) - def predefined_models(self) -> list[AIModelEntity]: + def predefined_models(self) -> List[AIModelEntity]: """ Get all predefined models for given provider. diff --git a/model-providers/model_providers/core/model_runtime/model_providers/__base/large_language_model.py b/model-providers/model_providers/core/model_runtime/model_providers/__base/large_language_model.py index 2b0a1e20..2b09bfdb 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/__base/large_language_model.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/__base/large_language_model.py @@ -3,8 +3,7 @@ import os import re import time from abc import abstractmethod -from collections.abc import Generator -from typing import Optional, Union +from typing import Generator, List, Optional, Union from model_providers.core.model_runtime.callbacks.base_callback import Callback from model_providers.core.model_runtime.callbacks.logging_callback import ( @@ -47,13 +46,13 @@ class LargeLanguageModel(AIModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: Optional[dict] = None, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> Union[LLMResult, Generator]: """ Invoke large language model @@ -170,13 +169,13 @@ class LargeLanguageModel(AIModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> Union[LLMResult, Generator]: """ Code block mode wrapper, ensure the response is a code block with output markdown quote @@ -290,7 +289,7 @@ if you are not sure about the structure. def _code_block_mode_stream_processor( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], input_generator: Generator[LLMResultChunk, None, None], ) -> Generator[LLMResultChunk, None, None]: """ @@ -428,13 +427,13 @@ if you are not sure about the structure. model: str, result: Generator, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> Generator: """ Invoke result generator @@ -498,10 +497,10 @@ if you are not sure about the structure. self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -525,8 +524,8 @@ if you are not sure about the structure. self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -539,7 +538,7 @@ if you are not sure about the structure. """ raise NotImplementedError - def enforce_stop_tokens(self, text: str, stop: list[str]) -> str: + def enforce_stop_tokens(self, text: str, stop: List[str]) -> str: """Cut off the text as soon as any stop words occur.""" return re.split("|".join(stop), text, maxsplit=1)[0] @@ -575,7 +574,7 @@ if you are not sure about the structure. index += 1 time.sleep(0.01) - def get_parameter_rules(self, model: str, credentials: dict) -> list[ParameterRule]: + def get_parameter_rules(self, model: str, credentials: dict) -> List[ParameterRule]: """ Get parameter rules @@ -658,13 +657,13 @@ if you are not sure about the structure. self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> None: """ Trigger before invoke callbacks @@ -706,13 +705,13 @@ if you are not sure about the structure. chunk: LLMResultChunk, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> None: """ Trigger new chunk callbacks @@ -755,13 +754,13 @@ if you are not sure about the structure. model: str, result: LLMResult, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> None: """ Trigger after invoke callbacks @@ -805,13 +804,13 @@ if you are not sure about the structure. model: str, ex: Exception, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> None: """ Trigger invoke error callbacks @@ -911,7 +910,7 @@ if you are not sure about the structure. f"Model Parameter {parameter_name} should be less than or equal to {parameter_rule.max}." ) elif parameter_rule.type == ParameterType.FLOAT: - if not isinstance(parameter_value, float | int): + if not isinstance(parameter_value, (float, int)): raise ValueError( f"Model Parameter {parameter_name} should be float." ) diff --git a/model-providers/model_providers/core/model_runtime/model_providers/__base/model_provider.py b/model-providers/model_providers/core/model_runtime/model_providers/__base/model_provider.py index 48f4a942..40fea584 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/__base/model_provider.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/__base/model_provider.py @@ -1,6 +1,7 @@ import importlib import os from abc import ABC, abstractmethod +from typing import Dict, List import yaml @@ -14,7 +15,7 @@ from model_providers.core.model_runtime.model_providers.__base.ai_model import A class ModelProvider(ABC): provider_schema: ProviderEntity = None - model_instance_map: dict[str, AIModel] = {} + model_instance_map: Dict[str, AIModel] = {} @abstractmethod def validate_provider_credentials(self, credentials: dict) -> None: @@ -65,7 +66,7 @@ class ModelProvider(ABC): return provider_schema - def models(self, model_type: ModelType) -> list[AIModelEntity]: + def models(self, model_type: ModelType) -> List[AIModelEntity]: """ Get all models for given model type diff --git a/model-providers/model_providers/core/model_runtime/model_providers/__base/rerank_model.py b/model-providers/model_providers/core/model_runtime/model_providers/__base/rerank_model.py index 6fdafde9..c37cc7d7 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/__base/rerank_model.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/__base/rerank_model.py @@ -1,6 +1,6 @@ import time from abc import abstractmethod -from typing import Optional +from typing import List, Optional from model_providers.core.model_runtime.entities.model_entities import ModelType from model_providers.core.model_runtime.entities.rerank_entities import RerankResult @@ -19,7 +19,7 @@ class RerankModel(AIModel): model: str, credentials: dict, query: str, - docs: list[str], + docs: List[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None, @@ -51,7 +51,7 @@ class RerankModel(AIModel): model: str, credentials: dict, query: str, - docs: list[str], + docs: List[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None, diff --git a/model-providers/model_providers/core/model_runtime/model_providers/__base/text2img_model.py b/model-providers/model_providers/core/model_runtime/model_providers/__base/text2img_model.py index 058c910e..c411b547 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/__base/text2img_model.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/__base/text2img_model.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import IO, Optional +from typing import IO, Optional, List from model_providers.core.model_runtime.entities.model_entities import ModelType from model_providers.core.model_runtime.model_providers.__base.ai_model import AIModel @@ -19,7 +19,7 @@ class Text2ImageModel(AIModel): prompt: str, model_parameters: dict, user: Optional[str] = None, - ) -> list[IO[bytes]]: + ) -> List[IO[bytes]]: """ Invoke Text2Image model @@ -44,7 +44,7 @@ class Text2ImageModel(AIModel): prompt: str, model_parameters: dict, user: Optional[str] = None, - ) -> list[IO[bytes]]: + ) -> List[IO[bytes]]: """ Invoke Text2Image model diff --git a/model-providers/model_providers/core/model_runtime/model_providers/__base/text_embedding_model.py b/model-providers/model_providers/core/model_runtime/model_providers/__base/text_embedding_model.py index d46b412e..101d2460 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/__base/text_embedding_model.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/__base/text_embedding_model.py @@ -1,6 +1,6 @@ import time from abc import abstractmethod -from typing import Optional +from typing import List, Optional from model_providers.core.model_runtime.entities.model_entities import ( ModelPropertyKey, @@ -23,7 +23,7 @@ class TextEmbeddingModel(AIModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -47,7 +47,7 @@ class TextEmbeddingModel(AIModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -62,7 +62,7 @@ class TextEmbeddingModel(AIModel): raise NotImplementedError @abstractmethod - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages diff --git a/model-providers/model_providers/core/model_runtime/model_providers/anthropic/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/anthropic/llm/llm.py index 032c2757..e13e3b06 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/anthropic/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/anthropic/llm/llm.py @@ -1,7 +1,6 @@ import base64 import mimetypes -from collections.abc import Generator -from typing import Optional, Union, cast +from typing import Optional, Union, cast, Generator, List, Dict, Type import anthropic import requests @@ -63,10 +62,10 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -92,9 +91,9 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -158,13 +157,13 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> Union[LLMResult, Generator]: """ Code block mode wrapper for invoking large language model @@ -203,12 +202,12 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, + user: Union[str, None] = None, response_format: str = "JSON", ) -> None: """ @@ -251,8 +250,8 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -297,7 +296,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: Message, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm chat response @@ -345,7 +344,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: Stream[MessageStreamEvent], - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm chat stream response @@ -424,8 +423,8 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): return credentials_kwargs def _convert_prompt_messages( - self, prompt_messages: list[PromptMessage] - ) -> tuple[str, list[dict]]: + self, prompt_messages: List[PromptMessage] + ) -> tuple[str, List[dict]]: """ Convert prompt messages to dict list and system """ @@ -560,7 +559,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): return message_text def _convert_messages_to_prompt_anthropic( - self, messages: list[PromptMessage] + self, messages: List[PromptMessage] ) -> str: """ Format a list of messages into a full prompt for the Anthropic model @@ -583,7 +582,7 @@ class AnthropicLargeLanguageModel(LargeLanguageModel): return text.rstrip() @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/_common.py b/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/_common.py index 6ae57a15..92bf471d 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/_common.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/_common.py @@ -1,3 +1,5 @@ +from typing import Dict, Type, List + import openai from httpx import Timeout @@ -29,7 +31,7 @@ class _CommonAzureOpenAI: return credentials_kwargs @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: return { InvokeConnectionError: [openai.APIConnectionError, openai.APITimeoutError], InvokeServerUnavailableError: [openai.InternalServerError], diff --git a/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/llm/llm.py index c712fd06..e1b1b6f0 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/llm/llm.py @@ -1,7 +1,6 @@ import copy import logging -from collections.abc import Generator -from typing import Optional, Union, cast +from typing import Optional, Union, cast, List, Generator import tiktoken from openai import AzureOpenAI, Stream @@ -60,10 +59,10 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -102,8 +101,8 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: model_mode = self._get_ai_model_entity( credentials.get("base_model_name"), model @@ -176,9 +175,9 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -215,7 +214,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): model: str, credentials: dict, response: Completion, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: assistant_text = response.choices[0].text @@ -257,7 +256,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): model: str, credentials: dict, response: Stream[Completion], - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: full_text = "" for chunk in response: @@ -321,10 +320,10 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -381,8 +380,8 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): model: str, credentials: dict, response: ChatCompletion, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> LLMResult: assistant_message = response.choices[0].message # assistant_message_tool_calls = assistant_message.tool_calls @@ -435,8 +434,8 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): model: str, credentials: dict, response: Stream[ChatCompletionChunk], - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> Generator: index = 0 full_assistant_content = "" @@ -545,8 +544,8 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): @staticmethod def _extract_response_tool_calls( - response_tool_calls: list[ChatCompletionMessageToolCall | ChoiceDeltaToolCall], - ) -> list[AssistantPromptMessage.ToolCall]: + response_tool_calls: List[Union[ChatCompletionMessageToolCall, ChoiceDeltaToolCall]], + ) -> List[AssistantPromptMessage.ToolCall]: tool_calls = [] if response_tool_calls: for response_tool_call in response_tool_calls: @@ -566,7 +565,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): @staticmethod def _extract_response_function_call( - response_function_call: FunctionCall | ChoiceDeltaFunctionCall, + response_function_call: Union[FunctionCall, ChoiceDeltaFunctionCall], ) -> AssistantPromptMessage.ToolCall: tool_call = None if response_function_call: @@ -651,7 +650,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): self, credentials: dict, text: str, - tools: Optional[list[PromptMessageTool]] = None, + tools: Optional[List[PromptMessageTool]] = None, ) -> int: try: encoding = tiktoken.encoding_for_model(credentials["base_model_name"]) @@ -668,8 +667,8 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): def _num_tokens_from_messages( self, credentials: dict, - messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """Calculate num tokens for gpt-3.5-turbo and gpt-4 with tiktoken package. @@ -743,7 +742,7 @@ class AzureOpenAILargeLanguageModel(_CommonAzureOpenAI, LargeLanguageModel): @staticmethod def _num_tokens_for_tools( - encoding: tiktoken.Encoding, tools: list[PromptMessageTool] + encoding: tiktoken.Encoding, tools: List[PromptMessageTool] ) -> int: num_tokens = 0 for tool in tools: diff --git a/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/text_embedding/text_embedding.py index 17e442fe..6bd4c0f6 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/azure_openai/text_embedding/text_embedding.py @@ -1,7 +1,7 @@ import base64 import copy import time -from typing import Optional, Union +from typing import Optional, Union, List import numpy as np import tiktoken @@ -35,7 +35,7 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: base_model_name = credentials["base_model_name"] @@ -51,7 +51,7 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel): context_size = self._get_context_size(model, credentials) max_chunks = self._get_max_chunks(model, credentials) - embeddings: list[list[float]] = [[] for _ in range(len(texts))] + embeddings: List[List[float]] = [[] for _ in range(len(texts))] tokens = [] indices = [] used_tokens = 0 @@ -81,8 +81,8 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel): used_tokens += embedding_used_tokens batched_embeddings += embeddings_batch - results: list[list[list[float]]] = [[] for _ in range(len(texts))] - num_tokens_in_batch: list[list[int]] = [[] for _ in range(len(texts))] + results: List[List[list[float]]] = [[] for _ in range(len(texts))] + num_tokens_in_batch: List[List[int]] = [[] for _ in range(len(texts))] for i in range(len(indices)): results[indices[i]].append(batched_embeddings[i]) num_tokens_in_batch[indices[i]].append(len(tokens[i])) @@ -112,7 +112,7 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel): embeddings=embeddings, usage=usage, model=base_model_name ) - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: if len(texts) == 0: return 0 @@ -168,9 +168,9 @@ class AzureOpenAITextEmbeddingModel(_CommonAzureOpenAI, TextEmbeddingModel): def _embedding_invoke( model: str, client: AzureOpenAI, - texts: Union[list[str], str], + texts: Union[List[str], str], extra_model_kwargs: dict, - ) -> tuple[list[list[float]], int]: + ) -> tuple[List[list[float]], int]: response = client.embeddings.create( input=texts, model=model, diff --git a/model-providers/model_providers/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py b/model-providers/model_providers/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py index 9fcffccc..95c822f4 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/baichuan/llm/baichuan_turbo.py @@ -1,8 +1,7 @@ -from collections.abc import Generator from enum import Enum from hashlib import md5 from json import dumps, loads -from typing import Any, Union +from typing import Any, Union, Dict, List, Generator from requests import post @@ -25,10 +24,10 @@ class BaichuanMessage: role: str = Role.USER.value content: str - usage: dict[str, int] = None + usage: Dict[str, int] = None stop_reason: str = "" - def to_dict(self) -> dict[str, Any]: + def to_dict(self) -> Dict[str, Any]: return { "role": self.role, "content": self.content, @@ -112,9 +111,9 @@ class BaichuanModel: self, model: str, stream: bool, - messages: list[BaichuanMessage], - parameters: dict[str, Any], - ) -> dict[str, Any]: + messages: List[BaichuanMessage], + parameters: Dict[str, Any], + ) -> Dict[str, Any]: if ( model == "baichuan2-turbo" or model == "baichuan2-turbo-192k" @@ -165,7 +164,7 @@ class BaichuanModel: else: raise BadRequestError(f"Unknown model: {model}") - def _build_headers(self, model: str, data: dict[str, Any]) -> dict[str, Any]: + def _build_headers(self, model: str, data: Dict[str, Any]) -> Dict[str, Any]: if ( model == "baichuan2-turbo" or model == "baichuan2-turbo-192k" @@ -187,8 +186,8 @@ class BaichuanModel: self, model: str, stream: bool, - messages: list[BaichuanMessage], - parameters: dict[str, Any], + messages: List[BaichuanMessage], + parameters: Dict[str, Any], timeout: int, ) -> Union[Generator, BaichuanMessage]: if ( diff --git a/model-providers/model_providers/core/model_runtime/model_providers/baichuan/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/baichuan/llm/llm.py index b1691b5f..14231d73 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/baichuan/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/baichuan/llm/llm.py @@ -1,5 +1,4 @@ -from collections.abc import Generator -from typing import cast +from typing import cast, Generator, List, Union, Dict, Type from model_providers.core.model_runtime.entities.llm_entities import ( LLMResult, @@ -49,13 +48,13 @@ class BaichuanLarguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: return self._generate( model=model, credentials=credentials, @@ -71,14 +70,14 @@ class BaichuanLarguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool] | None = None, + prompt_messages: List[PromptMessage], + tools: Union[List[PromptMessageTool], None] = None, ) -> int: return self._num_tokens_from_messages(prompt_messages) def _num_tokens_from_messages( self, - messages: list[PromptMessage], + messages: List[PromptMessage], ) -> int: """Calculate num tokens for baichuan model""" @@ -149,13 +148,13 @@ class BaichuanLarguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: if tools is not None and len(tools) > 0: raise InvokeBadRequestError("Baichuan model doesn't support tools") @@ -195,7 +194,7 @@ class BaichuanLarguageModel(LargeLanguageModel): def _handle_chat_generate_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: BaichuanMessage, ) -> LLMResult: @@ -216,7 +215,7 @@ class BaichuanLarguageModel(LargeLanguageModel): def _handle_chat_generate_stream_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: Generator[BaichuanMessage, None, None], ) -> Generator: @@ -258,7 +257,7 @@ class BaichuanLarguageModel(LargeLanguageModel): ) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/baichuan/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/baichuan/text_embedding/text_embedding.py index 0f008858..e2a6d0f9 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/baichuan/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/baichuan/text_embedding/text_embedding.py @@ -47,7 +47,7 @@ class BaichuanTextEmbeddingModel(TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -93,8 +93,8 @@ class BaichuanTextEmbeddingModel(TextEmbeddingModel): return result def embedding( - self, model: str, api_key, texts: list[str], user: Optional[str] = None - ) -> tuple[list[list[float]], int]: + self, model: str, api_key, texts: List[str], user: Optional[str] = None + ) -> tuple[List[list[float]], int]: """ Embed given texts @@ -154,7 +154,7 @@ class BaichuanTextEmbeddingModel(TextEmbeddingModel): return [data["embedding"] for data in embeddings], usage["total_tokens"] - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -183,7 +183,7 @@ class BaichuanTextEmbeddingModel(TextEmbeddingModel): raise CredentialsValidateFailedError("Invalid api key") @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: return { InvokeConnectionError: [], InvokeServerUnavailableError: [InternalServerError], diff --git a/model-providers/model_providers/core/model_runtime/model_providers/bedrock/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/bedrock/llm/llm.py index 48a9e990..5741efcd 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/bedrock/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/bedrock/llm/llm.py @@ -1,7 +1,6 @@ import json import logging -from collections.abc import Generator -from typing import Optional, Union +from typing import Dict, Generator, List, Optional, Type, Union import boto3 from botocore.config import Config @@ -48,10 +47,10 @@ class BedrockLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -77,8 +76,8 @@ class BedrockLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - messages: list[PromptMessage] | str, - tools: Optional[list[PromptMessageTool]] = None, + messages: Union[List[PromptMessage], str], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -99,7 +98,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): return self._get_num_tokens_by_gpt2(prompt) def _convert_messages_to_prompt( - self, model_prefix: str, messages: list[PromptMessage] + self, model_prefix: str, messages: List[PromptMessage] ) -> str: """ Format a list of messages into a full prompt for the Google model @@ -190,7 +189,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): return message_text def _convert_messages_to_prompt( - self, messages: list[PromptMessage], model_prefix: str + self, messages: List[PromptMessage], model_prefix: str ) -> str: """ Format a list of messages into a full prompt for the Anthropic, Amazon and Llama models @@ -216,9 +215,9 @@ class BedrockLargeLanguageModel(LargeLanguageModel): def _create_payload( self, model_prefix: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, ): """ @@ -282,9 +281,9 @@ class BedrockLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -356,7 +355,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm response @@ -436,7 +435,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm stream response @@ -551,7 +550,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): ) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the ermd = genai.GenerativeModel(model)ror type thrown to the caller @@ -570,7 +569,7 @@ class BedrockLargeLanguageModel(LargeLanguageModel): def _map_client_to_invoke_error( self, error_code: str, error_msg: str - ) -> type[InvokeError]: + ) -> Type[InvokeError]: """ Map client error to invoke error diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/__init__.py b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/_assets/icon_l_en.svg b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/_assets/icon_l_en.svg deleted file mode 100644 index a824d43d..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/_assets/icon_l_en.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/_assets/icon_s_en.svg b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/_assets/icon_s_en.svg deleted file mode 100644 index 466b4fce..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/_assets/icon_s_en.svg +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - - \ No newline at end of file diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/chatglm.py b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/chatglm.py deleted file mode 100644 index f0bd8825..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/chatglm.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging - -from model_providers.core.model_runtime.entities.model_entities import ModelType -from model_providers.core.model_runtime.errors.validate import ( - CredentialsValidateFailedError, -) -from model_providers.core.model_runtime.model_providers.__base.model_provider import ( - ModelProvider, -) - -logger = logging.getLogger(__name__) - - -class ChatGLMProvider(ModelProvider): - def validate_provider_credentials(self, credentials: dict) -> None: - """ - Validate provider credentials - - if validate failed, raise exception - - :param credentials: provider credentials, credentials form defined in `provider_credential_schema`. - """ - try: - model_instance = self.get_model_instance(ModelType.LLM) - - # Use `chatglm3-6b` model for validate, - model_instance.validate_credentials( - model="chatglm3-6b", credentials=credentials - ) - except CredentialsValidateFailedError as ex: - raise ex - except Exception as ex: - logger.exception( - f"{self.get_provider_schema().provider} credentials validate failed" - ) - raise ex diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/chatglm.yaml b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/chatglm.yaml deleted file mode 100644 index 0c1688c3..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/chatglm.yaml +++ /dev/null @@ -1,28 +0,0 @@ -provider: chatglm -label: - en_US: ChatGLM -icon_small: - en_US: icon_s_en.svg -icon_large: - en_US: icon_l_en.svg -background: "#F4F7FF" -help: - title: - en_US: Deploy ChatGLM to your local - zh_Hans: 部署您的本地 ChatGLM - url: - en_US: https://github.com/THUDM/ChatGLM3 -supported_model_types: - - llm -configurate_methods: - - predefined-model -provider_credential_schema: - credential_form_schemas: - - variable: api_base - label: - en_US: API URL - type: text-input - required: true - placeholder: - zh_Hans: 在此输入您的 API URL - en_US: Enter your API URL diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/__init__.py b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm2-6b-32k.yaml b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm2-6b-32k.yaml deleted file mode 100644 index d1075d74..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm2-6b-32k.yaml +++ /dev/null @@ -1,21 +0,0 @@ -model: chatglm2-6b-32k -label: - en_US: ChatGLM2-6B-32K -model_type: llm -features: - - agent-thought -model_properties: - mode: chat - context_size: 32000 -parameter_rules: - - name: temperature - use_template: temperature - - name: top_p - use_template: top_p - required: false - - name: max_tokens - use_template: max_tokens - required: true - default: 2000 - min: 1 - max: 32000 diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm2-6b.yaml b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm2-6b.yaml deleted file mode 100644 index e3cfeb90..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm2-6b.yaml +++ /dev/null @@ -1,21 +0,0 @@ -model: chatglm2-6b -label: - en_US: ChatGLM2-6B -model_type: llm -features: - - agent-thought -model_properties: - mode: chat - context_size: 2000 -parameter_rules: - - name: temperature - use_template: temperature - - name: top_p - use_template: top_p - required: false - - name: max_tokens - use_template: max_tokens - required: true - default: 256 - min: 1 - max: 2000 diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm3-6b-32k.yaml b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm3-6b-32k.yaml deleted file mode 100644 index 6f347435..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm3-6b-32k.yaml +++ /dev/null @@ -1,22 +0,0 @@ -model: chatglm3-6b-32k -label: - en_US: ChatGLM3-6B-32K -model_type: llm -features: - - tool-call - - agent-thought -model_properties: - mode: chat - context_size: 32000 -parameter_rules: - - name: temperature - use_template: temperature - - name: top_p - use_template: top_p - required: false - - name: max_tokens - use_template: max_tokens - required: true - default: 8000 - min: 1 - max: 32000 diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm3-6b.yaml b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm3-6b.yaml deleted file mode 100644 index d6d87e2e..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/chatglm3-6b.yaml +++ /dev/null @@ -1,22 +0,0 @@ -model: chatglm3-6b -label: - en_US: ChatGLM3-6B -model_type: llm -features: - - tool-call - - agent-thought -model_properties: - mode: chat - context_size: 8000 -parameter_rules: - - name: temperature - use_template: temperature - - name: top_p - use_template: top_p - required: false - - name: max_tokens - use_template: max_tokens - required: true - default: 256 - min: 1 - max: 8000 diff --git a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/llm.py deleted file mode 100644 index 1f798a26..00000000 --- a/model-providers/model_providers/core/model_runtime/model_providers/chatglm/llm/llm.py +++ /dev/null @@ -1,555 +0,0 @@ -import logging -from collections.abc import Generator -from os.path import join -from typing import Optional, cast - -from httpx import Timeout -from openai import ( - APIConnectionError, - APITimeoutError, - AuthenticationError, - ConflictError, - InternalServerError, - NotFoundError, - OpenAI, - PermissionDeniedError, - RateLimitError, - Stream, - UnprocessableEntityError, -) -from openai.types.chat import ChatCompletion, ChatCompletionChunk -from openai.types.chat.chat_completion_message import FunctionCall - -from model_providers.core.model_runtime.entities.llm_entities import ( - LLMResult, - LLMResultChunk, - LLMResultChunkDelta, -) -from model_providers.core.model_runtime.entities.message_entities import ( - AssistantPromptMessage, - PromptMessage, - PromptMessageTool, - SystemPromptMessage, - ToolPromptMessage, - UserPromptMessage, -) -from model_providers.core.model_runtime.errors.invoke import ( - InvokeAuthorizationError, - InvokeBadRequestError, - InvokeConnectionError, - InvokeError, - InvokeRateLimitError, - InvokeServerUnavailableError, -) -from model_providers.core.model_runtime.errors.validate import ( - CredentialsValidateFailedError, -) -from model_providers.core.model_runtime.model_providers.__base.large_language_model import ( - LargeLanguageModel, -) -from model_providers.core.model_runtime.utils import helper - -logger = logging.getLogger(__name__) - - -class ChatGLMLargeLanguageModel(LargeLanguageModel): - def _invoke( - self, - model: str, - credentials: dict, - prompt_messages: list[PromptMessage], - model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, - stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: - """ - Invoke large language model - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param tools: tools for tool calling - :param stop: stop words - :param stream: is stream response - :param user: unique user id - :return: full response or stream response chunk generator result - """ - # invoke model - return self._generate( - model=model, - credentials=credentials, - prompt_messages=prompt_messages, - model_parameters=model_parameters, - tools=tools, - stop=stop, - stream=stream, - user=user, - ) - - def get_num_tokens( - self, - model: str, - credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool] | None = None, - ) -> int: - """ - Get number of tokens for given prompt messages - - :param model: model name - :param credentials: model credentials - :param prompt_messages: prompt messages - :param tools: tools for tool calling - :return: - """ - return self._num_tokens_from_messages(messages=prompt_messages, tools=tools) - - def validate_credentials(self, model: str, credentials: dict) -> None: - """ - Validate model credentials - - :param model: model name - :param credentials: model credentials - :return: - """ - try: - self._invoke( - model=model, - credentials=credentials, - prompt_messages=[ - UserPromptMessage(content="ping"), - ], - model_parameters={ - "max_tokens": 16, - }, - ) - except Exception as e: - raise CredentialsValidateFailedError(str(e)) - - @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: - """ - Map model invoke error to unified error - The key is the error type thrown to the caller - The value is the error type thrown by the model, - which needs to be converted into a unified error type for the caller. - - :return: Invoke error mapping - """ - return { - InvokeConnectionError: [ - APIConnectionError, - APITimeoutError, - ], - InvokeServerUnavailableError: [ - InternalServerError, - ConflictError, - NotFoundError, - UnprocessableEntityError, - PermissionDeniedError, - ], - InvokeRateLimitError: [RateLimitError], - InvokeAuthorizationError: [AuthenticationError], - InvokeBadRequestError: [ValueError], - } - - def _generate( - self, - model: str, - credentials: dict, - prompt_messages: list[PromptMessage], - model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, - stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: - """ - Invoke large language model - - :param model: model name - :param credentials: credentials kwargs - :param prompt_messages: prompt messages - :param model_parameters: model parameters - :param stop: stop words - :param stream: is stream response - :param user: unique user id - :return: full response or stream response chunk generator result - """ - - self._check_chatglm_parameters( - model=model, model_parameters=model_parameters, tools=tools - ) - - kwargs = self._to_client_kwargs(credentials) - # init model client - client = OpenAI(**kwargs) - - extra_model_kwargs = {} - if stop: - extra_model_kwargs["stop"] = stop - - if user: - extra_model_kwargs["user"] = user - - if tools and len(tools) > 0: - extra_model_kwargs["functions"] = [ - helper.dump_model(tool) for tool in tools - ] - - result = client.chat.completions.create( - messages=[self._convert_prompt_message_to_dict(m) for m in prompt_messages], - model=model, - stream=stream, - **model_parameters, - **extra_model_kwargs, - ) - - if stream: - return self._handle_chat_generate_stream_response( - model=model, - credentials=credentials, - response=result, - tools=tools, - prompt_messages=prompt_messages, - ) - - return self._handle_chat_generate_response( - model=model, - credentials=credentials, - response=result, - tools=tools, - prompt_messages=prompt_messages, - ) - - def _check_chatglm_parameters( - self, model: str, model_parameters: dict, tools: list[PromptMessageTool] - ) -> None: - if model.find("chatglm2") != -1 and tools is not None and len(tools) > 0: - raise InvokeBadRequestError("ChatGLM2 does not support function calling") - - def _convert_prompt_message_to_dict(self, message: PromptMessage) -> dict: - """ - Convert PromptMessage to dict for OpenAI Compatibility API - """ - if isinstance(message, UserPromptMessage): - message = cast(UserPromptMessage, message) - if isinstance(message.content, str): - message_dict = {"role": "user", "content": message.content} - else: - raise ValueError("User message content must be str") - elif isinstance(message, AssistantPromptMessage): - message = cast(AssistantPromptMessage, message) - message_dict = {"role": "assistant", "content": message.content} - if message.tool_calls and len(message.tool_calls) > 0: - message_dict["function_call"] = { - "name": message.tool_calls[0].function.name, - "arguments": message.tool_calls[0].function.arguments, - } - elif isinstance(message, SystemPromptMessage): - message = cast(SystemPromptMessage, message) - message_dict = {"role": "system", "content": message.content} - elif isinstance(message, ToolPromptMessage): - # check if last message is user message - message = cast(ToolPromptMessage, message) - message_dict = {"role": "function", "content": message.content} - else: - raise ValueError(f"Unknown message type {type(message)}") - - return message_dict - - def _extract_response_tool_calls( - self, response_function_calls: list[FunctionCall] - ) -> list[AssistantPromptMessage.ToolCall]: - """ - Extract tool calls from response - - :param response_tool_calls: response tool calls - :return: list of tool calls - """ - tool_calls = [] - if response_function_calls: - for response_tool_call in response_function_calls: - function = AssistantPromptMessage.ToolCall.ToolCallFunction( - name=response_tool_call.name, arguments=response_tool_call.arguments - ) - - tool_call = AssistantPromptMessage.ToolCall( - id=0, type="function", function=function - ) - tool_calls.append(tool_call) - - return tool_calls - - def _to_client_kwargs(self, credentials: dict) -> dict: - """ - Convert invoke kwargs to client kwargs - - :param stream: is stream response - :param model_name: model name - :param credentials: credentials dict - :param model_parameters: model parameters - :return: client kwargs - """ - client_kwargs = { - "timeout": Timeout(315.0, read=300.0, write=10.0, connect=5.0), - "api_key": "1", - "base_url": join(credentials["api_base"], "v1"), - } - - return client_kwargs - - def _handle_chat_generate_stream_response( - self, - model: str, - credentials: dict, - response: Stream[ChatCompletionChunk], - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, - ) -> Generator: - full_response = "" - - for chunk in response: - if len(chunk.choices) == 0: - continue - - delta = chunk.choices[0] - - if delta.finish_reason is None and ( - delta.delta.content is None or delta.delta.content == "" - ): - continue - - # check if there is a tool call in the response - function_calls = None - if delta.delta.function_call: - function_calls = [delta.delta.function_call] - - assistant_message_tool_calls = self._extract_response_tool_calls( - function_calls if function_calls else [] - ) - - # transform assistant message to prompt message - assistant_prompt_message = AssistantPromptMessage( - content=delta.delta.content if delta.delta.content else "", - tool_calls=assistant_message_tool_calls, - ) - - if delta.finish_reason is not None: - # temp_assistant_prompt_message is used to calculate usage - temp_assistant_prompt_message = AssistantPromptMessage( - content=full_response, tool_calls=assistant_message_tool_calls - ) - - prompt_tokens = self._num_tokens_from_messages( - messages=prompt_messages, tools=tools - ) - completion_tokens = self._num_tokens_from_messages( - messages=[temp_assistant_prompt_message], tools=[] - ) - - usage = self._calc_response_usage( - model=model, - credentials=credentials, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - ) - - yield LLMResultChunk( - model=model, - prompt_messages=prompt_messages, - system_fingerprint=chunk.system_fingerprint, - delta=LLMResultChunkDelta( - index=delta.index, - message=assistant_prompt_message, - finish_reason=delta.finish_reason, - usage=usage, - ), - ) - else: - yield LLMResultChunk( - model=model, - prompt_messages=prompt_messages, - system_fingerprint=chunk.system_fingerprint, - delta=LLMResultChunkDelta( - index=delta.index, - message=assistant_prompt_message, - ), - ) - - full_response += delta.delta.content - - def _handle_chat_generate_response( - self, - model: str, - credentials: dict, - response: ChatCompletion, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, - ) -> LLMResult: - """ - Handle llm chat response - - :param model: model name - :param credentials: credentials - :param response: response - :param prompt_messages: prompt messages - :param tools: tools for tool calling - :return: llm response - """ - if len(response.choices) == 0: - raise InvokeServerUnavailableError("Empty response") - assistant_message = response.choices[0].message - - # convert function call to tool call - function_calls = assistant_message.function_call - tool_calls = self._extract_response_tool_calls( - [function_calls] if function_calls else [] - ) - - # transform assistant message to prompt message - assistant_prompt_message = AssistantPromptMessage( - content=assistant_message.content, tool_calls=tool_calls - ) - - prompt_tokens = self._num_tokens_from_messages( - messages=prompt_messages, tools=tools - ) - completion_tokens = self._num_tokens_from_messages( - messages=[assistant_prompt_message], tools=tools - ) - - usage = self._calc_response_usage( - model=model, - credentials=credentials, - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - ) - - response = LLMResult( - model=model, - prompt_messages=prompt_messages, - system_fingerprint=response.system_fingerprint, - usage=usage, - message=assistant_prompt_message, - ) - - return response - - def _num_tokens_from_string( - self, text: str, tools: Optional[list[PromptMessageTool]] = None - ) -> int: - """ - Calculate num tokens for text completion model with tiktoken package. - - :param model: model name - :param text: prompt text - :param tools: tools for tool calling - :return: number of tokens - """ - num_tokens = self._get_num_tokens_by_gpt2(text) - - if tools: - num_tokens += self._num_tokens_for_tools(tools) - - return num_tokens - - def _num_tokens_from_messages( - self, - messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, - ) -> int: - """Calculate num tokens for chatglm2 and chatglm3 with GPT2 tokenizer. - - it's too complex to calculate num tokens for chatglm2 and chatglm3 with ChatGLM tokenizer, - As a temporary solution we use GPT2 tokenizer instead. - - """ - - def tokens(text: str): - return self._get_num_tokens_by_gpt2(text) - - tokens_per_message = 3 - tokens_per_name = 1 - num_tokens = 0 - messages_dict = [self._convert_prompt_message_to_dict(m) for m in messages] - for message in messages_dict: - num_tokens += tokens_per_message - for key, value in message.items(): - if isinstance(value, list): - text = "" - for item in value: - if isinstance(item, dict) and item["type"] == "text": - text += item["text"] - value = text - - if key == "function_call": - for t_key, t_value in value.items(): - num_tokens += tokens(t_key) - if t_key == "function": - for f_key, f_value in t_value.items(): - num_tokens += tokens(f_key) - num_tokens += tokens(f_value) - else: - num_tokens += tokens(t_key) - num_tokens += tokens(t_value) - else: - num_tokens += tokens(str(value)) - - if key == "name": - num_tokens += tokens_per_name - - # every reply is primed with assistant - num_tokens += 3 - - if tools: - num_tokens += self._num_tokens_for_tools(tools) - - return num_tokens - - def _num_tokens_for_tools(self, tools: list[PromptMessageTool]) -> int: - """ - Calculate num tokens for tool calling - - :param encoding: encoding - :param tools: tools for tool calling - :return: number of tokens - """ - - def tokens(text: str): - return self._get_num_tokens_by_gpt2(text) - - num_tokens = 0 - for tool in tools: - # calculate num tokens for function object - num_tokens += tokens("name") - num_tokens += tokens(tool.name) - num_tokens += tokens("description") - num_tokens += tokens(tool.description) - parameters = tool.parameters - num_tokens += tokens("parameters") - num_tokens += tokens("type") - num_tokens += tokens(parameters.get("type")) - if "properties" in parameters: - num_tokens += tokens("properties") - for key, value in parameters.get("properties").items(): - num_tokens += tokens(key) - for field_key, field_value in value.items(): - num_tokens += tokens(field_key) - if field_key == "enum": - for enum_field in field_value: - num_tokens += 3 - num_tokens += tokens(enum_field) - else: - num_tokens += tokens(field_key) - num_tokens += tokens(str(field_value)) - if "required" in parameters: - num_tokens += tokens("required") - for required_field in parameters["required"]: - num_tokens += 3 - num_tokens += tokens(required_field) - - return num_tokens diff --git a/model-providers/model_providers/core/model_runtime/model_providers/cohere/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/cohere/llm/llm.py index 620a5b91..98612345 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/cohere/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/cohere/llm/llm.py @@ -1,6 +1,6 @@ import logging from collections.abc import Generator -from typing import Optional, Union, cast +from typing import Optional, Union, cast, List, Dict, Type import cohere from cohere.responses import Chat, Generations @@ -55,10 +55,10 @@ class CohereLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -103,8 +103,8 @@ class CohereLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -171,9 +171,9 @@ class CohereLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -216,7 +216,7 @@ class CohereLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: Generations, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm response @@ -256,7 +256,7 @@ class CohereLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: StreamingGenerations, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm stream response @@ -317,9 +317,9 @@ class CohereLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -377,8 +377,8 @@ class CohereLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: Chat, - prompt_messages: list[PromptMessage], - stop: Optional[list[str]] = None, + prompt_messages: List[PromptMessage], + stop: Optional[List[str]] = None, ) -> LLMResult: """ Handle llm chat response @@ -429,8 +429,8 @@ class CohereLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: StreamingChat, - prompt_messages: list[PromptMessage], - stop: Optional[list[str]] = None, + prompt_messages: List[PromptMessage], + stop: Optional[List[str]] = None, ) -> Generator: """ Handle llm chat stream response @@ -517,8 +517,8 @@ class CohereLargeLanguageModel(LargeLanguageModel): index += 1 def _convert_prompt_messages_to_message_and_chat_histories( - self, prompt_messages: list[PromptMessage] - ) -> tuple[str, list[dict]]: + self, prompt_messages: List[PromptMessage] + ) -> tuple[str, List[dict]]: """ Convert prompt messages to message and chat histories :param prompt_messages: prompt messages @@ -586,7 +586,7 @@ class CohereLargeLanguageModel(LargeLanguageModel): return response.length def _num_tokens_from_messages( - self, model: str, credentials: dict, messages: list[PromptMessage] + self, model: str, credentials: dict, messages: List[PromptMessage] ) -> int: """Calculate num tokens Cohere model.""" messages = [self._convert_prompt_message_to_dict(m) for m in messages] @@ -650,7 +650,7 @@ class CohereLargeLanguageModel(LargeLanguageModel): return entity @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/cohere/rerank/rerank.py b/model-providers/model_providers/core/model_runtime/model_providers/cohere/rerank/rerank.py index 86d0b22f..d07ffe8e 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/cohere/rerank/rerank.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/cohere/rerank/rerank.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, List, Dict, Type import cohere @@ -32,7 +32,7 @@ class CohereRerankModel(RerankModel): model: str, credentials: dict, query: str, - docs: list[str], + docs: List[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None, @@ -99,7 +99,7 @@ class CohereRerankModel(RerankModel): raise CredentialsValidateFailedError(str(ex)) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py index bf4821e9..49044053 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/cohere/text_embedding/text_embedding.py @@ -35,7 +35,7 @@ class CohereTextEmbeddingModel(TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -51,7 +51,7 @@ class CohereTextEmbeddingModel(TextEmbeddingModel): context_size = self._get_context_size(model, credentials) max_chunks = self._get_max_chunks(model, credentials) - embeddings: list[list[float]] = [[] for _ in range(len(texts))] + embeddings: List[List[float]] = [[] for _ in range(len(texts))] tokens = [] indices = [] used_tokens = 0 @@ -79,8 +79,8 @@ class CohereTextEmbeddingModel(TextEmbeddingModel): used_tokens += embedding_used_tokens batched_embeddings += embeddings_batch - results: list[list[list[float]]] = [[] for _ in range(len(texts))] - num_tokens_in_batch: list[list[int]] = [[] for _ in range(len(texts))] + results: List[List[list[float]]] = [[] for _ in range(len(texts))] + num_tokens_in_batch: List[List[int]] = [[] for _ in range(len(texts))] for i in range(len(indices)): results[indices[i]].append(batched_embeddings[i]) num_tokens_in_batch[indices[i]].append(len(tokens[i])) @@ -105,7 +105,7 @@ class CohereTextEmbeddingModel(TextEmbeddingModel): return TextEmbeddingResult(embeddings=embeddings, usage=usage, model=model) - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -161,8 +161,8 @@ class CohereTextEmbeddingModel(TextEmbeddingModel): raise CredentialsValidateFailedError(str(ex)) def _embedding_invoke( - self, model: str, credentials: dict, texts: list[str] - ) -> tuple[list[list[float]], int]: + self, model: str, credentials: dict, texts: List[str] + ) -> tuple[List[list[float]], int]: """ Invoke embedding model @@ -216,7 +216,7 @@ class CohereTextEmbeddingModel(TextEmbeddingModel): return usage @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/google/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/google/llm/llm.py index a23df8aa..c8a52ed7 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/google/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/google/llm/llm.py @@ -1,7 +1,6 @@ import json import logging -from collections.abc import Generator -from typing import Optional, Union +from typing import Optional, Union, Generator, List, Dict, Type import google.api_core.exceptions as exceptions import google.generativeai as genai @@ -66,10 +65,10 @@ class GoogleLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -102,8 +101,8 @@ class GoogleLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -118,7 +117,7 @@ class GoogleLargeLanguageModel(LargeLanguageModel): return self._get_num_tokens_by_gpt2(prompt) - def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str: + def _convert_messages_to_prompt(self, messages: List[PromptMessage]) -> str: """ Format a list of messages into a full prompt for the Google model @@ -155,10 +154,10 @@ class GoogleLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -249,7 +248,7 @@ class GoogleLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: GenerateContentResponse, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm response @@ -306,7 +305,7 @@ class GoogleLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, response: GenerateContentResponse, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm stream response @@ -416,7 +415,7 @@ class GoogleLargeLanguageModel(LargeLanguageModel): return glm_content @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the ermd = genai.GenerativeModel(model)ror type thrown to the caller @@ -472,7 +471,6 @@ class GoogleLargeLanguageModel(LargeLanguageModel): """ tool_call = None if response_function_call: - from google.protobuf import json_format if isinstance(response_function_call, FunctionCall): map_composite_dict = dict(response_function_call.args.items()) diff --git a/model-providers/model_providers/core/model_runtime/model_providers/groq/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/groq/llm/llm.py index 58a76581..0c63d759 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/groq/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/groq/llm/llm.py @@ -1,5 +1,4 @@ -from collections.abc import Generator -from typing import Optional, Union +from typing import Optional, Union, List, Generator from model_providers.core.model_runtime.entities.llm_entities import LLMResult from model_providers.core.model_runtime.entities.message_entities import ( @@ -16,10 +15,10 @@ class GroqLargeLanguageModel(OAIAPICompatLargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: diff --git a/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/_common.py b/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/_common.py index e14f2653..a11d6c74 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/_common.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/_common.py @@ -1,3 +1,5 @@ +from typing import Dict, Type, List + from huggingface_hub.utils import BadRequestError, HfHubHTTPError from model_providers.core.model_runtime.errors.invoke import ( @@ -8,5 +10,5 @@ from model_providers.core.model_runtime.errors.invoke import ( class _CommonHuggingfaceHub: @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: return {InvokeBadRequestError: [HfHubHTTPError, BadRequestError]} diff --git a/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/llm/llm.py index d47f0461..fdfad9ae 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/llm/llm.py @@ -1,5 +1,4 @@ -from collections.abc import Generator -from typing import Optional, Union +from typing import Optional, Union, List, Generator from huggingface_hub import InferenceClient from huggingface_hub.hf_api import HfApi @@ -44,10 +43,10 @@ class HuggingfaceHubLargeLanguageModel(_CommonHuggingfaceHub, LargeLanguageModel self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -81,8 +80,8 @@ class HuggingfaceHubLargeLanguageModel(_CommonHuggingfaceHub, LargeLanguageModel self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: prompt = self._convert_messages_to_prompt(prompt_messages) return self._get_num_tokens_by_gpt2(prompt) @@ -161,7 +160,7 @@ class HuggingfaceHubLargeLanguageModel(_CommonHuggingfaceHub, LargeLanguageModel return entity @staticmethod - def _get_customizable_model_parameter_rules() -> list[ParameterRule]: + def _get_customizable_model_parameter_rules() -> List[ParameterRule]: temperature_rule_dict = PARAMETER_RULE_TEMPLATE.get( DefaultParameterName.TEMPERATURE ).copy() @@ -253,7 +252,7 @@ class HuggingfaceHubLargeLanguageModel(_CommonHuggingfaceHub, LargeLanguageModel self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], response: Generator, ) -> Generator: index = -1 @@ -300,7 +299,7 @@ class HuggingfaceHubLargeLanguageModel(_CommonHuggingfaceHub, LargeLanguageModel self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], response: any, ) -> LLMResult: if isinstance(response, str): @@ -355,7 +354,7 @@ class HuggingfaceHubLargeLanguageModel(_CommonHuggingfaceHub, LargeLanguageModel return model_info.pipeline_tag - def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str: + def _convert_messages_to_prompt(self, messages: List[PromptMessage]) -> str: messages = messages.copy() # don't mutate the original list text = "".join( diff --git a/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/text_embedding/text_embedding.py index a0451017..0d85ab57 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/huggingface_hub/text_embedding/text_embedding.py @@ -1,6 +1,6 @@ import json import time -from typing import Optional +from typing import Optional, List import numpy as np import requests @@ -35,7 +35,7 @@ class HuggingfaceHubTextEmbeddingModel(_CommonHuggingfaceHub, TextEmbeddingModel self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: client = InferenceClient(token=credentials["huggingfacehub_api_token"]) @@ -62,7 +62,7 @@ class HuggingfaceHubTextEmbeddingModel(_CommonHuggingfaceHub, TextEmbeddingModel embeddings=self._mean_pooling(embeddings), usage=usage, model=model ) - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: num_tokens = 0 for text in texts: num_tokens += self._get_num_tokens_by_gpt2(text) @@ -132,12 +132,12 @@ class HuggingfaceHubTextEmbeddingModel(_CommonHuggingfaceHub, TextEmbeddingModel return entity # https://huggingface.co/docs/api-inference/detailed_parameters#feature-extraction-task - # Returned values are a list of floats, or a list[list[floats]] + # Returned values are a list of floats, or aList[List[floats]] # (depending on if you sent a string or a list of string, # and if the automatic reduction, usually mean_pooling for instance was applied for you or not. # This should be explained on the model's README.) @staticmethod - def _mean_pooling(embeddings: list) -> list[float]: + def _mean_pooling(embeddings: list) -> List[float]: # If automatic reduction by giving model, no need to mean_pooling. # For example one: List[List[float]] if not isinstance(embeddings[0][0], list): diff --git a/model-providers/model_providers/core/model_runtime/model_providers/jina/rerank/rerank.py b/model-providers/model_providers/core/model_runtime/model_providers/jina/rerank/rerank.py index 09a3b2fa..7af979e8 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/jina/rerank/rerank.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/jina/rerank/rerank.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, List, Dict, Type import httpx @@ -32,7 +32,7 @@ class JinaRerankModel(RerankModel): model: str, credentials: dict, query: str, - docs: list[str], + docs: List[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None, @@ -108,7 +108,7 @@ class JinaRerankModel(RerankModel): raise CredentialsValidateFailedError(str(ex)) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error """ diff --git a/model-providers/model_providers/core/model_runtime/model_providers/jina/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/jina/text_embedding/text_embedding.py index 48d8fecb..0cacbf73 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/jina/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/jina/text_embedding/text_embedding.py @@ -1,6 +1,6 @@ import time from json import JSONDecodeError, dumps -from typing import Optional +from typing import Optional, List, Dict, Type from requests import post @@ -34,7 +34,7 @@ class JinaTextEmbeddingModel(TextEmbeddingModel): """ api_base: str = "https://api.jina.ai/v1/embeddings" - models: list[str] = [ + models: List[str] = [ "jina-embeddings-v2-base-en", "jina-embeddings-v2-small-en", "jina-embeddings-v2-base-zh", @@ -45,7 +45,7 @@ class JinaTextEmbeddingModel(TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -113,7 +113,7 @@ class JinaTextEmbeddingModel(TextEmbeddingModel): return result - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -142,7 +142,7 @@ class JinaTextEmbeddingModel(TextEmbeddingModel): raise CredentialsValidateFailedError("Invalid api key") @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: return { InvokeConnectionError: [InvokeConnectionError], InvokeServerUnavailableError: [InvokeServerUnavailableError], diff --git a/model-providers/model_providers/core/model_runtime/model_providers/localai/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/localai/llm/llm.py index 0545b52d..6375595f 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/localai/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/localai/llm/llm.py @@ -1,5 +1,4 @@ -from collections.abc import Generator -from typing import cast +from typing import cast, List, Union, Dict, Type, Generator from httpx import Timeout from openai import ( @@ -64,13 +63,13 @@ class LocalAILarguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: return self._generate( model=model, credentials=credentials, @@ -86,14 +85,14 @@ class LocalAILarguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool] | None = None, + prompt_messages: List[PromptMessage], + tools: Union[List[PromptMessageTool], None] = None, ) -> int: # tools is not supported yet return self._num_tokens_from_messages(prompt_messages, tools=tools) def _num_tokens_from_messages( - self, messages: list[PromptMessage], tools: list[PromptMessageTool] + self, messages: List[PromptMessage], tools: List[PromptMessageTool] ) -> int: """ Calculate num tokens for baichuan model @@ -156,7 +155,7 @@ class LocalAILarguageModel(LargeLanguageModel): return num_tokens - def _num_tokens_for_tools(self, tools: list[PromptMessageTool]) -> int: + def _num_tokens_for_tools(self, tools: List[PromptMessageTool]) -> int: """ Calculate num tokens for tool calling @@ -224,7 +223,7 @@ class LocalAILarguageModel(LargeLanguageModel): def get_customizable_model_schema( self, model: str, credentials: dict - ) -> AIModelEntity | None: + ) -> Union[AIModelEntity, None]: completion_model = None if credentials["completion_type"] == "chat_completion": completion_model = LLMMode.CHAT.value @@ -286,13 +285,13 @@ class LocalAILarguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: kwargs = self._to_client_kwargs(credentials) # init model client client = OpenAI(**kwargs) @@ -414,7 +413,7 @@ class LocalAILarguageModel(LargeLanguageModel): return message_dict def _convert_prompt_message_to_completion_prompts( - self, messages: list[PromptMessage] + self, messages: List[PromptMessage] ) -> str: """ Convert PromptMessage to completion prompts @@ -438,7 +437,7 @@ class LocalAILarguageModel(LargeLanguageModel): def _handle_completion_generate_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: Completion, ) -> LLMResult: @@ -489,10 +488,10 @@ class LocalAILarguageModel(LargeLanguageModel): def _handle_chat_generate_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: ChatCompletion, - tools: list[PromptMessageTool], + tools: List[PromptMessageTool], ) -> LLMResult: """ Handle llm chat response @@ -547,10 +546,10 @@ class LocalAILarguageModel(LargeLanguageModel): def _handle_completion_generate_stream_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: Stream[Completion], - tools: list[PromptMessageTool], + tools: List[PromptMessageTool], ) -> Generator: full_response = "" @@ -613,10 +612,10 @@ class LocalAILarguageModel(LargeLanguageModel): def _handle_chat_generate_stream_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: Stream[ChatCompletionChunk], - tools: list[PromptMessageTool], + tools: List[PromptMessageTool], ) -> Generator: full_response = "" @@ -691,8 +690,8 @@ class LocalAILarguageModel(LargeLanguageModel): full_response += delta.delta.content def _extract_response_tool_calls( - self, response_function_calls: list[FunctionCall] - ) -> list[AssistantPromptMessage.ToolCall]: + self, response_function_calls: List[FunctionCall] + ) -> List[AssistantPromptMessage.ToolCall]: """ Extract tool calls from response @@ -714,7 +713,7 @@ class LocalAILarguageModel(LargeLanguageModel): return tool_calls @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/localai/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/localai/text_embedding/text_embedding.py index b42cfb3c..41522010 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/localai/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/localai/text_embedding/text_embedding.py @@ -1,6 +1,6 @@ import time from json import JSONDecodeError, dumps -from typing import Optional +from typing import Optional, List, Union, Dict, Type from requests import post from yarl import URL @@ -42,7 +42,7 @@ class LocalAITextEmbeddingModel(TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -121,7 +121,7 @@ class LocalAITextEmbeddingModel(TextEmbeddingModel): return result - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -138,7 +138,7 @@ class LocalAITextEmbeddingModel(TextEmbeddingModel): def _get_customizable_model_schema( self, model: str, credentials: dict - ) -> AIModelEntity | None: + ) -> Union[AIModelEntity, None]: """ Get customizable model schema @@ -177,7 +177,7 @@ class LocalAITextEmbeddingModel(TextEmbeddingModel): raise CredentialsValidateFailedError(f"Invalid credentials: {e}") @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: return { InvokeConnectionError: [InvokeConnectionError], InvokeServerUnavailableError: [InvokeServerUnavailableError], diff --git a/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/chat_completion.py b/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/chat_completion.py index 7f2edebc..f73876ee 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/chat_completion.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/chat_completion.py @@ -1,6 +1,5 @@ -from collections.abc import Generator from json import dumps, loads -from typing import Any, Union +from typing import Any, Union, Generator, List, Dict from requests import Response, post @@ -27,10 +26,10 @@ class MinimaxChatCompletion: model: str, api_key: str, group_id: str, - prompt_messages: list[MinimaxMessage], + prompt_messages: List[MinimaxMessage], model_parameters: dict, - tools: list[dict[str, Any]], - stop: list[str] | None, + tools: List[Dict[str, Any]], + stop: Union[List[str], None], stream: bool, user: str, ) -> Union[MinimaxMessage, Generator[MinimaxMessage, None, None]]: diff --git a/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/chat_completion_pro.py b/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/chat_completion_pro.py index cbe6979f..34e0857e 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/chat_completion_pro.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/chat_completion_pro.py @@ -1,6 +1,5 @@ -from collections.abc import Generator from json import dumps, loads -from typing import Any, Union +from typing import Any, Union, List, Dict, Generator from requests import Response, post @@ -28,10 +27,10 @@ class MinimaxChatCompletionPro: model: str, api_key: str, group_id: str, - prompt_messages: list[MinimaxMessage], + prompt_messages: List[MinimaxMessage], model_parameters: dict, - tools: list[dict[str, Any]], - stop: list[str] | None, + tools: List[Dict[str, Any]], + stop: Union[List[str], None], stream: bool, user: str, ) -> Union[MinimaxMessage, Generator[MinimaxMessage, None, None]]: diff --git a/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/llm.py index 1696bc7a..aea0f9e1 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/llm.py @@ -58,13 +58,13 @@ class MinimaxLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: return self._generate( model, credentials, @@ -110,13 +110,13 @@ class MinimaxLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool] | None = None, + prompt_messages: List[PromptMessage], + tools: Union[List[PromptMessageTool], None] = None, ) -> int: return self._num_tokens_from_messages(prompt_messages, tools) def _num_tokens_from_messages( - self, messages: list[PromptMessage], tools: list[PromptMessageTool] + self, messages: List[PromptMessage], tools: List[PromptMessageTool] ) -> int: """ Calculate num tokens for minimax model @@ -137,13 +137,13 @@ class MinimaxLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: """ use MinimaxChatCompletionPro as the type of client, anyway, MinimaxChatCompletion has the same interface """ @@ -227,7 +227,7 @@ class MinimaxLargeLanguageModel(LargeLanguageModel): def _handle_chat_generate_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: MinimaxMessage, ) -> LLMResult: @@ -250,7 +250,7 @@ class MinimaxLargeLanguageModel(LargeLanguageModel): def _handle_chat_generate_stream_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: Generator[MinimaxMessage, None, None], ) -> Generator[LLMResultChunk, None, None]: @@ -319,7 +319,7 @@ class MinimaxLargeLanguageModel(LargeLanguageModel): ) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/types.py b/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/types.py index 5e9d73dd..a2fc1220 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/types.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/minimax/llm/types.py @@ -11,11 +11,11 @@ class MinimaxMessage: role: str = Role.USER.value content: str - usage: dict[str, int] = None + usage: Dict[str, int] = None stop_reason: str = "" - function_call: dict[str, Any] = None + function_call: Dict[str, Any] = None - def to_dict(self) -> dict[str, Any]: + def to_dict(self) -> Dict[str, Any]: if self.function_call and self.role == MinimaxMessage.Role.ASSISTANT.value: return { "sender_type": "BOT", diff --git a/model-providers/model_providers/core/model_runtime/model_providers/minimax/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/minimax/text_embedding/text_embedding.py index ff764bb9..8832ea71 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/minimax/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/minimax/text_embedding/text_embedding.py @@ -1,6 +1,6 @@ import time from json import dumps -from typing import Optional +from typing import Optional, List, Dict, Type from requests import post @@ -44,7 +44,7 @@ class MinimaxTextEmbeddingModel(TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -103,7 +103,7 @@ class MinimaxTextEmbeddingModel(TextEmbeddingModel): return result - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -146,7 +146,7 @@ class MinimaxTextEmbeddingModel(TextEmbeddingModel): raise InternalServerError(msg) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/mistralai/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/mistralai/llm/llm.py index 364d4a92..6a3cf428 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/mistralai/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/mistralai/llm/llm.py @@ -1,5 +1,4 @@ -from collections.abc import Generator -from typing import Optional, Union +from typing import Optional, Union, List, Generator from model_providers.core.model_runtime.entities.llm_entities import LLMResult from model_providers.core.model_runtime.entities.message_entities import ( @@ -16,10 +15,10 @@ class MistralAILargeLanguageModel(OAIAPICompatLargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: diff --git a/model-providers/model_providers/core/model_runtime/model_providers/model_provider_factory.py b/model-providers/model_providers/core/model_runtime/model_providers/model_provider_factory.py index fbec3157..6adc96e2 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/model_provider_factory.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/model_provider_factory.py @@ -1,7 +1,7 @@ import importlib import logging import os -from typing import Optional, Union +from typing import Dict, List, Optional, Union from pydantic import BaseModel @@ -42,7 +42,7 @@ class ModelProviderExtension(BaseModel): class ModelProviderFactory: # init cache provider by default init_cache: bool = False - model_provider_extensions: dict[str, ModelProviderExtension] = None + model_provider_extensions: Dict[str, ModelProviderExtension] = None def __init__(self, init_cache: bool = False) -> None: # for cache in memory @@ -51,7 +51,7 @@ class ModelProviderFactory: def get_providers( self, provider_name: Union[str, set] = "" - ) -> list[ProviderEntity]: + ) -> List[ProviderEntity]: """ Get all providers :return: list of providers @@ -159,8 +159,8 @@ class ModelProviderFactory: self, provider: Optional[str] = None, model_type: Optional[ModelType] = None, - provider_configs: Optional[list[ProviderConfig]] = None, - ) -> list[SimpleProviderEntity]: + provider_configs: Optional[List[ProviderConfig]] = None, + ) -> List[SimpleProviderEntity]: """ Get all models for given model type @@ -234,7 +234,7 @@ class ModelProviderFactory: return model_provider_instance - def _get_model_provider_map(self) -> dict[str, ModelProviderExtension]: + def _get_model_provider_map(self) -> Dict[str, ModelProviderExtension]: if self.model_provider_extensions: return self.model_provider_extensions @@ -254,7 +254,7 @@ class ModelProviderFactory: position_map = get_position_map(model_providers_path) # traverse all model_provider_dir_paths - model_providers: list[ModelProviderExtension] = [] + model_providers: List[ModelProviderExtension] = [] for model_provider_dir_path in model_provider_dir_paths: # get model_provider dir name model_provider_name = os.path.basename(model_provider_dir_path) diff --git a/model-providers/model_providers/core/model_runtime/model_providers/moonshot/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/moonshot/llm/llm.py index fbf4bf29..027f00cd 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/moonshot/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/moonshot/llm/llm.py @@ -1,5 +1,5 @@ from collections.abc import Generator -from typing import Optional, Union +from typing import Optional, Union, List from model_providers.core.model_runtime.entities.llm_entities import LLMResult from model_providers.core.model_runtime.entities.message_entities import ( @@ -16,10 +16,10 @@ class MoonshotLargeLanguageModel(OAIAPICompatLargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: diff --git a/model-providers/model_providers/core/model_runtime/model_providers/ollama/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/ollama/llm/llm.py index 19a7cc58..b9ac688b 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/ollama/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/ollama/llm/llm.py @@ -3,7 +3,7 @@ import logging import re from collections.abc import Generator from decimal import Decimal -from typing import Optional, Union, cast +from typing import Optional, Union, cast, List, Dict, Type from urllib.parse import urljoin import requests @@ -63,10 +63,10 @@ class OllamaLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -97,8 +97,8 @@ class OllamaLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -159,9 +159,9 @@ class OllamaLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -258,7 +258,7 @@ class OllamaLargeLanguageModel(LargeLanguageModel): credentials: dict, completion_type: LLMMode, response: requests.Response, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm completion response @@ -310,7 +310,7 @@ class OllamaLargeLanguageModel(LargeLanguageModel): credentials: dict, completion_type: LLMMode, response: requests.Response, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm completion stream response @@ -462,7 +462,7 @@ class OllamaLargeLanguageModel(LargeLanguageModel): return message_dict - def _num_tokens_from_messages(self, messages: list[PromptMessage]) -> int: + def _num_tokens_from_messages(self, messages: List[PromptMessage]) -> int: """ Calculate num tokens. @@ -700,7 +700,7 @@ class OllamaLargeLanguageModel(LargeLanguageModel): return entity @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py index fd037190..bc8d7185 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/ollama/text_embedding/text_embedding.py @@ -2,7 +2,7 @@ import json import logging import time from decimal import Decimal -from typing import Optional +from typing import Optional, List, Dict, Type from urllib.parse import urljoin import numpy as np @@ -48,7 +48,7 @@ class OllamaEmbeddingModel(TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -123,7 +123,7 @@ class OllamaEmbeddingModel(TextEmbeddingModel): embeddings=batched_embeddings, usage=usage, model=model ) - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Approximate number of tokens for given messages using GPT2 tokenizer @@ -211,7 +211,7 @@ class OllamaEmbeddingModel(TextEmbeddingModel): return usage @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openai/_common.py b/model-providers/model_providers/core/model_runtime/model_providers/openai/_common.py index c459c20b..41464c68 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openai/_common.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openai/_common.py @@ -1,3 +1,5 @@ +from typing import Dict, Type, List + import openai from httpx import Timeout @@ -35,7 +37,7 @@ class _CommonOpenAI: return credentials_kwargs @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openai/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/openai/llm/llm.py index 0b886642..33f34bdd 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openai/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openai/llm/llm.py @@ -1,6 +1,6 @@ import logging from collections.abc import Generator -from typing import Optional, Union, cast +from typing import Optional, Union, cast, List import tiktoken from openai import OpenAI, Stream @@ -72,10 +72,10 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -128,13 +128,13 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> Union[LLMResult, Generator]: """ Code block mode wrapper for invoking large language model @@ -194,12 +194,12 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, + user: Union[str, None] = None, response_format: str = "JSON", ) -> None: """ @@ -242,12 +242,12 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, + user: Union[str, None] = None, response_format: str = "JSON", ) -> None: """ @@ -287,8 +287,8 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -366,7 +366,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): except Exception as ex: raise CredentialsValidateFailedError(str(ex)) - def remote_models(self, credentials: dict) -> list[AIModelEntity]: + def remote_models(self, credentials: dict) -> List[AIModelEntity]: """ Return remote models if credentials are provided. @@ -424,9 +424,9 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -479,7 +479,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): model: str, credentials: dict, response: Completion, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm completion response @@ -528,7 +528,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): model: str, credentials: dict, response: Stream[Completion], - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm completion stream response @@ -599,10 +599,10 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -676,8 +676,8 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): model: str, credentials: dict, response: ChatCompletion, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> LLMResult: """ Handle llm chat response @@ -740,8 +740,8 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): model: str, credentials: dict, response: Stream[ChatCompletionChunk], - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> Generator: """ Handle llm chat stream response @@ -851,8 +851,8 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): def _extract_response_tool_calls( self, - response_tool_calls: list[ChatCompletionMessageToolCall | ChoiceDeltaToolCall], - ) -> list[AssistantPromptMessage.ToolCall]: + response_tool_calls: List[Union[ChatCompletionMessageToolCall, ChoiceDeltaToolCall]], + ) -> List[AssistantPromptMessage.ToolCall]: """ Extract tool calls from response @@ -877,7 +877,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): return tool_calls def _extract_response_function_call( - self, response_function_call: FunctionCall | ChoiceDeltaFunctionCall + self, response_function_call: Union[FunctionCall, ChoiceDeltaFunctionCall] ) -> AssistantPromptMessage.ToolCall: """ Extract function call from response @@ -967,7 +967,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): return message_dict def _num_tokens_from_string( - self, model: str, text: str, tools: Optional[list[PromptMessageTool]] = None + self, model: str, text: str, tools: Optional[List[PromptMessageTool]] = None ) -> int: """ Calculate num tokens for text completion model with tiktoken package. @@ -992,8 +992,8 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): def _num_tokens_from_messages( self, model: str, - messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """Calculate num tokens for gpt-3.5-turbo and gpt-4 with tiktoken package. @@ -1068,7 +1068,7 @@ class OpenAILargeLanguageModel(_CommonOpenAI, LargeLanguageModel): return num_tokens def _num_tokens_for_tools( - self, encoding: tiktoken.Encoding, tools: list[PromptMessageTool] + self, encoding: tiktoken.Encoding, tools: List[PromptMessageTool] ) -> int: """ Calculate num tokens for tool calling with tiktoken package. diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openai/moderation/moderation.py b/model-providers/model_providers/core/model_runtime/model_providers/openai/moderation/moderation.py index 8ff1958a..69d5c810 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openai/moderation/moderation.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openai/moderation/moderation.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Optional, List from openai import OpenAI from openai.types import ModerationCreateResponse @@ -82,7 +82,7 @@ class OpenAIModerationModel(_CommonOpenAI, ModerationModel): raise CredentialsValidateFailedError(str(ex)) def _moderation_invoke( - self, model: str, client: OpenAI, texts: list[str] + self, model: str, client: OpenAI, texts: List[str] ) -> ModerationCreateResponse: """ Invoke moderation model diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openai/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/openai/text_embedding/text_embedding.py index ef04e13f..04db0f63 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openai/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openai/text_embedding/text_embedding.py @@ -1,6 +1,6 @@ import base64 import time -from typing import Optional, Union +from typing import Optional, Union, List, Tuple import numpy as np import tiktoken @@ -31,7 +31,7 @@ class OpenAITextEmbeddingModel(_CommonOpenAI, TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -58,7 +58,7 @@ class OpenAITextEmbeddingModel(_CommonOpenAI, TextEmbeddingModel): context_size = self._get_context_size(model, credentials) max_chunks = self._get_max_chunks(model, credentials) - embeddings: list[list[float]] = [[] for _ in range(len(texts))] + embeddings: List[List[float]] = [[] for _ in range(len(texts))] tokens = [] indices = [] used_tokens = 0 @@ -89,8 +89,8 @@ class OpenAITextEmbeddingModel(_CommonOpenAI, TextEmbeddingModel): used_tokens += embedding_used_tokens batched_embeddings += embeddings_batch - results: list[list[list[float]]] = [[] for _ in range(len(texts))] - num_tokens_in_batch: list[list[int]] = [[] for _ in range(len(texts))] + results: List[List[List[float]]] = [[] for _ in range(len(texts))] + num_tokens_in_batch: List[List[int]] = [[] for _ in range(len(texts))] for i in range(len(indices)): results[indices[i]].append(batched_embeddings[i]) num_tokens_in_batch[indices[i]].append(len(tokens[i])) @@ -118,7 +118,7 @@ class OpenAITextEmbeddingModel(_CommonOpenAI, TextEmbeddingModel): return TextEmbeddingResult(embeddings=embeddings, usage=usage, model=model) - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -167,9 +167,9 @@ class OpenAITextEmbeddingModel(_CommonOpenAI, TextEmbeddingModel): self, model: str, client: OpenAI, - texts: Union[list[str], str], + texts: Union[List[str], str], extra_model_kwargs: dict, - ) -> tuple[list[list[float]], int]: + ) -> Tuple[List[List[float]], int]: """ Invoke embedding model diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/_common.py b/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/_common.py index d5c3d879..e7c259fd 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/_common.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/_common.py @@ -1,3 +1,5 @@ +from typing import Dict, Type, List + import requests from model_providers.core.model_runtime.errors.invoke import ( @@ -12,7 +14,7 @@ from model_providers.core.model_runtime.errors.invoke import ( class _CommonOAI_API_Compat: @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py index 05a2c7c9..c365b769 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/llm/llm.py @@ -2,7 +2,7 @@ import json import logging from collections.abc import Generator from decimal import Decimal -from typing import Optional, Union, cast +from typing import Optional, Union, cast, List from urllib.parse import urljoin import requests @@ -61,10 +61,10 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -98,8 +98,8 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -282,10 +282,10 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -384,7 +384,7 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): model: str, credentials: dict, response: requests.Response, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm stream response @@ -516,7 +516,7 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): model: str, credentials: dict, response: requests.Response, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: response_json = response.json() @@ -649,7 +649,7 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): return message_dict def _num_tokens_from_string( - self, model: str, text: str, tools: Optional[list[PromptMessageTool]] = None + self, model: str, text: str, tools: Optional[List[PromptMessageTool]] = None ) -> int: """ Approximate num tokens for model with gpt2 tokenizer. @@ -669,8 +669,8 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): def _num_tokens_from_messages( self, model: str, - messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Approximate num tokens with GPT2 tokenizer. @@ -722,7 +722,7 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): return num_tokens - def _num_tokens_for_tools(self, tools: list[PromptMessageTool]) -> int: + def _num_tokens_for_tools(self, tools: List[PromptMessageTool]) -> int: """ Calculate num tokens for tool calling with tiktoken package. @@ -769,8 +769,8 @@ class OAIAPICompatLargeLanguageModel(_CommonOAI_API_Compat, LargeLanguageModel): return num_tokens def _extract_response_tool_calls( - self, response_tool_calls: list[dict] - ) -> list[AssistantPromptMessage.ToolCall]: + self, response_tool_calls: List[dict] + ) -> List[AssistantPromptMessage.ToolCall]: """ Extract tool calls from response diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/text_embedding/text_embedding.py index 0c71cfa0..c6f1bd78 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openai_api_compatible/text_embedding/text_embedding.py @@ -1,7 +1,7 @@ import json import time from decimal import Decimal -from typing import Optional +from typing import Optional, List from urllib.parse import urljoin import numpy as np @@ -40,7 +40,7 @@ class OAICompatEmbeddingModel(_CommonOAI_API_Compat, TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -131,7 +131,7 @@ class OAICompatEmbeddingModel(_CommonOAI_API_Compat, TextEmbeddingModel): embeddings=batched_embeddings, usage=usage, model=model ) - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Approximate number of tokens for given messages using GPT2 tokenizer diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openllm/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/openllm/llm/llm.py index 271eca7e..812f304a 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openllm/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openllm/llm/llm.py @@ -1,4 +1,4 @@ -from collections.abc import Generator +from typing import List, Union, Type, Dict, Generator from model_providers.core.model_runtime.entities.common_entities import I18nObject from model_providers.core.model_runtime.entities.llm_entities import ( @@ -54,13 +54,13 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: return self._generate( model, credentials, @@ -105,13 +105,13 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool] | None = None, + prompt_messages: List[PromptMessage], + tools: Union[List[PromptMessageTool], None] = None, ) -> int: return self._num_tokens_from_messages(prompt_messages, tools) def _num_tokens_from_messages( - self, messages: list[PromptMessage], tools: list[PromptMessageTool] + self, messages: List[PromptMessage], tools: List[PromptMessageTool] ) -> int: """ Calculate num tokens for OpenLLM model @@ -124,13 +124,13 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: client = OpenLLMGenerate() response = client.generate( model_name=model, @@ -183,7 +183,7 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): def _handle_chat_generate_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: OpenLLMGenerateMessage, ) -> LLMResult: @@ -206,7 +206,7 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): def _handle_chat_generate_stream_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: Generator[OpenLLMGenerateMessage, None, None], ) -> Generator[LLMResultChunk, None, None]: @@ -249,7 +249,7 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): def get_customizable_model_schema( self, model: str, credentials: dict - ) -> AIModelEntity | None: + ) -> Union[AIModelEntity, None]: """ used to define customizable model schema """ @@ -298,7 +298,7 @@ class OpenLLMLargeLanguageModel(LargeLanguageModel): return entity @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openllm/llm/openllm_generate.py b/model-providers/model_providers/core/model_runtime/model_providers/openllm/llm/openllm_generate.py index 05dc9488..1c0ab40d 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openllm/llm/openllm_generate.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openllm/llm/openllm_generate.py @@ -20,10 +20,10 @@ class OpenLLMGenerateMessage: role: str = Role.USER.value content: str - usage: dict[str, int] = None + usage: Dict[str, int] = None stop_reason: str = "" - def to_dict(self) -> dict[str, Any]: + def to_dict(self) -> Dict[str, Any]: return { "role": self.role, "content": self.content, @@ -40,9 +40,9 @@ class OpenLLMGenerate: server_url: str, model_name: str, stream: bool, - model_parameters: dict[str, Any], - stop: list[str], - prompt_messages: list[OpenLLMGenerateMessage], + model_parameters: Dict[str, Any], + stop: List[str], + prompt_messages: List[OpenLLMGenerateMessage], user: str, ) -> Union[Generator[OpenLLMGenerateMessage, None, None], OpenLLMGenerateMessage]: if not server_url: diff --git a/model-providers/model_providers/core/model_runtime/model_providers/openllm/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/openllm/text_embedding/text_embedding.py index 5e1ed3a7..c56442e1 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/openllm/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/openllm/text_embedding/text_embedding.py @@ -1,6 +1,6 @@ import time from json import dumps -from typing import Optional +from typing import Dict, List, Optional, Type from requests import post from requests.exceptions import ConnectionError, InvalidSchema, MissingSchema @@ -35,7 +35,7 @@ class OpenLLMTextEmbeddingModel(TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -89,7 +89,7 @@ class OpenLLMTextEmbeddingModel(TextEmbeddingModel): return result - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -118,7 +118,7 @@ class OpenLLMTextEmbeddingModel(TextEmbeddingModel): raise CredentialsValidateFailedError("Invalid server_url") @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/replicate/_common.py b/model-providers/model_providers/core/model_runtime/model_providers/replicate/_common.py index 582cb8aa..6299cfc0 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/replicate/_common.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/replicate/_common.py @@ -1,3 +1,5 @@ +from typing import Dict, List, Type + from replicate.exceptions import ModelError, ReplicateError from model_providers.core.model_runtime.errors.invoke import ( @@ -8,5 +10,5 @@ from model_providers.core.model_runtime.errors.invoke import ( class _CommonReplicate: @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: return {InvokeBadRequestError: [ReplicateError, ModelError]} diff --git a/model-providers/model_providers/core/model_runtime/model_providers/replicate/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/replicate/llm/llm.py index 987cb4d0..9b72f011 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/replicate/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/replicate/llm/llm.py @@ -1,5 +1,4 @@ -from collections.abc import Generator -from typing import Optional, Union +from typing import Generator, Optional, Union from replicate import Client as ReplicateClient from replicate.exceptions import ReplicateError @@ -43,10 +42,10 @@ class ReplicateLargeLanguageModel(_CommonReplicate, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -86,8 +85,8 @@ class ReplicateLargeLanguageModel(_CommonReplicate, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: prompt = self._convert_messages_to_prompt(prompt_messages) return self._get_num_tokens_by_gpt2(prompt) @@ -167,7 +166,7 @@ class ReplicateLargeLanguageModel(_CommonReplicate, LargeLanguageModel): @classmethod def _get_customizable_model_parameter_rules( cls, model: str, credentials: dict - ) -> list[ParameterRule]: + ) -> List[ParameterRule]: version = credentials["model_version"] client = ReplicateClient( @@ -215,8 +214,8 @@ class ReplicateLargeLanguageModel(_CommonReplicate, LargeLanguageModel): model: str, credentials: dict, prediction: Prediction, - stop: list[str], - prompt_messages: list[PromptMessage], + stop: List[str], + prompt_messages: List[PromptMessage], ) -> Generator: index = -1 current_completion: str = "" @@ -281,8 +280,8 @@ class ReplicateLargeLanguageModel(_CommonReplicate, LargeLanguageModel): model: str, credentials: dict, prediction: Prediction, - stop: list[str], - prompt_messages: list[PromptMessage], + stop: List[str], + prompt_messages: List[PromptMessage], ) -> LLMResult: current_completion: str = "" stop_condition_reached = False @@ -332,7 +331,7 @@ class ReplicateLargeLanguageModel(_CommonReplicate, LargeLanguageModel): elif param_type == "string": return "string" - def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str: + def _convert_messages_to_prompt(self, messages: List[PromptMessage]) -> str: messages = messages.copy() # don't mutate the original list text = "".join( diff --git a/model-providers/model_providers/core/model_runtime/model_providers/replicate/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/replicate/text_embedding/text_embedding.py index a6884360..e45bf19a 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/replicate/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/replicate/text_embedding/text_embedding.py @@ -1,6 +1,6 @@ import json import time -from typing import Optional +from typing import List, Optional from replicate import Client as ReplicateClient @@ -31,7 +31,7 @@ class ReplicateEmbeddingModel(_CommonReplicate, TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: client = ReplicateClient( @@ -52,7 +52,7 @@ class ReplicateEmbeddingModel(_CommonReplicate, TextEmbeddingModel): return TextEmbeddingResult(model=model, embeddings=embeddings, usage=usage) - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: num_tokens = 0 for text in texts: num_tokens += self._get_num_tokens_by_gpt2(text) @@ -124,8 +124,8 @@ class ReplicateEmbeddingModel(_CommonReplicate, TextEmbeddingModel): client: ReplicateClient, replicate_model_version: str, text_input_key: str, - texts: list[str], - ) -> list[list[float]]: + texts: List[str], + ) -> List[List[float]]: if text_input_key in ("text", "inputs"): embeddings = [] for text in texts: diff --git a/model-providers/model_providers/core/model_runtime/model_providers/spark/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/spark/llm/llm.py index c7ea29f3..02741fd7 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/spark/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/spark/llm/llm.py @@ -1,6 +1,6 @@ import threading from collections.abc import Generator -from typing import Optional, Union +from typing import Dict, List, Optional, Type, Union from model_providers.core.model_runtime.entities.llm_entities import ( LLMResult, @@ -37,10 +37,10 @@ class SparkLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -66,8 +66,8 @@ class SparkLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -109,9 +109,9 @@ class SparkLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -171,13 +171,12 @@ class SparkLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, client: SparkLLMClient, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm response :param model: model name - :param response: response :param prompt_messages: prompt messages :return: llm response """ @@ -222,7 +221,7 @@ class SparkLargeLanguageModel(LargeLanguageModel): model: str, credentials: dict, client: SparkLLMClient, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm stream response @@ -300,7 +299,7 @@ class SparkLargeLanguageModel(LargeLanguageModel): return message_text - def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str: + def _convert_messages_to_prompt(self, messages: List[PromptMessage]) -> str: """ Format a list of messages into a full prompt for the Anthropic model @@ -317,7 +316,7 @@ class SparkLargeLanguageModel(LargeLanguageModel): return text.rstrip() @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/togetherai/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/togetherai/llm/llm.py index c954affd..72b4be3c 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/togetherai/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/togetherai/llm/llm.py @@ -1,5 +1,5 @@ from collections.abc import Generator -from typing import Optional, Union +from typing import List, Optional, Union from model_providers.core.model_runtime.entities.llm_entities import LLMResult from model_providers.core.model_runtime.entities.message_entities import ( @@ -21,10 +21,10 @@ class TogetherAILargeLanguageModel(OAIAPICompatLargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -50,10 +50,10 @@ class TogetherAILargeLanguageModel(OAIAPICompatLargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -81,8 +81,8 @@ class TogetherAILargeLanguageModel(OAIAPICompatLargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: cred_with_endpoint = self._update_endpoint_url(credentials=credentials) diff --git a/model-providers/model_providers/core/model_runtime/model_providers/tongyi/_common.py b/model-providers/model_providers/core/model_runtime/model_providers/tongyi/_common.py index da62624a..b3497560 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/tongyi/_common.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/tongyi/_common.py @@ -1,3 +1,5 @@ +from typing import Dict, List, Type + from model_providers.core.model_runtime.errors.invoke import InvokeError @@ -11,7 +13,7 @@ class _CommonTongyi: return credentials_kwargs @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/tongyi/llm/_client.py b/model-providers/model_providers/core/model_runtime/model_providers/tongyi/llm/_client.py index d7bf35f3..8aac0c74 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/tongyi/llm/_client.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/tongyi/llm/_client.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Any, Dict, List, Optional from langchain.callbacks.manager import CallbackManagerForLLMRun from langchain.llms import Tongyi @@ -8,7 +8,7 @@ from langchain.schema import Generation, LLMResult class EnhanceTongyi(Tongyi): @property - def _default_params(self) -> dict[str, Any]: + def _default_params(self) -> Dict[str, Any]: """Get the default parameters for calling OpenAI API.""" normal_params = {"top_p": self.top_p, "api_key": self.dashscope_api_key} @@ -16,13 +16,13 @@ class EnhanceTongyi(Tongyi): def _generate( self, - prompts: list[str], - stop: Optional[list[str]] = None, + prompts: List[str], + stop: Optional[List[str]] = None, run_manager: Optional[CallbackManagerForLLMRun] = None, **kwargs: Any, ) -> LLMResult: generations = [] - params: dict[str, Any] = { + params: Dict[str, Any] = { **{"model": self.model_name}, **self._default_params, **kwargs, diff --git a/model-providers/model_providers/core/model_runtime/model_providers/tongyi/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/tongyi/llm/llm.py index f2b0741c..058027cc 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/tongyi/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/tongyi/llm/llm.py @@ -1,5 +1,5 @@ from collections.abc import Generator -from typing import Optional, Union +from typing import Dict, List, Optional, Type, Union from dashscope import get_tokenizer from dashscope.api_entities.dashscope_response import DashScopeAPIResponse @@ -50,10 +50,10 @@ class TongyiLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -79,14 +79,14 @@ class TongyiLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - callbacks: list[Callback] = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + callbacks: List[Callback] = None, + ) -> Union[LLMResult, Generator]: """ Wrapper for code block mode """ @@ -174,8 +174,8 @@ if you are not sure about the structure. self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -220,9 +220,9 @@ if you are not sure about the structure. self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - stop: Optional[list[str]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -289,7 +289,7 @@ if you are not sure about the structure. model: str, credentials: dict, response: DashScopeAPIResponse, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm response @@ -326,7 +326,7 @@ if you are not sure about the structure. model: str, credentials: dict, responses: Generator, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm stream response @@ -412,7 +412,7 @@ if you are not sure about the structure. return message_text - def _convert_messages_to_prompt(self, messages: list[PromptMessage]) -> str: + def _convert_messages_to_prompt(self, messages: List[PromptMessage]) -> str: """ Format a list of messages into a full prompt for the Anthropic model @@ -429,8 +429,8 @@ if you are not sure about the structure. return text.rstrip() def _convert_prompt_messages_to_tongyi_messages( - self, prompt_messages: list[PromptMessage] - ) -> list[dict]: + self, prompt_messages: List[PromptMessage] + ) -> List[dict]: """ Convert prompt messages to tongyi messages @@ -466,7 +466,7 @@ if you are not sure about the structure. return tongyi_messages @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/tongyi/tts/tts.py b/model-providers/model_providers/core/model_runtime/model_providers/tongyi/tts/tts.py index aa3e7f88..4ab2dd1e 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/tongyi/tts/tts.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/tongyi/tts/tts.py @@ -4,7 +4,7 @@ from io import BytesIO from typing import Optional import dashscope -from fastapi.responses import StreamingResponse +from fastapi.responses import Response, StreamingResponse from pydub import AudioSegment from model_providers.core.model_runtime.errors.invoke import InvokeBadRequestError diff --git a/model-providers/model_providers/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py b/model-providers/model_providers/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py index 39464d4e..2b6d82ac 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/wenxin/llm/ernie_bot.py @@ -1,9 +1,8 @@ -from collections.abc import Generator from datetime import datetime, timedelta from enum import Enum from json import dumps, loads from threading import Lock -from typing import Any, Union +from typing import Any, Dict, Generator, List, Union from requests import Response, post @@ -19,7 +18,7 @@ from model_providers.core.model_runtime.model_providers.wenxin.llm.ernie_bot_err ) # map api_key to access_token -baidu_access_tokens: dict[str, "BaiduAccessToken"] = {} +baidu_access_tokens: Dict[str, "BaiduAccessToken"] = {} baidu_access_tokens_lock = Lock() @@ -118,10 +117,10 @@ class ErnieMessage: role: str = Role.USER.value content: str - usage: dict[str, int] = None + usage: Dict[str, int] = None stop_reason: str = "" - def to_dict(self) -> dict[str, Any]: + def to_dict(self) -> Dict[str, Any]: return { "role": self.role, "content": self.content, @@ -156,11 +155,11 @@ class ErnieBotModel: self, model: str, stream: bool, - messages: list[ErnieMessage], - parameters: dict[str, Any], + messages: List[ErnieMessage], + parameters: Dict[str, Any], timeout: int, - tools: list[PromptMessageTool], - stop: list[str], + tools: List[PromptMessageTool], + stop: List[str], user: str, ) -> Union[Generator[ErnieMessage, None, None], ErnieMessage]: # check parameters @@ -243,15 +242,15 @@ class ErnieBotModel: token = BaiduAccessToken.get_access_token(self.api_key, self.secret_key) return token.access_token - def _copy_messages(self, messages: list[ErnieMessage]) -> list[ErnieMessage]: + def _copy_messages(self, messages: List[ErnieMessage]) -> List[ErnieMessage]: return [ErnieMessage(message.content, message.role) for message in messages] def _check_parameters( self, model: str, - parameters: dict[str, Any], - tools: list[PromptMessageTool], - stop: list[str], + parameters: Dict[str, Any], + tools: List[PromptMessageTool], + stop: List[str], ) -> None: if model not in self.api_bases: raise BadRequestError(f"Invalid model: {model}") @@ -276,13 +275,13 @@ class ErnieBotModel: def _build_request_body( self, model: str, - messages: list[ErnieMessage], + messages: List[ErnieMessage], stream: bool, - parameters: dict[str, Any], - tools: list[PromptMessageTool], - stop: list[str], + parameters: Dict[str, Any], + tools: List[PromptMessageTool], + stop: List[str], user: str, - ) -> dict[str, Any]: + ) -> Dict[str, Any]: # if model in self.function_calling_supports: # return self._build_function_calling_request_body(model, messages, parameters, tools, stop, user) return self._build_chat_request_body( @@ -292,13 +291,13 @@ class ErnieBotModel: def _build_function_calling_request_body( self, model: str, - messages: list[ErnieMessage], + messages: List[ErnieMessage], stream: bool, - parameters: dict[str, Any], - tools: list[PromptMessageTool], - stop: list[str], + parameters: Dict[str, Any], + tools: List[PromptMessageTool], + stop: List[str], user: str, - ) -> dict[str, Any]: + ) -> Dict[str, Any]: if len(messages) % 2 == 0: raise BadRequestError("The number of messages should be odd.") if messages[0].role == "function": @@ -311,12 +310,12 @@ class ErnieBotModel: def _build_chat_request_body( self, model: str, - messages: list[ErnieMessage], + messages: List[ErnieMessage], stream: bool, - parameters: dict[str, Any], - stop: list[str], + parameters: Dict[str, Any], + stop: List[str], user: str, - ) -> dict[str, Any]: + ) -> Dict[str, Any]: if len(messages) == 0: raise BadRequestError("The number of messages should not be zero.") diff --git a/model-providers/model_providers/core/model_runtime/model_providers/wenxin/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/wenxin/llm/llm.py index 1f7a638b..f53c9872 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/wenxin/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/wenxin/llm/llm.py @@ -1,5 +1,4 @@ -from collections.abc import Generator -from typing import Optional, Union, cast +from typing import Dict, Generator, List, Optional, Type, Union, cast from model_providers.core.model_runtime.callbacks.base_callback import Callback from model_providers.core.model_runtime.entities.llm_entities import ( @@ -59,13 +58,13 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: return self._generate( model=model, credentials=credentials, @@ -81,13 +80,13 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, - callbacks: list[Callback] = None, + callbacks: List[Callback] = None, ) -> Union[LLMResult, Generator]: """ Code block mode wrapper for invoking large language model @@ -140,12 +139,12 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, + user: Union[str, None] = None, response_format: str = "JSON", ) -> None: """ @@ -187,15 +186,15 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool] | None = None, + prompt_messages: List[PromptMessage], + tools: Union[List[PromptMessageTool], None] = None, ) -> int: # tools is not supported yet return self._num_tokens_from_messages(prompt_messages) def _num_tokens_from_messages( self, - messages: list[PromptMessage], + messages: List[PromptMessage], ) -> int: """Calculate num tokens for baichuan model""" @@ -234,13 +233,13 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: instance = ErnieBotModel( api_key=credentials["api_key"], secret_key=credentials["secret_key"], @@ -304,7 +303,7 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): def _handle_chat_generate_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: ErnieMessage, ) -> LLMResult: @@ -325,7 +324,7 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): def _handle_chat_generate_stream_response( self, model: str, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], credentials: dict, response: Generator[ErnieMessage, None, None], ) -> Generator: @@ -367,7 +366,7 @@ class ErnieBotLargeLanguageModel(LargeLanguageModel): ) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/xinference/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/xinference/llm/llm.py index aa139170..e521c505 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/xinference/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/xinference/llm/llm.py @@ -1,5 +1,5 @@ from collections.abc import Generator, Iterator -from typing import cast +from typing import Dict, List, Union, cast from openai import ( APIConnectionError, @@ -81,13 +81,13 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: """ invoke LLM @@ -168,8 +168,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool] | None = None, + prompt_messages: List[PromptMessage], + tools: Union[List[PromptMessageTool], None] = None, ) -> int: """ get number of tokens @@ -181,8 +181,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): def _num_tokens_from_messages( self, - messages: list[PromptMessage], - tools: list[PromptMessageTool], + messages: List[PromptMessage], + tools: List[PromptMessageTool], is_completion_model: bool = False, ) -> int: def tokens(text: str): @@ -240,7 +240,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): return num_tokens - def _num_tokens_for_tools(self, tools: list[PromptMessageTool]) -> int: + def _num_tokens_for_tools(self, tools: List[PromptMessageTool]) -> int: """ Calculate num tokens for tool calling @@ -284,7 +284,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): return num_tokens - def _convert_prompt_message_to_text(self, message: list[PromptMessage]) -> str: + def _convert_prompt_message_to_text(self, message: List[PromptMessage]) -> str: """ convert prompt message to text """ @@ -337,7 +337,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): def get_customizable_model_schema( self, model: str, credentials: dict - ) -> AIModelEntity | None: + ) -> Union[AIModelEntity, None]: """ used to define customizable model schema """ @@ -412,14 +412,14 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, extra_model_kwargs: XinferenceModelExtraParameter, - tools: list[PromptMessageTool] | None = None, - stop: list[str] | None = None, + tools: Union[List[PromptMessageTool], None] = None, + stop: Union[List[str], None] = None, stream: bool = True, - user: str | None = None, - ) -> LLMResult | Generator: + user: Union[str, None] = None, + ) -> Union[LLMResult, Generator]: """ generate text from LLM @@ -525,8 +525,10 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): def _extract_response_tool_calls( self, - response_tool_calls: list[ChatCompletionMessageToolCall | ChoiceDeltaToolCall], - ) -> list[AssistantPromptMessage.ToolCall]: + response_tool_calls: Union[ + List[ChatCompletionMessageToolCall, ChoiceDeltaToolCall] + ], + ) -> List[AssistantPromptMessage.ToolCall]: """ Extract tool calls from response @@ -551,7 +553,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): return tool_calls def _extract_response_function_call( - self, response_function_call: FunctionCall | ChoiceDeltaFunctionCall + self, response_function_call: Union[FunctionCall, ChoiceDeltaFunctionCall] ) -> AssistantPromptMessage.ToolCall: """ Extract function call from response @@ -576,8 +578,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool], + prompt_messages: List[PromptMessage], + tools: List[PromptMessageTool], resp: ChatCompletion, ) -> LLMResult: """ @@ -633,8 +635,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool], + prompt_messages: List[PromptMessage], + tools: List[PromptMessageTool], resp: Iterator[ChatCompletionChunk], ) -> Generator: """ @@ -721,8 +723,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool], + prompt_messages: List[PromptMessage], + tools: List[PromptMessageTool], resp: Completion, ) -> LLMResult: """ @@ -765,8 +767,8 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: list[PromptMessageTool], + prompt_messages: List[PromptMessage], + tools: List[PromptMessageTool], resp: Iterator[Completion], ) -> Generator: """ @@ -834,7 +836,7 @@ class XinferenceAILargeLanguageModel(LargeLanguageModel): full_response += delta.text @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/xinference/rerank/rerank.py b/model-providers/model_providers/core/model_runtime/model_providers/xinference/rerank/rerank.py index 4291d3c1..e35400b7 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/xinference/rerank/rerank.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/xinference/rerank/rerank.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Dict, List, Optional, Type, Union from xinference_client.client.restful.restful_client import ( Client, @@ -41,7 +41,7 @@ class XinferenceRerankModel(RerankModel): model: str, credentials: dict, query: str, - docs: list[str], + docs: List[str], score_threshold: Optional[float] = None, top_n: Optional[int] = None, user: Optional[str] = None, @@ -133,7 +133,7 @@ class XinferenceRerankModel(RerankModel): raise CredentialsValidateFailedError(str(ex)) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller @@ -152,7 +152,7 @@ class XinferenceRerankModel(RerankModel): def get_customizable_model_schema( self, model: str, credentials: dict - ) -> AIModelEntity | None: + ) -> Union[AIModelEntity, None]: """ used to define customizable model schema """ diff --git a/model-providers/model_providers/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py index 2cabf59b..cc3d30f0 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/xinference/text_embedding/text_embedding.py @@ -1,5 +1,5 @@ import time -from typing import Optional +from typing import Dict, List, Optional, Type, Union from xinference_client.client.restful.restful_client import ( Client, @@ -46,7 +46,7 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -116,7 +116,7 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): return result - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -167,7 +167,7 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): raise CredentialsValidateFailedError(e) @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: return { InvokeConnectionError: [InvokeConnectionError], InvokeServerUnavailableError: [InvokeServerUnavailableError], @@ -210,7 +210,7 @@ class XinferenceTextEmbeddingModel(TextEmbeddingModel): def get_customizable_model_schema( self, model: str, credentials: dict - ) -> AIModelEntity | None: + ) -> Union[AIModelEntity, None]: """ used to define customizable model schema """ diff --git a/model-providers/model_providers/core/model_runtime/model_providers/xinference/xinference_helper.py b/model-providers/model_providers/core/model_runtime/model_providers/xinference/xinference_helper.py index 6194a0cb..885ebf47 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/xinference/xinference_helper.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/xinference/xinference_helper.py @@ -1,5 +1,6 @@ from threading import Lock from time import time +from typing import List from requests.adapters import HTTPAdapter from requests.exceptions import ConnectionError, MissingSchema, Timeout @@ -10,7 +11,7 @@ from yarl import URL class XinferenceModelExtraParameter: model_format: str model_handle_type: str - model_ability: list[str] + model_ability: List[str] max_tokens: int = 512 context_length: int = 2048 support_function_call: bool = False @@ -19,7 +20,7 @@ class XinferenceModelExtraParameter: self, model_format: str, model_handle_type: str, - model_ability: list[str], + model_ability: List[str], support_function_call: bool, max_tokens: int, context_length: int, @@ -115,7 +116,7 @@ class XinferenceHelper: model_handle_type = "chat" else: raise NotImplementedError( - f"xinference model handle type {model_handle_type} is not supported" + f"xinference model handle type {response_json.get('model_type')} is not supported" ) support_function_call = "tools" in model_ability diff --git a/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/_common.py b/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/_common.py index 6f6595ed..4f7d4be3 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/_common.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/_common.py @@ -1,3 +1,5 @@ +from typing import Dict, List, Type + from model_providers.core.model_runtime.errors.invoke import ( InvokeAuthorizationError, InvokeBadRequestError, @@ -27,7 +29,7 @@ class _CommonZhipuaiAI: return credentials_kwargs @property - def _invoke_error_mapping(self) -> dict[type[InvokeError], list[type[Exception]]]: + def _invoke_error_mapping(self) -> Dict[Type[InvokeError], List[Type[Exception]]]: """ Map model invoke error to unified error The key is the error type thrown to the caller diff --git a/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/llm/llm.py b/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/llm/llm.py index 6ec0d178..d0ec7987 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/llm/llm.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/llm/llm.py @@ -1,5 +1,14 @@ -from collections.abc import Generator -from typing import Optional, Union +from typing import Generator, List, Optional, Union + +from zhipuai import ( + ZhipuAI, +) +from zhipuai.types.chat.chat_completion import ( + Completion, +) +from zhipuai.types.chat.chat_completion_chunk import ( + ChatCompletionChunk, +) from model_providers.core.model_runtime.entities.llm_entities import ( LLMResult, @@ -24,15 +33,6 @@ from model_providers.core.model_runtime.model_providers.__base.large_language_mo from model_providers.core.model_runtime.model_providers.zhipuai._common import ( _CommonZhipuaiAI, ) -from zhipuai import ( - ZhipuAI, -) -from zhipuai.types.chat.chat_completion import ( - Completion, -) -from zhipuai.types.chat.chat_completion_chunk import ( - ChatCompletionChunk, -) from model_providers.core.model_runtime.utils import helper GLM_JSON_MODE_PROMPT = """You should always follow the instructions and output a valid JSON object. @@ -53,10 +53,10 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -91,8 +91,8 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): ) # def _transform_json_prompts(self, model: str, credentials: dict, - # prompt_messages: list[PromptMessage], model_parameters: dict, - # tools: list[PromptMessageTool] | None = None, stop: list[str] | None = None, + # prompt_messages:List[PromptMessage], model_parameters: dict, + # tools:List[PromptMessageTool] | None = None, stop:List[str] | None = None, # stream: bool = True, user: str | None = None) \ # -> None: # """ @@ -126,8 +126,8 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): self, model: str, credentials: dict, - prompt_messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + prompt_messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> int: """ Get number of tokens for given prompt messages @@ -172,10 +172,10 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): self, model: str, credentials_kwargs: dict, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], model_parameters: dict, - tools: Optional[list[PromptMessageTool]] = None, - stop: Optional[list[str]] = None, + tools: Optional[List[PromptMessageTool]] = None, + stop: Optional[List[str]] = None, stream: bool = True, user: Optional[str] = None, ) -> Union[LLMResult, Generator]: @@ -205,7 +205,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): prompt_messages = prompt_messages[1:] # resolve zhipuai model not support system message and user message, assistant message must be in sequence - new_prompt_messages: list[PromptMessage] = [] + new_prompt_messages: List[PromptMessage] = [] for prompt_message in prompt_messages: copy_prompt_message = prompt_message.copy() if copy_prompt_message.role in [ @@ -375,9 +375,9 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): self, model: str, credentials: dict, - tools: Optional[list[PromptMessageTool]], + tools: Optional[List[PromptMessageTool]], response: Completion, - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> LLMResult: """ Handle llm response @@ -388,7 +388,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): :return: llm response """ text = "" - assistant_tool_calls: list[AssistantPromptMessage.ToolCall] = [] + assistant_tool_calls: List[AssistantPromptMessage.ToolCall] = [] for choice in response.choices: if choice.message.tool_calls: for tool_call in choice.message.tool_calls: @@ -430,9 +430,9 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): self, model: str, credentials: dict, - tools: Optional[list[PromptMessageTool]], + tools: Optional[List[PromptMessageTool]], responses: Generator[ChatCompletionChunk, None, None], - prompt_messages: list[PromptMessage], + prompt_messages: List[PromptMessage], ) -> Generator: """ Handle llm stream response @@ -454,7 +454,7 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): ): continue - assistant_tool_calls: list[AssistantPromptMessage.ToolCall] = [] + assistant_tool_calls: List[AssistantPromptMessage.ToolCall] = [] for tool_call in delta.delta.tool_calls or []: if tool_call.type == "function": assistant_tool_calls.append( @@ -531,8 +531,8 @@ class ZhipuAILargeLanguageModel(_CommonZhipuaiAI, LargeLanguageModel): def _convert_messages_to_prompt( self, - messages: list[PromptMessage], - tools: Optional[list[PromptMessageTool]] = None, + messages: List[PromptMessage], + tools: Optional[List[PromptMessageTool]] = None, ) -> str: """ :param messages: List of PromptMessage to combine. diff --git a/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py b/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py index af125b90..cd870b11 100644 --- a/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py +++ b/model-providers/model_providers/core/model_runtime/model_providers/zhipuai/text_embedding/text_embedding.py @@ -1,5 +1,9 @@ import time -from typing import Optional +from typing import List, Optional, Tuple + +from zhipuai import ( + ZhipuAI, +) from model_providers.core.model_runtime.entities.model_entities import PriceType from model_providers.core.model_runtime.entities.text_embedding_entities import ( @@ -15,9 +19,6 @@ from model_providers.core.model_runtime.model_providers.__base.text_embedding_mo from model_providers.core.model_runtime.model_providers.zhipuai._common import ( _CommonZhipuaiAI, ) -from zhipuai import ( - ZhipuAI, -) class ZhipuAITextEmbeddingModel(_CommonZhipuaiAI, TextEmbeddingModel): @@ -29,7 +30,7 @@ class ZhipuAITextEmbeddingModel(_CommonZhipuaiAI, TextEmbeddingModel): self, model: str, credentials: dict, - texts: list[str], + texts: List[str], user: Optional[str] = None, ) -> TextEmbeddingResult: """ @@ -54,7 +55,7 @@ class ZhipuAITextEmbeddingModel(_CommonZhipuaiAI, TextEmbeddingModel): model=model, ) - def get_num_tokens(self, model: str, credentials: dict, texts: list[str]) -> int: + def get_num_tokens(self, model: str, credentials: dict, texts: List[str]) -> int: """ Get number of tokens for given prompt messages @@ -95,8 +96,8 @@ class ZhipuAITextEmbeddingModel(_CommonZhipuaiAI, TextEmbeddingModel): raise CredentialsValidateFailedError(str(ex)) def embed_documents( - self, model: str, client: ZhipuAI, texts: list[str] - ) -> tuple[list[list[float]], int]: + self, model: str, client: ZhipuAI, texts: List[str] + ) -> Tuple[List[List[float]], int]: """Call out to ZhipuAI's embedding endpoint. Args: @@ -116,17 +117,6 @@ class ZhipuAITextEmbeddingModel(_CommonZhipuaiAI, TextEmbeddingModel): return [list(map(float, e)) for e in embeddings], embedding_used_tokens - def embed_query(self, text: str) -> list[float]: - """Call out to ZhipuAI's embedding endpoint. - - Args: - text: The text to embed. - - Returns: - Embeddings for the text. - """ - return self.embed_documents([text])[0] - def _calc_response_usage( self, model: str, credentials: dict, tokens: int ) -> EmbeddingUsage: diff --git a/model-providers/model_providers/core/model_runtime/schema_validators/common_validator.py b/model-providers/model_providers/core/model_runtime/schema_validators/common_validator.py index 8d56fb65..4b2ee2ee 100644 --- a/model-providers/model_providers/core/model_runtime/schema_validators/common_validator.py +++ b/model-providers/model_providers/core/model_runtime/schema_validators/common_validator.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import List, Optional from model_providers.core.model_runtime.entities.provider_entities import ( CredentialFormSchema, @@ -8,7 +8,7 @@ from model_providers.core.model_runtime.entities.provider_entities import ( class CommonValidator: def _validate_and_filter_credential_form_schemas( - self, credential_form_schemas: list[CredentialFormSchema], credentials: dict + self, credential_form_schemas: List[CredentialFormSchema], credentials: dict ) -> dict: need_validate_credential_form_schema_map = {} for credential_form_schema in credential_form_schemas: diff --git a/model-providers/model_providers/core/model_runtime/utils/encoders.py b/model-providers/model_providers/core/model_runtime/utils/encoders.py index 7c98c5e0..cea96079 100644 --- a/model-providers/model_providers/core/model_runtime/utils/encoders.py +++ b/model-providers/model_providers/core/model_runtime/utils/encoders.py @@ -54,7 +54,7 @@ def decimal_encoder(dec_value: Decimal) -> Union[int, float]: return float(dec_value) -ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = { +ENCODERS_BY_TYPE: Dict[type[Any], Callable[[Any], Any]] = { bytes: lambda o: o.decode(), Color: str, datetime.date: isoformat, @@ -85,9 +85,9 @@ ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = { def generate_encoders_by_class_tuples( - type_encoder_map: dict[Any, Callable[[Any], Any]], -) -> dict[Callable[[Any], Any], tuple[Any, ...]]: - encoders_by_class_tuples: dict[Callable[[Any], Any], tuple[Any, ...]] = defaultdict( + type_encoder_map: Dict[Any, Callable[[Any], Any]], +) -> Dict[Callable[[Any], Any], tuple[Any, ...]]: + encoders_by_class_tuples: Dict[Callable[[Any], Any], tuple[Any, ...]] = defaultdict( tuple ) for type_, encoder in type_encoder_map.items(): @@ -117,7 +117,7 @@ def jsonable_encoder( return encoder_instance(obj) if isinstance(obj, BaseModel): # TODO: remove when deprecating Pydantic v1 - encoders: dict[Any, Any] = {} + encoders: Dict[Any, Any] = {} if not PYDANTIC_V2: encoders = getattr(obj.__config__, "json_encoders", {}) # type: ignore[attr-defined] if custom_encoder: @@ -217,7 +217,7 @@ def jsonable_encoder( try: data = dict(obj) except Exception as e: - errors: list[Exception] = [] + errors: List[Exception] = [] errors.append(e) try: data = vars(obj) diff --git a/model-providers/model_providers/core/provider_manager.py b/model-providers/model_providers/core/provider_manager.py index e8703e45..ad511337 100644 --- a/model-providers/model_providers/core/provider_manager.py +++ b/model-providers/model_providers/core/provider_manager.py @@ -1,9 +1,7 @@ import json from collections import defaultdict from json import JSONDecodeError -from typing import Optional, Union - -from sqlalchemy.exc import IntegrityError +from typing import List, Optional, Union from model_providers.core.entities.model_entities import ( DefaultModelEntity, @@ -201,7 +199,7 @@ class ProviderManager: self, provider_entity: ProviderEntity, provider_credentials: dict, - provider_model_records: list[dict], + provider_model_records: List[dict], ) -> CustomConfiguration: """ Convert to custom configuration. @@ -266,8 +264,8 @@ class ProviderManager: ) def _extract_variables( - self, credential_form_schemas: list[CredentialFormSchema] - ) -> list[str]: + self, credential_form_schemas: List[CredentialFormSchema] + ) -> List[str]: """ Extract input form variables. diff --git a/model-providers/model_providers/core/utils/position_helper.py b/model-providers/model_providers/core/utils/position_helper.py index 55fd754c..f6c14d81 100644 --- a/model-providers/model_providers/core/utils/position_helper.py +++ b/model-providers/model_providers/core/utils/position_helper.py @@ -1,8 +1,6 @@ import logging import os -from collections import OrderedDict -from collections.abc import Callable -from typing import Any, AnyStr +from typing import Any, AnyStr, Callable, Dict, List, OrderedDict import yaml @@ -10,7 +8,7 @@ import yaml def get_position_map( folder_path: AnyStr, file_name: str = "_position.yaml", -) -> dict[str, int]: +) -> Dict[str, int]: """ Get the mapping from name to index from a YAML file :param folder_path: @@ -37,10 +35,10 @@ def get_position_map( def sort_by_position_map( - position_map: dict[str, int], - data: list[Any], + position_map: Dict[str, int], + data: List[Any], name_func: Callable[[Any], str], -) -> list[Any]: +) -> List[Any]: """ Sort the objects by the position map. If the name of the object is not in the position map, it will be put at the end. @@ -56,8 +54,8 @@ def sort_by_position_map( def sort_to_dict_by_position_map( - position_map: dict[str, int], - data: list[Any], + position_map: Dict[str, int], + data: List[Any], name_func: Callable[[Any], str], ) -> OrderedDict[str, Any]: """