chore: 添加虚拟环境到仓库
- 添加 backend_service/venv 虚拟环境 - 包含所有Python依赖包 - 注意：虚拟环境约393MB，包含12655个文件
This commit is contained in:
@@ -0,0 +1,33 @@
|
||||
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
||||
|
||||
from .runs import (
|
||||
Runs,
|
||||
AsyncRuns,
|
||||
RunsWithRawResponse,
|
||||
AsyncRunsWithRawResponse,
|
||||
RunsWithStreamingResponse,
|
||||
AsyncRunsWithStreamingResponse,
|
||||
)
|
||||
from .evals import (
|
||||
Evals,
|
||||
AsyncEvals,
|
||||
EvalsWithRawResponse,
|
||||
AsyncEvalsWithRawResponse,
|
||||
EvalsWithStreamingResponse,
|
||||
AsyncEvalsWithStreamingResponse,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Runs",
|
||||
"AsyncRuns",
|
||||
"RunsWithRawResponse",
|
||||
"AsyncRunsWithRawResponse",
|
||||
"RunsWithStreamingResponse",
|
||||
"AsyncRunsWithStreamingResponse",
|
||||
"Evals",
|
||||
"AsyncEvals",
|
||||
"EvalsWithRawResponse",
|
||||
"AsyncEvalsWithRawResponse",
|
||||
"EvalsWithStreamingResponse",
|
||||
"AsyncEvalsWithStreamingResponse",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,662 @@
|
||||
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Iterable, Optional
|
||||
from typing_extensions import Literal
|
||||
|
||||
import httpx
|
||||
|
||||
from ... import _legacy_response
|
||||
from ...types import eval_list_params, eval_create_params, eval_update_params
|
||||
from ..._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
|
||||
from ..._utils import maybe_transform, async_maybe_transform
|
||||
from ..._compat import cached_property
|
||||
from .runs.runs import (
|
||||
Runs,
|
||||
AsyncRuns,
|
||||
RunsWithRawResponse,
|
||||
AsyncRunsWithRawResponse,
|
||||
RunsWithStreamingResponse,
|
||||
AsyncRunsWithStreamingResponse,
|
||||
)
|
||||
from ..._resource import SyncAPIResource, AsyncAPIResource
|
||||
from ..._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper
|
||||
from ...pagination import SyncCursorPage, AsyncCursorPage
|
||||
from ..._base_client import AsyncPaginator, make_request_options
|
||||
from ...types.eval_list_response import EvalListResponse
|
||||
from ...types.eval_create_response import EvalCreateResponse
|
||||
from ...types.eval_delete_response import EvalDeleteResponse
|
||||
from ...types.eval_update_response import EvalUpdateResponse
|
||||
from ...types.eval_retrieve_response import EvalRetrieveResponse
|
||||
from ...types.shared_params.metadata import Metadata
|
||||
|
||||
__all__ = ["Evals", "AsyncEvals"]
|
||||
|
||||
|
||||
class Evals(SyncAPIResource):
|
||||
@cached_property
|
||||
def runs(self) -> Runs:
|
||||
return Runs(self._client)
|
||||
|
||||
@cached_property
|
||||
def with_raw_response(self) -> EvalsWithRawResponse:
|
||||
"""
|
||||
This property can be used as a prefix for any HTTP method call to return
|
||||
the raw response object instead of the parsed content.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers
|
||||
"""
|
||||
return EvalsWithRawResponse(self)
|
||||
|
||||
@cached_property
|
||||
def with_streaming_response(self) -> EvalsWithStreamingResponse:
|
||||
"""
|
||||
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#with_streaming_response
|
||||
"""
|
||||
return EvalsWithStreamingResponse(self)
|
||||
|
||||
def create(
|
||||
self,
|
||||
*,
|
||||
data_source_config: eval_create_params.DataSourceConfig,
|
||||
testing_criteria: Iterable[eval_create_params.TestingCriterion],
|
||||
metadata: Optional[Metadata] | Omit = omit,
|
||||
name: str | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> EvalCreateResponse:
|
||||
"""
|
||||
Create the structure of an evaluation that can be used to test a model's
|
||||
performance. An evaluation is a set of testing criteria and the config for a
|
||||
data source, which dictates the schema of the data used in the evaluation. After
|
||||
creating an evaluation, you can run it on different models and model parameters.
|
||||
We support several types of graders and datasources. For more information, see
|
||||
the [Evals guide](https://platform.openai.com/docs/guides/evals).
|
||||
|
||||
Args:
|
||||
data_source_config: The configuration for the data source used for the evaluation runs. Dictates the
|
||||
schema of the data used in the evaluation.
|
||||
|
||||
testing_criteria: A list of graders for all eval runs in this group. Graders can reference
|
||||
variables in the data source using double curly braces notation, like
|
||||
`{{item.variable_name}}`. To reference the model's output, use the `sample`
|
||||
namespace (ie, `{{sample.output_text}}`).
|
||||
|
||||
metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful
|
||||
for storing additional information about the object in a structured format, and
|
||||
querying for objects via API or the dashboard.
|
||||
|
||||
Keys are strings with a maximum length of 64 characters. Values are strings with
|
||||
a maximum length of 512 characters.
|
||||
|
||||
name: The name of the evaluation.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
return self._post(
|
||||
"/evals",
|
||||
body=maybe_transform(
|
||||
{
|
||||
"data_source_config": data_source_config,
|
||||
"testing_criteria": testing_criteria,
|
||||
"metadata": metadata,
|
||||
"name": name,
|
||||
},
|
||||
eval_create_params.EvalCreateParams,
|
||||
),
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=EvalCreateResponse,
|
||||
)
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> EvalRetrieveResponse:
|
||||
"""
|
||||
Get an evaluation by ID.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return self._get(
|
||||
f"/evals/{eval_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=EvalRetrieveResponse,
|
||||
)
|
||||
|
||||
def update(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
metadata: Optional[Metadata] | Omit = omit,
|
||||
name: str | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> EvalUpdateResponse:
|
||||
"""
|
||||
Update certain properties of an evaluation.
|
||||
|
||||
Args:
|
||||
metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful
|
||||
for storing additional information about the object in a structured format, and
|
||||
querying for objects via API or the dashboard.
|
||||
|
||||
Keys are strings with a maximum length of 64 characters. Values are strings with
|
||||
a maximum length of 512 characters.
|
||||
|
||||
name: Rename the evaluation.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return self._post(
|
||||
f"/evals/{eval_id}",
|
||||
body=maybe_transform(
|
||||
{
|
||||
"metadata": metadata,
|
||||
"name": name,
|
||||
},
|
||||
eval_update_params.EvalUpdateParams,
|
||||
),
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=EvalUpdateResponse,
|
||||
)
|
||||
|
||||
def list(
|
||||
self,
|
||||
*,
|
||||
after: str | Omit = omit,
|
||||
limit: int | Omit = omit,
|
||||
order: Literal["asc", "desc"] | Omit = omit,
|
||||
order_by: Literal["created_at", "updated_at"] | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> SyncCursorPage[EvalListResponse]:
|
||||
"""
|
||||
List evaluations for a project.
|
||||
|
||||
Args:
|
||||
after: Identifier for the last eval from the previous pagination request.
|
||||
|
||||
limit: Number of evals to retrieve.
|
||||
|
||||
order: Sort order for evals by timestamp. Use `asc` for ascending order or `desc` for
|
||||
descending order.
|
||||
|
||||
order_by: Evals can be ordered by creation time or last updated time. Use `created_at` for
|
||||
creation time or `updated_at` for last updated time.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
return self._get_api_list(
|
||||
"/evals",
|
||||
page=SyncCursorPage[EvalListResponse],
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
query=maybe_transform(
|
||||
{
|
||||
"after": after,
|
||||
"limit": limit,
|
||||
"order": order,
|
||||
"order_by": order_by,
|
||||
},
|
||||
eval_list_params.EvalListParams,
|
||||
),
|
||||
),
|
||||
model=EvalListResponse,
|
||||
)
|
||||
|
||||
def delete(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> EvalDeleteResponse:
|
||||
"""
|
||||
Delete an evaluation.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return self._delete(
|
||||
f"/evals/{eval_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=EvalDeleteResponse,
|
||||
)
|
||||
|
||||
|
||||
class AsyncEvals(AsyncAPIResource):
|
||||
@cached_property
|
||||
def runs(self) -> AsyncRuns:
|
||||
return AsyncRuns(self._client)
|
||||
|
||||
@cached_property
|
||||
def with_raw_response(self) -> AsyncEvalsWithRawResponse:
|
||||
"""
|
||||
This property can be used as a prefix for any HTTP method call to return
|
||||
the raw response object instead of the parsed content.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers
|
||||
"""
|
||||
return AsyncEvalsWithRawResponse(self)
|
||||
|
||||
@cached_property
|
||||
def with_streaming_response(self) -> AsyncEvalsWithStreamingResponse:
|
||||
"""
|
||||
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#with_streaming_response
|
||||
"""
|
||||
return AsyncEvalsWithStreamingResponse(self)
|
||||
|
||||
async def create(
|
||||
self,
|
||||
*,
|
||||
data_source_config: eval_create_params.DataSourceConfig,
|
||||
testing_criteria: Iterable[eval_create_params.TestingCriterion],
|
||||
metadata: Optional[Metadata] | Omit = omit,
|
||||
name: str | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> EvalCreateResponse:
|
||||
"""
|
||||
Create the structure of an evaluation that can be used to test a model's
|
||||
performance. An evaluation is a set of testing criteria and the config for a
|
||||
data source, which dictates the schema of the data used in the evaluation. After
|
||||
creating an evaluation, you can run it on different models and model parameters.
|
||||
We support several types of graders and datasources. For more information, see
|
||||
the [Evals guide](https://platform.openai.com/docs/guides/evals).
|
||||
|
||||
Args:
|
||||
data_source_config: The configuration for the data source used for the evaluation runs. Dictates the
|
||||
schema of the data used in the evaluation.
|
||||
|
||||
testing_criteria: A list of graders for all eval runs in this group. Graders can reference
|
||||
variables in the data source using double curly braces notation, like
|
||||
`{{item.variable_name}}`. To reference the model's output, use the `sample`
|
||||
namespace (ie, `{{sample.output_text}}`).
|
||||
|
||||
metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful
|
||||
for storing additional information about the object in a structured format, and
|
||||
querying for objects via API or the dashboard.
|
||||
|
||||
Keys are strings with a maximum length of 64 characters. Values are strings with
|
||||
a maximum length of 512 characters.
|
||||
|
||||
name: The name of the evaluation.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
return await self._post(
|
||||
"/evals",
|
||||
body=await async_maybe_transform(
|
||||
{
|
||||
"data_source_config": data_source_config,
|
||||
"testing_criteria": testing_criteria,
|
||||
"metadata": metadata,
|
||||
"name": name,
|
||||
},
|
||||
eval_create_params.EvalCreateParams,
|
||||
),
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=EvalCreateResponse,
|
||||
)
|
||||
|
||||
async def retrieve(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> EvalRetrieveResponse:
|
||||
"""
|
||||
Get an evaluation by ID.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return await self._get(
|
||||
f"/evals/{eval_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=EvalRetrieveResponse,
|
||||
)
|
||||
|
||||
async def update(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
metadata: Optional[Metadata] | Omit = omit,
|
||||
name: str | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> EvalUpdateResponse:
|
||||
"""
|
||||
Update certain properties of an evaluation.
|
||||
|
||||
Args:
|
||||
metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful
|
||||
for storing additional information about the object in a structured format, and
|
||||
querying for objects via API or the dashboard.
|
||||
|
||||
Keys are strings with a maximum length of 64 characters. Values are strings with
|
||||
a maximum length of 512 characters.
|
||||
|
||||
name: Rename the evaluation.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return await self._post(
|
||||
f"/evals/{eval_id}",
|
||||
body=await async_maybe_transform(
|
||||
{
|
||||
"metadata": metadata,
|
||||
"name": name,
|
||||
},
|
||||
eval_update_params.EvalUpdateParams,
|
||||
),
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=EvalUpdateResponse,
|
||||
)
|
||||
|
||||
def list(
|
||||
self,
|
||||
*,
|
||||
after: str | Omit = omit,
|
||||
limit: int | Omit = omit,
|
||||
order: Literal["asc", "desc"] | Omit = omit,
|
||||
order_by: Literal["created_at", "updated_at"] | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> AsyncPaginator[EvalListResponse, AsyncCursorPage[EvalListResponse]]:
|
||||
"""
|
||||
List evaluations for a project.
|
||||
|
||||
Args:
|
||||
after: Identifier for the last eval from the previous pagination request.
|
||||
|
||||
limit: Number of evals to retrieve.
|
||||
|
||||
order: Sort order for evals by timestamp. Use `asc` for ascending order or `desc` for
|
||||
descending order.
|
||||
|
||||
order_by: Evals can be ordered by creation time or last updated time. Use `created_at` for
|
||||
creation time or `updated_at` for last updated time.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
return self._get_api_list(
|
||||
"/evals",
|
||||
page=AsyncCursorPage[EvalListResponse],
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
query=maybe_transform(
|
||||
{
|
||||
"after": after,
|
||||
"limit": limit,
|
||||
"order": order,
|
||||
"order_by": order_by,
|
||||
},
|
||||
eval_list_params.EvalListParams,
|
||||
),
|
||||
),
|
||||
model=EvalListResponse,
|
||||
)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> EvalDeleteResponse:
|
||||
"""
|
||||
Delete an evaluation.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return await self._delete(
|
||||
f"/evals/{eval_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=EvalDeleteResponse,
|
||||
)
|
||||
|
||||
|
||||
class EvalsWithRawResponse:
|
||||
def __init__(self, evals: Evals) -> None:
|
||||
self._evals = evals
|
||||
|
||||
self.create = _legacy_response.to_raw_response_wrapper(
|
||||
evals.create,
|
||||
)
|
||||
self.retrieve = _legacy_response.to_raw_response_wrapper(
|
||||
evals.retrieve,
|
||||
)
|
||||
self.update = _legacy_response.to_raw_response_wrapper(
|
||||
evals.update,
|
||||
)
|
||||
self.list = _legacy_response.to_raw_response_wrapper(
|
||||
evals.list,
|
||||
)
|
||||
self.delete = _legacy_response.to_raw_response_wrapper(
|
||||
evals.delete,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def runs(self) -> RunsWithRawResponse:
|
||||
return RunsWithRawResponse(self._evals.runs)
|
||||
|
||||
|
||||
class AsyncEvalsWithRawResponse:
|
||||
def __init__(self, evals: AsyncEvals) -> None:
|
||||
self._evals = evals
|
||||
|
||||
self.create = _legacy_response.async_to_raw_response_wrapper(
|
||||
evals.create,
|
||||
)
|
||||
self.retrieve = _legacy_response.async_to_raw_response_wrapper(
|
||||
evals.retrieve,
|
||||
)
|
||||
self.update = _legacy_response.async_to_raw_response_wrapper(
|
||||
evals.update,
|
||||
)
|
||||
self.list = _legacy_response.async_to_raw_response_wrapper(
|
||||
evals.list,
|
||||
)
|
||||
self.delete = _legacy_response.async_to_raw_response_wrapper(
|
||||
evals.delete,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def runs(self) -> AsyncRunsWithRawResponse:
|
||||
return AsyncRunsWithRawResponse(self._evals.runs)
|
||||
|
||||
|
||||
class EvalsWithStreamingResponse:
|
||||
def __init__(self, evals: Evals) -> None:
|
||||
self._evals = evals
|
||||
|
||||
self.create = to_streamed_response_wrapper(
|
||||
evals.create,
|
||||
)
|
||||
self.retrieve = to_streamed_response_wrapper(
|
||||
evals.retrieve,
|
||||
)
|
||||
self.update = to_streamed_response_wrapper(
|
||||
evals.update,
|
||||
)
|
||||
self.list = to_streamed_response_wrapper(
|
||||
evals.list,
|
||||
)
|
||||
self.delete = to_streamed_response_wrapper(
|
||||
evals.delete,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def runs(self) -> RunsWithStreamingResponse:
|
||||
return RunsWithStreamingResponse(self._evals.runs)
|
||||
|
||||
|
||||
class AsyncEvalsWithStreamingResponse:
|
||||
def __init__(self, evals: AsyncEvals) -> None:
|
||||
self._evals = evals
|
||||
|
||||
self.create = async_to_streamed_response_wrapper(
|
||||
evals.create,
|
||||
)
|
||||
self.retrieve = async_to_streamed_response_wrapper(
|
||||
evals.retrieve,
|
||||
)
|
||||
self.update = async_to_streamed_response_wrapper(
|
||||
evals.update,
|
||||
)
|
||||
self.list = async_to_streamed_response_wrapper(
|
||||
evals.list,
|
||||
)
|
||||
self.delete = async_to_streamed_response_wrapper(
|
||||
evals.delete,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def runs(self) -> AsyncRunsWithStreamingResponse:
|
||||
return AsyncRunsWithStreamingResponse(self._evals.runs)
|
||||
@@ -0,0 +1,33 @@
|
||||
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
||||
|
||||
from .runs import (
|
||||
Runs,
|
||||
AsyncRuns,
|
||||
RunsWithRawResponse,
|
||||
AsyncRunsWithRawResponse,
|
||||
RunsWithStreamingResponse,
|
||||
AsyncRunsWithStreamingResponse,
|
||||
)
|
||||
from .output_items import (
|
||||
OutputItems,
|
||||
AsyncOutputItems,
|
||||
OutputItemsWithRawResponse,
|
||||
AsyncOutputItemsWithRawResponse,
|
||||
OutputItemsWithStreamingResponse,
|
||||
AsyncOutputItemsWithStreamingResponse,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"OutputItems",
|
||||
"AsyncOutputItems",
|
||||
"OutputItemsWithRawResponse",
|
||||
"AsyncOutputItemsWithRawResponse",
|
||||
"OutputItemsWithStreamingResponse",
|
||||
"AsyncOutputItemsWithStreamingResponse",
|
||||
"Runs",
|
||||
"AsyncRuns",
|
||||
"RunsWithRawResponse",
|
||||
"AsyncRunsWithRawResponse",
|
||||
"RunsWithStreamingResponse",
|
||||
"AsyncRunsWithStreamingResponse",
|
||||
]
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,315 @@
|
||||
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing_extensions import Literal
|
||||
|
||||
import httpx
|
||||
|
||||
from .... import _legacy_response
|
||||
from ...._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
|
||||
from ...._utils import maybe_transform
|
||||
from ...._compat import cached_property
|
||||
from ...._resource import SyncAPIResource, AsyncAPIResource
|
||||
from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper
|
||||
from ....pagination import SyncCursorPage, AsyncCursorPage
|
||||
from ...._base_client import AsyncPaginator, make_request_options
|
||||
from ....types.evals.runs import output_item_list_params
|
||||
from ....types.evals.runs.output_item_list_response import OutputItemListResponse
|
||||
from ....types.evals.runs.output_item_retrieve_response import OutputItemRetrieveResponse
|
||||
|
||||
__all__ = ["OutputItems", "AsyncOutputItems"]
|
||||
|
||||
|
||||
class OutputItems(SyncAPIResource):
|
||||
@cached_property
|
||||
def with_raw_response(self) -> OutputItemsWithRawResponse:
|
||||
"""
|
||||
This property can be used as a prefix for any HTTP method call to return
|
||||
the raw response object instead of the parsed content.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers
|
||||
"""
|
||||
return OutputItemsWithRawResponse(self)
|
||||
|
||||
@cached_property
|
||||
def with_streaming_response(self) -> OutputItemsWithStreamingResponse:
|
||||
"""
|
||||
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#with_streaming_response
|
||||
"""
|
||||
return OutputItemsWithStreamingResponse(self)
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
output_item_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> OutputItemRetrieveResponse:
|
||||
"""
|
||||
Get an evaluation run output item by ID.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
if not output_item_id:
|
||||
raise ValueError(f"Expected a non-empty value for `output_item_id` but received {output_item_id!r}")
|
||||
return self._get(
|
||||
f"/evals/{eval_id}/runs/{run_id}/output_items/{output_item_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=OutputItemRetrieveResponse,
|
||||
)
|
||||
|
||||
def list(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
after: str | Omit = omit,
|
||||
limit: int | Omit = omit,
|
||||
order: Literal["asc", "desc"] | Omit = omit,
|
||||
status: Literal["fail", "pass"] | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> SyncCursorPage[OutputItemListResponse]:
|
||||
"""
|
||||
Get a list of output items for an evaluation run.
|
||||
|
||||
Args:
|
||||
after: Identifier for the last output item from the previous pagination request.
|
||||
|
||||
limit: Number of output items to retrieve.
|
||||
|
||||
order: Sort order for output items by timestamp. Use `asc` for ascending order or
|
||||
`desc` for descending order. Defaults to `asc`.
|
||||
|
||||
status: Filter output items by status. Use `failed` to filter by failed output items or
|
||||
`pass` to filter by passed output items.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
return self._get_api_list(
|
||||
f"/evals/{eval_id}/runs/{run_id}/output_items",
|
||||
page=SyncCursorPage[OutputItemListResponse],
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
query=maybe_transform(
|
||||
{
|
||||
"after": after,
|
||||
"limit": limit,
|
||||
"order": order,
|
||||
"status": status,
|
||||
},
|
||||
output_item_list_params.OutputItemListParams,
|
||||
),
|
||||
),
|
||||
model=OutputItemListResponse,
|
||||
)
|
||||
|
||||
|
||||
class AsyncOutputItems(AsyncAPIResource):
|
||||
@cached_property
|
||||
def with_raw_response(self) -> AsyncOutputItemsWithRawResponse:
|
||||
"""
|
||||
This property can be used as a prefix for any HTTP method call to return
|
||||
the raw response object instead of the parsed content.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers
|
||||
"""
|
||||
return AsyncOutputItemsWithRawResponse(self)
|
||||
|
||||
@cached_property
|
||||
def with_streaming_response(self) -> AsyncOutputItemsWithStreamingResponse:
|
||||
"""
|
||||
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#with_streaming_response
|
||||
"""
|
||||
return AsyncOutputItemsWithStreamingResponse(self)
|
||||
|
||||
async def retrieve(
|
||||
self,
|
||||
output_item_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
run_id: str,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> OutputItemRetrieveResponse:
|
||||
"""
|
||||
Get an evaluation run output item by ID.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
if not output_item_id:
|
||||
raise ValueError(f"Expected a non-empty value for `output_item_id` but received {output_item_id!r}")
|
||||
return await self._get(
|
||||
f"/evals/{eval_id}/runs/{run_id}/output_items/{output_item_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=OutputItemRetrieveResponse,
|
||||
)
|
||||
|
||||
def list(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
after: str | Omit = omit,
|
||||
limit: int | Omit = omit,
|
||||
order: Literal["asc", "desc"] | Omit = omit,
|
||||
status: Literal["fail", "pass"] | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> AsyncPaginator[OutputItemListResponse, AsyncCursorPage[OutputItemListResponse]]:
|
||||
"""
|
||||
Get a list of output items for an evaluation run.
|
||||
|
||||
Args:
|
||||
after: Identifier for the last output item from the previous pagination request.
|
||||
|
||||
limit: Number of output items to retrieve.
|
||||
|
||||
order: Sort order for output items by timestamp. Use `asc` for ascending order or
|
||||
`desc` for descending order. Defaults to `asc`.
|
||||
|
||||
status: Filter output items by status. Use `failed` to filter by failed output items or
|
||||
`pass` to filter by passed output items.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
return self._get_api_list(
|
||||
f"/evals/{eval_id}/runs/{run_id}/output_items",
|
||||
page=AsyncCursorPage[OutputItemListResponse],
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
query=maybe_transform(
|
||||
{
|
||||
"after": after,
|
||||
"limit": limit,
|
||||
"order": order,
|
||||
"status": status,
|
||||
},
|
||||
output_item_list_params.OutputItemListParams,
|
||||
),
|
||||
),
|
||||
model=OutputItemListResponse,
|
||||
)
|
||||
|
||||
|
||||
class OutputItemsWithRawResponse:
|
||||
def __init__(self, output_items: OutputItems) -> None:
|
||||
self._output_items = output_items
|
||||
|
||||
self.retrieve = _legacy_response.to_raw_response_wrapper(
|
||||
output_items.retrieve,
|
||||
)
|
||||
self.list = _legacy_response.to_raw_response_wrapper(
|
||||
output_items.list,
|
||||
)
|
||||
|
||||
|
||||
class AsyncOutputItemsWithRawResponse:
|
||||
def __init__(self, output_items: AsyncOutputItems) -> None:
|
||||
self._output_items = output_items
|
||||
|
||||
self.retrieve = _legacy_response.async_to_raw_response_wrapper(
|
||||
output_items.retrieve,
|
||||
)
|
||||
self.list = _legacy_response.async_to_raw_response_wrapper(
|
||||
output_items.list,
|
||||
)
|
||||
|
||||
|
||||
class OutputItemsWithStreamingResponse:
|
||||
def __init__(self, output_items: OutputItems) -> None:
|
||||
self._output_items = output_items
|
||||
|
||||
self.retrieve = to_streamed_response_wrapper(
|
||||
output_items.retrieve,
|
||||
)
|
||||
self.list = to_streamed_response_wrapper(
|
||||
output_items.list,
|
||||
)
|
||||
|
||||
|
||||
class AsyncOutputItemsWithStreamingResponse:
|
||||
def __init__(self, output_items: AsyncOutputItems) -> None:
|
||||
self._output_items = output_items
|
||||
|
||||
self.retrieve = async_to_streamed_response_wrapper(
|
||||
output_items.retrieve,
|
||||
)
|
||||
self.list = async_to_streamed_response_wrapper(
|
||||
output_items.list,
|
||||
)
|
||||
@@ -0,0 +1,634 @@
|
||||
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
from typing_extensions import Literal
|
||||
|
||||
import httpx
|
||||
|
||||
from .... import _legacy_response
|
||||
from ...._types import Body, Omit, Query, Headers, NotGiven, omit, not_given
|
||||
from ...._utils import maybe_transform, async_maybe_transform
|
||||
from ...._compat import cached_property
|
||||
from ...._resource import SyncAPIResource, AsyncAPIResource
|
||||
from ...._response import to_streamed_response_wrapper, async_to_streamed_response_wrapper
|
||||
from .output_items import (
|
||||
OutputItems,
|
||||
AsyncOutputItems,
|
||||
OutputItemsWithRawResponse,
|
||||
AsyncOutputItemsWithRawResponse,
|
||||
OutputItemsWithStreamingResponse,
|
||||
AsyncOutputItemsWithStreamingResponse,
|
||||
)
|
||||
from ....pagination import SyncCursorPage, AsyncCursorPage
|
||||
from ....types.evals import run_list_params, run_create_params
|
||||
from ...._base_client import AsyncPaginator, make_request_options
|
||||
from ....types.shared_params.metadata import Metadata
|
||||
from ....types.evals.run_list_response import RunListResponse
|
||||
from ....types.evals.run_cancel_response import RunCancelResponse
|
||||
from ....types.evals.run_create_response import RunCreateResponse
|
||||
from ....types.evals.run_delete_response import RunDeleteResponse
|
||||
from ....types.evals.run_retrieve_response import RunRetrieveResponse
|
||||
|
||||
__all__ = ["Runs", "AsyncRuns"]
|
||||
|
||||
|
||||
class Runs(SyncAPIResource):
|
||||
@cached_property
|
||||
def output_items(self) -> OutputItems:
|
||||
return OutputItems(self._client)
|
||||
|
||||
@cached_property
|
||||
def with_raw_response(self) -> RunsWithRawResponse:
|
||||
"""
|
||||
This property can be used as a prefix for any HTTP method call to return
|
||||
the raw response object instead of the parsed content.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers
|
||||
"""
|
||||
return RunsWithRawResponse(self)
|
||||
|
||||
@cached_property
|
||||
def with_streaming_response(self) -> RunsWithStreamingResponse:
|
||||
"""
|
||||
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#with_streaming_response
|
||||
"""
|
||||
return RunsWithStreamingResponse(self)
|
||||
|
||||
def create(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
data_source: run_create_params.DataSource,
|
||||
metadata: Optional[Metadata] | Omit = omit,
|
||||
name: str | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> RunCreateResponse:
|
||||
"""
|
||||
Kicks off a new run for a given evaluation, specifying the data source, and what
|
||||
model configuration to use to test. The datasource will be validated against the
|
||||
schema specified in the config of the evaluation.
|
||||
|
||||
Args:
|
||||
data_source: Details about the run's data source.
|
||||
|
||||
metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful
|
||||
for storing additional information about the object in a structured format, and
|
||||
querying for objects via API or the dashboard.
|
||||
|
||||
Keys are strings with a maximum length of 64 characters. Values are strings with
|
||||
a maximum length of 512 characters.
|
||||
|
||||
name: The name of the run.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return self._post(
|
||||
f"/evals/{eval_id}/runs",
|
||||
body=maybe_transform(
|
||||
{
|
||||
"data_source": data_source,
|
||||
"metadata": metadata,
|
||||
"name": name,
|
||||
},
|
||||
run_create_params.RunCreateParams,
|
||||
),
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=RunCreateResponse,
|
||||
)
|
||||
|
||||
def retrieve(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> RunRetrieveResponse:
|
||||
"""
|
||||
Get an evaluation run by ID.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
return self._get(
|
||||
f"/evals/{eval_id}/runs/{run_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=RunRetrieveResponse,
|
||||
)
|
||||
|
||||
def list(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
after: str | Omit = omit,
|
||||
limit: int | Omit = omit,
|
||||
order: Literal["asc", "desc"] | Omit = omit,
|
||||
status: Literal["queued", "in_progress", "completed", "canceled", "failed"] | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> SyncCursorPage[RunListResponse]:
|
||||
"""
|
||||
Get a list of runs for an evaluation.
|
||||
|
||||
Args:
|
||||
after: Identifier for the last run from the previous pagination request.
|
||||
|
||||
limit: Number of runs to retrieve.
|
||||
|
||||
order: Sort order for runs by timestamp. Use `asc` for ascending order or `desc` for
|
||||
descending order. Defaults to `asc`.
|
||||
|
||||
status: Filter runs by status. One of `queued` | `in_progress` | `failed` | `completed`
|
||||
| `canceled`.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return self._get_api_list(
|
||||
f"/evals/{eval_id}/runs",
|
||||
page=SyncCursorPage[RunListResponse],
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
query=maybe_transform(
|
||||
{
|
||||
"after": after,
|
||||
"limit": limit,
|
||||
"order": order,
|
||||
"status": status,
|
||||
},
|
||||
run_list_params.RunListParams,
|
||||
),
|
||||
),
|
||||
model=RunListResponse,
|
||||
)
|
||||
|
||||
def delete(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> RunDeleteResponse:
|
||||
"""
|
||||
Delete an eval run.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
return self._delete(
|
||||
f"/evals/{eval_id}/runs/{run_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=RunDeleteResponse,
|
||||
)
|
||||
|
||||
def cancel(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> RunCancelResponse:
|
||||
"""
|
||||
Cancel an ongoing evaluation run.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
return self._post(
|
||||
f"/evals/{eval_id}/runs/{run_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=RunCancelResponse,
|
||||
)
|
||||
|
||||
|
||||
class AsyncRuns(AsyncAPIResource):
|
||||
@cached_property
|
||||
def output_items(self) -> AsyncOutputItems:
|
||||
return AsyncOutputItems(self._client)
|
||||
|
||||
@cached_property
|
||||
def with_raw_response(self) -> AsyncRunsWithRawResponse:
|
||||
"""
|
||||
This property can be used as a prefix for any HTTP method call to return
|
||||
the raw response object instead of the parsed content.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#accessing-raw-response-data-eg-headers
|
||||
"""
|
||||
return AsyncRunsWithRawResponse(self)
|
||||
|
||||
@cached_property
|
||||
def with_streaming_response(self) -> AsyncRunsWithStreamingResponse:
|
||||
"""
|
||||
An alternative to `.with_raw_response` that doesn't eagerly read the response body.
|
||||
|
||||
For more information, see https://www.github.com/openai/openai-python#with_streaming_response
|
||||
"""
|
||||
return AsyncRunsWithStreamingResponse(self)
|
||||
|
||||
async def create(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
data_source: run_create_params.DataSource,
|
||||
metadata: Optional[Metadata] | Omit = omit,
|
||||
name: str | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> RunCreateResponse:
|
||||
"""
|
||||
Kicks off a new run for a given evaluation, specifying the data source, and what
|
||||
model configuration to use to test. The datasource will be validated against the
|
||||
schema specified in the config of the evaluation.
|
||||
|
||||
Args:
|
||||
data_source: Details about the run's data source.
|
||||
|
||||
metadata: Set of 16 key-value pairs that can be attached to an object. This can be useful
|
||||
for storing additional information about the object in a structured format, and
|
||||
querying for objects via API or the dashboard.
|
||||
|
||||
Keys are strings with a maximum length of 64 characters. Values are strings with
|
||||
a maximum length of 512 characters.
|
||||
|
||||
name: The name of the run.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return await self._post(
|
||||
f"/evals/{eval_id}/runs",
|
||||
body=await async_maybe_transform(
|
||||
{
|
||||
"data_source": data_source,
|
||||
"metadata": metadata,
|
||||
"name": name,
|
||||
},
|
||||
run_create_params.RunCreateParams,
|
||||
),
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=RunCreateResponse,
|
||||
)
|
||||
|
||||
async def retrieve(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> RunRetrieveResponse:
|
||||
"""
|
||||
Get an evaluation run by ID.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
return await self._get(
|
||||
f"/evals/{eval_id}/runs/{run_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=RunRetrieveResponse,
|
||||
)
|
||||
|
||||
def list(
|
||||
self,
|
||||
eval_id: str,
|
||||
*,
|
||||
after: str | Omit = omit,
|
||||
limit: int | Omit = omit,
|
||||
order: Literal["asc", "desc"] | Omit = omit,
|
||||
status: Literal["queued", "in_progress", "completed", "canceled", "failed"] | Omit = omit,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> AsyncPaginator[RunListResponse, AsyncCursorPage[RunListResponse]]:
|
||||
"""
|
||||
Get a list of runs for an evaluation.
|
||||
|
||||
Args:
|
||||
after: Identifier for the last run from the previous pagination request.
|
||||
|
||||
limit: Number of runs to retrieve.
|
||||
|
||||
order: Sort order for runs by timestamp. Use `asc` for ascending order or `desc` for
|
||||
descending order. Defaults to `asc`.
|
||||
|
||||
status: Filter runs by status. One of `queued` | `in_progress` | `failed` | `completed`
|
||||
| `canceled`.
|
||||
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
return self._get_api_list(
|
||||
f"/evals/{eval_id}/runs",
|
||||
page=AsyncCursorPage[RunListResponse],
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers,
|
||||
extra_query=extra_query,
|
||||
extra_body=extra_body,
|
||||
timeout=timeout,
|
||||
query=maybe_transform(
|
||||
{
|
||||
"after": after,
|
||||
"limit": limit,
|
||||
"order": order,
|
||||
"status": status,
|
||||
},
|
||||
run_list_params.RunListParams,
|
||||
),
|
||||
),
|
||||
model=RunListResponse,
|
||||
)
|
||||
|
||||
async def delete(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> RunDeleteResponse:
|
||||
"""
|
||||
Delete an eval run.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
return await self._delete(
|
||||
f"/evals/{eval_id}/runs/{run_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=RunDeleteResponse,
|
||||
)
|
||||
|
||||
async def cancel(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
eval_id: str,
|
||||
# Use the following arguments if you need to pass additional parameters to the API that aren't available via kwargs.
|
||||
# The extra values given here take precedence over values defined on the client or passed to this method.
|
||||
extra_headers: Headers | None = None,
|
||||
extra_query: Query | None = None,
|
||||
extra_body: Body | None = None,
|
||||
timeout: float | httpx.Timeout | None | NotGiven = not_given,
|
||||
) -> RunCancelResponse:
|
||||
"""
|
||||
Cancel an ongoing evaluation run.
|
||||
|
||||
Args:
|
||||
extra_headers: Send extra headers
|
||||
|
||||
extra_query: Add additional query parameters to the request
|
||||
|
||||
extra_body: Add additional JSON properties to the request
|
||||
|
||||
timeout: Override the client-level default timeout for this request, in seconds
|
||||
"""
|
||||
if not eval_id:
|
||||
raise ValueError(f"Expected a non-empty value for `eval_id` but received {eval_id!r}")
|
||||
if not run_id:
|
||||
raise ValueError(f"Expected a non-empty value for `run_id` but received {run_id!r}")
|
||||
return await self._post(
|
||||
f"/evals/{eval_id}/runs/{run_id}",
|
||||
options=make_request_options(
|
||||
extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, timeout=timeout
|
||||
),
|
||||
cast_to=RunCancelResponse,
|
||||
)
|
||||
|
||||
|
||||
class RunsWithRawResponse:
|
||||
def __init__(self, runs: Runs) -> None:
|
||||
self._runs = runs
|
||||
|
||||
self.create = _legacy_response.to_raw_response_wrapper(
|
||||
runs.create,
|
||||
)
|
||||
self.retrieve = _legacy_response.to_raw_response_wrapper(
|
||||
runs.retrieve,
|
||||
)
|
||||
self.list = _legacy_response.to_raw_response_wrapper(
|
||||
runs.list,
|
||||
)
|
||||
self.delete = _legacy_response.to_raw_response_wrapper(
|
||||
runs.delete,
|
||||
)
|
||||
self.cancel = _legacy_response.to_raw_response_wrapper(
|
||||
runs.cancel,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def output_items(self) -> OutputItemsWithRawResponse:
|
||||
return OutputItemsWithRawResponse(self._runs.output_items)
|
||||
|
||||
|
||||
class AsyncRunsWithRawResponse:
|
||||
def __init__(self, runs: AsyncRuns) -> None:
|
||||
self._runs = runs
|
||||
|
||||
self.create = _legacy_response.async_to_raw_response_wrapper(
|
||||
runs.create,
|
||||
)
|
||||
self.retrieve = _legacy_response.async_to_raw_response_wrapper(
|
||||
runs.retrieve,
|
||||
)
|
||||
self.list = _legacy_response.async_to_raw_response_wrapper(
|
||||
runs.list,
|
||||
)
|
||||
self.delete = _legacy_response.async_to_raw_response_wrapper(
|
||||
runs.delete,
|
||||
)
|
||||
self.cancel = _legacy_response.async_to_raw_response_wrapper(
|
||||
runs.cancel,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def output_items(self) -> AsyncOutputItemsWithRawResponse:
|
||||
return AsyncOutputItemsWithRawResponse(self._runs.output_items)
|
||||
|
||||
|
||||
class RunsWithStreamingResponse:
|
||||
def __init__(self, runs: Runs) -> None:
|
||||
self._runs = runs
|
||||
|
||||
self.create = to_streamed_response_wrapper(
|
||||
runs.create,
|
||||
)
|
||||
self.retrieve = to_streamed_response_wrapper(
|
||||
runs.retrieve,
|
||||
)
|
||||
self.list = to_streamed_response_wrapper(
|
||||
runs.list,
|
||||
)
|
||||
self.delete = to_streamed_response_wrapper(
|
||||
runs.delete,
|
||||
)
|
||||
self.cancel = to_streamed_response_wrapper(
|
||||
runs.cancel,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def output_items(self) -> OutputItemsWithStreamingResponse:
|
||||
return OutputItemsWithStreamingResponse(self._runs.output_items)
|
||||
|
||||
|
||||
class AsyncRunsWithStreamingResponse:
|
||||
def __init__(self, runs: AsyncRuns) -> None:
|
||||
self._runs = runs
|
||||
|
||||
self.create = async_to_streamed_response_wrapper(
|
||||
runs.create,
|
||||
)
|
||||
self.retrieve = async_to_streamed_response_wrapper(
|
||||
runs.retrieve,
|
||||
)
|
||||
self.list = async_to_streamed_response_wrapper(
|
||||
runs.list,
|
||||
)
|
||||
self.delete = async_to_streamed_response_wrapper(
|
||||
runs.delete,
|
||||
)
|
||||
self.cancel = async_to_streamed_response_wrapper(
|
||||
runs.cancel,
|
||||
)
|
||||
|
||||
@cached_property
|
||||
def output_items(self) -> AsyncOutputItemsWithStreamingResponse:
|
||||
return AsyncOutputItemsWithStreamingResponse(self._runs.output_items)
|
||||
Reference in New Issue
Block a user