修改为东南天坐标系
This commit is contained in:
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -13,17 +13,19 @@
|
||||
# limitations under the License.
|
||||
"""Contains CLI utilities (styling, helpers)."""
|
||||
|
||||
import dataclasses
|
||||
import datetime
|
||||
import importlib.metadata
|
||||
import os
|
||||
import time
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Annotated, Optional
|
||||
from typing import TYPE_CHECKING, Annotated, Literal, Optional, Union
|
||||
|
||||
import click
|
||||
import typer
|
||||
|
||||
from huggingface_hub import __version__, constants
|
||||
from huggingface_hub import DatasetInfo, ModelInfo, SpaceInfo, __version__, constants
|
||||
from huggingface_hub.utils import ANSI, get_session, hf_raise_for_status, installation_method, logging
|
||||
|
||||
|
||||
@@ -110,28 +112,78 @@ RevisionOpt = Annotated[
|
||||
]
|
||||
|
||||
|
||||
LimitOpt = Annotated[
|
||||
int,
|
||||
typer.Option(help="Limit the number of results."),
|
||||
]
|
||||
|
||||
AuthorOpt = Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Filter by author or organization."),
|
||||
]
|
||||
|
||||
FilterOpt = Annotated[
|
||||
Optional[list[str]],
|
||||
typer.Option(help="Filter by tags (e.g. 'text-classification'). Can be used multiple times."),
|
||||
]
|
||||
|
||||
SearchOpt = Annotated[
|
||||
Optional[str],
|
||||
typer.Option(help="Search query."),
|
||||
]
|
||||
|
||||
|
||||
def repo_info_to_dict(info: Union[ModelInfo, DatasetInfo, SpaceInfo]) -> dict[str, object]:
|
||||
"""Convert repo info dataclasses to json-serializable dicts."""
|
||||
return {
|
||||
k: v.isoformat() if isinstance(v, datetime.datetime) else v
|
||||
for k, v in dataclasses.asdict(info).items()
|
||||
if v is not None
|
||||
}
|
||||
|
||||
|
||||
def make_expand_properties_parser(valid_properties: list[str]):
|
||||
"""Create a callback to parse and validate comma-separated expand properties."""
|
||||
|
||||
def _parse_expand_properties(value: Optional[str]) -> Optional[list[str]]:
|
||||
if value is None:
|
||||
return None
|
||||
properties = [p.strip() for p in value.split(",")]
|
||||
for prop in properties:
|
||||
if prop not in valid_properties:
|
||||
raise typer.BadParameter(
|
||||
f"Invalid expand property: '{prop}'. Valid values are: {', '.join(valid_properties)}"
|
||||
)
|
||||
return properties
|
||||
|
||||
return _parse_expand_properties
|
||||
|
||||
|
||||
### PyPI VERSION CHECKER
|
||||
|
||||
|
||||
def check_cli_update() -> None:
|
||||
def check_cli_update(library: Literal["huggingface_hub", "transformers"]) -> None:
|
||||
"""
|
||||
Check whether a newer version of `huggingface_hub` is available on PyPI.
|
||||
Check whether a newer version of a library is available on PyPI.
|
||||
|
||||
If a newer version is found, notify the user and suggest updating.
|
||||
If current version is a pre-release (e.g. `1.0.0.rc1`), or a dev version (e.g. `1.0.0.dev1`), no check is performed.
|
||||
|
||||
This function is called at the entry point of the CLI. It only performs the check once every 24 hours, and any error
|
||||
during the check is caught and logged, to avoid breaking the CLI.
|
||||
|
||||
Args:
|
||||
library: The library to check for updates. Currently supports "huggingface_hub" and "transformers".
|
||||
"""
|
||||
try:
|
||||
_check_cli_update()
|
||||
_check_cli_update(library)
|
||||
except Exception:
|
||||
# We don't want the CLI to fail on version checks, no matter the reason.
|
||||
logger.debug("Error while checking for CLI update.", exc_info=True)
|
||||
|
||||
|
||||
def _check_cli_update() -> None:
|
||||
current_version = importlib.metadata.version("huggingface_hub")
|
||||
def _check_cli_update(library: Literal["huggingface_hub", "transformers"]) -> None:
|
||||
current_version = importlib.metadata.version(library)
|
||||
|
||||
# Skip if current version is a pre-release or dev version
|
||||
if any(tag in current_version for tag in ["rc", "dev"]):
|
||||
@@ -148,27 +200,46 @@ def _check_cli_update() -> None:
|
||||
Path(constants.CHECK_FOR_UPDATE_DONE_PATH).touch()
|
||||
|
||||
# Check latest version from PyPI
|
||||
response = get_session().get("https://pypi.org/pypi/huggingface_hub/json", timeout=2)
|
||||
response = get_session().get(f"https://pypi.org/pypi/{library}/json", timeout=2)
|
||||
hf_raise_for_status(response)
|
||||
data = response.json()
|
||||
latest_version = data["info"]["version"]
|
||||
|
||||
# If latest version is different from current, notify user
|
||||
if current_version != latest_version:
|
||||
method = installation_method()
|
||||
if method == "brew":
|
||||
update_command = "brew upgrade huggingface-cli"
|
||||
elif method == "hf_installer" and os.name == "nt":
|
||||
update_command = 'powershell -NoProfile -Command "iwr -useb https://hf.co/cli/install.ps1 | iex"'
|
||||
elif method == "hf_installer":
|
||||
update_command = "curl -LsSf https://hf.co/cli/install.sh | bash -"
|
||||
else: # unknown => likely pip
|
||||
update_command = "pip install -U huggingface_hub"
|
||||
if library == "huggingface_hub":
|
||||
update_command = _get_huggingface_hub_update_command()
|
||||
else:
|
||||
update_command = _get_transformers_update_command()
|
||||
|
||||
click.echo(
|
||||
ANSI.yellow(
|
||||
f"A new version of huggingface_hub ({latest_version}) is available! "
|
||||
f"A new version of {library} ({latest_version}) is available! "
|
||||
f"You are using version {current_version}.\n"
|
||||
f"To update, run: {ANSI.bold(update_command)}\n",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _get_huggingface_hub_update_command() -> str:
|
||||
"""Return the command to update huggingface_hub."""
|
||||
method = installation_method()
|
||||
if method == "brew":
|
||||
return "brew upgrade huggingface-cli"
|
||||
elif method == "hf_installer" and os.name == "nt":
|
||||
return 'powershell -NoProfile -Command "iwr -useb https://hf.co/cli/install.ps1 | iex"'
|
||||
elif method == "hf_installer":
|
||||
return "curl -LsSf https://hf.co/cli/install.sh | bash -"
|
||||
else: # unknown => likely pip
|
||||
return "pip install -U huggingface_hub"
|
||||
|
||||
|
||||
def _get_transformers_update_command() -> str:
|
||||
"""Return the command to update transformers."""
|
||||
method = installation_method()
|
||||
if method == "hf_installer" and os.name == "nt":
|
||||
return 'powershell -NoProfile -Command "iwr -useb https://hf.co/cli/install.ps1 | iex" -WithTransformers'
|
||||
elif method == "hf_installer":
|
||||
return "curl -LsSf https://hf.co/cli/install.sh | bash -s -- --with-transformers"
|
||||
else: # brew/unknown => likely pip
|
||||
return "pip install -U transformers"
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with datasets on the Hugging Face Hub.
|
||||
|
||||
Usage:
|
||||
# list datasets on the Hub
|
||||
hf datasets ls
|
||||
|
||||
# list datasets with a search query
|
||||
hf datasets ls --search "code"
|
||||
|
||||
# get info about a dataset
|
||||
hf datasets info HuggingFaceFW/fineweb
|
||||
"""
|
||||
|
||||
import enum
|
||||
import json
|
||||
from typing import Annotated, Optional, get_args
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.hf_api import DatasetSort_T, ExpandDatasetProperty_T
|
||||
from huggingface_hub.utils import ANSI
|
||||
|
||||
from ._cli_utils import (
|
||||
AuthorOpt,
|
||||
FilterOpt,
|
||||
LimitOpt,
|
||||
RevisionOpt,
|
||||
SearchOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
make_expand_properties_parser,
|
||||
repo_info_to_dict,
|
||||
typer_factory,
|
||||
)
|
||||
|
||||
|
||||
_EXPAND_PROPERTIES = sorted(get_args(ExpandDatasetProperty_T))
|
||||
_SORT_OPTIONS = get_args(DatasetSort_T)
|
||||
DatasetSortEnum = enum.Enum("DatasetSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc]
|
||||
|
||||
|
||||
ExpandOpt = Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
help=f"Comma-separated properties to expand. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
||||
callback=make_expand_properties_parser(_EXPAND_PROPERTIES),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
datasets_cli = typer_factory(help="Interact with datasets on the Hub.")
|
||||
|
||||
|
||||
@datasets_cli.command("ls")
|
||||
def datasets_ls(
|
||||
search: SearchOpt = None,
|
||||
author: AuthorOpt = None,
|
||||
filter: FilterOpt = None,
|
||||
sort: Annotated[
|
||||
Optional[DatasetSortEnum],
|
||||
typer.Option(help="Sort results."),
|
||||
] = None,
|
||||
limit: LimitOpt = 10,
|
||||
expand: ExpandOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List datasets on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
sort_key = sort.value if sort else None
|
||||
results = [
|
||||
repo_info_to_dict(dataset_info)
|
||||
for dataset_info in api.list_datasets(
|
||||
filter=filter, author=author, search=search, sort=sort_key, limit=limit, expand=expand
|
||||
)
|
||||
]
|
||||
print(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
@datasets_cli.command("info")
|
||||
def datasets_info(
|
||||
dataset_id: Annotated[str, typer.Argument(help="The dataset ID (e.g. `username/repo-name`).")],
|
||||
revision: RevisionOpt = None,
|
||||
expand: ExpandOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get info about a dataset on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
info = api.dataset_info(repo_id=dataset_id, revision=revision, expand=expand) # type: ignore[arg-type]
|
||||
except RepositoryNotFoundError:
|
||||
print(f"Dataset {ANSI.bold(dataset_id)} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except RevisionNotFoundError:
|
||||
print(f"Revision {ANSI.bold(str(revision))} not found on {ANSI.bold(dataset_id)}.")
|
||||
raise typer.Exit(code=1)
|
||||
print(json.dumps(repo_info_to_dict(info), indent=2))
|
||||
@@ -83,7 +83,7 @@ def download(
|
||||
local_dir: Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-local-folder for more details.",
|
||||
help="If set, the downloaded file will be placed under this directory. Check out https://huggingface.co/docs/huggingface_hub/guides/download#download-files-to-a-local-folder for more details.",
|
||||
),
|
||||
] = None,
|
||||
force_download: Annotated[
|
||||
|
||||
@@ -17,12 +17,15 @@ from huggingface_hub import constants
|
||||
from huggingface_hub.cli._cli_utils import check_cli_update, typer_factory
|
||||
from huggingface_hub.cli.auth import auth_cli
|
||||
from huggingface_hub.cli.cache import cache_cli
|
||||
from huggingface_hub.cli.datasets import datasets_cli
|
||||
from huggingface_hub.cli.download import download
|
||||
from huggingface_hub.cli.inference_endpoints import ie_cli
|
||||
from huggingface_hub.cli.jobs import jobs_cli
|
||||
from huggingface_hub.cli.lfs import lfs_enable_largefiles, lfs_multipart_upload
|
||||
from huggingface_hub.cli.models import models_cli
|
||||
from huggingface_hub.cli.repo import repo_cli
|
||||
from huggingface_hub.cli.repo_files import repo_files_cli
|
||||
from huggingface_hub.cli.spaces import spaces_cli
|
||||
from huggingface_hub.cli.system import env, version
|
||||
from huggingface_hub.cli.upload import upload
|
||||
from huggingface_hub.cli.upload_large_folder import upload_large_folder
|
||||
@@ -45,16 +48,19 @@ app.command(help="Upload large files to the Hub.", hidden=True)(lfs_multipart_up
|
||||
# command groups
|
||||
app.add_typer(auth_cli, name="auth")
|
||||
app.add_typer(cache_cli, name="cache")
|
||||
app.add_typer(datasets_cli, name="datasets")
|
||||
app.add_typer(jobs_cli, name="jobs")
|
||||
app.add_typer(models_cli, name="models")
|
||||
app.add_typer(repo_cli, name="repo")
|
||||
app.add_typer(repo_files_cli, name="repo-files")
|
||||
app.add_typer(jobs_cli, name="jobs")
|
||||
app.add_typer(spaces_cli, name="spaces")
|
||||
app.add_typer(ie_cli, name="endpoints")
|
||||
|
||||
|
||||
def main():
|
||||
if not constants.HF_DEBUG:
|
||||
logging.set_verbosity_info()
|
||||
check_cli_update()
|
||||
check_cli_update("huggingface_hub")
|
||||
app()
|
||||
|
||||
|
||||
|
||||
@@ -23,6 +23,9 @@ Usage:
|
||||
# Stream logs from a job
|
||||
hf jobs logs <job-id>
|
||||
|
||||
# Stream resources usage stats and metrics from a job
|
||||
hf jobs stats <job-id>
|
||||
|
||||
# Inspect detailed information about a job
|
||||
hf jobs inspect <job-id>
|
||||
|
||||
@@ -53,17 +56,22 @@ Usage:
|
||||
"""
|
||||
|
||||
import json
|
||||
import multiprocessing
|
||||
import multiprocessing.pool
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from dataclasses import asdict
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Dict, Optional, Union
|
||||
from queue import Empty, Queue
|
||||
from typing import Annotated, Any, Callable, Dict, Iterable, Optional, TypeVar, Union
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub import SpaceHardware, get_token
|
||||
from huggingface_hub.errors import HfHubHTTPError
|
||||
from huggingface_hub.utils import logging
|
||||
from huggingface_hub.utils._cache_manager import _format_size
|
||||
from huggingface_hub.utils._dotenv import load_dotenv
|
||||
|
||||
from ._cli_utils import TokenOpt, get_hf_api, typer_factory
|
||||
@@ -72,6 +80,7 @@ from ._cli_utils import TokenOpt, get_hf_api, typer_factory
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
SUGGESTED_FLAVORS = [item.value for item in SpaceHardware if item.value != "zero-a10g"]
|
||||
STATS_UPDATE_MIN_INTERVAL = 0.1 # we set a limit here since there is one update per second per job
|
||||
|
||||
# Common job-related options
|
||||
ImageArg = Annotated[
|
||||
@@ -217,6 +226,13 @@ JobIdArg = Annotated[
|
||||
),
|
||||
]
|
||||
|
||||
JobIdsArg = Annotated[
|
||||
Optional[list[str]],
|
||||
typer.Argument(
|
||||
help="Job IDs",
|
||||
),
|
||||
]
|
||||
|
||||
ScheduledJobIdArg = Annotated[
|
||||
str,
|
||||
typer.Argument(
|
||||
@@ -224,18 +240,11 @@ ScheduledJobIdArg = Annotated[
|
||||
),
|
||||
]
|
||||
|
||||
RepoOpt = Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
help="Repository name for the script (creates ephemeral if not specified)",
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
jobs_cli = typer_factory(help="Run and manage Jobs on the Hub.")
|
||||
|
||||
|
||||
@jobs_cli.command("run", help="Run a Job")
|
||||
@jobs_cli.command("run", help="Run a Job", context_settings={"ignore_unknown_options": True})
|
||||
def jobs_run(
|
||||
image: ImageArg,
|
||||
command: CommandArg,
|
||||
@@ -312,14 +321,16 @@ def _matches_filters(job_properties: dict[str, str], filters: dict[str, str]) ->
|
||||
return True
|
||||
|
||||
|
||||
def _print_output(rows: list[list[Union[str, int]]], headers: list[str], fmt: Optional[str]) -> None:
|
||||
def _print_output(
|
||||
rows: list[list[Union[str, int]]], headers: list[str], aliases: list[str], fmt: Optional[str]
|
||||
) -> None:
|
||||
"""Print output according to the chosen format."""
|
||||
if fmt:
|
||||
# Use custom template if provided
|
||||
template = fmt
|
||||
for row in rows:
|
||||
line = template
|
||||
for i, field in enumerate(["id", "image", "command", "created", "status"]):
|
||||
for i, field in enumerate(aliases):
|
||||
placeholder = f"{{{{.{field}}}}}"
|
||||
if placeholder in line:
|
||||
line = line.replace(placeholder, str(row[i]))
|
||||
@@ -329,6 +340,112 @@ def _print_output(rows: list[list[Union[str, int]]], headers: list[str], fmt: Op
|
||||
print(_tabulate(rows, headers=headers))
|
||||
|
||||
|
||||
def _clear_line(n: int) -> None:
|
||||
LINE_UP = "\033[1A"
|
||||
LINE_CLEAR = "\x1b[2K"
|
||||
for i in range(n):
|
||||
print(LINE_UP, end=LINE_CLEAR)
|
||||
|
||||
|
||||
def _get_jobs_stats_rows(
|
||||
job_id: str, metrics_stream: Iterable[dict[str, Any]], table_headers: list[str]
|
||||
) -> Iterable[tuple[bool, str, list[list[Union[str, int]]]]]:
|
||||
for metrics in metrics_stream:
|
||||
row = [
|
||||
job_id,
|
||||
f"{metrics['cpu_usage_pct']}%",
|
||||
round(metrics["cpu_millicores"] / 1000.0, 1),
|
||||
f"{round(100 * metrics['memory_used_bytes'] / metrics['memory_total_bytes'], 2)}%",
|
||||
f"{_format_size(metrics['memory_used_bytes'])}B / {_format_size(metrics['memory_total_bytes'])}B",
|
||||
f"{_format_size(metrics['rx_bps'])}bps / {_format_size(metrics['tx_bps'])}bps",
|
||||
]
|
||||
if metrics["gpus"] and isinstance(metrics["gpus"], dict):
|
||||
rows = [row] + [[""] * len(row)] * (len(metrics["gpus"]) - 1)
|
||||
for row, gpu_id in zip(rows, sorted(metrics["gpus"])):
|
||||
gpu = metrics["gpus"][gpu_id]
|
||||
row += [
|
||||
f"{gpu['utilization']}%",
|
||||
f"{round(100 * gpu['memory_used_bytes'] / gpu['memory_total_bytes'], 2)}%",
|
||||
f"{_format_size(gpu['memory_used_bytes'])}B / {_format_size(gpu['memory_total_bytes'])}B",
|
||||
]
|
||||
else:
|
||||
row += ["N/A"] * (len(table_headers) - len(row))
|
||||
rows = [row]
|
||||
yield False, job_id, rows
|
||||
yield True, job_id, []
|
||||
|
||||
|
||||
@jobs_cli.command("stats", help="Fetch the resource usage statistics and metrics of Jobs")
|
||||
def jobs_stats(
|
||||
job_ids: JobIdsArg = None,
|
||||
namespace: NamespaceOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
api = get_hf_api(token=token)
|
||||
if namespace is None:
|
||||
namespace = api.whoami()["name"]
|
||||
if job_ids is None:
|
||||
job_ids = [
|
||||
job.id
|
||||
for job in api.list_jobs(namespace=namespace)
|
||||
if (job.status.stage if job.status else "UNKNOWN") in ("RUNNING", "UPDATING")
|
||||
]
|
||||
if len(job_ids) == 0:
|
||||
print("No running jobs found")
|
||||
return
|
||||
table_headers = [
|
||||
"JOB ID",
|
||||
"CPU %",
|
||||
"NUM CPU",
|
||||
"MEM %",
|
||||
"MEM USAGE",
|
||||
"NET I/O",
|
||||
"GPU UTIL %",
|
||||
"GPU MEM %",
|
||||
"GPU MEM USAGE",
|
||||
]
|
||||
headers_aliases = [
|
||||
"id",
|
||||
"cpu_usage_pct",
|
||||
"cpu_millicores",
|
||||
"memory_used_bytes_pct",
|
||||
"memory_used_bytes_and_total_bytes",
|
||||
"rx_bps_and_tx_bps",
|
||||
"gpu_utilization",
|
||||
"gpu_memory_used_bytes_pct",
|
||||
"gpu_memory_used_bytes_and_total_bytes",
|
||||
]
|
||||
with multiprocessing.pool.ThreadPool(len(job_ids)) as pool:
|
||||
rows_per_job_id: dict[str, list[list[Union[str, int]]]] = {}
|
||||
for job_id in job_ids:
|
||||
row: list[Union[str, int]] = [job_id]
|
||||
row += ["-- / --" if ("/" in header or "USAGE" in header) else "--" for header in table_headers[1:]]
|
||||
rows_per_job_id[job_id] = [row]
|
||||
last_update_time = time.time()
|
||||
total_rows = [row for job_id in rows_per_job_id for row in rows_per_job_id[job_id]]
|
||||
_print_output(total_rows, table_headers, headers_aliases, None)
|
||||
|
||||
kwargs_list = [
|
||||
{
|
||||
"job_id": job_id,
|
||||
"metrics_stream": api.fetch_job_metrics(job_id=job_id, namespace=namespace),
|
||||
"table_headers": table_headers,
|
||||
}
|
||||
for job_id in job_ids
|
||||
]
|
||||
for done, job_id, rows in iflatmap_unordered(pool, _get_jobs_stats_rows, kwargs_list=kwargs_list):
|
||||
if done:
|
||||
rows_per_job_id.pop(job_id, None)
|
||||
else:
|
||||
rows_per_job_id[job_id] = rows
|
||||
now = time.time()
|
||||
if now - last_update_time >= STATS_UPDATE_MIN_INTERVAL:
|
||||
_clear_line(2 + len(total_rows))
|
||||
total_rows = [row for job_id in rows_per_job_id for row in rows_per_job_id[job_id]]
|
||||
_print_output(total_rows, table_headers, headers_aliases, None)
|
||||
last_update_time = now
|
||||
|
||||
|
||||
@jobs_cli.command("ps", help="List Jobs")
|
||||
def jobs_ps(
|
||||
all: Annotated[
|
||||
@@ -362,6 +479,7 @@ def jobs_ps(
|
||||
jobs = api.list_jobs(namespace=namespace)
|
||||
# Define table headers
|
||||
table_headers = ["JOB ID", "IMAGE/SPACE", "COMMAND", "CREATED", "STATUS"]
|
||||
headers_aliases = ["id", "image", "command", "created", "status"]
|
||||
rows: list[list[Union[str, int]]] = []
|
||||
|
||||
filters: dict[str, str] = {}
|
||||
@@ -407,7 +525,7 @@ def jobs_ps(
|
||||
print(f"No jobs found{filters_msg}")
|
||||
return
|
||||
# Apply custom format if provided or use default tabular format
|
||||
_print_output(rows, table_headers, format)
|
||||
_print_output(rows, table_headers, headers_aliases, format)
|
||||
|
||||
except HfHubHTTPError as e:
|
||||
print(f"Error fetching jobs data: {e}")
|
||||
@@ -447,12 +565,15 @@ uv_app = typer_factory(help="Run UV scripts (Python with inline dependencies) on
|
||||
jobs_cli.add_typer(uv_app, name="uv")
|
||||
|
||||
|
||||
@uv_app.command("run", help="Run a UV script (local file or URL) on HF infrastructure")
|
||||
@uv_app.command(
|
||||
"run",
|
||||
help="Run a UV script (local file or URL) on HF infrastructure",
|
||||
context_settings={"ignore_unknown_options": True},
|
||||
)
|
||||
def jobs_uv_run(
|
||||
script: ScriptArg,
|
||||
script_args: ScriptArgsArg = None,
|
||||
image: ImageOpt = None,
|
||||
repo: RepoOpt = None,
|
||||
flavor: FlavorOpt = None,
|
||||
env: EnvOpt = None,
|
||||
secrets: SecretsOpt = None,
|
||||
@@ -489,7 +610,6 @@ def jobs_uv_run(
|
||||
flavor=flavor, # type: ignore[arg-type]
|
||||
timeout=timeout,
|
||||
namespace=namespace,
|
||||
_repo=repo,
|
||||
)
|
||||
# Always print the job ID to the user
|
||||
print(f"Job started with ID: {job.id}")
|
||||
@@ -505,7 +625,7 @@ scheduled_app = typer_factory(help="Create and manage scheduled Jobs on the Hub.
|
||||
jobs_cli.add_typer(scheduled_app, name="scheduled")
|
||||
|
||||
|
||||
@scheduled_app.command("run", help="Schedule a Job")
|
||||
@scheduled_app.command("run", help="Schedule a Job", context_settings={"ignore_unknown_options": True})
|
||||
def scheduled_run(
|
||||
schedule: ScheduleArg,
|
||||
image: ImageArg,
|
||||
@@ -581,6 +701,7 @@ def scheduled_ps(
|
||||
api = get_hf_api(token=token)
|
||||
scheduled_jobs = api.list_scheduled_jobs(namespace=namespace)
|
||||
table_headers = ["ID", "SCHEDULE", "IMAGE/SPACE", "COMMAND", "LAST RUN", "NEXT RUN", "SUSPEND"]
|
||||
headers_aliases = ["id", "schedule", "image", "command", "last", "next", "suspend"]
|
||||
rows: list[list[Union[str, int]]] = []
|
||||
filters: dict[str, str] = {}
|
||||
for f in filter or []:
|
||||
@@ -620,7 +741,7 @@ def scheduled_ps(
|
||||
)
|
||||
print(f"No scheduled jobs found{filters_msg}")
|
||||
return
|
||||
_print_output(rows, table_headers, format)
|
||||
_print_output(rows, table_headers, headers_aliases, format)
|
||||
|
||||
except HfHubHTTPError as e:
|
||||
print(f"Error fetching scheduled jobs data: {e}")
|
||||
@@ -683,7 +804,11 @@ scheduled_uv_app = typer_factory(help="Schedule UV scripts on HF infrastructure"
|
||||
scheduled_app.add_typer(scheduled_uv_app, name="uv")
|
||||
|
||||
|
||||
@scheduled_uv_app.command("run", help="Run a UV script (local file or URL) on HF infrastructure")
|
||||
@scheduled_uv_app.command(
|
||||
"run",
|
||||
help="Run a UV script (local file or URL) on HF infrastructure",
|
||||
context_settings={"ignore_unknown_options": True},
|
||||
)
|
||||
def scheduled_uv_run(
|
||||
schedule: ScheduleArg,
|
||||
script: ScriptArg,
|
||||
@@ -691,7 +816,6 @@ def scheduled_uv_run(
|
||||
suspend: SuspendOpt = None,
|
||||
concurrency: ConcurrencyOpt = None,
|
||||
image: ImageOpt = None,
|
||||
repo: RepoOpt = None,
|
||||
flavor: FlavorOpt = None,
|
||||
env: EnvOpt = None,
|
||||
secrets: SecretsOpt = None,
|
||||
@@ -730,7 +854,6 @@ def scheduled_uv_run(
|
||||
flavor=flavor, # type: ignore[arg-type]
|
||||
timeout=timeout,
|
||||
namespace=namespace,
|
||||
_repo=repo,
|
||||
)
|
||||
print(f"Scheduled Job created with ID: {job.id}")
|
||||
|
||||
@@ -770,3 +893,45 @@ def _get_extended_environ() -> Dict[str, str]:
|
||||
if (token := get_token()) is not None:
|
||||
extended_environ["HF_TOKEN"] = token
|
||||
return extended_environ
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def _write_generator_to_queue(queue: Queue[T], func: Callable[..., Iterable[T]], kwargs: dict) -> None:
|
||||
for result in func(**kwargs):
|
||||
queue.put(result)
|
||||
|
||||
|
||||
def iflatmap_unordered(
|
||||
pool: multiprocessing.pool.ThreadPool,
|
||||
func: Callable[..., Iterable[T]],
|
||||
*,
|
||||
kwargs_list: list[dict],
|
||||
) -> Iterable[T]:
|
||||
"""
|
||||
Takes a function that returns an iterable of items, and run it in parallel using threads to return the flattened iterable of items as they arrive.
|
||||
|
||||
This is inspired by those three `map()` variants, and is the mix of all three:
|
||||
|
||||
* `imap()`: like `map()` but returns an iterable instead of a list of results
|
||||
* `imap_unordered()`: like `imap()` but the output is sorted by time of arrival
|
||||
* `flatmap()`: like `map()` but given a function which returns a list, `flatmap()` returns the flattened list that is the concatenation of all the output lists
|
||||
"""
|
||||
queue: Queue[T] = Queue()
|
||||
async_results = [pool.apply_async(_write_generator_to_queue, (queue, func, kwargs)) for kwargs in kwargs_list]
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
yield queue.get(timeout=0.05)
|
||||
except Empty:
|
||||
if all(async_result.ready() for async_result in async_results) and queue.empty():
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
# we get the result in case there's an error to raise
|
||||
try:
|
||||
[async_result.get(timeout=0.05) for async_result in async_results]
|
||||
except multiprocessing.TimeoutError:
|
||||
pass
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with models on the Hugging Face Hub.
|
||||
|
||||
Usage:
|
||||
# list models on the Hub
|
||||
hf models ls
|
||||
|
||||
# list models with a search query
|
||||
hf models ls --search "llama"
|
||||
|
||||
# get info about a model
|
||||
hf models info Lightricks/LTX-2
|
||||
"""
|
||||
|
||||
import enum
|
||||
import json
|
||||
from typing import Annotated, Optional, get_args
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.hf_api import ExpandModelProperty_T, ModelSort_T
|
||||
from huggingface_hub.utils import ANSI
|
||||
|
||||
from ._cli_utils import (
|
||||
AuthorOpt,
|
||||
FilterOpt,
|
||||
LimitOpt,
|
||||
RevisionOpt,
|
||||
SearchOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
make_expand_properties_parser,
|
||||
repo_info_to_dict,
|
||||
typer_factory,
|
||||
)
|
||||
|
||||
|
||||
_EXPAND_PROPERTIES = sorted(get_args(ExpandModelProperty_T))
|
||||
_SORT_OPTIONS = get_args(ModelSort_T)
|
||||
ModelSortEnum = enum.Enum("ModelSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc]
|
||||
|
||||
|
||||
ExpandOpt = Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
help=f"Comma-separated properties to expand. Example: '--expand=downloads,likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
||||
callback=make_expand_properties_parser(_EXPAND_PROPERTIES),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
models_cli = typer_factory(help="Interact with models on the Hub.")
|
||||
|
||||
|
||||
@models_cli.command("ls")
|
||||
def models_ls(
|
||||
search: SearchOpt = None,
|
||||
author: AuthorOpt = None,
|
||||
filter: FilterOpt = None,
|
||||
sort: Annotated[
|
||||
Optional[ModelSortEnum],
|
||||
typer.Option(help="Sort results."),
|
||||
] = None,
|
||||
limit: LimitOpt = 10,
|
||||
expand: ExpandOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List models on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
sort_key = sort.value if sort else None
|
||||
results = [
|
||||
repo_info_to_dict(model_info)
|
||||
for model_info in api.list_models(
|
||||
filter=filter, author=author, search=search, sort=sort_key, limit=limit, expand=expand
|
||||
)
|
||||
]
|
||||
print(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
@models_cli.command("info")
|
||||
def models_info(
|
||||
model_id: Annotated[str, typer.Argument(help="The model ID (e.g. `username/repo-name`).")],
|
||||
revision: RevisionOpt = None,
|
||||
expand: ExpandOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get info about a model on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
info = api.model_info(repo_id=model_id, revision=revision, expand=expand) # type: ignore[arg-type]
|
||||
except RepositoryNotFoundError:
|
||||
print(f"Model {ANSI.bold(model_id)} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except RevisionNotFoundError:
|
||||
print(f"Revision {ANSI.bold(str(revision))} not found on {ANSI.bold(model_id)}.")
|
||||
raise typer.Exit(code=1)
|
||||
print(json.dumps(repo_info_to_dict(info), indent=2))
|
||||
@@ -27,22 +27,11 @@ from typing import Annotated, Optional
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import HfHubHTTPError, RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.utils import ANSI, logging
|
||||
from huggingface_hub.utils import ANSI
|
||||
|
||||
from ._cli_utils import (
|
||||
PrivateOpt,
|
||||
RepoIdArg,
|
||||
RepoType,
|
||||
RepoTypeOpt,
|
||||
RevisionOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
typer_factory,
|
||||
)
|
||||
from ._cli_utils import PrivateOpt, RepoIdArg, RepoType, RepoTypeOpt, RevisionOpt, TokenOpt, get_hf_api, typer_factory
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__)
|
||||
|
||||
repo_cli = typer_factory(help="Manage repos on the Hub.")
|
||||
tag_cli = typer_factory(help="Manage tags for a repo on the Hub.")
|
||||
branch_cli = typer_factory(help="Manage branches for a repo on the Hub.")
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
# Copyright 2026 The HuggingFace Team. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Contains commands to interact with spaces on the Hugging Face Hub.
|
||||
|
||||
Usage:
|
||||
# list spaces on the Hub
|
||||
hf spaces ls
|
||||
|
||||
# list spaces with a search query
|
||||
hf spaces ls --search "chatbot"
|
||||
|
||||
# get info about a space
|
||||
hf spaces info enzostvs/deepsite
|
||||
"""
|
||||
|
||||
import enum
|
||||
import json
|
||||
from typing import Annotated, Optional, get_args
|
||||
|
||||
import typer
|
||||
|
||||
from huggingface_hub.errors import RepositoryNotFoundError, RevisionNotFoundError
|
||||
from huggingface_hub.hf_api import ExpandSpaceProperty_T, SpaceSort_T
|
||||
from huggingface_hub.utils import ANSI
|
||||
|
||||
from ._cli_utils import (
|
||||
AuthorOpt,
|
||||
FilterOpt,
|
||||
LimitOpt,
|
||||
RevisionOpt,
|
||||
SearchOpt,
|
||||
TokenOpt,
|
||||
get_hf_api,
|
||||
make_expand_properties_parser,
|
||||
repo_info_to_dict,
|
||||
typer_factory,
|
||||
)
|
||||
|
||||
|
||||
_EXPAND_PROPERTIES = sorted(get_args(ExpandSpaceProperty_T))
|
||||
_SORT_OPTIONS = get_args(SpaceSort_T)
|
||||
SpaceSortEnum = enum.Enum("SpaceSortEnum", {s: s for s in _SORT_OPTIONS}, type=str) # type: ignore[misc]
|
||||
|
||||
|
||||
ExpandOpt = Annotated[
|
||||
Optional[str],
|
||||
typer.Option(
|
||||
help=f"Comma-separated properties to expand. Example: '--expand=likes,tags'. Valid: {', '.join(_EXPAND_PROPERTIES)}.",
|
||||
callback=make_expand_properties_parser(_EXPAND_PROPERTIES),
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
spaces_cli = typer_factory(help="Interact with spaces on the Hub.")
|
||||
|
||||
|
||||
@spaces_cli.command("ls")
|
||||
def spaces_ls(
|
||||
search: SearchOpt = None,
|
||||
author: AuthorOpt = None,
|
||||
filter: FilterOpt = None,
|
||||
sort: Annotated[
|
||||
Optional[SpaceSortEnum],
|
||||
typer.Option(help="Sort results."),
|
||||
] = None,
|
||||
limit: LimitOpt = 10,
|
||||
expand: ExpandOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""List spaces on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
sort_key = sort.value if sort else None
|
||||
results = [
|
||||
repo_info_to_dict(space_info)
|
||||
for space_info in api.list_spaces(
|
||||
filter=filter, author=author, search=search, sort=sort_key, limit=limit, expand=expand
|
||||
)
|
||||
]
|
||||
print(json.dumps(results, indent=2))
|
||||
|
||||
|
||||
@spaces_cli.command("info")
|
||||
def spaces_info(
|
||||
space_id: Annotated[str, typer.Argument(help="The space ID (e.g. `username/repo-name`).")],
|
||||
revision: RevisionOpt = None,
|
||||
expand: ExpandOpt = None,
|
||||
token: TokenOpt = None,
|
||||
) -> None:
|
||||
"""Get info about a space on the Hub."""
|
||||
api = get_hf_api(token=token)
|
||||
try:
|
||||
info = api.space_info(repo_id=space_id, revision=revision, expand=expand) # type: ignore[arg-type]
|
||||
except RepositoryNotFoundError:
|
||||
print(f"Space {ANSI.bold(space_id)} not found.")
|
||||
raise typer.Exit(code=1)
|
||||
except RevisionNotFoundError:
|
||||
print(f"Revision {ANSI.bold(str(revision))} not found on {ANSI.bold(space_id)}.")
|
||||
raise typer.Exit(code=1)
|
||||
print(json.dumps(repo_info_to_dict(info), indent=2))
|
||||
Reference in New Issue
Block a user