针对数据格式修改处理

This commit is contained in:
2026-04-19 16:28:10 +08:00
parent 2f075ce54c
commit 4981df0c02
4 changed files with 49 additions and 9636 deletions

View File

@@ -30,7 +30,8 @@
| 11 | `V_pi` | 半波电压 | 目标 |
- 自动忽略空行与行首行尾空格。
- 每行必须恰好 **11 列**,否则整文件解析失败并给出错误行号提示
- 每行必须恰好 **11 列**;也支持仿真导出的 **整行方括号** 写法,例如 `[a, b, ..., k]`(与无括号的 `a, b, ..., k` 等价)
- 否则整文件解析失败并给出错误行号提示。
## TXT 数据清洗流程(以 V_pi 为准)

File diff suppressed because it is too large Load Diff

View File

@@ -28,10 +28,29 @@ EXPECTED_COLS = 11
V_PI_TXT_1BASED_INDEX = 11
def _strip_optional_list_brackets(line: str) -> str:
"""
去掉仿真/导出常见的整行方括号包裹,例如::
[-2.15e-07, -10.0, ...] -> -2.15e-07, -10.0, ...
若行首无 ``[`` 或行尾无 ``]``,则原样返回(兼容无括号格式)。
"""
s = line.strip()
if len(s) >= 2 and s[0] == "[" and s[-1] == "]":
return s[1:-1].strip()
return s
def load_raw_txt(path: str | Path) -> pd.DataFrame:
"""
从 txt 读取数据逗号分隔、11 列浮点;跳过空行与纯空白行。
支持两种常见行格式(等价)::
a,b,c,...,k
[a, b, c, ..., k]
Raises:
FileNotFoundError: 文件不存在
ValueError: 行格式或列数不合法
@@ -51,6 +70,7 @@ def load_raw_txt(path: str | Path) -> pd.DataFrame:
line = raw.strip()
if not line:
continue
line = _strip_optional_list_brackets(line)
parts = [p.strip() for p in line.split(",")]
if len(parts) != EXPECTED_COLS:
bad_lines.append((line_no, f"列数={len(parts)},期望 {EXPECTED_COLS}"))

View File

@@ -10,7 +10,7 @@ import pandas as pd
import torch
from src.config import AppConfig, load_config
from src.data import INPUT_COLUMNS, TARGET_COLUMNS
from src.data import INPUT_COLUMNS, TARGET_COLUMNS, _strip_optional_list_brackets
from src.model import MLPRegressor
from src.preprocess import load_scalers
from src.trainer import load_weights
@@ -18,15 +18,39 @@ from src.trainer import load_weights
logger = logging.getLogger(__name__)
def _read_txt_inputs_eight_cols(path: Path) -> pd.DataFrame:
"""读取无表头 txt逗号分隔支持整行 ``[...]`` 包裹;取前 8 列为输入。"""
rows: list[list[float]] = []
with path.open("r", encoding="utf-8") as f:
for line_no, raw in enumerate(f, start=1):
s = raw.strip()
if not s:
continue
s = _strip_optional_list_brackets(s)
parts = [p.strip() for p in s.split(",")]
if len(parts) < len(INPUT_COLUMNS):
raise ValueError(
f"{path}{line_no} 行列数不足({len(parts)}),至少需要 {len(INPUT_COLUMNS)} 列输入"
)
try:
row = [float(parts[j]) for j in range(len(INPUT_COLUMNS))]
except ValueError as e:
raise ValueError(f"{path}{line_no} 行解析失败: {e}") from e
rows.append(row)
if not rows:
raise ValueError(f"{path} 无有效数据行")
return pd.DataFrame(rows, columns=INPUT_COLUMNS)
def _read_inputs_table(path: Path) -> pd.DataFrame:
"""读取 8 列输入:支持逗号分隔 txt 或 csv。"""
"""读取 8 列输入:支持逗号分隔 txt(可无表头、可带方括号)或 csv。"""
path = Path(path)
if not path.is_file():
raise FileNotFoundError(f"输入文件不存在: {path}")
if path.suffix.lower() in {".csv"}:
df = pd.read_csv(path)
else:
df = pd.read_csv(path, header=None, names=INPUT_COLUMNS)
df = _read_txt_inputs_eight_cols(path)
missing = [c for c in INPUT_COLUMNS if c not in df.columns]
if missing:
raise ValueError(f"输入缺少列: {missing};需要列 {INPUT_COLUMNS}")