针对数据格式修改处理
This commit is contained in:
@@ -30,7 +30,8 @@
|
||||
| 11 | `V_pi` | 半波电压 | 目标 |
|
||||
|
||||
- 自动忽略空行与行首行尾空格。
|
||||
- 每行必须恰好 **11 列**,否则整文件解析失败并给出错误行号提示。
|
||||
- 每行必须恰好 **11 列**;也支持仿真导出的 **整行方括号** 写法,例如 `[a, b, ..., k]`(与无括号的 `a, b, ..., k` 等价)。
|
||||
- 否则整文件解析失败并给出错误行号提示。
|
||||
|
||||
## TXT 数据清洗流程(以 V_pi 为准)
|
||||
|
||||
|
||||
9632
Sim_MZM_dataset.txt
9632
Sim_MZM_dataset.txt
File diff suppressed because it is too large
Load Diff
20
src/data.py
20
src/data.py
@@ -28,10 +28,29 @@ EXPECTED_COLS = 11
|
||||
V_PI_TXT_1BASED_INDEX = 11
|
||||
|
||||
|
||||
def _strip_optional_list_brackets(line: str) -> str:
|
||||
"""
|
||||
去掉仿真/导出常见的整行方括号包裹,例如::
|
||||
|
||||
[-2.15e-07, -10.0, ...] -> -2.15e-07, -10.0, ...
|
||||
|
||||
若行首无 ``[`` 或行尾无 ``]``,则原样返回(兼容无括号格式)。
|
||||
"""
|
||||
s = line.strip()
|
||||
if len(s) >= 2 and s[0] == "[" and s[-1] == "]":
|
||||
return s[1:-1].strip()
|
||||
return s
|
||||
|
||||
|
||||
def load_raw_txt(path: str | Path) -> pd.DataFrame:
|
||||
"""
|
||||
从 txt 读取数据:逗号分隔、11 列浮点;跳过空行与纯空白行。
|
||||
|
||||
支持两种常见行格式(等价)::
|
||||
|
||||
a,b,c,...,k
|
||||
[a, b, c, ..., k]
|
||||
|
||||
Raises:
|
||||
FileNotFoundError: 文件不存在
|
||||
ValueError: 行格式或列数不合法
|
||||
@@ -51,6 +70,7 @@ def load_raw_txt(path: str | Path) -> pd.DataFrame:
|
||||
line = raw.strip()
|
||||
if not line:
|
||||
continue
|
||||
line = _strip_optional_list_brackets(line)
|
||||
parts = [p.strip() for p in line.split(",")]
|
||||
if len(parts) != EXPECTED_COLS:
|
||||
bad_lines.append((line_no, f"列数={len(parts)},期望 {EXPECTED_COLS}"))
|
||||
|
||||
30
src/infer.py
30
src/infer.py
@@ -10,7 +10,7 @@ import pandas as pd
|
||||
import torch
|
||||
|
||||
from src.config import AppConfig, load_config
|
||||
from src.data import INPUT_COLUMNS, TARGET_COLUMNS
|
||||
from src.data import INPUT_COLUMNS, TARGET_COLUMNS, _strip_optional_list_brackets
|
||||
from src.model import MLPRegressor
|
||||
from src.preprocess import load_scalers
|
||||
from src.trainer import load_weights
|
||||
@@ -18,15 +18,39 @@ from src.trainer import load_weights
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _read_txt_inputs_eight_cols(path: Path) -> pd.DataFrame:
|
||||
"""读取无表头 txt:逗号分隔,支持整行 ``[...]`` 包裹;取前 8 列为输入。"""
|
||||
rows: list[list[float]] = []
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
for line_no, raw in enumerate(f, start=1):
|
||||
s = raw.strip()
|
||||
if not s:
|
||||
continue
|
||||
s = _strip_optional_list_brackets(s)
|
||||
parts = [p.strip() for p in s.split(",")]
|
||||
if len(parts) < len(INPUT_COLUMNS):
|
||||
raise ValueError(
|
||||
f"{path} 第 {line_no} 行列数不足({len(parts)}),至少需要 {len(INPUT_COLUMNS)} 列输入"
|
||||
)
|
||||
try:
|
||||
row = [float(parts[j]) for j in range(len(INPUT_COLUMNS))]
|
||||
except ValueError as e:
|
||||
raise ValueError(f"{path} 第 {line_no} 行解析失败: {e}") from e
|
||||
rows.append(row)
|
||||
if not rows:
|
||||
raise ValueError(f"{path} 无有效数据行")
|
||||
return pd.DataFrame(rows, columns=INPUT_COLUMNS)
|
||||
|
||||
|
||||
def _read_inputs_table(path: Path) -> pd.DataFrame:
|
||||
"""读取 8 列输入:支持逗号分隔 txt 或 csv。"""
|
||||
"""读取 8 列输入:支持逗号分隔 txt(可无表头、可带方括号)或 csv。"""
|
||||
path = Path(path)
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"输入文件不存在: {path}")
|
||||
if path.suffix.lower() in {".csv"}:
|
||||
df = pd.read_csv(path)
|
||||
else:
|
||||
df = pd.read_csv(path, header=None, names=INPUT_COLUMNS)
|
||||
df = _read_txt_inputs_eight_cols(path)
|
||||
missing = [c for c in INPUT_COLUMNS if c not in df.columns]
|
||||
if missing:
|
||||
raise ValueError(f"输入缺少列: {missing};需要列 {INPUT_COLUMNS}")
|
||||
|
||||
Reference in New Issue
Block a user