针对数据格式修改处理
This commit is contained in:
@@ -30,7 +30,8 @@
|
|||||||
| 11 | `V_pi` | 半波电压 | 目标 |
|
| 11 | `V_pi` | 半波电压 | 目标 |
|
||||||
|
|
||||||
- 自动忽略空行与行首行尾空格。
|
- 自动忽略空行与行首行尾空格。
|
||||||
- 每行必须恰好 **11 列**,否则整文件解析失败并给出错误行号提示。
|
- 每行必须恰好 **11 列**;也支持仿真导出的 **整行方括号** 写法,例如 `[a, b, ..., k]`(与无括号的 `a, b, ..., k` 等价)。
|
||||||
|
- 否则整文件解析失败并给出错误行号提示。
|
||||||
|
|
||||||
## TXT 数据清洗流程(以 V_pi 为准)
|
## TXT 数据清洗流程(以 V_pi 为准)
|
||||||
|
|
||||||
|
|||||||
9632
Sim_MZM_dataset.txt
9632
Sim_MZM_dataset.txt
File diff suppressed because it is too large
Load Diff
20
src/data.py
20
src/data.py
@@ -28,10 +28,29 @@ EXPECTED_COLS = 11
|
|||||||
V_PI_TXT_1BASED_INDEX = 11
|
V_PI_TXT_1BASED_INDEX = 11
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_optional_list_brackets(line: str) -> str:
|
||||||
|
"""
|
||||||
|
去掉仿真/导出常见的整行方括号包裹,例如::
|
||||||
|
|
||||||
|
[-2.15e-07, -10.0, ...] -> -2.15e-07, -10.0, ...
|
||||||
|
|
||||||
|
若行首无 ``[`` 或行尾无 ``]``,则原样返回(兼容无括号格式)。
|
||||||
|
"""
|
||||||
|
s = line.strip()
|
||||||
|
if len(s) >= 2 and s[0] == "[" and s[-1] == "]":
|
||||||
|
return s[1:-1].strip()
|
||||||
|
return s
|
||||||
|
|
||||||
|
|
||||||
def load_raw_txt(path: str | Path) -> pd.DataFrame:
|
def load_raw_txt(path: str | Path) -> pd.DataFrame:
|
||||||
"""
|
"""
|
||||||
从 txt 读取数据:逗号分隔、11 列浮点;跳过空行与纯空白行。
|
从 txt 读取数据:逗号分隔、11 列浮点;跳过空行与纯空白行。
|
||||||
|
|
||||||
|
支持两种常见行格式(等价)::
|
||||||
|
|
||||||
|
a,b,c,...,k
|
||||||
|
[a, b, c, ..., k]
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
FileNotFoundError: 文件不存在
|
FileNotFoundError: 文件不存在
|
||||||
ValueError: 行格式或列数不合法
|
ValueError: 行格式或列数不合法
|
||||||
@@ -51,6 +70,7 @@ def load_raw_txt(path: str | Path) -> pd.DataFrame:
|
|||||||
line = raw.strip()
|
line = raw.strip()
|
||||||
if not line:
|
if not line:
|
||||||
continue
|
continue
|
||||||
|
line = _strip_optional_list_brackets(line)
|
||||||
parts = [p.strip() for p in line.split(",")]
|
parts = [p.strip() for p in line.split(",")]
|
||||||
if len(parts) != EXPECTED_COLS:
|
if len(parts) != EXPECTED_COLS:
|
||||||
bad_lines.append((line_no, f"列数={len(parts)},期望 {EXPECTED_COLS}"))
|
bad_lines.append((line_no, f"列数={len(parts)},期望 {EXPECTED_COLS}"))
|
||||||
|
|||||||
30
src/infer.py
30
src/infer.py
@@ -10,7 +10,7 @@ import pandas as pd
|
|||||||
import torch
|
import torch
|
||||||
|
|
||||||
from src.config import AppConfig, load_config
|
from src.config import AppConfig, load_config
|
||||||
from src.data import INPUT_COLUMNS, TARGET_COLUMNS
|
from src.data import INPUT_COLUMNS, TARGET_COLUMNS, _strip_optional_list_brackets
|
||||||
from src.model import MLPRegressor
|
from src.model import MLPRegressor
|
||||||
from src.preprocess import load_scalers
|
from src.preprocess import load_scalers
|
||||||
from src.trainer import load_weights
|
from src.trainer import load_weights
|
||||||
@@ -18,15 +18,39 @@ from src.trainer import load_weights
|
|||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _read_txt_inputs_eight_cols(path: Path) -> pd.DataFrame:
|
||||||
|
"""读取无表头 txt:逗号分隔,支持整行 ``[...]`` 包裹;取前 8 列为输入。"""
|
||||||
|
rows: list[list[float]] = []
|
||||||
|
with path.open("r", encoding="utf-8") as f:
|
||||||
|
for line_no, raw in enumerate(f, start=1):
|
||||||
|
s = raw.strip()
|
||||||
|
if not s:
|
||||||
|
continue
|
||||||
|
s = _strip_optional_list_brackets(s)
|
||||||
|
parts = [p.strip() for p in s.split(",")]
|
||||||
|
if len(parts) < len(INPUT_COLUMNS):
|
||||||
|
raise ValueError(
|
||||||
|
f"{path} 第 {line_no} 行列数不足({len(parts)}),至少需要 {len(INPUT_COLUMNS)} 列输入"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
row = [float(parts[j]) for j in range(len(INPUT_COLUMNS))]
|
||||||
|
except ValueError as e:
|
||||||
|
raise ValueError(f"{path} 第 {line_no} 行解析失败: {e}") from e
|
||||||
|
rows.append(row)
|
||||||
|
if not rows:
|
||||||
|
raise ValueError(f"{path} 无有效数据行")
|
||||||
|
return pd.DataFrame(rows, columns=INPUT_COLUMNS)
|
||||||
|
|
||||||
|
|
||||||
def _read_inputs_table(path: Path) -> pd.DataFrame:
|
def _read_inputs_table(path: Path) -> pd.DataFrame:
|
||||||
"""读取 8 列输入:支持逗号分隔 txt 或 csv。"""
|
"""读取 8 列输入:支持逗号分隔 txt(可无表头、可带方括号)或 csv。"""
|
||||||
path = Path(path)
|
path = Path(path)
|
||||||
if not path.is_file():
|
if not path.is_file():
|
||||||
raise FileNotFoundError(f"输入文件不存在: {path}")
|
raise FileNotFoundError(f"输入文件不存在: {path}")
|
||||||
if path.suffix.lower() in {".csv"}:
|
if path.suffix.lower() in {".csv"}:
|
||||||
df = pd.read_csv(path)
|
df = pd.read_csv(path)
|
||||||
else:
|
else:
|
||||||
df = pd.read_csv(path, header=None, names=INPUT_COLUMNS)
|
df = _read_txt_inputs_eight_cols(path)
|
||||||
missing = [c for c in INPUT_COLUMNS if c not in df.columns]
|
missing = [c for c in INPUT_COLUMNS if c not in df.columns]
|
||||||
if missing:
|
if missing:
|
||||||
raise ValueError(f"输入缺少列: {missing};需要列 {INPUT_COLUMNS}")
|
raise ValueError(f"输入缺少列: {missing};需要列 {INPUT_COLUMNS}")
|
||||||
|
|||||||
Reference in New Issue
Block a user