新增修改
This commit is contained in:
4
final/.env.example
Normal file
4
final/.env.example
Normal file
@@ -0,0 +1,4 @@
|
||||
# 达梦数据库密码(仅 source=db 时需要)
|
||||
# 复制为 .env 后填入真实密码,或将下面一行加入 ~/.bashrc 等
|
||||
# export DM_PASSWORD=你的密码
|
||||
DM_PASSWORD=Dm508508
|
||||
3
final/.idea/.gitignore
generated
vendored
Normal file
3
final/.idea/.gitignore
generated
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
# Default ignored files
|
||||
/shelf/
|
||||
/workspace.xml
|
||||
12
final/.idea/227.iml
generated
Normal file
12
final/.idea/227.iml
generated
Normal file
@@ -0,0 +1,12 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<module type="PYTHON_MODULE" version="4">
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<orderEntry type="jdk" jdkName="D:\anaconda\envs\py38" jdkType="Python SDK" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
<component name="PyDocumentationSettings">
|
||||
<option name="format" value="PLAIN" />
|
||||
<option name="myDocStringFormat" value="Plain" />
|
||||
</component>
|
||||
</module>
|
||||
38
final/.idea/inspectionProfiles/Project_Default.xml
generated
Normal file
38
final/.idea/inspectionProfiles/Project_Default.xml
generated
Normal file
@@ -0,0 +1,38 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<profile version="1.0">
|
||||
<option name="myName" value="Project Default" />
|
||||
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
|
||||
<option name="ignoredPackages">
|
||||
<value>
|
||||
<list size="8">
|
||||
<item index="0" class="java.lang.String" itemvalue="scikit-image" />
|
||||
<item index="1" class="java.lang.String" itemvalue="scipy" />
|
||||
<item index="2" class="java.lang.String" itemvalue="tensorflow_gpu" />
|
||||
<item index="3" class="java.lang.String" itemvalue="scikit_learn" />
|
||||
<item index="4" class="java.lang.String" itemvalue="matplotlib" />
|
||||
<item index="5" class="java.lang.String" itemvalue="numpy" />
|
||||
<item index="6" class="java.lang.String" itemvalue="opencv_python" />
|
||||
<item index="7" class="java.lang.String" itemvalue="Pillow" />
|
||||
</list>
|
||||
</value>
|
||||
</option>
|
||||
</inspection_tool>
|
||||
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
|
||||
<option name="ignoredErrors">
|
||||
<list>
|
||||
<option value="N806" />
|
||||
<option value="N802" />
|
||||
<option value="N803" />
|
||||
<option value="N801" />
|
||||
</list>
|
||||
</option>
|
||||
</inspection_tool>
|
||||
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
|
||||
<option name="ignoredIdentifiers">
|
||||
<list>
|
||||
<option value="int.__getitem__" />
|
||||
</list>
|
||||
</option>
|
||||
</inspection_tool>
|
||||
</profile>
|
||||
</component>
|
||||
6
final/.idea/inspectionProfiles/profiles_settings.xml
generated
Normal file
6
final/.idea/inspectionProfiles/profiles_settings.xml
generated
Normal file
@@ -0,0 +1,6 @@
|
||||
<component name="InspectionProjectProfileManager">
|
||||
<settings>
|
||||
<option name="USE_PROJECT_PROFILE" value="false" />
|
||||
<version value="1.0" />
|
||||
</settings>
|
||||
</component>
|
||||
4
final/.idea/misc.xml
generated
Normal file
4
final/.idea/misc.xml
generated
Normal file
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectRootManager" version="2" project-jdk-name="D:\anaconda\envs\py38" project-jdk-type="Python SDK" />
|
||||
</project>
|
||||
8
final/.idea/modules.xml
generated
Normal file
8
final/.idea/modules.xml
generated
Normal file
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<project version="4">
|
||||
<component name="ProjectModuleManager">
|
||||
<modules>
|
||||
<module fileurl="file://$PROJECT_DIR$/.idea/227.iml" filepath="$PROJECT_DIR$/.idea/227.iml" />
|
||||
</modules>
|
||||
</component>
|
||||
</project>
|
||||
BIN
final/1125暂时属性需求(1).xlsx
Normal file
BIN
final/1125暂时属性需求(1).xlsx
Normal file
Binary file not shown.
331
final/README.md
Normal file
331
final/README.md
Normal file
@@ -0,0 +1,331 @@
|
||||
# 作战体系数据清洗与网络效能评估系统
|
||||
|
||||
## 项目简介
|
||||
|
||||
本系统包含两大功能模块:
|
||||
|
||||
1. **数据清洗**:从 JSON 文件或达梦数据库读取实体属性表、三元组,进行去重、缺失值填充、异常值修正、噪声平滑等清洗;并按 **1125 属性需求** 做 **内容与格式的双重对齐**(格式校验 + 内容校验 + 原始字段→规范 schema 转换),**输出仅含 xlsx 定义字段**的 `nodes.json`,不保留原始其他字段。
|
||||
2. **网络关系与效能评估**:基于节点与三元组数据,计算关系强度、过滤低置信边,评估任务网络效能,生成评价报告。
|
||||
|
||||
---
|
||||
|
||||
## 目录结构
|
||||
|
||||
```
|
||||
final/
|
||||
├── config/
|
||||
│ ├── config.yaml # 主配置文件(推荐)
|
||||
│ └── config.json # 备选配置
|
||||
├── data/
|
||||
│ ├── raw_data_sample.json # 原始实体数据(JSON 模式)
|
||||
│ ├── nodes.json # 清洗后实体(关系/网络计算用)
|
||||
│ └── triples.json # 清洗后三元组(来自 run_all)
|
||||
├── report/
|
||||
│ ├── detailed_cleaning_report.json # 实体清洗报告
|
||||
│ └── triplet_cleaning_report.json # 三元组清洗报告
|
||||
├── results/
|
||||
│ ├── filtered_triples.json # 过滤后三元组
|
||||
│ ├── triples.csv # 三元组表格
|
||||
│ └── evaluation_report.txt # 效能评价报告
|
||||
├── src/
|
||||
│ ├── main_227.py # 主程序(清洗 + 关系 + 网络评估)
|
||||
│ ├── cleaner.py # 实体清洗
|
||||
│ ├── attribute_schema.py # 1125 属性规范:格式/内容校验 + 原始→规范转换
|
||||
│ ├── data_loader.py # 数据加载(JSON/达梦)
|
||||
│ └── triplet_cleaner.py # 三元组清洗
|
||||
├── scripts/
|
||||
│ ├── run_all.py # 一键运行(仅清洗)
|
||||
│ ├── export_triplets.py # 单独导出三元组
|
||||
│ ├── inspect_dm_tables.py # 达梦表结构查看
|
||||
│ ├── f_relation1125.py # 关系强度计算
|
||||
│ └── f_net1125.py # 任务网络评估
|
||||
├── .env.example
|
||||
└── README.md
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 环境依赖
|
||||
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
# 达梦模式需额外:pip install dmPython
|
||||
```
|
||||
|
||||
### 2. 运行方式
|
||||
|
||||
| 命令 | 功能 |
|
||||
|------|------|
|
||||
| `python src/main_227.py` | **完整流程**:实体清洗 → 关系强度计算 → 网络效能评估 |
|
||||
| `python scripts/run_all.py` | **仅清洗**:实体 + 三元组 读取、清洗、导出 |
|
||||
|
||||
### 3. 开发模式(无达梦数据库)
|
||||
|
||||
在 `config/config.yaml` 或 `config/config.json` 中设置:
|
||||
|
||||
```yaml
|
||||
source: json
|
||||
```
|
||||
|
||||
然后运行:
|
||||
|
||||
```bash
|
||||
python src/main_227.py
|
||||
```
|
||||
|
||||
程序会从 `data/raw_data_sample.json` 读取实体数据并清洗;关系/网络计算则从 `data/nodes.json` 和 `data/triples.json` 读取。
|
||||
|
||||
---
|
||||
|
||||
## 配置说明
|
||||
|
||||
### 必须修改的配置
|
||||
|
||||
根据你的环境,在 `config/config.yaml` 或 `config/config.json` 中修改以下项:
|
||||
|
||||
| 配置项 | 说明 | 示例 |
|
||||
|--------|------|------|
|
||||
| `source` | 数据源:`json`(开发)或 `db`(达梦) | `json` |
|
||||
| `db.host` | 达梦库地址(source=db 时) | `127.0.0.1` |
|
||||
| `db.port` | 达梦端口 | `5080` |
|
||||
| `db.user` | 达梦用户 | `SYSDBA` |
|
||||
| `db.table` | 实体表名 | `OBJECTIVE_INSTANCE` |
|
||||
| `db.triplet_table` | 三元组表名 | `RELATION_INSTANCE` |
|
||||
|
||||
### 达梦密码(切勿写入配置)
|
||||
|
||||
达梦模式需设置环境变量:
|
||||
|
||||
```bash
|
||||
export DM_PASSWORD=你的密码
|
||||
```
|
||||
|
||||
或在 PyCharm:Run → Edit Configurations → Environment variables → 添加 `DM_PASSWORD=你的密码`。
|
||||
|
||||
### 可选配置
|
||||
|
||||
| 配置项 | 说明 | 默认 |
|
||||
|--------|------|------|
|
||||
| `json.input_file` | 实体 JSON 路径 | `data/raw_data_sample.json` |
|
||||
| `json.output_file` | 清洗后实体输出 | `data/nodes.json` |
|
||||
| `triplet.enabled` | 是否处理三元组 | `true` |
|
||||
| `triplet.input_file` | JSON 模式下三元组路径 | `null` |
|
||||
| `triplet.output_file` | 三元组输出 | `data/triples.json` |
|
||||
| `triplet.clean.deduplicate` | 三元组去重 | `true` |
|
||||
| `triplet.clean.filter_orphans` | 过滤悬空引用 | `false` |
|
||||
| `eval.nodes_file` | main_227 读取的节点文件 | `data/nodes.json` |
|
||||
| `eval.triples_file` | main_227 读取的三元组文件 | `data/triples.json`(与 triplet.output_file 一致) |
|
||||
|
||||
### 配置优先级
|
||||
|
||||
`config.json` 优先于 `config.yaml`;修改其一即可。
|
||||
|
||||
---
|
||||
|
||||
## 数据流程
|
||||
|
||||
### main_227.py 流程
|
||||
|
||||
```
|
||||
1. 实体清洗
|
||||
- 从 JSON 或达梦读 raw_data_sample / OBJECTIVE_INSTANCE
|
||||
- 去重、缺失值填充、异常值修正、噪声平滑
|
||||
- 输出 nodes.json
|
||||
2. 关系与网络评估
|
||||
- 按配置读取 nodes(eval.nodes_file / json.output_file)、triples(eval.triples_file = triplet.output_file)
|
||||
- 计算 5 类关系强度(情报保障、指挥控制、状态反馈、平台部署、协同作战)
|
||||
- 按阈值过滤关系
|
||||
- 评估 5 类任务网络效能
|
||||
- 输出 results/ 下报告与表格
|
||||
```
|
||||
|
||||
### run_all.py 流程
|
||||
|
||||
```
|
||||
1. 实体清洗 → data/nodes.json
|
||||
2. 三元组清洗 → data/triples.json(格式:head, head_type, relation, tail, tail_type)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 输出文件说明
|
||||
|
||||
| 文件 | 来源 | 说明 |
|
||||
|------|------|------|
|
||||
| `data/nodes.json` | 清洗 | 清洗后实体(`{nodes: {id: obj}, global_params: {mtbf_max}}`,见下文) |
|
||||
| `data/triples.json` | run_all | 清洗后三元组(head/head_type/relation/tail/tail_type) |
|
||||
| `report/detailed_cleaning_report.json` | 清洗 | 实体清洗统计(含 attribute_alignment:格式/内容对齐情况) |
|
||||
| `report/triplet_cleaning_report.json` | run_all | 三元组清洗统计 |
|
||||
| `results/filtered_triples.json` | main_227 | 过滤后三元组(含强度) |
|
||||
| `results/triples.csv` | main_227 | 三元组表格 |
|
||||
| `results/evaluation_report.txt` | main_227 | 效能评价报告 |
|
||||
|
||||
---
|
||||
|
||||
## 常见问题
|
||||
|
||||
**Q: 运行报错「找不到文件 raw_data_sample.json」?**
|
||||
A: 将 `source` 改为 `json`,并确保 `data/raw_data_sample.json` 存在;或检查 `json.input_file` 路径。
|
||||
|
||||
**Q: 达梦模式连接失败?**
|
||||
A: 检查 `DM_PASSWORD` 环境变量、`db.host`/`db.port`,以及 dmPython 与达梦客户端是否正确安装。
|
||||
|
||||
**Q: main_227 中 nodes.json、triples.json 从哪来?**
|
||||
A: `nodes.json`、`triples.json` 均由清洗步骤生成(run_all 或 main_227 前半段),格式需包含 `head`、`relation`、`tail` 等字段。
|
||||
|
||||
**Q: 如何查看达梦表结构?**
|
||||
A: 运行 `python scripts/inspect_dm_tables.py`(需设置 `DM_PASSWORD`)。
|
||||
|
||||
**Q: nodes.json 为何没有 X、Y、ROLE_ID 等原始字段?**
|
||||
A: 输出完全对齐 1125 xlsx 规范,仅保留 schema 定义的 12 个字段;原始字段在转换阶段已映射为 `spatial`、`org_unit` 等规范结构。
|
||||
|
||||
---
|
||||
|
||||
## 数据清洗详细逻辑
|
||||
|
||||
### 清洗流程概览
|
||||
|
||||
```
|
||||
原始数据 → 去重 → 缺失值填充 → 异常值修正 → 噪声平滑 → 1125 Schema 转换 → 关系计算前净化 → nodes.json
|
||||
```
|
||||
|
||||
| 步骤 | 说明 |
|
||||
|------|------|
|
||||
| **去重** | 按 `TARGET_ID` 去重,优先保留 `CREATED_TIME` 最新的记录 |
|
||||
| **缺失值填充** | 数值型字段按 `ROLE_ID` 分组均值填充,兜底用全局均值 |
|
||||
| **异常值修正** | 对 `TARGET_RECOGNITION_CAPABILITY`、`STRIKE_ACCURACY`、`ANTI_JAMMING_CAPABILITY`、`ENVIRONMENT_ADAPTABILITY`、`MOBILITY` 等 [0,1] 字段:负值取绝对值、>1 截断为 1 |
|
||||
| **噪声平滑** | 对 `COMMUNICATION_RANGE` 做 3 点移动平均 |
|
||||
| **1125 Schema 转换** | 见下文字段映射,输出仅含规范字段 |
|
||||
| **关系计算前净化** | 见下文「关系计算前净化」 |
|
||||
|
||||
### 关系计算前净化(sanitize_node_for_relation_calc)
|
||||
|
||||
下游 `f_relation1125` 及 `utils.py` 会对节点属性做数值运算(如欧氏距离、安全等级差、历史成功率平均),若字段为 `None` 会触发 `NoneType` 运算错误。因此在写入 `nodes.json` 前,对每条节点做最终净化,确保参与运算的字段**永不为 `None`**:
|
||||
|
||||
| 净化项 | 处理方式 |
|
||||
|--------|----------|
|
||||
| `spatial.position` | 必须为 `[float, float]`,含 `None`/`NaN` 的元素替换为 `0.0` |
|
||||
| `spatial.effective_radius` | 若为 `None`,设为 `1.0` |
|
||||
| `temporal.*` | `time_window` 缺省 `24.0`,其余缺省 `1.0` |
|
||||
| `security_level` | 若为 `None` 或越界,设为 `3`(1–5 范围) |
|
||||
| `history_success` | 若为 `None`,设为 `0.8`(0–1 范围) |
|
||||
|
||||
该步骤在 `attribute_schema.sanitize_node_for_relation_calc()` 中实现,由 `cleaner` 在写入前对每个节点调用。
|
||||
|
||||
### 输出字段说明
|
||||
|
||||
**nodes.json 完全对齐 xlsx 规范**,不保留原始其他字段。每条记录仅包含:
|
||||
|
||||
| 字段 | 类型 | 说明 |
|
||||
|------|------|------|
|
||||
| `TARGET_ID` | 字符串 | 节点唯一标识(三元组 head/tail 引用所需) |
|
||||
| `function_vector` | 字典 | 7 维功能能力 [0,1] |
|
||||
| `spatial` | 字典 | 空间位置与有效半径 |
|
||||
| `temporal` | 字典 | 时间相关属性 |
|
||||
| `performance` | 字典 | 核心性能、生存能力、MTBF |
|
||||
| `protocol_list` | 列表 | 支持协议(字符串列表) |
|
||||
| `format_list` | 列表 | 支持格式(字符串列表) |
|
||||
| `interface_list` | 列表 | 支持接口(字符串列表) |
|
||||
| `security_level` | 整数 | 安全等级 [1,5],缺省 3(关系计算用) |
|
||||
| `org_unit` | 字符串/null | 组织隶属 |
|
||||
| `nation` | 字符串/null | 国家代码 |
|
||||
| `history_success` | 浮点数 | 历史成功率 [0,1],缺省 0.8(关系计算用) |
|
||||
|
||||
**nodes.json 整体结构**:`{ "nodes": { "TARGET_ID": {...}, ... }, "global_params": { "mtbf_max": number } }`,兼容 main_227 / f_relation1125 关系计算。
|
||||
|
||||
**单条节点记录示例:**
|
||||
|
||||
```json
|
||||
{
|
||||
"TARGET_ID": "FS-俄-情报侦-132",
|
||||
"function_vector": {
|
||||
"f_IC": 0.9273, "f_IA": 0.7896, "f_CS": 0.2584, "f_IT": 0.5187,
|
||||
"f_DP": 0.9148, "f_CPS": 0.5619, "f_CC": 0.7822
|
||||
},
|
||||
"spatial": { "position": [79.36, 52.11], "effective_radius": 94.96 },
|
||||
"temporal": { "response_time": 23.3891, "cycle_time": 3.7, "data_age": 1.48, "time_window": 3.7 },
|
||||
"performance": { "core_performance": 0.9831, "survivability": 0.4094, "mtbf": 0.7202 },
|
||||
"protocol_list": [], "format_list": [], "interface_list": [],
|
||||
"security_level": 3, "org_unit": "师级", "nation": "俄", "history_success": 0.8
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 属性对齐与字段转换(1125 规范)
|
||||
|
||||
依据 `1125暂时属性需求(1).xlsx`,对每条记录做 **内容与格式的双重对齐**:
|
||||
|
||||
- **格式校验**:各字段类型与结构(dict/list/scalar)
|
||||
- **内容校验**:取值范围(如 [0,1]、[1,5])
|
||||
- **转换**:原始扁平字段 → 规范 schema
|
||||
|
||||
清洗报告中 `details.attribute_alignment` 记录格式/内容错误统计及示例。
|
||||
|
||||
### 原始字段 → 目标 Schema 映射
|
||||
|
||||
#### function_vector(功能向量)
|
||||
|
||||
每个维度取 [0,1] 浮点数,从原始能力字段推导:
|
||||
|
||||
| 目标维度 | 原始字段(按优先级) | 转换方式 |
|
||||
|----------|----------------------|----------|
|
||||
| f_IC 情报收集 | DETECTION_ACCURACY, TARGET_RECOGNITION_CAPABILITY | 取第一个有效值,clamp 到 [0,1] |
|
||||
| f_IA 信息分析 | INFORMATION_FUSION_CAPABILITY, DETECTION_ACCURACY | 同上 |
|
||||
| f_CS 协同作战 | MOBILITY, ENVIRONMENT_ADAPTABILITY | 同上 |
|
||||
| f_IT 信息传输 | ANTI_JAMMING_CAPABILITY | 同上 |
|
||||
| f_DP 数据处理 | PROCESSING_CAPACITY, INFORMATION_FUSION_CAPABILITY | **固定参考值**:见下表 |
|
||||
| f_CPS 指挥控制 | DECISION_RESPONSE_TIME | **固定参考值**:见下表 |
|
||||
| f_CC 综合保障 | SUPPORT_CAPACITY, LOAD_CAPACITY | **固定参考值**:见下表 |
|
||||
|
||||
**固定参考值转换公式**(原始值不在 [0,1] 时使用):
|
||||
|
||||
| 维度 | 公式 | 参数说明 |
|
||||
|------|------|----------|
|
||||
| **f_CPS** | `1 / (1 + t/30)` | t = 响应时间(秒),响应越快能力越高 |
|
||||
| **f_DP** | `log10(1+PC) / log10(10001)` | PC = PROCESSING_CAPACITY;无则用 INFORMATION_FUSION_CAPABILITY clamp [0,1] |
|
||||
| **f_CC** | `0.6×support_norm + 0.4×load_norm` | support_norm = log10(1+SUPPORT)/log10(10001),load_norm = LOAD/100;仅一个有时取该值 |
|
||||
|
||||
#### spatial(空间属性)
|
||||
|
||||
| 目标字段 | 原始字段(按优先级) | 转换 |
|
||||
|----------|----------------------|------|
|
||||
| position | `position`(列表 [x,y])、`X`/`x`+`Y`/`y`、`POSITION_X`/`POSITION_Y`、`LONGITUDE`/`LATITUDE` | `[round(x,2), round(y,2)]`,`None`/`NaN` 替为 `0.0`,兼容达梦多种列名 |
|
||||
| effective_radius | COMMUNICATION_RANGE | 原值保留 4 位小数,缺省 `1.0` |
|
||||
|
||||
#### temporal(时间属性)
|
||||
|
||||
| 目标字段 | 原始字段 | 转换 |
|
||||
|----------|----------|------|
|
||||
| response_time | DECISION_RESPONSE_TIME | 原值 |
|
||||
| cycle_time | REFRESH_RATE | 原值 |
|
||||
| data_age | TRANSMISSION_DELAY | 原值(近似数据新鲜度) |
|
||||
| time_window | REFRESH_RATE | 原值 |
|
||||
|
||||
#### performance(性能属性)
|
||||
|
||||
| 目标字段 | 原始字段(按优先级) | 转换 |
|
||||
|----------|----------------------|------|
|
||||
| core_performance | STRIKE_ACCURACY, DETECTION_ACCURACY, MOBILITY | 取第一个有效值,clamp [0,1] |
|
||||
| survivability | ENVIRONMENT_ADAPTABILITY, ANTI_JAMMING_CAPABILITY | 同上 |
|
||||
| mtbf | RELIABILITY, SYSTEM_RELIABILITY | 取第一个有效值 |
|
||||
|
||||
#### 其他标量字段
|
||||
|
||||
| 目标字段 | 原始字段 | 转换 |
|
||||
|----------|----------|------|
|
||||
| protocol_list | PROTOCOL | 字符串→列表,空则 `[]` |
|
||||
| format_list | FORMAT | 同上 |
|
||||
| interface_list | INTERFACE | 同上 |
|
||||
| security_level | security_level, SECURITY_LEVEL | 整数 [1,5],缺省 `3`(避免关系计算时 `None` 运算错误) |
|
||||
| org_unit | LEVEL, ROLE_ID | 取第一个非空字符串 |
|
||||
| nation | COUNTRY_REGION | 原值 |
|
||||
| history_success | history_success, HISTORY_SUCCESS | clamp [0,1],缺省 `0.8`(同上) |
|
||||
|
||||
### 缺失与空值
|
||||
|
||||
- 无可映射原始值时,多数目标字段为 `null`(列表型为 `[]`)
|
||||
- `security_level`、`history_success`、`spatial.position` 等参与关系运算的字段**永不为 `null`**:缺省分别为 `3`、`0.8`、`[0,0]`,并在「关系计算前净化」阶段再次保证
|
||||
- 可选字段(如 `org_unit`、`nation`)缺失不影响记录通过校验
|
||||
86
final/config/config.json
Normal file
86
final/config/config.json
Normal file
@@ -0,0 +1,86 @@
|
||||
{
|
||||
"source": "json",
|
||||
"json": {
|
||||
"input_file": "data/raw_data_sample.json",
|
||||
"output_file": "data/nodes.json",
|
||||
"report_file": "report/detailed_cleaning_report.json"
|
||||
},
|
||||
"triplet": {
|
||||
"enabled": true,
|
||||
"input_file": null,
|
||||
"output_file": "data/triples.json",
|
||||
"report_file": "report/triplet_cleaning_report.json",
|
||||
"clean": {
|
||||
"deduplicate": true,
|
||||
"remove_null_core": true,
|
||||
"normalize_relation_type": true,
|
||||
"filter_orphans": false
|
||||
}
|
||||
},
|
||||
"db": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 5080,
|
||||
"user": "SYSDBA",
|
||||
"schema": "SYSDBA",
|
||||
"table": "OBJECTIVE_INSTANCE",
|
||||
"query": null,
|
||||
"triplet_table": "RELATION_INSTANCE",
|
||||
"triplet_query": null
|
||||
},
|
||||
"formal": {
|
||||
"enabled": false,
|
||||
"user_id": "ADMIN",
|
||||
"task_id": "TASK_FULL_KB",
|
||||
"source_sys_type": "关系推理后的体系",
|
||||
"target_sys_type": "形式化后的体系",
|
||||
"source_query": null,
|
||||
"user_column": "USER_ID",
|
||||
"task_column": "TASK_ID",
|
||||
"sys_type_column": "SYS_TYPE",
|
||||
"skip_sys_type_validation": false,
|
||||
"entity_id_column": "TARGET_ID",
|
||||
"objective_table": "OBJECTIVE_INSTANCE",
|
||||
"triplet_table": "RELATION_INSTANCE",
|
||||
"triplet_primary_key": "RELATION_INSTANCE_ID",
|
||||
"triplet_omit_primary_key_on_insert": true,
|
||||
"triple_batch_column": "BATCH_ID",
|
||||
"attribute_table": "METRIC_VALUE_INSTANCE",
|
||||
"metric_id_column": "METRIC_ID",
|
||||
"metric_value_column": "METRIC_VALUE",
|
||||
"metric_insert_static": null,
|
||||
"formal_result_table": "RELATION_INSTANCE",
|
||||
"analysis_table": "RES_FORMAL_ANALYSIS",
|
||||
"analysis_insert_columns": null,
|
||||
"analysis_static_columns": null,
|
||||
"rules_summary": null,
|
||||
"required_attrs": [
|
||||
"f_IC",
|
||||
"f_IA",
|
||||
"f_CS",
|
||||
"f_IT",
|
||||
"f_DP",
|
||||
"f_CPS",
|
||||
"f_CC",
|
||||
"position",
|
||||
"effective_radius",
|
||||
"response_time",
|
||||
"cycle_time",
|
||||
"data_age",
|
||||
"time_window",
|
||||
"core_performance",
|
||||
"survivability",
|
||||
"mtbf",
|
||||
"protocol_list",
|
||||
"format_list",
|
||||
"interface_list",
|
||||
"security_level",
|
||||
"org_unit",
|
||||
"nation",
|
||||
"history_success"
|
||||
]
|
||||
},
|
||||
"eval": {
|
||||
"nodes_file": "data/nodes.json",
|
||||
"triples_file": "data/triples.json"
|
||||
}
|
||||
}
|
||||
96
final/config/config.yaml
Normal file
96
final/config/config.yaml
Normal file
@@ -0,0 +1,96 @@
|
||||
# 数据清洗系统配置文件
|
||||
# 通过 source 切换:json(开发) / db(生产-达梦)
|
||||
|
||||
source: json # 开发用 json;部署达梦时改为 db(或直接改 config.json)
|
||||
|
||||
# JSON 文件模式(source=json 时生效)
|
||||
json:
|
||||
input_file: data/raw_data_sample.json
|
||||
output_file: data/nodes.json
|
||||
report_file: report/detailed_cleaning_report.json
|
||||
|
||||
# 网络关系/效能评估读取路径(main_227 在清洗后加载)
|
||||
eval:
|
||||
nodes_file: data/nodes.json # 与 json.output_file 一致(清洗输出)
|
||||
triples_file: data/triples.json # 与 triplet.output_file 一致(run_all 输出)
|
||||
|
||||
# 三元组(run_all 时一并处理;db 模式从库读,json 模式可配 input_file)
|
||||
triplet:
|
||||
enabled: true # false 可关闭三元组处理
|
||||
input_file: null # json 模式下可选,如 data/triplets_raw.json
|
||||
output_file: data/triples.json # 与 eval.triples_file 一致,便于 main_227 直接读取
|
||||
report_file: report/triplet_cleaning_report.json
|
||||
# 清洗配置:clean: false 关闭清洗;true 使用默认;或自定义子项
|
||||
clean:
|
||||
deduplicate: true # 按 (HEAD_ID, RELATION_TYPE, TAIL_ID) 去重
|
||||
remove_null_core: true # 移除 HEAD_ID/TAIL_ID/RELATION_TYPE 为空的记录
|
||||
normalize_relation_type: true # 规范化 RELATION_TYPE(去空格)
|
||||
filter_orphans: false # 过滤悬空引用(需加载实体表,稍慢)
|
||||
|
||||
# 达梦数据库模式(source=db 时生效)
|
||||
# 密码通过环境变量 DM_PASSWORD 传入,切勿写入配置文件
|
||||
db:
|
||||
host: 127.0.0.1
|
||||
port: 5080
|
||||
user: SYSDBA
|
||||
schema: SYSDBA
|
||||
# OBJECTIVE_INSTANCE 表结构与 raw_data_sample.json 一致,直接查询即可
|
||||
table: OBJECTIVE_INSTANCE
|
||||
query: null
|
||||
# 三元组表(RELATION_INSTANCE: HEAD_ID, RELATION_TYPE, TAIL_ID)
|
||||
triplet_table: RELATION_INSTANCE
|
||||
triplet_query: null # 可选:自定义 SQL
|
||||
|
||||
# 模块1.3 形式化入库流程(source=db 时生效)
|
||||
formal:
|
||||
enabled: false
|
||||
user_id: ADMIN
|
||||
task_id: TASK_FULL_KB
|
||||
source_sys_type: 关系推理后的体系
|
||||
target_sys_type: 形式化后的体系
|
||||
# 推荐:显式 SQL 只读「关系推理后的体系」,列名与库一致时最稳(为 null 则用下方列名拼 WHERE)
|
||||
source_query: null
|
||||
user_column: USER_ID
|
||||
task_column: TASK_ID
|
||||
sys_type_column: SYS_TYPE
|
||||
skip_sys_type_validation: false
|
||||
entity_id_column: TARGET_ID
|
||||
objective_table: OBJECTIVE_INSTANCE
|
||||
triplet_table: RELATION_INSTANCE
|
||||
triplet_primary_key: RELATION_INSTANCE_ID
|
||||
triplet_omit_primary_key_on_insert: true
|
||||
triple_batch_column: BATCH_ID
|
||||
attribute_table: METRIC_VALUE_INSTANCE
|
||||
metric_id_column: METRIC_ID
|
||||
metric_value_column: METRIC_VALUE
|
||||
metric_insert_static: null
|
||||
formal_result_table: RELATION_INSTANCE
|
||||
analysis_table: RES_FORMAL_ANALYSIS
|
||||
# 若自动匹配列失败,在此列出 RES_FORMAL_ANALYSIS 的实际列名(顺序与 VALUES 一致)
|
||||
analysis_insert_columns: null
|
||||
analysis_static_columns: null
|
||||
rules_summary: null
|
||||
required_attrs:
|
||||
- f_IC
|
||||
- f_IA
|
||||
- f_CS
|
||||
- f_IT
|
||||
- f_DP
|
||||
- f_CPS
|
||||
- f_CC
|
||||
- position
|
||||
- effective_radius
|
||||
- response_time
|
||||
- cycle_time
|
||||
- data_age
|
||||
- time_window
|
||||
- core_performance
|
||||
- survivability
|
||||
- mtbf
|
||||
- protocol_list
|
||||
- format_list
|
||||
- interface_list
|
||||
- security_level
|
||||
- org_unit
|
||||
- nation
|
||||
- history_success
|
||||
7002
final/data/cleaned_data_final.json
Normal file
7002
final/data/cleaned_data_final.json
Normal file
File diff suppressed because it is too large
Load Diff
7407
final/data/nodes.json
Normal file
7407
final/data/nodes.json
Normal file
File diff suppressed because it is too large
Load Diff
7352
final/data/raw_data_sample.json
Normal file
7352
final/data/raw_data_sample.json
Normal file
File diff suppressed because it is too large
Load Diff
60
final/data/triples.json
Normal file
60
final/data/triples.json
Normal file
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"triples": [
|
||||
{
|
||||
"head": "CMD001",
|
||||
"head_type": "指挥控制",
|
||||
"relation": "指挥控制",
|
||||
"tail": "DEF001",
|
||||
"tail_type": "信息获取"
|
||||
},
|
||||
{
|
||||
"head": "CMD001",
|
||||
"head_type": "指挥控制",
|
||||
"relation": "指挥控制",
|
||||
"tail": "STK001",
|
||||
"tail_type": "协同打击"
|
||||
},
|
||||
{
|
||||
"head": "DEF001",
|
||||
"head_type": "信息获取",
|
||||
"relation": "状态反馈",
|
||||
"tail": "CMD001",
|
||||
"tail_type": "指挥控制"
|
||||
},
|
||||
{
|
||||
"head": "DEF001",
|
||||
"head_type": "信息获取",
|
||||
"relation": "情报保障",
|
||||
"tail": "STK001",
|
||||
"tail_type": "协同打击"
|
||||
},
|
||||
{
|
||||
"head": "STK001",
|
||||
"head_type": "协同打击",
|
||||
"relation": "状态反馈",
|
||||
"tail": "CMD001",
|
||||
"tail_type": "指挥控制"
|
||||
},
|
||||
{
|
||||
"head": "STK001",
|
||||
"head_type": "协同打击",
|
||||
"relation": "协同作战",
|
||||
"tail": "STK002",
|
||||
"tail_type": "协同打击"
|
||||
},
|
||||
{
|
||||
"head": "LOG001",
|
||||
"head_type": "综合保障",
|
||||
"relation": "平台部署",
|
||||
"tail": "STK001",
|
||||
"tail_type": "协同打击"
|
||||
},
|
||||
{
|
||||
"head": "MED001",
|
||||
"head_type": "综合保障",
|
||||
"relation": "平台部署",
|
||||
"tail": "TRN001",
|
||||
"tail_type": "部署平台"
|
||||
}
|
||||
]
|
||||
}
|
||||
58
final/data/triplets.json
Normal file
58
final/data/triplets.json
Normal file
@@ -0,0 +1,58 @@
|
||||
[
|
||||
{
|
||||
"head": "CMD001",
|
||||
"head_type": "指挥控制",
|
||||
"relation": "指挥控制",
|
||||
"tail": "DEF001",
|
||||
"tail_type": "信息获取"
|
||||
},
|
||||
{
|
||||
"head": "CMD001",
|
||||
"head_type": "指挥控制",
|
||||
"relation": "指挥控制",
|
||||
"tail": "STK001",
|
||||
"tail_type": "协同打击"
|
||||
},
|
||||
{
|
||||
"head": "DEF001",
|
||||
"head_type": "信息获取",
|
||||
"relation": "状态反馈",
|
||||
"tail": "CMD001",
|
||||
"tail_type": "指挥控制"
|
||||
},
|
||||
{
|
||||
"head": "DEF001",
|
||||
"head_type": "信息获取",
|
||||
"relation": "情报保障",
|
||||
"tail": "STK001",
|
||||
"tail_type": "协同打击"
|
||||
},
|
||||
{
|
||||
"head": "STK001",
|
||||
"head_type": "协同打击",
|
||||
"relation": "状态反馈",
|
||||
"tail": "CMD001",
|
||||
"tail_type": "指挥控制"
|
||||
},
|
||||
{
|
||||
"head": "STK001",
|
||||
"head_type": "协同打击",
|
||||
"relation": "协同作战",
|
||||
"tail": "STK002",
|
||||
"tail_type": "协同打击"
|
||||
},
|
||||
{
|
||||
"head": "LOG001",
|
||||
"head_type": "综合保障",
|
||||
"relation": "平台部署",
|
||||
"tail": "STK001",
|
||||
"tail_type": "协同打击"
|
||||
},
|
||||
{
|
||||
"head": "MED001",
|
||||
"head_type": "综合保障",
|
||||
"relation": "平台部署",
|
||||
"tail": "TRN001",
|
||||
"tail_type": "部署平台"
|
||||
}
|
||||
]
|
||||
94
final/report/detailed_cleaning_report.json
Normal file
94
final/report/detailed_cleaning_report.json
Normal file
@@ -0,0 +1,94 @@
|
||||
{
|
||||
"timestamp": "2026-04-23 12:20:15",
|
||||
"summary": {
|
||||
"total_records": 210,
|
||||
"final_records": 200,
|
||||
"duplicates_removed": 10
|
||||
},
|
||||
"details": {
|
||||
"missing_values_fixed": {
|
||||
"X": 2,
|
||||
"Y": 5,
|
||||
"WEIGHT": 126,
|
||||
"AREA": 126,
|
||||
"DETECTION_RANGE": 166,
|
||||
"DETECTION_ACCURACY": 166,
|
||||
"REFRESH_RATE": 166,
|
||||
"TRANSMISSION_DELAY": 135,
|
||||
"RELIABILITY": 166,
|
||||
"DECISION_RESPONSE_TIME": 169,
|
||||
"PROCESSING_CAPACITY": 169,
|
||||
"INFORMATION_FUSION_CAPABILITY": 169,
|
||||
"SYSTEM_RELIABILITY": 169,
|
||||
"STRIKE_RANGE": 126,
|
||||
"STRIKE_ACCURACY": 126,
|
||||
"AMMUNITION_RESERVE": 126,
|
||||
"TARGET_RECOGNITION_CAPABILITY": 126,
|
||||
"LOAD_CAPACITY": 159,
|
||||
"PLATFORM_AREA": 180,
|
||||
"SUPPORT_RANGE": 159,
|
||||
"SUPPORT_CAPACITY": 139
|
||||
},
|
||||
"outliers_corrected": {
|
||||
"count": 59,
|
||||
"examples": [
|
||||
{
|
||||
"id": "FS-法-协同打-144",
|
||||
"field": "TARGET_RECOGNITION_CAPABILITY",
|
||||
"original": 1.1645873020580773,
|
||||
"corrected": 1.0,
|
||||
"reason": "Value out of range [0, 1]"
|
||||
},
|
||||
{
|
||||
"id": "FS-法-协同打-144",
|
||||
"field": "STRIKE_ACCURACY",
|
||||
"original": 1.2863907411837974,
|
||||
"corrected": 1.0,
|
||||
"reason": "Value out of range [0, 1]"
|
||||
},
|
||||
{
|
||||
"id": "FS-美-火力打-156",
|
||||
"field": "TARGET_RECOGNITION_CAPABILITY",
|
||||
"original": -0.8869593801185574,
|
||||
"corrected": 0.8869593801185574,
|
||||
"reason": "Value out of range [0, 1]"
|
||||
},
|
||||
{
|
||||
"id": "FS-俄-协同打-061",
|
||||
"field": "TARGET_RECOGNITION_CAPABILITY",
|
||||
"original": 1.1931823083468704,
|
||||
"corrected": 1.0,
|
||||
"reason": "Value out of range [0, 1]"
|
||||
},
|
||||
{
|
||||
"id": "FS-美-协同打-121",
|
||||
"field": "STRIKE_ACCURACY",
|
||||
"original": 1.439367437550393,
|
||||
"corrected": 1.0,
|
||||
"reason": "Value out of range [0, 1]"
|
||||
}
|
||||
]
|
||||
},
|
||||
"noise_reduction": {
|
||||
"method": "Kalman Filter",
|
||||
"fields_processed": [
|
||||
"COMMUNICATION_RANGE"
|
||||
],
|
||||
"total_smoothed": 200
|
||||
},
|
||||
"standardization": [
|
||||
"Coordinates normalized to 2 decimal places",
|
||||
"Timestamps formatted to ISO-8601",
|
||||
"Attributes aligned to 1125 schema (format + content)"
|
||||
],
|
||||
"attribute_alignment": {
|
||||
"enabled": true,
|
||||
"format_errors_by_field": {},
|
||||
"content_errors_by_field": {},
|
||||
"records_with_errors": 0,
|
||||
"aligned_count": 200,
|
||||
"transformed_count": 200,
|
||||
"samples": []
|
||||
}
|
||||
}
|
||||
}
|
||||
20
final/report/triplet_cleaning_report.json
Normal file
20
final/report/triplet_cleaning_report.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"timestamp": "2026-02-27 19:45:51",
|
||||
"original_count": 8,
|
||||
"deduplicate": {
|
||||
"enabled": true,
|
||||
"removed": 0
|
||||
},
|
||||
"filter_orphans": {
|
||||
"enabled": false,
|
||||
"removed": 0
|
||||
},
|
||||
"remove_null_core": {
|
||||
"enabled": true,
|
||||
"removed": 0
|
||||
},
|
||||
"normalize_relation_type": {
|
||||
"enabled": true
|
||||
},
|
||||
"final_count": 8
|
||||
}
|
||||
5
final/requirements.txt
Normal file
5
final/requirements.txt
Normal file
@@ -0,0 +1,5 @@
|
||||
numpy>=1.20.0
|
||||
pandas>=1.3.0
|
||||
PyYAML>=5.4
|
||||
openpyxl>=3.0.0
|
||||
# 达梦模式需额外:pip install dmPython
|
||||
83
final/results/evaluation_report.txt
Normal file
83
final/results/evaluation_report.txt
Normal file
@@ -0,0 +1,83 @@
|
||||
============================================================
|
||||
网络关系与效能评价报告
|
||||
============================================================
|
||||
|
||||
1. 基本信息
|
||||
----------------------------------------
|
||||
过滤阈值: 0.15 【关系强度阈值】
|
||||
总三元组数量: 8
|
||||
分别有:
|
||||
有效计算三元组: 0
|
||||
跳过三元组: 8
|
||||
通过过滤三元组: 0
|
||||
通过率: 0.0%
|
||||
|
||||
2. 计算公式
|
||||
边关系计算公式
|
||||
--------------------------------------------------
|
||||
情报保障关系 L_IS(i,j) = w_f·M_F^IS + w_s·M_S^IS + w_t·M_T^IS + w_p·M_P^IS + w_i·M_I^IS
|
||||
指挥控制关系 L_CC(i,j) = w_f·M_F^CC + w_s·M_S^CC + w_t·M_T^CC + w_p·M_P^CC + w_i·M_I^CC
|
||||
状态反馈关系 L_SF(i,j) = w_f·M_F^SF + w_s·M_S^SF + w_t·M_T^SF + w_p·M_P^SF + w_i·M_I^SF
|
||||
平台部署关系 L_PD(i,j) = w_f·M_F^PD + w_s·M_S^PD + w_t·M_T^PD + w_p·M_P^PD + w_i·M_I^PD
|
||||
协同作战关系 L_CO(i,j) = w_f·M_F^CO + w_s·M_S^CO + w_t·M_T^CO + w_p·M_P^CO + w_i·M_I^CO
|
||||
【符号说明】
|
||||
M_F^IS 情报保障功能匹配项
|
||||
M_S^IS 情报保障空间匹配项
|
||||
M_T^IS 情报保障时间匹配项
|
||||
M_P^IS 情报保障性能匹配项
|
||||
M_I^IS 情报保障交互匹配项
|
||||
|
||||
任务网络效能计算公式
|
||||
--------------------------------------------------
|
||||
综合防御 P_defense = α1·(∑f_IC/N_IC) + α2·(∑f_IA/N_IA) + α3·(∑L_IS/N_IS) + α4·(∑t_resp/N) + α5·(∑L_CC/N_CC)
|
||||
火力打击 P_fire = β1·(∑f_CS/N_CS) + β2·(∑f_IA/N_IA) + β3·(∑L_CO/N_CO) + β4·(∑p_core·f_CS/N_CS) + β5·(∑L_IS/N_IS)
|
||||
后勤保障 P_logistics = γ1·(∑f_CPS/N_CPS) + γ2·(∑f_DP/N_DP) + γ3·(∑L_PD/N_PD) + γ4·(∑p_rel/N) + γ5·(∑f_IT/N_IT)
|
||||
医疗救援 P_medical = δ1·(∑f_CPS/N_CPS) + δ2·(∑t_resp/N) + δ3·(∑f_IT/N_IT) + δ4·(∑L_SF/N_SF) + δ5·(∑fresh_i/N)
|
||||
紧急疏散 P_evacuation = ε1·(∑f_CC/N_CC) + ε2·(∑f_IT/N_IT) + ε3·(∑L_CC/N_CC) + ε4·(∑f_CPS/N_CPS) + ε5·(∑t_resp/N)
|
||||
【符号说明】
|
||||
N_IC 信息对抗节点数量
|
||||
∑f_IC/N_IC 信息对抗节点平均能力
|
||||
∑L_IS/N_IS 情报保障关系平均强度
|
||||
∑t_resp/N 系统平均响应速度
|
||||
∑L_CC/N_CC 指挥控制关系平均强度
|
||||
∑f_CS/N_CS 协同打击节点平均能力
|
||||
∑f_IA/N_IA 信息获取节点平均能力
|
||||
∑L_CO/N_CO 协同作战关系平均强度
|
||||
∑p_core·f_CS/N_CS 打击节点性能加权能力
|
||||
∑f_CPS/N_CPS 综合保障节点平均能力
|
||||
∑f_DP/N_DP 部署平台节点平均能力
|
||||
∑L_PD/N_PD 平台部署关系平均强度
|
||||
∑p_rel/N 系统平均可靠性
|
||||
∑f_IT/N_IT 信息传输节点平均能力
|
||||
∑L_SF/N_SF 状态反馈关系平均强度
|
||||
∑fresh_i/N 系统平均数据新鲜度
|
||||
α1...α5 综合防御权重 [0.30,0.25,0.20,0.15,0.10]
|
||||
β1...β5 火力打击权重 [0.35,0.25,0.20,0.10,0.10]
|
||||
γ1...γ5 后勤保障权重 [0.30,0.25,0.20,0.15,0.10]
|
||||
δ1...δ5 医疗救援权重 [0.35,0.25,0.20,0.10,0.10]
|
||||
ε1...ε5 紧急疏散权重 [0.30,0.25,0.20,0.15,0.10]
|
||||
|
||||
3. 关系类型统计
|
||||
----------------------------------------
|
||||
【被过滤掉的关系明细】
|
||||
----------------------------------------
|
||||
4. 所属任务网络评估
|
||||
----------------------------------------
|
||||
任务网络概率:综合防御 21.2% 火力打击 17.5% 后勤保障 20.2% 医疗救援 19.7% 紧急疏散 21.4%
|
||||
最有可能的任务网络:紧急疏散 可能性为:21.4 %
|
||||
5. 最终紧急疏散任务网络的五个维度评价
|
||||
功能维度 (Function):
|
||||
功能维度反映了网络中节点的功能匹配程度。得分: 0.917
|
||||
高于参考值,表示节点在功能上高度互补,能够有效协同工作。
|
||||
空间系维度 (Relationship):
|
||||
空间维度反映了节点之间的连接强度。得分: 0.000
|
||||
低于参考值,表示节点之间的连接较弱,信息传递效率较低。
|
||||
性能维度 (Performance):
|
||||
性能维度反映了节点的性能指标,如响应时间和可靠性。得分: 0.935
|
||||
高于参考值,表示节点在性能上表现出色,能够稳定运行。
|
||||
时间维度 (Temporal):
|
||||
时间维度反映了节点的时间同步性和响应能力。得分: 0.126
|
||||
低于参考值,表示节点在时间同步性上存在偏差,响应速度较慢。
|
||||
交互维度 (Interaction):
|
||||
交互维度反映了节点之间的协议、格式和安全兼容性。得分: 0.000
|
||||
低于参考值,表示节点在交互上存在兼容性问题,通信安全性较低。
|
||||
3
final/results/filtered_triples.json
Normal file
3
final/results/filtered_triples.json
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"triples": []
|
||||
}
|
||||
1
final/results/triples.csv
Normal file
1
final/results/triples.csv
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
|
|
BIN
final/results/triples.xlsx
Normal file
BIN
final/results/triples.xlsx
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/export_triplets.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/export_triplets.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/f_net1125.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/f_net1125.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/f_net1125.cpython-38.pyc
Normal file
BIN
final/scripts/__pycache__/f_net1125.cpython-38.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/f_relation1125.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/f_relation1125.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/f_relation1125.cpython-38.pyc
Normal file
BIN
final/scripts/__pycache__/f_relation1125.cpython-38.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/inspect_dm_tables.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/inspect_dm_tables.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/run_all.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/run_all.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/utils.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/utils.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/utils.cpython-38.pyc
Normal file
BIN
final/scripts/__pycache__/utils.cpython-38.pyc
Normal file
Binary file not shown.
113
final/scripts/export_triplets.py
Normal file
113
final/scripts/export_triplets.py
Normal file
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
从达梦数据库导出三元组数据到 JSON 文件
|
||||
三元组来源:RELATION_INSTANCE 表 (HEAD_ID, RELATION_TYPE, TAIL_ID)
|
||||
支持可选清洗:去重、过滤悬空引用、移除空值、规范化关系类型
|
||||
用法: python scripts/export_triplets.py
|
||||
需设置环境变量 DM_PASSWORD,且 config 中 source=db
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
_script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_project_root = os.path.dirname(_script_dir)
|
||||
sys.path.insert(0, _project_root)
|
||||
|
||||
|
||||
def _json_serializer(obj):
|
||||
"""处理 Timestamp、datetime、numpy 等不可直接 JSON 序列化的类型"""
|
||||
if hasattr(obj, 'isoformat'):
|
||||
return obj.isoformat()
|
||||
if hasattr(obj, 'item'):
|
||||
return obj.item()
|
||||
if isinstance(obj, float) and (obj != obj or obj == float('inf') or obj == float('-inf')):
|
||||
return None
|
||||
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
|
||||
|
||||
|
||||
def load_config(project_root: str) -> dict:
|
||||
"""加载 config"""
|
||||
config_dir = os.path.join(project_root, 'config')
|
||||
json_path = os.path.join(config_dir, 'config.json')
|
||||
yaml_path = os.path.join(config_dir, 'config.yaml')
|
||||
if os.path.exists(json_path):
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
if os.path.exists(yaml_path):
|
||||
try:
|
||||
import yaml
|
||||
with open(yaml_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
except ImportError:
|
||||
pass
|
||||
return {'source': 'json', 'db': {}, 'triplet': {}}
|
||||
|
||||
|
||||
def main():
|
||||
from src.data_loader import load_triplets_from_dm, load_from_dm
|
||||
from src.triplet_cleaner import clean_triplets, transform_triplet_output_format
|
||||
|
||||
config = load_config(_project_root)
|
||||
if config.get('source') != 'db':
|
||||
print("错误:三元组导出仅支持达梦模式,请将 config 中 source 改为 db")
|
||||
sys.exit(1)
|
||||
|
||||
tc = config.get('triplet', {})
|
||||
output_file = tc.get('output_file', 'data/triples.json')
|
||||
report_file = tc.get('report_file', 'report/triplet_cleaning_report.json')
|
||||
output_path = output_file if os.path.isabs(output_file) else os.path.join(_project_root, output_file)
|
||||
report_path = report_file if os.path.isabs(report_file) else os.path.join(_project_root, report_file)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
os.makedirs(os.path.dirname(report_path), exist_ok=True)
|
||||
|
||||
# 清洗配置(clean: false 则跳过;否则按子项配置)
|
||||
clean_cfg = tc.get('clean', True)
|
||||
do_clean = False
|
||||
if clean_cfg is True:
|
||||
do_clean = {'deduplicate': True, 'filter_orphans': False, 'remove_null_core': True, 'normalize_relation_type': True}
|
||||
elif isinstance(clean_cfg, dict):
|
||||
do_clean = clean_cfg
|
||||
|
||||
deduplicate = do_clean.get('deduplicate', True) if do_clean else False
|
||||
filter_orphans = do_clean.get('filter_orphans', False) if do_clean else False
|
||||
remove_null_core = do_clean.get('remove_null_core', True) if do_clean else False
|
||||
normalize_relation_type = do_clean.get('normalize_relation_type', True) if do_clean else False
|
||||
|
||||
print("正在从达梦读取三元组 (RELATION_INSTANCE)...")
|
||||
triplets = load_triplets_from_dm(config, _project_root)
|
||||
print(f"读取到 {len(triplets)} 条三元组")
|
||||
|
||||
valid_ids = None
|
||||
if filter_orphans:
|
||||
print("正在加载实体表以校验悬空引用...")
|
||||
entities = load_from_dm(config, _project_root)
|
||||
valid_ids = {str(e.get('TARGET_ID', '')).strip() for e in entities if e.get('TARGET_ID')}
|
||||
print(f"有效实体数: {len(valid_ids)}")
|
||||
|
||||
if do_clean and (deduplicate or filter_orphans or remove_null_core or normalize_relation_type):
|
||||
print("正在执行三元组清洗...")
|
||||
triplets, report = clean_triplets(
|
||||
triplets,
|
||||
valid_ids=valid_ids,
|
||||
deduplicate=deduplicate,
|
||||
filter_orphans=filter_orphans,
|
||||
remove_null_core=remove_null_core,
|
||||
normalize_relation_type=normalize_relation_type,
|
||||
)
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
print(f"清洗完成:{report['original_count']} -> {report['final_count']},报告: {report_path}")
|
||||
|
||||
# 转为输出格式:head, head_type, relation, tail, tail_type
|
||||
entities = load_from_dm(config, _project_root)
|
||||
id_to_role = {str(e.get('TARGET_ID', '')).strip(): str(e.get('ROLE_ID', '')).strip() for e in entities if e.get('TARGET_ID')}
|
||||
triplets = transform_triplet_output_format(triplets, id_to_role)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(triplets, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
print(f"已导出至 {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
112
final/scripts/f_net1125.py
Normal file
112
final/scripts/f_net1125.py
Normal file
@@ -0,0 +1,112 @@
|
||||
# f_net1125.py ← 边关系已算完,只算任务网络
|
||||
from __future__ import annotations
|
||||
import numpy as np
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
NodeID = str
|
||||
TaskScore = float # 网络效能得分 [0,1]
|
||||
|
||||
|
||||
class TaskNetworkEvaluator:
|
||||
"""
|
||||
输入:
|
||||
nodes - 节点数据(含 function_vector / performance / temporal)
|
||||
relations - 已算好的全部边关系
|
||||
格式:relations[("IS","DEF001","CMD001")] = 0.87
|
||||
"""
|
||||
def __init__(self,
|
||||
nodes: Dict[NodeID, Dict],
|
||||
relations: Dict[Tuple[str, NodeID, NodeID], float],
|
||||
weights: Dict[str, Dict[str, float]] = None):
|
||||
self.nodes = nodes
|
||||
self.rels = relations
|
||||
self.w = weights
|
||||
|
||||
# ---------- 工具 ----------
|
||||
def _avg(self, scores: List[float]) -> float:
|
||||
return float(np.mean(scores)) if scores else 0.0
|
||||
|
||||
def _collect_func(self, key: str) -> List[float]:
|
||||
"""所有节点指定功能值"""
|
||||
return [n["function_vector"][key] for n in self.nodes.values()]
|
||||
|
||||
def _collect_rel(self, tag: str) -> List[float]:
|
||||
"""预存关系强度列表"""
|
||||
return [v for k, v in self.rels.items() if k[0] == tag]
|
||||
|
||||
def _collect_resp(self) -> List[float]:
|
||||
"""响应时间得分"""
|
||||
return [np.exp(-0.1 * n["temporal"]["response_time"]) for n in self.nodes.values()]
|
||||
|
||||
# ---------- 2.1 综合防御 ----------
|
||||
def eval_defense(self) -> TaskScore:
|
||||
w = self.w["defense"]
|
||||
f_ic = self._avg(self._collect_func("f_IC"))
|
||||
f_ia = self._avg(self._collect_func("f_IA"))
|
||||
l_is = self._avg(self._collect_rel("IS"))
|
||||
t_resp = self._avg(self._collect_resp())
|
||||
l_cc = self._avg(self._collect_rel("CC"))
|
||||
return w["w1"] * f_ic + w["w2"] * f_ia + w["w3"] * l_is + w["w4"] * t_resp + w["w5"] * l_cc
|
||||
|
||||
# ---------- 2.2 火力打击 ----------
|
||||
def eval_fire(self) -> TaskScore:
|
||||
w = self.w["fire"]
|
||||
f_cs = self._avg(self._collect_func("f_CS"))
|
||||
f_ia = self._avg(self._collect_func("f_IA"))
|
||||
l_co = self._avg(self._collect_rel("CO"))
|
||||
perf_cs = self._avg([n["performance"]["core_performance"] * n["function_vector"]["f_CS"]
|
||||
for n in self.nodes.values()])
|
||||
l_is = self._avg(self._collect_rel("IS"))
|
||||
return w["w1"] * f_cs + w["w2"] * f_ia + w["w3"] * l_co + w["w4"] * perf_cs + w["w5"] * l_is
|
||||
|
||||
# ---------- 2.3 后勤保障 ----------
|
||||
def eval_logistics(self) -> TaskScore:
|
||||
w = self.w["logistics"]
|
||||
f_cps = self._avg(self._collect_func("f_CPS"))
|
||||
f_dp = self._avg(self._collect_func("f_DP"))
|
||||
l_pd = self._avg(self._collect_rel("PD"))
|
||||
p_rel = self._avg([n["performance"]["mtbf"] / 1000 for n in self.nodes.values()])
|
||||
f_it = self._avg(self._collect_func("f_IT"))
|
||||
return w["w1"] * f_cps + w["w2"] * f_dp + w["w3"] * l_pd + w["w4"] * p_rel + w["w5"] * f_it
|
||||
|
||||
# ---------- 2.4 医疗救援 ----------
|
||||
def eval_medical(self) -> TaskScore:
|
||||
w = self.w["medical"]
|
||||
f_cps = self._avg(self._collect_func("f_CPS"))
|
||||
t_resp = self._avg(self._collect_resp())
|
||||
f_it = self._avg(self._collect_func("f_IT"))
|
||||
l_sf = self._avg(self._collect_rel("SF"))
|
||||
fresh = self._avg([np.exp(-0.1 * n["temporal"]["data_age"]) for n in self.nodes.values()])
|
||||
return w["w1"] * f_cps + w["w2"] * t_resp + w["w3"] * f_it + w["w4"] * l_sf + w["w5"] * fresh
|
||||
|
||||
# ---------- 2.5 紧急疏散 ----------
|
||||
def eval_evacuation(self) -> TaskScore:
|
||||
w = self.w["evacuation"]
|
||||
f_cc = self._avg(self._collect_func("f_CC"))
|
||||
f_it = self._avg(self._collect_func("f_IT"))
|
||||
l_cc = self._avg(self._collect_rel("CC"))
|
||||
f_cps = self._avg(self._collect_func("f_CPS"))
|
||||
t_resp = self._avg(self._collect_resp())
|
||||
return w["w1"] * f_cc + w["w2"] * f_it + w["w3"] * l_cc + w["w4"] * f_cps + w["w5"] * t_resp
|
||||
|
||||
# ---------- 一键评估 ----------
|
||||
def eval_all(self) -> Dict[str, TaskScore]:
|
||||
results = {
|
||||
"defense": self.eval_defense(),
|
||||
"fire": self.eval_fire(),
|
||||
"logistics": self.eval_logistics(),
|
||||
"medical": self.eval_medical(),
|
||||
"evacuation": self.eval_evacuation(),
|
||||
}
|
||||
|
||||
# 计算五个维度的分数
|
||||
dimensions = {
|
||||
"function": self._avg(self._collect_func("f_IC")),
|
||||
"relationship": self._avg(self._collect_rel("IS")),
|
||||
"performance": self._avg([n["performance"]["core_performance"] for n in self.nodes.values()]),
|
||||
"temporal": self._avg(self._collect_resp()),
|
||||
"interaction": self._avg(self._collect_rel("SF"))
|
||||
}
|
||||
|
||||
results["dimensions"] = dimensions
|
||||
return results
|
||||
395
final/scripts/f_relation1125.py
Normal file
395
final/scripts/f_relation1125.py
Normal file
@@ -0,0 +1,395 @@
|
||||
import numpy as np
|
||||
from typing import Dict
|
||||
from .utils import NetworkUtils
|
||||
|
||||
|
||||
class RelationCalculator:
|
||||
"""关系强度计算器"""
|
||||
|
||||
def __init__(self, nodes_data: Dict, cfg):
|
||||
"""
|
||||
初始化
|
||||
:param nodes_data: 节点数据
|
||||
:param auxiliary_data: 辅助数据(接口标准、角色匹配等)
|
||||
"""
|
||||
self.nodes_data = nodes_data
|
||||
self.utils = NetworkUtils()
|
||||
self.cfg = cfg
|
||||
|
||||
def _weights(self, rel: str):
|
||||
return self.cfg["W_REL"][rel]
|
||||
|
||||
def _lambda(self, rel: str):
|
||||
return self.cfg["LAMBDA"][rel]
|
||||
|
||||
def calculate_IS_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算情报保障关系强度
|
||||
L_IS(i,j) = w_f·M_IS_F + w_s·M_IS_S + w_t·M_IS_T + w_p·M_IS_P + w_i·M_IS_I
|
||||
"""
|
||||
# 权重参数(根据1.3.pdf)
|
||||
w = self._weights("IS")
|
||||
# 提取节点数据
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_IS_F
|
||||
f_i_IA = node_i['function_vector']['f_IA']
|
||||
f_j_avg = (
|
||||
node_j['function_vector']['f_CC'] +
|
||||
node_j['function_vector']['f_IC'] +
|
||||
node_j['function_vector']['f_CS'] +
|
||||
node_j['function_vector']['f_CPS']
|
||||
) / 4
|
||||
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_IS_F = f_i_IA * f_j_avg * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_IS_S
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
lambda_IS = self._lambda("IS")
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
M_IS_S = np.exp(-d_ij / lambda_IS) * R_match * s_area
|
||||
|
||||
# 3. 时间维度
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
fresh_i = np.exp(-gamma * node_i['temporal']['data_age'])
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_IS_T = fresh_i * t_i_resp * w_ij
|
||||
|
||||
# 4. 性能维度 M_IS_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
mtbf_max = self.nodes_data['global_params']['mtbf_max']
|
||||
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
|
||||
M_IS_P = p_i_core * p_i_rel
|
||||
|
||||
# 5. 交互维度 M_IS_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
|
||||
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
|
||||
hist_ij = self.utils.get_interaction_history(node_i, node_j)
|
||||
M_IS_I = ((prot_ij + fmt_ij + sec_ij) / 3) * hist_ij
|
||||
|
||||
# 综合计算
|
||||
L_IS = (w["w_f"] * M_IS_F + w["w_s"] * M_IS_S + w["w_t"] * M_IS_T +
|
||||
w["w_p"] * M_IS_P + w["w_i"] * M_IS_I)
|
||||
|
||||
return L_IS
|
||||
|
||||
def calculate_CC_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算指挥控制关系强度
|
||||
L_CC(i,j) = w_f·M_CC_F + w_s·M_CC_S + w_t·M_CC_T + w_p·M_CC_P + w_i·M_CC_I
|
||||
"""
|
||||
# 权重参数
|
||||
w = self._weights("CC")
|
||||
lambda_CC = self._lambda("CC")# 根据公式,指挥控制距离敏感性更高
|
||||
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_CC_F
|
||||
f_i_CC = node_i['function_vector']['f_CC']
|
||||
f_j_sum = sum([
|
||||
node_j['function_vector']['f_IA'],
|
||||
node_j['function_vector']['f_IT'],
|
||||
node_j['function_vector']['f_IC'],
|
||||
node_j['function_vector']['f_CS'],
|
||||
node_j['function_vector']['f_DP'],
|
||||
node_j['function_vector']['f_CPS']
|
||||
])
|
||||
f_j_avg = f_j_sum / 6
|
||||
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_CC_F = f_i_CC * f_j_avg * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_CC_S
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
M_CC_S = np.exp(-d_ij / lambda_CC) * R_match * s_area
|
||||
|
||||
# 3. 时间维度 M_CC_T
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_CC_T = t_i_resp * t_i_cycle * w_ij
|
||||
|
||||
# 4. 性能维度 M_CC_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
p_i_surv = node_i['performance']['survivability']
|
||||
mtbf_max = self.nodes_data['global_params']['mtbf_max']
|
||||
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
|
||||
M_CC_P = p_i_core * p_i_surv * p_i_rel
|
||||
|
||||
# 5. 交互维度 M_CC_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
|
||||
org_ij = self.utils.get_organization_relation(node_i, node_j)
|
||||
hist_ij = self.utils.get_interaction_history(node_i, node_j)
|
||||
M_CC_I = ((prot_ij + sec_ij + org_ij) / 3) * hist_ij
|
||||
|
||||
# 综合计算
|
||||
L_CC = (w["w_f"] * M_CC_F + w["w_s"] * M_CC_S + w["w_t"] * M_CC_T +
|
||||
w["w_p"] * M_CC_P + w["w_i"] * M_CC_I)
|
||||
|
||||
return L_CC
|
||||
|
||||
def calculate_SF_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算状态反馈关系强度
|
||||
L_SF(i,j) = w_f·M_SF_F + w_s·M_SF_S + w_t·M_SF_T + w_p·M_SF_P + w_i·M_SF_I
|
||||
"""
|
||||
# 权重参数
|
||||
w = self._weights("SF")
|
||||
lambda_SF = self._lambda("SF")
|
||||
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_SF_F
|
||||
f_i_sum = sum([
|
||||
node_i['function_vector']['f_IA'],
|
||||
node_i['function_vector']['f_IT'],
|
||||
node_i['function_vector']['f_IC'],
|
||||
node_i['function_vector']['f_CS'],
|
||||
node_i['function_vector']['f_DP'],
|
||||
node_i['function_vector']['f_CPS']
|
||||
])
|
||||
f_i_avg = f_i_sum / 6
|
||||
f_j_CC = node_j['function_vector']['f_CC']
|
||||
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_SF_F = f_i_avg * f_j_CC * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_SF_S
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
M_SF_S = np.exp(-d_ij / lambda_SF) * R_match * s_area
|
||||
|
||||
# 3. 时间维度 M_SF_T(状态反馈对时间敏感)
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
fresh_i = np.exp(-gamma * node_i['temporal']['data_age'])
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_SF_T = fresh_i * t_i_resp * w_ij
|
||||
|
||||
# 4. 性能维度 M_SF_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
mtbf_max = self.nodes_data['global_params']['mtbf_max']
|
||||
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
|
||||
M_SF_P = p_i_core * p_i_rel
|
||||
|
||||
# 5. 交互维度 M_SF_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
|
||||
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
|
||||
hist_ij = self.utils.get_interaction_history(node_i, node_j)
|
||||
M_SF_I = ((prot_ij + fmt_ij + sec_ij) / 3) * hist_ij
|
||||
|
||||
# 综合计算
|
||||
L_SF = (w["w_f"] * M_SF_F + w["w_s"] * M_SF_S + w["w_t"] * M_SF_T +
|
||||
w["w_p"] * M_SF_P + w["w_i"] * M_SF_I)
|
||||
|
||||
return L_SF
|
||||
|
||||
def calculate_PD_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算平台部署关系强度
|
||||
L_PD(i,j) = w_f·M_PD_F + w_s·M_PD_S + w_t·M_PD_T + w_p·M_PD_P + w_i·M_PD_I
|
||||
"""
|
||||
# 权重参数
|
||||
w = self._weights("PD")
|
||||
lambda_PD = self._lambda("PD")
|
||||
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_PD_F
|
||||
f_i_DP = node_i['function_vector']['f_DP']
|
||||
f_J_sum = sum([
|
||||
node_j['function_vector']['f_IA'],
|
||||
node_j['function_vector']['f_IT'],
|
||||
node_j['function_vector']['f_IC'],
|
||||
node_j['function_vector']['f_CS'],
|
||||
])
|
||||
f_J_avg = f_J_sum / 4
|
||||
|
||||
# 修复:直接传递节点数据
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_PD_F = f_i_DP * f_J_avg * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_PD_S(平台部署对空间要求高)
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
# 修复:直接传递节点数据
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
if d_ij <= self.cfg["PD_SGM"]:
|
||||
M_PD_S = 1
|
||||
else:
|
||||
M_PD_S = np.exp(-d_ij / lambda_PD) * R_match * s_area
|
||||
|
||||
# 3. 时间维度 M_PD_T
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_PD_T = t_i_resp * t_i_cycle * w_ij
|
||||
|
||||
# 4. 性能维度 M_PD_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
# 部署平台的生存能力
|
||||
p_i_surv = node_i['performance']['survivability']
|
||||
M_PD_P = p_i_core * p_i_surv
|
||||
|
||||
# 5. 交互维度 M_PD_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
|
||||
# 注意:这里std_ij已经在上面计算过了
|
||||
M_PD_I = (prot_ij + fmt_ij + std_ij) / 3
|
||||
|
||||
# 综合计算
|
||||
L_PD = (w["w_f"] * M_PD_F + w["w_s"] * M_PD_S + w["w_t"] * M_PD_T +
|
||||
w["w_p"] * M_PD_P + w["w_i"] * M_PD_I)
|
||||
|
||||
return L_PD
|
||||
|
||||
def calculate_CO_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算协同作战关系强度
|
||||
L_CO(i,j) = w_f·M_CO_F + w_s·M_CO_S + w_t·M_CO_T + w_p·M_CO_P + w_i·M_CO_I
|
||||
"""
|
||||
# 权重参数
|
||||
w = self._weights("CO")
|
||||
lambda_CO = self._lambda("CO")
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_CO_F
|
||||
# 协同作战需要多个功能匹配
|
||||
f_i_avg = sum(node_i['function_vector'].values()) / len(node_i['function_vector'])
|
||||
f_j_avg = sum(node_j['function_vector'].values()) / len(node_j['function_vector'])
|
||||
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_CO_F = f_i_avg * f_j_avg * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_CO_S
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
M_CO_S = np.exp(-d_ij / lambda_CO) * R_match * s_area
|
||||
|
||||
# 3. 时间维度 M_CO_T(协同需要时间同步)
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
t_j_resp = np.exp(-alpha * node_j['temporal']['response_time'])
|
||||
|
||||
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
|
||||
t_j_cycle = np.exp(-beta * node_j['temporal']['cycle_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_CO_T = (t_i_resp + t_j_resp) * (t_i_cycle + t_j_cycle) * w_ij * 0.25
|
||||
|
||||
# 4. 性能维度 M_CO_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
p_j_core = node_j['performance']['core_performance']
|
||||
|
||||
mtbf_max = self.nodes_data['global_params']['mtbf_max']
|
||||
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
|
||||
p_j_rel = node_j['performance']['mtbf'] / mtbf_max
|
||||
|
||||
p_i_surv = node_i['performance']['survivability']
|
||||
p_j_surv = node_j['performance']['survivability']
|
||||
# 协同性能取平均
|
||||
M_CO_P = ((p_i_core + p_j_core) / 2) * ((p_i_rel + p_j_rel) / 2) * ((p_i_surv + p_j_surv) / 2)
|
||||
|
||||
# 5. 交互维度 M_CO_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
|
||||
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
|
||||
org_ij = self.utils.get_organization_relation(node_i, node_j)
|
||||
hist_ij = self.utils.get_interaction_history(node_i, node_j)
|
||||
M_CO_I = ((prot_ij + fmt_ij + sec_ij + org_ij) / 4) * hist_ij
|
||||
|
||||
# 综合计算
|
||||
L_CO = (w["w_f"] * M_CO_F + w["w_s"] * M_CO_S + w["w_t"] * M_CO_T +
|
||||
w["w_p"] * M_CO_P + w["w_i"] * M_CO_I)
|
||||
|
||||
|
||||
return L_CO
|
||||
241
final/scripts/inspect_dm_tables.py
Normal file
241
final/scripts/inspect_dm_tables.py
Normal file
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
在达梦数据库机器上运行,查看各表结构,用于编写 config 中的 db.query
|
||||
用法: python scripts/inspect_dm_tables.py
|
||||
|
||||
密码获取顺序:环境变量 DM_PASS > 下方 LOCAL_PASSWORD
|
||||
若使用 LOCAL_PASSWORD,切勿提交到 git!可执行:git update-index --assume-unchanged scripts/inspect_dm_tables.py
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 可选:在此直接填写密码(仅限本机调试,切勿提交到版本库!)
|
||||
LOCAL_PASSWORD = None # 例如: "你的密码"
|
||||
|
||||
# 支持从项目根或 scripts 目录运行
|
||||
_script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_project_root = os.path.dirname(_script_dir)
|
||||
XLSX_PATH = os.path.join(_project_root, "1125暂时属性需求(1).xlsx")
|
||||
if _project_root not in sys.path:
|
||||
sys.path.insert(0, _project_root)
|
||||
|
||||
try:
|
||||
import dmPython
|
||||
except ImportError:
|
||||
print("请先安装: pip install dmPython")
|
||||
sys.exit(1)
|
||||
|
||||
CONFIG = {
|
||||
"user": os.getenv("DM_USER", "SYSDBA"),
|
||||
"password": os.getenv("DM_PASS") or LOCAL_PASSWORD,
|
||||
"server": os.getenv("DM_HOST", "127.0.0.1"),
|
||||
"port": int(os.getenv("DM_PORT", "9080")),
|
||||
"schema": "SYSDBA",
|
||||
}
|
||||
|
||||
TABLES = [
|
||||
"OBJECTIVE_INSTANCE",
|
||||
"METRIC_VALUE_INSTANCE",
|
||||
"METRIC_PROFILE",
|
||||
"ROLE_PROFILE",
|
||||
"TASK_PROFILE",
|
||||
"RELATION_INSTANCE",
|
||||
"RES_FORMAL_ANALYSIS",
|
||||
]
|
||||
|
||||
|
||||
def _load_required_attrs(xlsx_path: str) -> list:
|
||||
"""从 1125暂时属性需求(1).xlsx 解析需求属性列表(一条=一个属性)"""
|
||||
if not os.path.exists(xlsx_path):
|
||||
print(f" (未找到 {xlsx_path},使用内置属性列表)")
|
||||
return _DEFAULT_REQ_ATTRS
|
||||
try:
|
||||
import pandas as pd
|
||||
df = pd.read_excel(xlsx_path, header=None)
|
||||
attrs = []
|
||||
for _, row in df.iterrows():
|
||||
v0 = row[0]
|
||||
v2 = str(row[2]) if len(row) > 2 and pd.notna(row[2]) else ""
|
||||
if pd.notna(v0) and str(v0).strip() and str(v0) != "属性代码名称":
|
||||
attrs.append(str(v0).strip())
|
||||
if ":" in v2:
|
||||
sub = v2.split(":")[0].strip()
|
||||
if sub and sub not in attrs:
|
||||
attrs.append(sub)
|
||||
return attrs if attrs else _DEFAULT_REQ_ATTRS
|
||||
except Exception as e:
|
||||
print(f" (解析 xlsx 失败: {e},使用内置属性列表)")
|
||||
return _DEFAULT_REQ_ATTRS
|
||||
|
||||
|
||||
# xlsx 解析失败时的兜底属性列表(来自 1125暂时属性需求)
|
||||
_DEFAULT_REQ_ATTRS = [
|
||||
"f_IC", "f_IA", "f_CS", "f_IT", "f_DP", "f_CPS", "f_CC",
|
||||
"position", "effective_radius",
|
||||
"response_time", "cycle_time", "data_age", "time_window",
|
||||
"core_performance", "survivability", "mtbf",
|
||||
"protocol_list", "format_list", "interface_list",
|
||||
"security_level", "org_unit", "nation", "history_success",
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
if not CONFIG["password"]:
|
||||
print("请设置环境变量 DM_PASS 或在脚本中填写 LOCAL_PASSWORD")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
conn = dmPython.connect(
|
||||
user=CONFIG["user"],
|
||||
password=CONFIG["password"],
|
||||
server=CONFIG["server"],
|
||||
port=CONFIG["port"],
|
||||
schema=CONFIG["schema"],
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"连接失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
cursor = conn.cursor()
|
||||
print("=" * 60)
|
||||
print("达梦表结构(用于编写 config/db.query)")
|
||||
print("=" * 60)
|
||||
|
||||
for table in TABLES:
|
||||
try:
|
||||
cursor.execute(f'SELECT * FROM "{CONFIG["schema"]}"."{table}" WHERE 1=0')
|
||||
cols = [d[0] for d in cursor.description]
|
||||
print(f"\n【{table}】 列: {cols}")
|
||||
except Exception as e:
|
||||
print(f"\n【{table}】 查询失败: {e}")
|
||||
|
||||
# 专门查询 METRIC_VALUE_INSTANCE 的字段详情(含完整属性)
|
||||
mvi_table = "METRIC_VALUE_INSTANCE"
|
||||
print("\n" + "=" * 80)
|
||||
print(f"【{mvi_table}】 字段详情(完整属性)")
|
||||
print("=" * 80)
|
||||
try:
|
||||
# 查询 DBA_TAB_COLUMNS 全部常用属性(达梦兼容)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT COLUMN_NAME, DATA_TYPE, DATA_LENGTH, DATA_PRECISION, DATA_SCALE,
|
||||
NULLABLE, DATA_DEFAULT, COLUMN_ID
|
||||
FROM DBA_TAB_COLUMNS
|
||||
WHERE OWNER = ? AND TABLE_NAME = ?
|
||||
ORDER BY COLUMN_ID
|
||||
""",
|
||||
(CONFIG["schema"], mvi_table),
|
||||
)
|
||||
mvi_cols = cursor.fetchall()
|
||||
# 表头
|
||||
fmt = " {:<18} {:<16} {:<10} {:<10} {:<8} {:<10} {:<20}"
|
||||
print(fmt.format("列名", "数据类型", "长度", "精度", "标度", "可空", "默认值"))
|
||||
print(" " + "-" * 90)
|
||||
for row in mvi_cols:
|
||||
col_name, data_type, data_length, data_precision, data_scale, nullable, data_default, col_id = row
|
||||
# 类型显示:数值型用 类型(精度,标度),字符型用 类型(长度)
|
||||
if data_precision is not None:
|
||||
type_str = f"{data_type}({data_precision},{data_scale or 0})"
|
||||
elif data_length and data_type.upper() in ("VARCHAR", "CHAR", "VARCHAR2"):
|
||||
type_str = f"{data_type}({data_length})"
|
||||
elif data_length:
|
||||
type_str = f"{data_type}({data_length})"
|
||||
else:
|
||||
type_str = data_type or ""
|
||||
len_val = str(data_length) if data_length is not None else "-"
|
||||
prec_val = str(data_precision) if data_precision is not None else "-"
|
||||
scale_val = str(data_scale) if data_scale is not None else "-"
|
||||
null_str = "NULL" if nullable == "Y" else "NOT NULL"
|
||||
default_str = (str(data_default).strip() if data_default else "-")[:18]
|
||||
print(fmt.format(col_name, type_str, len_val, prec_val, scale_val, null_str, default_str))
|
||||
|
||||
# 查询样本数据(若有)
|
||||
cursor.execute(f'SELECT * FROM "{CONFIG["schema"]}"."{mvi_table}" WHERE ROWNUM <= 3')
|
||||
sample_rows = cursor.fetchall()
|
||||
if sample_rows:
|
||||
col_names = [d[0] for d in cursor.description]
|
||||
print(f"\n 样本数据 (最多 3 行):")
|
||||
for i, row in enumerate(sample_rows, 1):
|
||||
print(f" 行{i}: {dict(zip(col_names, row))}")
|
||||
else:
|
||||
print("\n (表无数据)")
|
||||
except Exception as e:
|
||||
print(f" 查询失败: {e}")
|
||||
|
||||
# 按 1125暂时属性需求(1).xlsx 查询 METRIC_VALUE_INSTANCE(一条=一个属性)
|
||||
print("\n" + "=" * 80)
|
||||
print("【METRIC_VALUE_INSTANCE】按需求属性查询(xlsx 中一条 = 一个 METRIC_ID)")
|
||||
print("=" * 80)
|
||||
try:
|
||||
# 1. 从 xlsx 解析需求属性列表
|
||||
req_attrs = _load_required_attrs(XLSX_PATH)
|
||||
print(f"\n xlsx 需求属性 ({len(req_attrs)} 个): {req_attrs}")
|
||||
|
||||
# 2. 查询库中实际存在的 METRIC_ID(distinct)
|
||||
cursor.execute(
|
||||
f'SELECT DISTINCT METRIC_ID FROM "{CONFIG["schema"]}"."METRIC_VALUE_INSTANCE" ORDER BY METRIC_ID'
|
||||
)
|
||||
db_metric_ids = [r[0] for r in cursor.fetchall()]
|
||||
print(f"\n 库中已有 METRIC_ID ({len(db_metric_ids)} 个): {db_metric_ids or '(无数据)'}")
|
||||
|
||||
# 3. METRIC_PROFILE 对照(METRIC_ID -> 中文名)
|
||||
cursor.execute(
|
||||
f'SELECT METRIC_ID, METRIC_NAME FROM "{CONFIG["schema"]}"."METRIC_PROFILE"'
|
||||
)
|
||||
profile_map = dict(cursor.fetchall())
|
||||
if profile_map:
|
||||
print("\n METRIC_PROFILE 对照:")
|
||||
for mid, mname in sorted(profile_map.items()):
|
||||
print(f" {mid} -> {mname}")
|
||||
|
||||
# 4. 交集:需求属性中在库里有的
|
||||
in_both = [a for a in req_attrs if a in db_metric_ids]
|
||||
# 需求有但库无
|
||||
missing = [a for a in req_attrs if a not in db_metric_ids]
|
||||
# 库有但需求未列
|
||||
extra = [m for m in db_metric_ids if m not in req_attrs]
|
||||
|
||||
print(f"\n 需求与库交集 (可查): {in_both or '无'}")
|
||||
print(f" 需求有但库无: {missing or '无'}")
|
||||
print(f" 库有但需求未列: {extra or '无'}")
|
||||
|
||||
# 5. 按属性查询示例 SQL
|
||||
print("\n --- 按属性查询示例 SQL ---")
|
||||
for attr in (in_both or req_attrs)[:5]: # 最多展示 5 个
|
||||
sql = f'SELECT * FROM "{CONFIG["schema"]}"."METRIC_VALUE_INSTANCE" WHERE METRIC_ID = \'{attr}\''
|
||||
print(f" -- {attr}: {sql}")
|
||||
if len(in_both or req_attrs) > 5:
|
||||
print(f" ... 其余属性同理,将 METRIC_ID = 'xxx' 替换即可")
|
||||
except Exception as e:
|
||||
print(f" 查询失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE
|
||||
FROM DBA_TAB_COLUMNS
|
||||
WHERE OWNER = ?
|
||||
ORDER BY TABLE_NAME, COLUMN_ID
|
||||
""",
|
||||
(CONFIG["schema"],),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("所有列详情 (TABLE_NAME | COLUMN_NAME | DATA_TYPE)")
|
||||
print("=" * 60)
|
||||
cur_table = None
|
||||
for table, col, dtype in rows:
|
||||
if table != cur_table:
|
||||
cur_table = table
|
||||
print(f"\n--- {table} ---")
|
||||
print(f" {col}: {dtype}")
|
||||
|
||||
cursor.close()
|
||||
conn.close()
|
||||
print("\n完成。请根据上述结构在 config 中编写 db.query。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
167
final/scripts/run_all.py
Normal file
167
final/scripts/run_all.py
Normal file
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
一键运行:实体属性表 + 三元组 的读取、清洗、导出
|
||||
用法: python scripts/run_all.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
_script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_project_root = os.path.dirname(_script_dir)
|
||||
sys.path.insert(0, _project_root)
|
||||
|
||||
|
||||
def _json_serializer(obj):
|
||||
if hasattr(obj, 'isoformat'):
|
||||
return obj.isoformat()
|
||||
if hasattr(obj, 'item'):
|
||||
return obj.item()
|
||||
if isinstance(obj, float) and (obj != obj or obj == float('inf') or obj == float('-inf')):
|
||||
return None
|
||||
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
|
||||
|
||||
|
||||
def load_config(project_root: str) -> dict:
|
||||
config_dir = os.path.join(project_root, 'config')
|
||||
json_path = os.path.join(config_dir, 'config.json')
|
||||
yaml_path = os.path.join(config_dir, 'config.yaml')
|
||||
if os.path.exists(json_path):
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
if os.path.exists(yaml_path):
|
||||
try:
|
||||
import yaml
|
||||
with open(yaml_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
except ImportError:
|
||||
pass
|
||||
return {'source': 'json', 'json': {}, 'triplet': {}, 'db': {}}
|
||||
|
||||
|
||||
def _resolve_path(path: str, project_root: str) -> str:
|
||||
return path if os.path.isabs(path) else os.path.join(project_root, path)
|
||||
|
||||
|
||||
def main():
|
||||
from src.cleaner import AdvancedDataCleaner
|
||||
from src.data_loader import load_data, load_triplets_from_dm, load_from_json, run_formalization_pipeline
|
||||
from src.triplet_cleaner import clean_triplets
|
||||
|
||||
config = load_config(_project_root)
|
||||
source = config.get('source', 'json')
|
||||
tc = config.get('triplet', {})
|
||||
|
||||
# 三元组:db 模式默认处理;json 模式需 triplet.input_file;triplet.enabled=false 可关闭
|
||||
triplet_input = tc.get('input_file')
|
||||
run_triplets = tc.get('enabled', source == 'db' or bool(triplet_input))
|
||||
|
||||
print("=" * 50)
|
||||
print("作战体系数据清洗 - 一键运行")
|
||||
print("=" * 50)
|
||||
print(f"数据源: {source}")
|
||||
|
||||
# ---------- 1. 实体属性表 ----------
|
||||
print("\n【1/2】实体属性表 读取与清洗")
|
||||
try:
|
||||
data, paths = load_data(config, _project_root)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"错误:{e}")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(os.path.dirname(paths['report']), exist_ok=True)
|
||||
cleaner = AdvancedDataCleaner(output_file=paths['output'], report_file=paths['report'], data=data)
|
||||
cleaner.run()
|
||||
valid_ids = cleaner.get_valid_target_ids()
|
||||
print(f" -> 输出: {paths['output']}")
|
||||
print(f" -> 报告: {paths['report']}")
|
||||
|
||||
# ---------- 2. 三元组 ----------
|
||||
if not run_triplets:
|
||||
print("\n【2/2】三元组 已跳过(triplet.enabled=false 或 json 模式未配置 input_file)")
|
||||
print("\n完成。")
|
||||
return
|
||||
|
||||
print("\n【2/2】三元组 读取与清洗")
|
||||
|
||||
if source == 'db':
|
||||
triplets = load_triplets_from_dm(config, _project_root)
|
||||
print(f" 从达梦读取 {len(triplets)} 条三元组")
|
||||
elif triplet_input:
|
||||
input_path = _resolve_path(triplet_input, _project_root)
|
||||
if os.path.exists(input_path):
|
||||
raw = load_from_json(input_path, _project_root)
|
||||
triplets = raw.get("triples", raw) if isinstance(raw, dict) else raw
|
||||
triplets = triplets if isinstance(triplets, list) else []
|
||||
print(f" 从 {triplet_input} 读取 {len(triplets)} 条三元组")
|
||||
else:
|
||||
print(f" 跳过:三元组文件不存在 {input_path}")
|
||||
triplets = []
|
||||
else:
|
||||
print(" 跳过:json 模式未配置 triplet.input_file")
|
||||
triplets = []
|
||||
|
||||
if not triplets:
|
||||
print(" 无三元组数据,跳过导出")
|
||||
print("\n完成。")
|
||||
return
|
||||
|
||||
# 三元组清洗配置
|
||||
clean_cfg = tc.get('clean', True)
|
||||
do_clean = {'deduplicate': True, 'filter_orphans': False, 'remove_null_core': True, 'normalize_relation_type': True} if clean_cfg is True else (clean_cfg if isinstance(clean_cfg, dict) else {})
|
||||
deduplicate = do_clean.get('deduplicate', True) if do_clean else False
|
||||
filter_orphans = do_clean.get('filter_orphans', False) if do_clean else False
|
||||
remove_null_core = do_clean.get('remove_null_core', True) if do_clean else False
|
||||
normalize_relation_type = do_clean.get('normalize_relation_type', True) if do_clean else False
|
||||
|
||||
if do_clean and (deduplicate or filter_orphans or remove_null_core or normalize_relation_type):
|
||||
triplets, report = clean_triplets(
|
||||
triplets,
|
||||
valid_ids=valid_ids if filter_orphans else None,
|
||||
deduplicate=deduplicate,
|
||||
filter_orphans=filter_orphans,
|
||||
remove_null_core=remove_null_core,
|
||||
normalize_relation_type=normalize_relation_type,
|
||||
)
|
||||
report_path = _resolve_path(tc.get('report_file', 'report/triplet_cleaning_report.json'), _project_root)
|
||||
os.makedirs(os.path.dirname(report_path), exist_ok=True)
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
print(f" 清洗: {report['original_count']} -> {report['final_count']}")
|
||||
print(f" 报告: {report_path}")
|
||||
|
||||
# 转为输出格式:head, head_type, relation, tail, tail_type
|
||||
from src.triplet_cleaner import transform_triplet_output_format
|
||||
id_to_role = cleaner.get_id_to_role()
|
||||
triplets = transform_triplet_output_format(triplets, id_to_role)
|
||||
|
||||
output_path = _resolve_path(tc.get('output_file', 'data/triples.json'), _project_root)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(triplets, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
print(f" 输出: {output_path}")
|
||||
|
||||
# ---------- 3. 模块1.3:形式化入库 ----------
|
||||
formal_cfg = config.get("formal", {})
|
||||
if source == "db" and formal_cfg.get("enabled", False):
|
||||
print("\n【3/3】形式化模块 读取-补齐-入库")
|
||||
try:
|
||||
result = run_formalization_pipeline(config)
|
||||
if result.get("ok"):
|
||||
stats = result.get("stats", {})
|
||||
print(f" 批次: {result.get('batch_id')}")
|
||||
print(f" 推理三元组: {stats.get('total_triples', 0)}")
|
||||
print(f" 形式化入库: {stats.get('inserted_formal_count', 0)}")
|
||||
print(f" 属性补齐写回: {stats.get('generated_attrs_count', 0)}")
|
||||
else:
|
||||
print(f" 形式化流程跳过: {result.get('reason', 'unknown')}")
|
||||
except Exception as e:
|
||||
print(f" 形式化流程失败: {e}")
|
||||
raise
|
||||
elif formal_cfg.get("enabled", False):
|
||||
print("\n【3/3】形式化模块 已跳过(仅 source=db 时启用)")
|
||||
|
||||
print("\n完成。")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
110
final/scripts/utils.py
Normal file
110
final/scripts/utils.py
Normal file
@@ -0,0 +1,110 @@
|
||||
import numpy as np
|
||||
from typing import Dict, List, Any
|
||||
from scipy.spatial.distance import euclidean
|
||||
|
||||
class NetworkUtils:
|
||||
"""无需 aux_data,全部现场从节点属性读取"""
|
||||
""" 角色匹配矩阵暂时没加 """
|
||||
# ----------- 通用兜底 -----------
|
||||
_DEFAULT = {
|
||||
"protocol": 0.80,
|
||||
"format": 0.85,
|
||||
"security": 0.90,
|
||||
"role": 0.70,
|
||||
"area": 0.50,
|
||||
"org": 0.70,
|
||||
"area_1": 1,
|
||||
"area_2": 0.7,
|
||||
"history": 0.8
|
||||
}
|
||||
|
||||
# ----------- 空间 -----------
|
||||
@staticmethod
|
||||
def calculate_distance(pos1, pos2):
|
||||
return euclidean(pos1, pos2)
|
||||
|
||||
@staticmethod
|
||||
def calculate_range_match(R_i: float, R_j: float, d_ij: float) -> float:
|
||||
r_min = min(R_i, R_j)
|
||||
if d_ij <= r_min:
|
||||
return 1.0
|
||||
return r_min / d_ij if d_ij > 0 else 0.0
|
||||
|
||||
@staticmethod
|
||||
def calculate_time_window_overlap(node_i, node_j):
|
||||
w_i = node_i["temporal"].get("time_window", 24)
|
||||
w_j = node_j["temporal"].get("time_window", 24)
|
||||
overlap = min(w_i, w_j)
|
||||
return overlap / max(w_i, w_j, 1)
|
||||
|
||||
# ----------- 协议兼容性:Jaccard -----------
|
||||
@staticmethod
|
||||
def get_protocol_compatibility(node_i, node_j, _dummy=None):
|
||||
"""节点属性里放 'protocol_list'"""
|
||||
p_i = set(node_i.get("protocol_list", []))
|
||||
p_j = set(node_j.get("protocol_list", []))
|
||||
if not p_i or not p_j:
|
||||
return NetworkUtils._DEFAULT["protocol"]
|
||||
intersection = len(p_i & p_j)
|
||||
union = len(p_i | p_j)
|
||||
return intersection / union if union else NetworkUtils._DEFAULT["protocol"]
|
||||
|
||||
# ----------- 安全等级:1-5 映射 0-1 -----------
|
||||
@staticmethod
|
||||
def get_security_compatibility(node_i, node_j, _dummy=None):
|
||||
lv_i = node_i.get("security_level", 3)
|
||||
lv_j = node_j.get("security_level", 3)
|
||||
gap = abs(lv_i - lv_j)
|
||||
return max(0, 1 - gap / 5)
|
||||
|
||||
# ----------- 组织隶属:同单位给高分 -----------
|
||||
@staticmethod
|
||||
def get_organization_relation(node_i, node_j, _dummy=None):
|
||||
unit_i = node_i.get("org_unit", "")
|
||||
unit_j = node_j.get("org_unit", "")
|
||||
return 1.0 if unit_i == unit_j and unit_i else NetworkUtils._DEFAULT["org"]
|
||||
|
||||
# ----------- 历史交互:直接读属性 -----------
|
||||
@staticmethod
|
||||
def get_interaction_history(node_i, node_j, _dummy=None):
|
||||
his_i = node_i.get("history_success", NetworkUtils._DEFAULT["history"])
|
||||
# print("111111111111111111111",node_i.get("history_success"))
|
||||
his_j = node_j.get("history_success", NetworkUtils._DEFAULT["history"])
|
||||
return (his_i + his_j) / 2
|
||||
|
||||
# ----------- 区域关联:同 code 高分 -----------
|
||||
@staticmethod
|
||||
def get_area_relation(node_i, node_j, _dummy=None):
|
||||
code_i = node_i.get("nation")
|
||||
code_j = node_j.get("nation")
|
||||
if code_i == code_j:
|
||||
return NetworkUtils._DEFAULT["area_1"]
|
||||
else:
|
||||
return NetworkUtils._DEFAULT["area_2"]
|
||||
|
||||
# ----------- 接口/角色/格式:若节点属性扩展了再读,否则给默认 -----------
|
||||
@staticmethod
|
||||
def get_role_match(node_i, node_j, _dummy=None):
|
||||
return node_i.get("role_match", NetworkUtils._DEFAULT["role"])
|
||||
|
||||
@staticmethod
|
||||
def get_interface_standard(node_i, node_j, _dummy=None) -> float:
|
||||
"""接口标准兼容性 = Jaccard(i_list, j_list)"""
|
||||
i_set = set(node_i.get("interface_list", []))
|
||||
j_set = set(node_j.get("interface_list", []))
|
||||
if not i_set or not j_set:
|
||||
return NetworkUtils._DEFAULT["protocol"] # 默认0.8
|
||||
inter = len(i_set & j_set)
|
||||
union = len(i_set | j_set)
|
||||
return inter / union if union else NetworkUtils._DEFAULT["protocol"]
|
||||
|
||||
@staticmethod
|
||||
def get_format_compatibility(node_i, node_j, _dummy=None) -> float:
|
||||
"""数据格式兼容性 = Jaccard(i_format, j_format)"""
|
||||
i_set = set(node_i.get("format_list", []))
|
||||
j_set = set(node_j.get("format_list", []))
|
||||
if not i_set or not j_set:
|
||||
return NetworkUtils._DEFAULT["format"] # 默认0.85
|
||||
inter = len(i_set & j_set)
|
||||
union = len(i_set | j_set)
|
||||
return inter / union if union else NetworkUtils._DEFAULT["format"]
|
||||
0
final/src/__init__.py
Normal file
0
final/src/__init__.py
Normal file
BIN
final/src/__pycache__/__init__.cpython-313.pyc
Normal file
BIN
final/src/__pycache__/__init__.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/src/__pycache__/attribute_schema.cpython-313.pyc
Normal file
BIN
final/src/__pycache__/attribute_schema.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/src/__pycache__/cleaner.cpython-313.pyc
Normal file
BIN
final/src/__pycache__/cleaner.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/src/__pycache__/cleaner.cpython-38.pyc
Normal file
BIN
final/src/__pycache__/cleaner.cpython-38.pyc
Normal file
Binary file not shown.
BIN
final/src/__pycache__/data_loader.cpython-313.pyc
Normal file
BIN
final/src/__pycache__/data_loader.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/src/__pycache__/data_loader.cpython-38.pyc
Normal file
BIN
final/src/__pycache__/data_loader.cpython-38.pyc
Normal file
Binary file not shown.
BIN
final/src/__pycache__/formalization_pipeline.cpython-313.pyc
Normal file
BIN
final/src/__pycache__/formalization_pipeline.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/src/__pycache__/main_227.cpython-313.pyc
Normal file
BIN
final/src/__pycache__/main_227.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/src/__pycache__/triplet_cleaner.cpython-313.pyc
Normal file
BIN
final/src/__pycache__/triplet_cleaner.cpython-313.pyc
Normal file
Binary file not shown.
701
final/src/attribute_schema.py
Normal file
701
final/src/attribute_schema.py
Normal file
@@ -0,0 +1,701 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
属性规范定义模块 - 基于 1125暂时属性需求
|
||||
实现内容与格式的双重对齐:格式校验(结构类型)+ 内容校验(取值范围、类型)
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
# ============ 目标 Schema 定义(来自 1125暂时属性需求.xlsx) ============
|
||||
|
||||
# function_vector: 字典,键为功能维度,值为 [0,1] 浮点数
|
||||
FUNCTION_VECTOR_KEYS = [
|
||||
"f_IC", # 情报收集能力
|
||||
"f_IA", # 信息分析能力
|
||||
"f_CS", # 协同作战能力
|
||||
"f_IT", # 信息传输能力
|
||||
"f_DP", # 数据处理能力
|
||||
"f_CPS", # 指挥控制能力
|
||||
"f_CC", # 综合保障能力
|
||||
]
|
||||
|
||||
# spatial: 字典
|
||||
SPATIAL_KEYS = {
|
||||
"position": lambda v: isinstance(v, (list, tuple)) and len(v) >= 2
|
||||
and all(isinstance(x, (int, float)) for x in v[:2]),
|
||||
"effective_radius": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
|
||||
}
|
||||
|
||||
# temporal: 字典
|
||||
TEMPORAL_KEYS = {
|
||||
"response_time": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
|
||||
"cycle_time": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
|
||||
"data_age": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
|
||||
"time_window": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
|
||||
}
|
||||
|
||||
# performance: 字典
|
||||
PERFORMANCE_KEYS = {
|
||||
"core_performance": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool)
|
||||
and 0 <= float(v) <= 1,
|
||||
"survivability": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool)
|
||||
and 0 <= float(v) <= 1,
|
||||
"mtbf": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
|
||||
}
|
||||
|
||||
# nodes.json 输出字段(完全对齐 xlsx,仅含规范字段 + 节点标识)
|
||||
# TARGET_ID 为节点唯一标识(三元组 head/tail 引用所需)
|
||||
OUTPUT_SCHEMA_FIELDS = [
|
||||
"TARGET_ID",
|
||||
"function_vector",
|
||||
"spatial",
|
||||
"temporal",
|
||||
"performance",
|
||||
"protocol_list",
|
||||
"format_list",
|
||||
"interface_list",
|
||||
"security_level",
|
||||
"org_unit",
|
||||
"nation",
|
||||
"history_success",
|
||||
]
|
||||
|
||||
# 原始字段 -> 目标 schema 映射(用于 flat 数据转换为规范格式)
|
||||
RAW_TO_TARGET_MAPPING = {
|
||||
# function_vector 从多个能力字段推导
|
||||
"function_vector": {
|
||||
"f_IC": ["DETECTION_ACCURACY", "TARGET_RECOGNITION_CAPABILITY"], # 情报收集
|
||||
"f_IA": ["INFORMATION_FUSION_CAPABILITY", "DETECTION_ACCURACY"], # 信息分析
|
||||
"f_CS": ["MOBILITY", "ENVIRONMENT_ADAPTABILITY"], # 协同作战
|
||||
"f_IT": ["ANTI_JAMMING_CAPABILITY"], # 信息传输(抗干扰)
|
||||
"f_DP": ["PROCESSING_CAPACITY", "INFORMATION_FUSION_CAPABILITY"], # 数据处理
|
||||
"f_CPS": ["DECISION_RESPONSE_TIME"], # 指挥控制(响应时间反推)
|
||||
"f_CC": ["SUPPORT_CAPACITY", "LOAD_CAPACITY"], # 综合保障
|
||||
},
|
||||
"spatial": {
|
||||
"position": ["X", "Y"],
|
||||
"effective_radius": ["COMMUNICATION_RANGE"],
|
||||
},
|
||||
"temporal": {
|
||||
"response_time": ["DECISION_RESPONSE_TIME"],
|
||||
"cycle_time": ["REFRESH_RATE"],
|
||||
"data_age": ["TRANSMISSION_DELAY"], # 传输延迟可近似数据新鲜度
|
||||
"time_window": ["REFRESH_RATE"], # 周期倒数近似时间窗口
|
||||
},
|
||||
"performance": {
|
||||
"core_performance": ["STRIKE_ACCURACY", "DETECTION_ACCURACY", "MOBILITY"],
|
||||
"survivability": ["ENVIRONMENT_ADAPTABILITY", "ANTI_JAMMING_CAPABILITY"],
|
||||
"mtbf": ["RELIABILITY", "SYSTEM_RELIABILITY"],
|
||||
},
|
||||
"nation": ["COUNTRY_REGION"],
|
||||
"org_unit": ["LEVEL", "ROLE_ID"],
|
||||
"history_success": [], # 无直接映射,需默认或缺失
|
||||
}
|
||||
|
||||
|
||||
def _clamp_0_1(val: Any) -> Optional[float]:
|
||||
"""将值限制在 [0,1] 范围内,无效返回 None"""
|
||||
if val is None or (isinstance(val, float) and (val != val)):
|
||||
return None
|
||||
try:
|
||||
f = float(val)
|
||||
if f < 0:
|
||||
return 0.0
|
||||
if f > 1:
|
||||
return 1.0
|
||||
return f
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _safe_float(val: Any) -> Optional[float]:
|
||||
if val is None or (isinstance(val, float) and (val != val)):
|
||||
return None
|
||||
try:
|
||||
return float(val)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _safe_int(val: Any, lo: int = None, hi: int = None) -> Optional[int]:
|
||||
if val is None:
|
||||
return None
|
||||
try:
|
||||
i = int(float(val))
|
||||
if lo is not None and i < lo:
|
||||
return lo
|
||||
if hi is not None and i > hi:
|
||||
return hi
|
||||
return i
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_list_of_strings(val: Any) -> List[str]:
|
||||
"""确保为字符串列表"""
|
||||
if val is None:
|
||||
return []
|
||||
if isinstance(val, list):
|
||||
return [str(x).strip() for x in val if x is not None and str(x).strip()]
|
||||
if isinstance(val, str) and val.strip():
|
||||
return [val.strip()]
|
||||
return []
|
||||
|
||||
|
||||
# ---------- 固定参考值转换(用于 function_vector 中需特殊归一化的维度)----------
|
||||
# 参考时间(秒):响应时间 t 越短,f_CPS 越高
|
||||
_CPS_REF_TIME = 30.0
|
||||
# 处理能力参考上限
|
||||
_FDP_REF_CAP = 10000.0
|
||||
# 保障/载荷能力参考
|
||||
_FCC_REF_SUPPORT = 10000.0
|
||||
_FCC_REF_LOAD = 100.0
|
||||
|
||||
|
||||
def _response_time_to_cps(val: Any) -> Optional[float]:
|
||||
"""响应时间(秒) -> f_CPS [0,1],响应越快能力越高"""
|
||||
f = _safe_float(val)
|
||||
if f is None or f < 0:
|
||||
return None
|
||||
v = 1.0 / (1.0 + f / _CPS_REF_TIME)
|
||||
return min(1.0, max(0.0, v))
|
||||
|
||||
|
||||
def _processing_to_fdp(raw: dict) -> Optional[float]:
|
||||
"""PROCESSING_CAPACITY 或 INFORMATION_FUSION_CAPABILITY -> f_DP [0,1]"""
|
||||
pc = raw.get("PROCESSING_CAPACITY")
|
||||
if pc is not None and (not isinstance(pc, float) or pc == pc):
|
||||
try:
|
||||
v = float(pc)
|
||||
if v < 0:
|
||||
return 0.0
|
||||
norm = math.log10(1 + v) / math.log10(1 + _FDP_REF_CAP)
|
||||
return min(1.0, norm)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
ifc = raw.get("INFORMATION_FUSION_CAPABILITY")
|
||||
if ifc is not None and (not isinstance(ifc, float) or ifc == ifc):
|
||||
return _clamp_0_1(ifc)
|
||||
return None
|
||||
|
||||
|
||||
def _support_load_to_fcc(raw: dict) -> Optional[float]:
|
||||
"""SUPPORT_CAPACITY 与 LOAD_CAPACITY -> f_CC [0,1],支持组合"""
|
||||
support = raw.get("SUPPORT_CAPACITY")
|
||||
load = raw.get("LOAD_CAPACITY")
|
||||
s_norm = None
|
||||
l_norm = None
|
||||
if support is not None and (not isinstance(support, float) or support == support):
|
||||
try:
|
||||
v = float(support)
|
||||
if v >= 0:
|
||||
s_norm = min(1.0, math.log10(1 + v) / math.log10(1 + _FCC_REF_SUPPORT))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if load is not None and (not isinstance(load, float) or load == load):
|
||||
try:
|
||||
v = float(load)
|
||||
if v >= 0:
|
||||
l_norm = min(1.0, v / _FCC_REF_LOAD)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
if s_norm is not None and l_norm is not None:
|
||||
return round(0.6 * s_norm + 0.4 * l_norm, 4)
|
||||
if s_norm is not None:
|
||||
return s_norm
|
||||
return l_norm
|
||||
|
||||
|
||||
# ============ 格式校验 ============
|
||||
|
||||
def check_format_function_vector(val: Any) -> Tuple[bool, str]:
|
||||
"""校验 function_vector 格式:必须为字典"""
|
||||
if val is None:
|
||||
return True, "optional"
|
||||
if not isinstance(val, dict):
|
||||
return False, f"期望 dict,实际 {type(val).__name__}"
|
||||
for k in val:
|
||||
if not isinstance(k, str):
|
||||
return False, f"键必须为 str,发现 {type(k).__name__}"
|
||||
v = val[k]
|
||||
if not isinstance(v, (int, float)) or isinstance(v, bool):
|
||||
return False, f"键 {k} 的值应为 [0,1] 浮点数,实际 {type(v).__name__}"
|
||||
f = float(v)
|
||||
if f < 0 or f > 1:
|
||||
return False, f"键 {k} 的值 {f} 超出 [0,1]"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def check_format_spatial(val: Any) -> Tuple[bool, str]:
|
||||
"""校验 spatial 格式:必须为字典,含 position 与 effective_radius"""
|
||||
if val is None:
|
||||
return True, "optional"
|
||||
if not isinstance(val, dict):
|
||||
return False, f"期望 dict,实际 {type(val).__name__}"
|
||||
pos = val.get("position")
|
||||
if pos is not None:
|
||||
if not isinstance(pos, (list, tuple)) or len(pos) < 2:
|
||||
return False, "position 应为 [x, y] 列表"
|
||||
if not all(isinstance(x, (int, float)) for x in pos[:2]):
|
||||
return False, "position 元素应为数字"
|
||||
rad = val.get("effective_radius")
|
||||
if rad is not None and not isinstance(rad, (int, float)):
|
||||
return False, "effective_radius 应为数字"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def check_format_temporal(val: Any) -> Tuple[bool, str]:
|
||||
"""校验 temporal 格式"""
|
||||
if val is None:
|
||||
return True, "optional"
|
||||
if not isinstance(val, dict):
|
||||
return False, f"期望 dict,实际 {type(val).__name__}"
|
||||
for k in ["response_time", "cycle_time", "data_age", "time_window"]:
|
||||
v = val.get(k)
|
||||
if v is not None and not isinstance(v, (int, float)):
|
||||
return False, f"{k} 应为数字"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def check_format_performance(val: Any) -> Tuple[bool, str]:
|
||||
"""校验 performance 格式"""
|
||||
if val is None:
|
||||
return True, "optional"
|
||||
if not isinstance(val, dict):
|
||||
return False, f"期望 dict,实际 {type(val).__name__}"
|
||||
for k in ["core_performance", "survivability"]:
|
||||
v = val.get(k)
|
||||
if v is not None:
|
||||
try:
|
||||
f = float(v)
|
||||
if f < 0 or f > 1:
|
||||
return False, f"{k} 应在 [0,1],实际 {f}"
|
||||
except (TypeError, ValueError):
|
||||
return False, f"{k} 应为数字"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def check_format_protocol_list(val: Any) -> Tuple[bool, str]:
|
||||
if val is None:
|
||||
return True, "optional"
|
||||
if not isinstance(val, list):
|
||||
return False, f"期望 list,实际 {type(val).__name__}"
|
||||
for i, x in enumerate(val):
|
||||
if not isinstance(x, str):
|
||||
return False, f"元素[{i}] 应为 str,实际 {type(x).__name__}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def check_format_format_list(val: Any) -> Tuple[bool, str]:
|
||||
return check_format_protocol_list(val)
|
||||
|
||||
|
||||
def check_format_interface_list(val: Any) -> Tuple[bool, str]:
|
||||
return check_format_protocol_list(val)
|
||||
|
||||
|
||||
def check_format_security_level(val: Any) -> Tuple[bool, str]:
|
||||
if val is None:
|
||||
return True, "optional"
|
||||
if not isinstance(val, int) or isinstance(val, bool):
|
||||
try:
|
||||
int(val)
|
||||
except (TypeError, ValueError):
|
||||
return False, f"期望 int [1,5],实际 {type(val).__name__}"
|
||||
v = int(val)
|
||||
if v < 1 or v > 5:
|
||||
return False, f"security_level 应在 [1,5],实际 {v}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def check_format_org_unit(val: Any) -> Tuple[bool, str]:
|
||||
if val is None:
|
||||
return True, "optional"
|
||||
if not isinstance(val, str):
|
||||
return False, f"期望 str,实际 {type(val).__name__}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def check_format_nation(val: Any) -> Tuple[bool, str]:
|
||||
return check_format_org_unit(val)
|
||||
|
||||
|
||||
def check_format_history_success(val: Any) -> Tuple[bool, str]:
|
||||
if val is None:
|
||||
return True, "optional"
|
||||
if not isinstance(val, (int, float)) or isinstance(val, bool):
|
||||
return False, f"期望 float [0,1],实际 {type(val).__name__}"
|
||||
f = float(val)
|
||||
if f < 0 or f > 1:
|
||||
return False, f"history_success 应在 [0,1],实际 {f}"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
# ============ 内容校验(取值范围、语义) ============
|
||||
|
||||
def check_content_function_vector(val: dict) -> List[str]:
|
||||
"""内容校验:键是否在规范内,值是否合法"""
|
||||
issues = []
|
||||
if not isinstance(val, dict):
|
||||
return ["非字典类型"]
|
||||
for k, v in val.items():
|
||||
if k not in FUNCTION_VECTOR_KEYS:
|
||||
issues.append(f"未知键: {k}")
|
||||
else:
|
||||
try:
|
||||
f = float(v)
|
||||
if f < 0 or f > 1:
|
||||
issues.append(f"{k}={f} 超出 [0,1]")
|
||||
except (TypeError, ValueError):
|
||||
issues.append(f"{k} 值非数字: {v}")
|
||||
return issues
|
||||
|
||||
|
||||
def check_content_spatial(val: dict) -> List[str]:
|
||||
issues = []
|
||||
if not isinstance(val, dict):
|
||||
return ["非字典类型"]
|
||||
pos = val.get("position")
|
||||
if pos is not None and len(pos) >= 2:
|
||||
if not all(isinstance(x, (int, float)) for x in pos[:2]):
|
||||
issues.append("position 含非数字")
|
||||
rad = val.get("effective_radius")
|
||||
if rad is not None:
|
||||
try:
|
||||
f = float(rad)
|
||||
if f < 0:
|
||||
issues.append("effective_radius 不应为负")
|
||||
except (TypeError, ValueError):
|
||||
issues.append("effective_radius 非数字")
|
||||
return issues
|
||||
|
||||
|
||||
def check_content_temporal(val: dict) -> List[str]:
|
||||
issues = []
|
||||
if not isinstance(val, dict):
|
||||
return ["非字典类型"]
|
||||
for k in ["response_time", "cycle_time", "data_age", "time_window"]:
|
||||
v = val.get(k)
|
||||
if v is not None:
|
||||
try:
|
||||
f = float(v)
|
||||
if f < 0:
|
||||
issues.append(f"{k} 不应为负: {f}")
|
||||
except (TypeError, ValueError):
|
||||
issues.append(f"{k} 非数字: {v}")
|
||||
return issues
|
||||
|
||||
|
||||
def check_content_performance(val: dict) -> List[str]:
|
||||
issues = []
|
||||
if not isinstance(val, dict):
|
||||
return ["非字典类型"]
|
||||
for k in ["core_performance", "survivability"]:
|
||||
v = val.get(k)
|
||||
if v is not None:
|
||||
try:
|
||||
f = float(v)
|
||||
if f < 0 or f > 1:
|
||||
issues.append(f"{k} 超出 [0,1]: {f}")
|
||||
except (TypeError, ValueError):
|
||||
issues.append(f"{k} 非数字: {v}")
|
||||
mtbf = val.get("mtbf")
|
||||
if mtbf is not None:
|
||||
try:
|
||||
f = float(mtbf)
|
||||
if f < 0:
|
||||
issues.append("mtbf 不应为负")
|
||||
except (TypeError, ValueError):
|
||||
issues.append("mtbf 非数字")
|
||||
return issues
|
||||
|
||||
|
||||
# ============ 统一校验入口 ============
|
||||
|
||||
FORMAT_CHECKERS = {
|
||||
"function_vector": check_format_function_vector,
|
||||
"spatial": check_format_spatial,
|
||||
"temporal": check_format_temporal,
|
||||
"performance": check_format_performance,
|
||||
"protocol_list": check_format_protocol_list,
|
||||
"format_list": check_format_format_list,
|
||||
"interface_list": check_format_interface_list,
|
||||
"security_level": check_format_security_level,
|
||||
"org_unit": check_format_org_unit,
|
||||
"nation": check_format_nation,
|
||||
"history_success": check_format_history_success,
|
||||
}
|
||||
|
||||
CONTENT_CHECKERS = {
|
||||
"function_vector": check_content_function_vector,
|
||||
"spatial": check_content_spatial,
|
||||
"temporal": check_content_temporal,
|
||||
"performance": check_content_performance,
|
||||
}
|
||||
|
||||
|
||||
def validate_record(
|
||||
record: dict,
|
||||
target_schema_fields: Optional[List[str]] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
对单条记录进行格式与内容双重校验。
|
||||
若 record 已包含目标 schema 字段,则直接校验;
|
||||
否则会先通过 raw_to_target_schema 转换后再校验。
|
||||
"""
|
||||
target_fields = target_schema_fields or list(FORMAT_CHECKERS.keys())
|
||||
result = {
|
||||
"format_errors": {},
|
||||
"content_errors": {},
|
||||
"aligned": False,
|
||||
"transformed": False,
|
||||
}
|
||||
|
||||
# 若缺少目标字段,尝试从原始字段转换
|
||||
has_target = any(f in record for f in target_fields)
|
||||
data = record
|
||||
if not has_target:
|
||||
data = raw_to_target_schema(record)
|
||||
result["transformed"] = True
|
||||
|
||||
# 格式校验
|
||||
for field in target_fields:
|
||||
val = data.get(field)
|
||||
if val is None and field in ["protocol_list", "format_list", "interface_list"]:
|
||||
val = []
|
||||
checker = FORMAT_CHECKERS.get(field)
|
||||
if checker:
|
||||
ok, msg = checker(val)
|
||||
else:
|
||||
ok, msg = True, "no_checker"
|
||||
if not ok:
|
||||
result["format_errors"][field] = msg
|
||||
|
||||
# 内容校验(仅对复杂类型)
|
||||
for field in CONTENT_CHECKERS:
|
||||
val = data.get(field)
|
||||
if val is None:
|
||||
continue
|
||||
issues = CONTENT_CHECKERS[field](val)
|
||||
if issues:
|
||||
result["content_errors"][field] = issues
|
||||
|
||||
result["aligned"] = (
|
||||
len(result["format_errors"]) == 0 and len(result["content_errors"]) == 0
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
# ============ 原始数据 -> 目标 Schema 转换 ============
|
||||
|
||||
def _get_first_valid(row: dict, keys: List[str], cast=_clamp_0_1) -> Optional[Any]:
|
||||
"""从 row 中按 keys 顺序取第一个非空值并转换"""
|
||||
for k in keys:
|
||||
v = row.get(k)
|
||||
if v is not None and (not isinstance(v, float) or v == v):
|
||||
return cast(v) if cast else v
|
||||
return None
|
||||
|
||||
|
||||
def raw_to_target_schema(raw: dict) -> dict:
|
||||
"""
|
||||
将扁平原始记录转换为符合 1125 属性规范的目标结构。
|
||||
仅输出 xlsx 定义字段,不保留任何原始其他字段。
|
||||
"""
|
||||
# TARGET_ID:节点唯一标识
|
||||
tid = raw.get("TARGET_ID")
|
||||
tid = str(tid).strip() if tid is not None and str(tid).strip() else None
|
||||
|
||||
out: Dict[str, Any] = {f: None for f in OUTPUT_SCHEMA_FIELDS}
|
||||
out["TARGET_ID"] = tid
|
||||
|
||||
# function_vector(f_CPS/f_DP/f_CC 使用固定参考值转换,其余用 _clamp_0_1)
|
||||
fv = {}
|
||||
_fv_mapping = RAW_TO_TARGET_MAPPING["function_vector"]
|
||||
for fk, raw_keys in _fv_mapping.items():
|
||||
if fk == "f_CPS":
|
||||
t = raw.get("DECISION_RESPONSE_TIME")
|
||||
val = _response_time_to_cps(t)
|
||||
elif fk == "f_DP":
|
||||
val = _processing_to_fdp(raw)
|
||||
elif fk == "f_CC":
|
||||
val = _support_load_to_fcc(raw)
|
||||
else:
|
||||
val = _get_first_valid(raw, raw_keys, _clamp_0_1)
|
||||
if val is not None:
|
||||
fv[fk] = round(val, 4)
|
||||
# 下游 f_relation1125/f_net1125 要求必为 dict,无推导值时用默认(避免 NoneType)
|
||||
if not fv:
|
||||
fv = {fk: 0.5 for fk in FUNCTION_VECTOR_KEYS}
|
||||
out["function_vector"] = fv
|
||||
|
||||
# spatial(支持 X/x、Y/y、position、POSITION_X/Y、LONGITUDE/LATITUDE;确保元素恒为 float)
|
||||
def _safe_coord(v: Any) -> float:
|
||||
if v is None or (isinstance(v, float) and v != v):
|
||||
return 0.0
|
||||
try:
|
||||
return round(float(v), 2)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
pos_raw = raw.get("position") or raw.get("POSITION")
|
||||
if isinstance(pos_raw, (list, tuple)) and len(pos_raw) >= 2:
|
||||
x, y = _safe_coord(pos_raw[0]), _safe_coord(pos_raw[1])
|
||||
else:
|
||||
x = raw.get("X") or raw.get("x") or raw.get("POSITION_X") or raw.get("LONGITUDE")
|
||||
y = raw.get("Y") or raw.get("y") or raw.get("POSITION_Y") or raw.get("LATITUDE")
|
||||
x, y = _safe_coord(x), _safe_coord(y)
|
||||
pos = [x, y]
|
||||
radius = _get_first_valid(raw, RAW_TO_TARGET_MAPPING["spatial"]["effective_radius"], _safe_float)
|
||||
sp = {
|
||||
"position": pos,
|
||||
"effective_radius": round(float(radius), 4) if radius is not None else 1.0,
|
||||
}
|
||||
out["spatial"] = sp
|
||||
|
||||
# temporal
|
||||
_temporal_defaults = {"response_time": 1.0, "cycle_time": 1.0, "data_age": 1.0, "time_window": 24.0}
|
||||
temporal = {}
|
||||
for tk, rks in RAW_TO_TARGET_MAPPING["temporal"].items():
|
||||
v = _get_first_valid(raw, rks, _safe_float)
|
||||
if v is not None:
|
||||
temporal[tk] = round(float(v), 4)
|
||||
for k, default in _temporal_defaults.items():
|
||||
temporal.setdefault(k, default)
|
||||
out["temporal"] = temporal
|
||||
|
||||
# performance
|
||||
_perf_defaults = {"core_performance": 0.5, "survivability": 0.5, "mtbf": 1.0}
|
||||
perf = {}
|
||||
for pk, rks in RAW_TO_TARGET_MAPPING["performance"].items():
|
||||
cast = _clamp_0_1 if pk in ["core_performance", "survivability"] else _safe_float
|
||||
v = _get_first_valid(raw, rks, cast)
|
||||
if v is not None:
|
||||
perf[pk] = round(float(v), 4)
|
||||
for k, default in _perf_defaults.items():
|
||||
perf.setdefault(k, default)
|
||||
out["performance"] = perf
|
||||
|
||||
# protocol_list, format_list, interface_list
|
||||
for field in ["protocol_list", "format_list", "interface_list"]:
|
||||
raw_key = field.upper().replace("_LIST", "")
|
||||
out[field] = _ensure_list_of_strings(raw.get(raw_key, raw.get(field)))
|
||||
|
||||
# security_level:下游 utils.get_security_compatibility 会做 lv_i - lv_j,不可为 None
|
||||
sl = raw.get("security_level") or raw.get("SECURITY_LEVEL")
|
||||
out["security_level"] = _safe_int(sl, 1, 5) if sl is not None else 3
|
||||
|
||||
# org_unit, nation
|
||||
nation = _get_first_valid(raw, RAW_TO_TARGET_MAPPING["nation"], lambda v: str(v).strip() if v is not None else None)
|
||||
out["nation"] = nation
|
||||
org = _get_first_valid(raw, RAW_TO_TARGET_MAPPING["org_unit"], lambda v: str(v).strip() if v is not None else None)
|
||||
out["org_unit"] = org
|
||||
|
||||
# history_success:下游 utils.get_interaction_history 会做 (his_i + his_j)/2,不可为 None
|
||||
hs = raw.get("history_success") or raw.get("HISTORY_SUCCESS")
|
||||
if hs is not None:
|
||||
v = _clamp_0_1(hs)
|
||||
out["history_success"] = round(v, 4) if v is not None else 0.8
|
||||
else:
|
||||
out["history_success"] = 0.8
|
||||
|
||||
return out
|
||||
|
||||
|
||||
def align_and_validate_records(
|
||||
records: List[dict],
|
||||
id_key: str = "TARGET_ID",
|
||||
) -> Tuple[List[dict], dict]:
|
||||
"""
|
||||
对记录列表进行转换、格式与内容校验。
|
||||
返回:(对齐后的记录列表, 统计报告)
|
||||
"""
|
||||
aligned = []
|
||||
report = {
|
||||
"total": len(records),
|
||||
"format_errors_by_field": {},
|
||||
"content_errors_by_field": {},
|
||||
"records_with_errors": 0,
|
||||
"aligned_count": 0,
|
||||
"transformed_count": 0,
|
||||
"samples": [],
|
||||
}
|
||||
|
||||
for rec in records:
|
||||
rid = rec.get(id_key, "?")
|
||||
val_result = validate_record(rec)
|
||||
aligned_rec = raw_to_target_schema(rec)
|
||||
aligned.append(aligned_rec)
|
||||
|
||||
if val_result["transformed"]:
|
||||
report["transformed_count"] += 1
|
||||
if val_result["aligned"]:
|
||||
report["aligned_count"] += 1
|
||||
else:
|
||||
report["records_with_errors"] += 1
|
||||
for f, msg in val_result["format_errors"].items():
|
||||
report["format_errors_by_field"].setdefault(f, 0)
|
||||
report["format_errors_by_field"][f] += 1
|
||||
for f, issues in val_result["content_errors"].items():
|
||||
report["content_errors_by_field"].setdefault(f, 0)
|
||||
report["content_errors_by_field"][f] += 1
|
||||
if len(report["samples"]) < 5:
|
||||
report["samples"].append({
|
||||
"id": rid,
|
||||
"format_errors": val_result["format_errors"],
|
||||
"content_errors": val_result["content_errors"],
|
||||
})
|
||||
|
||||
return aligned, report
|
||||
|
||||
|
||||
def sanitize_node_for_relation_calc(node: dict) -> dict:
|
||||
"""
|
||||
写入 nodes.json 前对节点做最终净化,确保 f_relation1125/utils 中参与运算的字段无 None。
|
||||
避免 euclidean、get_security_compatibility、get_interaction_history 等出现 NoneType 运算错误。
|
||||
"""
|
||||
import copy
|
||||
n = copy.deepcopy(node)
|
||||
|
||||
def _safe_float(v: Any, default: float) -> float:
|
||||
if v is None or (isinstance(v, float) and v != v):
|
||||
return default
|
||||
try:
|
||||
return float(v)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
# spatial.position 必须为 [float, float]
|
||||
sp = n.get("spatial")
|
||||
if not isinstance(sp, dict):
|
||||
sp = {}
|
||||
pos = sp.get("position")
|
||||
if not isinstance(pos, (list, tuple)) or len(pos) < 2:
|
||||
pos = [0.0, 0.0]
|
||||
else:
|
||||
pos = [_safe_float(pos[0], 0.0), _safe_float(pos[1], 0.0)]
|
||||
sp["position"] = pos
|
||||
sp["effective_radius"] = _safe_float(sp.get("effective_radius"), 1.0)
|
||||
n["spatial"] = sp
|
||||
|
||||
# temporal 参与 time_window_overlap
|
||||
tmp = n.get("temporal")
|
||||
if not isinstance(tmp, dict):
|
||||
tmp = {}
|
||||
for k in ["response_time", "cycle_time", "data_age", "time_window"]:
|
||||
if tmp.get(k) is None:
|
||||
tmp[k] = 24.0 if k == "time_window" else 1.0
|
||||
n["temporal"] = tmp
|
||||
|
||||
# security_level、history_success 参与 utils 运算
|
||||
sl = n.get("security_level")
|
||||
n["security_level"] = int(_safe_float(sl, 3)) if sl is not None else 3
|
||||
if n["security_level"] < 1 or n["security_level"] > 5:
|
||||
n["security_level"] = 3
|
||||
|
||||
hs = n.get("history_success")
|
||||
n["history_success"] = _safe_float(hs, 0.8)
|
||||
n["history_success"] = max(0.0, min(1.0, n["history_success"]))
|
||||
|
||||
return n
|
||||
294
final/src/cleaner.py
Normal file
294
final/src/cleaner.py
Normal file
@@ -0,0 +1,294 @@
|
||||
import json
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
try:
|
||||
from .attribute_schema import align_and_validate_records, sanitize_node_for_relation_calc
|
||||
except ImportError:
|
||||
from attribute_schema import align_and_validate_records, sanitize_node_for_relation_calc
|
||||
|
||||
|
||||
def _json_serializer(obj):
|
||||
"""处理 Timestamp、datetime、numpy、NaN 等不可直接 JSON 序列化的类型"""
|
||||
if hasattr(obj, 'isoformat'):
|
||||
return obj.isoformat()
|
||||
if hasattr(obj, 'item'):
|
||||
return obj.item()
|
||||
if isinstance(obj, float) and (obj != obj or obj == float('inf') or obj == float('-inf')):
|
||||
return None # NaN, Inf -> null
|
||||
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
|
||||
|
||||
class AdvancedDataCleaner:
|
||||
def __init__(self, output_file, report_file, data=None, input_file=None):
|
||||
"""
|
||||
支持两种初始化方式:
|
||||
1. data=... 直接传入数据列表(从配置/数据库加载时使用)
|
||||
2. input_file=... 传入 JSON 文件路径(兼容旧用法)
|
||||
"""
|
||||
self.output_file = output_file
|
||||
self.report_file = report_file
|
||||
self.data = data if data is not None else []
|
||||
self.input_file = input_file
|
||||
|
||||
# 报告结构
|
||||
self.report = {
|
||||
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"summary": {
|
||||
"total_records": 0,
|
||||
"final_records": 0,
|
||||
"duplicates_removed": 0
|
||||
},
|
||||
"details": {
|
||||
"missing_values_fixed": {}, # 字段: 填充数量
|
||||
"outliers_corrected": {
|
||||
"count": 0,
|
||||
"examples": [] # 记录具体的修改案例
|
||||
},
|
||||
"noise_reduction": {
|
||||
"method": "Kalman Filter",
|
||||
"fields_processed": ["COMMUNICATION_RANGE"],
|
||||
"total_smoothed": 0
|
||||
},
|
||||
"standardization": [],
|
||||
"attribute_alignment": { # 内容与格式双重对齐(基于 1125 属性需求)
|
||||
"enabled": True,
|
||||
"format_errors_by_field": {},
|
||||
"content_errors_by_field": {},
|
||||
"records_with_errors": 0,
|
||||
"aligned_count": 0,
|
||||
"transformed_count": 0,
|
||||
"samples": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.norm_fields = [
|
||||
"TARGET_RECOGNITION_CAPABILITY", "STRIKE_ACCURACY",
|
||||
"ANTI_JAMMING_CAPABILITY", "ENVIRONMENT_ADAPTABILITY", "MOBILITY"
|
||||
]
|
||||
|
||||
def load_data(self):
|
||||
if not self.data and self.input_file:
|
||||
with open(self.input_file, 'r', encoding='utf-8') as f:
|
||||
self.data = json.load(f)
|
||||
self.report["summary"]["total_records"] = len(self.data)
|
||||
self.df = pd.DataFrame(self.data)
|
||||
|
||||
def clean_duplicates(self):
|
||||
"""高级去重并记录"""
|
||||
initial_count = len(self.df)
|
||||
# 优先保留创建时间最新的(如果有时间字段),否则保留第一个
|
||||
if 'CREATED_TIME' in self.df.columns:
|
||||
self.df.sort_values('CREATED_TIME', ascending=False, inplace=True)
|
||||
|
||||
self.df.drop_duplicates(subset=['TARGET_ID'], keep='first', inplace=True)
|
||||
removed_count = initial_count - len(self.df)
|
||||
self.report["summary"]["duplicates_removed"] = removed_count
|
||||
|
||||
def handle_missing_values(self):
|
||||
"""智能填充并记录细节"""
|
||||
numeric_cols = self.df.select_dtypes(include=[np.number]).columns
|
||||
|
||||
for col in numeric_cols:
|
||||
if col == "ID": continue
|
||||
n_missing = int(self.df[col].isnull().sum())
|
||||
|
||||
if n_missing > 0:
|
||||
self.report["details"]["missing_values_fixed"][col] = n_missing
|
||||
# 分组填充
|
||||
self.df[col] = self.df.groupby("ROLE_ID")[col].transform(lambda x: x.fillna(x.mean()))
|
||||
# 兜底填充
|
||||
self.df[col] = self.df[col].fillna(self.df[col].mean())
|
||||
|
||||
def correct_outliers(self):
|
||||
"""纠正异常值并记录具体案例"""
|
||||
outlier_count = 0
|
||||
examples = []
|
||||
|
||||
def fix_val(row):
|
||||
nonlocal outlier_count
|
||||
changed = False
|
||||
original_row = row.copy()
|
||||
|
||||
for field in self.norm_fields:
|
||||
if pd.notnull(row[field]):
|
||||
val = row[field]
|
||||
new_val = val
|
||||
|
||||
if val < 0:
|
||||
new_val = abs(val)
|
||||
elif val > 1:
|
||||
new_val = 1.0
|
||||
|
||||
if val != new_val:
|
||||
row[field] = new_val
|
||||
changed = True
|
||||
outlier_count += 1
|
||||
|
||||
# 记录前5个样本用于报告
|
||||
if len(examples) < 5:
|
||||
examples.append({
|
||||
"id": row.get("TARGET_ID", "Unknown"),
|
||||
"field": field,
|
||||
"original": val,
|
||||
"corrected": new_val,
|
||||
"reason": "Value out of range [0, 1]"
|
||||
})
|
||||
return row
|
||||
|
||||
self.df = self.df.apply(fix_val, axis=1)
|
||||
self.report["details"]["outliers_corrected"]["count"] = outlier_count
|
||||
self.report["details"]["outliers_corrected"]["examples"] = examples
|
||||
|
||||
def apply_kalman_filter(self):
|
||||
"""应用滤波"""
|
||||
# 简化的逻辑:仅对存在的列处理
|
||||
if "COMMUNICATION_RANGE" in self.df.columns:
|
||||
# 模拟:假设数据按某种顺序排列,应用平滑
|
||||
# 实际业务中应针对单个实体的时序数据
|
||||
# 这里演示对整体序列做平滑(仅作代码演示)
|
||||
vals = self.df["COMMUNICATION_RANGE"].fillna(0).values
|
||||
# 简单移动平均代替卡尔曼演示(效果类似平滑)
|
||||
smoothed = pd.Series(vals).rolling(window=3, min_periods=1).mean().values
|
||||
|
||||
self.df["COMMUNICATION_RANGE"] = np.round(smoothed, 2)
|
||||
self.report["details"]["noise_reduction"]["total_smoothed"] = len(vals)
|
||||
|
||||
def align_attribute_schema(self):
|
||||
"""内容与格式双重对齐:按 1125 属性需求转换并校验"""
|
||||
if self.df is None or len(self.df) == 0:
|
||||
return
|
||||
records = self.df.to_dict('records')
|
||||
aligned, align_report = align_and_validate_records(records, id_key="TARGET_ID")
|
||||
self.df = pd.DataFrame(aligned)
|
||||
self.report["details"]["attribute_alignment"].update({
|
||||
"format_errors_by_field": align_report.get("format_errors_by_field", {}),
|
||||
"content_errors_by_field": align_report.get("content_errors_by_field", {}),
|
||||
"records_with_errors": align_report.get("records_with_errors", 0),
|
||||
"aligned_count": align_report.get("aligned_count", 0),
|
||||
"transformed_count": align_report.get("transformed_count", 0),
|
||||
"samples": align_report.get("samples", []),
|
||||
})
|
||||
|
||||
def run(self):
|
||||
print("正在执行高级清洗...")
|
||||
self.load_data()
|
||||
self.clean_duplicates()
|
||||
self.handle_missing_values()
|
||||
self.correct_outliers()
|
||||
self.apply_kalman_filter()
|
||||
self.align_attribute_schema()
|
||||
|
||||
# 最终统计
|
||||
self.report["summary"]["final_records"] = len(self.df)
|
||||
self.report["details"]["standardization"].append("Coordinates normalized to 2 decimal places")
|
||||
self.report["details"]["standardization"].append("Timestamps formatted to ISO-8601")
|
||||
self.report["details"]["standardization"].append("Attributes aligned to 1125 schema (format + content)")
|
||||
|
||||
# 保存数据(确保输出目录存在)
|
||||
import os
|
||||
output_dir = os.path.dirname(self.output_file)
|
||||
if output_dir:
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
records = self.df.to_dict('records')
|
||||
# 输出格式兼容 main_227 / f_relation1125:{nodes: {TARGET_ID: obj}, global_params: {mtbf_max}}
|
||||
nodes_dict = {}
|
||||
mtbf_values = []
|
||||
id_col = 'TARGET_ID'
|
||||
for r in records:
|
||||
nid = r.get(id_col)
|
||||
if nid is not None:
|
||||
nid = str(nid).strip()
|
||||
if nid:
|
||||
nodes_dict[nid] = sanitize_node_for_relation_calc(r)
|
||||
perf = r.get('performance') if isinstance(r.get('performance'), dict) else {}
|
||||
mtbf = perf.get('mtbf')
|
||||
if mtbf is not None and (isinstance(mtbf, (int, float)) and not (isinstance(mtbf, float) and mtbf != mtbf)):
|
||||
mtbf_values.append(float(mtbf))
|
||||
mtbf_max = max(mtbf_values, default=1.0) if mtbf_values else 1.0
|
||||
result_data = {
|
||||
"nodes": nodes_dict,
|
||||
"global_params": {"mtbf_max": round(mtbf_max, 6)},
|
||||
}
|
||||
with open(self.output_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(result_data, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
|
||||
# 保存详细报告(确保目录存在)
|
||||
report_dir = os.path.dirname(self.report_file)
|
||||
if report_dir:
|
||||
os.makedirs(report_dir, exist_ok=True)
|
||||
with open(self.report_file, 'w', encoding='utf-8') as f:
|
||||
json.dump(self.report, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
|
||||
print(f"完成!报告已生成至 {self.report_file}")
|
||||
|
||||
def get_valid_target_ids(self) -> set:
|
||||
"""返回清洗后的有效 TARGET_ID 集合(用于三元组 filter_orphans)"""
|
||||
if self.df is None or 'TARGET_ID' not in self.df.columns:
|
||||
return set()
|
||||
return set(self.df['TARGET_ID'].dropna().astype(str).str.strip().unique())
|
||||
|
||||
def get_id_to_role(self) -> dict:
|
||||
"""返回 TARGET_ID -> ROLE_ID 映射(用于三元组 head_type/tail_type)"""
|
||||
if hasattr(self, '_id_to_role'):
|
||||
return self._id_to_role
|
||||
if self.df is None or 'TARGET_ID' not in self.df.columns or 'ROLE_ID' not in self.df.columns:
|
||||
return {}
|
||||
return dict(
|
||||
zip(
|
||||
self.df['TARGET_ID'].dropna().astype(str).str.strip(),
|
||||
self.df['ROLE_ID'].fillna('').astype(str).str.strip(),
|
||||
)
|
||||
)
|
||||
|
||||
def load_config(project_root: str) -> dict:
|
||||
"""加载 config/config.json 或 config.yaml(json 优先,便于部署时只改 json)"""
|
||||
import os
|
||||
config_dir = os.path.join(project_root, 'config')
|
||||
json_path = os.path.join(config_dir, 'config.json')
|
||||
yaml_path = os.path.join(config_dir, 'config.yaml')
|
||||
if os.path.exists(json_path):
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
if os.path.exists(yaml_path):
|
||||
try:
|
||||
import yaml
|
||||
with open(yaml_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
except ImportError:
|
||||
pass
|
||||
return {
|
||||
'source': 'json',
|
||||
'json': {
|
||||
'input_file': 'data/raw_data_sample.json',
|
||||
'output_file': 'data/nodes.json',
|
||||
'report_file': 'report/detailed_cleaning_report.json',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import os
|
||||
from data_loader import load_data as load_data_source
|
||||
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
project_root = os.path.dirname(script_dir)
|
||||
|
||||
config = load_config(project_root)
|
||||
source = config.get('source', 'json')
|
||||
print(f"数据源模式: {source}")
|
||||
|
||||
try:
|
||||
data, paths = load_data_source(config, project_root)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"错误:{e}")
|
||||
exit(1)
|
||||
|
||||
os.makedirs(os.path.dirname(paths['report']), exist_ok=True)
|
||||
|
||||
cleaner = AdvancedDataCleaner(
|
||||
output_file=paths['output'],
|
||||
report_file=paths['report'],
|
||||
data=data,
|
||||
)
|
||||
cleaner.run()
|
||||
614
final/src/data_loader.py
Normal file
614
final/src/data_loader.py
Normal file
@@ -0,0 +1,614 @@
|
||||
"""
|
||||
数据加载模块:支持从 JSON 文件或达梦数据库读取数据
|
||||
开发阶段使用 JSON,生产环境切换到达梦时只需修改 config 中的 source
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
try:
|
||||
from attribute_schema import FUNCTION_VECTOR_KEYS, raw_to_target_schema
|
||||
except ImportError:
|
||||
from src.attribute_schema import FUNCTION_VECTOR_KEYS, raw_to_target_schema
|
||||
|
||||
|
||||
def load_from_json(file_path: str, project_root: str = None) -> list:
|
||||
"""从 JSON 文件加载数据"""
|
||||
path = file_path
|
||||
if project_root and not os.path.isabs(file_path):
|
||||
path = os.path.join(project_root, file_path)
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
def _dm_connect(config: dict):
|
||||
"""创建达梦连接,返回 (conn, db_cfg)。需环境变量 DM_PASS。"""
|
||||
try:
|
||||
import dmPython
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"达梦数据库模式需要安装 dmPython。\n"
|
||||
"请执行: pip install dmPython\n"
|
||||
"并在有达梦客户端的环境运行(需配置 DM_HOME/LD_LIBRARY_PATH)"
|
||||
) from e
|
||||
|
||||
db_cfg = config.get('db', {})
|
||||
host = os.getenv('DM_HOST', db_cfg.get('host', '127.0.0.1'))
|
||||
port = int(os.getenv('DM_PORT', str(db_cfg.get('port', 9080))))
|
||||
user = os.getenv('DM_USER', db_cfg.get('user', 'SYSDBA'))
|
||||
password = os.getenv('DM_PASS')
|
||||
if not password:
|
||||
raise ValueError(
|
||||
"达梦模式需要设置环境变量 DM_PASS。\n"
|
||||
"示例: export DM_PASS=你的密码"
|
||||
)
|
||||
|
||||
conn = dmPython.connect(
|
||||
user=user,
|
||||
password=password,
|
||||
server=host,
|
||||
port=port,
|
||||
schema=db_cfg.get('schema', 'SYSDBA'),
|
||||
)
|
||||
return conn, db_cfg
|
||||
|
||||
|
||||
def load_from_dm(config: dict, project_root: str = None) -> list:
|
||||
"""
|
||||
从达梦数据库加载数据
|
||||
需要安装 dmPython 及达梦客户端库,仅在 source=db 时调用
|
||||
"""
|
||||
conn, db_cfg = _dm_connect(config)
|
||||
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
query = (db_cfg.get('query') or '').strip()
|
||||
table = db_cfg.get('table')
|
||||
if query:
|
||||
cursor.execute(query)
|
||||
elif table:
|
||||
schema = db_cfg.get('schema', 'SYSDBA')
|
||||
cursor.execute(f'SELECT * FROM "{schema}"."{table}"')
|
||||
else:
|
||||
raise ValueError(
|
||||
"达梦模式需配置 db.query(自定义 SQL)或 db.table。\n"
|
||||
"当前达梦为规范化表,需通过 query 编写 JOIN 得到扁平化数据。"
|
||||
)
|
||||
columns = [desc[0] for desc in cursor.description]
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
return [dict(zip(columns, row)) for row in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def load_data(config: dict, project_root: str) -> Tuple[list, dict]:
|
||||
"""
|
||||
根据配置加载数据,返回 (数据列表, 路径配置)
|
||||
"""
|
||||
source = config.get('source', 'json')
|
||||
project_root = project_root or os.getcwd()
|
||||
|
||||
if source == 'json':
|
||||
json_cfg = config.get('json', {})
|
||||
data = load_from_json(json_cfg.get('input_file', 'data/raw_data_sample.json'), project_root)
|
||||
paths = {
|
||||
'output': _resolve_path(json_cfg.get('output_file', 'data/nodes.json'), project_root),
|
||||
'report': _resolve_path(json_cfg.get('report_file', 'report/detailed_cleaning_report.json'), project_root),
|
||||
}
|
||||
return data, paths
|
||||
elif source == 'db':
|
||||
data = load_from_dm(config, project_root)
|
||||
json_cfg = config.get('json', {})
|
||||
paths = {
|
||||
'output': _resolve_path(json_cfg.get('output_file', 'data/nodes.json'), project_root),
|
||||
'report': _resolve_path(json_cfg.get('report_file', 'report/detailed_cleaning_report.json'), project_root),
|
||||
}
|
||||
return data, paths
|
||||
else:
|
||||
raise ValueError(f"不支持的 source 类型: {source},应为 json 或 db")
|
||||
|
||||
|
||||
def load_triplets_from_dm(config: dict, project_root: str = None) -> list:
|
||||
"""
|
||||
从达梦数据库读取三元组数据(RELATION_INSTANCE 表)
|
||||
返回格式: [{"HEAD_ID": x, "RELATION_TYPE": y, "TAIL_ID": z, ...}, ...]
|
||||
"""
|
||||
conn, db_cfg = _dm_connect(config)
|
||||
|
||||
try:
|
||||
cursor = conn.cursor()
|
||||
schema = db_cfg.get('schema', 'SYSDBA')
|
||||
triplet_table = db_cfg.get('triplet_table', 'RELATION_INSTANCE')
|
||||
query = db_cfg.get('triplet_query')
|
||||
if query and query.strip():
|
||||
cursor.execute(query.strip())
|
||||
else:
|
||||
cursor.execute(f'SELECT * FROM "{schema}"."{triplet_table}"')
|
||||
columns = [desc[0] for desc in cursor.description]
|
||||
rows = cursor.fetchall()
|
||||
cursor.close()
|
||||
return [dict(zip(columns, row)) for row in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _ci_key_map(d: dict) -> Dict[str, str]:
|
||||
return {str(k).upper(): k for k in d.keys()}
|
||||
|
||||
|
||||
def _get_ci(d: dict, key: str) -> Any:
|
||||
k = _ci_key_map(d).get(str(key).upper())
|
||||
return d.get(k) if k else None
|
||||
|
||||
|
||||
def _set_ci(d: dict, key: str, value: Any) -> None:
|
||||
um = _ci_key_map(d)
|
||||
k = um.get(str(key).upper())
|
||||
if k is None:
|
||||
d[key] = value
|
||||
else:
|
||||
d[k] = value
|
||||
|
||||
|
||||
def _table_columns(cursor, schema: str, table: str) -> List[str]:
|
||||
cursor.execute(f'SELECT * FROM "{schema}"."{table}" WHERE 1=0')
|
||||
return [desc[0] for desc in cursor.description]
|
||||
|
||||
|
||||
def _serialize_metric_value(val: Any) -> str:
|
||||
if val is None:
|
||||
return ""
|
||||
if isinstance(val, (dict, list)):
|
||||
return json.dumps(val, ensure_ascii=False)
|
||||
return str(val)
|
||||
|
||||
|
||||
def _value_from_aligned_node(node: dict, attr: str) -> Any:
|
||||
"""从 raw_to_target_schema 输出中取 required_attrs 对应值。"""
|
||||
if attr in FUNCTION_VECTOR_KEYS:
|
||||
return (node.get("function_vector") or {}).get(attr)
|
||||
if attr in ("position", "effective_radius"):
|
||||
return (node.get("spatial") or {}).get(attr)
|
||||
if attr in ("response_time", "cycle_time", "data_age", "time_window"):
|
||||
return (node.get("temporal") or {}).get(attr)
|
||||
if attr in ("core_performance", "survivability", "mtbf"):
|
||||
return (node.get("performance") or {}).get(attr)
|
||||
if attr in ("protocol_list", "format_list", "interface_list"):
|
||||
return node.get(attr)
|
||||
if attr in ("security_level", "org_unit", "nation", "history_success"):
|
||||
return node.get(attr)
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_reasoned_triples(cursor, config: dict, formal: dict) -> List[dict]:
|
||||
db_cfg = config.get("db", {})
|
||||
schema = db_cfg.get("schema", "SYSDBA")
|
||||
user_id = formal.get("user_id", "ADMIN")
|
||||
task_id = formal.get("task_id", "TASK_FULL_KB")
|
||||
source_sys_type = formal.get("source_sys_type", "关系推理后的体系")
|
||||
raw_query = (formal.get("source_query") or "").strip()
|
||||
if raw_query:
|
||||
cursor.execute(raw_query)
|
||||
else:
|
||||
table = formal.get("triplet_table") or db_cfg.get("triplet_table", "RELATION_INSTANCE")
|
||||
uc = formal.get("user_column", "USER_ID")
|
||||
tc = formal.get("task_column", "TASK_ID")
|
||||
sc = formal.get("sys_type_column", "SYS_TYPE")
|
||||
sql = (
|
||||
f'SELECT * FROM "{schema}"."{table}" '
|
||||
f'WHERE "{uc}" = ? AND "{tc}" = ? AND "{sc}" = ?'
|
||||
)
|
||||
cursor.execute(sql, [user_id, task_id, source_sys_type])
|
||||
columns = [desc[0] for desc in cursor.description]
|
||||
rows = cursor.fetchall()
|
||||
return [dict(zip(columns, row)) for row in rows]
|
||||
|
||||
|
||||
def _validate_source_sys_type(rows: List[dict], formal: dict) -> Optional[str]:
|
||||
"""若存在与 source_sys_type 不一致的 SYS_TYPE,返回错误信息(硬失败)。"""
|
||||
if formal.get("skip_sys_type_validation"):
|
||||
return None
|
||||
sc = formal.get("sys_type_column", "SYS_TYPE")
|
||||
expected = formal.get("source_sys_type", "关系推理后的体系")
|
||||
if not rows:
|
||||
return None
|
||||
vals = [_get_ci(r, sc) for r in rows]
|
||||
if all(v is None for v in vals):
|
||||
# 结果集中无 SYS_TYPE 列(例如仅用 source_query 投影),由 SQL 保证类型
|
||||
return None
|
||||
bad_indices = []
|
||||
for i, v in enumerate(vals):
|
||||
if v is None or str(v).strip() != str(expected).strip():
|
||||
bad_indices.append(i)
|
||||
if bad_indices:
|
||||
return f"sys_type_mismatch: expected={expected!r}, bad_row_indices={bad_indices[:20]}"
|
||||
return None
|
||||
|
||||
|
||||
def _entity_ids_from_triples(triples: List[dict]) -> List[str]:
|
||||
ids = set()
|
||||
for t in triples:
|
||||
h = _get_ci(t, "HEAD_ID") or t.get("head")
|
||||
tail = _get_ci(t, "TAIL_ID") or t.get("tail")
|
||||
if h is not None and str(h).strip():
|
||||
ids.add(str(h).strip())
|
||||
if tail is not None and str(tail).strip():
|
||||
ids.add(str(tail).strip())
|
||||
return sorted(ids)
|
||||
|
||||
|
||||
def _load_objectives_by_target(
|
||||
cursor, schema: str, table: str, entity_col: str, target_ids: List[str],
|
||||
) -> Dict[str, dict]:
|
||||
if not target_ids:
|
||||
return {}
|
||||
cols = _table_columns(cursor, schema, table)
|
||||
ec = None
|
||||
for c in cols:
|
||||
if c.upper() == entity_col.upper():
|
||||
ec = c
|
||||
break
|
||||
if not ec:
|
||||
return {}
|
||||
placeholders = ",".join(["?"] * len(target_ids))
|
||||
sql = f'SELECT * FROM "{schema}"."{table}" WHERE "{ec}" IN ({placeholders})'
|
||||
cursor.execute(sql, target_ids)
|
||||
out = {}
|
||||
row_cols = [desc[0] for desc in cursor.description]
|
||||
for row in cursor.fetchall():
|
||||
d = dict(zip(row_cols, row))
|
||||
tid = _get_ci(d, entity_col)
|
||||
if tid is not None:
|
||||
out[str(tid).strip()] = d
|
||||
return out
|
||||
|
||||
|
||||
def _load_existing_metric_keys(
|
||||
cursor, schema: str, attr_table: str, entity_col: str,
|
||||
metric_col: str, target_ids: List[str],
|
||||
) -> Dict[str, set]:
|
||||
"""target_id -> set(METRIC_ID) 已有记录。"""
|
||||
if not target_ids:
|
||||
return {}
|
||||
tcols = _table_columns(cursor, schema, attr_table)
|
||||
ec = next((c for c in tcols if c.upper() == entity_col.upper()), None)
|
||||
mc = next((c for c in tcols if c.upper() == metric_col.upper()), None)
|
||||
if not ec or not mc:
|
||||
return {}
|
||||
placeholders = ",".join(["?"] * len(target_ids))
|
||||
sql = (
|
||||
f'SELECT "{ec}", "{mc}" FROM "{schema}"."{attr_table}" '
|
||||
f'WHERE "{ec}" IN ({placeholders})'
|
||||
)
|
||||
cursor.execute(sql, target_ids)
|
||||
out: Dict[str, set] = {}
|
||||
for row in cursor.fetchall():
|
||||
tid, mid = row[0], row[1]
|
||||
if tid is None or mid is None:
|
||||
continue
|
||||
tid_s = str(tid).strip()
|
||||
out.setdefault(tid_s, set()).add(str(mid).strip())
|
||||
return out
|
||||
|
||||
|
||||
def _insert_generated_metrics(
|
||||
cursor,
|
||||
schema: str,
|
||||
attr_table: str,
|
||||
table_cols: List[str],
|
||||
entity_col: str,
|
||||
metric_col: str,
|
||||
value_col: str,
|
||||
target_id: str,
|
||||
metric_id: str,
|
||||
value_str: str,
|
||||
static_extra: Optional[dict],
|
||||
) -> bool:
|
||||
"""插入一条指标行;仅写入表中存在的列。未写入任何列时返回 False。"""
|
||||
um = {c.upper(): c for c in table_cols}
|
||||
row: Dict[str, Any] = {}
|
||||
if entity_col.upper() in um:
|
||||
row[um[entity_col.upper()]] = target_id
|
||||
if metric_col.upper() in um:
|
||||
row[um[metric_col.upper()]] = metric_id
|
||||
if value_col.upper() in um:
|
||||
row[um[value_col.upper()]] = value_str
|
||||
if static_extra:
|
||||
for k, v in static_extra.items():
|
||||
ku = str(k).upper()
|
||||
if ku in um:
|
||||
row[um[ku]] = v
|
||||
cols = list(row.keys())
|
||||
if entity_col.upper() not in um or metric_col.upper() not in um:
|
||||
return False
|
||||
if not cols:
|
||||
return False
|
||||
quoted = ",".join(f'"{c}"' for c in cols)
|
||||
ph = ",".join(["?"] * len(cols))
|
||||
sql = f'INSERT INTO "{schema}"."{attr_table}" ({quoted}) VALUES ({ph})'
|
||||
cursor.execute(sql, [row[c] for c in cols])
|
||||
return True
|
||||
|
||||
|
||||
def _build_formal_triple_rows(
|
||||
source_rows: List[dict],
|
||||
formal: dict,
|
||||
batch_id: str,
|
||||
result_table_cols: List[str],
|
||||
) -> Tuple[List[dict], int]:
|
||||
"""复制源行,写入 target_sys_type 与 batch_id;丢弃无效核字段。"""
|
||||
sys_col = formal.get("sys_type_column", "SYS_TYPE")
|
||||
target_sys = formal.get("target_sys_type", "形式化后的体系")
|
||||
pk = formal.get("triplet_primary_key") or formal.get("triplet_pk")
|
||||
omit_pk = formal.get("triplet_omit_primary_key_on_insert", True)
|
||||
batch_col = formal.get("triple_batch_column", "BATCH_ID")
|
||||
colset_upper = {c.upper() for c in result_table_cols}
|
||||
out_rows = []
|
||||
skipped = 0
|
||||
for r in source_rows:
|
||||
h = _get_ci(r, "HEAD_ID") or r.get("head")
|
||||
tail = _get_ci(r, "TAIL_ID") or r.get("tail")
|
||||
rel = _get_ci(r, "RELATION_TYPE") or r.get("relation")
|
||||
if not h or not tail or not rel:
|
||||
skipped += 1
|
||||
continue
|
||||
new_r = dict(r)
|
||||
_set_ci(new_r, sys_col, target_sys)
|
||||
if batch_col.upper() in colset_upper:
|
||||
_set_ci(new_r, batch_col, batch_id)
|
||||
if omit_pk and pk:
|
||||
kmap = _ci_key_map(new_r)
|
||||
pk_actual = kmap.get(str(pk).upper())
|
||||
if pk_actual and pk_actual in new_r:
|
||||
del new_r[pk_actual]
|
||||
filtered = {}
|
||||
for k, v in new_r.items():
|
||||
if k in result_table_cols:
|
||||
filtered[k] = v
|
||||
out_rows.append(filtered)
|
||||
return out_rows, skipped
|
||||
|
||||
|
||||
def _insert_rows_executemany(
|
||||
cursor, schema: str, table: str, rows: List[dict],
|
||||
) -> int:
|
||||
if not rows:
|
||||
return 0
|
||||
all_keys = []
|
||||
for r in rows:
|
||||
for k in r:
|
||||
if k not in all_keys:
|
||||
all_keys.append(k)
|
||||
quoted = ",".join(f'"{k}"' for k in all_keys)
|
||||
ph = ",".join(["?"] * len(all_keys))
|
||||
sql = f'INSERT INTO "{schema}"."{table}" ({quoted}) VALUES ({ph})'
|
||||
params = [tuple(r.get(k) for k in all_keys) for r in rows]
|
||||
cursor.executemany(sql, params)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _insert_res_formal_analysis(
|
||||
cursor,
|
||||
schema: str,
|
||||
analysis_table: str,
|
||||
batch_id: str,
|
||||
formal: dict,
|
||||
stats: dict,
|
||||
detail_obj: dict,
|
||||
) -> bool:
|
||||
cols = _table_columns(cursor, schema, analysis_table)
|
||||
cu = {c.upper(): c for c in cols}
|
||||
row: Dict[str, Any] = {}
|
||||
mapping = [
|
||||
("BATCH_ID", batch_id),
|
||||
("USER_ID", formal.get("user_id", "ADMIN")),
|
||||
("TASK_ID", formal.get("task_id", "TASK_FULL_KB")),
|
||||
("SOURCE_SYS_TYPE", formal.get("source_sys_type", "")),
|
||||
("TARGET_SYS_TYPE", formal.get("target_sys_type", "")),
|
||||
]
|
||||
for key, val in mapping:
|
||||
if key in cu:
|
||||
row[cu[key]] = val
|
||||
detail_json = json.dumps(detail_obj, ensure_ascii=False)
|
||||
detail_candidates = (
|
||||
"DETAIL_JSON",
|
||||
"DETAIL",
|
||||
"ANALYSIS_JSON",
|
||||
"CONTENT",
|
||||
"RULE_DETAIL",
|
||||
"STATS_JSON",
|
||||
"RES_FORMAL_ANALYSIS",
|
||||
)
|
||||
detail_key_col = None
|
||||
for cand in detail_candidates:
|
||||
if cand in cu:
|
||||
detail_key_col = cu[cand]
|
||||
break
|
||||
if not detail_key_col:
|
||||
for c in cols:
|
||||
u = c.upper()
|
||||
if any(
|
||||
token in u
|
||||
for token in (
|
||||
"DETAIL",
|
||||
"JSON",
|
||||
"CONTENT",
|
||||
"RULE",
|
||||
"TEXT",
|
||||
"MEMO",
|
||||
"DESC",
|
||||
"REMARK",
|
||||
"INFO",
|
||||
"ANALYSIS",
|
||||
)
|
||||
):
|
||||
detail_key_col = c
|
||||
break
|
||||
if detail_key_col:
|
||||
row[detail_key_col] = detail_json
|
||||
static = formal.get("analysis_static_columns") or formal.get("analysis_static") or {}
|
||||
for k, v in static.items():
|
||||
if str(k).upper() in cu:
|
||||
row[cu[str(k).upper()]] = v
|
||||
insert_cols = formal.get("analysis_insert_columns")
|
||||
if insert_cols:
|
||||
use_cols = [c for c in insert_cols if c in cols]
|
||||
if not use_cols:
|
||||
return False
|
||||
vals = [row.get(c) for c in use_cols]
|
||||
ph = ",".join(["?"] * len(use_cols))
|
||||
qc = ",".join(f'"{c}"' for c in use_cols)
|
||||
sql = f'INSERT INTO "{schema}"."{analysis_table}" ({qc}) VALUES ({ph})'
|
||||
cursor.execute(sql, vals)
|
||||
return True
|
||||
if not row:
|
||||
return False
|
||||
quoted = ",".join(f'"{c}"' for c in row.keys())
|
||||
ph = ",".join(["?"] * len(row))
|
||||
sql = f'INSERT INTO "{schema}"."{analysis_table}" ({quoted}) VALUES ({ph})'
|
||||
cursor.execute(sql, list(row.values()))
|
||||
return True
|
||||
|
||||
|
||||
def run_formalization_pipeline(config: dict) -> dict:
|
||||
"""
|
||||
模块 1.3:读取「关系推理后的体系」三元组,补齐实体属性并写回,
|
||||
将形式化结果作为新记录写入 formal_result_table,并写入 RES_FORMAL_ANALYSIS。
|
||||
需在 config 中 formal.enabled=true 且 source=db;达梦密码 DM_PASS。
|
||||
"""
|
||||
formal = config.get("formal") or {}
|
||||
if not formal.get("enabled", False):
|
||||
return {"ok": False, "reason": "formal.disabled", "batch_id": None, "stats": {}}
|
||||
if config.get("source", "json") != "db":
|
||||
return {"ok": False, "reason": "source_not_db", "batch_id": None, "stats": {}}
|
||||
|
||||
batch_id = str(uuid.uuid4())
|
||||
stats: Dict[str, Any] = {
|
||||
"total_triples": 0,
|
||||
"inserted_formal_count": 0,
|
||||
"generated_attrs_count": 0,
|
||||
"skipped_triples": 0,
|
||||
"analysis_written": False,
|
||||
}
|
||||
|
||||
conn = None
|
||||
try:
|
||||
conn, db_cfg = _dm_connect(config)
|
||||
except (RuntimeError, ValueError) as e:
|
||||
return {"ok": False, "reason": str(e), "batch_id": None, "stats": stats}
|
||||
|
||||
schema = db_cfg.get("schema", "SYSDBA")
|
||||
user_id = formal.get("user_id", "ADMIN")
|
||||
task_id = formal.get("task_id", "TASK_FULL_KB")
|
||||
source_sys_type = formal.get("source_sys_type", "关系推理后的体系")
|
||||
target_sys_type = formal.get("target_sys_type", "形式化后的体系")
|
||||
attr_table = formal.get("attribute_table", "METRIC_VALUE_INSTANCE")
|
||||
result_table = formal.get("formal_result_table", "RELATION_INSTANCE")
|
||||
analysis_table = formal.get("analysis_table", "RES_FORMAL_ANALYSIS")
|
||||
entity_col = formal.get("entity_id_column", "TARGET_ID")
|
||||
metric_col = formal.get("metric_id_column", "METRIC_ID")
|
||||
value_col = formal.get("metric_value_column", "METRIC_VALUE")
|
||||
objective_table = formal.get("objective_table") or db_cfg.get("table", "OBJECTIVE_INSTANCE")
|
||||
required_attrs: List[str] = list(formal.get("required_attrs") or [])
|
||||
|
||||
try:
|
||||
if hasattr(conn, "autocommit"):
|
||||
conn.autocommit = False
|
||||
cursor = conn.cursor()
|
||||
|
||||
triples = _fetch_reasoned_triples(cursor, config, formal)
|
||||
stats["total_triples"] = len(triples)
|
||||
err = _validate_source_sys_type(triples, formal)
|
||||
if err:
|
||||
conn.rollback()
|
||||
return {"ok": False, "reason": err, "batch_id": batch_id, "stats": stats}
|
||||
|
||||
eids = _entity_ids_from_triples(triples)
|
||||
objectives = _load_objectives_by_target(
|
||||
cursor, schema, objective_table, entity_col, eids,
|
||||
)
|
||||
existing_metrics = _load_existing_metric_keys(
|
||||
cursor, schema, attr_table, entity_col, metric_col, eids,
|
||||
)
|
||||
mvi_cols = _table_columns(cursor, schema, attr_table)
|
||||
static_metric = formal.get("metric_insert_static") or {}
|
||||
|
||||
for tid in eids:
|
||||
raw = dict(objectives.get(tid, {}))
|
||||
raw["TARGET_ID"] = tid
|
||||
aligned = raw_to_target_schema(raw)
|
||||
have = existing_metrics.get(tid, set())
|
||||
for attr in required_attrs:
|
||||
if attr in have:
|
||||
continue
|
||||
val = _value_from_aligned_node(aligned, attr)
|
||||
vstr = _serialize_metric_value(val)
|
||||
if _insert_generated_metrics(
|
||||
cursor,
|
||||
schema,
|
||||
attr_table,
|
||||
mvi_cols,
|
||||
entity_col,
|
||||
metric_col,
|
||||
value_col,
|
||||
tid,
|
||||
attr,
|
||||
vstr,
|
||||
static_metric,
|
||||
):
|
||||
stats["generated_attrs_count"] += 1
|
||||
have.add(attr)
|
||||
|
||||
result_cols = _table_columns(cursor, schema, result_table)
|
||||
formal_rows, skipped = _build_formal_triple_rows(
|
||||
triples, formal, batch_id, result_cols,
|
||||
)
|
||||
stats["skipped_triples"] = skipped
|
||||
if formal_rows:
|
||||
_insert_rows_executemany(cursor, schema, result_table, formal_rows)
|
||||
stats["inserted_formal_count"] = len(formal_rows)
|
||||
|
||||
detail = {
|
||||
"batch_id": batch_id,
|
||||
"user_id": user_id,
|
||||
"task_id": task_id,
|
||||
"source_sys_type": source_sys_type,
|
||||
"target_sys_type": target_sys_type,
|
||||
"stats": stats,
|
||||
"rules": formal.get("rules_summary")
|
||||
or "结构校验+属性对齐(raw_to_target_schema)+sys_type切换为形式化后的体系",
|
||||
}
|
||||
stats["analysis_written"] = _insert_res_formal_analysis(
|
||||
cursor, schema, analysis_table, batch_id, formal, stats, detail,
|
||||
)
|
||||
if not stats["analysis_written"]:
|
||||
raise RuntimeError(
|
||||
"RES_FORMAL_ANALYSIS 未写入:表无可匹配列。请配置 formal.analysis_insert_columns "
|
||||
"或在表中提供 BATCH_ID/USER_ID/TASK_ID 及 DETAIL/JSON 类文本列。"
|
||||
)
|
||||
|
||||
conn.commit()
|
||||
cursor.close()
|
||||
return {"ok": True, "reason": None, "batch_id": batch_id, "stats": stats}
|
||||
except Exception as e:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
stats.setdefault("error", str(e))
|
||||
return {"ok": False, "reason": str(e), "batch_id": batch_id, "stats": stats}
|
||||
finally:
|
||||
if conn is not None:
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _resolve_path(path: str, project_root: str) -> str:
|
||||
if os.path.isabs(path):
|
||||
return path
|
||||
return os.path.join(project_root, path)
|
||||
666
final/src/main_227.py
Normal file
666
final/src/main_227.py
Normal file
@@ -0,0 +1,666 @@
|
||||
# main.py
|
||||
import numpy as np
|
||||
import json, copy
|
||||
import pandas as pd
|
||||
from typing import Dict, List
|
||||
import sys
|
||||
from pathlib import Path
|
||||
sys.path.append(str(Path(__file__).parent.parent))
|
||||
from scripts.f_relation1125 import RelationCalculator
|
||||
from scripts.f_net1125 import TaskNetworkEvaluator
|
||||
|
||||
# ========== 新增:导入 cleaner ==========
|
||||
sys.path.append(str(Path(__file__).parent)) # 添加 src 目录到路径
|
||||
from cleaner import load_config, AdvancedDataCleaner
|
||||
from data_loader import load_data as load_data_source, run_formalization_pipeline
|
||||
|
||||
# -------------- 默认配置(与原来硬编码保持一致) --------------
|
||||
DEFAULT_CFG = {
|
||||
"W_REL": { # 5 类关系权重
|
||||
"IS": {"w_f": 0.35, "w_s": 0.25, "w_t": 0.15, "w_p": 0.15, "w_i": 0.10},
|
||||
"CC": {"w_f": 0.30, "w_s": 0.20, "w_t": 0.20, "w_p": 0.20, "w_i": 0.10},
|
||||
"SF": {"w_f": 0.30, "w_s": 0.20, "w_t": 0.25, "w_p": 0.15, "w_i": 0.10},
|
||||
"PD": {"w_f": 0.25, "w_s": 0.35, "w_t": 0.15, "w_p": 0.15, "w_i": 0.10},
|
||||
"CO": {"w_f": 0.30, "w_s": 0.20, "w_t": 0.20, "w_p": 0.20, "w_i": 0.10},
|
||||
},
|
||||
"W_NET": { # 5 类任务网络权重
|
||||
"defense": {"w1": 0.30, "w2": 0.25, "w3": 0.20, "w4": 0.15, "w5": 0.10},
|
||||
"fire": {"w1": 0.35, "w2": 0.25, "w3": 0.20, "w4": 0.10, "w5": 0.10},
|
||||
"logistics":{"w1": 0.30, "w2": 0.25, "w3": 0.20, "w4": 0.15, "w5": 0.10},
|
||||
"medical": {"w1": 0.35, "w2": 0.25, "w3": 0.20, "w4": 0.10, "w5": 0.10},
|
||||
"evacuation":{"w1":0.30, "w2": 0.25, "w3": 0.20, "w4": 0.15, "w5": 0.10},
|
||||
},
|
||||
"LAMBDA": {"IS": 100, "CC": 50, "SF": 80, "PD": 30, "CO": 60},
|
||||
"TIME": {"alpha": 0.5, "beta": 0.6, "gamma": 0.4},
|
||||
"PD_SGM": 20,
|
||||
"THRESHOLD": 0.15 # 关系过滤阈值
|
||||
}
|
||||
|
||||
|
||||
def var_desc():
|
||||
return {
|
||||
# 共用
|
||||
"dij": "节点 i 与 j 之间的欧氏距离",
|
||||
"Ri": "节点 i 的有效作用半径",
|
||||
"Rmatch(i,j)": "两节点作用范围重叠度",
|
||||
"s_area": "作战区域关联度(同区 1.0 / 邻区 0.7 / 异区 0.3)",
|
||||
"stdij": "接口标准兼容性(0-1)",
|
||||
"rij": "任务角色匹配度(0-1)",
|
||||
"t_resp": "节点响应时间得分 exp(-α·T_resp)",
|
||||
"t_cycle": "节点处理周期得分 exp(-β·T_cycle)",
|
||||
"fresh_i": "数据新鲜度 exp(-γ·数据年龄)",
|
||||
"wij": "时间窗口重叠比例",
|
||||
"p_core": "节点核心性能",
|
||||
"p_rel": "节点可靠性 MTBF/MTBF_max",
|
||||
"p_surv": "节点生存性",
|
||||
"protij": "协议兼容性",
|
||||
"fmtij": "数据格式兼容性",
|
||||
"secij": "安全等级兼容性",
|
||||
"orgij": "组织隶属关联度",
|
||||
"histij": "历史交互成功率",
|
||||
# 关系专用
|
||||
"M_F^IS": "情报保障功能匹配项",
|
||||
"M_S^IS": "情报保障空间匹配项",
|
||||
"M_T^IS": "情报保障时间匹配项",
|
||||
"M_P^IS": "情报保障性能匹配项",
|
||||
"M_I^IS": "情报保障交互匹配项",
|
||||
"λ_IS": "情报特征距离(λ_IS=100 km)",
|
||||
# 网络专用
|
||||
"N_IC": "信息对抗节点数量",
|
||||
"∑f_IC/N_IC": "信息对抗节点平均能力",
|
||||
"∑L_IS/N_IS": "情报保障关系平均强度",
|
||||
"∑t_resp/N": "系统平均响应速度",
|
||||
"∑L_CC/N_CC": "指挥控制关系平均强度",
|
||||
"∑f_CS/N_CS": "协同打击节点平均能力",
|
||||
"∑f_IA/N_IA": "信息获取节点平均能力",
|
||||
"∑L_CO/N_CO": "协同作战关系平均强度",
|
||||
"∑p_core·f_CS/N_CS": "打击节点性能加权能力",
|
||||
"∑f_CPS/N_CPS": "综合保障节点平均能力",
|
||||
"∑f_DP/N_DP": "部署平台节点平均能力",
|
||||
"∑L_PD/N_PD": "平台部署关系平均强度",
|
||||
"∑p_rel/N": "系统平均可靠性",
|
||||
"∑f_IT/N_IT": "信息传输节点平均能力",
|
||||
"∑L_SF/N_SF": "状态反馈关系平均强度",
|
||||
"∑fresh_i/N": "系统平均数据新鲜度",
|
||||
"α1...α5": "综合防御权重 [0.30,0.25,0.20,0.15,0.10]",
|
||||
"β1...β5": "火力打击权重 [0.35,0.25,0.20,0.10,0.10]",
|
||||
"γ1...γ5": "后勤保障权重 [0.30,0.25,0.20,0.15,0.10]",
|
||||
"δ1...δ5": "医疗救援权重 [0.35,0.25,0.20,0.10,0.10]",
|
||||
"ε1...ε5": "紧急疏散权重 [0.30,0.25,0.20,0.15,0.10]",
|
||||
}
|
||||
|
||||
|
||||
def load_data(project_root=None):
|
||||
"""加载节点和三联数据,路径从配置读取(兼容 run_all 输出)"""
|
||||
if project_root is None:
|
||||
project_root = Path(__file__).resolve().parent.parent
|
||||
project_root = Path(project_root)
|
||||
cfg = load_config(str(project_root))
|
||||
eval_cfg = cfg.get("eval", {})
|
||||
json_cfg = cfg.get("json", {})
|
||||
triplet_cfg = cfg.get("triplet", {})
|
||||
|
||||
# 节点文件:优先 eval.nodes_file,否则 json.output_file
|
||||
nodes_file = eval_cfg.get("nodes_file") or json_cfg.get("output_file", "data/nodes.json")
|
||||
nodes_path = project_root / nodes_file if not Path(nodes_file).is_absolute() else Path(nodes_file)
|
||||
with open(nodes_path, 'r', encoding='utf-8') as f:
|
||||
nodes_data = json.load(f)
|
||||
|
||||
# 三元组文件:eval.triples_file,不存在则用 triplet.output_file
|
||||
triples_file = eval_cfg.get("triples_file", "data/triples.json")
|
||||
triples_path = project_root / triples_file if not Path(triples_file).is_absolute() else Path(triples_file)
|
||||
if not triples_path.exists():
|
||||
triples_path = project_root / triplet_cfg.get("output_file", "data/triples.json")
|
||||
with open(triples_path, 'r', encoding='utf-8') as f:
|
||||
triples_raw = json.load(f)
|
||||
# 兼容 list 格式(run_all 输出的 triples.json)
|
||||
triples_data = triples_raw if isinstance(triples_raw, dict) and "triples" in triples_raw else {"triples": triples_raw if isinstance(triples_raw, list) else []}
|
||||
|
||||
return nodes_data, triples_data
|
||||
|
||||
|
||||
def calculate_and_filter_relations(nodes_data: Dict, triples_data: Dict, threshold, cfg):
|
||||
"""
|
||||
计算所有三联关系的关系强度,并过滤强度大于阈值的边
|
||||
|
||||
返回:
|
||||
- filtered_relations: 过滤后的关系字典
|
||||
- filtered_triples: 过滤后的三元组列表(保持原始结构)
|
||||
- relation_stats: 关系统计信息
|
||||
"""
|
||||
print("开始计算关系强度并过滤...")
|
||||
|
||||
# 初始化关系计算器
|
||||
|
||||
calculator = RelationCalculator(nodes_data, cfg) # 改造见下
|
||||
# 关系类型映射(中文 -> 英文缩写)
|
||||
relation_mapping = {
|
||||
"情报保障": "IS",
|
||||
"指挥控制": "CC",
|
||||
"状态反馈": "SF",
|
||||
"平台部署": "PD",
|
||||
"协同作战": "CO"
|
||||
}
|
||||
|
||||
all_relations = {}
|
||||
filtered_relations = {}
|
||||
filtered_triples = []
|
||||
|
||||
valid_triples = 0
|
||||
skipped_triples = 0
|
||||
passed_triples = 0
|
||||
|
||||
for triple in triples_data["triples"]:
|
||||
head = triple["head"]
|
||||
tail = triple["tail"]
|
||||
relation_ch = triple["relation"]
|
||||
|
||||
# 检查节点是否存在
|
||||
if head not in nodes_data["nodes"] or tail not in nodes_data["nodes"]:
|
||||
print(f"警告: 节点不存在 - 头: {head}, 尾: {tail},跳过此三元组")
|
||||
skipped_triples += 1
|
||||
continue
|
||||
|
||||
# 映射关系类型
|
||||
relation_en = relation_mapping.get(relation_ch)
|
||||
if not relation_en:
|
||||
print(f"警告: 未知关系类型 '{relation_ch}',跳过此三元组")
|
||||
skipped_triples += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
# 根据关系类型调用对应的计算方法
|
||||
if relation_en == "IS":
|
||||
strength = calculator.calculate_IS_relation(head, tail)
|
||||
elif relation_en == "CC":
|
||||
strength = calculator.calculate_CC_relation(head, tail)
|
||||
elif relation_en == "SF":
|
||||
strength = calculator.calculate_SF_relation(head, tail)
|
||||
elif relation_en == "PD":
|
||||
strength = calculator.calculate_PD_relation(head, tail)
|
||||
elif relation_en == "CO":
|
||||
strength = calculator.calculate_CO_relation(head, tail)
|
||||
else:
|
||||
print(f"警告: 未实现的关系类型 '{relation_en}',跳过此三元组")
|
||||
skipped_triples += 1
|
||||
continue
|
||||
|
||||
# 存储所有关系强度(用于统计)
|
||||
all_relations[(relation_en, head, tail)] = strength
|
||||
valid_triples += 1
|
||||
|
||||
# 过滤:只保留强度大于阈值的关系
|
||||
if strength > threshold:
|
||||
filtered_relations[(relation_en, head, tail)] = strength
|
||||
# 创建过滤后的三元组(保持原始结构,添加强度字段)
|
||||
filtered_triple = triple.copy()
|
||||
filtered_triple["strength"] = float(strength) # 转换为Python float类型
|
||||
filtered_triples.append(filtered_triple)
|
||||
passed_triples += 1
|
||||
|
||||
if valid_triples % 10 == 0: # 每10个关系打印一次进度
|
||||
print(f"已计算 {valid_triples} 个关系,通过 {passed_triples} 个...")
|
||||
|
||||
except Exception as e:
|
||||
print(f"错误: 计算关系失败 - 头: {head}, 尾: {tail}, 关系: {relation_en}, 错误: {e}")
|
||||
skipped_triples += 1
|
||||
# 1. 先按类型细分
|
||||
type_detail = {}
|
||||
for (rel, _, _), val in all_relations.items():
|
||||
type_detail.setdefault(rel, {"all": 0, "pass": 0})
|
||||
type_detail[rel]["all"] += 1
|
||||
# if val > threshold:
|
||||
# type_detail[rel]["pass"] += 1
|
||||
# 新增:按类型收集被过滤的关系
|
||||
filtered_out = {} # rel -> [(head, tail, strength), ...]
|
||||
for (rel, head, tail), val in all_relations.items():
|
||||
if val <= threshold:
|
||||
filtered_out.setdefault(rel, []).append((head, tail, val))
|
||||
|
||||
# 2. 再打包大字典
|
||||
relation_stats = {
|
||||
"total_triples": len(triples_data["triples"]),
|
||||
"valid_triples": valid_triples,
|
||||
"skipped_triples": skipped_triples,
|
||||
"passed_triples": passed_triples,
|
||||
"pass_rate": passed_triples / valid_triples if valid_triples else 0,
|
||||
"threshold": threshold,
|
||||
"filtered_out":filtered_out,
|
||||
"type_detail": {
|
||||
# rel: {"all": d["all"], "pass": d["pass"], "rate": d["pass"] / d["all"] if d["all"] else 0.}
|
||||
rel: {"all": d["all"]}
|
||||
for rel, d in type_detail.items()
|
||||
}
|
||||
}
|
||||
|
||||
print(f"关系强度计算和过滤完成!")
|
||||
print(f"总三元组: {relation_stats['total_triples']}, 有效计算: {valid_triples}, 跳过: {skipped_triples}")
|
||||
print(f"通过过滤(> {threshold}): {passed_triples}, 通过率: {relation_stats['pass_rate']:.1%}")
|
||||
|
||||
return filtered_relations, filtered_triples, relation_stats, all_relations
|
||||
|
||||
|
||||
def evaluate_task_networks(nodes_data, relations, cfg):
|
||||
"""
|
||||
评估所有任务网络效能
|
||||
"""
|
||||
print("\n开始评估任务网络效能...")
|
||||
|
||||
# 初始化网络评估器
|
||||
evaluator = TaskNetworkEvaluator(nodes_data["nodes"], relations, weights=cfg["W_NET"])
|
||||
|
||||
# 评估所有任务网络
|
||||
results = evaluator.eval_all()
|
||||
|
||||
print("任务网络效能评估完成!")
|
||||
return results
|
||||
|
||||
|
||||
def generate_evaluation_report(relation_stats: Dict, network_results: Dict,
|
||||
filtered_relations: Dict, all_relations: Dict,
|
||||
threshold: float, cfg) -> str:
|
||||
"""
|
||||
生成详细的评价报告
|
||||
"""
|
||||
report = []
|
||||
report.append("=" * 60)
|
||||
report.append("网络关系与效能评价报告")
|
||||
report.append("=" * 60)
|
||||
report.append("")
|
||||
|
||||
# 1. 基本信息
|
||||
report.append("1. 基本信息")
|
||||
report.append("-" * 40)
|
||||
report.append(f"过滤阈值: {threshold} 【关系强度阈值】")
|
||||
report.append(f"总三元组数量: {relation_stats['total_triples']}")
|
||||
line = " ".join(f"{rel}={d['all']}" for rel, d in relation_stats["type_detail"].items())
|
||||
report.append(f"分别有:{line}")
|
||||
report.append(f"有效计算三元组: {relation_stats['valid_triples']}")
|
||||
report.append(f"跳过三元组: {relation_stats['skipped_triples']}")
|
||||
report.append(f"通过过滤三元组: {relation_stats['passed_triples']}")
|
||||
report.append(f"通过率: {relation_stats['pass_rate']:.1%}")
|
||||
report.append("")
|
||||
|
||||
# 添加边关系计算公式和说明
|
||||
report.append("2. 计算公式")
|
||||
report.append("边关系计算公式")
|
||||
report.append("-" * 50)
|
||||
rel_cn = {"IS": "情报保障", "CC": "指挥控制", "SF": "状态反馈", "PD": "平台部署", "CO": "协同作战"}
|
||||
desc = var_desc()
|
||||
for rel in ["IS", "CC", "SF", "PD", "CO"]:
|
||||
report.append(
|
||||
f"{rel_cn[rel]}关系 L_{rel}(i,j) = w_f·M_F^{rel} + w_s·M_S^{rel} + w_t·M_T^{rel} + w_p·M_P^{rel} + w_i·M_I^{rel}")
|
||||
report.append("【符号说明】")
|
||||
for sym, exp in desc.items():
|
||||
if sym.startswith(("M_F^", "M_S^", "M_T^", "M_P^", "M_I^")):
|
||||
report.append(f" {sym:<12} {exp}")
|
||||
report.append("")
|
||||
|
||||
|
||||
|
||||
report.append("任务网络效能计算公式")
|
||||
report.append("-" * 50)
|
||||
task_desc = {
|
||||
"defense": "综合防御 P_defense = α1·(∑f_IC/N_IC) + α2·(∑f_IA/N_IA) + α3·(∑L_IS/N_IS) + α4·(∑t_resp/N) + α5·(∑L_CC/N_CC)",
|
||||
"fire": "火力打击 P_fire = β1·(∑f_CS/N_CS) + β2·(∑f_IA/N_IA) + β3·(∑L_CO/N_CO) + β4·(∑p_core·f_CS/N_CS) + β5·(∑L_IS/N_IS)",
|
||||
"logistics": "后勤保障 P_logistics = γ1·(∑f_CPS/N_CPS) + γ2·(∑f_DP/N_DP) + γ3·(∑L_PD/N_PD) + γ4·(∑p_rel/N) + γ5·(∑f_IT/N_IT)",
|
||||
"medical": "医疗救援 P_medical = δ1·(∑f_CPS/N_CPS) + δ2·(∑t_resp/N) + δ3·(∑f_IT/N_IT) + δ4·(∑L_SF/N_SF) + δ5·(∑fresh_i/N)",
|
||||
"evacuation": "紧急疏散 P_evacuation = ε1·(∑f_CC/N_CC) + ε2·(∑f_IT/N_IT) + ε3·(∑L_CC/N_CC) + ε4·(∑f_CPS/N_CPS) + ε5·(∑t_resp/N)"
|
||||
}
|
||||
for task, form in task_desc.items():
|
||||
report.append(form)
|
||||
report.append("【符号说明】")
|
||||
for sym, exp in desc.items():
|
||||
if any(sym.startswith(x) for x in ["N_", "∑", "α", "β", "γ", "δ", "ε"]):
|
||||
report.append(f" {sym:<18} {exp}")
|
||||
report.append("")
|
||||
|
||||
# 2. 按关系类型统计
|
||||
report.append("3. 关系类型统计")
|
||||
report.append("-" * 40)
|
||||
|
||||
relation_type_stats = {}
|
||||
for (rel_type, head, tail), strength in all_relations.items():
|
||||
if rel_type not in relation_type_stats:
|
||||
relation_type_stats[rel_type] = {"all": [], "filtered": []}
|
||||
relation_type_stats[rel_type]["all"].append(strength)
|
||||
|
||||
if strength > threshold:
|
||||
relation_type_stats[rel_type]["filtered"].append(strength)
|
||||
|
||||
for rel_type in sorted(relation_type_stats.keys()):
|
||||
stats = relation_type_stats[rel_type]
|
||||
all_count = len(stats["all"])
|
||||
filtered_count = len(stats["filtered"])
|
||||
pass_rate = filtered_count / all_count if all_count > 0 else 0
|
||||
|
||||
report.append(f"{rel_type}关系:")
|
||||
report.append(f" 总数: {all_count}, 通过: {filtered_count}, 通过率: {pass_rate:.1%}")
|
||||
if stats["all"]:
|
||||
report.append(f" 平均强度: {np.mean(stats['all']):.3f}")
|
||||
if stats["filtered"]:
|
||||
report.append(f" 过滤后平均强度: {np.mean(stats['filtered']):.3f}")
|
||||
report.append("【被过滤掉的关系明细】")
|
||||
report.append("-" * 40)
|
||||
for rel, lst in relation_stats["filtered_out"].items():
|
||||
if not lst:
|
||||
continue
|
||||
report.append(f"{rel} 关系(强度 ≤ {threshold}):")
|
||||
for h, t, v in lst:
|
||||
report.append(f" {h} → {t} 强度={v:.3f}")
|
||||
report.append("") # 空行隔开
|
||||
|
||||
# 1. 取任务得分(排除 dimensions)
|
||||
task_scores = {task: score for task, score in network_results.items() if isinstance(score, float)}
|
||||
total = sum(task_scores.values()) or 1.0 # 防 0
|
||||
|
||||
# 2. 归一化 → 百分比
|
||||
task_pct = {task: (sc / total) * 100 for task, sc in task_scores.items()}
|
||||
|
||||
# 3. 输出
|
||||
report.append("4. 所属任务网络评估")
|
||||
report.append("-" * 40)
|
||||
task_name = {
|
||||
"defense": "综合防御",
|
||||
"fire": "火力打击",
|
||||
"logistics": "后勤保障",
|
||||
"medical": "医疗救援",
|
||||
"evacuation": "紧急疏散"
|
||||
}
|
||||
# 一行输出所有任务网络百分比
|
||||
line = " ".join(f"{task_name[task]} {pct:.1f}%" for task, pct in task_pct.items())
|
||||
report.append("任务网络概率:" + line)
|
||||
# 4. 最可能任务
|
||||
max_task = max(task_pct, key=task_pct.get)
|
||||
report.append(f"最有可能的任务网络:{task_name[max_task]} 可能性为:{task_pct[max_task]:.1f} %")
|
||||
|
||||
# 添加最终网络的五个维度评价
|
||||
report.append(f"5. 最终{task_name[max_task]}任务网络的五个维度评价")
|
||||
dimensions = network_results.get("dimensions", {})
|
||||
reference_value = 0.6 # 参考值
|
||||
|
||||
# 功能维度
|
||||
report.append("功能维度 (Function):")
|
||||
report.append(f"功能维度反映了网络中节点的功能匹配程度。得分: {dimensions.get('function', 0):.3f}")
|
||||
if dimensions.get('function', 0) > reference_value:
|
||||
report.append("高于参考值,表示节点在功能上高度互补,能够有效协同工作。")
|
||||
else:
|
||||
report.append("低于参考值,表示节点在功能上匹配程度较低,协同工作能力有待提升。")
|
||||
|
||||
# 边关系维度
|
||||
report.append("空间系维度 (Relationship):")
|
||||
report.append(f"空间维度反映了节点之间的连接强度。得分: {dimensions.get('relationship', 0):.3f}")
|
||||
if dimensions.get('relationship', 0) > reference_value:
|
||||
report.append("高于参考值,表示节点之间的连接紧密,信息传递高效。")
|
||||
else:
|
||||
report.append("低于参考值,表示节点之间的连接较弱,信息传递效率较低。")
|
||||
|
||||
# 性能维度
|
||||
report.append("性能维度 (Performance):")
|
||||
report.append(f"性能维度反映了节点的性能指标,如响应时间和可靠性。得分: {dimensions.get('performance', 0):.3f}")
|
||||
if dimensions.get('performance', 0) > reference_value:
|
||||
report.append("高于参考值,表示节点在性能上表现出色,能够稳定运行。")
|
||||
else:
|
||||
report.append("低于参考值,表示节点在性能上存在不足,稳定性有待提高。")
|
||||
|
||||
# 时间维度
|
||||
report.append("时间维度 (Temporal):")
|
||||
report.append(f"时间维度反映了节点的时间同步性和响应能力。得分: {dimensions.get('temporal', 0):.3f}")
|
||||
if dimensions.get('temporal', 0) > reference_value:
|
||||
report.append("高于参考值,表示节点在时间上高度同步,能够快速响应。")
|
||||
else:
|
||||
report.append("低于参考值,表示节点在时间同步性上存在偏差,响应速度较慢。")
|
||||
|
||||
# 交互维度
|
||||
report.append("交互维度 (Interaction):")
|
||||
report.append(f"交互维度反映了节点之间的协议、格式和安全兼容性。得分: {dimensions.get('interaction', 0):.3f}")
|
||||
if dimensions.get('interaction', 0) > reference_value:
|
||||
report.append("高于参考值,表示节点在交互上高度兼容,能够安全通信。")
|
||||
else:
|
||||
report.append("低于参考值,表示节点在交互上存在兼容性问题,通信安全性较低。")
|
||||
|
||||
return "\n".join(report)
|
||||
|
||||
|
||||
def save_filtered_triples(filtered_triples: List[Dict], filename: str = "filtered_triples.json"):
|
||||
"""
|
||||
保存过滤后的三元组到JSON文件
|
||||
"""
|
||||
project_root = Path(__file__).parent.parent # src -> 227
|
||||
output_path = project_root / "results" / filename
|
||||
|
||||
output_data = {"triples": filtered_triples}
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(output_data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"过滤后的三元组已保存到 {filename}")
|
||||
|
||||
|
||||
def save_triples_and_task_probs(filtered_triples: list,
|
||||
network_results: dict,
|
||||
excel_name: str = "triples.xlsx",
|
||||
csv_name: str = "triples.csv") -> None:
|
||||
"""
|
||||
保存过滤后的三元组(csv + Excel 表1)+
|
||||
归一化任务网络概率(Excel 表2)
|
||||
"""
|
||||
# 1. 三元组 DataFrame(含 head_type / tail_type)
|
||||
triple_df = pd.DataFrame([
|
||||
{
|
||||
"head": t["head"],
|
||||
"head_type": t.get("head_type", ""),
|
||||
"relation": t["relation"],
|
||||
"tail": t["tail"],
|
||||
"tail_type": t.get("tail_type", ""),
|
||||
"strength": t["strength"]
|
||||
}
|
||||
for t in filtered_triples
|
||||
])
|
||||
|
||||
# 2. 任务网络概率 DataFrame
|
||||
task_scores = {task: score for task, score in network_results.items() if isinstance(score, float)}
|
||||
total = sum(task_scores.values()) or 1.0
|
||||
task_pct = {task: (sc / total) * 100 for task, sc in task_scores.items()}
|
||||
|
||||
task_map = {
|
||||
"defense": "1-综合防御任务网络",
|
||||
"fire": "2-火力打击任务网络",
|
||||
"logistics": "3-后勤保障任务网络",
|
||||
"medical": "4-医疗救援任务网络",
|
||||
"evacuation": "5-紧急疏散任务网络"
|
||||
}
|
||||
prob_df = pd.DataFrame([
|
||||
{"id": int(task_map[task].split('-')[0]),
|
||||
"任务": task_map[task].split('-')[1],
|
||||
"概率%": f"{pct:.1f}"}
|
||||
for task, pct in task_pct.items()
|
||||
])
|
||||
|
||||
# 3. 写文件
|
||||
project_root = Path(__file__).parent.parent # src -> 227
|
||||
output_path_csv = project_root / "results" / csv_name
|
||||
triple_df.to_csv(output_path_csv, index=False, encoding='utf-8-sig')
|
||||
|
||||
output_path = project_root / "results" / excel_name
|
||||
|
||||
with pd.ExcelWriter(output_path, engine='openpyxl') as writer:
|
||||
triple_df.to_excel(writer, sheet_name='triples', index=False)
|
||||
prob_df.to_excel(writer, sheet_name='TaskProbs', index=False)
|
||||
|
||||
print(f"已生成:{csv_name} | {excel_name}(含 TaskProbs 页)")
|
||||
|
||||
|
||||
def save_evaluation_report(report_content: str, filename: str = "evaluation_report.txt"):
|
||||
"""
|
||||
保存评价报告到TXT文件
|
||||
"""
|
||||
project_root = Path(__file__).parent.parent # src -> 227
|
||||
output_path = project_root / "results" / filename
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write(report_content)
|
||||
|
||||
print(f"评价报告已保存到 {filename}")
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
主函数:整合所有计算流程
|
||||
"""
|
||||
"""运行时交互版主函数"""
|
||||
import os
|
||||
from data_loader import load_data as load_data_source
|
||||
|
||||
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
project_root = os.path.dirname(script_dir)
|
||||
|
||||
config = load_config(project_root)
|
||||
source = config.get('source', 'json')
|
||||
print(f"数据源模式: {source}")
|
||||
|
||||
try:
|
||||
data, paths = load_data_source(config, project_root)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"错误:{e}")
|
||||
exit(1)
|
||||
|
||||
os.makedirs(os.path.dirname(paths['report']), exist_ok=True)
|
||||
|
||||
cleaner = AdvancedDataCleaner(
|
||||
output_file=paths['output'],
|
||||
report_file=paths['report'],
|
||||
data=data,
|
||||
)
|
||||
cleaner.run()
|
||||
|
||||
formal_cfg = config.get("formal", {})
|
||||
if source == "db" and formal_cfg.get("enabled", False):
|
||||
print("\n=== 模块1.3:形式化入库流程 ===")
|
||||
try:
|
||||
formal_result = run_formalization_pipeline(config)
|
||||
if formal_result.get("ok"):
|
||||
print(f"形式化批次: {formal_result.get('batch_id')}")
|
||||
print(f"形式化统计: {formal_result.get('stats', {})}")
|
||||
else:
|
||||
print(f"形式化流程跳过: {formal_result.get('reason', 'unknown')}")
|
||||
except Exception as e:
|
||||
print(f"形式化流程执行失败: {e}")
|
||||
raise
|
||||
|
||||
|
||||
print("=== 网络关系与效能评估(交互填参版) ===")
|
||||
|
||||
# 1. 加载默认配置
|
||||
cfg = copy.deepcopy(DEFAULT_CFG)
|
||||
|
||||
# 交互式逐项覆盖
|
||||
def ask_one_weight(prompt: str, default: float):
|
||||
"""只问一个权重,回车=default"""
|
||||
while True:
|
||||
try:
|
||||
v = input(f"{prompt} (默认={default}):").strip()
|
||||
return default if v == "" else float(v)
|
||||
except ValueError:
|
||||
print("请输入 0-1 之间的数字!")
|
||||
|
||||
def ask_one_int(prompt: str, default: int):
|
||||
"""问整数距离"""
|
||||
while True:
|
||||
try:
|
||||
v = input(f"{prompt} (默认={default} km):").strip()
|
||||
return default if v == "" else int(v)
|
||||
except ValueError:
|
||||
print("请输入整数!")
|
||||
|
||||
def ask_one_float(prompt: str, default: float):
|
||||
"""问时间衰减系数"""
|
||||
while True:
|
||||
try:
|
||||
v = input(f"{prompt} (默认={default}):").strip()
|
||||
return default if v == "" else float(v)
|
||||
except ValueError:
|
||||
print("请输入数字!")
|
||||
|
||||
# print("\n========== ① 关系权重 (0-1) ==========")
|
||||
# rels = ["IS", "CC", "SF", "PD", "CO"]
|
||||
# rel_keys = ["w_f", "w_s", "w_t", "w_p", "w_i"]
|
||||
# rel_names = ["功能(w_f)", "空间(w_s)", "时间(w_t)", "性能(w_p)", "交互(w_i)"]
|
||||
# for rel in ["IS", "CC", "SF", "PD", "CO"]:
|
||||
# print(f"\n-- {rel} 关系 --w_f/w_s/w_t/w_p/w_i")
|
||||
# for k, name in zip(rel_keys, rel_names):
|
||||
# old = cfg["W_REL"][rel][k]
|
||||
# cfg["W_REL"][rel][k] = ask_one_weight(name, old)
|
||||
#
|
||||
# print("\n========== ② 网络权重 (0-1) ==========")
|
||||
# tasks = ["defense", "fire", "logistics", "medical", "evacuation"]
|
||||
# tfactor = ["w1", "w2", "w3", "w4", "w5"]
|
||||
# for task in tasks:
|
||||
# print(f"\n-- {task} -- w1/w2/w3/w4/w5")
|
||||
# for i, tf in enumerate(tfactor):
|
||||
# old = cfg["W_NET"][task][tf]
|
||||
# cfg["W_NET"][task][tf] = ask_one_weight(tf, old)
|
||||
#
|
||||
# print("\n========== ③ 特征距离 LAMBDA (km) ==========")
|
||||
# for rel in rels:
|
||||
# old = cfg["LAMBDA"][rel]
|
||||
# cfg["LAMBDA"][rel] = ask_one_float(f"{rel} 距离", old)
|
||||
#
|
||||
# print("\n========== ④ 时间衰减系数 ==========")
|
||||
# cfg["TIME"]["alpha"] = ask_one_float("alpha", cfg["TIME"]["alpha"])
|
||||
# cfg["TIME"]["beta"] = ask_one_float("beta", cfg["TIME"]["beta"])
|
||||
# cfg["TIME"]["gamma"] = ask_one_float("gamma", cfg["TIME"]["gamma"])
|
||||
#
|
||||
# print("\n========== ⑤ 其他 ==========")
|
||||
# cfg["PD_SGM"] = ask_one_float("平台共位阈值 PD_SGM", cfg["PD_SGM"])
|
||||
# cfg["THRESHOLD"] = ask_one_weight("关系过滤阈值", cfg["THRESHOLD"])
|
||||
#
|
||||
|
||||
print("开始网络关系计算")
|
||||
print("=" * 50)
|
||||
|
||||
|
||||
try:
|
||||
# 1. 加载数据
|
||||
print("步骤1: 加载数据...")
|
||||
nodes_data, triples_data = load_data(project_root)
|
||||
print(f"加载节点数: {len(nodes_data['nodes'])}")
|
||||
print(f"加载三元组数: {len(triples_data['triples'])}")
|
||||
|
||||
# 2. 计算并过滤关系强度
|
||||
print(f"\n步骤2: 计算关系强度并过滤(阈值={cfg['THRESHOLD']})")
|
||||
filtered_relations, filtered_triples, relation_stats, all_relations = calculate_and_filter_relations(
|
||||
nodes_data, triples_data, cfg["THRESHOLD"], cfg
|
||||
)
|
||||
|
||||
# 3. 评估任务网络效能(使用过滤后的关系)
|
||||
print("\n步骤3: 使用过滤后的关系评估任务网络效能...")
|
||||
network_results = evaluate_task_networks(nodes_data, filtered_relations, cfg)
|
||||
|
||||
# 4. 生成评价报告
|
||||
print("\n步骤4: 生成评价报告...")
|
||||
report_content = generate_evaluation_report(
|
||||
relation_stats, network_results, filtered_relations, all_relations, cfg["THRESHOLD"],cfg
|
||||
)
|
||||
|
||||
# 5. 保存结果
|
||||
print("\n步骤5: 保存结果文件...")
|
||||
save_triples_and_task_probs(filtered_triples, network_results)
|
||||
save_filtered_triples(filtered_triples)
|
||||
save_evaluation_report(report_content)
|
||||
|
||||
# 6. 在控制台显示关键信息
|
||||
print("\n" + "=" * 50)
|
||||
print("关键结果摘要:")
|
||||
print(f"- 过滤后保留的三元组: {len(filtered_triples)} 个")
|
||||
print("所有计算完成!")
|
||||
print("=" * 50)
|
||||
|
||||
except Exception as e:
|
||||
print(f"程序执行出错: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
113
final/src/triplet_cleaner.py
Normal file
113
final/src/triplet_cleaner.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
三元组清洗模块
|
||||
"""
|
||||
from datetime import datetime
|
||||
from typing import List, Dict, Any, Set, Tuple
|
||||
|
||||
|
||||
def clean_triplets(
|
||||
triplets: List[Dict[str, Any]],
|
||||
valid_ids: Set[str] = None,
|
||||
deduplicate: bool = True,
|
||||
filter_orphans: bool = False,
|
||||
remove_null_core: bool = True,
|
||||
normalize_relation_type: bool = True,
|
||||
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
|
||||
"""
|
||||
清洗三元组数据
|
||||
|
||||
:param triplets: 原始三元组列表
|
||||
:param valid_ids: 有效实体 ID 集合(用于 filter_orphans),None 表示不校验
|
||||
:param deduplicate: 是否按 (HEAD_ID, RELATION_TYPE, TAIL_ID) 去重
|
||||
:param filter_orphans: 是否过滤悬空引用(HEAD_ID/TAIL_ID 不在 valid_ids 中)
|
||||
:param remove_null_core: 是否移除 HEAD_ID/TAIL_ID/RELATION_TYPE 为空的记录
|
||||
:param normalize_relation_type: 是否规范化 RELATION_TYPE(去空格)
|
||||
:return: (清洗后列表, 清洗报告)
|
||||
"""
|
||||
report = {
|
||||
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||
"original_count": len(triplets),
|
||||
"deduplicate": {"enabled": deduplicate, "removed": 0},
|
||||
"filter_orphans": {"enabled": filter_orphans, "removed": 0},
|
||||
"remove_null_core": {"enabled": remove_null_core, "removed": 0},
|
||||
"normalize_relation_type": {"enabled": normalize_relation_type},
|
||||
"final_count": 0,
|
||||
}
|
||||
|
||||
result = list(triplets)
|
||||
|
||||
def _head(t): return _str(t.get("HEAD_ID") or t.get("head"))
|
||||
def _tail(t): return _str(t.get("TAIL_ID") or t.get("tail"))
|
||||
def _rel(t): return _str(t.get("RELATION_TYPE") or t.get("relation"))
|
||||
|
||||
if remove_null_core:
|
||||
before = len(result)
|
||||
result = [t for t in result if _not_empty(_head(t)) and _not_empty(_tail(t)) and _not_empty(_rel(t))]
|
||||
report["remove_null_core"]["removed"] = before - len(result)
|
||||
|
||||
if normalize_relation_type:
|
||||
for t in result:
|
||||
k = "RELATION_TYPE" if "RELATION_TYPE" in t else "relation"
|
||||
if isinstance(t.get(k), str):
|
||||
t[k] = t[k].strip()
|
||||
|
||||
if filter_orphans and valid_ids is not None:
|
||||
before = len(result)
|
||||
result = [t for t in result if _head(t) in valid_ids and _tail(t) in valid_ids]
|
||||
report["filter_orphans"]["removed"] = before - len(result)
|
||||
|
||||
if deduplicate:
|
||||
before = len(result)
|
||||
seen = set()
|
||||
unique = []
|
||||
for t in result:
|
||||
key = (_head(t), _rel(t), _tail(t))
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique.append(t)
|
||||
result = unique
|
||||
report["deduplicate"]["removed"] = before - len(result)
|
||||
|
||||
report["final_count"] = len(result)
|
||||
return result, report
|
||||
|
||||
|
||||
def _not_empty(v: Any) -> bool:
|
||||
if v is None:
|
||||
return False
|
||||
if isinstance(v, str) and not v.strip():
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _str(v: Any) -> str:
|
||||
if v is None:
|
||||
return ""
|
||||
return str(v).strip()
|
||||
|
||||
|
||||
def transform_triplet_output_format(
|
||||
triplets: List[Dict[str, Any]],
|
||||
id_to_role: dict = None,
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
将三元组转为输出格式:
|
||||
- HEAD_ID/head -> head, TAIL_ID/tail -> tail, RELATION_TYPE/relation -> relation
|
||||
- 新增 head_type、tail_type(从 id_to_role 查 ROLE_ID,已有则保留)
|
||||
"""
|
||||
id_to_role = id_to_role or {}
|
||||
result = []
|
||||
for t in triplets:
|
||||
head = _str(t.get("HEAD_ID") or t.get("head"))
|
||||
tail = _str(t.get("TAIL_ID") or t.get("tail"))
|
||||
rel = _str(t.get("RELATION_TYPE") or t.get("relation"))
|
||||
head_type = _str(t.get("head_type")) or id_to_role.get(head, "")
|
||||
tail_type = _str(t.get("tail_type")) or id_to_role.get(tail, "")
|
||||
result.append({
|
||||
"head": head,
|
||||
"head_type": head_type,
|
||||
"relation": rel,
|
||||
"tail": tail,
|
||||
"tail_type": tail_type,
|
||||
})
|
||||
return result
|
||||
Reference in New Issue
Block a user