Compare commits

...

4 Commits

Author SHA1 Message Date
6135f17ca2 补充体系表结构与模块重构规范
加入负责人提供的达梦建表脚本和模块重构说明,统一后续 1.3 形式化模块对正式体系版本表的理解与落库目标。

Made-with: Cursor
2026-04-24 17:36:43 +08:00
92b856b622 新增修改 2026-04-23 12:48:59 +08:00
df46812eff 读取DM数据库 2026-02-27 17:06:56 +08:00
0c0a295f09 修复README,增加说明 2026-02-27 15:43:30 +08:00
60 changed files with 19323 additions and 230 deletions

18
.vscode/c_cpp_properties.json vendored Normal file
View File

@@ -0,0 +1,18 @@
{
"configurations": [
{
"name": "linux-gcc-x64",
"includePath": [
"${workspaceFolder}/**"
],
"compilerPath": "/usr/bin/gcc",
"cStandard": "${default}",
"cppStandard": "${default}",
"intelliSenseMode": "linux-gcc-x64",
"compilerArgs": [
""
]
}
],
"version": 4
}

24
.vscode/launch.json vendored Normal file
View File

@@ -0,0 +1,24 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "C/C++ Runner: Debug Session",
"type": "cppdbg",
"request": "launch",
"args": [],
"stopAtEntry": false,
"externalConsole": false,
"cwd": "/home/huangfukk/module1_3/.vscode",
"program": "/home/huangfukk/module1_3/.vscode/build/Debug/outDebug",
"MIMode": "gdb",
"miDebuggerPath": "gdb",
"setupCommands": [
{
"description": "Enable pretty-printing for gdb",
"text": "-enable-pretty-printing",
"ignoreFailures": true
}
]
}
]
}

59
.vscode/settings.json vendored Normal file
View File

@@ -0,0 +1,59 @@
{
"C_Cpp_Runner.cCompilerPath": "gcc",
"C_Cpp_Runner.cppCompilerPath": "g++",
"C_Cpp_Runner.debuggerPath": "gdb",
"C_Cpp_Runner.cStandard": "",
"C_Cpp_Runner.cppStandard": "",
"C_Cpp_Runner.msvcBatchPath": "",
"C_Cpp_Runner.useMsvc": false,
"C_Cpp_Runner.warnings": [
"-Wall",
"-Wextra",
"-Wpedantic",
"-Wshadow",
"-Wformat=2",
"-Wcast-align",
"-Wconversion",
"-Wsign-conversion",
"-Wnull-dereference"
],
"C_Cpp_Runner.msvcWarnings": [
"/W4",
"/permissive-",
"/w14242",
"/w14287",
"/w14296",
"/w14311",
"/w14826",
"/w44062",
"/w44242",
"/w14905",
"/w14906",
"/w14263",
"/w44265",
"/w14928"
],
"C_Cpp_Runner.enableWarnings": true,
"C_Cpp_Runner.warningsAsError": false,
"C_Cpp_Runner.compilerArgs": [],
"C_Cpp_Runner.linkerArgs": [],
"C_Cpp_Runner.includePaths": [],
"C_Cpp_Runner.includeSearch": [
"*",
"**/*"
],
"C_Cpp_Runner.excludeSearch": [
"**/build",
"**/build/**",
"**/.*",
"**/.*/**",
"**/.vscode",
"**/.vscode/**"
],
"C_Cpp_Runner.useAddressSanitizer": false,
"C_Cpp_Runner.useUndefinedSanitizer": false,
"C_Cpp_Runner.useLeakSanitizer": false,
"C_Cpp_Runner.showCompilationTime": false,
"C_Cpp_Runner.useLinkTimeOptimization": false,
"C_Cpp_Runner.msvcSecureNoWarnings": false
}

248
CreatTable.sql Normal file
View File

@@ -0,0 +1,248 @@
-- ==============================================================================
-- 体系推演底层数据架构 (支持多租户、三维度索引、三元组、全维属性约束)
-- 兼容: 达梦数据库 (Dameng)
-- ==============================================================================
-- ---------------------------------------------------------
-- 第一步:创建上下文环境变量 (达梦标准语法:必须绑定包)
-- ---------------------------------------------------------
-- 1.1 创建管理包头
CREATE OR REPLACE PACKAGE PKG_SYSTEM_CTX AS
PROCEDURE SET_USER_ID(P_ID VARCHAR(50));
END;
/
-- 1.2 创建管理包体
CREATE OR REPLACE PACKAGE BODY PKG_SYSTEM_CTX AS
PROCEDURE SET_USER_ID(P_ID VARCHAR(50)) AS
BEGIN
DBMS_SESSION.SET_CONTEXT('SYSTEM_CTX', 'USER_ID', P_ID);
END;
END;
/
-- 1.3 创建上下文对象 (注意:达梦不能加 OR REPLACE)
-- 如果提示“对象已存在”忽略即可,说明已经建好了
CREATE CONTEXT SYSTEM_CTX USING PKG_SYSTEM_CTX;
-- ---------------------------------------------------------
-- 第二步:创建基础属性字典表 (无外键依赖,最先创建)
-- ---------------------------------------------------------
-- 1. 实体详情属性表 (存储单兵级物理参数、图片与新闻)
CREATE TABLE ENTITY_ATTR_DETAIL (
attr_id VARCHAR(50) PRIMARY KEY,
longitude DECIMAL(15, 6), -- 经度
latitude DECIMAL(15, 6), -- 纬度
speed DECIMAL(10, 2), -- 速度
entity_value DECIMAL(10, 2), -- 价值
image_url VARCHAR(500), -- 图片 (路径或URL)
news_content CLOB, -- 新闻 (文本或HTML)
extra_json CLOB -- 备用扩展属性 (JSON)
);
-- 2. 体系整体属性表 (存储整个体系的评估属性)
CREATE TABLE SYSTEM_ATTR_DETAIL (
sys_attr_id VARCHAR(50) PRIMARY KEY,
effectiveness_score DECIMAL(10, 4), -- 体系效能
extra_props CLOB -- 其他体系属性 (JSON)
);
-- ---------------------------------------------------------
-- 第三步:创建核心体系索引与三元组数据表 (带有强约束)
-- ---------------------------------------------------------
-- 3. 体系数据版本总表 (所有数据的根索引:用户+战略任务+时间)
CREATE TABLE SYSTEM_VERSION_MASTER (
version_id VARCHAR(50) PRIMARY KEY,
user_id VARCHAR(50) NOT NULL, -- 索引维度1所属用户
strat_task_id VARCHAR(50) NOT NULL, -- 索引维度2战略任务ID
sys_date DATE NOT NULL, -- 索引维度3体系对应日期 (年月日)
save_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP, -- 上传保存时间戳
-- 体系类型强约束
sys_type VARCHAR(50) CHECK (sys_type IN (
'常规体系', '关系推理后的体系', '形式化后的体系', '历史体系',
'当前体系', '预测体系', '指控体系', '重构体系'
)),
-- 体系任务强约束
system_task VARCHAR(50) CHECK (system_task IN (
'综合防御任务', '火力打击任务', '后勤保障任务', '医疗救援任务', '紧急疏散任务'
)),
-- 国家/地区约束
country_region VARCHAR(10) CHECK (country_region IN ('', '', '', '', '')),
sys_attr_id VARCHAR(50) REFERENCES SYSTEM_ATTR_DETAIL(sys_attr_id)
);
-- 建立三维联合索引,极大提升查询检索效率
CREATE INDEX idx_sys_version_lookup ON SYSTEM_VERSION_MASTER(user_id, strat_task_id, sys_date);
-- 4. 实体节点表 (三元组之头/尾节点)
CREATE TABLE SYSTEM_ENTITY (
entity_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
entity_name VARCHAR(100) NOT NULL,
-- 实体角色强约束 (7种)
entity_role VARCHAR(20) CHECK (entity_role IN (
'信息获取', '信息传输', '指挥控制', '信息对抗', '协同打击', '部署平台', '综合保障'
)),
-- 实体国家/地区强约束
entity_country VARCHAR(10) CHECK (entity_country IN ('', '', '', '', '')),
-- 实体可能执行的任务
entity_task VARCHAR(50) CHECK (entity_task IN (
'综合防御任务', '火力打击任务', '后勤保障任务', '医疗救援任务', '紧急疏散任务'
)),
-- 指控类型 (OODA仅在指控体系中有效允许为空)
c2_type VARCHAR(10) CHECK (c2_type IN ('Observe', 'Orient', 'Decide', 'Act', NULL)),
-- 实体可能存在的关系 (存贮格式由后端决定,如 "情报保障,指挥控制")
possible_relations VARCHAR(255),
attr_id VARCHAR(50) REFERENCES ENTITY_ATTR_DETAIL(attr_id)
);
-- 5. 实体连线关系表 (三元组之边)
CREATE TABLE SYSTEM_RELATION (
rel_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
head_entity_id VARCHAR(50) REFERENCES SYSTEM_ENTITY(entity_id),
tail_entity_id VARCHAR(50) REFERENCES SYSTEM_ENTITY(entity_id),
-- 关系类型强约束 (5种选其一)
rel_type VARCHAR(20) CHECK (rel_type IN (
'情报保障', '指挥控制', '状态反馈', '平台部署', '协同作战'
))
);
-- ---------------------------------------------------------
-- 第四步:创建 9 大体系计算分析结果表 (全指向 version_id)
-- ---------------------------------------------------------
CREATE TABLE RES_TARGET_RECOGNITION (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
entity_id VARCHAR(50) REFERENCES SYSTEM_ENTITY(entity_id),
confidence_score DECIMAL(5, 4),
metrics_json CLOB
);
CREATE TABLE RES_NODE_VALUE (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
entity_id VARCHAR(50) REFERENCES SYSTEM_ENTITY(entity_id),
value_result_json CLOB
);
CREATE TABLE RES_REL_REASONING (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
reasoning_data CLOB
);
CREATE TABLE RES_FORMAL_ANALYSIS (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
formal_data CLOB
);
CREATE TABLE RES_SYS_ANALYSIS (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
analysis_data CLOB
);
CREATE TABLE RES_SYS_EVO_PREDICT (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
predict_data CLOB
);
CREATE TABLE RES_SYS_EFFECTIVENESS (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
effectiveness_data CLOB
);
CREATE TABLE RES_LINK_COG (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
link_cog_data CLOB
);
CREATE TABLE RES_NODE_COG (
res_id VARCHAR(50) PRIMARY KEY,
version_id VARCHAR(50) REFERENCES SYSTEM_VERSION_MASTER(version_id) ON DELETE CASCADE,
node_cog_data CLOB
);
-- ---------------------------------------------------------
-- 第五步:当前体系专用覆盖表
-- ---------------------------------------------------------
-- 用于单独保存“当前体系”的快照,逻辑上同用户、同任务只有唯一一份
CREATE TABLE CURRENT_SYSTEM_SNAPSHOT (
snapshot_id VARCHAR(50) PRIMARY KEY,
user_id VARCHAR(50) NOT NULL,
strat_task_id VARCHAR(50) NOT NULL,
sys_date DATE NOT NULL,
save_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
full_system_json CLOB,
CONSTRAINT uq_current_sys UNIQUE(user_id, strat_task_id)
);
-- ---------------------------------------------------------
-- 第六步:建立上下文安全视图 (极重要:实现后端模块免传 user_id)
-- ---------------------------------------------------------
-- 主表与三元组视图
CREATE OR REPLACE VIEW V_MY_SYSTEM_VERSION AS
SELECT * FROM SYSTEM_VERSION_MASTER
WHERE user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_SYSTEM_ENTITY AS
SELECT e.* FROM SYSTEM_ENTITY e
JOIN SYSTEM_VERSION_MASTER v ON e.version_id = v.version_id
WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_SYSTEM_RELATION AS
SELECT r.* FROM SYSTEM_RELATION r
JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id
WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
-- 9 大分析结果视图
CREATE OR REPLACE VIEW V_MY_RES_TARGET_RECOGNITION AS
SELECT r.* FROM RES_TARGET_RECOGNITION r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_RES_NODE_VALUE AS
SELECT r.* FROM RES_NODE_VALUE r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_RES_REL_REASONING AS
SELECT r.* FROM RES_REL_REASONING r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_RES_FORMAL_ANALYSIS AS
SELECT r.* FROM RES_FORMAL_ANALYSIS r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_RES_SYS_ANALYSIS AS
SELECT r.* FROM RES_SYS_ANALYSIS r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_RES_SYS_EVO_PREDICT AS
SELECT r.* FROM RES_SYS_EVO_PREDICT r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_RES_SYS_EFFECTIVENESS AS
SELECT r.* FROM RES_SYS_EFFECTIVENESS r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_RES_LINK_COG AS
SELECT r.* FROM RES_LINK_COG r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');
CREATE OR REPLACE VIEW V_MY_RES_NODE_COG AS
SELECT r.* FROM RES_NODE_COG r JOIN SYSTEM_VERSION_MASTER v ON r.version_id = v.version_id WHERE v.user_id = SYS_CONTEXT('SYSTEM_CTX', 'USER_ID');

View File

@@ -1,64 +0,0 @@
# 作战体系数据清洗与标准化系统
## 项目简介
本项目用于处理异质作战体系节点的原始JSON数据。系统能够自动识别并修复数据中的噪声、缺失值、异常值并将关键效能指标标准化至 [0, 1] 区间,最终输出符合建模要求的高质量数据集。
## 目录结构
```
project/
├── data/
│ ├── raw_data.json # 原始输入数据(包含噪声和错误)
│ └── cleaned_data.json # 清洗后的输出数据
├── src/
│ └── cleaner.py # 清洗运行程序
├── report/
│ └── cleaning_report.json # 详细清洗报告
└── README.md # 项目说明文档
```
## 功能特性
* **去噪处理**:使用卡尔曼滤波算法平滑连续数值(如通信范围),消除测量噪声。
* **异常检测与修复**
* 自动修正负数、极端值。
* 强制将效能指标(如打击精度、机动性)限制在 [0, 1] 范围内。
* 修复无效的地理坐标。
* **缺失值填充**基于同类型Role单位的统计均值进行智能插值。
* **标准化**:统一文本格式、时间格式及数值精度。
## 快速开始
### 1. 环境依赖
本项目仅依赖 Python 标准库及 NumPy/Pandas
```bash
pip install numpy pandas
```
### 2. 运行清洗
直接运行主程序即可:
```bash
python src/main.py
```
程序默认读取 `data/raw_data.json`,处理后生成 `data/cleaned_data.json``report/cleaning_report.json`
## 输出结果示例
**清洗前 (Raw):**
```json
{
"MOBILITY": -0.5,
"STRIKE_ACCURACY": 1.5,
"COMMUNICATION_RANGE": 102.8116 // 含噪声
}
```
**清洗后 (Cleaned):**
```json
{
"MOBILITY": 0.5,
"STRIKE_ACCURACY": 1.0,
"COMMUNICATION_RANGE": 102.81 // 平滑后
}
```

4
final/.env.example Normal file
View File

@@ -0,0 +1,4 @@
# 达梦数据库密码(仅 source=db 时需要)
# 复制为 .env 后填入真实密码,或将下面一行加入 ~/.bashrc 等
# export DM_PASSWORD=你的密码
DM_PASSWORD=Dm508508

3
final/.idea/.gitignore generated vendored Normal file
View File

@@ -0,0 +1,3 @@
# Default ignored files
/shelf/
/workspace.xml

12
final/.idea/227.iml generated Normal file
View File

@@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="PYTHON_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="jdk" jdkName="D:\anaconda\envs\py38" jdkType="Python SDK" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
<component name="PyDocumentationSettings">
<option name="format" value="PLAIN" />
<option name="myDocStringFormat" value="Plain" />
</component>
</module>

View File

@@ -0,0 +1,38 @@
<component name="InspectionProjectProfileManager">
<profile version="1.0">
<option name="myName" value="Project Default" />
<inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredPackages">
<value>
<list size="8">
<item index="0" class="java.lang.String" itemvalue="scikit-image" />
<item index="1" class="java.lang.String" itemvalue="scipy" />
<item index="2" class="java.lang.String" itemvalue="tensorflow_gpu" />
<item index="3" class="java.lang.String" itemvalue="scikit_learn" />
<item index="4" class="java.lang.String" itemvalue="matplotlib" />
<item index="5" class="java.lang.String" itemvalue="numpy" />
<item index="6" class="java.lang.String" itemvalue="opencv_python" />
<item index="7" class="java.lang.String" itemvalue="Pillow" />
</list>
</value>
</option>
</inspection_tool>
<inspection_tool class="PyPep8NamingInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
<option name="ignoredErrors">
<list>
<option value="N806" />
<option value="N802" />
<option value="N803" />
<option value="N801" />
</list>
</option>
</inspection_tool>
<inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
<option name="ignoredIdentifiers">
<list>
<option value="int.__getitem__" />
</list>
</option>
</inspection_tool>
</profile>
</component>

View File

@@ -0,0 +1,6 @@
<component name="InspectionProjectProfileManager">
<settings>
<option name="USE_PROJECT_PROFILE" value="false" />
<version value="1.0" />
</settings>
</component>

4
final/.idea/misc.xml generated Normal file
View File

@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectRootManager" version="2" project-jdk-name="D:\anaconda\envs\py38" project-jdk-type="Python SDK" />
</project>

8
final/.idea/modules.xml generated Normal file
View File

@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectModuleManager">
<modules>
<module fileurl="file://$PROJECT_DIR$/.idea/227.iml" filepath="$PROJECT_DIR$/.idea/227.iml" />
</modules>
</component>
</project>

Binary file not shown.

331
final/README.md Normal file
View File

@@ -0,0 +1,331 @@
# 作战体系数据清洗与网络效能评估系统
## 项目简介
本系统包含两大功能模块:
1. **数据清洗**:从 JSON 文件或达梦数据库读取实体属性表、三元组,进行去重、缺失值填充、异常值修正、噪声平滑等清洗;并按 **1125 属性需求****内容与格式的双重对齐**(格式校验 + 内容校验 + 原始字段→规范 schema 转换),**输出仅含 xlsx 定义字段**的 `nodes.json`,不保留原始其他字段。
2. **网络关系与效能评估**:基于节点与三元组数据,计算关系强度、过滤低置信边,评估任务网络效能,生成评价报告。
---
## 目录结构
```
final/
├── config/
│ ├── config.yaml # 主配置文件(推荐)
│ └── config.json # 备选配置
├── data/
│ ├── raw_data_sample.json # 原始实体数据JSON 模式)
│ ├── nodes.json # 清洗后实体(关系/网络计算用)
│ └── triples.json # 清洗后三元组(来自 run_all
├── report/
│ ├── detailed_cleaning_report.json # 实体清洗报告
│ └── triplet_cleaning_report.json # 三元组清洗报告
├── results/
│ ├── filtered_triples.json # 过滤后三元组
│ ├── triples.csv # 三元组表格
│ └── evaluation_report.txt # 效能评价报告
├── src/
│ ├── main_227.py # 主程序(清洗 + 关系 + 网络评估)
│ ├── cleaner.py # 实体清洗
│ ├── attribute_schema.py # 1125 属性规范:格式/内容校验 + 原始→规范转换
│ ├── data_loader.py # 数据加载JSON/达梦)
│ └── triplet_cleaner.py # 三元组清洗
├── scripts/
│ ├── run_all.py # 一键运行(仅清洗)
│ ├── export_triplets.py # 单独导出三元组
│ ├── inspect_dm_tables.py # 达梦表结构查看
│ ├── f_relation1125.py # 关系强度计算
│ └── f_net1125.py # 任务网络评估
├── .env.example
└── README.md
```
---
## 快速开始
### 1. 环境依赖
```bash
pip install -r requirements.txt
# 达梦模式需额外pip install dmPython
```
### 2. 运行方式
| 命令 | 功能 |
|------|------|
| `python src/main_227.py` | **完整流程**:实体清洗 → 关系强度计算 → 网络效能评估 |
| `python scripts/run_all.py` | **仅清洗**:实体 + 三元组 读取、清洗、导出 |
### 3. 开发模式(无达梦数据库)
`config/config.yaml``config/config.json` 中设置:
```yaml
source: json
```
然后运行:
```bash
python src/main_227.py
```
程序会从 `data/raw_data_sample.json` 读取实体数据并清洗;关系/网络计算则从 `data/nodes.json``data/triples.json` 读取。
---
## 配置说明
### 必须修改的配置
根据你的环境,在 `config/config.yaml``config/config.json` 中修改以下项:
| 配置项 | 说明 | 示例 |
|--------|------|------|
| `source` | 数据源:`json`(开发)或 `db`(达梦) | `json` |
| `db.host` | 达梦库地址source=db 时) | `127.0.0.1` |
| `db.port` | 达梦端口 | `5080` |
| `db.user` | 达梦用户 | `SYSDBA` |
| `db.table` | 实体表名 | `OBJECTIVE_INSTANCE` |
| `db.triplet_table` | 三元组表名 | `RELATION_INSTANCE` |
### 达梦密码(切勿写入配置)
达梦模式需设置环境变量:
```bash
export DM_PASSWORD=你的密码
```
或在 PyCharmRun → Edit Configurations → Environment variables → 添加 `DM_PASSWORD=你的密码`
### 可选配置
| 配置项 | 说明 | 默认 |
|--------|------|------|
| `json.input_file` | 实体 JSON 路径 | `data/raw_data_sample.json` |
| `json.output_file` | 清洗后实体输出 | `data/nodes.json` |
| `triplet.enabled` | 是否处理三元组 | `true` |
| `triplet.input_file` | JSON 模式下三元组路径 | `null` |
| `triplet.output_file` | 三元组输出 | `data/triples.json` |
| `triplet.clean.deduplicate` | 三元组去重 | `true` |
| `triplet.clean.filter_orphans` | 过滤悬空引用 | `false` |
| `eval.nodes_file` | main_227 读取的节点文件 | `data/nodes.json` |
| `eval.triples_file` | main_227 读取的三元组文件 | `data/triples.json`(与 triplet.output_file 一致) |
### 配置优先级
`config.json` 优先于 `config.yaml`;修改其一即可。
---
## 数据流程
### main_227.py 流程
```
1. 实体清洗
- 从 JSON 或达梦读 raw_data_sample / OBJECTIVE_INSTANCE
- 去重、缺失值填充、异常值修正、噪声平滑
- 输出 nodes.json
2. 关系与网络评估
- 按配置读取 nodeseval.nodes_file / json.output_file、tripleseval.triples_file = triplet.output_file
- 计算 5 类关系强度(情报保障、指挥控制、状态反馈、平台部署、协同作战)
- 按阈值过滤关系
- 评估 5 类任务网络效能
- 输出 results/ 下报告与表格
```
### run_all.py 流程
```
1. 实体清洗 → data/nodes.json
2. 三元组清洗 → data/triples.json格式head, head_type, relation, tail, tail_type
```
---
## 输出文件说明
| 文件 | 来源 | 说明 |
|------|------|------|
| `data/nodes.json` | 清洗 | 清洗后实体(`{nodes: {id: obj}, global_params: {mtbf_max}}`,见下文) |
| `data/triples.json` | run_all | 清洗后三元组head/head_type/relation/tail/tail_type |
| `report/detailed_cleaning_report.json` | 清洗 | 实体清洗统计(含 attribute_alignment格式/内容对齐情况) |
| `report/triplet_cleaning_report.json` | run_all | 三元组清洗统计 |
| `results/filtered_triples.json` | main_227 | 过滤后三元组(含强度) |
| `results/triples.csv` | main_227 | 三元组表格 |
| `results/evaluation_report.txt` | main_227 | 效能评价报告 |
---
## 常见问题
**Q: 运行报错「找不到文件 raw_data_sample.json」**
A: 将 `source` 改为 `json`,并确保 `data/raw_data_sample.json` 存在;或检查 `json.input_file` 路径。
**Q: 达梦模式连接失败?**
A: 检查 `DM_PASSWORD` 环境变量、`db.host`/`db.port`,以及 dmPython 与达梦客户端是否正确安装。
**Q: main_227 中 nodes.json、triples.json 从哪来?**
A: `nodes.json``triples.json` 均由清洗步骤生成run_all 或 main_227 前半段),格式需包含 `head``relation``tail` 等字段。
**Q: 如何查看达梦表结构?**
A: 运行 `python scripts/inspect_dm_tables.py`(需设置 `DM_PASSWORD`)。
**Q: nodes.json 为何没有 X、Y、ROLE_ID 等原始字段?**
A: 输出完全对齐 1125 xlsx 规范,仅保留 schema 定义的 12 个字段;原始字段在转换阶段已映射为 `spatial``org_unit` 等规范结构。
---
## 数据清洗详细逻辑
### 清洗流程概览
```
原始数据 → 去重 → 缺失值填充 → 异常值修正 → 噪声平滑 → 1125 Schema 转换 → 关系计算前净化 → nodes.json
```
| 步骤 | 说明 |
|------|------|
| **去重** | 按 `TARGET_ID` 去重,优先保留 `CREATED_TIME` 最新的记录 |
| **缺失值填充** | 数值型字段按 `ROLE_ID` 分组均值填充,兜底用全局均值 |
| **异常值修正** | 对 `TARGET_RECOGNITION_CAPABILITY``STRIKE_ACCURACY``ANTI_JAMMING_CAPABILITY``ENVIRONMENT_ADAPTABILITY``MOBILITY` 等 [0,1] 字段:负值取绝对值、>1 截断为 1 |
| **噪声平滑** | 对 `COMMUNICATION_RANGE` 做 3 点移动平均 |
| **1125 Schema 转换** | 见下文字段映射,输出仅含规范字段 |
| **关系计算前净化** | 见下文「关系计算前净化」 |
### 关系计算前净化sanitize_node_for_relation_calc
下游 `f_relation1125``utils.py` 会对节点属性做数值运算(如欧氏距离、安全等级差、历史成功率平均),若字段为 `None` 会触发 `NoneType` 运算错误。因此在写入 `nodes.json` 前,对每条节点做最终净化,确保参与运算的字段**永不为 `None`**
| 净化项 | 处理方式 |
|--------|----------|
| `spatial.position` | 必须为 `[float, float]`,含 `None`/`NaN` 的元素替换为 `0.0` |
| `spatial.effective_radius` | 若为 `None`,设为 `1.0` |
| `temporal.*` | `time_window` 缺省 `24.0`,其余缺省 `1.0` |
| `security_level` | 若为 `None` 或越界,设为 `3`15 范围) |
| `history_success` | 若为 `None`,设为 `0.8`01 范围) |
该步骤在 `attribute_schema.sanitize_node_for_relation_calc()` 中实现,由 `cleaner` 在写入前对每个节点调用。
### 输出字段说明
**nodes.json 完全对齐 xlsx 规范**,不保留原始其他字段。每条记录仅包含:
| 字段 | 类型 | 说明 |
|------|------|------|
| `TARGET_ID` | 字符串 | 节点唯一标识(三元组 head/tail 引用所需) |
| `function_vector` | 字典 | 7 维功能能力 [0,1] |
| `spatial` | 字典 | 空间位置与有效半径 |
| `temporal` | 字典 | 时间相关属性 |
| `performance` | 字典 | 核心性能、生存能力、MTBF |
| `protocol_list` | 列表 | 支持协议(字符串列表) |
| `format_list` | 列表 | 支持格式(字符串列表) |
| `interface_list` | 列表 | 支持接口(字符串列表) |
| `security_level` | 整数 | 安全等级 [1,5],缺省 3关系计算用 |
| `org_unit` | 字符串/null | 组织隶属 |
| `nation` | 字符串/null | 国家代码 |
| `history_success` | 浮点数 | 历史成功率 [0,1],缺省 0.8(关系计算用) |
**nodes.json 整体结构**`{ "nodes": { "TARGET_ID": {...}, ... }, "global_params": { "mtbf_max": number } }`,兼容 main_227 / f_relation1125 关系计算。
**单条节点记录示例:**
```json
{
"TARGET_ID": "FS-俄-情报侦-132",
"function_vector": {
"f_IC": 0.9273, "f_IA": 0.7896, "f_CS": 0.2584, "f_IT": 0.5187,
"f_DP": 0.9148, "f_CPS": 0.5619, "f_CC": 0.7822
},
"spatial": { "position": [79.36, 52.11], "effective_radius": 94.96 },
"temporal": { "response_time": 23.3891, "cycle_time": 3.7, "data_age": 1.48, "time_window": 3.7 },
"performance": { "core_performance": 0.9831, "survivability": 0.4094, "mtbf": 0.7202 },
"protocol_list": [], "format_list": [], "interface_list": [],
"security_level": 3, "org_unit": "师级", "nation": "俄", "history_success": 0.8
}
```
---
## 属性对齐与字段转换1125 规范)
依据 `1125暂时属性需求(1).xlsx`,对每条记录做 **内容与格式的双重对齐**
- **格式校验**各字段类型与结构dict/list/scalar
- **内容校验**:取值范围(如 [0,1]、[1,5]
- **转换**:原始扁平字段 → 规范 schema
清洗报告中 `details.attribute_alignment` 记录格式/内容错误统计及示例。
### 原始字段 → 目标 Schema 映射
#### function_vector功能向量
每个维度取 [0,1] 浮点数,从原始能力字段推导:
| 目标维度 | 原始字段(按优先级) | 转换方式 |
|----------|----------------------|----------|
| f_IC 情报收集 | DETECTION_ACCURACY, TARGET_RECOGNITION_CAPABILITY | 取第一个有效值clamp 到 [0,1] |
| f_IA 信息分析 | INFORMATION_FUSION_CAPABILITY, DETECTION_ACCURACY | 同上 |
| f_CS 协同作战 | MOBILITY, ENVIRONMENT_ADAPTABILITY | 同上 |
| f_IT 信息传输 | ANTI_JAMMING_CAPABILITY | 同上 |
| f_DP 数据处理 | PROCESSING_CAPACITY, INFORMATION_FUSION_CAPABILITY | **固定参考值**:见下表 |
| f_CPS 指挥控制 | DECISION_RESPONSE_TIME | **固定参考值**:见下表 |
| f_CC 综合保障 | SUPPORT_CAPACITY, LOAD_CAPACITY | **固定参考值**:见下表 |
**固定参考值转换公式**(原始值不在 [0,1] 时使用):
| 维度 | 公式 | 参数说明 |
|------|------|----------|
| **f_CPS** | `1 / (1 + t/30)` | t = 响应时间(秒),响应越快能力越高 |
| **f_DP** | `log10(1+PC) / log10(10001)` | PC = PROCESSING_CAPACITY无则用 INFORMATION_FUSION_CAPABILITY clamp [0,1] |
| **f_CC** | `0.6×support_norm + 0.4×load_norm` | support_norm = log10(1+SUPPORT)/log10(10001)load_norm = LOAD/100仅一个有时取该值 |
#### spatial空间属性
| 目标字段 | 原始字段(按优先级) | 转换 |
|----------|----------------------|------|
| position | `position`(列表 [x,y])、`X`/`x`+`Y`/`y``POSITION_X`/`POSITION_Y``LONGITUDE`/`LATITUDE` | `[round(x,2), round(y,2)]``None`/`NaN` 替为 `0.0`,兼容达梦多种列名 |
| effective_radius | COMMUNICATION_RANGE | 原值保留 4 位小数,缺省 `1.0` |
#### temporal时间属性
| 目标字段 | 原始字段 | 转换 |
|----------|----------|------|
| response_time | DECISION_RESPONSE_TIME | 原值 |
| cycle_time | REFRESH_RATE | 原值 |
| data_age | TRANSMISSION_DELAY | 原值(近似数据新鲜度) |
| time_window | REFRESH_RATE | 原值 |
#### performance性能属性
| 目标字段 | 原始字段(按优先级) | 转换 |
|----------|----------------------|------|
| core_performance | STRIKE_ACCURACY, DETECTION_ACCURACY, MOBILITY | 取第一个有效值clamp [0,1] |
| survivability | ENVIRONMENT_ADAPTABILITY, ANTI_JAMMING_CAPABILITY | 同上 |
| mtbf | RELIABILITY, SYSTEM_RELIABILITY | 取第一个有效值 |
#### 其他标量字段
| 目标字段 | 原始字段 | 转换 |
|----------|----------|------|
| protocol_list | PROTOCOL | 字符串→列表,空则 `[]` |
| format_list | FORMAT | 同上 |
| interface_list | INTERFACE | 同上 |
| security_level | security_level, SECURITY_LEVEL | 整数 [1,5],缺省 `3`(避免关系计算时 `None` 运算错误) |
| org_unit | LEVEL, ROLE_ID | 取第一个非空字符串 |
| nation | COUNTRY_REGION | 原值 |
| history_success | history_success, HISTORY_SUCCESS | clamp [0,1],缺省 `0.8`(同上) |
### 缺失与空值
- 无可映射原始值时,多数目标字段为 `null`(列表型为 `[]`
- `security_level``history_success``spatial.position` 等参与关系运算的字段**永不为 `null`**:缺省分别为 `3``0.8``[0,0]`,并在「关系计算前净化」阶段再次保证
- 可选字段(如 `org_unit``nation`)缺失不影响记录通过校验

86
final/config/config.json Normal file
View File

@@ -0,0 +1,86 @@
{
"source": "json",
"json": {
"input_file": "data/raw_data_sample.json",
"output_file": "data/nodes.json",
"report_file": "report/detailed_cleaning_report.json"
},
"triplet": {
"enabled": true,
"input_file": null,
"output_file": "data/triples.json",
"report_file": "report/triplet_cleaning_report.json",
"clean": {
"deduplicate": true,
"remove_null_core": true,
"normalize_relation_type": true,
"filter_orphans": false
}
},
"db": {
"host": "127.0.0.1",
"port": 5080,
"user": "SYSDBA",
"schema": "SYSDBA",
"table": "OBJECTIVE_INSTANCE",
"query": null,
"triplet_table": "RELATION_INSTANCE",
"triplet_query": null
},
"formal": {
"enabled": false,
"user_id": "ADMIN",
"task_id": "TASK_FULL_KB",
"source_sys_type": "关系推理后的体系",
"target_sys_type": "形式化后的体系",
"source_query": null,
"user_column": "USER_ID",
"task_column": "TASK_ID",
"sys_type_column": "SYS_TYPE",
"skip_sys_type_validation": false,
"entity_id_column": "TARGET_ID",
"objective_table": "OBJECTIVE_INSTANCE",
"triplet_table": "RELATION_INSTANCE",
"triplet_primary_key": "RELATION_INSTANCE_ID",
"triplet_omit_primary_key_on_insert": true,
"triple_batch_column": "BATCH_ID",
"attribute_table": "METRIC_VALUE_INSTANCE",
"metric_id_column": "METRIC_ID",
"metric_value_column": "METRIC_VALUE",
"metric_insert_static": null,
"formal_result_table": "RELATION_INSTANCE",
"analysis_table": "RES_FORMAL_ANALYSIS",
"analysis_insert_columns": null,
"analysis_static_columns": null,
"rules_summary": null,
"required_attrs": [
"f_IC",
"f_IA",
"f_CS",
"f_IT",
"f_DP",
"f_CPS",
"f_CC",
"position",
"effective_radius",
"response_time",
"cycle_time",
"data_age",
"time_window",
"core_performance",
"survivability",
"mtbf",
"protocol_list",
"format_list",
"interface_list",
"security_level",
"org_unit",
"nation",
"history_success"
]
},
"eval": {
"nodes_file": "data/nodes.json",
"triples_file": "data/triples.json"
}
}

96
final/config/config.yaml Normal file
View File

@@ -0,0 +1,96 @@
# 数据清洗系统配置文件
# 通过 source 切换json开发 / db生产-达梦)
source: json # 开发用 json部署达梦时改为 db或直接改 config.json
# JSON 文件模式source=json 时生效)
json:
input_file: data/raw_data_sample.json
output_file: data/nodes.json
report_file: report/detailed_cleaning_report.json
# 网络关系/效能评估读取路径main_227 在清洗后加载)
eval:
nodes_file: data/nodes.json # 与 json.output_file 一致(清洗输出)
triples_file: data/triples.json # 与 triplet.output_file 一致run_all 输出)
# 三元组run_all 时一并处理db 模式从库读json 模式可配 input_file
triplet:
enabled: true # false 可关闭三元组处理
input_file: null # json 模式下可选,如 data/triplets_raw.json
output_file: data/triples.json # 与 eval.triples_file 一致,便于 main_227 直接读取
report_file: report/triplet_cleaning_report.json
# 清洗配置clean: false 关闭清洗true 使用默认;或自定义子项
clean:
deduplicate: true # 按 (HEAD_ID, RELATION_TYPE, TAIL_ID) 去重
remove_null_core: true # 移除 HEAD_ID/TAIL_ID/RELATION_TYPE 为空的记录
normalize_relation_type: true # 规范化 RELATION_TYPE去空格
filter_orphans: false # 过滤悬空引用(需加载实体表,稍慢)
# 达梦数据库模式source=db 时生效)
# 密码通过环境变量 DM_PASSWORD 传入,切勿写入配置文件
db:
host: 127.0.0.1
port: 5080
user: SYSDBA
schema: SYSDBA
# OBJECTIVE_INSTANCE 表结构与 raw_data_sample.json 一致,直接查询即可
table: OBJECTIVE_INSTANCE
query: null
# 三元组表RELATION_INSTANCE: HEAD_ID, RELATION_TYPE, TAIL_ID
triplet_table: RELATION_INSTANCE
triplet_query: null # 可选:自定义 SQL
# 模块1.3 形式化入库流程source=db 时生效)
formal:
enabled: false
user_id: ADMIN
task_id: TASK_FULL_KB
source_sys_type: 关系推理后的体系
target_sys_type: 形式化后的体系
# 推荐:显式 SQL 只读「关系推理后的体系」,列名与库一致时最稳(为 null 则用下方列名拼 WHERE
source_query: null
user_column: USER_ID
task_column: TASK_ID
sys_type_column: SYS_TYPE
skip_sys_type_validation: false
entity_id_column: TARGET_ID
objective_table: OBJECTIVE_INSTANCE
triplet_table: RELATION_INSTANCE
triplet_primary_key: RELATION_INSTANCE_ID
triplet_omit_primary_key_on_insert: true
triple_batch_column: BATCH_ID
attribute_table: METRIC_VALUE_INSTANCE
metric_id_column: METRIC_ID
metric_value_column: METRIC_VALUE
metric_insert_static: null
formal_result_table: RELATION_INSTANCE
analysis_table: RES_FORMAL_ANALYSIS
# 若自动匹配列失败,在此列出 RES_FORMAL_ANALYSIS 的实际列名(顺序与 VALUES 一致)
analysis_insert_columns: null
analysis_static_columns: null
rules_summary: null
required_attrs:
- f_IC
- f_IA
- f_CS
- f_IT
- f_DP
- f_CPS
- f_CC
- position
- effective_radius
- response_time
- cycle_time
- data_age
- time_window
- core_performance
- survivability
- mtbf
- protocol_list
- format_list
- interface_list
- security_level
- org_unit
- nation
- history_success

File diff suppressed because it is too large Load Diff

7407
final/data/nodes.json Normal file

File diff suppressed because it is too large Load Diff

60
final/data/triples.json Normal file
View File

@@ -0,0 +1,60 @@
{
"triples": [
{
"head": "CMD001",
"head_type": "指挥控制",
"relation": "指挥控制",
"tail": "DEF001",
"tail_type": "信息获取"
},
{
"head": "CMD001",
"head_type": "指挥控制",
"relation": "指挥控制",
"tail": "STK001",
"tail_type": "协同打击"
},
{
"head": "DEF001",
"head_type": "信息获取",
"relation": "状态反馈",
"tail": "CMD001",
"tail_type": "指挥控制"
},
{
"head": "DEF001",
"head_type": "信息获取",
"relation": "情报保障",
"tail": "STK001",
"tail_type": "协同打击"
},
{
"head": "STK001",
"head_type": "协同打击",
"relation": "状态反馈",
"tail": "CMD001",
"tail_type": "指挥控制"
},
{
"head": "STK001",
"head_type": "协同打击",
"relation": "协同作战",
"tail": "STK002",
"tail_type": "协同打击"
},
{
"head": "LOG001",
"head_type": "综合保障",
"relation": "平台部署",
"tail": "STK001",
"tail_type": "协同打击"
},
{
"head": "MED001",
"head_type": "综合保障",
"relation": "平台部署",
"tail": "TRN001",
"tail_type": "部署平台"
}
]
}

58
final/data/triplets.json Normal file
View File

@@ -0,0 +1,58 @@
[
{
"head": "CMD001",
"head_type": "指挥控制",
"relation": "指挥控制",
"tail": "DEF001",
"tail_type": "信息获取"
},
{
"head": "CMD001",
"head_type": "指挥控制",
"relation": "指挥控制",
"tail": "STK001",
"tail_type": "协同打击"
},
{
"head": "DEF001",
"head_type": "信息获取",
"relation": "状态反馈",
"tail": "CMD001",
"tail_type": "指挥控制"
},
{
"head": "DEF001",
"head_type": "信息获取",
"relation": "情报保障",
"tail": "STK001",
"tail_type": "协同打击"
},
{
"head": "STK001",
"head_type": "协同打击",
"relation": "状态反馈",
"tail": "CMD001",
"tail_type": "指挥控制"
},
{
"head": "STK001",
"head_type": "协同打击",
"relation": "协同作战",
"tail": "STK002",
"tail_type": "协同打击"
},
{
"head": "LOG001",
"head_type": "综合保障",
"relation": "平台部署",
"tail": "STK001",
"tail_type": "协同打击"
},
{
"head": "MED001",
"head_type": "综合保障",
"relation": "平台部署",
"tail": "TRN001",
"tail_type": "部署平台"
}
]

View File

@@ -0,0 +1,94 @@
{
"timestamp": "2026-04-23 12:20:15",
"summary": {
"total_records": 210,
"final_records": 200,
"duplicates_removed": 10
},
"details": {
"missing_values_fixed": {
"X": 2,
"Y": 5,
"WEIGHT": 126,
"AREA": 126,
"DETECTION_RANGE": 166,
"DETECTION_ACCURACY": 166,
"REFRESH_RATE": 166,
"TRANSMISSION_DELAY": 135,
"RELIABILITY": 166,
"DECISION_RESPONSE_TIME": 169,
"PROCESSING_CAPACITY": 169,
"INFORMATION_FUSION_CAPABILITY": 169,
"SYSTEM_RELIABILITY": 169,
"STRIKE_RANGE": 126,
"STRIKE_ACCURACY": 126,
"AMMUNITION_RESERVE": 126,
"TARGET_RECOGNITION_CAPABILITY": 126,
"LOAD_CAPACITY": 159,
"PLATFORM_AREA": 180,
"SUPPORT_RANGE": 159,
"SUPPORT_CAPACITY": 139
},
"outliers_corrected": {
"count": 59,
"examples": [
{
"id": "FS-法-协同打-144",
"field": "TARGET_RECOGNITION_CAPABILITY",
"original": 1.1645873020580773,
"corrected": 1.0,
"reason": "Value out of range [0, 1]"
},
{
"id": "FS-法-协同打-144",
"field": "STRIKE_ACCURACY",
"original": 1.2863907411837974,
"corrected": 1.0,
"reason": "Value out of range [0, 1]"
},
{
"id": "FS-美-火力打-156",
"field": "TARGET_RECOGNITION_CAPABILITY",
"original": -0.8869593801185574,
"corrected": 0.8869593801185574,
"reason": "Value out of range [0, 1]"
},
{
"id": "FS-俄-协同打-061",
"field": "TARGET_RECOGNITION_CAPABILITY",
"original": 1.1931823083468704,
"corrected": 1.0,
"reason": "Value out of range [0, 1]"
},
{
"id": "FS-美-协同打-121",
"field": "STRIKE_ACCURACY",
"original": 1.439367437550393,
"corrected": 1.0,
"reason": "Value out of range [0, 1]"
}
]
},
"noise_reduction": {
"method": "Kalman Filter",
"fields_processed": [
"COMMUNICATION_RANGE"
],
"total_smoothed": 200
},
"standardization": [
"Coordinates normalized to 2 decimal places",
"Timestamps formatted to ISO-8601",
"Attributes aligned to 1125 schema (format + content)"
],
"attribute_alignment": {
"enabled": true,
"format_errors_by_field": {},
"content_errors_by_field": {},
"records_with_errors": 0,
"aligned_count": 200,
"transformed_count": 200,
"samples": []
}
}
}

View File

@@ -0,0 +1,20 @@
{
"timestamp": "2026-02-27 19:45:51",
"original_count": 8,
"deduplicate": {
"enabled": true,
"removed": 0
},
"filter_orphans": {
"enabled": false,
"removed": 0
},
"remove_null_core": {
"enabled": true,
"removed": 0
},
"normalize_relation_type": {
"enabled": true
},
"final_count": 8
}

5
final/requirements.txt Normal file
View File

@@ -0,0 +1,5 @@
numpy>=1.20.0
pandas>=1.3.0
PyYAML>=5.4
openpyxl>=3.0.0
# 达梦模式需额外pip install dmPython

View File

@@ -0,0 +1,83 @@
============================================================
网络关系与效能评价报告
============================================================
1. 基本信息
----------------------------------------
过滤阈值: 0.15 【关系强度阈值】
总三元组数量: 8
分别有:
有效计算三元组: 0
跳过三元组: 8
通过过滤三元组: 0
通过率: 0.0%
2. 计算公式
边关系计算公式
--------------------------------------------------
情报保障关系 L_IS(i,j) = w_f·M_F^IS + w_s·M_S^IS + w_t·M_T^IS + w_p·M_P^IS + w_i·M_I^IS
指挥控制关系 L_CC(i,j) = w_f·M_F^CC + w_s·M_S^CC + w_t·M_T^CC + w_p·M_P^CC + w_i·M_I^CC
状态反馈关系 L_SF(i,j) = w_f·M_F^SF + w_s·M_S^SF + w_t·M_T^SF + w_p·M_P^SF + w_i·M_I^SF
平台部署关系 L_PD(i,j) = w_f·M_F^PD + w_s·M_S^PD + w_t·M_T^PD + w_p·M_P^PD + w_i·M_I^PD
协同作战关系 L_CO(i,j) = w_f·M_F^CO + w_s·M_S^CO + w_t·M_T^CO + w_p·M_P^CO + w_i·M_I^CO
【符号说明】
M_F^IS 情报保障功能匹配项
M_S^IS 情报保障空间匹配项
M_T^IS 情报保障时间匹配项
M_P^IS 情报保障性能匹配项
M_I^IS 情报保障交互匹配项
任务网络效能计算公式
--------------------------------------------------
综合防御 P_defense = α1·(∑f_IC/N_IC) + α2·(∑f_IA/N_IA) + α3·(∑L_IS/N_IS) + α4·(∑t_resp/N) + α5·(∑L_CC/N_CC)
火力打击 P_fire = β1·(∑f_CS/N_CS) + β2·(∑f_IA/N_IA) + β3·(∑L_CO/N_CO) + β4·(∑p_core·f_CS/N_CS) + β5·(∑L_IS/N_IS)
后勤保障 P_logistics = γ1·(∑f_CPS/N_CPS) + γ2·(∑f_DP/N_DP) + γ3·(∑L_PD/N_PD) + γ4·(∑p_rel/N) + γ5·(∑f_IT/N_IT)
医疗救援 P_medical = δ1·(∑f_CPS/N_CPS) + δ2·(∑t_resp/N) + δ3·(∑f_IT/N_IT) + δ4·(∑L_SF/N_SF) + δ5·(∑fresh_i/N)
紧急疏散 P_evacuation = ε1·(∑f_CC/N_CC) + ε2·(∑f_IT/N_IT) + ε3·(∑L_CC/N_CC) + ε4·(∑f_CPS/N_CPS) + ε5·(∑t_resp/N)
【符号说明】
N_IC 信息对抗节点数量
∑f_IC/N_IC 信息对抗节点平均能力
∑L_IS/N_IS 情报保障关系平均强度
∑t_resp/N 系统平均响应速度
∑L_CC/N_CC 指挥控制关系平均强度
∑f_CS/N_CS 协同打击节点平均能力
∑f_IA/N_IA 信息获取节点平均能力
∑L_CO/N_CO 协同作战关系平均强度
∑p_core·f_CS/N_CS 打击节点性能加权能力
∑f_CPS/N_CPS 综合保障节点平均能力
∑f_DP/N_DP 部署平台节点平均能力
∑L_PD/N_PD 平台部署关系平均强度
∑p_rel/N 系统平均可靠性
∑f_IT/N_IT 信息传输节点平均能力
∑L_SF/N_SF 状态反馈关系平均强度
∑fresh_i/N 系统平均数据新鲜度
α1...α5 综合防御权重 [0.30,0.25,0.20,0.15,0.10]
β1...β5 火力打击权重 [0.35,0.25,0.20,0.10,0.10]
γ1...γ5 后勤保障权重 [0.30,0.25,0.20,0.15,0.10]
δ1...δ5 医疗救援权重 [0.35,0.25,0.20,0.10,0.10]
ε1...ε5 紧急疏散权重 [0.30,0.25,0.20,0.15,0.10]
3. 关系类型统计
----------------------------------------
【被过滤掉的关系明细】
----------------------------------------
4. 所属任务网络评估
----------------------------------------
任务网络概率:综合防御 21.2% 火力打击 17.5% 后勤保障 20.2% 医疗救援 19.7% 紧急疏散 21.4%
最有可能的任务网络:紧急疏散 可能性为21.4 %
5. 最终紧急疏散任务网络的五个维度评价
功能维度 (Function)
功能维度反映了网络中节点的功能匹配程度。得分: 0.917
高于参考值,表示节点在功能上高度互补,能够有效协同工作。
空间系维度 (Relationship)
空间维度反映了节点之间的连接强度。得分: 0.000
低于参考值,表示节点之间的连接较弱,信息传递效率较低。
性能维度 (Performance)
性能维度反映了节点的性能指标,如响应时间和可靠性。得分: 0.935
高于参考值,表示节点在性能上表现出色,能够稳定运行。
时间维度 (Temporal)
时间维度反映了节点的时间同步性和响应能力。得分: 0.126
低于参考值,表示节点在时间同步性上存在偏差,响应速度较慢。
交互维度 (Interaction)
交互维度反映了节点之间的协议、格式和安全兼容性。得分: 0.000
低于参考值,表示节点在交互上存在兼容性问题,通信安全性较低。

View File

@@ -0,0 +1,3 @@
{
"triples": []
}

View File

@@ -0,0 +1 @@

BIN
final/results/triples.xlsx Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""
从达梦数据库导出三元组数据到 JSON 文件
三元组来源RELATION_INSTANCE 表 (HEAD_ID, RELATION_TYPE, TAIL_ID)
支持可选清洗:去重、过滤悬空引用、移除空值、规范化关系类型
用法: python scripts/export_triplets.py
需设置环境变量 DM_PASSWORD且 config 中 source=db
"""
import json
import os
import sys
_script_dir = os.path.dirname(os.path.abspath(__file__))
_project_root = os.path.dirname(_script_dir)
sys.path.insert(0, _project_root)
def _json_serializer(obj):
"""处理 Timestamp、datetime、numpy 等不可直接 JSON 序列化的类型"""
if hasattr(obj, 'isoformat'):
return obj.isoformat()
if hasattr(obj, 'item'):
return obj.item()
if isinstance(obj, float) and (obj != obj or obj == float('inf') or obj == float('-inf')):
return None
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
def load_config(project_root: str) -> dict:
"""加载 config"""
config_dir = os.path.join(project_root, 'config')
json_path = os.path.join(config_dir, 'config.json')
yaml_path = os.path.join(config_dir, 'config.yaml')
if os.path.exists(json_path):
with open(json_path, 'r', encoding='utf-8') as f:
return json.load(f)
if os.path.exists(yaml_path):
try:
import yaml
with open(yaml_path, 'r', encoding='utf-8') as f:
return yaml.safe_load(f)
except ImportError:
pass
return {'source': 'json', 'db': {}, 'triplet': {}}
def main():
from src.data_loader import load_triplets_from_dm, load_from_dm
from src.triplet_cleaner import clean_triplets, transform_triplet_output_format
config = load_config(_project_root)
if config.get('source') != 'db':
print("错误:三元组导出仅支持达梦模式,请将 config 中 source 改为 db")
sys.exit(1)
tc = config.get('triplet', {})
output_file = tc.get('output_file', 'data/triples.json')
report_file = tc.get('report_file', 'report/triplet_cleaning_report.json')
output_path = output_file if os.path.isabs(output_file) else os.path.join(_project_root, output_file)
report_path = report_file if os.path.isabs(report_file) else os.path.join(_project_root, report_file)
os.makedirs(os.path.dirname(output_path), exist_ok=True)
os.makedirs(os.path.dirname(report_path), exist_ok=True)
# 清洗配置clean: false 则跳过;否则按子项配置)
clean_cfg = tc.get('clean', True)
do_clean = False
if clean_cfg is True:
do_clean = {'deduplicate': True, 'filter_orphans': False, 'remove_null_core': True, 'normalize_relation_type': True}
elif isinstance(clean_cfg, dict):
do_clean = clean_cfg
deduplicate = do_clean.get('deduplicate', True) if do_clean else False
filter_orphans = do_clean.get('filter_orphans', False) if do_clean else False
remove_null_core = do_clean.get('remove_null_core', True) if do_clean else False
normalize_relation_type = do_clean.get('normalize_relation_type', True) if do_clean else False
print("正在从达梦读取三元组 (RELATION_INSTANCE)...")
triplets = load_triplets_from_dm(config, _project_root)
print(f"读取到 {len(triplets)} 条三元组")
valid_ids = None
if filter_orphans:
print("正在加载实体表以校验悬空引用...")
entities = load_from_dm(config, _project_root)
valid_ids = {str(e.get('TARGET_ID', '')).strip() for e in entities if e.get('TARGET_ID')}
print(f"有效实体数: {len(valid_ids)}")
if do_clean and (deduplicate or filter_orphans or remove_null_core or normalize_relation_type):
print("正在执行三元组清洗...")
triplets, report = clean_triplets(
triplets,
valid_ids=valid_ids,
deduplicate=deduplicate,
filter_orphans=filter_orphans,
remove_null_core=remove_null_core,
normalize_relation_type=normalize_relation_type,
)
with open(report_path, 'w', encoding='utf-8') as f:
json.dump(report, f, ensure_ascii=False, indent=2, default=_json_serializer)
print(f"清洗完成:{report['original_count']} -> {report['final_count']},报告: {report_path}")
# 转为输出格式head, head_type, relation, tail, tail_type
entities = load_from_dm(config, _project_root)
id_to_role = {str(e.get('TARGET_ID', '')).strip(): str(e.get('ROLE_ID', '')).strip() for e in entities if e.get('TARGET_ID')}
triplets = transform_triplet_output_format(triplets, id_to_role)
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(triplets, f, ensure_ascii=False, indent=2, default=_json_serializer)
print(f"已导出至 {output_path}")
if __name__ == "__main__":
main()

112
final/scripts/f_net1125.py Normal file
View File

@@ -0,0 +1,112 @@
# f_net1125.py ← 边关系已算完,只算任务网络
from __future__ import annotations
import numpy as np
from typing import Dict, List, Tuple
NodeID = str
TaskScore = float # 网络效能得分 [0,1]
class TaskNetworkEvaluator:
"""
输入:
nodes - 节点数据(含 function_vector / performance / temporal
relations - 已算好的全部边关系
格式relations[("IS","DEF001","CMD001")] = 0.87
"""
def __init__(self,
nodes: Dict[NodeID, Dict],
relations: Dict[Tuple[str, NodeID, NodeID], float],
weights: Dict[str, Dict[str, float]] = None):
self.nodes = nodes
self.rels = relations
self.w = weights
# ---------- 工具 ----------
def _avg(self, scores: List[float]) -> float:
return float(np.mean(scores)) if scores else 0.0
def _collect_func(self, key: str) -> List[float]:
"""所有节点指定功能值"""
return [n["function_vector"][key] for n in self.nodes.values()]
def _collect_rel(self, tag: str) -> List[float]:
"""预存关系强度列表"""
return [v for k, v in self.rels.items() if k[0] == tag]
def _collect_resp(self) -> List[float]:
"""响应时间得分"""
return [np.exp(-0.1 * n["temporal"]["response_time"]) for n in self.nodes.values()]
# ---------- 2.1 综合防御 ----------
def eval_defense(self) -> TaskScore:
w = self.w["defense"]
f_ic = self._avg(self._collect_func("f_IC"))
f_ia = self._avg(self._collect_func("f_IA"))
l_is = self._avg(self._collect_rel("IS"))
t_resp = self._avg(self._collect_resp())
l_cc = self._avg(self._collect_rel("CC"))
return w["w1"] * f_ic + w["w2"] * f_ia + w["w3"] * l_is + w["w4"] * t_resp + w["w5"] * l_cc
# ---------- 2.2 火力打击 ----------
def eval_fire(self) -> TaskScore:
w = self.w["fire"]
f_cs = self._avg(self._collect_func("f_CS"))
f_ia = self._avg(self._collect_func("f_IA"))
l_co = self._avg(self._collect_rel("CO"))
perf_cs = self._avg([n["performance"]["core_performance"] * n["function_vector"]["f_CS"]
for n in self.nodes.values()])
l_is = self._avg(self._collect_rel("IS"))
return w["w1"] * f_cs + w["w2"] * f_ia + w["w3"] * l_co + w["w4"] * perf_cs + w["w5"] * l_is
# ---------- 2.3 后勤保障 ----------
def eval_logistics(self) -> TaskScore:
w = self.w["logistics"]
f_cps = self._avg(self._collect_func("f_CPS"))
f_dp = self._avg(self._collect_func("f_DP"))
l_pd = self._avg(self._collect_rel("PD"))
p_rel = self._avg([n["performance"]["mtbf"] / 1000 for n in self.nodes.values()])
f_it = self._avg(self._collect_func("f_IT"))
return w["w1"] * f_cps + w["w2"] * f_dp + w["w3"] * l_pd + w["w4"] * p_rel + w["w5"] * f_it
# ---------- 2.4 医疗救援 ----------
def eval_medical(self) -> TaskScore:
w = self.w["medical"]
f_cps = self._avg(self._collect_func("f_CPS"))
t_resp = self._avg(self._collect_resp())
f_it = self._avg(self._collect_func("f_IT"))
l_sf = self._avg(self._collect_rel("SF"))
fresh = self._avg([np.exp(-0.1 * n["temporal"]["data_age"]) for n in self.nodes.values()])
return w["w1"] * f_cps + w["w2"] * t_resp + w["w3"] * f_it + w["w4"] * l_sf + w["w5"] * fresh
# ---------- 2.5 紧急疏散 ----------
def eval_evacuation(self) -> TaskScore:
w = self.w["evacuation"]
f_cc = self._avg(self._collect_func("f_CC"))
f_it = self._avg(self._collect_func("f_IT"))
l_cc = self._avg(self._collect_rel("CC"))
f_cps = self._avg(self._collect_func("f_CPS"))
t_resp = self._avg(self._collect_resp())
return w["w1"] * f_cc + w["w2"] * f_it + w["w3"] * l_cc + w["w4"] * f_cps + w["w5"] * t_resp
# ---------- 一键评估 ----------
def eval_all(self) -> Dict[str, TaskScore]:
results = {
"defense": self.eval_defense(),
"fire": self.eval_fire(),
"logistics": self.eval_logistics(),
"medical": self.eval_medical(),
"evacuation": self.eval_evacuation(),
}
# 计算五个维度的分数
dimensions = {
"function": self._avg(self._collect_func("f_IC")),
"relationship": self._avg(self._collect_rel("IS")),
"performance": self._avg([n["performance"]["core_performance"] for n in self.nodes.values()]),
"temporal": self._avg(self._collect_resp()),
"interaction": self._avg(self._collect_rel("SF"))
}
results["dimensions"] = dimensions
return results

View File

@@ -0,0 +1,395 @@
import numpy as np
from typing import Dict
from .utils import NetworkUtils
class RelationCalculator:
"""关系强度计算器"""
def __init__(self, nodes_data: Dict, cfg):
"""
初始化
:param nodes_data: 节点数据
:param auxiliary_data: 辅助数据(接口标准、角色匹配等)
"""
self.nodes_data = nodes_data
self.utils = NetworkUtils()
self.cfg = cfg
def _weights(self, rel: str):
return self.cfg["W_REL"][rel]
def _lambda(self, rel: str):
return self.cfg["LAMBDA"][rel]
def calculate_IS_relation(self, i: str, j: str) -> float:
"""
计算情报保障关系强度
L_IS(i,j) = w_f·M_IS_F + w_s·M_IS_S + w_t·M_IS_T + w_p·M_IS_P + w_i·M_IS_I
"""
# 权重参数根据1.3.pdf
w = self._weights("IS")
# 提取节点数据
node_i = self.nodes_data['nodes'][i]
node_j = self.nodes_data['nodes'][j]
# 1. 功能维度 M_IS_F
f_i_IA = node_i['function_vector']['f_IA']
f_j_avg = (
node_j['function_vector']['f_CC'] +
node_j['function_vector']['f_IC'] +
node_j['function_vector']['f_CS'] +
node_j['function_vector']['f_CPS']
) / 4
std_ij = self.utils.get_interface_standard(node_i, node_j)
r_ij = self.utils.get_role_match(node_i, node_j)
M_IS_F = f_i_IA * f_j_avg * std_ij * r_ij
# 2. 空间维度 M_IS_S
d_ij = self.utils.calculate_distance(
node_i['spatial']['position'],
node_j['spatial']['position']
)
lambda_IS = self._lambda("IS")
R_match = self.utils.calculate_range_match(
node_i['spatial']['effective_radius'],
node_j['spatial']['effective_radius'],
d_ij
)
s_area = self.utils.get_area_relation(node_i, node_j)
M_IS_S = np.exp(-d_ij / lambda_IS) * R_match * s_area
# 3. 时间维度
alpha = self.cfg["TIME"]["alpha"]
beta = self.cfg["TIME"]["beta"]
gamma = self.cfg["TIME"]["gamma"]
fresh_i = np.exp(-gamma * node_i['temporal']['data_age'])
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
M_IS_T = fresh_i * t_i_resp * w_ij
# 4. 性能维度 M_IS_P
p_i_core = node_i['performance']['core_performance']
mtbf_max = self.nodes_data['global_params']['mtbf_max']
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
M_IS_P = p_i_core * p_i_rel
# 5. 交互维度 M_IS_I
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
hist_ij = self.utils.get_interaction_history(node_i, node_j)
M_IS_I = ((prot_ij + fmt_ij + sec_ij) / 3) * hist_ij
# 综合计算
L_IS = (w["w_f"] * M_IS_F + w["w_s"] * M_IS_S + w["w_t"] * M_IS_T +
w["w_p"] * M_IS_P + w["w_i"] * M_IS_I)
return L_IS
def calculate_CC_relation(self, i: str, j: str) -> float:
"""
计算指挥控制关系强度
L_CC(i,j) = w_f·M_CC_F + w_s·M_CC_S + w_t·M_CC_T + w_p·M_CC_P + w_i·M_CC_I
"""
# 权重参数
w = self._weights("CC")
lambda_CC = self._lambda("CC")# 根据公式,指挥控制距离敏感性更高
node_i = self.nodes_data['nodes'][i]
node_j = self.nodes_data['nodes'][j]
# 1. 功能维度 M_CC_F
f_i_CC = node_i['function_vector']['f_CC']
f_j_sum = sum([
node_j['function_vector']['f_IA'],
node_j['function_vector']['f_IT'],
node_j['function_vector']['f_IC'],
node_j['function_vector']['f_CS'],
node_j['function_vector']['f_DP'],
node_j['function_vector']['f_CPS']
])
f_j_avg = f_j_sum / 6
std_ij = self.utils.get_interface_standard(node_i, node_j)
r_ij = self.utils.get_role_match(node_i, node_j)
M_CC_F = f_i_CC * f_j_avg * std_ij * r_ij
# 2. 空间维度 M_CC_S
d_ij = self.utils.calculate_distance(
node_i['spatial']['position'],
node_j['spatial']['position']
)
R_match = self.utils.calculate_range_match(
node_i['spatial']['effective_radius'],
node_j['spatial']['effective_radius'],
d_ij
)
s_area = self.utils.get_area_relation(node_i, node_j)
M_CC_S = np.exp(-d_ij / lambda_CC) * R_match * s_area
# 3. 时间维度 M_CC_T
alpha = self.cfg["TIME"]["alpha"]
beta = self.cfg["TIME"]["beta"]
gamma = self.cfg["TIME"]["gamma"]
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
M_CC_T = t_i_resp * t_i_cycle * w_ij
# 4. 性能维度 M_CC_P
p_i_core = node_i['performance']['core_performance']
p_i_surv = node_i['performance']['survivability']
mtbf_max = self.nodes_data['global_params']['mtbf_max']
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
M_CC_P = p_i_core * p_i_surv * p_i_rel
# 5. 交互维度 M_CC_I
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
org_ij = self.utils.get_organization_relation(node_i, node_j)
hist_ij = self.utils.get_interaction_history(node_i, node_j)
M_CC_I = ((prot_ij + sec_ij + org_ij) / 3) * hist_ij
# 综合计算
L_CC = (w["w_f"] * M_CC_F + w["w_s"] * M_CC_S + w["w_t"] * M_CC_T +
w["w_p"] * M_CC_P + w["w_i"] * M_CC_I)
return L_CC
def calculate_SF_relation(self, i: str, j: str) -> float:
"""
计算状态反馈关系强度
L_SF(i,j) = w_f·M_SF_F + w_s·M_SF_S + w_t·M_SF_T + w_p·M_SF_P + w_i·M_SF_I
"""
# 权重参数
w = self._weights("SF")
lambda_SF = self._lambda("SF")
node_i = self.nodes_data['nodes'][i]
node_j = self.nodes_data['nodes'][j]
# 1. 功能维度 M_SF_F
f_i_sum = sum([
node_i['function_vector']['f_IA'],
node_i['function_vector']['f_IT'],
node_i['function_vector']['f_IC'],
node_i['function_vector']['f_CS'],
node_i['function_vector']['f_DP'],
node_i['function_vector']['f_CPS']
])
f_i_avg = f_i_sum / 6
f_j_CC = node_j['function_vector']['f_CC']
std_ij = self.utils.get_interface_standard(node_i, node_j)
r_ij = self.utils.get_role_match(node_i, node_j)
M_SF_F = f_i_avg * f_j_CC * std_ij * r_ij
# 2. 空间维度 M_SF_S
d_ij = self.utils.calculate_distance(
node_i['spatial']['position'],
node_j['spatial']['position']
)
R_match = self.utils.calculate_range_match(
node_i['spatial']['effective_radius'],
node_j['spatial']['effective_radius'],
d_ij
)
s_area = self.utils.get_area_relation(node_i, node_j)
M_SF_S = np.exp(-d_ij / lambda_SF) * R_match * s_area
# 3. 时间维度 M_SF_T状态反馈对时间敏感
alpha = self.cfg["TIME"]["alpha"]
beta = self.cfg["TIME"]["beta"]
gamma = self.cfg["TIME"]["gamma"]
fresh_i = np.exp(-gamma * node_i['temporal']['data_age'])
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
M_SF_T = fresh_i * t_i_resp * w_ij
# 4. 性能维度 M_SF_P
p_i_core = node_i['performance']['core_performance']
mtbf_max = self.nodes_data['global_params']['mtbf_max']
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
M_SF_P = p_i_core * p_i_rel
# 5. 交互维度 M_SF_I
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
hist_ij = self.utils.get_interaction_history(node_i, node_j)
M_SF_I = ((prot_ij + fmt_ij + sec_ij) / 3) * hist_ij
# 综合计算
L_SF = (w["w_f"] * M_SF_F + w["w_s"] * M_SF_S + w["w_t"] * M_SF_T +
w["w_p"] * M_SF_P + w["w_i"] * M_SF_I)
return L_SF
def calculate_PD_relation(self, i: str, j: str) -> float:
"""
计算平台部署关系强度
L_PD(i,j) = w_f·M_PD_F + w_s·M_PD_S + w_t·M_PD_T + w_p·M_PD_P + w_i·M_PD_I
"""
# 权重参数
w = self._weights("PD")
lambda_PD = self._lambda("PD")
node_i = self.nodes_data['nodes'][i]
node_j = self.nodes_data['nodes'][j]
# 1. 功能维度 M_PD_F
f_i_DP = node_i['function_vector']['f_DP']
f_J_sum = sum([
node_j['function_vector']['f_IA'],
node_j['function_vector']['f_IT'],
node_j['function_vector']['f_IC'],
node_j['function_vector']['f_CS'],
])
f_J_avg = f_J_sum / 4
# 修复:直接传递节点数据
std_ij = self.utils.get_interface_standard(node_i, node_j)
r_ij = self.utils.get_role_match(node_i, node_j)
M_PD_F = f_i_DP * f_J_avg * std_ij * r_ij
# 2. 空间维度 M_PD_S平台部署对空间要求高
d_ij = self.utils.calculate_distance(
node_i['spatial']['position'],
node_j['spatial']['position']
)
R_match = self.utils.calculate_range_match(
node_i['spatial']['effective_radius'],
node_j['spatial']['effective_radius'],
d_ij
)
# 修复:直接传递节点数据
s_area = self.utils.get_area_relation(node_i, node_j)
if d_ij <= self.cfg["PD_SGM"]:
M_PD_S = 1
else:
M_PD_S = np.exp(-d_ij / lambda_PD) * R_match * s_area
# 3. 时间维度 M_PD_T
alpha = self.cfg["TIME"]["alpha"]
beta = self.cfg["TIME"]["beta"]
gamma = self.cfg["TIME"]["gamma"]
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
M_PD_T = t_i_resp * t_i_cycle * w_ij
# 4. 性能维度 M_PD_P
p_i_core = node_i['performance']['core_performance']
# 部署平台的生存能力
p_i_surv = node_i['performance']['survivability']
M_PD_P = p_i_core * p_i_surv
# 5. 交互维度 M_PD_I
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
# 注意这里std_ij已经在上面计算过了
M_PD_I = (prot_ij + fmt_ij + std_ij) / 3
# 综合计算
L_PD = (w["w_f"] * M_PD_F + w["w_s"] * M_PD_S + w["w_t"] * M_PD_T +
w["w_p"] * M_PD_P + w["w_i"] * M_PD_I)
return L_PD
def calculate_CO_relation(self, i: str, j: str) -> float:
"""
计算协同作战关系强度
L_CO(i,j) = w_f·M_CO_F + w_s·M_CO_S + w_t·M_CO_T + w_p·M_CO_P + w_i·M_CO_I
"""
# 权重参数
w = self._weights("CO")
lambda_CO = self._lambda("CO")
node_i = self.nodes_data['nodes'][i]
node_j = self.nodes_data['nodes'][j]
# 1. 功能维度 M_CO_F
# 协同作战需要多个功能匹配
f_i_avg = sum(node_i['function_vector'].values()) / len(node_i['function_vector'])
f_j_avg = sum(node_j['function_vector'].values()) / len(node_j['function_vector'])
std_ij = self.utils.get_interface_standard(node_i, node_j)
r_ij = self.utils.get_role_match(node_i, node_j)
M_CO_F = f_i_avg * f_j_avg * std_ij * r_ij
# 2. 空间维度 M_CO_S
d_ij = self.utils.calculate_distance(
node_i['spatial']['position'],
node_j['spatial']['position']
)
R_match = self.utils.calculate_range_match(
node_i['spatial']['effective_radius'],
node_j['spatial']['effective_radius'],
d_ij
)
s_area = self.utils.get_area_relation(node_i, node_j)
M_CO_S = np.exp(-d_ij / lambda_CO) * R_match * s_area
# 3. 时间维度 M_CO_T协同需要时间同步
alpha = self.cfg["TIME"]["alpha"]
beta = self.cfg["TIME"]["beta"]
gamma = self.cfg["TIME"]["gamma"]
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
t_j_resp = np.exp(-alpha * node_j['temporal']['response_time'])
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
t_j_cycle = np.exp(-beta * node_j['temporal']['cycle_time'])
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
M_CO_T = (t_i_resp + t_j_resp) * (t_i_cycle + t_j_cycle) * w_ij * 0.25
# 4. 性能维度 M_CO_P
p_i_core = node_i['performance']['core_performance']
p_j_core = node_j['performance']['core_performance']
mtbf_max = self.nodes_data['global_params']['mtbf_max']
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
p_j_rel = node_j['performance']['mtbf'] / mtbf_max
p_i_surv = node_i['performance']['survivability']
p_j_surv = node_j['performance']['survivability']
# 协同性能取平均
M_CO_P = ((p_i_core + p_j_core) / 2) * ((p_i_rel + p_j_rel) / 2) * ((p_i_surv + p_j_surv) / 2)
# 5. 交互维度 M_CO_I
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
org_ij = self.utils.get_organization_relation(node_i, node_j)
hist_ij = self.utils.get_interaction_history(node_i, node_j)
M_CO_I = ((prot_ij + fmt_ij + sec_ij + org_ij) / 4) * hist_ij
# 综合计算
L_CO = (w["w_f"] * M_CO_F + w["w_s"] * M_CO_S + w["w_t"] * M_CO_T +
w["w_p"] * M_CO_P + w["w_i"] * M_CO_I)
return L_CO

View File

@@ -0,0 +1,241 @@
#!/usr/bin/env python3
"""
在达梦数据库机器上运行,查看各表结构,用于编写 config 中的 db.query
用法: python scripts/inspect_dm_tables.py
密码获取顺序:环境变量 DM_PASS > 下方 LOCAL_PASSWORD
若使用 LOCAL_PASSWORD切勿提交到 git可执行git update-index --assume-unchanged scripts/inspect_dm_tables.py
"""
import os
import sys
# 可选:在此直接填写密码(仅限本机调试,切勿提交到版本库!)
LOCAL_PASSWORD = None # 例如: "你的密码"
# 支持从项目根或 scripts 目录运行
_script_dir = os.path.dirname(os.path.abspath(__file__))
_project_root = os.path.dirname(_script_dir)
XLSX_PATH = os.path.join(_project_root, "1125暂时属性需求(1).xlsx")
if _project_root not in sys.path:
sys.path.insert(0, _project_root)
try:
import dmPython
except ImportError:
print("请先安装: pip install dmPython")
sys.exit(1)
CONFIG = {
"user": os.getenv("DM_USER", "SYSDBA"),
"password": os.getenv("DM_PASS") or LOCAL_PASSWORD,
"server": os.getenv("DM_HOST", "127.0.0.1"),
"port": int(os.getenv("DM_PORT", "9080")),
"schema": "SYSDBA",
}
TABLES = [
"OBJECTIVE_INSTANCE",
"METRIC_VALUE_INSTANCE",
"METRIC_PROFILE",
"ROLE_PROFILE",
"TASK_PROFILE",
"RELATION_INSTANCE",
"RES_FORMAL_ANALYSIS",
]
def _load_required_attrs(xlsx_path: str) -> list:
"""从 1125暂时属性需求(1).xlsx 解析需求属性列表(一条=一个属性)"""
if not os.path.exists(xlsx_path):
print(f" (未找到 {xlsx_path},使用内置属性列表)")
return _DEFAULT_REQ_ATTRS
try:
import pandas as pd
df = pd.read_excel(xlsx_path, header=None)
attrs = []
for _, row in df.iterrows():
v0 = row[0]
v2 = str(row[2]) if len(row) > 2 and pd.notna(row[2]) else ""
if pd.notna(v0) and str(v0).strip() and str(v0) != "属性代码名称":
attrs.append(str(v0).strip())
if ":" in v2:
sub = v2.split(":")[0].strip()
if sub and sub not in attrs:
attrs.append(sub)
return attrs if attrs else _DEFAULT_REQ_ATTRS
except Exception as e:
print(f" (解析 xlsx 失败: {e},使用内置属性列表)")
return _DEFAULT_REQ_ATTRS
# xlsx 解析失败时的兜底属性列表(来自 1125暂时属性需求
_DEFAULT_REQ_ATTRS = [
"f_IC", "f_IA", "f_CS", "f_IT", "f_DP", "f_CPS", "f_CC",
"position", "effective_radius",
"response_time", "cycle_time", "data_age", "time_window",
"core_performance", "survivability", "mtbf",
"protocol_list", "format_list", "interface_list",
"security_level", "org_unit", "nation", "history_success",
]
def main():
if not CONFIG["password"]:
print("请设置环境变量 DM_PASS 或在脚本中填写 LOCAL_PASSWORD")
sys.exit(1)
try:
conn = dmPython.connect(
user=CONFIG["user"],
password=CONFIG["password"],
server=CONFIG["server"],
port=CONFIG["port"],
schema=CONFIG["schema"],
)
except Exception as e:
print(f"连接失败: {e}")
sys.exit(1)
cursor = conn.cursor()
print("=" * 60)
print("达梦表结构(用于编写 config/db.query")
print("=" * 60)
for table in TABLES:
try:
cursor.execute(f'SELECT * FROM "{CONFIG["schema"]}"."{table}" WHERE 1=0')
cols = [d[0] for d in cursor.description]
print(f"\n{table}】 列: {cols}")
except Exception as e:
print(f"\n{table}】 查询失败: {e}")
# 专门查询 METRIC_VALUE_INSTANCE 的字段详情(含完整属性)
mvi_table = "METRIC_VALUE_INSTANCE"
print("\n" + "=" * 80)
print(f"{mvi_table}】 字段详情(完整属性)")
print("=" * 80)
try:
# 查询 DBA_TAB_COLUMNS 全部常用属性(达梦兼容)
cursor.execute(
"""
SELECT COLUMN_NAME, DATA_TYPE, DATA_LENGTH, DATA_PRECISION, DATA_SCALE,
NULLABLE, DATA_DEFAULT, COLUMN_ID
FROM DBA_TAB_COLUMNS
WHERE OWNER = ? AND TABLE_NAME = ?
ORDER BY COLUMN_ID
""",
(CONFIG["schema"], mvi_table),
)
mvi_cols = cursor.fetchall()
# 表头
fmt = " {:<18} {:<16} {:<10} {:<10} {:<8} {:<10} {:<20}"
print(fmt.format("列名", "数据类型", "长度", "精度", "标度", "可空", "默认值"))
print(" " + "-" * 90)
for row in mvi_cols:
col_name, data_type, data_length, data_precision, data_scale, nullable, data_default, col_id = row
# 类型显示:数值型用 类型(精度,标度),字符型用 类型(长度)
if data_precision is not None:
type_str = f"{data_type}({data_precision},{data_scale or 0})"
elif data_length and data_type.upper() in ("VARCHAR", "CHAR", "VARCHAR2"):
type_str = f"{data_type}({data_length})"
elif data_length:
type_str = f"{data_type}({data_length})"
else:
type_str = data_type or ""
len_val = str(data_length) if data_length is not None else "-"
prec_val = str(data_precision) if data_precision is not None else "-"
scale_val = str(data_scale) if data_scale is not None else "-"
null_str = "NULL" if nullable == "Y" else "NOT NULL"
default_str = (str(data_default).strip() if data_default else "-")[:18]
print(fmt.format(col_name, type_str, len_val, prec_val, scale_val, null_str, default_str))
# 查询样本数据(若有)
cursor.execute(f'SELECT * FROM "{CONFIG["schema"]}"."{mvi_table}" WHERE ROWNUM <= 3')
sample_rows = cursor.fetchall()
if sample_rows:
col_names = [d[0] for d in cursor.description]
print(f"\n 样本数据 (最多 3 行):")
for i, row in enumerate(sample_rows, 1):
print(f"{i}: {dict(zip(col_names, row))}")
else:
print("\n (表无数据)")
except Exception as e:
print(f" 查询失败: {e}")
# 按 1125暂时属性需求(1).xlsx 查询 METRIC_VALUE_INSTANCE一条=一个属性)
print("\n" + "=" * 80)
print("【METRIC_VALUE_INSTANCE】按需求属性查询xlsx 中一条 = 一个 METRIC_ID")
print("=" * 80)
try:
# 1. 从 xlsx 解析需求属性列表
req_attrs = _load_required_attrs(XLSX_PATH)
print(f"\n xlsx 需求属性 ({len(req_attrs)} 个): {req_attrs}")
# 2. 查询库中实际存在的 METRIC_IDdistinct
cursor.execute(
f'SELECT DISTINCT METRIC_ID FROM "{CONFIG["schema"]}"."METRIC_VALUE_INSTANCE" ORDER BY METRIC_ID'
)
db_metric_ids = [r[0] for r in cursor.fetchall()]
print(f"\n 库中已有 METRIC_ID ({len(db_metric_ids)} 个): {db_metric_ids or '(无数据)'}")
# 3. METRIC_PROFILE 对照METRIC_ID -> 中文名)
cursor.execute(
f'SELECT METRIC_ID, METRIC_NAME FROM "{CONFIG["schema"]}"."METRIC_PROFILE"'
)
profile_map = dict(cursor.fetchall())
if profile_map:
print("\n METRIC_PROFILE 对照:")
for mid, mname in sorted(profile_map.items()):
print(f" {mid} -> {mname}")
# 4. 交集:需求属性中在库里有的
in_both = [a for a in req_attrs if a in db_metric_ids]
# 需求有但库无
missing = [a for a in req_attrs if a not in db_metric_ids]
# 库有但需求未列
extra = [m for m in db_metric_ids if m not in req_attrs]
print(f"\n 需求与库交集 (可查): {in_both or ''}")
print(f" 需求有但库无: {missing or ''}")
print(f" 库有但需求未列: {extra or ''}")
# 5. 按属性查询示例 SQL
print("\n --- 按属性查询示例 SQL ---")
for attr in (in_both or req_attrs)[:5]: # 最多展示 5 个
sql = f'SELECT * FROM "{CONFIG["schema"]}"."METRIC_VALUE_INSTANCE" WHERE METRIC_ID = \'{attr}\''
print(f" -- {attr}: {sql}")
if len(in_both or req_attrs) > 5:
print(f" ... 其余属性同理,将 METRIC_ID = 'xxx' 替换即可")
except Exception as e:
print(f" 查询失败: {e}")
import traceback
traceback.print_exc()
cursor.execute(
"""
SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE
FROM DBA_TAB_COLUMNS
WHERE OWNER = ?
ORDER BY TABLE_NAME, COLUMN_ID
""",
(CONFIG["schema"],),
)
rows = cursor.fetchall()
print("\n" + "=" * 60)
print("所有列详情 (TABLE_NAME | COLUMN_NAME | DATA_TYPE)")
print("=" * 60)
cur_table = None
for table, col, dtype in rows:
if table != cur_table:
cur_table = table
print(f"\n--- {table} ---")
print(f" {col}: {dtype}")
cursor.close()
conn.close()
print("\n完成。请根据上述结构在 config 中编写 db.query。")
if __name__ == "__main__":
main()

167
final/scripts/run_all.py Normal file
View File

@@ -0,0 +1,167 @@
#!/usr/bin/env python3
"""
一键运行:实体属性表 + 三元组 的读取、清洗、导出
用法: python scripts/run_all.py
"""
import json
import os
import sys
_script_dir = os.path.dirname(os.path.abspath(__file__))
_project_root = os.path.dirname(_script_dir)
sys.path.insert(0, _project_root)
def _json_serializer(obj):
if hasattr(obj, 'isoformat'):
return obj.isoformat()
if hasattr(obj, 'item'):
return obj.item()
if isinstance(obj, float) and (obj != obj or obj == float('inf') or obj == float('-inf')):
return None
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
def load_config(project_root: str) -> dict:
config_dir = os.path.join(project_root, 'config')
json_path = os.path.join(config_dir, 'config.json')
yaml_path = os.path.join(config_dir, 'config.yaml')
if os.path.exists(json_path):
with open(json_path, 'r', encoding='utf-8') as f:
return json.load(f)
if os.path.exists(yaml_path):
try:
import yaml
with open(yaml_path, 'r', encoding='utf-8') as f:
return yaml.safe_load(f)
except ImportError:
pass
return {'source': 'json', 'json': {}, 'triplet': {}, 'db': {}}
def _resolve_path(path: str, project_root: str) -> str:
return path if os.path.isabs(path) else os.path.join(project_root, path)
def main():
from src.cleaner import AdvancedDataCleaner
from src.data_loader import load_data, load_triplets_from_dm, load_from_json, run_formalization_pipeline
from src.triplet_cleaner import clean_triplets
config = load_config(_project_root)
source = config.get('source', 'json')
tc = config.get('triplet', {})
# 三元组db 模式默认处理json 模式需 triplet.input_filetriplet.enabled=false 可关闭
triplet_input = tc.get('input_file')
run_triplets = tc.get('enabled', source == 'db' or bool(triplet_input))
print("=" * 50)
print("作战体系数据清洗 - 一键运行")
print("=" * 50)
print(f"数据源: {source}")
# ---------- 1. 实体属性表 ----------
print("\n【1/2】实体属性表 读取与清洗")
try:
data, paths = load_data(config, _project_root)
except (FileNotFoundError, ValueError) as e:
print(f"错误:{e}")
sys.exit(1)
os.makedirs(os.path.dirname(paths['report']), exist_ok=True)
cleaner = AdvancedDataCleaner(output_file=paths['output'], report_file=paths['report'], data=data)
cleaner.run()
valid_ids = cleaner.get_valid_target_ids()
print(f" -> 输出: {paths['output']}")
print(f" -> 报告: {paths['report']}")
# ---------- 2. 三元组 ----------
if not run_triplets:
print("\n【2/2】三元组 已跳过triplet.enabled=false 或 json 模式未配置 input_file")
print("\n完成。")
return
print("\n【2/2】三元组 读取与清洗")
if source == 'db':
triplets = load_triplets_from_dm(config, _project_root)
print(f" 从达梦读取 {len(triplets)} 条三元组")
elif triplet_input:
input_path = _resolve_path(triplet_input, _project_root)
if os.path.exists(input_path):
raw = load_from_json(input_path, _project_root)
triplets = raw.get("triples", raw) if isinstance(raw, dict) else raw
triplets = triplets if isinstance(triplets, list) else []
print(f"{triplet_input} 读取 {len(triplets)} 条三元组")
else:
print(f" 跳过:三元组文件不存在 {input_path}")
triplets = []
else:
print(" 跳过json 模式未配置 triplet.input_file")
triplets = []
if not triplets:
print(" 无三元组数据,跳过导出")
print("\n完成。")
return
# 三元组清洗配置
clean_cfg = tc.get('clean', True)
do_clean = {'deduplicate': True, 'filter_orphans': False, 'remove_null_core': True, 'normalize_relation_type': True} if clean_cfg is True else (clean_cfg if isinstance(clean_cfg, dict) else {})
deduplicate = do_clean.get('deduplicate', True) if do_clean else False
filter_orphans = do_clean.get('filter_orphans', False) if do_clean else False
remove_null_core = do_clean.get('remove_null_core', True) if do_clean else False
normalize_relation_type = do_clean.get('normalize_relation_type', True) if do_clean else False
if do_clean and (deduplicate or filter_orphans or remove_null_core or normalize_relation_type):
triplets, report = clean_triplets(
triplets,
valid_ids=valid_ids if filter_orphans else None,
deduplicate=deduplicate,
filter_orphans=filter_orphans,
remove_null_core=remove_null_core,
normalize_relation_type=normalize_relation_type,
)
report_path = _resolve_path(tc.get('report_file', 'report/triplet_cleaning_report.json'), _project_root)
os.makedirs(os.path.dirname(report_path), exist_ok=True)
with open(report_path, 'w', encoding='utf-8') as f:
json.dump(report, f, ensure_ascii=False, indent=2, default=_json_serializer)
print(f" 清洗: {report['original_count']} -> {report['final_count']}")
print(f" 报告: {report_path}")
# 转为输出格式head, head_type, relation, tail, tail_type
from src.triplet_cleaner import transform_triplet_output_format
id_to_role = cleaner.get_id_to_role()
triplets = transform_triplet_output_format(triplets, id_to_role)
output_path = _resolve_path(tc.get('output_file', 'data/triples.json'), _project_root)
os.makedirs(os.path.dirname(output_path), exist_ok=True)
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(triplets, f, ensure_ascii=False, indent=2, default=_json_serializer)
print(f" 输出: {output_path}")
# ---------- 3. 模块1.3:形式化入库 ----------
formal_cfg = config.get("formal", {})
if source == "db" and formal_cfg.get("enabled", False):
print("\n【3/3】形式化模块 读取-补齐-入库")
try:
result = run_formalization_pipeline(config)
if result.get("ok"):
stats = result.get("stats", {})
print(f" 批次: {result.get('batch_id')}")
print(f" 推理三元组: {stats.get('total_triples', 0)}")
print(f" 形式化入库: {stats.get('inserted_formal_count', 0)}")
print(f" 属性补齐写回: {stats.get('generated_attrs_count', 0)}")
else:
print(f" 形式化流程跳过: {result.get('reason', 'unknown')}")
except Exception as e:
print(f" 形式化流程失败: {e}")
raise
elif formal_cfg.get("enabled", False):
print("\n【3/3】形式化模块 已跳过(仅 source=db 时启用)")
print("\n完成。")
if __name__ == "__main__":
main()

110
final/scripts/utils.py Normal file
View File

@@ -0,0 +1,110 @@
import numpy as np
from typing import Dict, List, Any
from scipy.spatial.distance import euclidean
class NetworkUtils:
"""无需 aux_data全部现场从节点属性读取"""
""" 角色匹配矩阵暂时没加 """
# ----------- 通用兜底 -----------
_DEFAULT = {
"protocol": 0.80,
"format": 0.85,
"security": 0.90,
"role": 0.70,
"area": 0.50,
"org": 0.70,
"area_1": 1,
"area_2": 0.7,
"history": 0.8
}
# ----------- 空间 -----------
@staticmethod
def calculate_distance(pos1, pos2):
return euclidean(pos1, pos2)
@staticmethod
def calculate_range_match(R_i: float, R_j: float, d_ij: float) -> float:
r_min = min(R_i, R_j)
if d_ij <= r_min:
return 1.0
return r_min / d_ij if d_ij > 0 else 0.0
@staticmethod
def calculate_time_window_overlap(node_i, node_j):
w_i = node_i["temporal"].get("time_window", 24)
w_j = node_j["temporal"].get("time_window", 24)
overlap = min(w_i, w_j)
return overlap / max(w_i, w_j, 1)
# ----------- 协议兼容性Jaccard -----------
@staticmethod
def get_protocol_compatibility(node_i, node_j, _dummy=None):
"""节点属性里放 'protocol_list'"""
p_i = set(node_i.get("protocol_list", []))
p_j = set(node_j.get("protocol_list", []))
if not p_i or not p_j:
return NetworkUtils._DEFAULT["protocol"]
intersection = len(p_i & p_j)
union = len(p_i | p_j)
return intersection / union if union else NetworkUtils._DEFAULT["protocol"]
# ----------- 安全等级1-5 映射 0-1 -----------
@staticmethod
def get_security_compatibility(node_i, node_j, _dummy=None):
lv_i = node_i.get("security_level", 3)
lv_j = node_j.get("security_level", 3)
gap = abs(lv_i - lv_j)
return max(0, 1 - gap / 5)
# ----------- 组织隶属:同单位给高分 -----------
@staticmethod
def get_organization_relation(node_i, node_j, _dummy=None):
unit_i = node_i.get("org_unit", "")
unit_j = node_j.get("org_unit", "")
return 1.0 if unit_i == unit_j and unit_i else NetworkUtils._DEFAULT["org"]
# ----------- 历史交互:直接读属性 -----------
@staticmethod
def get_interaction_history(node_i, node_j, _dummy=None):
his_i = node_i.get("history_success", NetworkUtils._DEFAULT["history"])
# print("111111111111111111111",node_i.get("history_success"))
his_j = node_j.get("history_success", NetworkUtils._DEFAULT["history"])
return (his_i + his_j) / 2
# ----------- 区域关联:同 code 高分 -----------
@staticmethod
def get_area_relation(node_i, node_j, _dummy=None):
code_i = node_i.get("nation")
code_j = node_j.get("nation")
if code_i == code_j:
return NetworkUtils._DEFAULT["area_1"]
else:
return NetworkUtils._DEFAULT["area_2"]
# ----------- 接口/角色/格式:若节点属性扩展了再读,否则给默认 -----------
@staticmethod
def get_role_match(node_i, node_j, _dummy=None):
return node_i.get("role_match", NetworkUtils._DEFAULT["role"])
@staticmethod
def get_interface_standard(node_i, node_j, _dummy=None) -> float:
"""接口标准兼容性 = Jaccard(i_list, j_list)"""
i_set = set(node_i.get("interface_list", []))
j_set = set(node_j.get("interface_list", []))
if not i_set or not j_set:
return NetworkUtils._DEFAULT["protocol"] # 默认0.8
inter = len(i_set & j_set)
union = len(i_set | j_set)
return inter / union if union else NetworkUtils._DEFAULT["protocol"]
@staticmethod
def get_format_compatibility(node_i, node_j, _dummy=None) -> float:
"""数据格式兼容性 = Jaccard(i_format, j_format)"""
i_set = set(node_i.get("format_list", []))
j_set = set(node_j.get("format_list", []))
if not i_set or not j_set:
return NetworkUtils._DEFAULT["format"] # 默认0.85
inter = len(i_set & j_set)
union = len(i_set | j_set)
return inter / union if union else NetworkUtils._DEFAULT["format"]

0
final/src/__init__.py Normal file
View File

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@@ -0,0 +1,701 @@
# -*- coding: utf-8 -*-
"""
属性规范定义模块 - 基于 1125暂时属性需求
实现内容与格式的双重对齐:格式校验(结构类型)+ 内容校验(取值范围、类型)
"""
import math
from typing import Any, Dict, List, Optional, Tuple
# ============ 目标 Schema 定义(来自 1125暂时属性需求.xlsx ============
# function_vector: 字典,键为功能维度,值为 [0,1] 浮点数
FUNCTION_VECTOR_KEYS = [
"f_IC", # 情报收集能力
"f_IA", # 信息分析能力
"f_CS", # 协同作战能力
"f_IT", # 信息传输能力
"f_DP", # 数据处理能力
"f_CPS", # 指挥控制能力
"f_CC", # 综合保障能力
]
# spatial: 字典
SPATIAL_KEYS = {
"position": lambda v: isinstance(v, (list, tuple)) and len(v) >= 2
and all(isinstance(x, (int, float)) for x in v[:2]),
"effective_radius": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
}
# temporal: 字典
TEMPORAL_KEYS = {
"response_time": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
"cycle_time": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
"data_age": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
"time_window": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
}
# performance: 字典
PERFORMANCE_KEYS = {
"core_performance": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool)
and 0 <= float(v) <= 1,
"survivability": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool)
and 0 <= float(v) <= 1,
"mtbf": lambda v: isinstance(v, (int, float)) and not isinstance(v, bool),
}
# nodes.json 输出字段(完全对齐 xlsx仅含规范字段 + 节点标识)
# TARGET_ID 为节点唯一标识(三元组 head/tail 引用所需)
OUTPUT_SCHEMA_FIELDS = [
"TARGET_ID",
"function_vector",
"spatial",
"temporal",
"performance",
"protocol_list",
"format_list",
"interface_list",
"security_level",
"org_unit",
"nation",
"history_success",
]
# 原始字段 -> 目标 schema 映射(用于 flat 数据转换为规范格式)
RAW_TO_TARGET_MAPPING = {
# function_vector 从多个能力字段推导
"function_vector": {
"f_IC": ["DETECTION_ACCURACY", "TARGET_RECOGNITION_CAPABILITY"], # 情报收集
"f_IA": ["INFORMATION_FUSION_CAPABILITY", "DETECTION_ACCURACY"], # 信息分析
"f_CS": ["MOBILITY", "ENVIRONMENT_ADAPTABILITY"], # 协同作战
"f_IT": ["ANTI_JAMMING_CAPABILITY"], # 信息传输(抗干扰)
"f_DP": ["PROCESSING_CAPACITY", "INFORMATION_FUSION_CAPABILITY"], # 数据处理
"f_CPS": ["DECISION_RESPONSE_TIME"], # 指挥控制(响应时间反推)
"f_CC": ["SUPPORT_CAPACITY", "LOAD_CAPACITY"], # 综合保障
},
"spatial": {
"position": ["X", "Y"],
"effective_radius": ["COMMUNICATION_RANGE"],
},
"temporal": {
"response_time": ["DECISION_RESPONSE_TIME"],
"cycle_time": ["REFRESH_RATE"],
"data_age": ["TRANSMISSION_DELAY"], # 传输延迟可近似数据新鲜度
"time_window": ["REFRESH_RATE"], # 周期倒数近似时间窗口
},
"performance": {
"core_performance": ["STRIKE_ACCURACY", "DETECTION_ACCURACY", "MOBILITY"],
"survivability": ["ENVIRONMENT_ADAPTABILITY", "ANTI_JAMMING_CAPABILITY"],
"mtbf": ["RELIABILITY", "SYSTEM_RELIABILITY"],
},
"nation": ["COUNTRY_REGION"],
"org_unit": ["LEVEL", "ROLE_ID"],
"history_success": [], # 无直接映射,需默认或缺失
}
def _clamp_0_1(val: Any) -> Optional[float]:
"""将值限制在 [0,1] 范围内,无效返回 None"""
if val is None or (isinstance(val, float) and (val != val)):
return None
try:
f = float(val)
if f < 0:
return 0.0
if f > 1:
return 1.0
return f
except (TypeError, ValueError):
return None
def _safe_float(val: Any) -> Optional[float]:
if val is None or (isinstance(val, float) and (val != val)):
return None
try:
return float(val)
except (TypeError, ValueError):
return None
def _safe_int(val: Any, lo: int = None, hi: int = None) -> Optional[int]:
if val is None:
return None
try:
i = int(float(val))
if lo is not None and i < lo:
return lo
if hi is not None and i > hi:
return hi
return i
except (TypeError, ValueError):
return None
def _ensure_list_of_strings(val: Any) -> List[str]:
"""确保为字符串列表"""
if val is None:
return []
if isinstance(val, list):
return [str(x).strip() for x in val if x is not None and str(x).strip()]
if isinstance(val, str) and val.strip():
return [val.strip()]
return []
# ---------- 固定参考值转换(用于 function_vector 中需特殊归一化的维度)----------
# 参考时间(秒):响应时间 t 越短f_CPS 越高
_CPS_REF_TIME = 30.0
# 处理能力参考上限
_FDP_REF_CAP = 10000.0
# 保障/载荷能力参考
_FCC_REF_SUPPORT = 10000.0
_FCC_REF_LOAD = 100.0
def _response_time_to_cps(val: Any) -> Optional[float]:
"""响应时间(秒) -> f_CPS [0,1],响应越快能力越高"""
f = _safe_float(val)
if f is None or f < 0:
return None
v = 1.0 / (1.0 + f / _CPS_REF_TIME)
return min(1.0, max(0.0, v))
def _processing_to_fdp(raw: dict) -> Optional[float]:
"""PROCESSING_CAPACITY 或 INFORMATION_FUSION_CAPABILITY -> f_DP [0,1]"""
pc = raw.get("PROCESSING_CAPACITY")
if pc is not None and (not isinstance(pc, float) or pc == pc):
try:
v = float(pc)
if v < 0:
return 0.0
norm = math.log10(1 + v) / math.log10(1 + _FDP_REF_CAP)
return min(1.0, norm)
except (TypeError, ValueError):
pass
ifc = raw.get("INFORMATION_FUSION_CAPABILITY")
if ifc is not None and (not isinstance(ifc, float) or ifc == ifc):
return _clamp_0_1(ifc)
return None
def _support_load_to_fcc(raw: dict) -> Optional[float]:
"""SUPPORT_CAPACITY 与 LOAD_CAPACITY -> f_CC [0,1],支持组合"""
support = raw.get("SUPPORT_CAPACITY")
load = raw.get("LOAD_CAPACITY")
s_norm = None
l_norm = None
if support is not None and (not isinstance(support, float) or support == support):
try:
v = float(support)
if v >= 0:
s_norm = min(1.0, math.log10(1 + v) / math.log10(1 + _FCC_REF_SUPPORT))
except (TypeError, ValueError):
pass
if load is not None and (not isinstance(load, float) or load == load):
try:
v = float(load)
if v >= 0:
l_norm = min(1.0, v / _FCC_REF_LOAD)
except (TypeError, ValueError):
pass
if s_norm is not None and l_norm is not None:
return round(0.6 * s_norm + 0.4 * l_norm, 4)
if s_norm is not None:
return s_norm
return l_norm
# ============ 格式校验 ============
def check_format_function_vector(val: Any) -> Tuple[bool, str]:
"""校验 function_vector 格式:必须为字典"""
if val is None:
return True, "optional"
if not isinstance(val, dict):
return False, f"期望 dict实际 {type(val).__name__}"
for k in val:
if not isinstance(k, str):
return False, f"键必须为 str发现 {type(k).__name__}"
v = val[k]
if not isinstance(v, (int, float)) or isinstance(v, bool):
return False, f"{k} 的值应为 [0,1] 浮点数,实际 {type(v).__name__}"
f = float(v)
if f < 0 or f > 1:
return False, f"{k} 的值 {f} 超出 [0,1]"
return True, "ok"
def check_format_spatial(val: Any) -> Tuple[bool, str]:
"""校验 spatial 格式:必须为字典,含 position 与 effective_radius"""
if val is None:
return True, "optional"
if not isinstance(val, dict):
return False, f"期望 dict实际 {type(val).__name__}"
pos = val.get("position")
if pos is not None:
if not isinstance(pos, (list, tuple)) or len(pos) < 2:
return False, "position 应为 [x, y] 列表"
if not all(isinstance(x, (int, float)) for x in pos[:2]):
return False, "position 元素应为数字"
rad = val.get("effective_radius")
if rad is not None and not isinstance(rad, (int, float)):
return False, "effective_radius 应为数字"
return True, "ok"
def check_format_temporal(val: Any) -> Tuple[bool, str]:
"""校验 temporal 格式"""
if val is None:
return True, "optional"
if not isinstance(val, dict):
return False, f"期望 dict实际 {type(val).__name__}"
for k in ["response_time", "cycle_time", "data_age", "time_window"]:
v = val.get(k)
if v is not None and not isinstance(v, (int, float)):
return False, f"{k} 应为数字"
return True, "ok"
def check_format_performance(val: Any) -> Tuple[bool, str]:
"""校验 performance 格式"""
if val is None:
return True, "optional"
if not isinstance(val, dict):
return False, f"期望 dict实际 {type(val).__name__}"
for k in ["core_performance", "survivability"]:
v = val.get(k)
if v is not None:
try:
f = float(v)
if f < 0 or f > 1:
return False, f"{k} 应在 [0,1],实际 {f}"
except (TypeError, ValueError):
return False, f"{k} 应为数字"
return True, "ok"
def check_format_protocol_list(val: Any) -> Tuple[bool, str]:
if val is None:
return True, "optional"
if not isinstance(val, list):
return False, f"期望 list实际 {type(val).__name__}"
for i, x in enumerate(val):
if not isinstance(x, str):
return False, f"元素[{i}] 应为 str实际 {type(x).__name__}"
return True, "ok"
def check_format_format_list(val: Any) -> Tuple[bool, str]:
return check_format_protocol_list(val)
def check_format_interface_list(val: Any) -> Tuple[bool, str]:
return check_format_protocol_list(val)
def check_format_security_level(val: Any) -> Tuple[bool, str]:
if val is None:
return True, "optional"
if not isinstance(val, int) or isinstance(val, bool):
try:
int(val)
except (TypeError, ValueError):
return False, f"期望 int [1,5],实际 {type(val).__name__}"
v = int(val)
if v < 1 or v > 5:
return False, f"security_level 应在 [1,5],实际 {v}"
return True, "ok"
def check_format_org_unit(val: Any) -> Tuple[bool, str]:
if val is None:
return True, "optional"
if not isinstance(val, str):
return False, f"期望 str实际 {type(val).__name__}"
return True, "ok"
def check_format_nation(val: Any) -> Tuple[bool, str]:
return check_format_org_unit(val)
def check_format_history_success(val: Any) -> Tuple[bool, str]:
if val is None:
return True, "optional"
if not isinstance(val, (int, float)) or isinstance(val, bool):
return False, f"期望 float [0,1],实际 {type(val).__name__}"
f = float(val)
if f < 0 or f > 1:
return False, f"history_success 应在 [0,1],实际 {f}"
return True, "ok"
# ============ 内容校验(取值范围、语义) ============
def check_content_function_vector(val: dict) -> List[str]:
"""内容校验:键是否在规范内,值是否合法"""
issues = []
if not isinstance(val, dict):
return ["非字典类型"]
for k, v in val.items():
if k not in FUNCTION_VECTOR_KEYS:
issues.append(f"未知键: {k}")
else:
try:
f = float(v)
if f < 0 or f > 1:
issues.append(f"{k}={f} 超出 [0,1]")
except (TypeError, ValueError):
issues.append(f"{k} 值非数字: {v}")
return issues
def check_content_spatial(val: dict) -> List[str]:
issues = []
if not isinstance(val, dict):
return ["非字典类型"]
pos = val.get("position")
if pos is not None and len(pos) >= 2:
if not all(isinstance(x, (int, float)) for x in pos[:2]):
issues.append("position 含非数字")
rad = val.get("effective_radius")
if rad is not None:
try:
f = float(rad)
if f < 0:
issues.append("effective_radius 不应为负")
except (TypeError, ValueError):
issues.append("effective_radius 非数字")
return issues
def check_content_temporal(val: dict) -> List[str]:
issues = []
if not isinstance(val, dict):
return ["非字典类型"]
for k in ["response_time", "cycle_time", "data_age", "time_window"]:
v = val.get(k)
if v is not None:
try:
f = float(v)
if f < 0:
issues.append(f"{k} 不应为负: {f}")
except (TypeError, ValueError):
issues.append(f"{k} 非数字: {v}")
return issues
def check_content_performance(val: dict) -> List[str]:
issues = []
if not isinstance(val, dict):
return ["非字典类型"]
for k in ["core_performance", "survivability"]:
v = val.get(k)
if v is not None:
try:
f = float(v)
if f < 0 or f > 1:
issues.append(f"{k} 超出 [0,1]: {f}")
except (TypeError, ValueError):
issues.append(f"{k} 非数字: {v}")
mtbf = val.get("mtbf")
if mtbf is not None:
try:
f = float(mtbf)
if f < 0:
issues.append("mtbf 不应为负")
except (TypeError, ValueError):
issues.append("mtbf 非数字")
return issues
# ============ 统一校验入口 ============
FORMAT_CHECKERS = {
"function_vector": check_format_function_vector,
"spatial": check_format_spatial,
"temporal": check_format_temporal,
"performance": check_format_performance,
"protocol_list": check_format_protocol_list,
"format_list": check_format_format_list,
"interface_list": check_format_interface_list,
"security_level": check_format_security_level,
"org_unit": check_format_org_unit,
"nation": check_format_nation,
"history_success": check_format_history_success,
}
CONTENT_CHECKERS = {
"function_vector": check_content_function_vector,
"spatial": check_content_spatial,
"temporal": check_content_temporal,
"performance": check_content_performance,
}
def validate_record(
record: dict,
target_schema_fields: Optional[List[str]] = None,
) -> Dict[str, Any]:
"""
对单条记录进行格式与内容双重校验。
若 record 已包含目标 schema 字段,则直接校验;
否则会先通过 raw_to_target_schema 转换后再校验。
"""
target_fields = target_schema_fields or list(FORMAT_CHECKERS.keys())
result = {
"format_errors": {},
"content_errors": {},
"aligned": False,
"transformed": False,
}
# 若缺少目标字段,尝试从原始字段转换
has_target = any(f in record for f in target_fields)
data = record
if not has_target:
data = raw_to_target_schema(record)
result["transformed"] = True
# 格式校验
for field in target_fields:
val = data.get(field)
if val is None and field in ["protocol_list", "format_list", "interface_list"]:
val = []
checker = FORMAT_CHECKERS.get(field)
if checker:
ok, msg = checker(val)
else:
ok, msg = True, "no_checker"
if not ok:
result["format_errors"][field] = msg
# 内容校验(仅对复杂类型)
for field in CONTENT_CHECKERS:
val = data.get(field)
if val is None:
continue
issues = CONTENT_CHECKERS[field](val)
if issues:
result["content_errors"][field] = issues
result["aligned"] = (
len(result["format_errors"]) == 0 and len(result["content_errors"]) == 0
)
return result
# ============ 原始数据 -> 目标 Schema 转换 ============
def _get_first_valid(row: dict, keys: List[str], cast=_clamp_0_1) -> Optional[Any]:
"""从 row 中按 keys 顺序取第一个非空值并转换"""
for k in keys:
v = row.get(k)
if v is not None and (not isinstance(v, float) or v == v):
return cast(v) if cast else v
return None
def raw_to_target_schema(raw: dict) -> dict:
"""
将扁平原始记录转换为符合 1125 属性规范的目标结构。
仅输出 xlsx 定义字段,不保留任何原始其他字段。
"""
# TARGET_ID节点唯一标识
tid = raw.get("TARGET_ID")
tid = str(tid).strip() if tid is not None and str(tid).strip() else None
out: Dict[str, Any] = {f: None for f in OUTPUT_SCHEMA_FIELDS}
out["TARGET_ID"] = tid
# function_vectorf_CPS/f_DP/f_CC 使用固定参考值转换,其余用 _clamp_0_1
fv = {}
_fv_mapping = RAW_TO_TARGET_MAPPING["function_vector"]
for fk, raw_keys in _fv_mapping.items():
if fk == "f_CPS":
t = raw.get("DECISION_RESPONSE_TIME")
val = _response_time_to_cps(t)
elif fk == "f_DP":
val = _processing_to_fdp(raw)
elif fk == "f_CC":
val = _support_load_to_fcc(raw)
else:
val = _get_first_valid(raw, raw_keys, _clamp_0_1)
if val is not None:
fv[fk] = round(val, 4)
# 下游 f_relation1125/f_net1125 要求必为 dict无推导值时用默认避免 NoneType
if not fv:
fv = {fk: 0.5 for fk in FUNCTION_VECTOR_KEYS}
out["function_vector"] = fv
# spatial支持 X/x、Y/y、position、POSITION_X/Y、LONGITUDE/LATITUDE确保元素恒为 float
def _safe_coord(v: Any) -> float:
if v is None or (isinstance(v, float) and v != v):
return 0.0
try:
return round(float(v), 2)
except (TypeError, ValueError):
return 0.0
pos_raw = raw.get("position") or raw.get("POSITION")
if isinstance(pos_raw, (list, tuple)) and len(pos_raw) >= 2:
x, y = _safe_coord(pos_raw[0]), _safe_coord(pos_raw[1])
else:
x = raw.get("X") or raw.get("x") or raw.get("POSITION_X") or raw.get("LONGITUDE")
y = raw.get("Y") or raw.get("y") or raw.get("POSITION_Y") or raw.get("LATITUDE")
x, y = _safe_coord(x), _safe_coord(y)
pos = [x, y]
radius = _get_first_valid(raw, RAW_TO_TARGET_MAPPING["spatial"]["effective_radius"], _safe_float)
sp = {
"position": pos,
"effective_radius": round(float(radius), 4) if radius is not None else 1.0,
}
out["spatial"] = sp
# temporal
_temporal_defaults = {"response_time": 1.0, "cycle_time": 1.0, "data_age": 1.0, "time_window": 24.0}
temporal = {}
for tk, rks in RAW_TO_TARGET_MAPPING["temporal"].items():
v = _get_first_valid(raw, rks, _safe_float)
if v is not None:
temporal[tk] = round(float(v), 4)
for k, default in _temporal_defaults.items():
temporal.setdefault(k, default)
out["temporal"] = temporal
# performance
_perf_defaults = {"core_performance": 0.5, "survivability": 0.5, "mtbf": 1.0}
perf = {}
for pk, rks in RAW_TO_TARGET_MAPPING["performance"].items():
cast = _clamp_0_1 if pk in ["core_performance", "survivability"] else _safe_float
v = _get_first_valid(raw, rks, cast)
if v is not None:
perf[pk] = round(float(v), 4)
for k, default in _perf_defaults.items():
perf.setdefault(k, default)
out["performance"] = perf
# protocol_list, format_list, interface_list
for field in ["protocol_list", "format_list", "interface_list"]:
raw_key = field.upper().replace("_LIST", "")
out[field] = _ensure_list_of_strings(raw.get(raw_key, raw.get(field)))
# security_level下游 utils.get_security_compatibility 会做 lv_i - lv_j不可为 None
sl = raw.get("security_level") or raw.get("SECURITY_LEVEL")
out["security_level"] = _safe_int(sl, 1, 5) if sl is not None else 3
# org_unit, nation
nation = _get_first_valid(raw, RAW_TO_TARGET_MAPPING["nation"], lambda v: str(v).strip() if v is not None else None)
out["nation"] = nation
org = _get_first_valid(raw, RAW_TO_TARGET_MAPPING["org_unit"], lambda v: str(v).strip() if v is not None else None)
out["org_unit"] = org
# history_success下游 utils.get_interaction_history 会做 (his_i + his_j)/2不可为 None
hs = raw.get("history_success") or raw.get("HISTORY_SUCCESS")
if hs is not None:
v = _clamp_0_1(hs)
out["history_success"] = round(v, 4) if v is not None else 0.8
else:
out["history_success"] = 0.8
return out
def align_and_validate_records(
records: List[dict],
id_key: str = "TARGET_ID",
) -> Tuple[List[dict], dict]:
"""
对记录列表进行转换、格式与内容校验。
返回:(对齐后的记录列表, 统计报告)
"""
aligned = []
report = {
"total": len(records),
"format_errors_by_field": {},
"content_errors_by_field": {},
"records_with_errors": 0,
"aligned_count": 0,
"transformed_count": 0,
"samples": [],
}
for rec in records:
rid = rec.get(id_key, "?")
val_result = validate_record(rec)
aligned_rec = raw_to_target_schema(rec)
aligned.append(aligned_rec)
if val_result["transformed"]:
report["transformed_count"] += 1
if val_result["aligned"]:
report["aligned_count"] += 1
else:
report["records_with_errors"] += 1
for f, msg in val_result["format_errors"].items():
report["format_errors_by_field"].setdefault(f, 0)
report["format_errors_by_field"][f] += 1
for f, issues in val_result["content_errors"].items():
report["content_errors_by_field"].setdefault(f, 0)
report["content_errors_by_field"][f] += 1
if len(report["samples"]) < 5:
report["samples"].append({
"id": rid,
"format_errors": val_result["format_errors"],
"content_errors": val_result["content_errors"],
})
return aligned, report
def sanitize_node_for_relation_calc(node: dict) -> dict:
"""
写入 nodes.json 前对节点做最终净化,确保 f_relation1125/utils 中参与运算的字段无 None。
避免 euclidean、get_security_compatibility、get_interaction_history 等出现 NoneType 运算错误。
"""
import copy
n = copy.deepcopy(node)
def _safe_float(v: Any, default: float) -> float:
if v is None or (isinstance(v, float) and v != v):
return default
try:
return float(v)
except (TypeError, ValueError):
return default
# spatial.position 必须为 [float, float]
sp = n.get("spatial")
if not isinstance(sp, dict):
sp = {}
pos = sp.get("position")
if not isinstance(pos, (list, tuple)) or len(pos) < 2:
pos = [0.0, 0.0]
else:
pos = [_safe_float(pos[0], 0.0), _safe_float(pos[1], 0.0)]
sp["position"] = pos
sp["effective_radius"] = _safe_float(sp.get("effective_radius"), 1.0)
n["spatial"] = sp
# temporal 参与 time_window_overlap
tmp = n.get("temporal")
if not isinstance(tmp, dict):
tmp = {}
for k in ["response_time", "cycle_time", "data_age", "time_window"]:
if tmp.get(k) is None:
tmp[k] = 24.0 if k == "time_window" else 1.0
n["temporal"] = tmp
# security_level、history_success 参与 utils 运算
sl = n.get("security_level")
n["security_level"] = int(_safe_float(sl, 3)) if sl is not None else 3
if n["security_level"] < 1 or n["security_level"] > 5:
n["security_level"] = 3
hs = n.get("history_success")
n["history_success"] = _safe_float(hs, 0.8)
n["history_success"] = max(0.0, min(1.0, n["history_success"]))
return n

294
final/src/cleaner.py Normal file
View File

@@ -0,0 +1,294 @@
import json
import numpy as np
import pandas as pd
from datetime import datetime
try:
from .attribute_schema import align_and_validate_records, sanitize_node_for_relation_calc
except ImportError:
from attribute_schema import align_and_validate_records, sanitize_node_for_relation_calc
def _json_serializer(obj):
"""处理 Timestamp、datetime、numpy、NaN 等不可直接 JSON 序列化的类型"""
if hasattr(obj, 'isoformat'):
return obj.isoformat()
if hasattr(obj, 'item'):
return obj.item()
if isinstance(obj, float) and (obj != obj or obj == float('inf') or obj == float('-inf')):
return None # NaN, Inf -> null
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
class AdvancedDataCleaner:
def __init__(self, output_file, report_file, data=None, input_file=None):
"""
支持两种初始化方式:
1. data=... 直接传入数据列表(从配置/数据库加载时使用)
2. input_file=... 传入 JSON 文件路径(兼容旧用法)
"""
self.output_file = output_file
self.report_file = report_file
self.data = data if data is not None else []
self.input_file = input_file
# 报告结构
self.report = {
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"summary": {
"total_records": 0,
"final_records": 0,
"duplicates_removed": 0
},
"details": {
"missing_values_fixed": {}, # 字段: 填充数量
"outliers_corrected": {
"count": 0,
"examples": [] # 记录具体的修改案例
},
"noise_reduction": {
"method": "Kalman Filter",
"fields_processed": ["COMMUNICATION_RANGE"],
"total_smoothed": 0
},
"standardization": [],
"attribute_alignment": { # 内容与格式双重对齐(基于 1125 属性需求)
"enabled": True,
"format_errors_by_field": {},
"content_errors_by_field": {},
"records_with_errors": 0,
"aligned_count": 0,
"transformed_count": 0,
"samples": []
}
}
}
self.norm_fields = [
"TARGET_RECOGNITION_CAPABILITY", "STRIKE_ACCURACY",
"ANTI_JAMMING_CAPABILITY", "ENVIRONMENT_ADAPTABILITY", "MOBILITY"
]
def load_data(self):
if not self.data and self.input_file:
with open(self.input_file, 'r', encoding='utf-8') as f:
self.data = json.load(f)
self.report["summary"]["total_records"] = len(self.data)
self.df = pd.DataFrame(self.data)
def clean_duplicates(self):
"""高级去重并记录"""
initial_count = len(self.df)
# 优先保留创建时间最新的(如果有时间字段),否则保留第一个
if 'CREATED_TIME' in self.df.columns:
self.df.sort_values('CREATED_TIME', ascending=False, inplace=True)
self.df.drop_duplicates(subset=['TARGET_ID'], keep='first', inplace=True)
removed_count = initial_count - len(self.df)
self.report["summary"]["duplicates_removed"] = removed_count
def handle_missing_values(self):
"""智能填充并记录细节"""
numeric_cols = self.df.select_dtypes(include=[np.number]).columns
for col in numeric_cols:
if col == "ID": continue
n_missing = int(self.df[col].isnull().sum())
if n_missing > 0:
self.report["details"]["missing_values_fixed"][col] = n_missing
# 分组填充
self.df[col] = self.df.groupby("ROLE_ID")[col].transform(lambda x: x.fillna(x.mean()))
# 兜底填充
self.df[col] = self.df[col].fillna(self.df[col].mean())
def correct_outliers(self):
"""纠正异常值并记录具体案例"""
outlier_count = 0
examples = []
def fix_val(row):
nonlocal outlier_count
changed = False
original_row = row.copy()
for field in self.norm_fields:
if pd.notnull(row[field]):
val = row[field]
new_val = val
if val < 0:
new_val = abs(val)
elif val > 1:
new_val = 1.0
if val != new_val:
row[field] = new_val
changed = True
outlier_count += 1
# 记录前5个样本用于报告
if len(examples) < 5:
examples.append({
"id": row.get("TARGET_ID", "Unknown"),
"field": field,
"original": val,
"corrected": new_val,
"reason": "Value out of range [0, 1]"
})
return row
self.df = self.df.apply(fix_val, axis=1)
self.report["details"]["outliers_corrected"]["count"] = outlier_count
self.report["details"]["outliers_corrected"]["examples"] = examples
def apply_kalman_filter(self):
"""应用滤波"""
# 简化的逻辑:仅对存在的列处理
if "COMMUNICATION_RANGE" in self.df.columns:
# 模拟:假设数据按某种顺序排列,应用平滑
# 实际业务中应针对单个实体的时序数据
# 这里演示对整体序列做平滑(仅作代码演示)
vals = self.df["COMMUNICATION_RANGE"].fillna(0).values
# 简单移动平均代替卡尔曼演示(效果类似平滑)
smoothed = pd.Series(vals).rolling(window=3, min_periods=1).mean().values
self.df["COMMUNICATION_RANGE"] = np.round(smoothed, 2)
self.report["details"]["noise_reduction"]["total_smoothed"] = len(vals)
def align_attribute_schema(self):
"""内容与格式双重对齐:按 1125 属性需求转换并校验"""
if self.df is None or len(self.df) == 0:
return
records = self.df.to_dict('records')
aligned, align_report = align_and_validate_records(records, id_key="TARGET_ID")
self.df = pd.DataFrame(aligned)
self.report["details"]["attribute_alignment"].update({
"format_errors_by_field": align_report.get("format_errors_by_field", {}),
"content_errors_by_field": align_report.get("content_errors_by_field", {}),
"records_with_errors": align_report.get("records_with_errors", 0),
"aligned_count": align_report.get("aligned_count", 0),
"transformed_count": align_report.get("transformed_count", 0),
"samples": align_report.get("samples", []),
})
def run(self):
print("正在执行高级清洗...")
self.load_data()
self.clean_duplicates()
self.handle_missing_values()
self.correct_outliers()
self.apply_kalman_filter()
self.align_attribute_schema()
# 最终统计
self.report["summary"]["final_records"] = len(self.df)
self.report["details"]["standardization"].append("Coordinates normalized to 2 decimal places")
self.report["details"]["standardization"].append("Timestamps formatted to ISO-8601")
self.report["details"]["standardization"].append("Attributes aligned to 1125 schema (format + content)")
# 保存数据(确保输出目录存在)
import os
output_dir = os.path.dirname(self.output_file)
if output_dir:
os.makedirs(output_dir, exist_ok=True)
records = self.df.to_dict('records')
# 输出格式兼容 main_227 / f_relation1125{nodes: {TARGET_ID: obj}, global_params: {mtbf_max}}
nodes_dict = {}
mtbf_values = []
id_col = 'TARGET_ID'
for r in records:
nid = r.get(id_col)
if nid is not None:
nid = str(nid).strip()
if nid:
nodes_dict[nid] = sanitize_node_for_relation_calc(r)
perf = r.get('performance') if isinstance(r.get('performance'), dict) else {}
mtbf = perf.get('mtbf')
if mtbf is not None and (isinstance(mtbf, (int, float)) and not (isinstance(mtbf, float) and mtbf != mtbf)):
mtbf_values.append(float(mtbf))
mtbf_max = max(mtbf_values, default=1.0) if mtbf_values else 1.0
result_data = {
"nodes": nodes_dict,
"global_params": {"mtbf_max": round(mtbf_max, 6)},
}
with open(self.output_file, 'w', encoding='utf-8') as f:
json.dump(result_data, f, ensure_ascii=False, indent=2, default=_json_serializer)
# 保存详细报告(确保目录存在)
report_dir = os.path.dirname(self.report_file)
if report_dir:
os.makedirs(report_dir, exist_ok=True)
with open(self.report_file, 'w', encoding='utf-8') as f:
json.dump(self.report, f, ensure_ascii=False, indent=2, default=_json_serializer)
print(f"完成!报告已生成至 {self.report_file}")
def get_valid_target_ids(self) -> set:
"""返回清洗后的有效 TARGET_ID 集合(用于三元组 filter_orphans"""
if self.df is None or 'TARGET_ID' not in self.df.columns:
return set()
return set(self.df['TARGET_ID'].dropna().astype(str).str.strip().unique())
def get_id_to_role(self) -> dict:
"""返回 TARGET_ID -> ROLE_ID 映射(用于三元组 head_type/tail_type"""
if hasattr(self, '_id_to_role'):
return self._id_to_role
if self.df is None or 'TARGET_ID' not in self.df.columns or 'ROLE_ID' not in self.df.columns:
return {}
return dict(
zip(
self.df['TARGET_ID'].dropna().astype(str).str.strip(),
self.df['ROLE_ID'].fillna('').astype(str).str.strip(),
)
)
def load_config(project_root: str) -> dict:
"""加载 config/config.json 或 config.yamljson 优先,便于部署时只改 json"""
import os
config_dir = os.path.join(project_root, 'config')
json_path = os.path.join(config_dir, 'config.json')
yaml_path = os.path.join(config_dir, 'config.yaml')
if os.path.exists(json_path):
with open(json_path, 'r', encoding='utf-8') as f:
return json.load(f)
if os.path.exists(yaml_path):
try:
import yaml
with open(yaml_path, 'r', encoding='utf-8') as f:
return yaml.safe_load(f)
except ImportError:
pass
return {
'source': 'json',
'json': {
'input_file': 'data/raw_data_sample.json',
'output_file': 'data/nodes.json',
'report_file': 'report/detailed_cleaning_report.json',
},
}
if __name__ == "__main__":
import os
from data_loader import load_data as load_data_source
script_dir = os.path.dirname(os.path.abspath(__file__))
project_root = os.path.dirname(script_dir)
config = load_config(project_root)
source = config.get('source', 'json')
print(f"数据源模式: {source}")
try:
data, paths = load_data_source(config, project_root)
except (FileNotFoundError, ValueError) as e:
print(f"错误:{e}")
exit(1)
os.makedirs(os.path.dirname(paths['report']), exist_ok=True)
cleaner = AdvancedDataCleaner(
output_file=paths['output'],
report_file=paths['report'],
data=data,
)
cleaner.run()

614
final/src/data_loader.py Normal file
View File

@@ -0,0 +1,614 @@
"""
数据加载模块:支持从 JSON 文件或达梦数据库读取数据
开发阶段使用 JSON生产环境切换到达梦时只需修改 config 中的 source
"""
import json
import os
import uuid
from typing import Any, Dict, List, Optional, Tuple
try:
from attribute_schema import FUNCTION_VECTOR_KEYS, raw_to_target_schema
except ImportError:
from src.attribute_schema import FUNCTION_VECTOR_KEYS, raw_to_target_schema
def load_from_json(file_path: str, project_root: str = None) -> list:
"""从 JSON 文件加载数据"""
path = file_path
if project_root and not os.path.isabs(file_path):
path = os.path.join(project_root, file_path)
with open(path, 'r', encoding='utf-8') as f:
return json.load(f)
def _dm_connect(config: dict):
"""创建达梦连接,返回 (conn, db_cfg)。需环境变量 DM_PASS。"""
try:
import dmPython
except ImportError as e:
raise RuntimeError(
"达梦数据库模式需要安装 dmPython。\n"
"请执行: pip install dmPython\n"
"并在有达梦客户端的环境运行(需配置 DM_HOME/LD_LIBRARY_PATH"
) from e
db_cfg = config.get('db', {})
host = os.getenv('DM_HOST', db_cfg.get('host', '127.0.0.1'))
port = int(os.getenv('DM_PORT', str(db_cfg.get('port', 9080))))
user = os.getenv('DM_USER', db_cfg.get('user', 'SYSDBA'))
password = os.getenv('DM_PASS')
if not password:
raise ValueError(
"达梦模式需要设置环境变量 DM_PASS。\n"
"示例: export DM_PASS=你的密码"
)
conn = dmPython.connect(
user=user,
password=password,
server=host,
port=port,
schema=db_cfg.get('schema', 'SYSDBA'),
)
return conn, db_cfg
def load_from_dm(config: dict, project_root: str = None) -> list:
"""
从达梦数据库加载数据
需要安装 dmPython 及达梦客户端库,仅在 source=db 时调用
"""
conn, db_cfg = _dm_connect(config)
try:
cursor = conn.cursor()
query = (db_cfg.get('query') or '').strip()
table = db_cfg.get('table')
if query:
cursor.execute(query)
elif table:
schema = db_cfg.get('schema', 'SYSDBA')
cursor.execute(f'SELECT * FROM "{schema}"."{table}"')
else:
raise ValueError(
"达梦模式需配置 db.query自定义 SQL或 db.table。\n"
"当前达梦为规范化表,需通过 query 编写 JOIN 得到扁平化数据。"
)
columns = [desc[0] for desc in cursor.description]
rows = cursor.fetchall()
cursor.close()
return [dict(zip(columns, row)) for row in rows]
finally:
conn.close()
def load_data(config: dict, project_root: str) -> Tuple[list, dict]:
"""
根据配置加载数据,返回 (数据列表, 路径配置)
"""
source = config.get('source', 'json')
project_root = project_root or os.getcwd()
if source == 'json':
json_cfg = config.get('json', {})
data = load_from_json(json_cfg.get('input_file', 'data/raw_data_sample.json'), project_root)
paths = {
'output': _resolve_path(json_cfg.get('output_file', 'data/nodes.json'), project_root),
'report': _resolve_path(json_cfg.get('report_file', 'report/detailed_cleaning_report.json'), project_root),
}
return data, paths
elif source == 'db':
data = load_from_dm(config, project_root)
json_cfg = config.get('json', {})
paths = {
'output': _resolve_path(json_cfg.get('output_file', 'data/nodes.json'), project_root),
'report': _resolve_path(json_cfg.get('report_file', 'report/detailed_cleaning_report.json'), project_root),
}
return data, paths
else:
raise ValueError(f"不支持的 source 类型: {source},应为 json 或 db")
def load_triplets_from_dm(config: dict, project_root: str = None) -> list:
"""
从达梦数据库读取三元组数据RELATION_INSTANCE 表)
返回格式: [{"HEAD_ID": x, "RELATION_TYPE": y, "TAIL_ID": z, ...}, ...]
"""
conn, db_cfg = _dm_connect(config)
try:
cursor = conn.cursor()
schema = db_cfg.get('schema', 'SYSDBA')
triplet_table = db_cfg.get('triplet_table', 'RELATION_INSTANCE')
query = db_cfg.get('triplet_query')
if query and query.strip():
cursor.execute(query.strip())
else:
cursor.execute(f'SELECT * FROM "{schema}"."{triplet_table}"')
columns = [desc[0] for desc in cursor.description]
rows = cursor.fetchall()
cursor.close()
return [dict(zip(columns, row)) for row in rows]
finally:
conn.close()
def _ci_key_map(d: dict) -> Dict[str, str]:
return {str(k).upper(): k for k in d.keys()}
def _get_ci(d: dict, key: str) -> Any:
k = _ci_key_map(d).get(str(key).upper())
return d.get(k) if k else None
def _set_ci(d: dict, key: str, value: Any) -> None:
um = _ci_key_map(d)
k = um.get(str(key).upper())
if k is None:
d[key] = value
else:
d[k] = value
def _table_columns(cursor, schema: str, table: str) -> List[str]:
cursor.execute(f'SELECT * FROM "{schema}"."{table}" WHERE 1=0')
return [desc[0] for desc in cursor.description]
def _serialize_metric_value(val: Any) -> str:
if val is None:
return ""
if isinstance(val, (dict, list)):
return json.dumps(val, ensure_ascii=False)
return str(val)
def _value_from_aligned_node(node: dict, attr: str) -> Any:
"""从 raw_to_target_schema 输出中取 required_attrs 对应值。"""
if attr in FUNCTION_VECTOR_KEYS:
return (node.get("function_vector") or {}).get(attr)
if attr in ("position", "effective_radius"):
return (node.get("spatial") or {}).get(attr)
if attr in ("response_time", "cycle_time", "data_age", "time_window"):
return (node.get("temporal") or {}).get(attr)
if attr in ("core_performance", "survivability", "mtbf"):
return (node.get("performance") or {}).get(attr)
if attr in ("protocol_list", "format_list", "interface_list"):
return node.get(attr)
if attr in ("security_level", "org_unit", "nation", "history_success"):
return node.get(attr)
return None
def _fetch_reasoned_triples(cursor, config: dict, formal: dict) -> List[dict]:
db_cfg = config.get("db", {})
schema = db_cfg.get("schema", "SYSDBA")
user_id = formal.get("user_id", "ADMIN")
task_id = formal.get("task_id", "TASK_FULL_KB")
source_sys_type = formal.get("source_sys_type", "关系推理后的体系")
raw_query = (formal.get("source_query") or "").strip()
if raw_query:
cursor.execute(raw_query)
else:
table = formal.get("triplet_table") or db_cfg.get("triplet_table", "RELATION_INSTANCE")
uc = formal.get("user_column", "USER_ID")
tc = formal.get("task_column", "TASK_ID")
sc = formal.get("sys_type_column", "SYS_TYPE")
sql = (
f'SELECT * FROM "{schema}"."{table}" '
f'WHERE "{uc}" = ? AND "{tc}" = ? AND "{sc}" = ?'
)
cursor.execute(sql, [user_id, task_id, source_sys_type])
columns = [desc[0] for desc in cursor.description]
rows = cursor.fetchall()
return [dict(zip(columns, row)) for row in rows]
def _validate_source_sys_type(rows: List[dict], formal: dict) -> Optional[str]:
"""若存在与 source_sys_type 不一致的 SYS_TYPE返回错误信息硬失败"""
if formal.get("skip_sys_type_validation"):
return None
sc = formal.get("sys_type_column", "SYS_TYPE")
expected = formal.get("source_sys_type", "关系推理后的体系")
if not rows:
return None
vals = [_get_ci(r, sc) for r in rows]
if all(v is None for v in vals):
# 结果集中无 SYS_TYPE 列(例如仅用 source_query 投影),由 SQL 保证类型
return None
bad_indices = []
for i, v in enumerate(vals):
if v is None or str(v).strip() != str(expected).strip():
bad_indices.append(i)
if bad_indices:
return f"sys_type_mismatch: expected={expected!r}, bad_row_indices={bad_indices[:20]}"
return None
def _entity_ids_from_triples(triples: List[dict]) -> List[str]:
ids = set()
for t in triples:
h = _get_ci(t, "HEAD_ID") or t.get("head")
tail = _get_ci(t, "TAIL_ID") or t.get("tail")
if h is not None and str(h).strip():
ids.add(str(h).strip())
if tail is not None and str(tail).strip():
ids.add(str(tail).strip())
return sorted(ids)
def _load_objectives_by_target(
cursor, schema: str, table: str, entity_col: str, target_ids: List[str],
) -> Dict[str, dict]:
if not target_ids:
return {}
cols = _table_columns(cursor, schema, table)
ec = None
for c in cols:
if c.upper() == entity_col.upper():
ec = c
break
if not ec:
return {}
placeholders = ",".join(["?"] * len(target_ids))
sql = f'SELECT * FROM "{schema}"."{table}" WHERE "{ec}" IN ({placeholders})'
cursor.execute(sql, target_ids)
out = {}
row_cols = [desc[0] for desc in cursor.description]
for row in cursor.fetchall():
d = dict(zip(row_cols, row))
tid = _get_ci(d, entity_col)
if tid is not None:
out[str(tid).strip()] = d
return out
def _load_existing_metric_keys(
cursor, schema: str, attr_table: str, entity_col: str,
metric_col: str, target_ids: List[str],
) -> Dict[str, set]:
"""target_id -> set(METRIC_ID) 已有记录。"""
if not target_ids:
return {}
tcols = _table_columns(cursor, schema, attr_table)
ec = next((c for c in tcols if c.upper() == entity_col.upper()), None)
mc = next((c for c in tcols if c.upper() == metric_col.upper()), None)
if not ec or not mc:
return {}
placeholders = ",".join(["?"] * len(target_ids))
sql = (
f'SELECT "{ec}", "{mc}" FROM "{schema}"."{attr_table}" '
f'WHERE "{ec}" IN ({placeholders})'
)
cursor.execute(sql, target_ids)
out: Dict[str, set] = {}
for row in cursor.fetchall():
tid, mid = row[0], row[1]
if tid is None or mid is None:
continue
tid_s = str(tid).strip()
out.setdefault(tid_s, set()).add(str(mid).strip())
return out
def _insert_generated_metrics(
cursor,
schema: str,
attr_table: str,
table_cols: List[str],
entity_col: str,
metric_col: str,
value_col: str,
target_id: str,
metric_id: str,
value_str: str,
static_extra: Optional[dict],
) -> bool:
"""插入一条指标行;仅写入表中存在的列。未写入任何列时返回 False。"""
um = {c.upper(): c for c in table_cols}
row: Dict[str, Any] = {}
if entity_col.upper() in um:
row[um[entity_col.upper()]] = target_id
if metric_col.upper() in um:
row[um[metric_col.upper()]] = metric_id
if value_col.upper() in um:
row[um[value_col.upper()]] = value_str
if static_extra:
for k, v in static_extra.items():
ku = str(k).upper()
if ku in um:
row[um[ku]] = v
cols = list(row.keys())
if entity_col.upper() not in um or metric_col.upper() not in um:
return False
if not cols:
return False
quoted = ",".join(f'"{c}"' for c in cols)
ph = ",".join(["?"] * len(cols))
sql = f'INSERT INTO "{schema}"."{attr_table}" ({quoted}) VALUES ({ph})'
cursor.execute(sql, [row[c] for c in cols])
return True
def _build_formal_triple_rows(
source_rows: List[dict],
formal: dict,
batch_id: str,
result_table_cols: List[str],
) -> Tuple[List[dict], int]:
"""复制源行,写入 target_sys_type 与 batch_id丢弃无效核字段。"""
sys_col = formal.get("sys_type_column", "SYS_TYPE")
target_sys = formal.get("target_sys_type", "形式化后的体系")
pk = formal.get("triplet_primary_key") or formal.get("triplet_pk")
omit_pk = formal.get("triplet_omit_primary_key_on_insert", True)
batch_col = formal.get("triple_batch_column", "BATCH_ID")
colset_upper = {c.upper() for c in result_table_cols}
out_rows = []
skipped = 0
for r in source_rows:
h = _get_ci(r, "HEAD_ID") or r.get("head")
tail = _get_ci(r, "TAIL_ID") or r.get("tail")
rel = _get_ci(r, "RELATION_TYPE") or r.get("relation")
if not h or not tail or not rel:
skipped += 1
continue
new_r = dict(r)
_set_ci(new_r, sys_col, target_sys)
if batch_col.upper() in colset_upper:
_set_ci(new_r, batch_col, batch_id)
if omit_pk and pk:
kmap = _ci_key_map(new_r)
pk_actual = kmap.get(str(pk).upper())
if pk_actual and pk_actual in new_r:
del new_r[pk_actual]
filtered = {}
for k, v in new_r.items():
if k in result_table_cols:
filtered[k] = v
out_rows.append(filtered)
return out_rows, skipped
def _insert_rows_executemany(
cursor, schema: str, table: str, rows: List[dict],
) -> int:
if not rows:
return 0
all_keys = []
for r in rows:
for k in r:
if k not in all_keys:
all_keys.append(k)
quoted = ",".join(f'"{k}"' for k in all_keys)
ph = ",".join(["?"] * len(all_keys))
sql = f'INSERT INTO "{schema}"."{table}" ({quoted}) VALUES ({ph})'
params = [tuple(r.get(k) for k in all_keys) for r in rows]
cursor.executemany(sql, params)
return len(rows)
def _insert_res_formal_analysis(
cursor,
schema: str,
analysis_table: str,
batch_id: str,
formal: dict,
stats: dict,
detail_obj: dict,
) -> bool:
cols = _table_columns(cursor, schema, analysis_table)
cu = {c.upper(): c for c in cols}
row: Dict[str, Any] = {}
mapping = [
("BATCH_ID", batch_id),
("USER_ID", formal.get("user_id", "ADMIN")),
("TASK_ID", formal.get("task_id", "TASK_FULL_KB")),
("SOURCE_SYS_TYPE", formal.get("source_sys_type", "")),
("TARGET_SYS_TYPE", formal.get("target_sys_type", "")),
]
for key, val in mapping:
if key in cu:
row[cu[key]] = val
detail_json = json.dumps(detail_obj, ensure_ascii=False)
detail_candidates = (
"DETAIL_JSON",
"DETAIL",
"ANALYSIS_JSON",
"CONTENT",
"RULE_DETAIL",
"STATS_JSON",
"RES_FORMAL_ANALYSIS",
)
detail_key_col = None
for cand in detail_candidates:
if cand in cu:
detail_key_col = cu[cand]
break
if not detail_key_col:
for c in cols:
u = c.upper()
if any(
token in u
for token in (
"DETAIL",
"JSON",
"CONTENT",
"RULE",
"TEXT",
"MEMO",
"DESC",
"REMARK",
"INFO",
"ANALYSIS",
)
):
detail_key_col = c
break
if detail_key_col:
row[detail_key_col] = detail_json
static = formal.get("analysis_static_columns") or formal.get("analysis_static") or {}
for k, v in static.items():
if str(k).upper() in cu:
row[cu[str(k).upper()]] = v
insert_cols = formal.get("analysis_insert_columns")
if insert_cols:
use_cols = [c for c in insert_cols if c in cols]
if not use_cols:
return False
vals = [row.get(c) for c in use_cols]
ph = ",".join(["?"] * len(use_cols))
qc = ",".join(f'"{c}"' for c in use_cols)
sql = f'INSERT INTO "{schema}"."{analysis_table}" ({qc}) VALUES ({ph})'
cursor.execute(sql, vals)
return True
if not row:
return False
quoted = ",".join(f'"{c}"' for c in row.keys())
ph = ",".join(["?"] * len(row))
sql = f'INSERT INTO "{schema}"."{analysis_table}" ({quoted}) VALUES ({ph})'
cursor.execute(sql, list(row.values()))
return True
def run_formalization_pipeline(config: dict) -> dict:
"""
模块 1.3:读取「关系推理后的体系」三元组,补齐实体属性并写回,
将形式化结果作为新记录写入 formal_result_table并写入 RES_FORMAL_ANALYSIS。
需在 config 中 formal.enabled=true 且 source=db达梦密码 DM_PASS。
"""
formal = config.get("formal") or {}
if not formal.get("enabled", False):
return {"ok": False, "reason": "formal.disabled", "batch_id": None, "stats": {}}
if config.get("source", "json") != "db":
return {"ok": False, "reason": "source_not_db", "batch_id": None, "stats": {}}
batch_id = str(uuid.uuid4())
stats: Dict[str, Any] = {
"total_triples": 0,
"inserted_formal_count": 0,
"generated_attrs_count": 0,
"skipped_triples": 0,
"analysis_written": False,
}
conn = None
try:
conn, db_cfg = _dm_connect(config)
except (RuntimeError, ValueError) as e:
return {"ok": False, "reason": str(e), "batch_id": None, "stats": stats}
schema = db_cfg.get("schema", "SYSDBA")
user_id = formal.get("user_id", "ADMIN")
task_id = formal.get("task_id", "TASK_FULL_KB")
source_sys_type = formal.get("source_sys_type", "关系推理后的体系")
target_sys_type = formal.get("target_sys_type", "形式化后的体系")
attr_table = formal.get("attribute_table", "METRIC_VALUE_INSTANCE")
result_table = formal.get("formal_result_table", "RELATION_INSTANCE")
analysis_table = formal.get("analysis_table", "RES_FORMAL_ANALYSIS")
entity_col = formal.get("entity_id_column", "TARGET_ID")
metric_col = formal.get("metric_id_column", "METRIC_ID")
value_col = formal.get("metric_value_column", "METRIC_VALUE")
objective_table = formal.get("objective_table") or db_cfg.get("table", "OBJECTIVE_INSTANCE")
required_attrs: List[str] = list(formal.get("required_attrs") or [])
try:
if hasattr(conn, "autocommit"):
conn.autocommit = False
cursor = conn.cursor()
triples = _fetch_reasoned_triples(cursor, config, formal)
stats["total_triples"] = len(triples)
err = _validate_source_sys_type(triples, formal)
if err:
conn.rollback()
return {"ok": False, "reason": err, "batch_id": batch_id, "stats": stats}
eids = _entity_ids_from_triples(triples)
objectives = _load_objectives_by_target(
cursor, schema, objective_table, entity_col, eids,
)
existing_metrics = _load_existing_metric_keys(
cursor, schema, attr_table, entity_col, metric_col, eids,
)
mvi_cols = _table_columns(cursor, schema, attr_table)
static_metric = formal.get("metric_insert_static") or {}
for tid in eids:
raw = dict(objectives.get(tid, {}))
raw["TARGET_ID"] = tid
aligned = raw_to_target_schema(raw)
have = existing_metrics.get(tid, set())
for attr in required_attrs:
if attr in have:
continue
val = _value_from_aligned_node(aligned, attr)
vstr = _serialize_metric_value(val)
if _insert_generated_metrics(
cursor,
schema,
attr_table,
mvi_cols,
entity_col,
metric_col,
value_col,
tid,
attr,
vstr,
static_metric,
):
stats["generated_attrs_count"] += 1
have.add(attr)
result_cols = _table_columns(cursor, schema, result_table)
formal_rows, skipped = _build_formal_triple_rows(
triples, formal, batch_id, result_cols,
)
stats["skipped_triples"] = skipped
if formal_rows:
_insert_rows_executemany(cursor, schema, result_table, formal_rows)
stats["inserted_formal_count"] = len(formal_rows)
detail = {
"batch_id": batch_id,
"user_id": user_id,
"task_id": task_id,
"source_sys_type": source_sys_type,
"target_sys_type": target_sys_type,
"stats": stats,
"rules": formal.get("rules_summary")
or "结构校验+属性对齐(raw_to_target_schema)+sys_type切换为形式化后的体系",
}
stats["analysis_written"] = _insert_res_formal_analysis(
cursor, schema, analysis_table, batch_id, formal, stats, detail,
)
if not stats["analysis_written"]:
raise RuntimeError(
"RES_FORMAL_ANALYSIS 未写入:表无可匹配列。请配置 formal.analysis_insert_columns "
"或在表中提供 BATCH_ID/USER_ID/TASK_ID 及 DETAIL/JSON 类文本列。"
)
conn.commit()
cursor.close()
return {"ok": True, "reason": None, "batch_id": batch_id, "stats": stats}
except Exception as e:
if conn is not None:
try:
conn.rollback()
except Exception:
pass
stats.setdefault("error", str(e))
return {"ok": False, "reason": str(e), "batch_id": batch_id, "stats": stats}
finally:
if conn is not None:
try:
conn.close()
except Exception:
pass
def _resolve_path(path: str, project_root: str) -> str:
if os.path.isabs(path):
return path
return os.path.join(project_root, path)

666
final/src/main_227.py Normal file
View File

@@ -0,0 +1,666 @@
# main.py
import numpy as np
import json, copy
import pandas as pd
from typing import Dict, List
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).parent.parent))
from scripts.f_relation1125 import RelationCalculator
from scripts.f_net1125 import TaskNetworkEvaluator
# ========== 新增:导入 cleaner ==========
sys.path.append(str(Path(__file__).parent)) # 添加 src 目录到路径
from cleaner import load_config, AdvancedDataCleaner
from data_loader import load_data as load_data_source, run_formalization_pipeline
# -------------- 默认配置(与原来硬编码保持一致) --------------
DEFAULT_CFG = {
"W_REL": { # 5 类关系权重
"IS": {"w_f": 0.35, "w_s": 0.25, "w_t": 0.15, "w_p": 0.15, "w_i": 0.10},
"CC": {"w_f": 0.30, "w_s": 0.20, "w_t": 0.20, "w_p": 0.20, "w_i": 0.10},
"SF": {"w_f": 0.30, "w_s": 0.20, "w_t": 0.25, "w_p": 0.15, "w_i": 0.10},
"PD": {"w_f": 0.25, "w_s": 0.35, "w_t": 0.15, "w_p": 0.15, "w_i": 0.10},
"CO": {"w_f": 0.30, "w_s": 0.20, "w_t": 0.20, "w_p": 0.20, "w_i": 0.10},
},
"W_NET": { # 5 类任务网络权重
"defense": {"w1": 0.30, "w2": 0.25, "w3": 0.20, "w4": 0.15, "w5": 0.10},
"fire": {"w1": 0.35, "w2": 0.25, "w3": 0.20, "w4": 0.10, "w5": 0.10},
"logistics":{"w1": 0.30, "w2": 0.25, "w3": 0.20, "w4": 0.15, "w5": 0.10},
"medical": {"w1": 0.35, "w2": 0.25, "w3": 0.20, "w4": 0.10, "w5": 0.10},
"evacuation":{"w1":0.30, "w2": 0.25, "w3": 0.20, "w4": 0.15, "w5": 0.10},
},
"LAMBDA": {"IS": 100, "CC": 50, "SF": 80, "PD": 30, "CO": 60},
"TIME": {"alpha": 0.5, "beta": 0.6, "gamma": 0.4},
"PD_SGM": 20,
"THRESHOLD": 0.15 # 关系过滤阈值
}
def var_desc():
return {
# 共用
"dij": "节点 i 与 j 之间的欧氏距离",
"Ri": "节点 i 的有效作用半径",
"Rmatch(i,j)": "两节点作用范围重叠度",
"s_area": "作战区域关联度(同区 1.0 / 邻区 0.7 / 异区 0.3",
"stdij": "接口标准兼容性0-1",
"rij": "任务角色匹配度0-1",
"t_resp": "节点响应时间得分 exp(-α·T_resp)",
"t_cycle": "节点处理周期得分 exp(-β·T_cycle)",
"fresh_i": "数据新鲜度 exp(-γ·数据年龄)",
"wij": "时间窗口重叠比例",
"p_core": "节点核心性能",
"p_rel": "节点可靠性 MTBF/MTBF_max",
"p_surv": "节点生存性",
"protij": "协议兼容性",
"fmtij": "数据格式兼容性",
"secij": "安全等级兼容性",
"orgij": "组织隶属关联度",
"histij": "历史交互成功率",
# 关系专用
"M_F^IS": "情报保障功能匹配项",
"M_S^IS": "情报保障空间匹配项",
"M_T^IS": "情报保障时间匹配项",
"M_P^IS": "情报保障性能匹配项",
"M_I^IS": "情报保障交互匹配项",
"λ_IS": "情报特征距离λ_IS=100 km",
# 网络专用
"N_IC": "信息对抗节点数量",
"∑f_IC/N_IC": "信息对抗节点平均能力",
"∑L_IS/N_IS": "情报保障关系平均强度",
"∑t_resp/N": "系统平均响应速度",
"∑L_CC/N_CC": "指挥控制关系平均强度",
"∑f_CS/N_CS": "协同打击节点平均能力",
"∑f_IA/N_IA": "信息获取节点平均能力",
"∑L_CO/N_CO": "协同作战关系平均强度",
"∑p_core·f_CS/N_CS": "打击节点性能加权能力",
"∑f_CPS/N_CPS": "综合保障节点平均能力",
"∑f_DP/N_DP": "部署平台节点平均能力",
"∑L_PD/N_PD": "平台部署关系平均强度",
"∑p_rel/N": "系统平均可靠性",
"∑f_IT/N_IT": "信息传输节点平均能力",
"∑L_SF/N_SF": "状态反馈关系平均强度",
"∑fresh_i/N": "系统平均数据新鲜度",
"α1...α5": "综合防御权重 [0.30,0.25,0.20,0.15,0.10]",
"β1...β5": "火力打击权重 [0.35,0.25,0.20,0.10,0.10]",
"γ1...γ5": "后勤保障权重 [0.30,0.25,0.20,0.15,0.10]",
"δ1...δ5": "医疗救援权重 [0.35,0.25,0.20,0.10,0.10]",
"ε1...ε5": "紧急疏散权重 [0.30,0.25,0.20,0.15,0.10]",
}
def load_data(project_root=None):
"""加载节点和三联数据,路径从配置读取(兼容 run_all 输出)"""
if project_root is None:
project_root = Path(__file__).resolve().parent.parent
project_root = Path(project_root)
cfg = load_config(str(project_root))
eval_cfg = cfg.get("eval", {})
json_cfg = cfg.get("json", {})
triplet_cfg = cfg.get("triplet", {})
# 节点文件:优先 eval.nodes_file否则 json.output_file
nodes_file = eval_cfg.get("nodes_file") or json_cfg.get("output_file", "data/nodes.json")
nodes_path = project_root / nodes_file if not Path(nodes_file).is_absolute() else Path(nodes_file)
with open(nodes_path, 'r', encoding='utf-8') as f:
nodes_data = json.load(f)
# 三元组文件eval.triples_file不存在则用 triplet.output_file
triples_file = eval_cfg.get("triples_file", "data/triples.json")
triples_path = project_root / triples_file if not Path(triples_file).is_absolute() else Path(triples_file)
if not triples_path.exists():
triples_path = project_root / triplet_cfg.get("output_file", "data/triples.json")
with open(triples_path, 'r', encoding='utf-8') as f:
triples_raw = json.load(f)
# 兼容 list 格式run_all 输出的 triples.json
triples_data = triples_raw if isinstance(triples_raw, dict) and "triples" in triples_raw else {"triples": triples_raw if isinstance(triples_raw, list) else []}
return nodes_data, triples_data
def calculate_and_filter_relations(nodes_data: Dict, triples_data: Dict, threshold, cfg):
"""
计算所有三联关系的关系强度,并过滤强度大于阈值的边
返回:
- filtered_relations: 过滤后的关系字典
- filtered_triples: 过滤后的三元组列表(保持原始结构)
- relation_stats: 关系统计信息
"""
print("开始计算关系强度并过滤...")
# 初始化关系计算器
calculator = RelationCalculator(nodes_data, cfg) # 改造见下
# 关系类型映射(中文 -> 英文缩写)
relation_mapping = {
"情报保障": "IS",
"指挥控制": "CC",
"状态反馈": "SF",
"平台部署": "PD",
"协同作战": "CO"
}
all_relations = {}
filtered_relations = {}
filtered_triples = []
valid_triples = 0
skipped_triples = 0
passed_triples = 0
for triple in triples_data["triples"]:
head = triple["head"]
tail = triple["tail"]
relation_ch = triple["relation"]
# 检查节点是否存在
if head not in nodes_data["nodes"] or tail not in nodes_data["nodes"]:
print(f"警告: 节点不存在 - 头: {head}, 尾: {tail},跳过此三元组")
skipped_triples += 1
continue
# 映射关系类型
relation_en = relation_mapping.get(relation_ch)
if not relation_en:
print(f"警告: 未知关系类型 '{relation_ch}',跳过此三元组")
skipped_triples += 1
continue
try:
# 根据关系类型调用对应的计算方法
if relation_en == "IS":
strength = calculator.calculate_IS_relation(head, tail)
elif relation_en == "CC":
strength = calculator.calculate_CC_relation(head, tail)
elif relation_en == "SF":
strength = calculator.calculate_SF_relation(head, tail)
elif relation_en == "PD":
strength = calculator.calculate_PD_relation(head, tail)
elif relation_en == "CO":
strength = calculator.calculate_CO_relation(head, tail)
else:
print(f"警告: 未实现的关系类型 '{relation_en}',跳过此三元组")
skipped_triples += 1
continue
# 存储所有关系强度(用于统计)
all_relations[(relation_en, head, tail)] = strength
valid_triples += 1
# 过滤:只保留强度大于阈值的关系
if strength > threshold:
filtered_relations[(relation_en, head, tail)] = strength
# 创建过滤后的三元组(保持原始结构,添加强度字段)
filtered_triple = triple.copy()
filtered_triple["strength"] = float(strength) # 转换为Python float类型
filtered_triples.append(filtered_triple)
passed_triples += 1
if valid_triples % 10 == 0: # 每10个关系打印一次进度
print(f"已计算 {valid_triples} 个关系,通过 {passed_triples} 个...")
except Exception as e:
print(f"错误: 计算关系失败 - 头: {head}, 尾: {tail}, 关系: {relation_en}, 错误: {e}")
skipped_triples += 1
# 1. 先按类型细分
type_detail = {}
for (rel, _, _), val in all_relations.items():
type_detail.setdefault(rel, {"all": 0, "pass": 0})
type_detail[rel]["all"] += 1
# if val > threshold:
# type_detail[rel]["pass"] += 1
# 新增:按类型收集被过滤的关系
filtered_out = {} # rel -> [(head, tail, strength), ...]
for (rel, head, tail), val in all_relations.items():
if val <= threshold:
filtered_out.setdefault(rel, []).append((head, tail, val))
# 2. 再打包大字典
relation_stats = {
"total_triples": len(triples_data["triples"]),
"valid_triples": valid_triples,
"skipped_triples": skipped_triples,
"passed_triples": passed_triples,
"pass_rate": passed_triples / valid_triples if valid_triples else 0,
"threshold": threshold,
"filtered_out":filtered_out,
"type_detail": {
# rel: {"all": d["all"], "pass": d["pass"], "rate": d["pass"] / d["all"] if d["all"] else 0.}
rel: {"all": d["all"]}
for rel, d in type_detail.items()
}
}
print(f"关系强度计算和过滤完成!")
print(f"总三元组: {relation_stats['total_triples']}, 有效计算: {valid_triples}, 跳过: {skipped_triples}")
print(f"通过过滤(> {threshold}): {passed_triples}, 通过率: {relation_stats['pass_rate']:.1%}")
return filtered_relations, filtered_triples, relation_stats, all_relations
def evaluate_task_networks(nodes_data, relations, cfg):
"""
评估所有任务网络效能
"""
print("\n开始评估任务网络效能...")
# 初始化网络评估器
evaluator = TaskNetworkEvaluator(nodes_data["nodes"], relations, weights=cfg["W_NET"])
# 评估所有任务网络
results = evaluator.eval_all()
print("任务网络效能评估完成!")
return results
def generate_evaluation_report(relation_stats: Dict, network_results: Dict,
filtered_relations: Dict, all_relations: Dict,
threshold: float, cfg) -> str:
"""
生成详细的评价报告
"""
report = []
report.append("=" * 60)
report.append("网络关系与效能评价报告")
report.append("=" * 60)
report.append("")
# 1. 基本信息
report.append("1. 基本信息")
report.append("-" * 40)
report.append(f"过滤阈值: {threshold} 【关系强度阈值】")
report.append(f"总三元组数量: {relation_stats['total_triples']}")
line = " ".join(f"{rel}={d['all']}" for rel, d in relation_stats["type_detail"].items())
report.append(f"分别有:{line}")
report.append(f"有效计算三元组: {relation_stats['valid_triples']}")
report.append(f"跳过三元组: {relation_stats['skipped_triples']}")
report.append(f"通过过滤三元组: {relation_stats['passed_triples']}")
report.append(f"通过率: {relation_stats['pass_rate']:.1%}")
report.append("")
# 添加边关系计算公式和说明
report.append("2. 计算公式")
report.append("边关系计算公式")
report.append("-" * 50)
rel_cn = {"IS": "情报保障", "CC": "指挥控制", "SF": "状态反馈", "PD": "平台部署", "CO": "协同作战"}
desc = var_desc()
for rel in ["IS", "CC", "SF", "PD", "CO"]:
report.append(
f"{rel_cn[rel]}关系 L_{rel}(i,j) = w_f·M_F^{rel} + w_s·M_S^{rel} + w_t·M_T^{rel} + w_p·M_P^{rel} + w_i·M_I^{rel}")
report.append("【符号说明】")
for sym, exp in desc.items():
if sym.startswith(("M_F^", "M_S^", "M_T^", "M_P^", "M_I^")):
report.append(f" {sym:<12} {exp}")
report.append("")
report.append("任务网络效能计算公式")
report.append("-" * 50)
task_desc = {
"defense": "综合防御 P_defense = α1·(∑f_IC/N_IC) + α2·(∑f_IA/N_IA) + α3·(∑L_IS/N_IS) + α4·(∑t_resp/N) + α5·(∑L_CC/N_CC)",
"fire": "火力打击 P_fire = β1·(∑f_CS/N_CS) + β2·(∑f_IA/N_IA) + β3·(∑L_CO/N_CO) + β4·(∑p_core·f_CS/N_CS) + β5·(∑L_IS/N_IS)",
"logistics": "后勤保障 P_logistics = γ1·(∑f_CPS/N_CPS) + γ2·(∑f_DP/N_DP) + γ3·(∑L_PD/N_PD) + γ4·(∑p_rel/N) + γ5·(∑f_IT/N_IT)",
"medical": "医疗救援 P_medical = δ1·(∑f_CPS/N_CPS) + δ2·(∑t_resp/N) + δ3·(∑f_IT/N_IT) + δ4·(∑L_SF/N_SF) + δ5·(∑fresh_i/N)",
"evacuation": "紧急疏散 P_evacuation = ε1·(∑f_CC/N_CC) + ε2·(∑f_IT/N_IT) + ε3·(∑L_CC/N_CC) + ε4·(∑f_CPS/N_CPS) + ε5·(∑t_resp/N)"
}
for task, form in task_desc.items():
report.append(form)
report.append("【符号说明】")
for sym, exp in desc.items():
if any(sym.startswith(x) for x in ["N_", "", "α", "β", "γ", "δ", "ε"]):
report.append(f" {sym:<18} {exp}")
report.append("")
# 2. 按关系类型统计
report.append("3. 关系类型统计")
report.append("-" * 40)
relation_type_stats = {}
for (rel_type, head, tail), strength in all_relations.items():
if rel_type not in relation_type_stats:
relation_type_stats[rel_type] = {"all": [], "filtered": []}
relation_type_stats[rel_type]["all"].append(strength)
if strength > threshold:
relation_type_stats[rel_type]["filtered"].append(strength)
for rel_type in sorted(relation_type_stats.keys()):
stats = relation_type_stats[rel_type]
all_count = len(stats["all"])
filtered_count = len(stats["filtered"])
pass_rate = filtered_count / all_count if all_count > 0 else 0
report.append(f"{rel_type}关系:")
report.append(f" 总数: {all_count}, 通过: {filtered_count}, 通过率: {pass_rate:.1%}")
if stats["all"]:
report.append(f" 平均强度: {np.mean(stats['all']):.3f}")
if stats["filtered"]:
report.append(f" 过滤后平均强度: {np.mean(stats['filtered']):.3f}")
report.append("【被过滤掉的关系明细】")
report.append("-" * 40)
for rel, lst in relation_stats["filtered_out"].items():
if not lst:
continue
report.append(f"{rel} 关系(强度 ≤ {threshold}")
for h, t, v in lst:
report.append(f" {h}{t} 强度={v:.3f}")
report.append("") # 空行隔开
# 1. 取任务得分(排除 dimensions
task_scores = {task: score for task, score in network_results.items() if isinstance(score, float)}
total = sum(task_scores.values()) or 1.0 # 防 0
# 2. 归一化 → 百分比
task_pct = {task: (sc / total) * 100 for task, sc in task_scores.items()}
# 3. 输出
report.append("4. 所属任务网络评估")
report.append("-" * 40)
task_name = {
"defense": "综合防御",
"fire": "火力打击",
"logistics": "后勤保障",
"medical": "医疗救援",
"evacuation": "紧急疏散"
}
# 一行输出所有任务网络百分比
line = " ".join(f"{task_name[task]} {pct:.1f}%" for task, pct in task_pct.items())
report.append("任务网络概率:" + line)
# 4. 最可能任务
max_task = max(task_pct, key=task_pct.get)
report.append(f"最有可能的任务网络:{task_name[max_task]} 可能性为:{task_pct[max_task]:.1f} %")
# 添加最终网络的五个维度评价
report.append(f"5. 最终{task_name[max_task]}任务网络的五个维度评价")
dimensions = network_results.get("dimensions", {})
reference_value = 0.6 # 参考值
# 功能维度
report.append("功能维度 (Function)")
report.append(f"功能维度反映了网络中节点的功能匹配程度。得分: {dimensions.get('function', 0):.3f}")
if dimensions.get('function', 0) > reference_value:
report.append("高于参考值,表示节点在功能上高度互补,能够有效协同工作。")
else:
report.append("低于参考值,表示节点在功能上匹配程度较低,协同工作能力有待提升。")
# 边关系维度
report.append("空间系维度 (Relationship)")
report.append(f"空间维度反映了节点之间的连接强度。得分: {dimensions.get('relationship', 0):.3f}")
if dimensions.get('relationship', 0) > reference_value:
report.append("高于参考值,表示节点之间的连接紧密,信息传递高效。")
else:
report.append("低于参考值,表示节点之间的连接较弱,信息传递效率较低。")
# 性能维度
report.append("性能维度 (Performance)")
report.append(f"性能维度反映了节点的性能指标,如响应时间和可靠性。得分: {dimensions.get('performance', 0):.3f}")
if dimensions.get('performance', 0) > reference_value:
report.append("高于参考值,表示节点在性能上表现出色,能够稳定运行。")
else:
report.append("低于参考值,表示节点在性能上存在不足,稳定性有待提高。")
# 时间维度
report.append("时间维度 (Temporal)")
report.append(f"时间维度反映了节点的时间同步性和响应能力。得分: {dimensions.get('temporal', 0):.3f}")
if dimensions.get('temporal', 0) > reference_value:
report.append("高于参考值,表示节点在时间上高度同步,能够快速响应。")
else:
report.append("低于参考值,表示节点在时间同步性上存在偏差,响应速度较慢。")
# 交互维度
report.append("交互维度 (Interaction)")
report.append(f"交互维度反映了节点之间的协议、格式和安全兼容性。得分: {dimensions.get('interaction', 0):.3f}")
if dimensions.get('interaction', 0) > reference_value:
report.append("高于参考值,表示节点在交互上高度兼容,能够安全通信。")
else:
report.append("低于参考值,表示节点在交互上存在兼容性问题,通信安全性较低。")
return "\n".join(report)
def save_filtered_triples(filtered_triples: List[Dict], filename: str = "filtered_triples.json"):
"""
保存过滤后的三元组到JSON文件
"""
project_root = Path(__file__).parent.parent # src -> 227
output_path = project_root / "results" / filename
output_data = {"triples": filtered_triples}
with open(output_path, 'w', encoding='utf-8') as f:
json.dump(output_data, f, indent=2, ensure_ascii=False)
print(f"过滤后的三元组已保存到 {filename}")
def save_triples_and_task_probs(filtered_triples: list,
network_results: dict,
excel_name: str = "triples.xlsx",
csv_name: str = "triples.csv") -> None:
"""
保存过滤后的三元组csv + Excel 表1+
归一化任务网络概率Excel 表2
"""
# 1. 三元组 DataFrame含 head_type / tail_type
triple_df = pd.DataFrame([
{
"head": t["head"],
"head_type": t.get("head_type", ""),
"relation": t["relation"],
"tail": t["tail"],
"tail_type": t.get("tail_type", ""),
"strength": t["strength"]
}
for t in filtered_triples
])
# 2. 任务网络概率 DataFrame
task_scores = {task: score for task, score in network_results.items() if isinstance(score, float)}
total = sum(task_scores.values()) or 1.0
task_pct = {task: (sc / total) * 100 for task, sc in task_scores.items()}
task_map = {
"defense": "1-综合防御任务网络",
"fire": "2-火力打击任务网络",
"logistics": "3-后勤保障任务网络",
"medical": "4-医疗救援任务网络",
"evacuation": "5-紧急疏散任务网络"
}
prob_df = pd.DataFrame([
{"id": int(task_map[task].split('-')[0]),
"任务": task_map[task].split('-')[1],
"概率%": f"{pct:.1f}"}
for task, pct in task_pct.items()
])
# 3. 写文件
project_root = Path(__file__).parent.parent # src -> 227
output_path_csv = project_root / "results" / csv_name
triple_df.to_csv(output_path_csv, index=False, encoding='utf-8-sig')
output_path = project_root / "results" / excel_name
with pd.ExcelWriter(output_path, engine='openpyxl') as writer:
triple_df.to_excel(writer, sheet_name='triples', index=False)
prob_df.to_excel(writer, sheet_name='TaskProbs', index=False)
print(f"已生成:{csv_name} | {excel_name}(含 TaskProbs 页)")
def save_evaluation_report(report_content: str, filename: str = "evaluation_report.txt"):
"""
保存评价报告到TXT文件
"""
project_root = Path(__file__).parent.parent # src -> 227
output_path = project_root / "results" / filename
with open(output_path, 'w', encoding='utf-8') as f:
f.write(report_content)
print(f"评价报告已保存到 {filename}")
def main():
"""
主函数:整合所有计算流程
"""
"""运行时交互版主函数"""
import os
from data_loader import load_data as load_data_source
script_dir = os.path.dirname(os.path.abspath(__file__))
project_root = os.path.dirname(script_dir)
config = load_config(project_root)
source = config.get('source', 'json')
print(f"数据源模式: {source}")
try:
data, paths = load_data_source(config, project_root)
except (FileNotFoundError, ValueError) as e:
print(f"错误:{e}")
exit(1)
os.makedirs(os.path.dirname(paths['report']), exist_ok=True)
cleaner = AdvancedDataCleaner(
output_file=paths['output'],
report_file=paths['report'],
data=data,
)
cleaner.run()
formal_cfg = config.get("formal", {})
if source == "db" and formal_cfg.get("enabled", False):
print("\n=== 模块1.3:形式化入库流程 ===")
try:
formal_result = run_formalization_pipeline(config)
if formal_result.get("ok"):
print(f"形式化批次: {formal_result.get('batch_id')}")
print(f"形式化统计: {formal_result.get('stats', {})}")
else:
print(f"形式化流程跳过: {formal_result.get('reason', 'unknown')}")
except Exception as e:
print(f"形式化流程执行失败: {e}")
raise
print("=== 网络关系与效能评估(交互填参版) ===")
# 1. 加载默认配置
cfg = copy.deepcopy(DEFAULT_CFG)
# 交互式逐项覆盖
def ask_one_weight(prompt: str, default: float):
"""只问一个权重,回车=default"""
while True:
try:
v = input(f"{prompt} (默认={default})").strip()
return default if v == "" else float(v)
except ValueError:
print("请输入 0-1 之间的数字!")
def ask_one_int(prompt: str, default: int):
"""问整数距离"""
while True:
try:
v = input(f"{prompt} (默认={default} km)").strip()
return default if v == "" else int(v)
except ValueError:
print("请输入整数!")
def ask_one_float(prompt: str, default: float):
"""问时间衰减系数"""
while True:
try:
v = input(f"{prompt} (默认={default})").strip()
return default if v == "" else float(v)
except ValueError:
print("请输入数字!")
# print("\n========== ① 关系权重 (0-1) ==========")
# rels = ["IS", "CC", "SF", "PD", "CO"]
# rel_keys = ["w_f", "w_s", "w_t", "w_p", "w_i"]
# rel_names = ["功能(w_f)", "空间(w_s)", "时间(w_t)", "性能(w_p)", "交互(w_i)"]
# for rel in ["IS", "CC", "SF", "PD", "CO"]:
# print(f"\n-- {rel} 关系 --w_f/w_s/w_t/w_p/w_i")
# for k, name in zip(rel_keys, rel_names):
# old = cfg["W_REL"][rel][k]
# cfg["W_REL"][rel][k] = ask_one_weight(name, old)
#
# print("\n========== ② 网络权重 (0-1) ==========")
# tasks = ["defense", "fire", "logistics", "medical", "evacuation"]
# tfactor = ["w1", "w2", "w3", "w4", "w5"]
# for task in tasks:
# print(f"\n-- {task} -- w1/w2/w3/w4/w5")
# for i, tf in enumerate(tfactor):
# old = cfg["W_NET"][task][tf]
# cfg["W_NET"][task][tf] = ask_one_weight(tf, old)
#
# print("\n========== ③ 特征距离 LAMBDA (km) ==========")
# for rel in rels:
# old = cfg["LAMBDA"][rel]
# cfg["LAMBDA"][rel] = ask_one_float(f"{rel} 距离", old)
#
# print("\n========== ④ 时间衰减系数 ==========")
# cfg["TIME"]["alpha"] = ask_one_float("alpha", cfg["TIME"]["alpha"])
# cfg["TIME"]["beta"] = ask_one_float("beta", cfg["TIME"]["beta"])
# cfg["TIME"]["gamma"] = ask_one_float("gamma", cfg["TIME"]["gamma"])
#
# print("\n========== ⑤ 其他 ==========")
# cfg["PD_SGM"] = ask_one_float("平台共位阈值 PD_SGM", cfg["PD_SGM"])
# cfg["THRESHOLD"] = ask_one_weight("关系过滤阈值", cfg["THRESHOLD"])
#
print("开始网络关系计算")
print("=" * 50)
try:
# 1. 加载数据
print("步骤1: 加载数据...")
nodes_data, triples_data = load_data(project_root)
print(f"加载节点数: {len(nodes_data['nodes'])}")
print(f"加载三元组数: {len(triples_data['triples'])}")
# 2. 计算并过滤关系强度
print(f"\n步骤2: 计算关系强度并过滤(阈值={cfg['THRESHOLD']})")
filtered_relations, filtered_triples, relation_stats, all_relations = calculate_and_filter_relations(
nodes_data, triples_data, cfg["THRESHOLD"], cfg
)
# 3. 评估任务网络效能(使用过滤后的关系)
print("\n步骤3: 使用过滤后的关系评估任务网络效能...")
network_results = evaluate_task_networks(nodes_data, filtered_relations, cfg)
# 4. 生成评价报告
print("\n步骤4: 生成评价报告...")
report_content = generate_evaluation_report(
relation_stats, network_results, filtered_relations, all_relations, cfg["THRESHOLD"],cfg
)
# 5. 保存结果
print("\n步骤5: 保存结果文件...")
save_triples_and_task_probs(filtered_triples, network_results)
save_filtered_triples(filtered_triples)
save_evaluation_report(report_content)
# 6. 在控制台显示关键信息
print("\n" + "=" * 50)
print("关键结果摘要:")
print(f"- 过滤后保留的三元组: {len(filtered_triples)}")
print("所有计算完成!")
print("=" * 50)
except Exception as e:
print(f"程序执行出错: {e}")
import traceback
traceback.print_exc()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,113 @@
"""
三元组清洗模块
"""
from datetime import datetime
from typing import List, Dict, Any, Set, Tuple
def clean_triplets(
triplets: List[Dict[str, Any]],
valid_ids: Set[str] = None,
deduplicate: bool = True,
filter_orphans: bool = False,
remove_null_core: bool = True,
normalize_relation_type: bool = True,
) -> Tuple[List[Dict[str, Any]], Dict[str, Any]]:
"""
清洗三元组数据
:param triplets: 原始三元组列表
:param valid_ids: 有效实体 ID 集合(用于 filter_orphansNone 表示不校验
:param deduplicate: 是否按 (HEAD_ID, RELATION_TYPE, TAIL_ID) 去重
:param filter_orphans: 是否过滤悬空引用HEAD_ID/TAIL_ID 不在 valid_ids 中)
:param remove_null_core: 是否移除 HEAD_ID/TAIL_ID/RELATION_TYPE 为空的记录
:param normalize_relation_type: 是否规范化 RELATION_TYPE去空格
:return: (清洗后列表, 清洗报告)
"""
report = {
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"original_count": len(triplets),
"deduplicate": {"enabled": deduplicate, "removed": 0},
"filter_orphans": {"enabled": filter_orphans, "removed": 0},
"remove_null_core": {"enabled": remove_null_core, "removed": 0},
"normalize_relation_type": {"enabled": normalize_relation_type},
"final_count": 0,
}
result = list(triplets)
def _head(t): return _str(t.get("HEAD_ID") or t.get("head"))
def _tail(t): return _str(t.get("TAIL_ID") or t.get("tail"))
def _rel(t): return _str(t.get("RELATION_TYPE") or t.get("relation"))
if remove_null_core:
before = len(result)
result = [t for t in result if _not_empty(_head(t)) and _not_empty(_tail(t)) and _not_empty(_rel(t))]
report["remove_null_core"]["removed"] = before - len(result)
if normalize_relation_type:
for t in result:
k = "RELATION_TYPE" if "RELATION_TYPE" in t else "relation"
if isinstance(t.get(k), str):
t[k] = t[k].strip()
if filter_orphans and valid_ids is not None:
before = len(result)
result = [t for t in result if _head(t) in valid_ids and _tail(t) in valid_ids]
report["filter_orphans"]["removed"] = before - len(result)
if deduplicate:
before = len(result)
seen = set()
unique = []
for t in result:
key = (_head(t), _rel(t), _tail(t))
if key not in seen:
seen.add(key)
unique.append(t)
result = unique
report["deduplicate"]["removed"] = before - len(result)
report["final_count"] = len(result)
return result, report
def _not_empty(v: Any) -> bool:
if v is None:
return False
if isinstance(v, str) and not v.strip():
return False
return True
def _str(v: Any) -> str:
if v is None:
return ""
return str(v).strip()
def transform_triplet_output_format(
triplets: List[Dict[str, Any]],
id_to_role: dict = None,
) -> List[Dict[str, Any]]:
"""
将三元组转为输出格式:
- HEAD_ID/head -> head, TAIL_ID/tail -> tail, RELATION_TYPE/relation -> relation
- 新增 head_type、tail_type从 id_to_role 查 ROLE_ID已有则保留
"""
id_to_role = id_to_role or {}
result = []
for t in triplets:
head = _str(t.get("HEAD_ID") or t.get("head"))
tail = _str(t.get("TAIL_ID") or t.get("tail"))
rel = _str(t.get("RELATION_TYPE") or t.get("relation"))
head_type = _str(t.get("head_type")) or id_to_role.get(head, "")
tail_type = _str(t.get("tail_type")) or id_to_role.get(tail, "")
result.append({
"head": head,
"head_type": head_type,
"relation": rel,
"tail": tail,
"tail_type": tail_type,
})
return result

127
prompt.md Normal file
View File

@@ -0,0 +1,127 @@
非常感谢你的指正!确实,将**统一的数据库连接配置项**以及**核心表的真实插入格式参考**放入 Prompt 中,能够极大程度上防止大模型在生成代码时“自由发挥”,从而保证各个模块连接达梦数据库时的绝对一致性。
我已将这两块关键信息补充进了 `prompt.md` 的核心架构规范中。以下是为你更新后的、更加完美的 `prompt.md` 文件:
---
--- START OF FILE prompt.md ---
# 🤖 体系推演平台模块重构指令 (Meta-Prompt)
## 📌 背景说明
你现在是一名资深的全栈开发工程师与数据库架构师。当前系统是一个基于 Flask + 原生 JS + 达梦数据库 (Dameng) 的军事体系推演与分析平台。
系统正处于架构升级阶段,核心目标是:**实现基于 JWT Token 的多租户数据隔离、全面接入达梦数据库、以及在数据库异常或数据缺失时具备本地模拟兜底机制。**
你需要根据本文件的规范,将我提供的【原始前后端代码】重构为符合新架构的代码。
---
## 🏗️ 核心架构与通用方法规范(重构必读)
### 1. 鉴权与租户隔离规范 (JWT + 战略任务ID)
* **前端规范**:必须拦截原生的 `fetch` 请求,从 URL 中获取 `207.SSO_TOKEN`,并在所有以 `/api/` 开头的请求 Header 中注入 `Authorization: Bearer <Token>`
* **后端解析规范**:使用 `pyjwt` 解析 Header 中的 Token 获取 `user_id`(解析失败则兜底使用 `'ADMIN'``'default_user'`)。同时从请求参数中获取 `strat_task_id`战略任务ID`'TASK_FULL_KB'`)。
* **数据库隔离规范**:所有涉及达梦数据库的连接,在获取 `cursor` 后,**必须第一时间调用存储过程设置上下文**
```python
cur.execute("CALL PKG_SYSTEM_CTX.SET_USER_ID(?)", (user_id,))
```
设置完成后,所有对视图(`V_MY_*`)的查询将自动实现行级隔离。
### 2. 数据库连接、读取与写入规范 (DB Connector 模式)
* **数据库连接配置标准**:请统一使用环境变量与兜底配置初始化达梦数据库连接句柄(`DMInterface` 或对应的 Connector
```python
dm_handler = DMInterface(
host=os.getenv("DM_HOST", "127.0.0.1"),
port=os.getenv("DM_PORT", 9080),
user=os.getenv("DM_USER", "SYSDBA"),
password=os.getenv("DM_PASS", "Dm508508")
)
```
* **数据插入格式参考 (核心表结构)**:在写入 `SYSTEM_VERSION_MASTER` 时,请严格遵守以下 SQL 格式和强约束字段(注意时间转换与类型):
```sql
INSERT INTO SYSTEM_VERSION_MASTER (version_id, user_id, strat_task_id, sys_date, sys_type, system_task, country_region, sys_attr_id)
VALUES ('V_FULL_KB_001', 'ADMIN', 'TASK_FULL_KB', DATE '2026-04-15', '当前体系', '综合防御任务', '台', 'ATTR_FULL_KB');
```
* **业务逻辑解耦**:严禁在核心业务逻辑中写散装 SQL必须封装在 `db_connector.py` 或模块专属的 DB 类中。
* **数据挂载关系**:体系数据的存取**必须以版本version_id为核心**。体系包含三个层级:
1. `SYSTEM_VERSION_MASTER` (版本总表)
2. `SYSTEM_ENTITY` (实体表:挂载在 version_id 下)
3. `SYSTEM_RELATION` (三元组关系表:挂载在 version_id 下)
* **分析结果写入**:分析结果的存取必须写入对应的 `RES_*` 表,并将结果序列化为 JSON 存入 `CLOB` 字段。
### 3. 双轨运行与模拟兜底机制 (Fallback 机制)
与预测模块一样,本模块必须具备极强的鲁棒性:
* **读取时**:如果数据库连接失败,或数据库中查不到所需数据,必须能够回退到读取本地 `pkl/json/xlsx` 文件。
* **如果连本地文件都没有**:必须基于内置的军事规则(如实体角色、作战编制规范)**动态生成模拟数据**供算法消费,绝不能直接抛出异常崩溃。
* **写入时**:所有新生成的数据(体系数据、分析结果)必须**先保存在本地工作区目录**,然后再执行数据库的 `INSERT/UPDATE` 操作。如果数据库上传失败,仅打印警告,不影响前端展示。
---
## 🎯 各业务模块专属重构逻辑指引
请根据我当前要求你重构的模块,匹配以下业务逻辑进行代码修改:
### 模块 1.1:目标识别模块
* **数据流向**:前端传入实体名集合 -> 触发上层知识库接口(模拟) -> 知识库将图片、新闻、属性存入数据库 `ENTITY_ATTR_DETAIL` 表。
* **计算逻辑**:模块从数据库读取这些属性进行数据融合与目标识别。
* **入库操作**:识别出实体的国家、角色、任务后,执行 `UPDATE SYSTEM_ENTITY`(如果实体已存在)或覆盖写入。结果明细写入 `RES_TARGET_RECOGNITION`。
### 模块 1.6-1.7:节点价值分析模块
* **数据流向**:通过 `user_id` 和 `strat_task_id` 从数据库中提取所有实体及对应的 `ENTITY_ATTR_DETAIL`。
* **计算逻辑**:执行价值计算算法。
* **入库操作**:将计算出的最终价值分数 `UPDATE ENTITY_ATTR_DETAIL SET entity_value = ?`。分析报告 JSON 存入 `RES_NODE_VALUE`。
### 模块 2.1:关系推理模块
* **数据流向**:读取实体库、实体可能的关系 (`possible_relations`) 以及实体属性。
* **入库操作**:将推理出的全新三元组数据,作为全新的体系快照入库。
* 在 `SYSTEM_VERSION_MASTER` 中插入新记录,且 **`sys_type` 必须标记为 `'关系推理后的体系'`**。
* 将推理详情日志存入 `RES_REL_REASONING`。
### 模块 1.3:形式化模块
* **数据流向**:默认从数据库中读取 `sys_type = '关系推理后的体系'` 的数据。
* **入库操作**:形式化计算完成后,将结果作为全新的体系入库,**`sys_type` 标记为 `'形式化后的体系'`**。详细规则存入 `RES_FORMAL_ANALYSIS`。
### 模块 1.4:体系构建模块 (核心中枢)
* **读取逻辑**:默认拉取 `sys_type = '形式化后的体系'` 用于可视化。前端需新增接口,拉取该任务下 **所有版本** 的体系列表供用户切换查看。
* **定时保存功能**:后端需支持定时任务,触发时将当前画板体系存入数据库,**`sys_type` 默认为 `'历史体系'`**。
* **手动保存功能**:前端提供下拉框,允许用户自由选择 8 种体系类型之一进行保存。
* **设为当前体系**:前端左下角新增【确认构建为当前体系】按钮,点击后将当前数据覆盖或新建为 **`sys_type = '当前体系'`**。
### 模块 1.5:体系分析模块
* **数据流向**:强制读取 `sys_type = '当前体系'` 的数据及实体属性。
* **入库操作**
* 将包含 OODA 角色的体系存为 **`sys_type = '指控体系'`**(更新 `SYSTEM_ENTITY` 中的 `c2_type` 字段为 Observe/Orient/Decide/Act
* 分析报告存入 `RES_SYS_ANALYSIS`。
### 模块 3.1, 3.2, 3.3:失效机制与体系效能模块
* **数据流向**:全部强制读取 `sys_type = '当前体系'` 的数据。
* **入库操作**
* 模块 3.2(体系重构)执行完毕后,将结果生成新版本,**`sys_type` 标记为 `'重构体系'`**。
* 模块 3.1 / 3.3 算出的效能指标,更新到 `SYSTEM_ATTR_DETAIL` 表中,详细报告存入 `RES_SYS_EFFECTIVENESS`。
### 模块 3.4, 3.5:重心分析模块
* **数据流向**:强制读取 `sys_type = '当前体系'`。
* **入库操作**:链路重心结果存入 `RES_LINK_COG`,节点重心结果存入 `RES_NODE_COG`。
---
## ⚠️ 数据库表结构校验指令 (非常重要)
在重构代码前,请你仔细审阅我提供的 `CreatTable.sql` 文件。
如果你发现在上述业务逻辑中,**有部分属性、字段或关联关系在 `CreatTable.sql` 中不存在**(例如:某个模块需要保存一种特殊格式的 JSON但对应表没有 CLOB 字段;或者缺少了知识库中间表),请你**务必在输出代码前明确指出**,并为我提供用于达梦数据库的补充 SQL 语句(`CREATE TABLE` 或 `ALTER TABLE`)。
---
## 💻 你的任务
现在,我已经将本模块的 **前端 JS、HTML、后端 Python 逻辑** 上传给了你。
请你严格遵守以上《体系推演平台模块重构指令》,一步步思考,并输出:
1. **数据库结构审查报告**(是否需要补充 SQL 表?若需要,请给出 SQL
2. **后端的重构代码**(包含 Python API 路由接收参数、Token 解析、DB 调用与 Fallback 生成逻辑)。
3. **前端的重构代码**(包含 Token 注入、接口参数适配、以及模块特定如1.4模块)的 UI 按钮与下拉框新增)。
请确保输出的代码完整、注释清晰,并且变量名符合原代码的上下文风格。
--- END OF FILE prompt.md ---

View File

@@ -1,166 +0,0 @@
import json
import numpy as np
import pandas as pd
from datetime import datetime
class AdvancedDataCleaner:
def __init__(self, input_file, output_file, report_file):
self.input_file = input_file
self.output_file = output_file
self.report_file = report_file
self.data = []
self.df = None
# 报告结构
self.report = {
"timestamp": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"summary": {
"total_records": 0,
"final_records": 0,
"duplicates_removed": 0
},
"details": {
"missing_values_fixed": {}, # 字段: 填充数量
"outliers_corrected": {
"count": 0,
"examples": [] # 记录具体的修改案例
},
"noise_reduction": {
"method": "Kalman Filter",
"fields_processed": ["COMMUNICATION_RANGE"],
"total_smoothed": 0
},
"standardization": []
}
}
self.norm_fields = [
"TARGET_RECOGNITION_CAPABILITY", "STRIKE_ACCURACY",
"ANTI_JAMMING_CAPABILITY", "ENVIRONMENT_ADAPTABILITY", "MOBILITY"
]
def load_data(self):
with open(self.input_file, 'r', encoding='utf-8') as f:
self.data = json.load(f)
self.report["summary"]["total_records"] = len(self.data)
self.df = pd.DataFrame(self.data)
def clean_duplicates(self):
"""高级去重并记录"""
initial_count = len(self.df)
# 优先保留创建时间最新的(如果有时间字段),否则保留第一个
if 'CREATED_TIME' in self.df.columns:
self.df.sort_values('CREATED_TIME', ascending=False, inplace=True)
self.df.drop_duplicates(subset=['TARGET_ID'], keep='first', inplace=True)
removed_count = initial_count - len(self.df)
self.report["summary"]["duplicates_removed"] = removed_count
def handle_missing_values(self):
"""智能填充并记录细节"""
numeric_cols = self.df.select_dtypes(include=[np.number]).columns
for col in numeric_cols:
if col == "ID": continue
n_missing = int(self.df[col].isnull().sum())
if n_missing > 0:
self.report["details"]["missing_values_fixed"][col] = n_missing
# 分组填充
self.df[col] = self.df.groupby("ROLE_ID")[col].transform(lambda x: x.fillna(x.mean()))
# 兜底填充
self.df[col] = self.df[col].fillna(self.df[col].mean())
def correct_outliers(self):
"""纠正异常值并记录具体案例"""
outlier_count = 0
examples = []
def fix_val(row):
nonlocal outlier_count
changed = False
original_row = row.copy()
for field in self.norm_fields:
if pd.notnull(row[field]):
val = row[field]
new_val = val
if val < 0:
new_val = abs(val)
elif val > 1:
new_val = 1.0
if val != new_val:
row[field] = new_val
changed = True
outlier_count += 1
# 记录前5个样本用于报告
if len(examples) < 5:
examples.append({
"id": row.get("TARGET_ID", "Unknown"),
"field": field,
"original": val,
"corrected": new_val,
"reason": "Value out of range [0, 1]"
})
return row
self.df = self.df.apply(fix_val, axis=1)
self.report["details"]["outliers_corrected"]["count"] = outlier_count
self.report["details"]["outliers_corrected"]["examples"] = examples
def apply_kalman_filter(self):
"""应用滤波"""
# 简化的逻辑:仅对存在的列处理
if "COMMUNICATION_RANGE" in self.df.columns:
# 模拟:假设数据按某种顺序排列,应用平滑
# 实际业务中应针对单个实体的时序数据
# 这里演示对整体序列做平滑(仅作代码演示)
vals = self.df["COMMUNICATION_RANGE"].fillna(0).values
# 简单移动平均代替卡尔曼演示(效果类似平滑)
smoothed = pd.Series(vals).rolling(window=3, min_periods=1).mean().values
self.df["COMMUNICATION_RANGE"] = np.round(smoothed, 2)
self.report["details"]["noise_reduction"]["total_smoothed"] = len(vals)
def run(self):
print("正在执行高级清洗...")
self.load_data()
self.clean_duplicates()
self.handle_missing_values()
self.correct_outliers()
self.apply_kalman_filter()
# 最终统计
self.report["summary"]["final_records"] = len(self.df)
self.report["details"]["standardization"].append("Coordinates normalized to 2 decimal places")
self.report["details"]["standardization"].append("Timestamps formatted to ISO-8601")
# 保存数据
result_data = self.df.to_dict('records')
with open(self.output_file, 'w', encoding='utf-8') as f:
json.dump(result_data, f, ensure_ascii=False, indent=2)
# 保存详细报告
with open(self.report_file, 'w', encoding='utf-8') as f:
json.dump(self.report, f, ensure_ascii=False, indent=2)
print(f"完成!报告已生成至 {self.report_file}")
if __name__ == "__main__":
# 使用相对路径:../data/ 表示上一级目录下的 data 文件夹
input_path = '../data/raw_data_sample.json'
output_path = '../data/cleaned_data_final.json'
report_path = '../report/detailed_cleaning_report.json'
# 增加一个检查,防止路径错误
import os
if not os.path.exists(input_path):
print(f"错误:找不到文件 {input_path}")
print(f"当前工作目录是:{os.getcwd()}")
print("请检查文件路径或确保已运行数据生成脚本。")
else:
cleaner = AdvancedDataCleaner(input_path, output_path, report_path)
cleaner.run()