新增修改
This commit is contained in:
BIN
final/scripts/__pycache__/export_triplets.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/export_triplets.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/f_net1125.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/f_net1125.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/f_net1125.cpython-38.pyc
Normal file
BIN
final/scripts/__pycache__/f_net1125.cpython-38.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/f_relation1125.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/f_relation1125.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/f_relation1125.cpython-38.pyc
Normal file
BIN
final/scripts/__pycache__/f_relation1125.cpython-38.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/inspect_dm_tables.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/inspect_dm_tables.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/run_all.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/run_all.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/utils.cpython-313.pyc
Normal file
BIN
final/scripts/__pycache__/utils.cpython-313.pyc
Normal file
Binary file not shown.
BIN
final/scripts/__pycache__/utils.cpython-38.pyc
Normal file
BIN
final/scripts/__pycache__/utils.cpython-38.pyc
Normal file
Binary file not shown.
113
final/scripts/export_triplets.py
Normal file
113
final/scripts/export_triplets.py
Normal file
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
从达梦数据库导出三元组数据到 JSON 文件
|
||||
三元组来源:RELATION_INSTANCE 表 (HEAD_ID, RELATION_TYPE, TAIL_ID)
|
||||
支持可选清洗:去重、过滤悬空引用、移除空值、规范化关系类型
|
||||
用法: python scripts/export_triplets.py
|
||||
需设置环境变量 DM_PASSWORD,且 config 中 source=db
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
_script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_project_root = os.path.dirname(_script_dir)
|
||||
sys.path.insert(0, _project_root)
|
||||
|
||||
|
||||
def _json_serializer(obj):
|
||||
"""处理 Timestamp、datetime、numpy 等不可直接 JSON 序列化的类型"""
|
||||
if hasattr(obj, 'isoformat'):
|
||||
return obj.isoformat()
|
||||
if hasattr(obj, 'item'):
|
||||
return obj.item()
|
||||
if isinstance(obj, float) and (obj != obj or obj == float('inf') or obj == float('-inf')):
|
||||
return None
|
||||
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
|
||||
|
||||
|
||||
def load_config(project_root: str) -> dict:
|
||||
"""加载 config"""
|
||||
config_dir = os.path.join(project_root, 'config')
|
||||
json_path = os.path.join(config_dir, 'config.json')
|
||||
yaml_path = os.path.join(config_dir, 'config.yaml')
|
||||
if os.path.exists(json_path):
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
if os.path.exists(yaml_path):
|
||||
try:
|
||||
import yaml
|
||||
with open(yaml_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
except ImportError:
|
||||
pass
|
||||
return {'source': 'json', 'db': {}, 'triplet': {}}
|
||||
|
||||
|
||||
def main():
|
||||
from src.data_loader import load_triplets_from_dm, load_from_dm
|
||||
from src.triplet_cleaner import clean_triplets, transform_triplet_output_format
|
||||
|
||||
config = load_config(_project_root)
|
||||
if config.get('source') != 'db':
|
||||
print("错误:三元组导出仅支持达梦模式,请将 config 中 source 改为 db")
|
||||
sys.exit(1)
|
||||
|
||||
tc = config.get('triplet', {})
|
||||
output_file = tc.get('output_file', 'data/triples.json')
|
||||
report_file = tc.get('report_file', 'report/triplet_cleaning_report.json')
|
||||
output_path = output_file if os.path.isabs(output_file) else os.path.join(_project_root, output_file)
|
||||
report_path = report_file if os.path.isabs(report_file) else os.path.join(_project_root, report_file)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
os.makedirs(os.path.dirname(report_path), exist_ok=True)
|
||||
|
||||
# 清洗配置(clean: false 则跳过;否则按子项配置)
|
||||
clean_cfg = tc.get('clean', True)
|
||||
do_clean = False
|
||||
if clean_cfg is True:
|
||||
do_clean = {'deduplicate': True, 'filter_orphans': False, 'remove_null_core': True, 'normalize_relation_type': True}
|
||||
elif isinstance(clean_cfg, dict):
|
||||
do_clean = clean_cfg
|
||||
|
||||
deduplicate = do_clean.get('deduplicate', True) if do_clean else False
|
||||
filter_orphans = do_clean.get('filter_orphans', False) if do_clean else False
|
||||
remove_null_core = do_clean.get('remove_null_core', True) if do_clean else False
|
||||
normalize_relation_type = do_clean.get('normalize_relation_type', True) if do_clean else False
|
||||
|
||||
print("正在从达梦读取三元组 (RELATION_INSTANCE)...")
|
||||
triplets = load_triplets_from_dm(config, _project_root)
|
||||
print(f"读取到 {len(triplets)} 条三元组")
|
||||
|
||||
valid_ids = None
|
||||
if filter_orphans:
|
||||
print("正在加载实体表以校验悬空引用...")
|
||||
entities = load_from_dm(config, _project_root)
|
||||
valid_ids = {str(e.get('TARGET_ID', '')).strip() for e in entities if e.get('TARGET_ID')}
|
||||
print(f"有效实体数: {len(valid_ids)}")
|
||||
|
||||
if do_clean and (deduplicate or filter_orphans or remove_null_core or normalize_relation_type):
|
||||
print("正在执行三元组清洗...")
|
||||
triplets, report = clean_triplets(
|
||||
triplets,
|
||||
valid_ids=valid_ids,
|
||||
deduplicate=deduplicate,
|
||||
filter_orphans=filter_orphans,
|
||||
remove_null_core=remove_null_core,
|
||||
normalize_relation_type=normalize_relation_type,
|
||||
)
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
print(f"清洗完成:{report['original_count']} -> {report['final_count']},报告: {report_path}")
|
||||
|
||||
# 转为输出格式:head, head_type, relation, tail, tail_type
|
||||
entities = load_from_dm(config, _project_root)
|
||||
id_to_role = {str(e.get('TARGET_ID', '')).strip(): str(e.get('ROLE_ID', '')).strip() for e in entities if e.get('TARGET_ID')}
|
||||
triplets = transform_triplet_output_format(triplets, id_to_role)
|
||||
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(triplets, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
print(f"已导出至 {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
112
final/scripts/f_net1125.py
Normal file
112
final/scripts/f_net1125.py
Normal file
@@ -0,0 +1,112 @@
|
||||
# f_net1125.py ← 边关系已算完,只算任务网络
|
||||
from __future__ import annotations
|
||||
import numpy as np
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
NodeID = str
|
||||
TaskScore = float # 网络效能得分 [0,1]
|
||||
|
||||
|
||||
class TaskNetworkEvaluator:
|
||||
"""
|
||||
输入:
|
||||
nodes - 节点数据(含 function_vector / performance / temporal)
|
||||
relations - 已算好的全部边关系
|
||||
格式:relations[("IS","DEF001","CMD001")] = 0.87
|
||||
"""
|
||||
def __init__(self,
|
||||
nodes: Dict[NodeID, Dict],
|
||||
relations: Dict[Tuple[str, NodeID, NodeID], float],
|
||||
weights: Dict[str, Dict[str, float]] = None):
|
||||
self.nodes = nodes
|
||||
self.rels = relations
|
||||
self.w = weights
|
||||
|
||||
# ---------- 工具 ----------
|
||||
def _avg(self, scores: List[float]) -> float:
|
||||
return float(np.mean(scores)) if scores else 0.0
|
||||
|
||||
def _collect_func(self, key: str) -> List[float]:
|
||||
"""所有节点指定功能值"""
|
||||
return [n["function_vector"][key] for n in self.nodes.values()]
|
||||
|
||||
def _collect_rel(self, tag: str) -> List[float]:
|
||||
"""预存关系强度列表"""
|
||||
return [v for k, v in self.rels.items() if k[0] == tag]
|
||||
|
||||
def _collect_resp(self) -> List[float]:
|
||||
"""响应时间得分"""
|
||||
return [np.exp(-0.1 * n["temporal"]["response_time"]) for n in self.nodes.values()]
|
||||
|
||||
# ---------- 2.1 综合防御 ----------
|
||||
def eval_defense(self) -> TaskScore:
|
||||
w = self.w["defense"]
|
||||
f_ic = self._avg(self._collect_func("f_IC"))
|
||||
f_ia = self._avg(self._collect_func("f_IA"))
|
||||
l_is = self._avg(self._collect_rel("IS"))
|
||||
t_resp = self._avg(self._collect_resp())
|
||||
l_cc = self._avg(self._collect_rel("CC"))
|
||||
return w["w1"] * f_ic + w["w2"] * f_ia + w["w3"] * l_is + w["w4"] * t_resp + w["w5"] * l_cc
|
||||
|
||||
# ---------- 2.2 火力打击 ----------
|
||||
def eval_fire(self) -> TaskScore:
|
||||
w = self.w["fire"]
|
||||
f_cs = self._avg(self._collect_func("f_CS"))
|
||||
f_ia = self._avg(self._collect_func("f_IA"))
|
||||
l_co = self._avg(self._collect_rel("CO"))
|
||||
perf_cs = self._avg([n["performance"]["core_performance"] * n["function_vector"]["f_CS"]
|
||||
for n in self.nodes.values()])
|
||||
l_is = self._avg(self._collect_rel("IS"))
|
||||
return w["w1"] * f_cs + w["w2"] * f_ia + w["w3"] * l_co + w["w4"] * perf_cs + w["w5"] * l_is
|
||||
|
||||
# ---------- 2.3 后勤保障 ----------
|
||||
def eval_logistics(self) -> TaskScore:
|
||||
w = self.w["logistics"]
|
||||
f_cps = self._avg(self._collect_func("f_CPS"))
|
||||
f_dp = self._avg(self._collect_func("f_DP"))
|
||||
l_pd = self._avg(self._collect_rel("PD"))
|
||||
p_rel = self._avg([n["performance"]["mtbf"] / 1000 for n in self.nodes.values()])
|
||||
f_it = self._avg(self._collect_func("f_IT"))
|
||||
return w["w1"] * f_cps + w["w2"] * f_dp + w["w3"] * l_pd + w["w4"] * p_rel + w["w5"] * f_it
|
||||
|
||||
# ---------- 2.4 医疗救援 ----------
|
||||
def eval_medical(self) -> TaskScore:
|
||||
w = self.w["medical"]
|
||||
f_cps = self._avg(self._collect_func("f_CPS"))
|
||||
t_resp = self._avg(self._collect_resp())
|
||||
f_it = self._avg(self._collect_func("f_IT"))
|
||||
l_sf = self._avg(self._collect_rel("SF"))
|
||||
fresh = self._avg([np.exp(-0.1 * n["temporal"]["data_age"]) for n in self.nodes.values()])
|
||||
return w["w1"] * f_cps + w["w2"] * t_resp + w["w3"] * f_it + w["w4"] * l_sf + w["w5"] * fresh
|
||||
|
||||
# ---------- 2.5 紧急疏散 ----------
|
||||
def eval_evacuation(self) -> TaskScore:
|
||||
w = self.w["evacuation"]
|
||||
f_cc = self._avg(self._collect_func("f_CC"))
|
||||
f_it = self._avg(self._collect_func("f_IT"))
|
||||
l_cc = self._avg(self._collect_rel("CC"))
|
||||
f_cps = self._avg(self._collect_func("f_CPS"))
|
||||
t_resp = self._avg(self._collect_resp())
|
||||
return w["w1"] * f_cc + w["w2"] * f_it + w["w3"] * l_cc + w["w4"] * f_cps + w["w5"] * t_resp
|
||||
|
||||
# ---------- 一键评估 ----------
|
||||
def eval_all(self) -> Dict[str, TaskScore]:
|
||||
results = {
|
||||
"defense": self.eval_defense(),
|
||||
"fire": self.eval_fire(),
|
||||
"logistics": self.eval_logistics(),
|
||||
"medical": self.eval_medical(),
|
||||
"evacuation": self.eval_evacuation(),
|
||||
}
|
||||
|
||||
# 计算五个维度的分数
|
||||
dimensions = {
|
||||
"function": self._avg(self._collect_func("f_IC")),
|
||||
"relationship": self._avg(self._collect_rel("IS")),
|
||||
"performance": self._avg([n["performance"]["core_performance"] for n in self.nodes.values()]),
|
||||
"temporal": self._avg(self._collect_resp()),
|
||||
"interaction": self._avg(self._collect_rel("SF"))
|
||||
}
|
||||
|
||||
results["dimensions"] = dimensions
|
||||
return results
|
||||
395
final/scripts/f_relation1125.py
Normal file
395
final/scripts/f_relation1125.py
Normal file
@@ -0,0 +1,395 @@
|
||||
import numpy as np
|
||||
from typing import Dict
|
||||
from .utils import NetworkUtils
|
||||
|
||||
|
||||
class RelationCalculator:
|
||||
"""关系强度计算器"""
|
||||
|
||||
def __init__(self, nodes_data: Dict, cfg):
|
||||
"""
|
||||
初始化
|
||||
:param nodes_data: 节点数据
|
||||
:param auxiliary_data: 辅助数据(接口标准、角色匹配等)
|
||||
"""
|
||||
self.nodes_data = nodes_data
|
||||
self.utils = NetworkUtils()
|
||||
self.cfg = cfg
|
||||
|
||||
def _weights(self, rel: str):
|
||||
return self.cfg["W_REL"][rel]
|
||||
|
||||
def _lambda(self, rel: str):
|
||||
return self.cfg["LAMBDA"][rel]
|
||||
|
||||
def calculate_IS_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算情报保障关系强度
|
||||
L_IS(i,j) = w_f·M_IS_F + w_s·M_IS_S + w_t·M_IS_T + w_p·M_IS_P + w_i·M_IS_I
|
||||
"""
|
||||
# 权重参数(根据1.3.pdf)
|
||||
w = self._weights("IS")
|
||||
# 提取节点数据
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_IS_F
|
||||
f_i_IA = node_i['function_vector']['f_IA']
|
||||
f_j_avg = (
|
||||
node_j['function_vector']['f_CC'] +
|
||||
node_j['function_vector']['f_IC'] +
|
||||
node_j['function_vector']['f_CS'] +
|
||||
node_j['function_vector']['f_CPS']
|
||||
) / 4
|
||||
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_IS_F = f_i_IA * f_j_avg * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_IS_S
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
lambda_IS = self._lambda("IS")
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
M_IS_S = np.exp(-d_ij / lambda_IS) * R_match * s_area
|
||||
|
||||
# 3. 时间维度
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
fresh_i = np.exp(-gamma * node_i['temporal']['data_age'])
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_IS_T = fresh_i * t_i_resp * w_ij
|
||||
|
||||
# 4. 性能维度 M_IS_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
mtbf_max = self.nodes_data['global_params']['mtbf_max']
|
||||
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
|
||||
M_IS_P = p_i_core * p_i_rel
|
||||
|
||||
# 5. 交互维度 M_IS_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
|
||||
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
|
||||
hist_ij = self.utils.get_interaction_history(node_i, node_j)
|
||||
M_IS_I = ((prot_ij + fmt_ij + sec_ij) / 3) * hist_ij
|
||||
|
||||
# 综合计算
|
||||
L_IS = (w["w_f"] * M_IS_F + w["w_s"] * M_IS_S + w["w_t"] * M_IS_T +
|
||||
w["w_p"] * M_IS_P + w["w_i"] * M_IS_I)
|
||||
|
||||
return L_IS
|
||||
|
||||
def calculate_CC_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算指挥控制关系强度
|
||||
L_CC(i,j) = w_f·M_CC_F + w_s·M_CC_S + w_t·M_CC_T + w_p·M_CC_P + w_i·M_CC_I
|
||||
"""
|
||||
# 权重参数
|
||||
w = self._weights("CC")
|
||||
lambda_CC = self._lambda("CC")# 根据公式,指挥控制距离敏感性更高
|
||||
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_CC_F
|
||||
f_i_CC = node_i['function_vector']['f_CC']
|
||||
f_j_sum = sum([
|
||||
node_j['function_vector']['f_IA'],
|
||||
node_j['function_vector']['f_IT'],
|
||||
node_j['function_vector']['f_IC'],
|
||||
node_j['function_vector']['f_CS'],
|
||||
node_j['function_vector']['f_DP'],
|
||||
node_j['function_vector']['f_CPS']
|
||||
])
|
||||
f_j_avg = f_j_sum / 6
|
||||
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_CC_F = f_i_CC * f_j_avg * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_CC_S
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
M_CC_S = np.exp(-d_ij / lambda_CC) * R_match * s_area
|
||||
|
||||
# 3. 时间维度 M_CC_T
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_CC_T = t_i_resp * t_i_cycle * w_ij
|
||||
|
||||
# 4. 性能维度 M_CC_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
p_i_surv = node_i['performance']['survivability']
|
||||
mtbf_max = self.nodes_data['global_params']['mtbf_max']
|
||||
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
|
||||
M_CC_P = p_i_core * p_i_surv * p_i_rel
|
||||
|
||||
# 5. 交互维度 M_CC_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
|
||||
org_ij = self.utils.get_organization_relation(node_i, node_j)
|
||||
hist_ij = self.utils.get_interaction_history(node_i, node_j)
|
||||
M_CC_I = ((prot_ij + sec_ij + org_ij) / 3) * hist_ij
|
||||
|
||||
# 综合计算
|
||||
L_CC = (w["w_f"] * M_CC_F + w["w_s"] * M_CC_S + w["w_t"] * M_CC_T +
|
||||
w["w_p"] * M_CC_P + w["w_i"] * M_CC_I)
|
||||
|
||||
return L_CC
|
||||
|
||||
def calculate_SF_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算状态反馈关系强度
|
||||
L_SF(i,j) = w_f·M_SF_F + w_s·M_SF_S + w_t·M_SF_T + w_p·M_SF_P + w_i·M_SF_I
|
||||
"""
|
||||
# 权重参数
|
||||
w = self._weights("SF")
|
||||
lambda_SF = self._lambda("SF")
|
||||
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_SF_F
|
||||
f_i_sum = sum([
|
||||
node_i['function_vector']['f_IA'],
|
||||
node_i['function_vector']['f_IT'],
|
||||
node_i['function_vector']['f_IC'],
|
||||
node_i['function_vector']['f_CS'],
|
||||
node_i['function_vector']['f_DP'],
|
||||
node_i['function_vector']['f_CPS']
|
||||
])
|
||||
f_i_avg = f_i_sum / 6
|
||||
f_j_CC = node_j['function_vector']['f_CC']
|
||||
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_SF_F = f_i_avg * f_j_CC * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_SF_S
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
M_SF_S = np.exp(-d_ij / lambda_SF) * R_match * s_area
|
||||
|
||||
# 3. 时间维度 M_SF_T(状态反馈对时间敏感)
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
fresh_i = np.exp(-gamma * node_i['temporal']['data_age'])
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_SF_T = fresh_i * t_i_resp * w_ij
|
||||
|
||||
# 4. 性能维度 M_SF_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
mtbf_max = self.nodes_data['global_params']['mtbf_max']
|
||||
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
|
||||
M_SF_P = p_i_core * p_i_rel
|
||||
|
||||
# 5. 交互维度 M_SF_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
|
||||
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
|
||||
hist_ij = self.utils.get_interaction_history(node_i, node_j)
|
||||
M_SF_I = ((prot_ij + fmt_ij + sec_ij) / 3) * hist_ij
|
||||
|
||||
# 综合计算
|
||||
L_SF = (w["w_f"] * M_SF_F + w["w_s"] * M_SF_S + w["w_t"] * M_SF_T +
|
||||
w["w_p"] * M_SF_P + w["w_i"] * M_SF_I)
|
||||
|
||||
return L_SF
|
||||
|
||||
def calculate_PD_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算平台部署关系强度
|
||||
L_PD(i,j) = w_f·M_PD_F + w_s·M_PD_S + w_t·M_PD_T + w_p·M_PD_P + w_i·M_PD_I
|
||||
"""
|
||||
# 权重参数
|
||||
w = self._weights("PD")
|
||||
lambda_PD = self._lambda("PD")
|
||||
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_PD_F
|
||||
f_i_DP = node_i['function_vector']['f_DP']
|
||||
f_J_sum = sum([
|
||||
node_j['function_vector']['f_IA'],
|
||||
node_j['function_vector']['f_IT'],
|
||||
node_j['function_vector']['f_IC'],
|
||||
node_j['function_vector']['f_CS'],
|
||||
])
|
||||
f_J_avg = f_J_sum / 4
|
||||
|
||||
# 修复:直接传递节点数据
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_PD_F = f_i_DP * f_J_avg * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_PD_S(平台部署对空间要求高)
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
# 修复:直接传递节点数据
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
if d_ij <= self.cfg["PD_SGM"]:
|
||||
M_PD_S = 1
|
||||
else:
|
||||
M_PD_S = np.exp(-d_ij / lambda_PD) * R_match * s_area
|
||||
|
||||
# 3. 时间维度 M_PD_T
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_PD_T = t_i_resp * t_i_cycle * w_ij
|
||||
|
||||
# 4. 性能维度 M_PD_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
# 部署平台的生存能力
|
||||
p_i_surv = node_i['performance']['survivability']
|
||||
M_PD_P = p_i_core * p_i_surv
|
||||
|
||||
# 5. 交互维度 M_PD_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
|
||||
# 注意:这里std_ij已经在上面计算过了
|
||||
M_PD_I = (prot_ij + fmt_ij + std_ij) / 3
|
||||
|
||||
# 综合计算
|
||||
L_PD = (w["w_f"] * M_PD_F + w["w_s"] * M_PD_S + w["w_t"] * M_PD_T +
|
||||
w["w_p"] * M_PD_P + w["w_i"] * M_PD_I)
|
||||
|
||||
return L_PD
|
||||
|
||||
def calculate_CO_relation(self, i: str, j: str) -> float:
|
||||
"""
|
||||
计算协同作战关系强度
|
||||
L_CO(i,j) = w_f·M_CO_F + w_s·M_CO_S + w_t·M_CO_T + w_p·M_CO_P + w_i·M_CO_I
|
||||
"""
|
||||
# 权重参数
|
||||
w = self._weights("CO")
|
||||
lambda_CO = self._lambda("CO")
|
||||
node_i = self.nodes_data['nodes'][i]
|
||||
node_j = self.nodes_data['nodes'][j]
|
||||
|
||||
# 1. 功能维度 M_CO_F
|
||||
# 协同作战需要多个功能匹配
|
||||
f_i_avg = sum(node_i['function_vector'].values()) / len(node_i['function_vector'])
|
||||
f_j_avg = sum(node_j['function_vector'].values()) / len(node_j['function_vector'])
|
||||
|
||||
std_ij = self.utils.get_interface_standard(node_i, node_j)
|
||||
r_ij = self.utils.get_role_match(node_i, node_j)
|
||||
M_CO_F = f_i_avg * f_j_avg * std_ij * r_ij
|
||||
|
||||
# 2. 空间维度 M_CO_S
|
||||
d_ij = self.utils.calculate_distance(
|
||||
node_i['spatial']['position'],
|
||||
node_j['spatial']['position']
|
||||
)
|
||||
|
||||
R_match = self.utils.calculate_range_match(
|
||||
node_i['spatial']['effective_radius'],
|
||||
node_j['spatial']['effective_radius'],
|
||||
d_ij
|
||||
)
|
||||
|
||||
s_area = self.utils.get_area_relation(node_i, node_j)
|
||||
M_CO_S = np.exp(-d_ij / lambda_CO) * R_match * s_area
|
||||
|
||||
# 3. 时间维度 M_CO_T(协同需要时间同步)
|
||||
alpha = self.cfg["TIME"]["alpha"]
|
||||
beta = self.cfg["TIME"]["beta"]
|
||||
gamma = self.cfg["TIME"]["gamma"]
|
||||
|
||||
t_i_resp = np.exp(-alpha * node_i['temporal']['response_time'])
|
||||
t_j_resp = np.exp(-alpha * node_j['temporal']['response_time'])
|
||||
|
||||
t_i_cycle = np.exp(-beta * node_i['temporal']['cycle_time'])
|
||||
t_j_cycle = np.exp(-beta * node_j['temporal']['cycle_time'])
|
||||
|
||||
w_ij = self.utils.calculate_time_window_overlap(node_i, node_j)
|
||||
|
||||
M_CO_T = (t_i_resp + t_j_resp) * (t_i_cycle + t_j_cycle) * w_ij * 0.25
|
||||
|
||||
# 4. 性能维度 M_CO_P
|
||||
p_i_core = node_i['performance']['core_performance']
|
||||
p_j_core = node_j['performance']['core_performance']
|
||||
|
||||
mtbf_max = self.nodes_data['global_params']['mtbf_max']
|
||||
p_i_rel = node_i['performance']['mtbf'] / mtbf_max
|
||||
p_j_rel = node_j['performance']['mtbf'] / mtbf_max
|
||||
|
||||
p_i_surv = node_i['performance']['survivability']
|
||||
p_j_surv = node_j['performance']['survivability']
|
||||
# 协同性能取平均
|
||||
M_CO_P = ((p_i_core + p_j_core) / 2) * ((p_i_rel + p_j_rel) / 2) * ((p_i_surv + p_j_surv) / 2)
|
||||
|
||||
# 5. 交互维度 M_CO_I
|
||||
prot_ij = self.utils.get_protocol_compatibility(node_i, node_j)
|
||||
fmt_ij = self.utils.get_format_compatibility(node_i, node_j)
|
||||
sec_ij = self.utils.get_security_compatibility(node_i, node_j)
|
||||
org_ij = self.utils.get_organization_relation(node_i, node_j)
|
||||
hist_ij = self.utils.get_interaction_history(node_i, node_j)
|
||||
M_CO_I = ((prot_ij + fmt_ij + sec_ij + org_ij) / 4) * hist_ij
|
||||
|
||||
# 综合计算
|
||||
L_CO = (w["w_f"] * M_CO_F + w["w_s"] * M_CO_S + w["w_t"] * M_CO_T +
|
||||
w["w_p"] * M_CO_P + w["w_i"] * M_CO_I)
|
||||
|
||||
|
||||
return L_CO
|
||||
241
final/scripts/inspect_dm_tables.py
Normal file
241
final/scripts/inspect_dm_tables.py
Normal file
@@ -0,0 +1,241 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
在达梦数据库机器上运行,查看各表结构,用于编写 config 中的 db.query
|
||||
用法: python scripts/inspect_dm_tables.py
|
||||
|
||||
密码获取顺序:环境变量 DM_PASS > 下方 LOCAL_PASSWORD
|
||||
若使用 LOCAL_PASSWORD,切勿提交到 git!可执行:git update-index --assume-unchanged scripts/inspect_dm_tables.py
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
# 可选:在此直接填写密码(仅限本机调试,切勿提交到版本库!)
|
||||
LOCAL_PASSWORD = None # 例如: "你的密码"
|
||||
|
||||
# 支持从项目根或 scripts 目录运行
|
||||
_script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_project_root = os.path.dirname(_script_dir)
|
||||
XLSX_PATH = os.path.join(_project_root, "1125暂时属性需求(1).xlsx")
|
||||
if _project_root not in sys.path:
|
||||
sys.path.insert(0, _project_root)
|
||||
|
||||
try:
|
||||
import dmPython
|
||||
except ImportError:
|
||||
print("请先安装: pip install dmPython")
|
||||
sys.exit(1)
|
||||
|
||||
CONFIG = {
|
||||
"user": os.getenv("DM_USER", "SYSDBA"),
|
||||
"password": os.getenv("DM_PASS") or LOCAL_PASSWORD,
|
||||
"server": os.getenv("DM_HOST", "127.0.0.1"),
|
||||
"port": int(os.getenv("DM_PORT", "9080")),
|
||||
"schema": "SYSDBA",
|
||||
}
|
||||
|
||||
TABLES = [
|
||||
"OBJECTIVE_INSTANCE",
|
||||
"METRIC_VALUE_INSTANCE",
|
||||
"METRIC_PROFILE",
|
||||
"ROLE_PROFILE",
|
||||
"TASK_PROFILE",
|
||||
"RELATION_INSTANCE",
|
||||
"RES_FORMAL_ANALYSIS",
|
||||
]
|
||||
|
||||
|
||||
def _load_required_attrs(xlsx_path: str) -> list:
|
||||
"""从 1125暂时属性需求(1).xlsx 解析需求属性列表(一条=一个属性)"""
|
||||
if not os.path.exists(xlsx_path):
|
||||
print(f" (未找到 {xlsx_path},使用内置属性列表)")
|
||||
return _DEFAULT_REQ_ATTRS
|
||||
try:
|
||||
import pandas as pd
|
||||
df = pd.read_excel(xlsx_path, header=None)
|
||||
attrs = []
|
||||
for _, row in df.iterrows():
|
||||
v0 = row[0]
|
||||
v2 = str(row[2]) if len(row) > 2 and pd.notna(row[2]) else ""
|
||||
if pd.notna(v0) and str(v0).strip() and str(v0) != "属性代码名称":
|
||||
attrs.append(str(v0).strip())
|
||||
if ":" in v2:
|
||||
sub = v2.split(":")[0].strip()
|
||||
if sub and sub not in attrs:
|
||||
attrs.append(sub)
|
||||
return attrs if attrs else _DEFAULT_REQ_ATTRS
|
||||
except Exception as e:
|
||||
print(f" (解析 xlsx 失败: {e},使用内置属性列表)")
|
||||
return _DEFAULT_REQ_ATTRS
|
||||
|
||||
|
||||
# xlsx 解析失败时的兜底属性列表(来自 1125暂时属性需求)
|
||||
_DEFAULT_REQ_ATTRS = [
|
||||
"f_IC", "f_IA", "f_CS", "f_IT", "f_DP", "f_CPS", "f_CC",
|
||||
"position", "effective_radius",
|
||||
"response_time", "cycle_time", "data_age", "time_window",
|
||||
"core_performance", "survivability", "mtbf",
|
||||
"protocol_list", "format_list", "interface_list",
|
||||
"security_level", "org_unit", "nation", "history_success",
|
||||
]
|
||||
|
||||
|
||||
def main():
|
||||
if not CONFIG["password"]:
|
||||
print("请设置环境变量 DM_PASS 或在脚本中填写 LOCAL_PASSWORD")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
conn = dmPython.connect(
|
||||
user=CONFIG["user"],
|
||||
password=CONFIG["password"],
|
||||
server=CONFIG["server"],
|
||||
port=CONFIG["port"],
|
||||
schema=CONFIG["schema"],
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"连接失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
cursor = conn.cursor()
|
||||
print("=" * 60)
|
||||
print("达梦表结构(用于编写 config/db.query)")
|
||||
print("=" * 60)
|
||||
|
||||
for table in TABLES:
|
||||
try:
|
||||
cursor.execute(f'SELECT * FROM "{CONFIG["schema"]}"."{table}" WHERE 1=0')
|
||||
cols = [d[0] for d in cursor.description]
|
||||
print(f"\n【{table}】 列: {cols}")
|
||||
except Exception as e:
|
||||
print(f"\n【{table}】 查询失败: {e}")
|
||||
|
||||
# 专门查询 METRIC_VALUE_INSTANCE 的字段详情(含完整属性)
|
||||
mvi_table = "METRIC_VALUE_INSTANCE"
|
||||
print("\n" + "=" * 80)
|
||||
print(f"【{mvi_table}】 字段详情(完整属性)")
|
||||
print("=" * 80)
|
||||
try:
|
||||
# 查询 DBA_TAB_COLUMNS 全部常用属性(达梦兼容)
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT COLUMN_NAME, DATA_TYPE, DATA_LENGTH, DATA_PRECISION, DATA_SCALE,
|
||||
NULLABLE, DATA_DEFAULT, COLUMN_ID
|
||||
FROM DBA_TAB_COLUMNS
|
||||
WHERE OWNER = ? AND TABLE_NAME = ?
|
||||
ORDER BY COLUMN_ID
|
||||
""",
|
||||
(CONFIG["schema"], mvi_table),
|
||||
)
|
||||
mvi_cols = cursor.fetchall()
|
||||
# 表头
|
||||
fmt = " {:<18} {:<16} {:<10} {:<10} {:<8} {:<10} {:<20}"
|
||||
print(fmt.format("列名", "数据类型", "长度", "精度", "标度", "可空", "默认值"))
|
||||
print(" " + "-" * 90)
|
||||
for row in mvi_cols:
|
||||
col_name, data_type, data_length, data_precision, data_scale, nullable, data_default, col_id = row
|
||||
# 类型显示:数值型用 类型(精度,标度),字符型用 类型(长度)
|
||||
if data_precision is not None:
|
||||
type_str = f"{data_type}({data_precision},{data_scale or 0})"
|
||||
elif data_length and data_type.upper() in ("VARCHAR", "CHAR", "VARCHAR2"):
|
||||
type_str = f"{data_type}({data_length})"
|
||||
elif data_length:
|
||||
type_str = f"{data_type}({data_length})"
|
||||
else:
|
||||
type_str = data_type or ""
|
||||
len_val = str(data_length) if data_length is not None else "-"
|
||||
prec_val = str(data_precision) if data_precision is not None else "-"
|
||||
scale_val = str(data_scale) if data_scale is not None else "-"
|
||||
null_str = "NULL" if nullable == "Y" else "NOT NULL"
|
||||
default_str = (str(data_default).strip() if data_default else "-")[:18]
|
||||
print(fmt.format(col_name, type_str, len_val, prec_val, scale_val, null_str, default_str))
|
||||
|
||||
# 查询样本数据(若有)
|
||||
cursor.execute(f'SELECT * FROM "{CONFIG["schema"]}"."{mvi_table}" WHERE ROWNUM <= 3')
|
||||
sample_rows = cursor.fetchall()
|
||||
if sample_rows:
|
||||
col_names = [d[0] for d in cursor.description]
|
||||
print(f"\n 样本数据 (最多 3 行):")
|
||||
for i, row in enumerate(sample_rows, 1):
|
||||
print(f" 行{i}: {dict(zip(col_names, row))}")
|
||||
else:
|
||||
print("\n (表无数据)")
|
||||
except Exception as e:
|
||||
print(f" 查询失败: {e}")
|
||||
|
||||
# 按 1125暂时属性需求(1).xlsx 查询 METRIC_VALUE_INSTANCE(一条=一个属性)
|
||||
print("\n" + "=" * 80)
|
||||
print("【METRIC_VALUE_INSTANCE】按需求属性查询(xlsx 中一条 = 一个 METRIC_ID)")
|
||||
print("=" * 80)
|
||||
try:
|
||||
# 1. 从 xlsx 解析需求属性列表
|
||||
req_attrs = _load_required_attrs(XLSX_PATH)
|
||||
print(f"\n xlsx 需求属性 ({len(req_attrs)} 个): {req_attrs}")
|
||||
|
||||
# 2. 查询库中实际存在的 METRIC_ID(distinct)
|
||||
cursor.execute(
|
||||
f'SELECT DISTINCT METRIC_ID FROM "{CONFIG["schema"]}"."METRIC_VALUE_INSTANCE" ORDER BY METRIC_ID'
|
||||
)
|
||||
db_metric_ids = [r[0] for r in cursor.fetchall()]
|
||||
print(f"\n 库中已有 METRIC_ID ({len(db_metric_ids)} 个): {db_metric_ids or '(无数据)'}")
|
||||
|
||||
# 3. METRIC_PROFILE 对照(METRIC_ID -> 中文名)
|
||||
cursor.execute(
|
||||
f'SELECT METRIC_ID, METRIC_NAME FROM "{CONFIG["schema"]}"."METRIC_PROFILE"'
|
||||
)
|
||||
profile_map = dict(cursor.fetchall())
|
||||
if profile_map:
|
||||
print("\n METRIC_PROFILE 对照:")
|
||||
for mid, mname in sorted(profile_map.items()):
|
||||
print(f" {mid} -> {mname}")
|
||||
|
||||
# 4. 交集:需求属性中在库里有的
|
||||
in_both = [a for a in req_attrs if a in db_metric_ids]
|
||||
# 需求有但库无
|
||||
missing = [a for a in req_attrs if a not in db_metric_ids]
|
||||
# 库有但需求未列
|
||||
extra = [m for m in db_metric_ids if m not in req_attrs]
|
||||
|
||||
print(f"\n 需求与库交集 (可查): {in_both or '无'}")
|
||||
print(f" 需求有但库无: {missing or '无'}")
|
||||
print(f" 库有但需求未列: {extra or '无'}")
|
||||
|
||||
# 5. 按属性查询示例 SQL
|
||||
print("\n --- 按属性查询示例 SQL ---")
|
||||
for attr in (in_both or req_attrs)[:5]: # 最多展示 5 个
|
||||
sql = f'SELECT * FROM "{CONFIG["schema"]}"."METRIC_VALUE_INSTANCE" WHERE METRIC_ID = \'{attr}\''
|
||||
print(f" -- {attr}: {sql}")
|
||||
if len(in_both or req_attrs) > 5:
|
||||
print(f" ... 其余属性同理,将 METRIC_ID = 'xxx' 替换即可")
|
||||
except Exception as e:
|
||||
print(f" 查询失败: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
cursor.execute(
|
||||
"""
|
||||
SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE
|
||||
FROM DBA_TAB_COLUMNS
|
||||
WHERE OWNER = ?
|
||||
ORDER BY TABLE_NAME, COLUMN_ID
|
||||
""",
|
||||
(CONFIG["schema"],),
|
||||
)
|
||||
rows = cursor.fetchall()
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("所有列详情 (TABLE_NAME | COLUMN_NAME | DATA_TYPE)")
|
||||
print("=" * 60)
|
||||
cur_table = None
|
||||
for table, col, dtype in rows:
|
||||
if table != cur_table:
|
||||
cur_table = table
|
||||
print(f"\n--- {table} ---")
|
||||
print(f" {col}: {dtype}")
|
||||
|
||||
cursor.close()
|
||||
conn.close()
|
||||
print("\n完成。请根据上述结构在 config 中编写 db.query。")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
167
final/scripts/run_all.py
Normal file
167
final/scripts/run_all.py
Normal file
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
一键运行:实体属性表 + 三元组 的读取、清洗、导出
|
||||
用法: python scripts/run_all.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
_script_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
_project_root = os.path.dirname(_script_dir)
|
||||
sys.path.insert(0, _project_root)
|
||||
|
||||
|
||||
def _json_serializer(obj):
|
||||
if hasattr(obj, 'isoformat'):
|
||||
return obj.isoformat()
|
||||
if hasattr(obj, 'item'):
|
||||
return obj.item()
|
||||
if isinstance(obj, float) and (obj != obj or obj == float('inf') or obj == float('-inf')):
|
||||
return None
|
||||
raise TypeError(f'Object of type {type(obj).__name__} is not JSON serializable')
|
||||
|
||||
|
||||
def load_config(project_root: str) -> dict:
|
||||
config_dir = os.path.join(project_root, 'config')
|
||||
json_path = os.path.join(config_dir, 'config.json')
|
||||
yaml_path = os.path.join(config_dir, 'config.yaml')
|
||||
if os.path.exists(json_path):
|
||||
with open(json_path, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
if os.path.exists(yaml_path):
|
||||
try:
|
||||
import yaml
|
||||
with open(yaml_path, 'r', encoding='utf-8') as f:
|
||||
return yaml.safe_load(f)
|
||||
except ImportError:
|
||||
pass
|
||||
return {'source': 'json', 'json': {}, 'triplet': {}, 'db': {}}
|
||||
|
||||
|
||||
def _resolve_path(path: str, project_root: str) -> str:
|
||||
return path if os.path.isabs(path) else os.path.join(project_root, path)
|
||||
|
||||
|
||||
def main():
|
||||
from src.cleaner import AdvancedDataCleaner
|
||||
from src.data_loader import load_data, load_triplets_from_dm, load_from_json, run_formalization_pipeline
|
||||
from src.triplet_cleaner import clean_triplets
|
||||
|
||||
config = load_config(_project_root)
|
||||
source = config.get('source', 'json')
|
||||
tc = config.get('triplet', {})
|
||||
|
||||
# 三元组:db 模式默认处理;json 模式需 triplet.input_file;triplet.enabled=false 可关闭
|
||||
triplet_input = tc.get('input_file')
|
||||
run_triplets = tc.get('enabled', source == 'db' or bool(triplet_input))
|
||||
|
||||
print("=" * 50)
|
||||
print("作战体系数据清洗 - 一键运行")
|
||||
print("=" * 50)
|
||||
print(f"数据源: {source}")
|
||||
|
||||
# ---------- 1. 实体属性表 ----------
|
||||
print("\n【1/2】实体属性表 读取与清洗")
|
||||
try:
|
||||
data, paths = load_data(config, _project_root)
|
||||
except (FileNotFoundError, ValueError) as e:
|
||||
print(f"错误:{e}")
|
||||
sys.exit(1)
|
||||
|
||||
os.makedirs(os.path.dirname(paths['report']), exist_ok=True)
|
||||
cleaner = AdvancedDataCleaner(output_file=paths['output'], report_file=paths['report'], data=data)
|
||||
cleaner.run()
|
||||
valid_ids = cleaner.get_valid_target_ids()
|
||||
print(f" -> 输出: {paths['output']}")
|
||||
print(f" -> 报告: {paths['report']}")
|
||||
|
||||
# ---------- 2. 三元组 ----------
|
||||
if not run_triplets:
|
||||
print("\n【2/2】三元组 已跳过(triplet.enabled=false 或 json 模式未配置 input_file)")
|
||||
print("\n完成。")
|
||||
return
|
||||
|
||||
print("\n【2/2】三元组 读取与清洗")
|
||||
|
||||
if source == 'db':
|
||||
triplets = load_triplets_from_dm(config, _project_root)
|
||||
print(f" 从达梦读取 {len(triplets)} 条三元组")
|
||||
elif triplet_input:
|
||||
input_path = _resolve_path(triplet_input, _project_root)
|
||||
if os.path.exists(input_path):
|
||||
raw = load_from_json(input_path, _project_root)
|
||||
triplets = raw.get("triples", raw) if isinstance(raw, dict) else raw
|
||||
triplets = triplets if isinstance(triplets, list) else []
|
||||
print(f" 从 {triplet_input} 读取 {len(triplets)} 条三元组")
|
||||
else:
|
||||
print(f" 跳过:三元组文件不存在 {input_path}")
|
||||
triplets = []
|
||||
else:
|
||||
print(" 跳过:json 模式未配置 triplet.input_file")
|
||||
triplets = []
|
||||
|
||||
if not triplets:
|
||||
print(" 无三元组数据,跳过导出")
|
||||
print("\n完成。")
|
||||
return
|
||||
|
||||
# 三元组清洗配置
|
||||
clean_cfg = tc.get('clean', True)
|
||||
do_clean = {'deduplicate': True, 'filter_orphans': False, 'remove_null_core': True, 'normalize_relation_type': True} if clean_cfg is True else (clean_cfg if isinstance(clean_cfg, dict) else {})
|
||||
deduplicate = do_clean.get('deduplicate', True) if do_clean else False
|
||||
filter_orphans = do_clean.get('filter_orphans', False) if do_clean else False
|
||||
remove_null_core = do_clean.get('remove_null_core', True) if do_clean else False
|
||||
normalize_relation_type = do_clean.get('normalize_relation_type', True) if do_clean else False
|
||||
|
||||
if do_clean and (deduplicate or filter_orphans or remove_null_core or normalize_relation_type):
|
||||
triplets, report = clean_triplets(
|
||||
triplets,
|
||||
valid_ids=valid_ids if filter_orphans else None,
|
||||
deduplicate=deduplicate,
|
||||
filter_orphans=filter_orphans,
|
||||
remove_null_core=remove_null_core,
|
||||
normalize_relation_type=normalize_relation_type,
|
||||
)
|
||||
report_path = _resolve_path(tc.get('report_file', 'report/triplet_cleaning_report.json'), _project_root)
|
||||
os.makedirs(os.path.dirname(report_path), exist_ok=True)
|
||||
with open(report_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(report, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
print(f" 清洗: {report['original_count']} -> {report['final_count']}")
|
||||
print(f" 报告: {report_path}")
|
||||
|
||||
# 转为输出格式:head, head_type, relation, tail, tail_type
|
||||
from src.triplet_cleaner import transform_triplet_output_format
|
||||
id_to_role = cleaner.get_id_to_role()
|
||||
triplets = transform_triplet_output_format(triplets, id_to_role)
|
||||
|
||||
output_path = _resolve_path(tc.get('output_file', 'data/triples.json'), _project_root)
|
||||
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(triplets, f, ensure_ascii=False, indent=2, default=_json_serializer)
|
||||
print(f" 输出: {output_path}")
|
||||
|
||||
# ---------- 3. 模块1.3:形式化入库 ----------
|
||||
formal_cfg = config.get("formal", {})
|
||||
if source == "db" and formal_cfg.get("enabled", False):
|
||||
print("\n【3/3】形式化模块 读取-补齐-入库")
|
||||
try:
|
||||
result = run_formalization_pipeline(config)
|
||||
if result.get("ok"):
|
||||
stats = result.get("stats", {})
|
||||
print(f" 批次: {result.get('batch_id')}")
|
||||
print(f" 推理三元组: {stats.get('total_triples', 0)}")
|
||||
print(f" 形式化入库: {stats.get('inserted_formal_count', 0)}")
|
||||
print(f" 属性补齐写回: {stats.get('generated_attrs_count', 0)}")
|
||||
else:
|
||||
print(f" 形式化流程跳过: {result.get('reason', 'unknown')}")
|
||||
except Exception as e:
|
||||
print(f" 形式化流程失败: {e}")
|
||||
raise
|
||||
elif formal_cfg.get("enabled", False):
|
||||
print("\n【3/3】形式化模块 已跳过(仅 source=db 时启用)")
|
||||
|
||||
print("\n完成。")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
110
final/scripts/utils.py
Normal file
110
final/scripts/utils.py
Normal file
@@ -0,0 +1,110 @@
|
||||
import numpy as np
|
||||
from typing import Dict, List, Any
|
||||
from scipy.spatial.distance import euclidean
|
||||
|
||||
class NetworkUtils:
|
||||
"""无需 aux_data,全部现场从节点属性读取"""
|
||||
""" 角色匹配矩阵暂时没加 """
|
||||
# ----------- 通用兜底 -----------
|
||||
_DEFAULT = {
|
||||
"protocol": 0.80,
|
||||
"format": 0.85,
|
||||
"security": 0.90,
|
||||
"role": 0.70,
|
||||
"area": 0.50,
|
||||
"org": 0.70,
|
||||
"area_1": 1,
|
||||
"area_2": 0.7,
|
||||
"history": 0.8
|
||||
}
|
||||
|
||||
# ----------- 空间 -----------
|
||||
@staticmethod
|
||||
def calculate_distance(pos1, pos2):
|
||||
return euclidean(pos1, pos2)
|
||||
|
||||
@staticmethod
|
||||
def calculate_range_match(R_i: float, R_j: float, d_ij: float) -> float:
|
||||
r_min = min(R_i, R_j)
|
||||
if d_ij <= r_min:
|
||||
return 1.0
|
||||
return r_min / d_ij if d_ij > 0 else 0.0
|
||||
|
||||
@staticmethod
|
||||
def calculate_time_window_overlap(node_i, node_j):
|
||||
w_i = node_i["temporal"].get("time_window", 24)
|
||||
w_j = node_j["temporal"].get("time_window", 24)
|
||||
overlap = min(w_i, w_j)
|
||||
return overlap / max(w_i, w_j, 1)
|
||||
|
||||
# ----------- 协议兼容性:Jaccard -----------
|
||||
@staticmethod
|
||||
def get_protocol_compatibility(node_i, node_j, _dummy=None):
|
||||
"""节点属性里放 'protocol_list'"""
|
||||
p_i = set(node_i.get("protocol_list", []))
|
||||
p_j = set(node_j.get("protocol_list", []))
|
||||
if not p_i or not p_j:
|
||||
return NetworkUtils._DEFAULT["protocol"]
|
||||
intersection = len(p_i & p_j)
|
||||
union = len(p_i | p_j)
|
||||
return intersection / union if union else NetworkUtils._DEFAULT["protocol"]
|
||||
|
||||
# ----------- 安全等级:1-5 映射 0-1 -----------
|
||||
@staticmethod
|
||||
def get_security_compatibility(node_i, node_j, _dummy=None):
|
||||
lv_i = node_i.get("security_level", 3)
|
||||
lv_j = node_j.get("security_level", 3)
|
||||
gap = abs(lv_i - lv_j)
|
||||
return max(0, 1 - gap / 5)
|
||||
|
||||
# ----------- 组织隶属:同单位给高分 -----------
|
||||
@staticmethod
|
||||
def get_organization_relation(node_i, node_j, _dummy=None):
|
||||
unit_i = node_i.get("org_unit", "")
|
||||
unit_j = node_j.get("org_unit", "")
|
||||
return 1.0 if unit_i == unit_j and unit_i else NetworkUtils._DEFAULT["org"]
|
||||
|
||||
# ----------- 历史交互:直接读属性 -----------
|
||||
@staticmethod
|
||||
def get_interaction_history(node_i, node_j, _dummy=None):
|
||||
his_i = node_i.get("history_success", NetworkUtils._DEFAULT["history"])
|
||||
# print("111111111111111111111",node_i.get("history_success"))
|
||||
his_j = node_j.get("history_success", NetworkUtils._DEFAULT["history"])
|
||||
return (his_i + his_j) / 2
|
||||
|
||||
# ----------- 区域关联:同 code 高分 -----------
|
||||
@staticmethod
|
||||
def get_area_relation(node_i, node_j, _dummy=None):
|
||||
code_i = node_i.get("nation")
|
||||
code_j = node_j.get("nation")
|
||||
if code_i == code_j:
|
||||
return NetworkUtils._DEFAULT["area_1"]
|
||||
else:
|
||||
return NetworkUtils._DEFAULT["area_2"]
|
||||
|
||||
# ----------- 接口/角色/格式:若节点属性扩展了再读,否则给默认 -----------
|
||||
@staticmethod
|
||||
def get_role_match(node_i, node_j, _dummy=None):
|
||||
return node_i.get("role_match", NetworkUtils._DEFAULT["role"])
|
||||
|
||||
@staticmethod
|
||||
def get_interface_standard(node_i, node_j, _dummy=None) -> float:
|
||||
"""接口标准兼容性 = Jaccard(i_list, j_list)"""
|
||||
i_set = set(node_i.get("interface_list", []))
|
||||
j_set = set(node_j.get("interface_list", []))
|
||||
if not i_set or not j_set:
|
||||
return NetworkUtils._DEFAULT["protocol"] # 默认0.8
|
||||
inter = len(i_set & j_set)
|
||||
union = len(i_set | j_set)
|
||||
return inter / union if union else NetworkUtils._DEFAULT["protocol"]
|
||||
|
||||
@staticmethod
|
||||
def get_format_compatibility(node_i, node_j, _dummy=None) -> float:
|
||||
"""数据格式兼容性 = Jaccard(i_format, j_format)"""
|
||||
i_set = set(node_i.get("format_list", []))
|
||||
j_set = set(node_j.get("format_list", []))
|
||||
if not i_set or not j_set:
|
||||
return NetworkUtils._DEFAULT["format"] # 默认0.85
|
||||
inter = len(i_set & j_set)
|
||||
union = len(i_set | j_set)
|
||||
return inter / union if union else NetworkUtils._DEFAULT["format"]
|
||||
Reference in New Issue
Block a user