优化知识库结构与分类器提示词

tools/rag/ingest.py

@@ -103,16 +103,23 @@ def get_documents(directory: Path):
                     with open(file_path, 'r', encoding='utf-8') as f:
                         count = 0
                         for line in f:
+                            line = line.strip()
+                            if not line:
+                                continue
                             try:
                                 record = json.loads(line)
+                                # 优先使用 record 中的 "text" 字段，若无则用整条记录的 JSON 作为文档内容
                                 if 'text' in record and isinstance(record['text'], str):
-                                    documents.append({
-                                        "text": record['text'],
-                                        "metadata": {"source": str(file_path.name)}
-                                    })
-                                    count += 1
+                                    text = record['text']
+                                else:
+                                    text = json.dumps(record, ensure_ascii=False)
+                                documents.append({
+                                    "text": text,
+                                    "metadata": {"source": str(file_path.name)}
+                                })
+                                count += 1
                             except json.JSONDecodeError:
-                                logging.warning(f"跳过无效的JSON行: {line.strip()}")
+                                logging.warning(f"跳过无效的JSON行: {line[:80]}{'...' if len(line) > 80 else ''}")
                     if count > 0:
                         logging.info(f"成功处理NDJSON文件: {file_path.name}, 提取了 {count} 个文档。")
                 # 对其他所有文件类型，使用unstructured