5f2a43f7 by 柴进

:bug: 一刀砍干净 HistoryManager 所有 O(N) stat 全扫路径

继上次只修 hot path 之后, 把残留的所有冷热路径都改成 raw json
直接操作, 彻底消灭 load_history_index 内的 stat 循环.

修改:
1. _migrate_paths_once: 启动时一次性路径归一化, 抽样 5 条 60% 阈值
   决定是否需要全量迁移, 之后所有路径都是当前 base_path 下的有效绝对路径
2. load_history_index: 简化为 raw read + from_dict + sort, 0 stat
   N=513 时只需几 ms (旧版 ~60ms)
3. delete_history_item: 改 raw json filter
4. _cleanup_old_records: 改 raw json sort + slice
5. 删除已无调用方的 _save_history_index / _fix_history_path

至此 hot path (生成完成, 点击历史项) 与 cold path (删除, 启动清理)
都不再触发 N 次 stat. 唯一保留全量读的 load_history_index 也只有
refresh_history 一处真实调用方.
1 parent 3253b221
......@@ -812,6 +812,13 @@ class HistoryManager:
self.logger.debug(f"历史记录管理器初始化完成,存储路径: {self.base_path}")
# 启动时一次性把过期绝对路径归一化到当前 base_path,
# 之后 load_history_index 不再做任何 stat 循环
try:
self._migrate_paths_once()
except Exception as e:
self.logger.warning(f"启动路径迁移失败 (可忽略): {e}")
def save_generation(self, image_bytes: bytes, prompt: str, reference_images: List[bytes],
aspect_ratio: str, image_size: str, model: str) -> str:
"""保存生成的图片到历史记录
......@@ -929,80 +936,106 @@ class HistoryManager:
pass
return None
def _fix_history_path(self, stored_path: Path, timestamp: str) -> Path:
"""修正历史记录中的路径,使其指向当前 base_path
def _migrate_paths_once(self):
"""启动时一次性路径归一化(取代过去每次 load 都修正的循环)。
历史背景:旧版本 index.json 存绝对路径。.app 重打包或存储位置变更
后,老路径失效。过去做法是 load_history_index 每次都全扫 stat 修正,
N=513 时主线程阻塞 ~60ms × N 次/会话,Mac 上累计触发 jetsam SIGKILL。
存储迁移后,index 里的绝对路径可能指向旧位置(如 .app 内部),
而实际文件已在新的 base_path 下。用 timestamp + 文件名重建路径。
现在改成只在启动时跑一次:
1. 抽样前 5 条,60% 以上路径有效 → 跳过(绝大多数启动走这条 fast path)
2. 否则全量重写所有路径并 save,之后 load_history_index 直接信任 raw
"""
if stored_path.exists():
return stored_path
if not self.history_index_file.exists():
return
try:
with open(self.history_index_file, 'r', encoding='utf-8') as f:
raw = json.load(f)
except Exception as e:
self.logger.warning(f"_migrate_paths_once 读取 index 失败: {e}")
return
if not isinstance(raw, list) or not raw:
return
sample = [d for d in raw[:5] if isinstance(d, dict)]
if not sample:
return
ok = 0
for d in sample:
p = d.get('generated_image_path')
if p and Path(p).exists():
ok += 1
# 60% 阈值:少数损坏可能是孤立文件被人手动删,不需要全迁移
if ok * 10 >= len(sample) * 6:
self.logger.info(
f"[migrate_paths] 抽样 {ok}/{len(sample)} 路径有效,跳过迁移"
)
return
# 尝试在当前 base_path 下找到对应文件
corrected = self.base_path / timestamp / stored_path.name
if corrected.exists():
return corrected
self.logger.info(
f"[migrate_paths] 抽样 {ok}/{len(sample)} 路径有效,开始全量迁移 {len(raw)} 条"
)
changed = 0
for d in raw:
if not isinstance(d, dict):
continue
ts = d.get('timestamp')
if not ts:
continue
gen_p = d.get('generated_image_path')
if gen_p and not Path(gen_p).exists():
cand = self.base_path / ts / Path(gen_p).name
if cand.exists():
d['generated_image_path'] = str(cand)
changed += 1
refs = d.get('reference_image_paths') or []
new_refs = []
for r in refs:
if Path(r).exists():
new_refs.append(r)
else:
cand = self.base_path / ts / Path(r).name
if cand.exists():
new_refs.append(str(cand))
changed += 1
else:
new_refs.append(r)
d['reference_image_paths'] = new_refs
return stored_path
if changed > 0:
self.logger.info(f"[migrate_paths] 修正 {changed} 个路径, 写回 index.json")
try:
with open(self.history_index_file, 'w', encoding='utf-8') as f:
json.dump(raw, f, ensure_ascii=False, indent=2)
except Exception as e:
self.logger.error(f"[migrate_paths] 写回失败: {e}")
def load_history_index(self) -> List[HistoryItem]:
"""加载历史记录索引
"""加载历史记录索引(仅 raw read + from_dict + sort)。
Returns:
历史记录项列表,按时间戳倒序排列
路径修正已下放到 __init__ 时的 _migrate_paths_once,
load 路径完全没有 stat 系统调用,N=513 时只需几 ms。
"""
if not self.history_index_file.exists():
return []
try:
self.logger.info("[load_history_index] 开始读取 index.json")
_flush_logs()
with open(self.history_index_file, 'r', encoding='utf-8') as f:
data = json.load(f)
self.logger.info(f"[load_history_index] JSON 解析完成, {len(data)} 条原始数据")
_flush_logs()
history_items = [HistoryItem.from_dict(item) for item in data]
total = len(history_items)
# 修正可能过期的绝对路径(存储迁移后旧路径不再有效)
# 每条 item 对每个 path 都要 stat; 这是 UI 主线程上可能拖死的点
needs_save = False
for idx, item in enumerate(history_items):
fixed_gen = self._fix_history_path(item.generated_image_path, item.timestamp)
if fixed_gen != item.generated_image_path:
item.generated_image_path = fixed_gen
needs_save = True
fixed_refs = []
for ref_path in item.reference_image_paths:
fixed_ref = self._fix_history_path(ref_path, item.timestamp)
if fixed_ref != ref_path:
needs_save = True
fixed_refs.append(fixed_ref)
item.reference_image_paths = fixed_refs
if (idx + 1) % 20 == 0:
self.logger.info(f"[load_history_index] 路径修正进度 {idx + 1}/{total}")
_flush_logs()
self.logger.info(f"[load_history_index] 路径修正完成, needs_save={needs_save}")
_flush_logs()
# 路径修正后持久化,避免每次都修正
if needs_save:
self.logger.info("检测到历史记录路径变更,已自动修正")
self._save_history_index(history_items)
_flush_logs()
# 按时间戳倒序排列
history_items.sort(key=lambda x: x.timestamp, reverse=True)
self.logger.info(f"[load_history_index] 返回 {total} 条")
_flush_logs()
return history_items
if not isinstance(data, list):
return []
items: List[HistoryItem] = []
for d in data:
if not isinstance(d, dict):
continue
try:
items.append(HistoryItem.from_dict(d))
except Exception:
continue
items.sort(key=lambda x: x.timestamp, reverse=True)
return items
except Exception as e:
self.logger.error(f"加载历史记录索引失败: {e}", exc_info=True)
_flush_logs()
return []
def get_history_item(self, timestamp: str) -> Optional[HistoryItem]:
......@@ -1066,14 +1099,26 @@ class HistoryManager:
if record_dir.exists():
shutil.rmtree(record_dir)
# 更新索引文件
history_items = self.load_history_index()
history_items = [item for item in history_items if item.timestamp != timestamp]
self._save_history_index(history_items)
# 直接对 raw json 操作, 避免 load_history_index O(N) 全扫
if not self.history_index_file.exists():
return True
try:
with open(self.history_index_file, 'r', encoding='utf-8') as f:
raw = json.load(f)
if not isinstance(raw, list):
raw = []
except Exception:
raw = []
raw = [d for d in raw if isinstance(d, dict) and d.get('timestamp') != timestamp]
try:
with open(self.history_index_file, 'w', encoding='utf-8') as f:
json.dump(raw, f, ensure_ascii=False, indent=2)
except Exception as e:
self.logger.error(f"删除后写回索引失败: {e}")
return True
except Exception as e:
print(f"删除历史记录失败: {e}")
self.logger.error(f"删除历史记录失败: {e}")
return False
def _update_history_index(self, history_item: HistoryItem):
......@@ -1107,40 +1152,46 @@ class HistoryManager:
except Exception as e:
self.logger.error(f"_update_history_index 写入索引失败: {e}")
def _save_history_index(self, history_items: List[HistoryItem]):
"""保存历史记录索引到文件
Args:
history_items: 历史记录项列表
"""
try:
data = [item.to_dict() for item in history_items]
with open(self.history_index_file, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
except Exception as e:
print(f"保存历史记录索引失败: {e}")
def _cleanup_old_records(self):
"""清理旧的历史记录,保持最大数量限制。max_history_count <= 0 表示不限制。"""
if self.max_history_count <= 0:
return
history_items = self.load_history_index()
if len(history_items) > self.max_history_count:
# 保留最新的记录
items_to_keep = history_items[:self.max_history_count]
items_to_remove = history_items[self.max_history_count:]
# 删除多余记录的文件
for item in items_to_remove:
record_dir = self.base_path / item.timestamp
if record_dir.exists():
try:
shutil.rmtree(record_dir)
except Exception as e:
print(f"删除旧历史记录失败 {item.timestamp}: {e}")
if not self.history_index_file.exists():
return
try:
with open(self.history_index_file, 'r', encoding='utf-8') as f:
raw = json.load(f)
if not isinstance(raw, list):
return
except Exception as e:
self.logger.error(f"_cleanup_old_records 读取索引失败: {e}")
return
# 倒序排列,保留前 N 条
raw.sort(key=lambda d: d.get('timestamp', '') if isinstance(d, dict) else '', reverse=True)
if len(raw) <= self.max_history_count:
return
keep = raw[:self.max_history_count]
remove = raw[self.max_history_count:]
for d in remove:
if not isinstance(d, dict):
continue
ts = d.get('timestamp')
if not ts:
continue
record_dir = self.base_path / ts
if record_dir.exists():
try:
shutil.rmtree(record_dir)
except Exception as e:
self.logger.warning(f"删除旧记录失败 {ts}: {e}")
# 更新索引文件
self._save_history_index(items_to_keep)
try:
with open(self.history_index_file, 'w', encoding='utf-8') as f:
json.dump(keep, f, ensure_ascii=False, indent=2)
except Exception as e:
self.logger.error(f"_cleanup_old_records 写回索引失败: {e}")
class LoginDialog(QDialog):
......