提升暴雨产图效率
This commit is contained in:
wzy-warehouse
+107
-82
@@ -36,8 +36,7 @@ _thread_pool = concurrent.futures.ThreadPoolExecutor(
|
||||
)
|
||||
logger.info(f"QGIS 线程池初始化: {_worker_threads} workers")
|
||||
|
||||
# 去重锁：in_progress_locks[disaster_time] = asyncio.Lock
|
||||
# 同一灾害时间的并发请求会排队，避免重复产图
|
||||
# 去重锁
|
||||
_in_progress_locks: dict[str, asyncio.Lock] = {}
|
||||
_locks_lock = asyncio.Lock()
|
||||
|
||||
@@ -375,7 +374,62 @@ async def export_map(req: QgisMapExportRequest):
|
||||
|
||||
|
||||
def _generate_batch_maps(models: list, config: dict, disaster_time: str) -> None:
|
||||
"""通过 QGIS Python 3.12 子进程批量生成专题图（单次 DLL 加载）"""
|
||||
"""并行启动多个 QGIS 子进程，每个处理一部分模板"""
|
||||
import json, math, concurrent.futures
|
||||
|
||||
# 并行度：取配置的 worker 数和模板数的较小值
|
||||
max_workers = getattr(settings, "QGIS_PARALLEL_WORKERS", 4)
|
||||
workers = min(max_workers, len(models))
|
||||
chunk_size = math.ceil(len(models) / workers)
|
||||
|
||||
chunks = []
|
||||
for i in range(0, len(models), chunk_size):
|
||||
chunks.append(models[i:i + chunk_size])
|
||||
|
||||
logger.info(
|
||||
f"[批量产图] {len(models)} 张图 → {len(chunks)} 个并行子进程 "
|
||||
f"(每进程 {chunk_size} 张)"
|
||||
)
|
||||
|
||||
errors = []
|
||||
all_results = []
|
||||
|
||||
def _run_chunk(chunk_models: list, chunk_idx: int) -> list:
|
||||
"""单个子进程处理一批模板"""
|
||||
return _generate_maps_subprocess(chunk_models, config, chunk_idx)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=len(chunks)) as executor:
|
||||
futures = {
|
||||
executor.submit(_run_chunk, chunk, i): i
|
||||
for i, chunk in enumerate(chunks)
|
||||
}
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
chunk_idx = futures[future]
|
||||
try:
|
||||
results = future.result()
|
||||
all_results.extend(results)
|
||||
except Exception as e:
|
||||
logger.error(f"[批量产图] 子进程 {chunk_idx} 失败: {e}")
|
||||
errors.append(str(e))
|
||||
|
||||
if errors and not all_results:
|
||||
raise RuntimeError(f"所有子进程均失败: {'; '.join(errors[:2])}")
|
||||
|
||||
success_count = sum(1 for r in all_results if "error" not in r)
|
||||
fail_count = len(all_results) - success_count
|
||||
logger.info(f"[批量产图] 完成: 成功={success_count}, 失败={fail_count}")
|
||||
for r in all_results:
|
||||
if "error" not in r:
|
||||
logger.info(f" OK {r.get('output', 'N/A')}")
|
||||
else:
|
||||
logger.error(f" FAIL {r['name']}: {r.get('error', 'unknown')}")
|
||||
if success_count == 0 and fail_count > 0:
|
||||
first_err = all_results[0].get("error", "unknown")
|
||||
raise RuntimeError(f"所有模型均失败 ({fail_count}张): {first_err}")
|
||||
|
||||
|
||||
def _generate_maps_subprocess(chunk_models: list, config: dict, chunk_idx: int) -> list:
|
||||
"""单个 QGIS 子进程，处理一批模板，返回结果列表"""
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -383,93 +437,64 @@ def _generate_batch_maps(models: list, config: dict, disaster_time: str) -> None
|
||||
get_qgis_python_path, get_runner_script, build_qgis_subprocess_env,
|
||||
)
|
||||
|
||||
request_data = json.dumps(
|
||||
{"config": config, "models": chunk_models},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
tmp_json = tempfile.NamedTemporaryFile(
|
||||
suffix=".json", delete=False, mode="w", encoding="utf-8"
|
||||
)
|
||||
tmp_json.write(request_data)
|
||||
tmp_json.close()
|
||||
|
||||
try:
|
||||
logger.info(f"[批量产图] 开始: {len(models)} 张专题图, batch={disaster_time}")
|
||||
from config import settings
|
||||
qgis_root = getattr(settings, "QGIS_ROOT", "D:/QGIS")
|
||||
python_path = get_qgis_python_path(qgis_root)
|
||||
if not python_path:
|
||||
raise RuntimeError("未找到 QGIS Python 3.12 解释器")
|
||||
runner_script = get_runner_script()
|
||||
cmd = [python_path, runner_script, tmp_json.name]
|
||||
|
||||
# 构建子进程请求 JSON —— 批量格式
|
||||
request_data = json.dumps(
|
||||
{"config": config, "models": models},
|
||||
ensure_ascii=False,
|
||||
logger.info(f"[子进程{chunk_idx}] 启动: {len(chunk_models)} 张图")
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
timeout=600,
|
||||
env=build_qgis_subprocess_env(qgis_root),
|
||||
)
|
||||
|
||||
# 将请求 JSON 写入临时文件（避免 stdin 管道问题）
|
||||
tmp_json = tempfile.NamedTemporaryFile(
|
||||
suffix=".json", delete=False, mode="w", encoding="utf-8"
|
||||
)
|
||||
tmp_json.write(request_data)
|
||||
tmp_json.close()
|
||||
|
||||
finally:
|
||||
try:
|
||||
from config import settings
|
||||
qgis_root = getattr(settings, "QGIS_ROOT", "D:/QGIS")
|
||||
python_path = get_qgis_python_path(qgis_root)
|
||||
if not python_path:
|
||||
raise RuntimeError("未找到 QGIS Python 3.12 解释器")
|
||||
runner_script = get_runner_script()
|
||||
cmd = [python_path, runner_script, tmp_json.name]
|
||||
os.remove(tmp_json.name)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
logger.info(f"[批量产图] 启动 QGIS 子进程: {python_path}...")
|
||||
stdout_text = result.stdout.decode("utf-8", errors="replace").strip()
|
||||
parsed_output = None
|
||||
if stdout_text:
|
||||
for line in reversed(stdout_text.split("\n")):
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
parsed_output = json.loads(line)
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
result = subprocess.run(
|
||||
cmd,
|
||||
capture_output=True,
|
||||
timeout=600, # 10 分钟超时（15 张模板 × ~40s/张）
|
||||
env=build_qgis_subprocess_env(qgis_root),
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
os.remove(tmp_json.name)
|
||||
except OSError:
|
||||
pass
|
||||
if parsed_output is not None:
|
||||
results = parsed_output.get("results", [])
|
||||
ok = sum(1 for r in results if "error" not in r)
|
||||
logger.info(f"[子进程{chunk_idx}] 完成: {ok}/{len(results)}")
|
||||
return results
|
||||
|
||||
# 解析子进程输出 —— 优先检查 stdout 是否有有效结果
|
||||
stdout_text = result.stdout.decode("utf-8", errors="replace").strip()
|
||||
parsed_output = None
|
||||
if stdout_text:
|
||||
for line in reversed(stdout_text.split("\n")):
|
||||
line = line.strip()
|
||||
if line.startswith("{"):
|
||||
try:
|
||||
parsed_output = json.loads(line)
|
||||
break
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
if parsed_output is not None:
|
||||
batch_results = parsed_output.get("results", [])
|
||||
success_count = sum(1 for r in batch_results if "error" not in r)
|
||||
fail_count = len(batch_results) - success_count
|
||||
logger.info(
|
||||
f"[批量产图] 完成: 成功={success_count}, 失败={fail_count}"
|
||||
)
|
||||
for r in batch_results:
|
||||
if "error" not in r:
|
||||
logger.info(f" OK {r.get('output', 'N/A')}")
|
||||
else:
|
||||
logger.error(f" FAIL {r.get('error', 'unknown')}")
|
||||
if success_count == 0 and fail_count > 0:
|
||||
first_err = batch_results[0].get("error", "unknown")
|
||||
raise RuntimeError(
|
||||
f"QGIS 子进程所有模型均失败 ({fail_count}张): {first_err}"
|
||||
)
|
||||
elif result.returncode != 0:
|
||||
stderr_text = result.stderr.decode("utf-8", errors="replace").strip()
|
||||
logger.error(f"[批量产图] QGIS 子进程失败 (exit={result.returncode}):")
|
||||
for line in stderr_text.split("\n"):
|
||||
logger.error(f" {line}")
|
||||
raise RuntimeError(
|
||||
f"QGIS 子进程失败: {stderr_text[:300]}"
|
||||
)
|
||||
else:
|
||||
logger.warning("[批量产图] 子进程无有效输出，exit code = 0")
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.error(f"[批量产图] QGIS 子进程超时 (600s)")
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"[批量产图] 产图失败: {e}", exc_info=True)
|
||||
raise
|
||||
if result.returncode != 0:
|
||||
stderr_text = result.stderr.decode("utf-8", errors="replace").strip()
|
||||
raise RuntimeError(f"[子进程{chunk_idx}] 失败: {stderr_text[:200]}")
|
||||
|
||||
logger.warning(f"[子进程{chunk_idx}] 无输出")
|
||||
return []
|
||||
|
||||
# ============================================================
|
||||
# 清理函数
|
||||
|
||||
Reference in New Issue
Block a user