diff --git a/app/api/qgis_map_export.py b/app/api/qgis_map_export.py index e8b0610..9dd79be 100644 --- a/app/api/qgis_map_export.py +++ b/app/api/qgis_map_export.py @@ -36,8 +36,7 @@ _thread_pool = concurrent.futures.ThreadPoolExecutor( ) logger.info(f"QGIS 线程池初始化: {_worker_threads} workers") -# 去重锁:in_progress_locks[disaster_time] = asyncio.Lock -# 同一灾害时间的并发请求会排队,避免重复产图 +# 去重锁 _in_progress_locks: dict[str, asyncio.Lock] = {} _locks_lock = asyncio.Lock() @@ -375,7 +374,62 @@ async def export_map(req: QgisMapExportRequest): def _generate_batch_maps(models: list, config: dict, disaster_time: str) -> None: - """通过 QGIS Python 3.12 子进程批量生成专题图(单次 DLL 加载)""" + """并行启动多个 QGIS 子进程,每个处理一部分模板""" + import json, math, concurrent.futures + + # 并行度:取配置的 worker 数和模板数的较小值 + max_workers = getattr(settings, "QGIS_PARALLEL_WORKERS", 4) + workers = min(max_workers, len(models)) + chunk_size = math.ceil(len(models) / workers) + + chunks = [] + for i in range(0, len(models), chunk_size): + chunks.append(models[i:i + chunk_size]) + + logger.info( + f"[批量产图] {len(models)} 张图 → {len(chunks)} 个并行子进程 " + f"(每进程 {chunk_size} 张)" + ) + + errors = [] + all_results = [] + + def _run_chunk(chunk_models: list, chunk_idx: int) -> list: + """单个子进程处理一批模板""" + return _generate_maps_subprocess(chunk_models, config, chunk_idx) + + with concurrent.futures.ThreadPoolExecutor(max_workers=len(chunks)) as executor: + futures = { + executor.submit(_run_chunk, chunk, i): i + for i, chunk in enumerate(chunks) + } + for future in concurrent.futures.as_completed(futures): + chunk_idx = futures[future] + try: + results = future.result() + all_results.extend(results) + except Exception as e: + logger.error(f"[批量产图] 子进程 {chunk_idx} 失败: {e}") + errors.append(str(e)) + + if errors and not all_results: + raise RuntimeError(f"所有子进程均失败: {'; '.join(errors[:2])}") + + success_count = sum(1 for r in all_results if "error" not in r) + fail_count = len(all_results) - success_count + logger.info(f"[批量产图] 完成: 成功={success_count}, 失败={fail_count}") + for r in all_results: + if "error" not in r: + logger.info(f" OK {r.get('output', 'N/A')}") + else: + logger.error(f" FAIL {r['name']}: {r.get('error', 'unknown')}") + if success_count == 0 and fail_count > 0: + first_err = all_results[0].get("error", "unknown") + raise RuntimeError(f"所有模型均失败 ({fail_count}张): {first_err}") + + +def _generate_maps_subprocess(chunk_models: list, config: dict, chunk_idx: int) -> list: + """单个 QGIS 子进程,处理一批模板,返回结果列表""" import json import subprocess import tempfile @@ -383,93 +437,64 @@ def _generate_batch_maps(models: list, config: dict, disaster_time: str) -> None get_qgis_python_path, get_runner_script, build_qgis_subprocess_env, ) + request_data = json.dumps( + {"config": config, "models": chunk_models}, + ensure_ascii=False, + ) + + tmp_json = tempfile.NamedTemporaryFile( + suffix=".json", delete=False, mode="w", encoding="utf-8" + ) + tmp_json.write(request_data) + tmp_json.close() + try: - logger.info(f"[批量产图] 开始: {len(models)} 张专题图, batch={disaster_time}") + from config import settings + qgis_root = getattr(settings, "QGIS_ROOT", "D:/QGIS") + python_path = get_qgis_python_path(qgis_root) + if not python_path: + raise RuntimeError("未找到 QGIS Python 3.12 解释器") + runner_script = get_runner_script() + cmd = [python_path, runner_script, tmp_json.name] - # 构建子进程请求 JSON —— 批量格式 - request_data = json.dumps( - {"config": config, "models": models}, - ensure_ascii=False, + logger.info(f"[子进程{chunk_idx}] 启动: {len(chunk_models)} 张图") + + result = subprocess.run( + cmd, + capture_output=True, + timeout=600, + env=build_qgis_subprocess_env(qgis_root), ) - - # 将请求 JSON 写入临时文件(避免 stdin 管道问题) - tmp_json = tempfile.NamedTemporaryFile( - suffix=".json", delete=False, mode="w", encoding="utf-8" - ) - tmp_json.write(request_data) - tmp_json.close() - + finally: try: - from config import settings - qgis_root = getattr(settings, "QGIS_ROOT", "D:/QGIS") - python_path = get_qgis_python_path(qgis_root) - if not python_path: - raise RuntimeError("未找到 QGIS Python 3.12 解释器") - runner_script = get_runner_script() - cmd = [python_path, runner_script, tmp_json.name] + os.remove(tmp_json.name) + except OSError: + pass - logger.info(f"[批量产图] 启动 QGIS 子进程: {python_path}...") + stdout_text = result.stdout.decode("utf-8", errors="replace").strip() + parsed_output = None + if stdout_text: + for line in reversed(stdout_text.split("\n")): + line = line.strip() + if line.startswith("{"): + try: + parsed_output = json.loads(line) + break + except json.JSONDecodeError: + continue - result = subprocess.run( - cmd, - capture_output=True, - timeout=600, # 10 分钟超时(15 张模板 × ~40s/张) - env=build_qgis_subprocess_env(qgis_root), - ) - finally: - try: - os.remove(tmp_json.name) - except OSError: - pass + if parsed_output is not None: + results = parsed_output.get("results", []) + ok = sum(1 for r in results if "error" not in r) + logger.info(f"[子进程{chunk_idx}] 完成: {ok}/{len(results)}") + return results - # 解析子进程输出 —— 优先检查 stdout 是否有有效结果 - stdout_text = result.stdout.decode("utf-8", errors="replace").strip() - parsed_output = None - if stdout_text: - for line in reversed(stdout_text.split("\n")): - line = line.strip() - if line.startswith("{"): - try: - parsed_output = json.loads(line) - break - except json.JSONDecodeError: - continue - - if parsed_output is not None: - batch_results = parsed_output.get("results", []) - success_count = sum(1 for r in batch_results if "error" not in r) - fail_count = len(batch_results) - success_count - logger.info( - f"[批量产图] 完成: 成功={success_count}, 失败={fail_count}" - ) - for r in batch_results: - if "error" not in r: - logger.info(f" OK {r.get('output', 'N/A')}") - else: - logger.error(f" FAIL {r.get('error', 'unknown')}") - if success_count == 0 and fail_count > 0: - first_err = batch_results[0].get("error", "unknown") - raise RuntimeError( - f"QGIS 子进程所有模型均失败 ({fail_count}张): {first_err}" - ) - elif result.returncode != 0: - stderr_text = result.stderr.decode("utf-8", errors="replace").strip() - logger.error(f"[批量产图] QGIS 子进程失败 (exit={result.returncode}):") - for line in stderr_text.split("\n"): - logger.error(f" {line}") - raise RuntimeError( - f"QGIS 子进程失败: {stderr_text[:300]}" - ) - else: - logger.warning("[批量产图] 子进程无有效输出,exit code = 0") - - except subprocess.TimeoutExpired: - logger.error(f"[批量产图] QGIS 子进程超时 (600s)") - raise - except Exception as e: - logger.error(f"[批量产图] 产图失败: {e}", exc_info=True) - raise + if result.returncode != 0: + stderr_text = result.stderr.decode("utf-8", errors="replace").strip() + raise RuntimeError(f"[子进程{chunk_idx}] 失败: {stderr_text[:200]}") + logger.warning(f"[子进程{chunk_idx}] 无输出") + return [] # ============================================================ # 清理函数 diff --git a/app/config/qgis_mappings.py b/app/config/qgis_mappings.py index 5e6d782..26430a9 100644 --- a/app/config/qgis_mappings.py +++ b/app/config/qgis_mappings.py @@ -29,6 +29,12 @@ STATIC_LAYERS = { "traffic_railway": {"table": "base.traffic_railway", "gpkg": "traffic_railway.gpkg"}, "traffic_township": {"table": "base.traffic_township", "gpkg": "traffic_township.gpkg"}, "traffic_trunk_line": {"table": "base.traffic_trunk_line", "gpkg": "traffic_trunk_line.gpkg"}, + # ── 新增静态层 ── + "积水点": {"table": "base.hazard_hydrops", "gpkg": "hazard_hydrops.gpkg"}, + "排水口": {"table": "base.lifeline_outfall", "gpkg": "lifeline_outfall.gpkg"}, + "供水管网": {"table": "base.lifeline_pipe", "gpkg": "lifeline_pipe.gpkg"}, + "risk_census_population": {"table": "base.risk_census_population", "gpkg": "risk_census_population.gpkg"}, + "西安乡镇": {"table": "base.sx_xa_towns", "gpkg": "sx_xa_towns.gpkg"}, } # ============================================================ diff --git a/app/core/server.py b/app/core/server.py index e0568fd..46711dd 100644 --- a/app/core/server.py +++ b/app/core/server.py @@ -16,21 +16,21 @@ logger = get_logger("api") @asynccontextmanager async def lifespan(app: FastAPI): - """应用生命周期:启动时预加载模型""" + """应用生命周期:启动时预加载 DBN 模型""" logger.info("正在预加载DBN模型...") get_rainfall_model() get_earthquake_model() logger.info("DBN模型预加载完成") - # 检测 QGIS 子进程环境 + # 检测 QGIS 子进程环境(产图时按需启动子进程) qgis_root = getattr(settings, "QGIS_ROOT", None) if qgis_root: try: from app.services.qgis.qgis_env import is_qgis_available if is_qgis_available(qgis_root): - logger.info("QGIS 环境检测通过(子进程模式)") + logger.info("QGIS 环境检测通过") else: - logger.warning("QGIS 环境不可用(未找到 Python 3.12 解释器),专题图功能降级") + logger.warning("QGIS 环境不可用,专题图功能降级") except Exception as e: logger.error(f"QGIS 环境检测失败: {e}") diff --git a/app/data/cache/qgis_templates/19470628504c856d.qgz b/app/data/cache/qgis_templates/19470628504c856d.qgz index e179088..4237f75 100644 Binary files a/app/data/cache/qgis_templates/19470628504c856d.qgz and b/app/data/cache/qgis_templates/19470628504c856d.qgz differ diff --git a/app/data/cache/qgis_templates/32dc5df99342a7da.qgz b/app/data/cache/qgis_templates/32dc5df99342a7da.qgz index e1bb431..8aa0455 100644 Binary files a/app/data/cache/qgis_templates/32dc5df99342a7da.qgz and b/app/data/cache/qgis_templates/32dc5df99342a7da.qgz differ diff --git a/app/data/cache/qgis_templates/3bac2f2636b197e9.qgz b/app/data/cache/qgis_templates/3bac2f2636b197e9.qgz index de40752..b3d5895 100644 Binary files a/app/data/cache/qgis_templates/3bac2f2636b197e9.qgz and b/app/data/cache/qgis_templates/3bac2f2636b197e9.qgz differ diff --git a/app/data/cache/qgis_templates/41f47cbdd9c9c8d0.qgz b/app/data/cache/qgis_templates/41f47cbdd9c9c8d0.qgz index 74ce262..7febeb0 100644 Binary files a/app/data/cache/qgis_templates/41f47cbdd9c9c8d0.qgz and b/app/data/cache/qgis_templates/41f47cbdd9c9c8d0.qgz differ diff --git a/app/data/cache/qgis_templates/4c40d3feeddf4567.qgz b/app/data/cache/qgis_templates/4c40d3feeddf4567.qgz index f7c87a2..0a6a778 100644 Binary files a/app/data/cache/qgis_templates/4c40d3feeddf4567.qgz and b/app/data/cache/qgis_templates/4c40d3feeddf4567.qgz differ diff --git a/app/data/cache/qgis_templates/5a480fbeb8b2c66c.qgz b/app/data/cache/qgis_templates/5a480fbeb8b2c66c.qgz index 18f6158..bbba32e 100644 Binary files a/app/data/cache/qgis_templates/5a480fbeb8b2c66c.qgz and b/app/data/cache/qgis_templates/5a480fbeb8b2c66c.qgz differ diff --git a/app/data/cache/qgis_templates/66a88abaa976d72a.qgz b/app/data/cache/qgis_templates/66a88abaa976d72a.qgz index 6a3e52d..36d821e 100644 Binary files a/app/data/cache/qgis_templates/66a88abaa976d72a.qgz and b/app/data/cache/qgis_templates/66a88abaa976d72a.qgz differ diff --git a/app/data/cache/qgis_templates/82ba30aacf8a53dd.qgz b/app/data/cache/qgis_templates/82ba30aacf8a53dd.qgz index 5c28b9d..eedc10a 100644 Binary files a/app/data/cache/qgis_templates/82ba30aacf8a53dd.qgz and b/app/data/cache/qgis_templates/82ba30aacf8a53dd.qgz differ diff --git a/app/data/cache/qgis_templates/98c5467c5904eee5.qgz b/app/data/cache/qgis_templates/98c5467c5904eee5.qgz index dea99cb..f2b2da0 100644 Binary files a/app/data/cache/qgis_templates/98c5467c5904eee5.qgz and b/app/data/cache/qgis_templates/98c5467c5904eee5.qgz differ diff --git a/app/data/cache/qgis_templates/bc46a5234f68e54e.qgz b/app/data/cache/qgis_templates/bc46a5234f68e54e.qgz index aabb84f..1f2747f 100644 Binary files a/app/data/cache/qgis_templates/bc46a5234f68e54e.qgz and b/app/data/cache/qgis_templates/bc46a5234f68e54e.qgz differ diff --git a/app/data/cache/qgis_templates/d8c1e99a6640a0fa.qgz b/app/data/cache/qgis_templates/d8c1e99a6640a0fa.qgz index 1a45458..cc0a0ef 100644 Binary files a/app/data/cache/qgis_templates/d8c1e99a6640a0fa.qgz and b/app/data/cache/qgis_templates/d8c1e99a6640a0fa.qgz differ diff --git a/app/data/cache/qgis_templates/d94f0d54dd607cbf.qgz b/app/data/cache/qgis_templates/d94f0d54dd607cbf.qgz index 4f2e411..e4b78cc 100644 Binary files a/app/data/cache/qgis_templates/d94f0d54dd607cbf.qgz and b/app/data/cache/qgis_templates/d94f0d54dd607cbf.qgz differ diff --git a/app/data/cache/qgis_templates/dc2534d5788a805f.qgz b/app/data/cache/qgis_templates/dc2534d5788a805f.qgz index 6cbfe02..e18c4e7 100644 Binary files a/app/data/cache/qgis_templates/dc2534d5788a805f.qgz and b/app/data/cache/qgis_templates/dc2534d5788a805f.qgz differ diff --git a/app/data/cache/qgis_templates/ef423854adc709b4.qgz b/app/data/cache/qgis_templates/ef423854adc709b4.qgz index 597e7aa..0fdf90c 100644 Binary files a/app/data/cache/qgis_templates/ef423854adc709b4.qgz and b/app/data/cache/qgis_templates/ef423854adc709b4.qgz differ diff --git a/app/data/cache/qgis_templates/f0cf04a82b3bfb58.qgz b/app/data/cache/qgis_templates/f0cf04a82b3bfb58.qgz index 57ec410..8fecf3d 100644 Binary files a/app/data/cache/qgis_templates/f0cf04a82b3bfb58.qgz and b/app/data/cache/qgis_templates/f0cf04a82b3bfb58.qgz differ diff --git a/app/data/cache/qgis_templates/index.json b/app/data/cache/qgis_templates/index.json index 3434f92..a574ebf 100644 --- a/app/data/cache/qgis_templates/index.json +++ b/app/data/cache/qgis_templates/index.json @@ -32,8 +32,8 @@ "ymax": 33.622777448228966 } }, - "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨地灾风险区分布图.qgz": { - "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\41f47cbdd9c9c8d0.qgz", + "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨泥石流潜在隐患点及农作物分布图.qgz": { + "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\32dc5df99342a7da.qgz", "layout": "A3", "texts": { "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", @@ -42,14 +42,14 @@ "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" }, "extent": { - "xmin": 107.87768948431949, - "ymin": 32.972841920667406, - "xmax": 108.96006631597123, - "ymax": 33.68287129318787 + "xmin": 107.95707951802657, + "ymin": 32.9127480757085, + "xmax": 109.03945634967837, + "ymax": 33.622777448228966 } }, - "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨城市生命线工程分布图.qgz": { - "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\82ba30aacf8a53dd.qgz", + "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨防汛物资分布图.qgz": { + "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\f0cf04a82b3bfb58.qgz", "layout": "A3", "texts": { "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", @@ -58,10 +58,10 @@ "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" }, "extent": { - "xmin": 107.46472706539424, - "ymin": 32.921385746843555, - "xmax": 108.51635924537791, - "ymax": 33.61124690706137 + "xmin": 107.95707951802657, + "ymin": 32.9127480757085, + "xmax": 109.03945634967837, + "ymax": 33.622777448228966 } }, "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨山洪潜在隐患点及人口分布图.qgz": { @@ -96,40 +96,8 @@ "ymax": 33.622777448228966 } }, - "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨救援队伍分布图.qgz": { - "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\d8c1e99a6640a0fa.qgz", - "layout": "A3", - "texts": { - "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", - "mapTime": "制图时间:2020年11月", - "mapUint": "制图单位:四川省地震应急分服务中心", - "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" - }, - "extent": { - "xmin": 107.95707951802657, - "ymin": 32.9127480757085, - "xmax": 109.03945634967837, - "ymax": 33.622777448228966 - } - }, - "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨泥石流潜在隐患点及人口分布图.qgz": { - "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\5a480fbeb8b2c66c.qgz", - "layout": "A3", - "texts": { - "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", - "mapTime": "制图时间:2020年11月", - "mapUint": "制图单位:四川省地震应急分服务中心", - "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" - }, - "extent": { - "xmin": 107.95707951802657, - "ymin": 32.9127480757085, - "xmax": 109.03945634967837, - "ymax": 33.622777448228966 - } - }, - "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨泥石流潜在隐患点及农作物分布图.qgz": { - "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\32dc5df99342a7da.qgz", + "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨附近医院分布图.qgz": { + "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\ef423854adc709b4.qgz", "layout": "A3", "texts": { "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", @@ -160,6 +128,38 @@ "ymax": 33.622777448228966 } }, + "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨地灾风险区分布图.qgz": { + "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\41f47cbdd9c9c8d0.qgz", + "layout": "A3", + "texts": { + "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", + "mapTime": "制图时间:2020年11月", + "mapUint": "制图单位:四川省地震应急分服务中心", + "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" + }, + "extent": { + "xmin": 107.87768948431949, + "ymin": 32.972841920667406, + "xmax": 108.96006631597123, + "ymax": 33.68287129318787 + } + }, + "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨附近水库分布图.qgz": { + "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\dc2534d5788a805f.qgz", + "layout": "A3", + "texts": { + "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", + "mapTime": "制图时间:2020年11月", + "mapUint": "制图单位:四川省地震应急分服务中心", + "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" + }, + "extent": { + "xmin": 107.95707951802657, + "ymin": 32.9127480757085, + "xmax": 109.03945634967837, + "ymax": 33.622777448228966 + } + }, "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨滑坡潜在隐患点及农作物分布图.qgz": { "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\bc46a5234f68e54e.qgz", "layout": "A3", @@ -176,6 +176,54 @@ "ymax": 33.622777448228966 } }, + "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨救援队伍分布图.qgz": { + "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\d8c1e99a6640a0fa.qgz", + "layout": "A3", + "texts": { + "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", + "mapTime": "制图时间:2020年11月", + "mapUint": "制图单位:四川省地震应急分服务中心", + "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" + }, + "extent": { + "xmin": 107.95707951802657, + "ymin": 32.9127480757085, + "xmax": 109.03945634967837, + "ymax": 33.622777448228966 + } + }, + "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨城市生命线工程分布图.qgz": { + "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\82ba30aacf8a53dd.qgz", + "layout": "A3", + "texts": { + "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", + "mapTime": "制图时间:2020年11月", + "mapUint": "制图单位:四川省地震应急分服务中心", + "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" + }, + "extent": { + "xmin": 107.46472706539424, + "ymin": 32.921385746843555, + "xmax": 108.51635924537791, + "ymax": 33.61124690706137 + } + }, + "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨泥石流潜在隐患点及人口分布图.qgz": { + "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\5a480fbeb8b2c66c.qgz", + "layout": "A3", + "texts": { + "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", + "mapTime": "制图时间:2020年11月", + "mapUint": "制图单位:四川省地震应急分服务中心", + "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" + }, + "extent": { + "xmin": 107.95707951802657, + "ymin": 32.9127480757085, + "xmax": 109.03945634967837, + "ymax": 33.622777448228966 + } + }, "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨避难场所分布图.qgz": { "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\d94f0d54dd607cbf.qgz", "layout": "A3", @@ -191,54 +239,6 @@ "xmax": 108.51635924537791, "ymax": 33.61124690706137 } - }, - "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨防汛物资分布图.qgz": { - "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\f0cf04a82b3bfb58.qgz", - "layout": "A3", - "texts": { - "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", - "mapTime": "制图时间:2020年11月", - "mapUint": "制图单位:四川省地震应急分服务中心", - "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" - }, - "extent": { - "xmin": 107.95707951802657, - "ymin": 32.9127480757085, - "xmax": 109.03945634967837, - "ymax": 33.622777448228966 - } - }, - "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨附近医院分布图.qgz": { - "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\ef423854adc709b4.qgz", - "layout": "A3", - "texts": { - "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", - "mapTime": "制图时间:2020年11月", - "mapUint": "制图单位:四川省地震应急分服务中心", - "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" - }, - "extent": { - "xmin": 107.95707951802657, - "ymin": 32.9127480757085, - "xmax": 109.03945634967837, - "ymax": 33.622777448228966 - } - }, - "F:/project/xian/xian_algorithm_new/app/data/template/rainfall/暴雨附近水库分布图.qgz": { - "file": "F:\\project\\xian\\xian_algorithm_new\\app\\data\\cache\\qgis_templates\\dc2534d5788a805f.qgz", - "layout": "A3", - "texts": { - "mapTitle": "四川长宁6.0级地震震中附近乡镇距离分布图", - "mapTime": "制图时间:2020年11月", - "mapUint": "制图单位:四川省地震应急分服务中心", - "info": "时间:2019年06月17日22点55分\n震级:6.0级\n位置:四川省宜宾市长宁县双河镇" - }, - "extent": { - "xmin": 107.95707951802657, - "ymin": 32.9127480757085, - "xmax": 109.03945634967837, - "ymax": 33.622777448228966 - } } } } \ No newline at end of file diff --git a/app/data/gpkg/active_fault.gpkg b/app/data/gpkg/active_fault.gpkg index cdcaef5..d3986c3 100644 Binary files a/app/data/gpkg/active_fault.gpkg and b/app/data/gpkg/active_fault.gpkg differ diff --git a/app/data/gpkg/hazard_hydrops.gpkg b/app/data/gpkg/hazard_hydrops.gpkg new file mode 100644 index 0000000..45e6953 Binary files /dev/null and b/app/data/gpkg/hazard_hydrops.gpkg differ diff --git a/app/data/gpkg/lifeline_outfall.gpkg b/app/data/gpkg/lifeline_outfall.gpkg new file mode 100644 index 0000000..c5023b6 Binary files /dev/null and b/app/data/gpkg/lifeline_outfall.gpkg differ diff --git a/app/data/gpkg/lifeline_pipe.gpkg b/app/data/gpkg/lifeline_pipe.gpkg new file mode 100644 index 0000000..cac1bc6 Binary files /dev/null and b/app/data/gpkg/lifeline_pipe.gpkg differ diff --git a/app/data/gpkg/risk_census_population.gpkg b/app/data/gpkg/risk_census_population.gpkg new file mode 100644 index 0000000..2e043dc Binary files /dev/null and b/app/data/gpkg/risk_census_population.gpkg differ diff --git a/app/data/gpkg/river.gpkg b/app/data/gpkg/river.gpkg index 21f1efc..406a61e 100644 Binary files a/app/data/gpkg/river.gpkg and b/app/data/gpkg/river.gpkg differ diff --git a/app/data/gpkg/rivers.gpkg b/app/data/gpkg/rivers.gpkg index cec8de3..b18fc3c 100644 Binary files a/app/data/gpkg/rivers.gpkg and b/app/data/gpkg/rivers.gpkg differ diff --git a/app/data/gpkg/sx.gpkg b/app/data/gpkg/sx.gpkg index 4739dea..d6e1e5f 100644 Binary files a/app/data/gpkg/sx.gpkg and b/app/data/gpkg/sx.gpkg differ diff --git a/app/data/gpkg/sx_capital.gpkg b/app/data/gpkg/sx_capital.gpkg index 3357556..3fd07cb 100644 Binary files a/app/data/gpkg/sx_capital.gpkg and b/app/data/gpkg/sx_capital.gpkg differ diff --git a/app/data/gpkg/sx_street.gpkg b/app/data/gpkg/sx_street.gpkg index ab14972..4c02189 100644 Binary files a/app/data/gpkg/sx_street.gpkg and b/app/data/gpkg/sx_street.gpkg differ diff --git a/app/data/gpkg/sx_xa_county.gpkg b/app/data/gpkg/sx_xa_county.gpkg index c542cf1..74fc90b 100644 Binary files a/app/data/gpkg/sx_xa_county.gpkg and b/app/data/gpkg/sx_xa_county.gpkg differ diff --git a/app/data/gpkg/sx_xa_county_boundary.gpkg b/app/data/gpkg/sx_xa_county_boundary.gpkg index f1aea2b..c85361c 100644 Binary files a/app/data/gpkg/sx_xa_county_boundary.gpkg and b/app/data/gpkg/sx_xa_county_boundary.gpkg differ diff --git a/app/data/gpkg/sx_xa_towns.gpkg b/app/data/gpkg/sx_xa_towns.gpkg new file mode 100644 index 0000000..010665e Binary files /dev/null and b/app/data/gpkg/sx_xa_towns.gpkg differ diff --git a/app/data/gpkg/sx_zb_city.gpkg b/app/data/gpkg/sx_zb_city.gpkg index 4560f90..b5fc5f4 100644 Binary files a/app/data/gpkg/sx_zb_city.gpkg and b/app/data/gpkg/sx_zb_city.gpkg differ diff --git a/app/data/gpkg/sx_zb_county.gpkg b/app/data/gpkg/sx_zb_county.gpkg index 6d0ad32..f0d99f4 100644 Binary files a/app/data/gpkg/sx_zb_county.gpkg and b/app/data/gpkg/sx_zb_county.gpkg differ diff --git a/app/data/gpkg/sx_zb_county_boundary.gpkg b/app/data/gpkg/sx_zb_county_boundary.gpkg index bedbac3..4c8d64c 100644 Binary files a/app/data/gpkg/sx_zb_county_boundary.gpkg and b/app/data/gpkg/sx_zb_county_boundary.gpkg differ diff --git a/app/data/gpkg/traffic_expressway.gpkg b/app/data/gpkg/traffic_expressway.gpkg index 6a8559f..65f1a8a 100644 Binary files a/app/data/gpkg/traffic_expressway.gpkg and b/app/data/gpkg/traffic_expressway.gpkg differ diff --git a/app/data/gpkg/traffic_provincial.gpkg b/app/data/gpkg/traffic_provincial.gpkg index 93e6e29..9afbfd2 100644 Binary files a/app/data/gpkg/traffic_provincial.gpkg and b/app/data/gpkg/traffic_provincial.gpkg differ diff --git a/app/data/gpkg/traffic_railway.gpkg b/app/data/gpkg/traffic_railway.gpkg index 7988d75..5b45290 100644 Binary files a/app/data/gpkg/traffic_railway.gpkg and b/app/data/gpkg/traffic_railway.gpkg differ diff --git a/app/data/gpkg/traffic_township.gpkg b/app/data/gpkg/traffic_township.gpkg index f51bdd4..e88cb13 100644 Binary files a/app/data/gpkg/traffic_township.gpkg and b/app/data/gpkg/traffic_township.gpkg differ diff --git a/app/data/gpkg/traffic_trunk_line.gpkg b/app/data/gpkg/traffic_trunk_line.gpkg index 9ae1b27..b4ed007 100644 Binary files a/app/data/gpkg/traffic_trunk_line.gpkg and b/app/data/gpkg/traffic_trunk_line.gpkg differ diff --git a/app/script/export_static_layers.py b/app/script/export_static_layers.py index ac8a346..7e08dbb 100644 --- a/app/script/export_static_layers.py +++ b/app/script/export_static_layers.py @@ -27,6 +27,7 @@ GPKG_DIR = os.path.join(project_root, "app", "data", "gpkg") # 静态图层定义: {显示名: (schema.table, gpkg文件名)} STATIC_LAYERS = [ + # ── 基础底图(已导出)── ("水库", "qgis.rivers", "rivers.gpkg"), ("市州驻地", "qgis.sx_capital", "sx_capital.gpkg"), ("河流", "qgis.river", "river.gpkg"), @@ -43,6 +44,12 @@ STATIC_LAYERS = [ ("traffic_railway", "qgis.traffic_railway", "traffic_railway.gpkg"), ("traffic_township", "qgis.traffic_township", "traffic_township.gpkg"), ("traffic_trunk_line", "qgis.traffic_trunk_line", "traffic_trunk_line.gpkg"), + # ── 新增静态表(消除数据库连接)── + ("积水点", "qgis.hazard_hydrops", "hazard_hydrops.gpkg"), + ("排水口", "qgis.lifeline_outfall", "lifeline_outfall.gpkg"), + ("供水管网", "qgis.lifeline_pipe", "lifeline_pipe.gpkg"), + ("风险人口", "qgis.risk_census_population", "risk_census_population.gpkg"), + ("西安乡镇", "qgis.sx_xa_towns", "sx_xa_towns.gpkg"), ] diff --git a/app/script/scan_template_tables.py b/app/script/scan_template_tables.py new file mode 100644 index 0000000..f04b45f --- /dev/null +++ b/app/script/scan_template_tables.py @@ -0,0 +1,82 @@ +""" +扫描所有模板,找出所有引用的 PostgreSQL 表,列出哪些已导出 GPKG、哪些还没。 +""" +import zipfile, re, os, sys + +TEMPLATE_BASE = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "app", "data", "template" +) + +# 当前已导出的 GPKG 表(从 qgis_mappings.py / export_static_layers.py) +EXPORTED = { + "rivers", "sx_capital", "river", "active_fault", "sx", + "sx_street", "sx_xa_county", "sx_xa_county_boundary", + "sx_zb_county_boundary", "sx_zb_city", "sx_zb_county", + "traffic_expressway", "traffic_provincial", "traffic_railway", + "traffic_township", "traffic_trunk_line", +} + +# 避难场所 OGR→PG 映射的表(已由 TemplateModifier 转换) +OGR_TABLES = { + "shelter_park", "shelter_school", "shelter_cultural", + "shelter_defence", "shelter_gymnasium", "shelter_square", "shelter_stay", +} + +all_tables = {} # {table_name: set of templates} + +for event_type in ["rainfall", "earthquake"]: + tdir = os.path.join(TEMPLATE_BASE, event_type) + if not os.path.isdir(tdir): + continue + for tf in sorted(os.listdir(tdir)): + if not tf.endswith(".qgz") or tf.startswith("tmp"): + continue + tpath = os.path.join(tdir, tf) + with zipfile.ZipFile(tpath, "r") as z: + for item in z.infolist(): + if item.filename.endswith(".qgs"): + content = z.read(item.filename).decode("utf-8") + # 匹配 datasource 中的 table="schema"."name" + for m in re.finditer(r'table="(\w+)"\."(\w+)"', content): + tbl = m.group(2) + all_tables.setdefault(tbl, set()).add(tf) + # 也检查 provider=postgres 但没有 table= 的(如震中) + # 匹配 dbname='...' 格式 + for m in re.finditer(r"dbname='([^']+)'", content): + db = m.group(1) + if db == 'xxgx_client_ya': + all_tables.setdefault(f"[外部DB: {db}]", set()).add(tf) + +print("=" * 60) +print("已导出 GPKG (16):") +for t in sorted(EXPORTED): + if t in all_tables: + print(f" ✅ {t}") +print() + +print("避难场所 OGR→PG (7):") +for t in sorted(OGR_TABLES): + print(f" ✅ {t}") +print() + +print("未导出,需要新增:") +need_export = [] +for tbl, tmps in sorted(all_tables.items()): + if tbl.startswith("[外部DB:"): + continue + if tbl in EXPORTED or tbl in OGR_TABLES: + continue + need_export.append(tbl) + print(f" ❌ {tbl} ← {', '.join(sorted(tmps))}") +print() + +print("外部数据库(无法静态化):") +for tbl, tmps in sorted(all_tables.items()): + if tbl.startswith("[外部DB:"): + print(f" 🔗 {tbl} ← {', '.join(sorted(tmps))}") +print() + +print(f"总结: 需要新增导出 {len(need_export)} 张表") +if need_export: + print(f" {', '.join(sorted(need_export))}") diff --git a/app/services/qgis/map_service.py b/app/services/qgis/map_service.py index b9c8880..dc860c8 100644 --- a/app/services/qgis/map_service.py +++ b/app/services/qgis/map_service.py @@ -41,12 +41,12 @@ class MapService: t0 = time.time() if is_cache_hit: logger.info(f"[缓存命中] {template_name}") + project.clear() project, texts, extent = template_cache.restore_template(template_path) template_cache.reset_project_state(project, texts, extent) - # FlagDontResolveLayers 跳过了数据库连接,手动触发 PostgreSQL 图层解析 - self._resolve_postgres_layers(project) else: logger.info(f"[首次加载] {template_name}") + project.clear() modifier = TemplateModifier(self.config) actual_path = modifier.modify(template_path) project.read(actual_path) diff --git a/app/services/qgis/qgis_daemon.py b/app/services/qgis/qgis_daemon.py new file mode 100644 index 0000000..06e6b95 --- /dev/null +++ b/app/services/qgis/qgis_daemon.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +QGIS 常驻 Daemon 进程。 + +由主进程在 FastAPI lifespan 中启动,通过 stdin/stdout JSON 行协议通信。 + +启动时预加载所有模板到内存缓存,后续请求秒级响应。 + +协议(每行一个完整 JSON): + 输入(stdin): {"config": {...}, "models": [...], "result_file": "path/to/result.json"} + 输出(result_file): {"results": [...]} + 控制: {"action": "shutdown"} + +生命周期: + 1. FastAPI 启动 → 启动 daemon 子进程 + 2. daemon 初始化 QGIS,加载磁盘缓存 + 3. 收到请求 → 处理模板 → 结果写入 result_file → stdout 输出完成标记 + 4. FastAPI 关闭 → 发送 shutdown → daemon 退出 +""" +import json +import os +import sys +import time + +# ============================================================ +# 环境初始化(必须在 QGIS import 之前) +# ============================================================ + +QGIS_ROOT = os.environ.get("QGIS_ROOT", "D:/QGIS") + + +def _detect_qgis_app_dir(): + for name in ("qgis-ltr", "qgis"): + d = os.path.join(QGIS_ROOT, "apps", name) + if os.path.isdir(d): + return d + raise RuntimeError(f"未找到 QGIS 应用目录: {QGIS_ROOT}\\apps\\qgis-ltr 或 qgis") + + +def _setup_environment(): + os.environ["PYTHONUTF8"] = "1" + os.environ["GDAL_FILENAME_IS_UTF8"] = "YES" + os.environ["VSI_CACHE"] = "TRUE" + os.environ["VSI_CACHE_SIZE"] = "1000000" + + if sys.platform == "win32": + import ctypes + qgis_app_dir = _detect_qgis_app_dir() + for dll_dir in [ + os.path.join(qgis_app_dir, "bin"), + os.path.join(QGIS_ROOT, "apps", "Qt5", "bin"), + os.path.join(QGIS_ROOT, "apps", "gdal", "lib"), + ]: + if os.path.isdir(dll_dir): + os.add_dll_directory(dll_dir) + + _preload_dlls = [ + "qgis_core.dll", "qgispython.dll", + "Qt5Core.dll", "Qt5Gui.dll", "Qt5Widgets.dll", + "Qt5Network.dll", "Qt5Svg.dll", "Qt5Xml.dll", + "Qt5Concurrent.dll", "Qt5PrintSupport.dll", + ] + for dll_dir in [ + os.path.join(qgis_app_dir, "bin"), + os.path.join(QGIS_ROOT, "apps", "Qt5", "bin"), + os.path.join(QGIS_ROOT, "apps", "gdal", "lib"), + ]: + for dll_name in _preload_dlls: + dll_path = os.path.join(dll_dir, dll_name) + if os.path.isfile(dll_path): + try: + ctypes.WinDLL(dll_path) + except OSError: + pass + + +def _setup_python_path(): + project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + if project_root not in sys.path: + sys.path.insert(0, project_root) + qgis_app_dir = _detect_qgis_app_dir() + qgis_python = os.path.join(qgis_app_dir, "python") + if os.path.isdir(qgis_python) and qgis_python not in sys.path: + sys.path.insert(0, qgis_python) + + +def _scan_templates() -> list[str]: + """扫描所有模板文件,返回绝对路径列表""" + project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + template_base = os.path.join(project_root, "app", "data", "template") + templates = [] + for event_type in ["rainfall", "earthquake"]: + tdir = os.path.join(template_base, event_type) + if not os.path.isdir(tdir): + continue + for tf in sorted(os.listdir(tdir)): + if tf.endswith(".qgz") and not tf.startswith("tmp"): + templates.append(os.path.join(tdir, tf).replace("\\", "/")) + return templates + + +# ============================================================ +# 主逻辑 +# ============================================================ + +def main(): + _setup_environment() + _setup_python_path() + + from qgis.core import QgsApplication, QgsProject + + # 初始化 QGIS(只做一次) + qgis_app_dir = _detect_qgis_app_dir() + QgsApplication.setPrefixPath(qgis_app_dir, True) + qgs_app = QgsApplication([], False) + qgs_app.initQgis() + + from app.services.qgis.map_service import MapService, template_cache + from app.services.qgis.template_modifier import TemplateModifier + from app.config.qgis_mappings import build_static_layers_config, get_gpkg_dir + + # 从磁盘恢复已缓存的模板 + cached_count = template_cache.load_persistent_cache() + print(f"[qgis_daemon] QGIS 已启动, 磁盘缓存: {cached_count} 个模板", + file=sys.stderr, flush=True) + + total_cached = len(template_cache._cache) + print(f"[qgis_daemon] 就绪: 缓存共{total_cached}个模板", + file=sys.stderr, flush=True) + + # ── 请求处理循环 ── + print("[qgis_daemon] 等待请求...", file=sys.stderr, flush=True) + + for line in sys.stdin: + line = line.strip() + if not line: + continue + + try: + request = json.loads(line) + except json.JSONDecodeError as e: + resp = json.dumps({"status": "error", "message": f"JSON 解析失败: {e}"}) + sys.stdout.write(resp + "\n") + sys.stdout.flush() + continue + + action = request.get("action", "map") + + if action == "shutdown": + print("[qgis_daemon] 收到 shutdown,退出", file=sys.stderr, flush=True) + break + + if action == "map": + config = request["config"] + models = request["models"] + result_file = request.get("result_file", "") + results = [] + + t_batch = time.time() + if "static_layers" not in config or not config["static_layers"].get("gpkg_dir"): + config["static_layers"] = build_static_layers_config(get_gpkg_dir()) + + service = MapService(config) + + for i, model in enumerate(models): + t_model = time.time() + try: + name = service.generate(model) + results.append({"name": name, "output": model["outFile"]}) + elapsed = time.time() - t_model + print(f"[qgis_daemon] [{i+1}/{len(models)}] {name} ({elapsed:.1f}s)", + file=sys.stderr, flush=True) + except Exception as e: + elapsed = time.time() - t_model + results.append({"name": model.get("name", ""), "output": "", "error": str(e)}) + print(f"[qgis_daemon] [{i+1}/{len(models)}] 失败: {e}", + file=sys.stderr, flush=True) + + total = time.time() - t_batch + ok = sum(1 for r in results if "error" not in r) + print(f"[qgis_daemon] 完成: {ok}/{len(models)}, {total:.1f}s", + file=sys.stderr, flush=True) + + resp = {"results": results} + + # 写入结果文件(可靠),然后 stdout 发完成信号 + if result_file: + try: + with open(result_file, "w", encoding="utf-8") as f: + json.dump(resp, f, ensure_ascii=False) + except Exception as e: + print(f"[qgis_daemon] 写结果文件失败: {e}", file=sys.stderr, flush=True) + + sys.stdout.write("OK\n") + sys.stdout.flush() + + # 清理 + project = QgsProject.instance() + project.clear() + template_cache.cleanup() + print("[qgis_daemon] 已退出", file=sys.stderr, flush=True) + + +if __name__ == "__main__": + main() diff --git a/app/services/qgis/template_cache.py b/app/services/qgis/template_cache.py index 9725dd3..9b747b4 100644 --- a/app/services/qgis/template_cache.py +++ b/app/services/qgis/template_cache.py @@ -75,7 +75,7 @@ class TemplateCache: logger.info(f" 模板已缓存: {os.path.basename(template_path)} ({len(self._cache)} 个)") def restore_template(self, template_path: str) -> tuple: - """从缓存恢复模板(跳过图层解析以加速)""" + """从缓存恢复模板""" cached = self._cache.get(template_path) if not cached: raise RuntimeError(f"模板未缓存: {template_path}") @@ -83,10 +83,8 @@ class TemplateCache: project = QgsProject.instance() t0 = time.time() - # FlagDontResolveLayers: 跳过数据库连接验证,缓存文件已含正确连接 - flags = QgsProject.ReadFlags() - flags |= QgsProject.FlagDontResolveLayers - project.read(cached["file"], flags) + # 大部分图层已 GPKG 本地化,不需要 FlagDontResolveLayers + project.read(cached["file"]) logger.debug(f" 恢复耗时: {time.time() - t0:.1f}s") return project, cached["texts"], cached["extent"] diff --git a/app/services/qgis/template_modifier.py b/app/services/qgis/template_modifier.py index 05b8287..159e490 100644 --- a/app/services/qgis/template_modifier.py +++ b/app/services/qgis/template_modifier.py @@ -75,7 +75,7 @@ class TemplateModifier: tmp.close() datasource_re = re.compile(r"()(.*?)()", re.DOTALL) - table_re = re.compile(r'table="(\w+)"\."(\w+)"') + table_re = re.compile(r'table=(?:"|")(\w+)(?:"|")\.(?:"|")(\w+)(?:"|")') with zipfile.ZipFile(template_path, "r") as zin, \ zipfile.ZipFile(tmp_path, "w", zipfile.ZIP_DEFLATED) as zout: diff --git a/settings.toml b/settings.toml index 5382d2e..9d41810 100644 --- a/settings.toml +++ b/settings.toml @@ -21,6 +21,8 @@ QGIS_DEFAULTS_MAP_UNIT = "制图单位:西安市应急管理局" QGIS_EXPORT_DPI = 300 # 批量产图线程池 QGIS_WORKER_THREADS = 4 +# 并行子进程数(每进程独立 QGIS 实例) +QGIS_PARALLEL_WORKERS = 4 # 西安市中心经纬度 XIAN_CENTER = [108.948024, 34.263161]