From 555792cc44bf195d50d88b778081e6be5cf373bf Mon Sep 17 00:00:00 2001 From: cfdaily Date: Sun, 3 May 2026 10:48:31 +0800 Subject: [PATCH] auto-sync: 2026-05-03 10:48:31 --- data_platform/daily_all_update.py | 36 +++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/data_platform/daily_all_update.py b/data_platform/daily_all_update.py index 3646bfd8..bb57a7d6 100644 --- a/data_platform/daily_all_update.py +++ b/data_platform/daily_all_update.py @@ -426,22 +426,33 @@ def run_15min_update(codes: List[str]) -> dict: # ======================== vnpy DB写入 ======================== def _write_vnpy_db(values: list, label: str): - """增量写入vnpy SQLite DB""" + """增量写入vnpy SQLite DB - 本地临时DB写入后ATTACH导入NAS DB""" logger.info("写入vnpy DB [%s]: %d 条记录", label, len(values)) + local_tmp = f"/tmp/vnpy_update_{label}_{int(time.time())}.db" try: + # 1. 本地临时DB写入增量 + conn = sqlite3.connect(local_tmp, timeout=30) + c = conn.cursor() + c.execute("PRAGMA journal_mode=WAL") + c.executemany( + """INSERT OR REPLACE INTO dbbardata + (symbol,exchange,datetime,interval,volume,turnover,open_interest, + open_price,high_price,low_price,close_price) + VALUES (?,?,?,?,?,?,?,?,?,?,?)""", + values, + ) + conn.commit() + conn.close() + logger.info(" 本地写入完成: %s (%d rows)", local_tmp, len(values)) + + # 2. ATTACH到NAS DB,批量导入 conn = sqlite3.connect(str(VNPY_DB_PATH), timeout=120) c = conn.cursor() c.execute("PRAGMA journal_mode=WAL") - for j in range(0, len(values), DB_BATCH_SIZE): - c.executemany( - """INSERT OR REPLACE INTO dbbardata - (symbol,exchange,datetime,interval,volume,turnover,open_interest, - open_price,high_price,low_price,close_price) - VALUES (?,?,?,?,?,?,?,?,?,?,?)""", - values[j : j + DB_BATCH_SIZE], - ) - conn.commit() - logger.info(" DB批次 %d/%d", j // DB_BATCH_SIZE + 1, (len(values) - 1) // DB_BATCH_SIZE + 1) + c.execute(f"ATTACH DATABASE ? AS tmpdb", (local_tmp,)) + c.execute("INSERT OR REPLACE INTO main.dbbardata SELECT * FROM tmpdb.dbbardata") + conn.commit() + c.execute("DETACH DATABASE tmpdb") # 更新overview c.execute( @@ -454,6 +465,9 @@ def _write_vnpy_db(values: list, label: str): logger.info("✅ vnpy DB [%s] 写入完成", label) except Exception as e: logger.error("❌ vnpy DB [%s] 写入失败: %s", label, e) + finally: + if os.path.exists(local_tmp): + os.remove(local_tmp) # ======================== 主入口 ========================