20260327 今日开发完成：三个策略新增+结构化适配+消息风控+任务跟踪

2026-03-27 16:44:01 +08:00
parent f043f123a1
commit dd1b06ba4a
41 changed files with 12115 additions and 0 deletions
@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+启动全量分钟数据下载
+"""
+import os
+import sys
+import time
+import json
+import subprocess
+from datetime import datetime
+
+print("="*70)
+print("🚀 赵云启动全量分钟数据下载")
+print("="*70)
+
+# 配置信息
+config = {
+    "base_dir": "/Users/chufeng/nas/stock/minute_kline",
+    "data_source": "akshare",
+    "timeframe": "15min",  # 从15分钟数据开始
+    "date_range": {
+        "start": "2021-01-01",
+        "end": "2026-03-27"
+    },
+    "stock_count": 5500,  # 全市场A股数量
+    "download_mode": "full",
+    "batch_size": 100,
+    "max_workers": 5,
+    "retry_count": 3,
+    "log_file": "/Users/chufeng/nas/stock/minute_kline/logs/full_download_{}.log".format(
+        datetime.now().strftime("%Y%m%d_%H%M%S")
+    )
+}
+
+# 保存配置
+config_file = os.path.join(config["base_dir"], "full_download_config.json")
+with open(config_file, 'w', encoding='utf-8') as f:
+    json.dump(config, f, ensure_ascii=False, indent=2)
+
+print("📊 配置信息:")
+print(f"  存储路径: {config['base_dir']}")
+print(f"  数据粒度: {config['timeframe']}")
+print(f"  股票数量: {config['stock_count']} 只")
+print(f"  时间范围: {config['date_range']['start']} 至 {config['date_range']['end']}")
+print(f"  下载模式: {config['download_mode']}")
+print(f"  批次大小: {config['batch_size']} 只/批")
+print(f"  并发数: {config['max_workers']}")
+
+# 检查已有数据
+print("\n🔍 检查已有数据...")
+existing_files = 0
+if os.path.exists(os.path.join(config["base_dir"], config["timeframe"])):
+    existing_files = len([f for f in os.listdir(os.path.join(config["base_dir"], config["timeframe"])) 
+                         if f.endswith('.parquet')])
+    print(f"  已下载股票: {existing_files} 只")
+    print(f"  待下载股票: {config['stock_count'] - existing_files} 只")
+
+print("\n🎯 启动下载命令:")
+print("="*70)
+
+# 创建启动脚本
+start_script = """#!/bin/bash
+# 赵云全量分钟数据下载启动脚本
+# 开始时间: {}
+
+cd /Users/chufeng/.openclaw/sanguo_projects/sanguo_quant_live/zhaoyun-data/scripts/data_acquisition
+
+echo "🚀 开始全量分钟数据下载..."
+echo "📊 目标: 下载{}只A股的{}数据"
+echo "⏱️ 开始时间: $(date)"
+
+# 使用稳定下载器开始下载
+python3 -c "
+import sys
+import os
+sys.path.append(os.path.dirname(os.path.abspath(__file__)))
+
+from minute_kline_collector import MinuteKlineCollector
+
+collector = MinuteKlineCollector(
+    base_dir='{}',
+    timeframe='{}',
+    start_date='{}',
+    end_date='{}',
+    batch_size={},
+    max_workers={},
+    retry_count={}
+)
+
+print('🎯 赵云开始全量下载任务...')
+collector.download_all_stocks()
+print('✅ 全量下载任务完成！')
+"
+
+echo "⏱️ 结束时间: $(date)"
+echo "📈 下载总结: 请查看 {}/reports/"
+""".format(
+    datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
+    config["stock_count"],
+    config["timeframe"],
+    config["base_dir"],
+    config["timeframe"],
+    config["date_range"]["start"],
+    config["date_range"]["end"],
+    config["batch_size"],
+    config["max_workers"],
+    config["retry_count"],
+    config["base_dir"]
+)
+
+# 保存启动脚本
+script_file = os.path.join(os.path.dirname(__file__), "start_full_download.sh")
+with open(script_file, 'w', encoding='utf-8') as f:
+    f.write(start_script)
+
+os.chmod(script_file, 0o755)
+
+print(f"启动脚本: {script_file}")
+print("="*70)
+
+print("\n📋 执行步骤:")
+print("1. ✅ 配置文件已生成")
+print("2. ✅ 启动脚本已创建")
+print("3. 🚀 准备开始全量下载")
+
+print("\n⏱️ 时间预估:")
+estimated_hours = (config["stock_count"] - existing_files) / (config["batch_size"] * 60) * 2
+print(f"  预计完成时间: {estimated_hours:.1f} 小时")
+print(f"  预计完成日期: {(datetime.now().timestamp() + estimated_hours * 3600):%Y-%m-%d %H:%M}")
+
+print("\n" + "="*70)
+print("🎯 赵云立即开始执行全量下载！")
+print("="*70)
+
+# 立即启动下载
+print("\n🚀 启动下载进程...")
+try:
+    # 启动下载进程
+    subprocess.Popen(
+        ["nohup", "bash", script_file, "&"],
+        stdout=open(config["log_file"], 'w'),
+        stderr=open(config["log_file"], 'a')
+    )
+    print(f"✅ 下载进程已启动，日志文件: {config['log_file']}")
+    print("📊 可以使用以下命令监控进度:")
+    print(f"  tail -f {config['log_file']}")
+    
+except Exception as e:
+    print(f"❌ 启动失败: {e}")
+    print("\n💡 备用方案: 手动运行启动脚本")
+    print(f"  bash {script_file}")
+
+print("\n" + "="*70)
+print("📡 赵云确认: 全量分钟数据下载已启动！")
+print("="*70)