#!/usr/bin/env python3 """ 简单稳定的分钟数据下载器 """ import os import sys import time import json import logging from datetime import datetime, timedelta from typing import List, Optional import warnings warnings.filterwarnings('ignore') import pandas as pd try: import akshare as ak AKSHARE_AVAILABLE = True except ImportError: AKSHARE_AVAILABLE = False print("❌ AKShare未安装，请运行: pip install akshare") sys.exit(1) # 配置日志 logging.basicConfig( level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s' ) logger = logging.getLogger(__name__) class SimpleMinuteDownloader: """简单稳定的分钟数据下载器""" def __init__( self, base_dir: str = "/Users/chufeng/nas/stock/minute_kline", timeframe: str = "15min", start_date: str = "2021-01-01", end_date: str = None, batch_size: int = 100, max_workers: int = 5, retry_count: int = 3 ): """初始化下载器""" self.base_dir = base_dir self.timeframe = timeframe self.start_date = start_date self.end_date = end_date or datetime.now().strftime("%Y-%m-%d") self.batch_size = batch_size self.max_workers = max_workers self.retry_count = retry_count # 创建目录 self.data_dir = os.path.join(self.base_dir, self.timeframe) self.log_dir = os.path.join(self.base_dir, "logs") self.report_dir = os.path.join(self.base_dir, "reports") os.makedirs(self.data_dir, exist_ok=True) os.makedirs(self.log_dir, exist_ok=True) os.makedirs(self.report_dir, exist_ok=True) # 下载状态 self.download_stats = { "total_stocks": 0, "downloaded_stocks": 0, "failed_stocks": 0, "start_time": datetime.now(), "end_time": None } logger.info(f"初始化下载器: {self.timeframe} 数据") logger.info(f"数据目录: {self.data_dir}") logger.info(f"时间范围: {self.start_date} 至 {self.end_date}") def get_all_a_stock_codes(self) -> List[str]: """获取所有A股代码""" logger.info("获取A股代码列表...") try: # 使用AKShare获取A股代码 stock_info = ak.stock_info_a_code_name() if stock_info is not None and not stock_info.empty: # 提取股票代码，格式化为带市场前缀 codes = [] for _, row in stock_info.iterrows(): code = str(row.get('code', '')) if not code: continue # 添加市场前缀 if code.startswith('6'): codes.append(f"sh{code}") elif code.startswith('0') or code.startswith('3'): codes.append(f"sz{code}") logger.info(f"获取到 {len(codes)} 只A股代码") return codes else: logger.warning("AKShare返回空数据，使用预定义列表") return self._get_default_stock_codes() except Exception as e: logger.error(f"获取A股代码失败: {e}") return self._get_default_stock_codes() def _get_default_stock_codes(self) -> List[str]: """获取默认股票代码列表（前1000只）""" codes = [] # 上证股票 for i in range(1, 600): codes.append(f"sh600{i:03d}") # 深证股票 for i in range(1, 300): codes.append(f"sz000{i:03d}") for i in range(1, 100): codes.append(f"sz300{i:03d}") logger.info(f"使用默认股票代码列表: {len(codes)} 只") return codes def download_stock_data(self, stock_code: str) -> bool: """下载单只股票的分钟数据""" for attempt in range(self.retry_count): try: logger.info(f"下载 {stock_code} {self.timeframe} 数据...") # 根据时间粒度设置参数 period_map = { '1min': '1', '5min': '5', '15min': '15', '30min': '30', '60min': '60' } period = period_map.get(self.timeframe, '15') # 下载数据 data = ak.stock_zh_a_minute( symbol=stock_code, period=period, adjust='hfq' ) # 检查数据是否有效 if data is None: logger.warning(f"{stock_code}: 返回None数据，跳过") return False if hasattr(data, 'empty') and data.empty: logger.warning(f"{stock_code}: 数据为空，跳过") return False # 数据清理 data = data.copy() if hasattr(data, 'columns'): data.columns = [col.strip() if isinstance(col, str) else col for col in data.columns] # 保存数据 parquet_file = os.path.join(self.data_dir, f"{stock_code}_{self.timeframe}.parquet") csv_file = os.path.join(self.data_dir, f"{stock_code}_{self.timeframe}.csv") # 保存为Parquet data.to_parquet(parquet_file, compression='snappy') # 保存为CSV（便于查看） data.to_csv(csv_file, index=False) logger.info(f"✅ {stock_code}: 下载成功 {len(data)} 条记录") logger.info(f" 保存文件: {parquet_file} ({os.path.getsize(parquet_file) // 1024} KB)") return True except Exception as e: logger.error(f"❌ {stock_code}: 下载失败 (尝试 {attempt + 1}/{self.retry_count}) - {str(e)[:100]}") if attempt < self.retry_count - 1: time.sleep(2) # 重试前等待 else: return False return False def download_all_stocks(self, all_stocks: bool = True): """下载所有股票数据""" logger.info("="*70) logger.info("🚀 开始全量分钟数据下载") logger.info("="*70) # 获取股票代码 stock_codes = self.get_all_a_stock_codes() if not stock_codes: logger.error("❌ 无法获取股票代码列表") return self.download_stats["total_stocks"] = len(stock_codes) # 检查已下载的股票 downloaded_stocks = set() if os.path.exists(self.data_dir): for file in os.listdir(self.data_dir): if file.endswith(f"_{self.timeframe}.parquet"): stock_code = file.replace(f"_{self.timeframe}.parquet", "") downloaded_stocks.add(stock_code) # 过滤已下载的股票 if not all_stocks and downloaded_stocks: remaining_stocks = [code for code in stock_codes if code not in downloaded_stocks] logger.info(f"已下载 {len(downloaded_stocks)} 只，剩余 {len(remaining_stocks)} 只") stock_codes = remaining_stocks logger.info(f"📊 开始下载 {len(stock_codes)} 只股票") # 分批下载 success_count = 0 fail_count = 0 for i, stock_code in enumerate(stock_codes, 1): logger.info(f"\n📈 进度: {i}/{len(stock_codes)} ({i/len(stock_codes)*100:.1f}%)") if self.download_stock_data(stock_code): success_count += 1 else: fail_count += 1 # 更新统计 self.download_stats["downloaded_stocks"] = success_count self.download_stats["failed_stocks"] = fail_count # 每下载10只股票保存一次进度 if i % 10 == 0 or i == len(stock_codes): self._save_progress_report() # 控制下载速度 time.sleep(0.5) # 避免请求过快 # 完成下载 self.download_stats["end_time"] = datetime.now() self._save_final_report() logger.info("="*70) logger.info(f"✅ 下载完成!") logger.info(f" 成功: {success_count} 只") logger.info(f" 失败: {fail_count} 只") logger.info(f" 成功率: {success_count/(success_count+fail_count)*100:.1f}%") logger.info(f" 总耗时: {self.download_stats['end_time'] - self.download_stats['start_time']}") logger.info("="*70) def _save_progress_report(self): """保存进度报告""" # 创建可序列化的统计副本 serializable_stats = self.download_stats.copy() # 转换datetime对象为字符串 if isinstance(serializable_stats.get("start_time"), datetime): serializable_stats["start_time"] = serializable_stats["start_time"].isoformat() if isinstance(serializable_stats.get("end_time"), datetime): serializable_stats["end_time"] = serializable_stats["end_time"].isoformat() report = { "timestamp": datetime.now().isoformat(), "timeframe": self.timeframe, "stats": serializable_stats, "progress": { "percentage": self.download_stats["downloaded_stocks"] / max(1, self.download_stats["total_stocks"]) * 100, "estimated_time_left": self._estimate_time_left() } } report_file = os.path.join(self.report_dir, f"progress_{self.timeframe}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json") with open(report_file, 'w', encoding='utf-8') as f: json.dump(report, f, ensure_ascii=False, indent=2) def _save_final_report(self): """保存最终报告""" # 创建可序列化的统计副本 serializable_stats = self.download_stats.copy() # 转换datetime对象为字符串 if isinstance(serializable_stats.get("start_time"), datetime): serializable_stats["start_time"] = serializable_stats["start_time"].isoformat() if isinstance(serializable_stats.get("end_time"), datetime): serializable_stats["end_time"] = serializable_stats["end_time"].isoformat() report = { "timestamp": datetime.now().isoformat(), "timeframe": self.timeframe, "summary": serializable_stats, "duration": str(self.download_stats["end_time"] - self.download_stats["start_time"]) if self.download_stats["end_time"] else None, "success_rate": self.download_stats["downloaded_stocks"] / max(1, self.download_stats["total_stocks"]) * 100 } report_file = os.path.join(self.report_dir, f"final_{self.timeframe}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json") with open(report_file, 'w', encoding='utf-8') as f: json.dump(report, f, ensure_ascii=False, indent=2) logger.info(f"📄 最终报告已保存: {report_file}") def _estimate_time_left(self) -> str: """估计剩余时间""" downloaded = self.download_stats["downloaded_stocks"] total = self.download_stats["total_stocks"] if downloaded == 0: return "未知" elapsed = (datetime.now() - self.download_stats["start_time"]).total_seconds() time_per_stock = elapsed / downloaded remaining_stocks = total - downloaded remaining_seconds = time_per_stock * remaining_stocks if remaining_seconds < 60: return f"{int(remaining_seconds)}秒" elif remaining_seconds < 3600: return f"{int(remaining_seconds/60)}分钟" else: return f"{int(remaining_seconds/3600)}小时{int((remaining_seconds%3600)/60)}分钟" def main(): """主函数""" import argparse parser = argparse.ArgumentParser(description="简单稳定的分钟数据下载器") parser.add_argument("--timeframe", default="15min", help="时间粒度: 1min, 5min, 15min, 30min, 60min") parser.add_argument("--start-date", default="2021-01-01", help="开始日期") parser.add_argument("--end-date", default=None, help="结束日期") parser.add_argument("--batch-size", type=int, default=100, help="批次大小") parser.add_argument("--max-workers", type=int, default=5, help="最大工作线程数") parser.add_argument("--all-stocks", action="store_true", help="下载所有股票（包括已下载的）") args = parser.parse_args() print("="*70) print("🚀 赵云分钟数据下载器启动") print("="*70) downloader = SimpleMinuteDownloader( timeframe=args.timeframe, start_date=args.start_date, end_date=args.end_date, batch_size=args.batch_size, max_workers=args.max_workers ) downloader.download_all_stocks(all_stocks=args.all_stocks) if __name__ == "__main__": main()