1079 lines
35 KiB
Python
1079 lines
35 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
概念异动检测 - 基于超额收益(Alpha)的方法
|
||
|
||
核心思想：
|
||
- 异动 = 概念表现与市场整体的显著偏离
|
||
- Alpha = 概念涨幅 - 大盘涨幅（超额收益）
|
||
- 用 Z-Score 衡量 Alpha 是否显著偏离历史正常范围
|
||
|
||
优点：
|
||
- 自适应：不同概念有不同波动率，自动适应
|
||
- 相对比较：不看绝对涨跌，看相对表现
|
||
- 无需规则维护：统计方法自动学习"正常范围"
|
||
- 通用：可迁移到其他市场
|
||
"""
|
||
|
||
import pandas as pd
|
||
import numpy as np
|
||
from datetime import datetime, timedelta
|
||
from sqlalchemy import create_engine, text
|
||
from elasticsearch import Elasticsearch
|
||
from clickhouse_driver import Client
|
||
from collections import deque, defaultdict
|
||
import time
|
||
import logging
|
||
import json
|
||
import os
|
||
import hashlib
|
||
import argparse
|
||
from typing import Dict, List, Optional, Tuple
|
||
from dataclasses import dataclass, field
|
||
|
||
# ==================== 配置 ====================
|
||
|
||
MYSQL_ENGINE = create_engine(
|
||
"mysql+pymysql://root:Zzl5588161!@222.128.1.157:33060/stock",
|
||
echo=False
|
||
)
|
||
|
||
ES_CLIENT = Elasticsearch(['http://222.128.1.157:19200'])
|
||
INDEX_NAME = 'concept_library_v3'
|
||
|
||
CLICKHOUSE_CONFIG = {
|
||
'host': '222.128.1.157',
|
||
'port': 18000,
|
||
'user': 'default',
|
||
'password': 'Zzl33818!',
|
||
'database': 'stock'
|
||
}
|
||
|
||
HIERARCHY_FILE = 'concept_hierarchy_v3.json'
|
||
REFERENCE_INDEX = '000001.SH'
|
||
|
||
# ==================== Alpha 异动检测配置 ====================
|
||
|
||
ALPHA_CONFIG = {
|
||
# Z-Score 阈值
|
||
'zscore_threshold': 2.0, # |Z| > 2.0 视为异动（降低阈值）
|
||
|
||
# 绝对 Alpha 阈值（当历史数据不足时使用）
|
||
'absolute_alpha_threshold': 1.5, # |Alpha| > 1.5% 视为异动
|
||
|
||
# 历史窗口
|
||
'history_window': 60, # 保留最近60个数据点用于计算均值/标准差
|
||
'min_history': 5, # 最少需要5个数据点才用Z-Score（降低要求）
|
||
|
||
# 冷却时间
|
||
'cooldown_minutes': 8, # 同一概念触发异动后的冷却时间
|
||
|
||
# 显示控制
|
||
'max_alerts_per_minute': 15, # 每分钟最多显示的异动数
|
||
'min_alpha_abs': 0.5, # 最小超额收益绝对值（过滤微小波动）
|
||
|
||
# 重要性评分权重
|
||
'importance_weights': {
|
||
'alpha_zscore': 0.35, # Alpha Z-Score 绝对值
|
||
'alpha_abs': 0.25, # 超额收益绝对值
|
||
'rank_in_minute': 0.20, # 当分钟内的排名
|
||
'limit_up_count': 0.20, # 涨停数
|
||
}
|
||
}
|
||
|
||
# ==================== 日志配置 ====================
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||
handlers=[
|
||
logging.FileHandler(f'concept_alert_alpha_{datetime.now().strftime("%Y%m%d")}.log', encoding='utf-8'),
|
||
logging.StreamHandler()
|
||
]
|
||
)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# ==================== 数据结构 ====================
|
||
|
||
@dataclass
|
||
class AlphaStats:
|
||
"""概念的Alpha统计信息"""
|
||
history: deque = field(default_factory=lambda: deque(maxlen=ALPHA_CONFIG['history_window']))
|
||
mean: float = 0.0
|
||
std: float = 1.0
|
||
last_update: datetime = None
|
||
|
||
def update(self, alpha: float, timestamp: datetime):
|
||
"""更新统计"""
|
||
self.history.append(alpha)
|
||
self.last_update = timestamp
|
||
|
||
if len(self.history) >= 2:
|
||
self.mean = np.mean(self.history)
|
||
self.std = max(np.std(self.history), 0.1) # 避免除零
|
||
|
||
def get_zscore(self, alpha: float) -> float:
|
||
"""计算Z-Score"""
|
||
if len(self.history) < ALPHA_CONFIG['min_history']:
|
||
return 0.0 # 数据不足，不触发
|
||
return (alpha - self.mean) / self.std
|
||
|
||
def is_ready(self) -> bool:
|
||
"""是否有足够数据进行检测"""
|
||
return len(self.history) >= ALPHA_CONFIG['min_history']
|
||
|
||
|
||
# ==================== 全局变量 ====================
|
||
|
||
ch_client = None
|
||
|
||
# 每个概念的Alpha统计
|
||
alpha_stats: Dict[str, AlphaStats] = defaultdict(AlphaStats)
|
||
|
||
# 冷却记录
|
||
cooldown_cache: Dict[str, datetime] = {}
|
||
|
||
# 当前分钟的数据缓存（用于计算排名）
|
||
current_minute_data: List[dict] = []
|
||
|
||
|
||
def get_ch_client():
|
||
global ch_client
|
||
if ch_client is None:
|
||
ch_client = Client(**CLICKHOUSE_CONFIG)
|
||
return ch_client
|
||
|
||
|
||
def generate_id(name: str) -> str:
|
||
return hashlib.md5(name.encode('utf-8')).hexdigest()[:16]
|
||
|
||
|
||
def code_to_ch_format(code: str) -> str:
|
||
if not code or len(code) != 6 or not code.isdigit():
|
||
return None
|
||
if code.startswith('6'):
|
||
return f"{code}.SH"
|
||
elif code.startswith('0') or code.startswith('3'):
|
||
return f"{code}.SZ"
|
||
else:
|
||
return f"{code}.BJ"
|
||
|
||
|
||
# ==================== 概念数据获取 ====================
|
||
|
||
def get_all_concepts():
|
||
"""从ES获取所有叶子概念"""
|
||
concepts = []
|
||
|
||
query = {
|
||
"query": {"match_all": {}},
|
||
"size": 100,
|
||
"_source": ["concept_id", "concept", "stocks"]
|
||
}
|
||
|
||
resp = ES_CLIENT.search(index=INDEX_NAME, body=query, scroll='2m')
|
||
scroll_id = resp['_scroll_id']
|
||
hits = resp['hits']['hits']
|
||
|
||
while len(hits) > 0:
|
||
for hit in hits:
|
||
source = hit['_source']
|
||
concept_info = {
|
||
'concept_id': source.get('concept_id'),
|
||
'concept_name': source.get('concept'),
|
||
'stocks': [],
|
||
'concept_type': 'leaf'
|
||
}
|
||
|
||
if 'stocks' in source and isinstance(source['stocks'], list):
|
||
for stock in source['stocks']:
|
||
if isinstance(stock, dict) and 'code' in stock and stock['code']:
|
||
concept_info['stocks'].append(stock['code'])
|
||
|
||
if concept_info['stocks']:
|
||
concepts.append(concept_info)
|
||
|
||
resp = ES_CLIENT.scroll(scroll_id=scroll_id, scroll='2m')
|
||
scroll_id = resp['_scroll_id']
|
||
hits = resp['hits']['hits']
|
||
|
||
ES_CLIENT.clear_scroll(scroll_id=scroll_id)
|
||
return concepts
|
||
|
||
|
||
def load_hierarchy_concepts(leaf_concepts: list) -> list:
|
||
"""加载层级结构，生成母概念"""
|
||
hierarchy_path = os.path.join(os.path.dirname(__file__), HIERARCHY_FILE)
|
||
if not os.path.exists(hierarchy_path):
|
||
return []
|
||
|
||
with open(hierarchy_path, 'r', encoding='utf-8') as f:
|
||
hierarchy_data = json.load(f)
|
||
|
||
concept_to_stocks = {c['concept_name']: set(c['stocks']) for c in leaf_concepts}
|
||
parent_concepts = []
|
||
|
||
for lv1 in hierarchy_data.get('hierarchy', []):
|
||
lv1_name = lv1.get('lv1', '')
|
||
lv1_stocks = set()
|
||
|
||
for child in lv1.get('children', []):
|
||
lv2_name = child.get('lv2', '')
|
||
lv2_stocks = set()
|
||
|
||
if 'children' in child:
|
||
for lv3_child in child.get('children', []):
|
||
lv3_name = lv3_child.get('lv3', '')
|
||
lv3_stocks = set()
|
||
|
||
for concept_name in lv3_child.get('concepts', []):
|
||
if concept_name in concept_to_stocks:
|
||
lv3_stocks.update(concept_to_stocks[concept_name])
|
||
|
||
if lv3_stocks:
|
||
parent_concepts.append({
|
||
'concept_id': generate_id(f"lv3_{lv3_name}"),
|
||
'concept_name': f"[三级] {lv3_name}",
|
||
'stocks': list(lv3_stocks),
|
||
'concept_type': 'lv3'
|
||
})
|
||
lv2_stocks.update(lv3_stocks)
|
||
else:
|
||
for concept_name in child.get('concepts', []):
|
||
if concept_name in concept_to_stocks:
|
||
lv2_stocks.update(concept_to_stocks[concept_name])
|
||
|
||
if lv2_stocks:
|
||
parent_concepts.append({
|
||
'concept_id': generate_id(f"lv2_{lv2_name}"),
|
||
'concept_name': f"[二级] {lv2_name}",
|
||
'stocks': list(lv2_stocks),
|
||
'concept_type': 'lv2'
|
||
})
|
||
lv1_stocks.update(lv2_stocks)
|
||
|
||
if lv1_stocks:
|
||
parent_concepts.append({
|
||
'concept_id': generate_id(f"lv1_{lv1_name}"),
|
||
'concept_name': f"[一级] {lv1_name}",
|
||
'stocks': list(lv1_stocks),
|
||
'concept_type': 'lv1'
|
||
})
|
||
|
||
return parent_concepts
|
||
|
||
|
||
# ==================== 价格数据获取 ====================
|
||
|
||
def get_base_prices(stock_codes: list, current_date: str) -> dict:
|
||
"""获取昨收价"""
|
||
if not stock_codes:
|
||
return {}
|
||
|
||
valid_codes = [code for code in stock_codes if code and len(code) == 6 and code.isdigit()]
|
||
if not valid_codes:
|
||
return {}
|
||
|
||
stock_codes_str = "','".join(valid_codes)
|
||
|
||
query = f"""
|
||
SELECT SECCODE, F002N
|
||
FROM ea_trade
|
||
WHERE SECCODE IN ('{stock_codes_str}')
|
||
AND TRADEDATE = (
|
||
SELECT MAX(TRADEDATE) FROM ea_trade WHERE TRADEDATE < '{current_date}'
|
||
)
|
||
AND F002N IS NOT NULL AND F002N > 0
|
||
"""
|
||
|
||
try:
|
||
with MYSQL_ENGINE.connect() as conn:
|
||
result = conn.execute(text(query))
|
||
return {row[0]: float(row[1]) for row in result if row[1] and float(row[1]) > 0}
|
||
except Exception as e:
|
||
logger.error(f"获取基准价格失败: {e}")
|
||
return {}
|
||
|
||
|
||
def get_latest_prices(stock_codes: list) -> dict:
|
||
"""获取最新价格"""
|
||
if not stock_codes:
|
||
return {}
|
||
|
||
client = get_ch_client()
|
||
|
||
ch_codes = []
|
||
code_mapping = {}
|
||
for code in stock_codes:
|
||
ch_code = code_to_ch_format(code)
|
||
if ch_code:
|
||
ch_codes.append(ch_code)
|
||
code_mapping[ch_code] = code
|
||
|
||
if not ch_codes:
|
||
return {}
|
||
|
||
ch_codes_str = "','".join(ch_codes)
|
||
|
||
query = f"""
|
||
SELECT code, close, timestamp
|
||
FROM (
|
||
SELECT code, close, timestamp,
|
||
ROW_NUMBER() OVER (PARTITION BY code ORDER BY timestamp DESC) as rn
|
||
FROM stock_minute
|
||
WHERE code IN ('{ch_codes_str}')
|
||
AND toDate(timestamp) = today()
|
||
)
|
||
WHERE rn = 1
|
||
"""
|
||
|
||
try:
|
||
result = client.execute(query)
|
||
prices = {}
|
||
for row in result:
|
||
ch_code, close, ts = row
|
||
if close and close > 0:
|
||
pure_code = code_mapping.get(ch_code)
|
||
if pure_code:
|
||
prices[pure_code] = {'close': float(close), 'timestamp': ts}
|
||
return prices
|
||
except Exception as e:
|
||
logger.error(f"获取最新价格失败: {e}")
|
||
return {}
|
||
|
||
|
||
def get_prices_at_time(stock_codes: list, timestamp: datetime) -> dict:
|
||
"""获取指定时间的价格"""
|
||
if not stock_codes:
|
||
return {}
|
||
|
||
client = get_ch_client()
|
||
|
||
ch_codes = []
|
||
code_mapping = {}
|
||
for code in stock_codes:
|
||
ch_code = code_to_ch_format(code)
|
||
if ch_code:
|
||
ch_codes.append(ch_code)
|
||
code_mapping[ch_code] = code
|
||
|
||
if not ch_codes:
|
||
return {}
|
||
|
||
ch_codes_str = "','".join(ch_codes)
|
||
ts_str = timestamp.strftime('%Y-%m-%d %H:%M:%S')
|
||
|
||
query = f"""
|
||
SELECT code, close, timestamp
|
||
FROM stock_minute
|
||
WHERE code IN ('{ch_codes_str}')
|
||
AND timestamp = '{ts_str}'
|
||
"""
|
||
|
||
try:
|
||
result = client.execute(query)
|
||
prices = {}
|
||
for row in result:
|
||
ch_code, close, ts = row
|
||
if close and close > 0:
|
||
pure_code = code_mapping.get(ch_code)
|
||
if pure_code:
|
||
prices[pure_code] = {'close': float(close), 'timestamp': ts}
|
||
return prices
|
||
except Exception as e:
|
||
logger.error(f"获取历史价格失败: {e}")
|
||
return {}
|
||
|
||
|
||
def get_index_change(index_code: str, trade_date: str, timestamp: datetime = None) -> Optional[float]:
|
||
"""获取指数涨跌幅"""
|
||
client = get_ch_client()
|
||
|
||
try:
|
||
# 获取昨收
|
||
code_no_suffix = index_code.split('.')[0]
|
||
with MYSQL_ENGINE.connect() as conn:
|
||
prev_result = conn.execute(text("""
|
||
SELECT F006N FROM ea_exchangetrade
|
||
WHERE INDEXCODE = :code AND TRADEDATE < :today
|
||
ORDER BY TRADEDATE DESC LIMIT 1
|
||
"""), {'code': code_no_suffix, 'today': trade_date}).fetchone()
|
||
|
||
if not prev_result or not prev_result[0]:
|
||
return None
|
||
prev_close = float(prev_result[0])
|
||
|
||
# 获取当前价格
|
||
if timestamp:
|
||
ts_str = timestamp.strftime('%Y-%m-%d %H:%M:%S')
|
||
query = f"""
|
||
SELECT close FROM index_minute
|
||
WHERE code = '{index_code}' AND timestamp = '{ts_str}'
|
||
LIMIT 1
|
||
"""
|
||
else:
|
||
query = f"""
|
||
SELECT close FROM index_minute
|
||
WHERE code = '{index_code}' AND toDate(timestamp) = today()
|
||
ORDER BY timestamp DESC LIMIT 1
|
||
"""
|
||
|
||
result = client.execute(query)
|
||
if not result:
|
||
return None
|
||
|
||
current_price = float(result[0][0])
|
||
return (current_price - prev_close) / prev_close * 100
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取指数涨跌幅失败: {e}")
|
||
return None
|
||
|
||
|
||
def get_index_data(index_code: str, trade_date: str, timestamp: datetime = None) -> Optional[dict]:
|
||
"""获取指数完整数据"""
|
||
client = get_ch_client()
|
||
|
||
try:
|
||
code_no_suffix = index_code.split('.')[0]
|
||
with MYSQL_ENGINE.connect() as conn:
|
||
prev_result = conn.execute(text("""
|
||
SELECT F006N FROM ea_exchangetrade
|
||
WHERE INDEXCODE = :code AND TRADEDATE < :today
|
||
ORDER BY TRADEDATE DESC LIMIT 1
|
||
"""), {'code': code_no_suffix, 'today': trade_date}).fetchone()
|
||
|
||
if not prev_result or not prev_result[0]:
|
||
return None
|
||
prev_close = float(prev_result[0])
|
||
|
||
if timestamp:
|
||
ts_str = timestamp.strftime('%Y-%m-%d %H:%M:%S')
|
||
query = f"""
|
||
SELECT close, timestamp FROM index_minute
|
||
WHERE code = '{index_code}' AND timestamp = '{ts_str}'
|
||
LIMIT 1
|
||
"""
|
||
else:
|
||
query = f"""
|
||
SELECT close, timestamp FROM index_minute
|
||
WHERE code = '{index_code}' AND toDate(timestamp) = today()
|
||
ORDER BY timestamp DESC LIMIT 1
|
||
"""
|
||
|
||
result = client.execute(query)
|
||
if not result:
|
||
return None
|
||
|
||
current_price, ts = result[0]
|
||
change_pct = (float(current_price) - prev_close) / prev_close * 100
|
||
|
||
return {
|
||
'code': index_code,
|
||
'price': float(current_price),
|
||
'prev_close': prev_close,
|
||
'change_pct': round(change_pct, 4),
|
||
'timestamp': ts
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.error(f"获取指数数据失败: {e}")
|
||
return None
|
||
|
||
|
||
# ==================== 概念统计计算 ====================
|
||
|
||
def calculate_concept_stats(concepts: list, stock_changes: dict, index_change: float) -> list:
|
||
"""
|
||
计算概念统计，包含超额收益(Alpha)
|
||
Alpha = 概念涨幅 - 指数涨幅
|
||
"""
|
||
stats = []
|
||
|
||
for concept in concepts:
|
||
concept_id = concept['concept_id']
|
||
concept_name = concept['concept_name']
|
||
stock_codes = concept['stocks']
|
||
concept_type = concept.get('concept_type', 'leaf')
|
||
|
||
changes = []
|
||
limit_up_count = 0
|
||
limit_down_count = 0
|
||
|
||
for code in stock_codes:
|
||
if code in stock_changes:
|
||
change_pct = stock_changes[code]
|
||
changes.append(change_pct)
|
||
|
||
if change_pct >= 9.8:
|
||
limit_up_count += 1
|
||
elif change_pct <= -9.8:
|
||
limit_down_count += 1
|
||
|
||
if not changes:
|
||
continue
|
||
|
||
avg_change = np.mean(changes)
|
||
|
||
# 核心：计算超额收益 Alpha
|
||
alpha = avg_change - index_change
|
||
|
||
stats.append({
|
||
'concept_id': concept_id,
|
||
'concept_name': concept_name,
|
||
'change_pct': round(avg_change, 4),
|
||
'alpha': round(alpha, 4), # 超额收益
|
||
'index_change': round(index_change, 4),
|
||
'stock_count': len(changes),
|
||
'concept_type': concept_type,
|
||
'limit_up_count': limit_up_count,
|
||
'limit_down_count': limit_down_count,
|
||
})
|
||
|
||
# 按Alpha排序
|
||
stats.sort(key=lambda x: x['alpha'], reverse=True)
|
||
for i, item in enumerate(stats):
|
||
item['alpha_rank'] = i + 1
|
||
|
||
return stats
|
||
|
||
|
||
# ==================== Alpha 异动检测 ====================
|
||
|
||
def check_cooldown(concept_id: str, current_time: datetime) -> bool:
|
||
"""检查冷却"""
|
||
if concept_id in cooldown_cache:
|
||
last_alert = cooldown_cache[concept_id]
|
||
if current_time - last_alert < timedelta(minutes=ALPHA_CONFIG['cooldown_minutes']):
|
||
return True
|
||
return False
|
||
|
||
|
||
def set_cooldown(concept_id: str, current_time: datetime):
|
||
"""设置冷却"""
|
||
cooldown_cache[concept_id] = current_time
|
||
|
||
|
||
def calculate_importance(
|
||
alpha_zscore: float,
|
||
alpha: float,
|
||
alpha_rank: int,
|
||
total_concepts: int,
|
||
limit_up_count: int
|
||
) -> float:
|
||
"""计算重要性评分"""
|
||
weights = ALPHA_CONFIG['importance_weights']
|
||
|
||
# Z-Score 分数（归一化到0-1）
|
||
zscore_score = min(abs(alpha_zscore) / 5.0, 1.0)
|
||
|
||
# Alpha 绝对值分数
|
||
alpha_score = min(abs(alpha) / 3.0, 1.0)
|
||
|
||
# 排名分数（越靠前/越靠后越重要）
|
||
# alpha_rank 小表示强势，alpha_rank 大表示弱势
|
||
rank_ratio = alpha_rank / max(total_concepts, 1)
|
||
rank_score = max(1 - rank_ratio, rank_ratio) # 两端都重要
|
||
|
||
# 涨停分数
|
||
limit_score = min(limit_up_count / 3.0, 1.0)
|
||
|
||
total = (
|
||
weights['alpha_zscore'] * zscore_score +
|
||
weights['alpha_abs'] * alpha_score +
|
||
weights['rank_in_minute'] * rank_score +
|
||
weights['limit_up_count'] * limit_score
|
||
)
|
||
|
||
return round(total, 4)
|
||
|
||
|
||
def detect_alpha_alerts(
|
||
stats: list,
|
||
index_data: dict,
|
||
trade_date: str,
|
||
current_time: datetime
|
||
) -> list:
|
||
"""
|
||
基于Alpha Z-Score的异动检测
|
||
|
||
检测逻辑：
|
||
1. 如果有足够历史数据（>= min_history）：用 Z-Score 判断
|
||
2. 如果历史数据不足：用绝对 Alpha 阈值判断
|
||
"""
|
||
global alpha_stats
|
||
|
||
alerts = []
|
||
total_concepts = len(stats)
|
||
|
||
for stat in stats:
|
||
concept_id = stat['concept_id']
|
||
concept_name = stat['concept_name']
|
||
alpha = stat['alpha']
|
||
change_pct = stat['change_pct']
|
||
alpha_rank = stat['alpha_rank']
|
||
limit_up_count = stat['limit_up_count']
|
||
limit_down_count = stat['limit_down_count']
|
||
|
||
# 更新该概念的Alpha统计
|
||
concept_stats = alpha_stats[concept_id]
|
||
|
||
# 计算Z-Score（在更新前计算，用历史数据）
|
||
alpha_zscore = concept_stats.get_zscore(alpha)
|
||
|
||
# 更新统计
|
||
concept_stats.update(alpha, current_time)
|
||
|
||
# Alpha 太小，不值得关注
|
||
if abs(alpha) < ALPHA_CONFIG['min_alpha_abs']:
|
||
continue
|
||
|
||
# 检查冷却
|
||
if check_cooldown(concept_id, current_time):
|
||
continue
|
||
|
||
# 判断是否触发异动
|
||
is_alert = False
|
||
trigger_reason = ""
|
||
|
||
if concept_stats.is_ready():
|
||
# 方式1：有足够历史数据，用 Z-Score
|
||
if abs(alpha_zscore) >= ALPHA_CONFIG['zscore_threshold']:
|
||
is_alert = True
|
||
trigger_reason = f"Z={alpha_zscore:.2f}"
|
||
else:
|
||
# 方式2：历史数据不足，用绝对 Alpha 阈值
|
||
if abs(alpha) >= ALPHA_CONFIG['absolute_alpha_threshold']:
|
||
is_alert = True
|
||
trigger_reason = f"Alpha={alpha:+.2f}%"
|
||
alpha_zscore = alpha / 0.5 # 估算一个Z值（假设std=0.5）
|
||
|
||
if not is_alert:
|
||
continue
|
||
|
||
# 判断异动类型
|
||
if alpha > 0:
|
||
alert_type = 'alpha_surge' # 超额上涨
|
||
else:
|
||
alert_type = 'alpha_drop' # 超额下跌
|
||
|
||
# 计算重要性
|
||
importance = calculate_importance(
|
||
alpha_zscore, alpha, alpha_rank, total_concepts, limit_up_count
|
||
)
|
||
|
||
alert = {
|
||
'concept_id': concept_id,
|
||
'concept_name': concept_name,
|
||
'alert_type': alert_type,
|
||
'alert_time': current_time,
|
||
'trade_date': trade_date,
|
||
'change_pct': change_pct,
|
||
'alpha': alpha,
|
||
'alpha_zscore': round(alpha_zscore, 4),
|
||
'alpha_mean': round(concept_stats.mean, 4),
|
||
'alpha_std': round(concept_stats.std, 4),
|
||
'index_change_pct': stat['index_change'],
|
||
'alpha_rank': alpha_rank,
|
||
'limit_up_count': limit_up_count,
|
||
'limit_down_count': limit_down_count,
|
||
'stock_count': stat['stock_count'],
|
||
'concept_type': stat['concept_type'],
|
||
'importance_score': importance,
|
||
'index_code': REFERENCE_INDEX,
|
||
'index_price': index_data['price'] if index_data else None,
|
||
}
|
||
|
||
alerts.append(alert)
|
||
set_cooldown(concept_id, current_time)
|
||
|
||
# 日志
|
||
direction = "📈 超额上涨" if alert_type == 'alpha_surge' else "📉 超额下跌"
|
||
logger.info(
|
||
f"{direction}: {concept_name} "
|
||
f"Alpha={alpha:+.2f}% ({trigger_reason}) "
|
||
f"概念{change_pct:+.2f}% vs 大盘{stat['index_change']:+.2f}%"
|
||
)
|
||
|
||
# 按重要性排序，限制数量
|
||
alerts.sort(key=lambda x: x['importance_score'], reverse=True)
|
||
return alerts[:ALPHA_CONFIG['max_alerts_per_minute']]
|
||
|
||
|
||
# ==================== 数据持久化 ====================
|
||
|
||
def save_alerts_to_mysql(alerts: list):
|
||
"""保存异动到MySQL"""
|
||
if not alerts:
|
||
return 0
|
||
|
||
saved = 0
|
||
with MYSQL_ENGINE.begin() as conn:
|
||
for alert in alerts:
|
||
try:
|
||
# 映射 alert_type 到数据库格式
|
||
db_alert_type = 'surge_up' if alert['alert_type'] == 'alpha_surge' else 'surge_down'
|
||
|
||
insert_sql = text("""
|
||
INSERT INTO concept_minute_alert
|
||
(concept_id, concept_name, alert_time, alert_type, trade_date,
|
||
change_pct, prev_change_pct, change_delta,
|
||
limit_up_count, prev_limit_up_count, limit_up_delta,
|
||
rank_position, prev_rank_position, rank_delta,
|
||
index_code, index_price, index_change_pct,
|
||
stock_count, concept_type, zscore, importance_score, extra_info)
|
||
VALUES
|
||
(:concept_id, :concept_name, :alert_time, :alert_type, :trade_date,
|
||
:change_pct, :prev_change_pct, :change_delta,
|
||
:limit_up_count, 0, 0,
|
||
:rank_position, NULL, NULL,
|
||
:index_code, :index_price, :index_change_pct,
|
||
:stock_count, :concept_type, :zscore, :importance_score, :extra_info)
|
||
""")
|
||
|
||
params = {
|
||
'concept_id': alert['concept_id'],
|
||
'concept_name': alert['concept_name'],
|
||
'alert_time': alert['alert_time'],
|
||
'alert_type': db_alert_type,
|
||
'trade_date': alert['trade_date'],
|
||
'change_pct': alert['change_pct'],
|
||
'prev_change_pct': alert['index_change_pct'], # 用大盘涨幅作为参照
|
||
'change_delta': alert['alpha'], # Alpha 作为变化量
|
||
'limit_up_count': alert['limit_up_count'],
|
||
'rank_position': alert['alpha_rank'],
|
||
'index_code': alert['index_code'],
|
||
'index_price': alert['index_price'],
|
||
'index_change_pct': alert['index_change_pct'],
|
||
'stock_count': alert['stock_count'],
|
||
'concept_type': alert['concept_type'],
|
||
'zscore': alert['alpha_zscore'],
|
||
'importance_score': alert['importance_score'],
|
||
'extra_info': json.dumps({
|
||
'alpha': alert['alpha'],
|
||
'alpha_zscore': alert['alpha_zscore'],
|
||
'alpha_mean': alert['alpha_mean'],
|
||
'alpha_std': alert['alpha_std'],
|
||
'limit_down_count': alert['limit_down_count'],
|
||
}, ensure_ascii=False)
|
||
}
|
||
|
||
conn.execute(insert_sql, params)
|
||
saved += 1
|
||
|
||
except Exception as e:
|
||
logger.error(f"保存异动失败: {alert['concept_name']} - {e}")
|
||
|
||
return saved
|
||
|
||
|
||
def save_index_snapshot(index_data: dict, trade_date: str):
|
||
"""保存指数快照"""
|
||
if not index_data:
|
||
return
|
||
|
||
try:
|
||
with MYSQL_ENGINE.begin() as conn:
|
||
conn.execute(text("""
|
||
REPLACE INTO index_minute_snapshot
|
||
(index_code, trade_date, snapshot_time, price, prev_close, change_pct)
|
||
VALUES (:index_code, :trade_date, :snapshot_time, :price, :prev_close, :change_pct)
|
||
"""), {
|
||
'index_code': index_data['code'],
|
||
'trade_date': trade_date,
|
||
'snapshot_time': index_data['timestamp'],
|
||
'price': index_data['price'],
|
||
'prev_close': index_data.get('prev_close'),
|
||
'change_pct': index_data.get('change_pct')
|
||
})
|
||
except Exception as e:
|
||
logger.error(f"保存指数快照失败: {e}")
|
||
|
||
|
||
# ==================== 交易时间 ====================
|
||
|
||
def is_trading_time() -> bool:
|
||
now = datetime.now()
|
||
if now.weekday() >= 5:
|
||
return False
|
||
|
||
current_time = now.hour * 60 + now.minute
|
||
morning = (9 * 60 + 30 <= current_time <= 11 * 60 + 30)
|
||
afternoon = (13 * 60 <= current_time <= 15 * 60)
|
||
return morning or afternoon
|
||
|
||
|
||
def get_next_update_time() -> int:
|
||
now = datetime.now()
|
||
if is_trading_time():
|
||
return 60 - now.second
|
||
|
||
hour, minute = now.hour, now.minute
|
||
if hour < 9 or (hour == 9 and minute < 30):
|
||
target = now.replace(hour=9, minute=30, second=0)
|
||
elif (hour == 11 and minute >= 30) or hour == 12:
|
||
target = now.replace(hour=13, minute=0, second=0)
|
||
elif hour >= 15:
|
||
target = (now + timedelta(days=1)).replace(hour=9, minute=30, second=0)
|
||
else:
|
||
target = now + timedelta(minutes=1)
|
||
|
||
return max(60, int((target - now).total_seconds()))
|
||
|
||
|
||
# ==================== 主运行逻辑 ====================
|
||
|
||
def run_once(concepts: list, all_stocks: list, trade_date: str, timestamp: datetime = None) -> Tuple[int, int]:
|
||
"""执行一次检测"""
|
||
timestamp = timestamp or datetime.now()
|
||
|
||
# 获取基准价格
|
||
base_prices = get_base_prices(all_stocks, trade_date)
|
||
if not base_prices:
|
||
logger.warning("无法获取基准价格")
|
||
return 0, 0
|
||
|
||
# 获取当前价格
|
||
if timestamp.date() == datetime.now().date():
|
||
latest_prices = get_latest_prices(all_stocks)
|
||
else:
|
||
latest_prices = get_prices_at_time(all_stocks, timestamp)
|
||
|
||
if not latest_prices:
|
||
logger.warning("无法获取最新价格")
|
||
return 0, 0
|
||
|
||
# 计算股票涨跌幅
|
||
stock_changes = {}
|
||
for code, price_data in latest_prices.items():
|
||
if code in base_prices and base_prices[code] > 0:
|
||
change = (price_data['close'] - base_prices[code]) / base_prices[code] * 100
|
||
stock_changes[code] = round(change, 4)
|
||
|
||
if not stock_changes:
|
||
return 0, 0
|
||
|
||
# 获取指数数据
|
||
index_data = get_index_data(REFERENCE_INDEX, trade_date, timestamp)
|
||
if not index_data:
|
||
logger.warning("无法获取指数数据")
|
||
return 0, 0
|
||
|
||
index_change = index_data['change_pct']
|
||
save_index_snapshot(index_data, trade_date)
|
||
|
||
# 计算概念统计（包含Alpha）
|
||
stats = calculate_concept_stats(concepts, stock_changes, index_change)
|
||
|
||
# 检测异动
|
||
alerts = detect_alpha_alerts(stats, index_data, trade_date, timestamp)
|
||
|
||
# 保存
|
||
if alerts:
|
||
saved = save_alerts_to_mysql(alerts)
|
||
logger.info(f"💾 保存了 {saved} 条异动")
|
||
|
||
return len(stats), len(alerts)
|
||
|
||
|
||
def run_realtime():
|
||
"""实时运行"""
|
||
logger.info("=" * 60)
|
||
logger.info("🚀 启动概念异动检测服务（Alpha Z-Score 方法）")
|
||
logger.info("=" * 60)
|
||
logger.info(f"配置: Z-Score阈值={ALPHA_CONFIG['zscore_threshold']}, 冷却={ALPHA_CONFIG['cooldown_minutes']}分钟")
|
||
|
||
# 加载概念
|
||
leaf_concepts = get_all_concepts()
|
||
parent_concepts = load_hierarchy_concepts(leaf_concepts)
|
||
all_concepts = leaf_concepts + parent_concepts
|
||
|
||
all_stocks = set()
|
||
for c in all_concepts:
|
||
all_stocks.update(c['stocks'])
|
||
all_stocks = list(all_stocks)
|
||
|
||
logger.info(f"监控 {len(all_concepts)} 个概念, {len(all_stocks)} 只股票")
|
||
|
||
total_alerts = 0
|
||
last_reload = datetime.now()
|
||
|
||
while True:
|
||
try:
|
||
now = datetime.now()
|
||
trade_date = now.strftime('%Y-%m-%d')
|
||
|
||
# 每小时重载概念
|
||
if (now - last_reload).total_seconds() > 3600:
|
||
leaf_concepts = get_all_concepts()
|
||
parent_concepts = load_hierarchy_concepts(leaf_concepts)
|
||
all_concepts = leaf_concepts + parent_concepts
|
||
all_stocks = list(set(s for c in all_concepts for s in c['stocks']))
|
||
last_reload = now
|
||
logger.info(f"重载概念: {len(all_concepts)} 个")
|
||
|
||
if not is_trading_time():
|
||
wait = get_next_update_time()
|
||
logger.info(f"⏰ 非交易时间，等待 {wait//60} 分钟")
|
||
time.sleep(min(wait, 300))
|
||
continue
|
||
|
||
updated, alert_count = run_once(all_concepts, all_stocks, trade_date)
|
||
total_alerts += alert_count
|
||
|
||
if alert_count > 0:
|
||
logger.info(f"📊 本次 {alert_count} 条异动，累计 {total_alerts} 条")
|
||
|
||
time.sleep(60 - datetime.now().second)
|
||
|
||
except KeyboardInterrupt:
|
||
logger.info("\n停止服务")
|
||
break
|
||
except Exception as e:
|
||
logger.error(f"错误: {e}")
|
||
import traceback
|
||
traceback.print_exc()
|
||
time.sleep(60)
|
||
|
||
|
||
def run_backtest(trade_date: str, clear_existing: bool = True):
|
||
"""回测"""
|
||
global alpha_stats, cooldown_cache
|
||
|
||
logger.info("=" * 60)
|
||
logger.info(f"🔄 回测: {trade_date} (Alpha Z-Score 方法)")
|
||
logger.info("=" * 60)
|
||
|
||
# 清空状态
|
||
alpha_stats.clear()
|
||
cooldown_cache.clear()
|
||
|
||
if clear_existing:
|
||
with MYSQL_ENGINE.begin() as conn:
|
||
conn.execute(text("DELETE FROM concept_minute_alert WHERE trade_date = :date"), {'date': trade_date})
|
||
conn.execute(text("DELETE FROM index_minute_snapshot WHERE trade_date = :date"), {'date': trade_date})
|
||
logger.info(f"已清除 {trade_date} 的数据")
|
||
|
||
# 加载概念
|
||
leaf_concepts = get_all_concepts()
|
||
parent_concepts = load_hierarchy_concepts(leaf_concepts)
|
||
all_concepts = leaf_concepts + parent_concepts
|
||
|
||
all_stocks = list(set(s for c in all_concepts for s in c['stocks']))
|
||
logger.info(f"概念: {len(all_concepts)}, 股票: {len(all_stocks)}")
|
||
|
||
# 获取分钟时间戳
|
||
client = get_ch_client()
|
||
result = client.execute(f"""
|
||
SELECT DISTINCT timestamp FROM stock_minute
|
||
WHERE toDate(timestamp) = '{trade_date}'
|
||
ORDER BY timestamp
|
||
""")
|
||
timestamps = [row[0] for row in result]
|
||
|
||
if not timestamps:
|
||
logger.error(f"未找到 {trade_date} 的数据")
|
||
return
|
||
|
||
logger.info(f"时间点: {len(timestamps)}")
|
||
|
||
total_alerts = 0
|
||
|
||
for i, ts in enumerate(timestamps):
|
||
updated, alerts = run_once(all_concepts, all_stocks, trade_date, ts)
|
||
total_alerts += alerts
|
||
|
||
if (i + 1) % 30 == 0:
|
||
logger.info(f"进度: {i+1}/{len(timestamps)} ({(i+1)*100//len(timestamps)}%), 异动: {total_alerts}")
|
||
|
||
logger.info("=" * 60)
|
||
logger.info(f"✅ 回测完成! 检测到 {total_alerts} 条异动")
|
||
logger.info("=" * 60)
|
||
|
||
|
||
def show_status():
|
||
"""显示状态"""
|
||
print("\n" + "=" * 60)
|
||
print("概念异动检测服务 - Alpha Z-Score 方法")
|
||
print("=" * 60)
|
||
|
||
print(f"\n当前时间: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||
print(f"交易时间: {'是' if is_trading_time() else '否'}")
|
||
print(f"Z-Score阈值: {ALPHA_CONFIG['zscore_threshold']}")
|
||
|
||
try:
|
||
with MYSQL_ENGINE.connect() as conn:
|
||
result = conn.execute(text("""
|
||
SELECT alert_type, COUNT(*), AVG(change_delta)
|
||
FROM concept_minute_alert
|
||
WHERE trade_date = CURDATE()
|
||
GROUP BY alert_type
|
||
"""))
|
||
|
||
print("\n今日异动统计:")
|
||
rows = list(result)
|
||
if rows:
|
||
for row in rows:
|
||
type_name = {'surge_up': '超额上涨', 'surge_down': '超额下跌'}.get(row[0], row[0])
|
||
avg_alpha = f"{row[2]:+.2f}%" if row[2] else "-"
|
||
print(f" {type_name}: {row[1]} 条 (平均Alpha: {avg_alpha})")
|
||
else:
|
||
print(" 暂无异动")
|
||
|
||
print("\n最新异动 (前10条):")
|
||
result = conn.execute(text("""
|
||
SELECT concept_name, alert_type, alert_time, change_pct,
|
||
change_delta, index_change_pct, zscore
|
||
FROM concept_minute_alert
|
||
WHERE trade_date = CURDATE()
|
||
ORDER BY alert_time DESC
|
||
LIMIT 10
|
||
"""))
|
||
|
||
rows = list(result)
|
||
if rows:
|
||
print(f" {'概念':<18} | {'类型':<8} | {'时间':<5} | {'涨幅':>6} | {'Alpha':>6} | {'大盘':>5} | {'Z':>5}")
|
||
print(" " + "-" * 75)
|
||
for row in rows:
|
||
name = (row[0][:16] + '..') if len(row[0]) > 18 else row[0]
|
||
t = {'surge_up': '超额涨', 'surge_down': '超额跌'}.get(row[1], row[1][:6])
|
||
time_s = row[2].strftime('%H:%M') if row[2] else '-'
|
||
chg = f"{row[3]:+.2f}%" if row[3] else '-'
|
||
alpha = f"{row[4]:+.2f}%" if row[4] else '-'
|
||
idx = f"{row[5]:+.1f}%" if row[5] else '-'
|
||
z = f"{row[6]:.1f}" if row[6] else '-'
|
||
print(f" {name:<18} | {t:<8} | {time_s:<5} | {chg:>6} | {alpha:>6} | {idx:>5} | {z:>5}")
|
||
|
||
except Exception as e:
|
||
print(f" 查询失败: {e}")
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description='概念异动检测 - Alpha Z-Score 方法')
|
||
parser.add_argument('command', nargs='?', default='realtime',
|
||
choices=['realtime', 'once', 'status', 'backtest'],
|
||
help='命令')
|
||
parser.add_argument('--date', '-d', type=str, default=None, help='回测日期')
|
||
parser.add_argument('--keep', '-k', action='store_true', help='保留已有数据')
|
||
|
||
args = parser.parse_args()
|
||
|
||
if args.command == 'realtime':
|
||
run_realtime()
|
||
elif args.command == 'once':
|
||
leaf_concepts = get_all_concepts()
|
||
parent_concepts = load_hierarchy_concepts(leaf_concepts)
|
||
all_concepts = leaf_concepts + parent_concepts
|
||
all_stocks = list(set(s for c in all_concepts for s in c['stocks']))
|
||
trade_date = datetime.now().strftime('%Y-%m-%d')
|
||
run_once(all_concepts, all_stocks, trade_date)
|
||
elif args.command == 'status':
|
||
show_status()
|
||
elif args.command == 'backtest':
|
||
trade_date = args.date or datetime.now().strftime('%Y-%m-%d')
|
||
run_backtest(trade_date, not args.keep)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|