From f58937bf284b8bdc49188178d0a1af08355ea20f Mon Sep 17 00:00:00 2001 From: foolcage <5533061@qq.com> Date: Tue, 17 Jan 2023 18:10:30 +0800 Subject: [PATCH] add emotion data from provider jqka --- src/zvt/domain/__init__.py | 6 + src/zvt/domain/emotion/__init__.py | 13 ++ src/zvt/domain/emotion/emotion.py | 87 ++++++++ src/zvt/fill_project.py | 6 +- src/zvt/recorders/__init__.py | 6 + src/zvt/recorders/em/em_api.py | 1 + src/zvt/recorders/jqka/__init__.py | 19 ++ .../jqka/emotion/JqkaEmotionRecorder.py | 207 ++++++++++++++++++ src/zvt/recorders/jqka/emotion/__init__.py | 13 ++ src/zvt/recorders/jqka/jqka_api.py | 101 +++++++++ 10 files changed, 456 insertions(+), 3 deletions(-) create mode 100644 src/zvt/domain/emotion/__init__.py create mode 100644 src/zvt/domain/emotion/emotion.py create mode 100644 src/zvt/recorders/jqka/__init__.py create mode 100644 src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py create mode 100644 src/zvt/recorders/jqka/emotion/__init__.py create mode 100644 src/zvt/recorders/jqka/jqka_api.py diff --git a/src/zvt/domain/__init__.py b/src/zvt/domain/__init__.py index 5af29524..a3e1b388 100644 --- a/src/zvt/domain/__init__.py +++ b/src/zvt/domain/__init__.py @@ -190,3 +190,9 @@ def get_future_name(code): from .actor import __all__ as _actor_all __all__ += _actor_all + +# import all from submodule emotion +from .emotion import * +from .emotion import __all__ as _emotion_all + +__all__ += _emotion_all diff --git a/src/zvt/domain/emotion/__init__.py b/src/zvt/domain/emotion/__init__.py new file mode 100644 index 00000000..d7225685 --- /dev/null +++ b/src/zvt/domain/emotion/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# the __all__ is generated +__all__ = [] + +# __init__.py structure: +# common code of the package +# export interface in __all__ which contains __all__ of its sub modules + +# import all from submodule emotion +from .emotion import * +from .emotion import __all__ as _emotion_all + +__all__ += _emotion_all diff --git a/src/zvt/domain/emotion/emotion.py b/src/zvt/domain/emotion/emotion.py new file mode 100644 index 00000000..42b0fe1b --- /dev/null +++ b/src/zvt/domain/emotion/emotion.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +from sqlalchemy import Column, String, Integer, DateTime, Boolean, Float +from sqlalchemy.orm import declarative_base + +from zvt.contract import Mixin +from zvt.contract.register import register_schema + +EmotionBase = declarative_base() + + +class LimitUpInfo(EmotionBase, Mixin): + __tablename__ = "limit_up_info" + + code = Column(String(length=32)) + name = Column(String(length=32)) + #: 是否新股 + is_new = Column(Boolean) + #: 是否回封，是就是打开过，否相反 + is_again_limit = Column(Boolean) + #: 涨停打开次数,0代表封住就没开板 + open_count = Column(Integer) + #: 首次封板时间 + first_limit_up_time = Column(DateTime) + #: 最后封板时间 + last_limit_up_time = Column(DateTime) + #: 涨停类型:换手板，一字板 + limit_up_type = Column(String) + #: 封单金额 + order_amount = Column(String) + #: 最近一年封板成功率 + success_rate = Column(Float) + #: 流通市值 + currency_value = Column(Float) + #: 涨幅 + change_pct = Column(Float) + #: 换手率 + turnover_rate = Column(Float) + #: 涨停原因 + reason = Column(String) + #: 几天几板 + high_days = Column(String) + #: 最近几板，不一定是连板 + high_days_count = Column(Integer) + + +class LimitDownInfo(EmotionBase, Mixin): + __tablename__ = "limit_down_info" + + code = Column(String(length=32)) + name = Column(String(length=32)) + #: 是否新股 + is_new = Column(Boolean) + #: 是否回封，是就是打开过，否相反 + is_again_limit = Column(Boolean) + #: 流通市值 + currency_value = Column(Float) + #: 涨幅 + change_pct = Column(Float) + #: 换手率 + turnover_rate = Column(Float) + + +class Emotion(EmotionBase, Mixin): + __tablename__ = "emotion" + #: 涨停数量 + limit_up_count = Column(Integer) + #: 炸板数 + limit_up_open_count = Column(Integer) + #: 涨停封板成功率 + limit_up_success_rate = Column(Float) + + #: 连板高度 + max_height = Column(Integer) + #: 连板数x个数 相加 + continuous_power = Column(Integer) + + #: 跌停数量 + limit_down_count = Column(Integer) + #: 跌停打开 + limit_down_open_count = Column(Integer) + #: 跌停封板成功率 + limit_down_success_rate = Column(Float) + + +register_schema(providers=["jqka"], db_name="emotion", schema_base=EmotionBase) +# the __all__ is generated +__all__ = ["LimitUpInfo", "LimitDownInfo", "Emotion"] diff --git a/src/zvt/fill_project.py b/src/zvt/fill_project.py index d3d4ea66..6e92d78c 100644 --- a/src/zvt/fill_project.py +++ b/src/zvt/fill_project.py @@ -90,10 +90,10 @@ def gen_kdata_schemas(): # gen_exports("ml") # gen_exports("utils") # gen_exports('informer') - gen_exports("api") + # gen_exports("api") # gen_exports('trader') # gen_exports('autocode') # gen_exports("ml") # gen_kdata_schemas() - # gen_exports("recorders") - # gen_exports("domain") + gen_exports("recorders") + gen_exports("domain") diff --git a/src/zvt/recorders/__init__.py b/src/zvt/recorders/__init__.py index d273c9e8..a0b017ae 100644 --- a/src/zvt/recorders/__init__.py +++ b/src/zvt/recorders/__init__.py @@ -102,6 +102,12 @@ def init_main_index(provider="exchange"): __all__ += _consts_all +# import all from submodule jqka +from .jqka import * +from .jqka import __all__ as _jqka_all + +__all__ += _jqka_all + # import all from submodule eastmoney from .eastmoney import * from .eastmoney import __all__ as _eastmoney_all diff --git a/src/zvt/recorders/em/em_api.py b/src/zvt/recorders/em/em_api.py index 39a3e9ae..493ac08b 100644 --- a/src/zvt/recorders/em/em_api.py +++ b/src/zvt/recorders/em/em_api.py @@ -720,6 +720,7 @@ def to_zvt_code(code): __all__ = [ "get_treasury_yield", "get_ii_holder_report_dates", + "get_dragon_and_tiger_list", "get_dragon_and_tiger", "get_holder_report_dates", "get_free_holder_report_dates", diff --git a/src/zvt/recorders/jqka/__init__.py b/src/zvt/recorders/jqka/__init__.py new file mode 100644 index 00000000..48d5641b --- /dev/null +++ b/src/zvt/recorders/jqka/__init__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# the __all__ is generated +__all__ = [] + +# __init__.py structure: +# common code of the package +# export interface in __all__ which contains __all__ of its sub modules + +# import all from submodule jqka_api +from .jqka_api import * +from .jqka_api import __all__ as _jqka_api_all + +__all__ += _jqka_api_all + +# import all from submodule emotion +from .emotion import * +from .emotion import __all__ as _emotion_all + +__all__ += _emotion_all diff --git a/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py b/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py new file mode 100644 index 00000000..9a8587dd --- /dev/null +++ b/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py @@ -0,0 +1,207 @@ +# -*- coding: utf-8 -*- +from typing import List + +import pandas as pd + +from zvt.api.utils import china_stock_code_to_id +from zvt.contract.api import df_to_db +from zvt.contract.recorder import TimestampsDataRecorder +from zvt.domain import Stock +from zvt.domain.emotion.emotion import LimitUpInfo, LimitDownInfo, Emotion +from zvt.recorders.jqka import jqka_api +from zvt.utils import to_time_str, next_date, current_date, to_pd_timestamp + + +def _get_high_days_count(high_days_str: str): + if not high_days_str or (high_days_str == "首板"): + return 1 + return int(high_days_str[3]) + + +class JqkaLimitUpRecorder(TimestampsDataRecorder): + entity_provider = "em" + entity_schema = Stock + + provider = "jqka" + data_schema = LimitUpInfo + + def init_entities(self): + # fake entity to for trigger run + self.entities = [Stock(id="stock_sz_000001")] + + def init_timestamps(self, entity_item) -> List[pd.Timestamp]: + latest_infos = LimitUpInfo.query_data( + provider=self.provider, order=LimitUpInfo.timestamp.desc(), limit=1, return_type="domain" + ) + if latest_infos: + start_date = latest_infos[0].timestamp + else: + # 最近一年半的数据 + start_date = next_date(current_date(), -365 - 366 / 2) + return pd.date_range(start=start_date, end=pd.Timestamp.now(), freq="B").tolist() + + def record(self, entity, start, end, size, timestamps): + for timestamp in timestamps: + the_date = to_time_str(timestamp) + self.logger.info(f"record {self.data_schema} to {the_date}") + limit_ups = jqka_api.get_limit_up(date=the_date) + if limit_ups: + records = [] + for data in limit_ups: + entity_id = china_stock_code_to_id(code=data["code"]) + record = { + "id": "{}_{}".format(entity_id, the_date), + "entity_id": entity_id, + "timestamp": to_pd_timestamp(the_date), + "code": data["code"], + "name": data["name"], + "is_new": data["is_new"], + "is_again_limit": data["is_again_limit"], + "open_count": data["open_num"] if data["open_num"] else 0, + "first_limit_up_time": pd.Timestamp.fromtimestamp(int(data["first_limit_up_time"])), + "last_limit_up_time": pd.Timestamp.fromtimestamp(int(data["last_limit_up_time"])), + "limit_up_type": data["limit_up_type"], + "order_amount": data["order_amount"], + "success_rate": data["limit_up_suc_rate"], + "currency_value": data["currency_value"], + "change_pct": data["change_rate"] / 100, + "turnover_rate": data["turnover_rate"] / 100, + "reason": data["reason_type"], + "high_days": data["high_days"], + "high_days_count": _get_high_days_count(data["high_days"]), + } + records.append(record) + df = pd.DataFrame.from_records(records) + df_to_db( + data_schema=self.data_schema, + df=df, + provider=self.provider, + force_update=True, + drop_duplicates=True, + ) + + +class JqkaLimitDownRecorder(TimestampsDataRecorder): + entity_provider = "em" + entity_schema = Stock + + provider = "jqka" + data_schema = LimitDownInfo + + def init_entities(self): + # fake entity to for trigger run + self.entities = [Stock(id="stock_sz_000001")] + + def init_timestamps(self, entity_item) -> List[pd.Timestamp]: + latest_infos = LimitDownInfo.query_data( + provider=self.provider, order=LimitDownInfo.timestamp.desc(), limit=1, return_type="domain" + ) + if latest_infos: + start_date = latest_infos[0].timestamp + else: + # 最近一年半的数据 + start_date = next_date(current_date(), -365 - 366 / 2) + return pd.date_range(start=start_date, end=pd.Timestamp.now(), freq="B").tolist() + + def record(self, entity, start, end, size, timestamps): + for timestamp in timestamps: + the_date = to_time_str(timestamp) + self.logger.info(f"record {self.data_schema} to {the_date}") + limit_downs = jqka_api.get_limit_down(date=the_date) + if limit_downs: + records = [] + for data in limit_downs: + entity_id = china_stock_code_to_id(code=data["code"]) + record = { + "id": "{}_{}".format(entity_id, the_date), + "entity_id": entity_id, + "timestamp": to_pd_timestamp(the_date), + "code": data["code"], + "name": data["name"], + "is_new": data["is_new"], + "is_again_limit": data["is_again_limit"], + "currency_value": data["currency_value"], + "change_pct": data["change_rate"] / 100, + "turnover_rate": data["turnover_rate"] / 100, + } + records.append(record) + df = pd.DataFrame.from_records(records) + df_to_db( + data_schema=self.data_schema, + df=df, + provider=self.provider, + force_update=True, + drop_duplicates=True, + ) + + +def _cal_power_and_max_height(continuous_limit_up: dict): + max_height = 0 + power = 0 + for item in continuous_limit_up: + if max_height < item["height"]: + max_height = item["height"] + power = power + item["height"] * item["number"] + return max_height, power + + +class JqkaEmotionRecorder(TimestampsDataRecorder): + entity_provider = "em" + entity_schema = Stock + + provider = "jqka" + data_schema = Emotion + + def init_entities(self): + # fake entity to for trigger run + self.entities = [Stock(id="stock_sz_000001")] + + def init_timestamps(self, entity_item) -> List[pd.Timestamp]: + latest_infos = Emotion.query_data( + provider=self.provider, order=Emotion.timestamp.desc(), limit=1, return_type="domain" + ) + if latest_infos: + start_date = latest_infos[0].timestamp + else: + # 最近一年半的数据 + start_date = next_date(current_date(), -365 - 366 / 2) + return pd.date_range(start=start_date, end=pd.Timestamp.now(), freq="B").tolist() + + def record(self, entity, start, end, size, timestamps): + for timestamp in timestamps: + the_date = to_time_str(timestamp) + self.logger.info(f"record {self.data_schema} to {the_date}") + limit_stats = jqka_api.get_limit_stats(date=the_date) + continuous_limit_up = jqka_api.get_continuous_limit_up(date=the_date) + max_height, continuous_power = _cal_power_and_max_height(continuous_limit_up=continuous_limit_up) + + if limit_stats: + # 大盘 + entity_id = "stock_sh_000001" + record = { + "id": "{}_{}".format(entity_id, the_date), + "entity_id": entity_id, + "timestamp": to_pd_timestamp(the_date), + "limit_up_count": limit_stats["limit_up_count"]["today"]["num"], + "limit_up_open_count": limit_stats["limit_up_count"]["today"]["open_num"], + "limit_up_success_rate": limit_stats["limit_up_count"]["today"]["rate"], + "limit_down_count": limit_stats["limit_down_count"]["today"]["num"], + "limit_down_open_count": limit_stats["limit_down_count"]["today"]["open_num"], + "limit_down_success_rate": limit_stats["limit_down_count"]["today"]["rate"], + "max_height": max_height, + "continuous_power": continuous_power, + } + df = pd.DataFrame.from_records([record]) + df_to_db( + data_schema=self.data_schema, + df=df, + provider=self.provider, + force_update=True, + drop_duplicates=True, + ) + + +if __name__ == "__main__": + JqkaEmotionRecorder().run() +# the __all__ is generated +__all__ = ["JqkaLimitUpRecorder", "JqkaLimitDownRecorder", "JqkaEmotionRecorder"] diff --git a/src/zvt/recorders/jqka/emotion/__init__.py b/src/zvt/recorders/jqka/emotion/__init__.py new file mode 100644 index 00000000..d8285f22 --- /dev/null +++ b/src/zvt/recorders/jqka/emotion/__init__.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# the __all__ is generated +__all__ = [] + +# __init__.py structure: +# common code of the package +# export interface in __all__ which contains __all__ of its sub modules + +# import all from submodule JqkaEmotionRecorder +from .JqkaEmotionRecorder import * +from .JqkaEmotionRecorder import __all__ as _JqkaEmotionRecorder_all + +__all__ += _JqkaEmotionRecorder_all diff --git a/src/zvt/recorders/jqka/jqka_api.py b/src/zvt/recorders/jqka/jqka_api.py new file mode 100644 index 00000000..aa467eee --- /dev/null +++ b/src/zvt/recorders/jqka/jqka_api.py @@ -0,0 +1,101 @@ +# -*- coding: utf-8 -*- + +import requests + +from zvt.utils import now_timestamp, to_time_str, TIME_FORMAT_DAY1, chrome_copy_header_to_dict + +_JKQA_HEADER = chrome_copy_header_to_dict( + """ +Accept: application/json, text/plain, */* +Accept-Encoding: gzip, deflate, br +Accept-Language: zh-CN,zh;q=0.9,en;q=0.8 +Connection: keep-alive +Host: data.10jqka.com.cn +Referer: https://data.10jqka.com.cn/datacenterph/limitup/limtupInfo.html?fontzoom=no&client_userid=cA2fp&share_hxapp=gsc&share_action=webpage_share.1&back_source=wxhy +sec-ch-ua: "Not_A Brand";v="99", "Google Chrome";v="109", "Chromium";v="109" +sec-ch-ua-mobile: ?1 +sec-ch-ua-platform: "Android" +Sec-Fetch-Dest: empty +Sec-Fetch-Mode: cors +Sec-Fetch-Site: same-origin +User-Agent: Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Mobile Safari/537.36 +""" +) + + +def get_continuous_limit_up(date: str): + date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1) + url = f"https://data.10jqka.com.cn/dataapi/limit_up/continuous_limit_up?filter=HS,GEM2STAR&date={date_str}" + resp = requests.get(url, headers=_JKQA_HEADER) + if resp.status_code == 200: + json_result = resp.json() + if json_result: + return json_result["data"] + raise RuntimeError(f"request jkqa data code: {resp.status_code}, error: {resp.text}") + + +def get_limit_stats(date: str): + date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1) + url = f"https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool?page=1&limit=1&field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&date={date_str}&order_field=330324&order_type=0&_={now_timestamp()}" + resp = requests.get(url, headers=_JKQA_HEADER) + if resp.status_code == 200: + json_result = resp.json() + if json_result: + return { + "limit_up_count": json_result["data"]["limit_up_count"], + "limit_down_count": json_result["data"]["limit_down_count"], + } + raise RuntimeError(f"request jkqa data code: {resp.status_code}, error: {resp.text}") + + +def get_limit_up(date: str): + date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1) + url = f"https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=330324&order_type=0&date={date_str}" + return get_jkqa_data(url=url) + + +def get_limit_down(date: str): + date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1) + url = f"https://data.10jqka.com.cn/dataapi/limit_up/lower_limit_pool?page=1&limit=15&field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=330324&order_type=0&date={date_str}" + return get_jkqa_data(url=url) + + +def get_jkqa_data(url, pn=1, ps=2000, fetch_all=True, headers=_JKQA_HEADER): + url = url + f"&page={pn}&limit={ps}&_={now_timestamp()}" + print(url) + resp = requests.get(url, headers=headers) + if resp.status_code == 200: + json_result = resp.json() + if json_result and json_result["data"]: + data: list = json_result["data"]["info"] + if fetch_all: + if pn < json_result["data"]["page"]["page"]: + next_data = get_jkqa_data( + pn=pn + 1, + ps=ps, + fetch_all=fetch_all, + ) + if next_data: + data = data + next_data + return data + else: + return data + else: + return data + else: + return data + return None + raise RuntimeError(f"request jkqa data code: {resp.status_code}, error: {resp.text}") + + +if __name__ == "__main__": + # result = get_limit_up(date="20210716") + # print(result) + # result = get_limit_stats(date="20210716") + # print(result) + # result = get_limit_down(date="20210716") + # print(result) + result = get_continuous_limit_up(date="20210716") + print(result) +# the __all__ is generated +__all__ = ["get_continuous_limit_up", "get_limit_stats", "get_limit_up", "get_limit_down", "get_jkqa_data"]