From 5391df8d74bd9a71863eaeaa77aebd373e33d773 Mon Sep 17 00:00:00 2001 From: foolcage <5533061@qq.com> Date: Fri, 24 Jun 2022 18:35:08 +0800 Subject: [PATCH] Ignore jq account error --- examples/report_utils.py | 179 ++++++++++-------- src/zvt/informer/informer.py | 54 +++--- .../base_china_stock_finance_recorder.py | 2 +- 3 files changed, 125 insertions(+), 110 deletions(-) diff --git a/examples/report_utils.py b/examples/report_utils.py index ec809905..a8dbc270 100644 --- a/examples/report_utils.py +++ b/examples/report_utils.py @@ -170,98 +170,109 @@ def report_top_entities( em_group_over_write=True, return_type=TopType.positive, ): - if periods is None: - periods = [7, 30, 365] - if not adjust_type: - adjust_type = default_adjust_type(entity_type=entity_type) - kdata_schema = get_kdata_schema(entity_type=entity_type, adjust_type=adjust_type) - entity_schema = get_entity_schema(entity_type=entity_type) + error_count = 0 - target_date = get_latest_kdata_date(provider=data_provider, entity_type=entity_type, adjust_type=adjust_type) + while error_count <= 10: + try: + if periods is None: + periods = [7, 30, 365] + if not adjust_type: + adjust_type = default_adjust_type(entity_type=entity_type) + kdata_schema = get_kdata_schema(entity_type=entity_type, adjust_type=adjust_type) + entity_schema = get_entity_schema(entity_type=entity_type) - filter_entity_ids = get_entity_ids_by_filter( - provider=entity_provider, - ignore_st=ignore_st, - ignore_new_stock=ignore_new_stock, - entity_schema=entity_schema, - target_date=target_date, - entity_ids=entity_ids, - ) + target_date = get_latest_kdata_date( + provider=data_provider, entity_type=entity_type, adjust_type=adjust_type + ) - if not filter_entity_ids: - msg = f"{entity_type} no entity_ids selected" - logger.error(msg) - informer.send_message(zvt_config["email_username"], "report_top_stats error", msg) - return + filter_entity_ids = get_entity_ids_by_filter( + provider=entity_provider, + ignore_st=ignore_st, + ignore_new_stock=ignore_new_stock, + entity_schema=entity_schema, + target_date=target_date, + entity_ids=entity_ids, + ) - filter_turnover_df = kdata_schema.query_data( - filters=[ - kdata_schema.turnover >= turnover_threshold, - kdata_schema.turnover_rate >= turnover_rate_threshold, - ], - provider=data_provider, - start_timestamp=target_date, - index="entity_id", - columns=["entity_id", "code"], - ) - if filter_entity_ids: - filter_entity_ids = set(filter_entity_ids) & set(filter_turnover_df.index.tolist()) - else: - filter_entity_ids = filter_turnover_df.index.tolist() + if not filter_entity_ids: + msg = f"{entity_type} no entity_ids selected" + logger.error(msg) + informer.send_message(zvt_config["email_username"], "report_top_stats error", msg) + return - if not filter_entity_ids: - msg = f"{entity_type} no entity_ids selected" - logger.error(msg) - informer.send_message(zvt_config["email_username"], "report_top_stats error", msg) - return + filter_turnover_df = kdata_schema.query_data( + filters=[ + kdata_schema.turnover >= turnover_threshold, + kdata_schema.turnover_rate >= turnover_rate_threshold, + ], + provider=data_provider, + start_timestamp=target_date, + index="entity_id", + columns=["entity_id", "code"], + ) + if filter_entity_ids: + filter_entity_ids = set(filter_entity_ids) & set(filter_turnover_df.index.tolist()) + else: + filter_entity_ids = filter_turnover_df.index.tolist() - logger.info(f"{entity_type} filter_entity_ids size: {len(filter_entity_ids)}") - filters = [kdata_schema.entity_id.in_(filter_entity_ids)] - for i, period in enumerate(periods): - interval = period - if target_date.weekday() + 1 < interval: - interval = interval + 2 - start = next_date(target_date, -interval) - positive_df, negative_df = get_top_performance_entities( - entity_type=entity_type, - start_timestamp=start, - kdata_filters=filters, - pct=1, - show_name=True, - entity_provider=entity_provider, - data_provider=data_provider, - return_type=return_type, - ) + if not filter_entity_ids: + msg = f"{entity_type} no entity_ids selected" + logger.error(msg) + informer.send_message(zvt_config["email_username"], "report_top_stats error", msg) + return - if return_type == TopType.positive: - tag = "最靓仔" - df = positive_df - else: - tag = "谁有我惨" - df = negative_df + logger.info(f"{entity_type} filter_entity_ids size: {len(filter_entity_ids)}") + filters = [kdata_schema.entity_id.in_(filter_entity_ids)] + for i, period in enumerate(periods): + interval = period + if target_date.weekday() + 1 < interval: + interval = interval + 2 + start = next_date(target_date, -interval) + positive_df, negative_df = get_top_performance_entities( + entity_type=entity_type, + start_timestamp=start, + kdata_filters=filters, + pct=1, + show_name=True, + entity_provider=entity_provider, + data_provider=data_provider, + return_type=return_type, + ) - if i == 0: - inform( - informer, - entity_ids=df.index[:top_count].tolist(), - target_date=target_date, - title=f"{entity_type} {period}日内 {tag}", - entity_provider=entity_provider, - entity_type=entity_type, - em_group=em_group, - em_group_over_write=em_group_over_write, - ) - else: - inform( - informer, - entity_ids=df.index[:top_count].tolist(), - target_date=target_date, - title=f"{entity_type} {period}日内 {tag}", - entity_provider=entity_provider, - entity_type=entity_type, - em_group=em_group, - em_group_over_write=False, - ) + if return_type == TopType.positive: + tag = "最靓仔" + df = positive_df + else: + tag = "谁有我惨" + df = negative_df + + if i == 0: + inform( + informer, + entity_ids=df.index[:top_count].tolist(), + target_date=target_date, + title=f"{entity_type} {period}日内 {tag}", + entity_provider=entity_provider, + entity_type=entity_type, + em_group=em_group, + em_group_over_write=em_group_over_write, + ) + else: + inform( + informer, + entity_ids=df.index[:top_count].tolist(), + target_date=target_date, + title=f"{entity_type} {period}日内 {tag}", + entity_provider=entity_provider, + entity_type=entity_type, + em_group=em_group, + em_group_over_write=False, + ) + break + except Exception as e: + logger.exception("report error:{}".format(e)) + time.sleep(30) + error_count = error_count + 1 if __name__ == "__main__": diff --git a/src/zvt/informer/informer.py b/src/zvt/informer/informer.py index dea34317..b262508f 100644 --- a/src/zvt/informer/informer.py +++ b/src/zvt/informer/informer.py @@ -27,36 +27,40 @@ def __init__(self, ssl=True) -> None: def send_message_(self, to_user, title, body, **kwargs): host = zvt_config["smtp_host"] port = zvt_config["smtp_port"] - if self.ssl: - try: - smtp_client = smtplib.SMTP_SSL(host=host, port=port) - except: - smtp_client = smtplib.SMTP_SSL() - else: - try: - smtp_client = smtplib.SMTP(host=host, port=port) - except: - smtp_client = smtplib.SMTP() - - smtp_client.connect(host=host, port=port) - smtp_client.login(zvt_config["email_username"], zvt_config["email_password"]) - msg = MIMEMultipart("alternative") - msg["Subject"] = Header(title).encode() - msg["From"] = "{} <{}>".format(Header("zvt").encode(), zvt_config["email_username"]) - if type(to_user) is list: - msg["To"] = ", ".join(to_user) - else: - msg["To"] = to_user - msg["Message-id"] = email.utils.make_msgid() - msg["Date"] = email.utils.formatdate() - - plain_text = MIMEText(body, _subtype="plain", _charset="UTF-8") - msg.attach(plain_text) + smtp_client = None try: + if self.ssl: + try: + smtp_client = smtplib.SMTP_SSL(host=host, port=port) + except: + smtp_client = smtplib.SMTP_SSL() + else: + try: + smtp_client = smtplib.SMTP(host=host, port=port) + except: + smtp_client = smtplib.SMTP() + + smtp_client.connect(host=host, port=port) + smtp_client.login(zvt_config["email_username"], zvt_config["email_password"]) + msg = MIMEMultipart("alternative") + msg["Subject"] = Header(title).encode() + msg["From"] = "{} <{}>".format(Header("zvt").encode(), zvt_config["email_username"]) + if type(to_user) is list: + msg["To"] = ", ".join(to_user) + else: + msg["To"] = to_user + msg["Message-id"] = email.utils.make_msgid() + msg["Date"] = email.utils.formatdate() + + plain_text = MIMEText(body, _subtype="plain", _charset="UTF-8") + msg.attach(plain_text) smtp_client.sendmail(zvt_config["email_username"], to_user, msg.as_string()) except Exception as e: self.logger.exception("send email failed", e) + finally: + if smtp_client: + smtp_client.quit() def send_message(self, to_user, title, body, sub_size=20, with_sender=True, **kwargs): if type(to_user) is list and sub_size: diff --git a/src/zvt/recorders/eastmoney/finance/base_china_stock_finance_recorder.py b/src/zvt/recorders/eastmoney/finance/base_china_stock_finance_recorder.py index 815de6c4..b9559ece 100644 --- a/src/zvt/recorders/eastmoney/finance/base_china_stock_finance_recorder.py +++ b/src/zvt/recorders/eastmoney/finance/base_china_stock_finance_recorder.py @@ -77,7 +77,7 @@ def __init__( except Exception as e: self.fetch_jq_timestamp = False self.logger.warning( - f"joinquant account not ok,the timestamp(publish date) for finance would be not correct", e + f"joinquant account not ok,the timestamp(publish date) for finance would be not correct. {e}" ) def init_timestamps(self, entity):