diff --git a/auto_email/__init__.py b/auto_email/__init__.py new file mode 100644 index 0000000..2058bfa --- /dev/null +++ b/auto_email/__init__.py @@ -0,0 +1,22 @@ +# 尝试次数 +TRY_TIMES = 3 +# 最小等待时间(秒) +MIN_WAIT_TIME = 1 +# 最大等待时间(秒) +MAX_WAIT_TIME = 3 + +# 程序异常短信配置 +ERROR_EMAIL_CONFIG = { + # SMTP服务器地址 + "smtp_server": "smtp.163.com", + # 连接SMTP的端口 + "port": 994, + # 发件人邮箱地址，请确保开启了SMTP邮件服务！ + "sender": "EchoLiu618@163.com", + # 授权码--用于登录第三方邮件客户端的专用密码，不是邮箱密码 + "authorization_code": "OKPQLIIVLVGRZYVH", + # 收件人邮箱地址 + "receivers": ["1515783401@qq.com"], + # 尝试次数 + "retry_times": 3, +} diff --git a/auto_email/error_email.py b/auto_email/error_email.py new file mode 100644 index 0000000..16dd93b --- /dev/null +++ b/auto_email/error_email.py @@ -0,0 +1,77 @@ +import datetime +import logging +import smtplib +from email.mime.text import MIMEText + +from tenacity import retry, stop_after_attempt, wait_random + +from auto_email import ERROR_EMAIL_CONFIG, TRY_TIMES, MIN_WAIT_TIME, MAX_WAIT_TIME +from log import HOSTNAME + + +@retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME), reraise=True, + after=lambda x: logging.warning("发送邮件失败！")) +def send_email(email_config, massage): + smtp_server = email_config["smtp_server"] + port = email_config["port"] + sender = email_config["sender"] + authorization_code = email_config["authorization_code"] + receivers = email_config["receivers"] + mail = smtplib.SMTP_SSL(smtp_server, port) # 连接SMTP服务 + mail.login(sender, authorization_code) # 登录到SMTP服务 + mail.sendmail(sender, receivers, massage.as_string()) # 发送邮件 + mail.quit() + logging.info(f"成功发送了一封邮件到[{','.join(receivers)}]") + + +def send_error_email(program_name, error_name, error_detail): + """ + 程序出错时发送邮件提醒 + :param program_name: 运行的程序名 + :param error_name: 错误名 + :param error_detail: 错误的详细信息 + :return: + """ + + # SMTP 服务器配置 + sender = ERROR_EMAIL_CONFIG["sender"] + receivers = ERROR_EMAIL_CONFIG["receivers"] + + # 获取程序出错的时间 + error_time = datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d %H:%M:%S:%f") + # 邮件内容 + subject = f"【程序异常提醒】{program_name}({HOSTNAME}) {error_time}" # 邮件的标题 + content = f'''
+
+

程序运行异常通知

+
+

程序：【{program_name}】运行过程中出现异常错误，下面是具体的异常信息，请及时核查处理！

+ + + + + + + + + + + + + + + + +
程序异常详细信息
异常简述{error_name}
异常详情{error_detail}
+
+
+
+ ''' # 邮件的正文部分 + # 实例化一个文本对象 + massage = MIMEText(content, 'html', 'utf-8') + massage['Subject'] = subject # 标题 + massage['From'] = sender # 发件人 + receivers_str = ','.join(receivers) + massage['To'] = receivers_str # 收件人 + + send_email(ERROR_EMAIL_CONFIG, massage) diff --git a/cron.py b/cron.py deleted file mode 100644 index 7355f93..0000000 --- a/cron.py +++ /dev/null @@ -1,23 +0,0 @@ -from datetime import datetime, timedelta -from time import sleep, time - -import fcbsync_dbphoto_mysql - -if __name__ == '__main__': - while 1: - if datetime.now().hour >= 8 and datetime.now().hour < 18: - # 白天间隔10分钟 - sleep_time = 10 * 60 - else: - # 夜间间隔20分钟 - sleep_time = 20 * 60 - - # 执行同步 - start_time = time() - fcbsync_dbphoto_mysql.main() - end_time = time() - elapsed_time = end_time - start_time - if elapsed_time > 30: - print('警告！执行时间超过30秒，请检查程序运行状态！') - print(f'下一次同步时间：{datetime.now() + timedelta(seconds=sleep_time)}') - sleep(sleep_time) diff --git a/docker-compose.yml b/docker-compose.yml index e809ae1..8d5fe2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,4 +6,4 @@ services: context: . container_name: fcb_ai_db_sync hostname: fcb_ai_db_sync - command: ["cron.py"] \ No newline at end of file + command: ["fcb_ai_db_sync.py"] \ No newline at end of file diff --git a/fcb_ai_db_sync.py b/fcb_ai_db_sync.py new file mode 100644 index 0000000..80386d6 --- /dev/null +++ b/fcb_ai_db_sync.py @@ -0,0 +1,38 @@ +import logging.config +import traceback +from datetime import datetime, timedelta +from time import sleep, time + +import fcbsync_dbphoto_mysql +from auto_email.error_email import send_error_email +from log import LOGGING_CONFIG + +if __name__ == '__main__': + program_name = 'AI赋能数据库同步脚本' + logging.config.dictConfig(LOGGING_CONFIG) + + try: + logging.info(f"【{program_name}】开始运行") + while 1: + if 8 <= datetime.now().hour < 18: + # 白天间隔10分钟 + sleep_time = 10 * 60 + else: + # 夜间间隔20分钟 + sleep_time = 20 * 60 + + # 执行同步 + start_time = time() + fcbsync_dbphoto_mysql.main() + end_time = time() + elapsed_time = end_time - start_time + if elapsed_time > 30: + error_message = f'警告！同步脚本执行时间高达{elapsed_time}秒，请检查程序运行状态！' + logging.error(error_message) + send_error_email(program_name, '超时警告', error_message) + logging.info(f'下一次同步时间：{datetime.now() + timedelta(seconds=sleep_time)}') + sleep(sleep_time) + except Exception as e: + error_logger = logging.getLogger("error") + error_logger.error(traceback.format_exc()) + send_error_email(program_name, repr(e), traceback.format_exc()) diff --git a/fcbsync_dbphoto_mysql.py b/fcbsync_dbphoto_mysql.py index 478e983..68bd457 100644 --- a/fcbsync_dbphoto_mysql.py +++ b/fcbsync_dbphoto_mysql.py @@ -2,11 +2,14 @@ 功能：将mysql从源数据表同步目标数据表 主要用于ocr 识别及自动脱敏程序 """ import configparser +import logging #import sys import os import platform import re import time +from datetime import datetime + # from tkinter import EXCEPTION import requests import json @@ -149,14 +152,14 @@ def sync_dbtarget(source_cursor,source_conn,target_cursor,target_db,i_sync_fiel target_db.commit() except Exception as e: target_db.rollback() - print('插入失败imp_'+i_table_name+",原因"+str(e)) + logging.error('插入失败imp_'+i_table_name+",原因"+str(e)) continue sql = "" j = j + 1 # 每一千条打印 if divmod(j, 1000)[1] == 0: - print("已经插入"+i_imp_table_name+":" +str(j) + "条") - print("插入"+i_imp_table_name+":" + str(j - 1) + "条") + logging.info("已经插入"+i_imp_table_name+":" +str(j) + "条") + logging.info("插入"+i_imp_table_name+":" + str(j - 1) + "条") if v_pk_phhd[1:] != '': v_sql =''; v_mess=''; @@ -182,10 +185,10 @@ def sync_dbtarget(source_cursor,source_conn,target_cursor,target_db,i_sync_fiel try: source_cursor.execute(v_sql) source_conn.commit() - print(v_mess+'成功!'+' from:'+i_table_name+' ,to:'+i_imp_table_name) + logging.info(v_mess+'成功!'+' from:'+i_table_name+' ,to:'+i_imp_table_name) except Exception as e: source_conn.rollback() - print(v_mess+"失败,原因"+str(e)) + logging.error(v_mess+"失败,原因"+str(e)) def main(): # path = os.path.abspath(os.path.dirname(__file__)) @@ -297,7 +300,7 @@ def main(): target_conn.commit except Exception as e: target_conn.rollback() - print("删除目标表失败"+"imp_开头表,原因"+str(e)) + logging.error("删除目标表失败"+"imp_开头表,原因"+str(e)) source_cursor = source_conn.cursor() try: @@ -312,7 +315,7 @@ def main(): source_conn.commit except Exception as e: source_conn.rollback() - print("删除源表失败"+"imp_开头表,原因"+str(e)) + logging.error("删除源表失败"+"imp_开头表,原因"+str(e)) #v_sql ="select date_format(date_sub(max(sync_date),interval 1 day) , '%Y-%m-%d') pk_maxvalue from wzx2017.sys_sybnc_date" @@ -350,14 +353,14 @@ def main(): v_title = "花费{mtime:.0f}分{stime:.0f}秒时间,同步成功!".format( mtime=time_elapsed // 60, stime=time_elapsed % 60) - print(v_title) + logging.info(v_title) v_sql = 'insert into sys_sync_date(sync_date,sync_direct,depiction) values(CURRENT_TIMESTAMP,"1","{}") '.format(v_title) try: target_cursor.execute(v_sql) target_conn.commit() except Exception as e: - print('追加记录至sys_sybnc_date失败？' + ",原因" + str(e)) + logging.error('追加记录至sys_sybnc_date失败？' + ",原因" + str(e)) target_conn.rollback() v_sql = 'call pro_sync_waitfor_ocrdata(1,@dd);' @@ -365,7 +368,7 @@ def main(): target_cursor.execute(v_sql) target_conn.commit() except Exception as e: - print('同步至待ocr 识别完成失败(pro_sync_waitfor_ocrdata)？' + ",原因" + str(e)) + logging.error('同步至待ocr 识别完成失败(pro_sync_waitfor_ocrdata)？' + ",原因" + str(e)) target_conn.rollback() v_sql = 'select * from sys_sync_tables where sync_direct="2" and del_flag=0 order by 1 ' @@ -393,14 +396,14 @@ def main(): v_title = "花费{mtime:.0f}分{stime:.0f}秒时间,同步成功!".format( mtime=time_elapsed // 60, stime=time_elapsed % 60) - print(v_title) + logging.info(v_title) v_sql = 'insert into sys_sync_date(sync_date,sync_direct,depiction) values(CURRENT_TIMESTAMP,"2","{}") '.format(v_title) try: target_cursor.execute(v_sql) target_conn.commit() except Exception as e: - print('追加记录至sys_sybnc_date失败？' + ",原因" + str(e)) + logging.error('追加记录至sys_sybnc_date失败？' + ",原因" + str(e)) target_conn.rollback() v_sql = 'call pro_photoocr_update(1,@dd);' @@ -408,7 +411,7 @@ def main(): source_cursor.execute(v_sql) source_conn.commit() except Exception as e: - print('ocr识别结果同步完成失败(pro_photoocr_update)？' + ",原因" + str(e)) + logging.error('ocr识别结果同步完成失败(pro_photoocr_update)？' + ",原因" + str(e)) source_conn.rollback() v_sql = "select a.pk_phhd from imp_zx_phapply a left join zx_phhd b on a.pk_phhd=b.pk_phhd where a.cStatus='2' and b.pk_phhd is null;" @@ -427,7 +430,7 @@ def main(): source_conn.commit() except Exception as e: v_error='(存在待ocr识别但没有成功的案子，自动处理失败' + ",原因" + str(e)+')' - print(v_error) + logging.error(v_error) source_conn.rollback() v_sql = "select count(1) jl from zx_phhd where paint_flag in ('1','2');" @@ -456,7 +459,11 @@ def main(): ) # v_head=v_mess+v_head ## 定义上午11点和中午12点的时间对象 - v_hour=time.strftime("%H", time.localtime(time.time())) - if ('09'<=v_hour<'10' or '17'<=v_hour<'18') or v_paint>300 : + now = datetime.now() + v_hour = now.hour + v_minute = now.minute + if ((v_hour == 8 and 30 <= v_minute < 45) + or (v_hour == 11 and 0 <= v_minute < 15) + or (v_hour == 17 and 0 <= v_minute < 15)) or v_paint>300 : send_data(html_msg, v_head,v_paint) # os._exit(0) \ No newline at end of file diff --git a/log/__init__.py b/log/__init__.py new file mode 100644 index 0000000..d2e9472 --- /dev/null +++ b/log/__init__.py @@ -0,0 +1,70 @@ +import os +import socket + +# 获取主机名，方便区分容器 +HOSTNAME = socket.gethostname() +# 检测日志文件的路径是否存在，不存在则创建 +LOG_PATHS = [ + f"log/{HOSTNAME}/error", +] +for path in LOG_PATHS: + if not os.path.exists(path): + os.makedirs(path) + +# 配置字典 +LOGGING_CONFIG = { + 'version': 1, # 必需，指定配置格式的版本 + 'disable_existing_loggers': False, # 是否禁用已经存在的logger实例 + + # formatters定义了不同格式的日志样式 + 'formatters': { + 'standard': { + 'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S', + }, + }, + + # handlers定义了不同类型的日志处理器 + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', # 控制台处理器 + 'level': 'DEBUG', + 'formatter': 'standard', + 'stream': 'ext://sys.stdout', # 输出到标准输出，默认编码跟随系统，一般为UTF-8 + }, + 'file': { + 'class': 'logging.handlers.TimedRotatingFileHandler', # 文件处理器，支持日志滚动 + 'level': 'INFO', + 'formatter': 'standard', + 'filename': f'log/{HOSTNAME}/fcb_photo_review.log', # 日志文件路径 + 'when': 'midnight', + 'interval': 1, + 'backupCount': 14, # 保留的备份文件数量 + 'encoding': 'utf-8', # 显式指定文件编码为UTF-8以支持中文 + }, + 'error': { + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'level': 'INFO', + 'formatter': 'standard', + 'filename': f'log/{HOSTNAME}/error/fcb_photo_review_error.log', + 'when': 'midnight', + 'interval': 1, + 'backupCount': 14, + 'encoding': 'utf-8', + }, + }, + + # loggers定义了日志记录器 + 'loggers': { + '': { # 根记录器 + 'handlers': ['console', 'file'], # 关联的处理器 + 'level': 'DEBUG', # 根记录器的级别 + 'propagate': False, # 是否向上级传播日志信息 + }, + 'error': { + 'handlers': ['console', 'file', 'error'], + 'level': 'DEBUG', + 'propagate': False, + }, + }, +} diff --git a/requirements.txt b/requirements.txt index 7367a10..b82e529 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ pycryptodome==3.21.0 pymysql==1.1.1 requests==2.32.3 sshtunnel==0.4.0 +tenacity==8.5.0 ufile==3.2.9 xlrd==2.0.1 \ No newline at end of file