优化日志管理

2025-02-24 14:35:58 +08:00
parent 7e2c0cfd16
commit 3532249a6c
8 changed files with 232 additions and 40 deletions
--- a/auto_email/init.py
+++ b/auto_email/init.py
@@ -0,0 +1,22 @@
+# 尝试次数
+TRY_TIMES = 3
+# 最小等待时间(秒)
+MIN_WAIT_TIME = 1
+# 最大等待时间(秒)
+MAX_WAIT_TIME = 3
+
+# 程序异常短信配置
+ERROR_EMAIL_CONFIG = {
+    # SMTP服务器地址
+    "smtp_server": "smtp.163.com",
+    # 连接SMTP的端口
+    "port": 994,
+    # 发件人邮箱地址，请确保开启了SMTP邮件服务！
+    "sender": "EchoLiu618@163.com",
+    # 授权码--用于登录第三方邮件客户端的专用密码，不是邮箱密码
+    "authorization_code": "OKPQLIIVLVGRZYVH",
+    # 收件人邮箱地址
+    "receivers": ["1515783401@qq.com"],
+    # 尝试次数
+    "retry_times": 3,
+}
--- a/auto_email/error_email.py
+++ b/auto_email/error_email.py
@@ -0,0 +1,77 @@
+import datetime
+import logging
+import smtplib
+from email.mime.text import MIMEText
+
+from tenacity import retry, stop_after_attempt, wait_random
+
+from auto_email import ERROR_EMAIL_CONFIG, TRY_TIMES, MIN_WAIT_TIME, MAX_WAIT_TIME
+from log import HOSTNAME
+
+
+@retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME), reraise=True,
+       after=lambda x: logging.warning("发送邮件失败！"))
+def send_email(email_config, massage):
+    smtp_server = email_config["smtp_server"]
+    port = email_config["port"]
+    sender = email_config["sender"]
+    authorization_code = email_config["authorization_code"]
+    receivers = email_config["receivers"]
+    mail = smtplib.SMTP_SSL(smtp_server, port)  # 连接SMTP服务
+    mail.login(sender, authorization_code)  # 登录到SMTP服务
+    mail.sendmail(sender, receivers, massage.as_string())  # 发送邮件
+    mail.quit()
+    logging.info(f"成功发送了一封邮件到[{','.join(receivers)}]")
+
+
+def send_error_email(program_name, error_name, error_detail):
+    """
+    程序出错时发送邮件提醒
+    :param program_name: 运行的程序名
+    :param error_name: 错误名
+    :param error_detail: 错误的详细信息
+    :return:
+    """
+
+    # SMTP 服务器配置
+    sender = ERROR_EMAIL_CONFIG["sender"]
+    receivers = ERROR_EMAIL_CONFIG["receivers"]
+
+    # 获取程序出错的时间
+    error_time = datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d %H:%M:%S:%f")
+    # 邮件内容
+    subject = f"【程序异常提醒】{program_name}({HOSTNAME}) {error_time}"  # 邮件的标题
+    content = f'''<div class="emailcontent" style="width:100%;max-width:720px;text-align:left;margin:0 auto;padding-top:80px;padding-bottom:20px">
+        <div class="emailtitle">
+            <h1 style="color:#fff;background:#51a0e3;line-height:70px;font-size:24px;font-weight:400;padding-left:40px;margin:0">程序运行异常通知</h1>
+            <div class="emailtext" style="background:#fff;padding:20px 32px 20px">
+                <p style="color:#6e6e6e;font-size:13px;line-height:24px">程序：<span style="color:red;">【{program_name}】</span>运行过程中出现异常错误，下面是具体的异常信息，请及时核查处理！</p>
+                <table cellpadding="0" cellspacing="0" border="0" style="width:100%;border-top:1px solid #eee;border-left:1px solid #eee;color:#6e6e6e;font-size:16px;font-weight:normal">
+                    <thead>
+                        <tr>
+                            <th colspan="2" style="padding:10px 0;border-right:1px solid #eee;border-bottom:1px solid #eee;text-align:center;background:#f8f8f8">程序异常详细信息</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        <tr>
+                            <td style="padding:10px 0;border-right:1px solid #eee;border-bottom:1px solid #eee;text-align:center;width:100px">异常简述</td>
+                            <td style="padding:10px 20px 10px 30px;border-right:1px solid #eee;border-bottom:1px solid #eee;line-height:30px">{error_name}</td>
+                        </tr>
+                        <tr>
+                            <td style="padding:10px 0;border-right:1px solid #eee;border-bottom:1px solid #eee;text-align:center">异常详情</td>
+                            <td style="padding:10px 20px 10px 30px;border-right:1px solid #eee;border-bottom:1px solid #eee;line-height:30px">{error_detail}</td>
+                        </tr>
+                    </tbody>
+                </table>
+            </div>
+        </div>
+    </div>
+        '''  # 邮件的正文部分
+    # 实例化一个文本对象
+    massage = MIMEText(content, 'html', 'utf-8')
+    massage['Subject'] = subject  # 标题
+    massage['From'] = sender  # 发件人
+    receivers_str = ','.join(receivers)
+    massage['To'] = receivers_str  # 收件人
+
+    send_email(ERROR_EMAIL_CONFIG, massage)
--- a/cron.py
+++ b/cron.py
@@ -1,23 +0,0 @@
-from datetime import datetime, timedelta
-from time import sleep, time
-
-import fcbsync_dbphoto_mysql
-
-if __name__ == '__main__':
-    while 1:
-        if datetime.now().hour >= 8 and datetime.now().hour < 18:
-            # 白天间隔10分钟
-            sleep_time = 10 * 60
-        else:
-            # 夜间间隔20分钟
-            sleep_time = 20 * 60
-
-        # 执行同步
-        start_time = time()
-        fcbsync_dbphoto_mysql.main()
-        end_time = time()
-        elapsed_time = end_time - start_time
-        if elapsed_time > 30:
-            print('警告！执行时间超过30秒，请检查程序运行状态！')
-        print(f'下一次同步时间：{datetime.now() + timedelta(seconds=sleep_time)}')
-        sleep(sleep_time)
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -6,4 +6,4 @@ services:
      context: .
    container_name: fcb_ai_db_sync
    hostname: fcb_ai_db_sync
-    command: ["cron.py"]
+    command: ["fcb_ai_db_sync.py"]
--- a/fcb_ai_db_sync.py
+++ b/fcb_ai_db_sync.py
@@ -0,0 +1,38 @@
+import logging.config
+import traceback
+from datetime import datetime, timedelta
+from time import sleep, time
+
+import fcbsync_dbphoto_mysql
+from auto_email.error_email import send_error_email
+from log import LOGGING_CONFIG
+
+if __name__ == '__main__':
+    program_name = 'AI赋能数据库同步脚本'
+    logging.config.dictConfig(LOGGING_CONFIG)
+
+    try:
+        logging.info(f"【{program_name}】开始运行")
+        while 1:
+            if 8 <= datetime.now().hour < 18:
+                # 白天间隔10分钟
+                sleep_time = 10 * 60
+            else:
+                # 夜间间隔20分钟
+                sleep_time = 20 * 60
+
+            # 执行同步
+            start_time = time()
+            fcbsync_dbphoto_mysql.main()
+            end_time = time()
+            elapsed_time = end_time - start_time
+            if elapsed_time > 30:
+                error_message = f'警告！同步脚本执行时间高达{elapsed_time}秒，请检查程序运行状态！'
+                logging.error(error_message)
+                send_error_email(program_name, '超时警告', error_message)
+            logging.info(f'下一次同步时间：{datetime.now() + timedelta(seconds=sleep_time)}')
+            sleep(sleep_time)
+    except Exception as e:
+        error_logger = logging.getLogger("error")
+        error_logger.error(traceback.format_exc())
+        send_error_email(program_name, repr(e), traceback.format_exc())
--- a/fcbsync_dbphoto_mysql.py
+++ b/fcbsync_dbphoto_mysql.py
@@ -2,11 +2,14 @@
 功能：将mysql从源数据表同步目标数据表 主要用于ocr 识别及自动脱敏程序
 """
 import configparser
+import logging
 #import sys
 import os
 import platform
 import re
 import time
+from datetime import datetime
+
 # from tkinter import EXCEPTION
 import requests
 import json
@@ -149,14 +152,14 @@ def  sync_dbtarget(source_cursor,source_conn,target_cursor,target_db,i_sync_fiel
                target_db.commit()
            except Exception as e:
                target_db.rollback()
-                print('插入失败imp_'+i_table_name+",原因"+str(e))
+                logging.error('插入失败imp_'+i_table_name+",原因"+str(e))
                continue
            sql = ""
        j = j + 1
        # 每一千条打印
        if divmod(j, 1000)[1] == 0:
-            print("已经插入"+i_imp_table_name+":" +str(j) + "条")
-    print("插入"+i_imp_table_name+":" + str(j - 1) + "条")
+            logging.info("已经插入"+i_imp_table_name+":" +str(j) + "条")
+    logging.info("插入"+i_imp_table_name+":" + str(j - 1) + "条")
    if v_pk_phhd[1:] != '':
        v_sql ='';
        v_mess='';
@@ -182,10 +185,10 @@ def  sync_dbtarget(source_cursor,source_conn,target_cursor,target_db,i_sync_fiel
            try:
                source_cursor.execute(v_sql)
                source_conn.commit()
-                print(v_mess+'成功!'+' from:'+i_table_name+' ,to:'+i_imp_table_name)
+                logging.info(v_mess+'成功!'+' from:'+i_table_name+' ,to:'+i_imp_table_name)
            except Exception as e:
                source_conn.rollback()
-                print(v_mess+"失败,原因"+str(e))
+                logging.error(v_mess+"失败,原因"+str(e))

 def main():
    # path = os.path.abspath(os.path.dirname(__file__))
@@ -297,7 +300,7 @@ def main():
                    target_conn.commit
    except Exception as e:
        target_conn.rollback()
-        print("删除目标表失败"+"imp_开头表,原因"+str(e))
+        logging.error("删除目标表失败"+"imp_开头表,原因"+str(e))

    source_cursor = source_conn.cursor()
    try:
@@ -312,7 +315,7 @@ def main():
                    source_conn.commit
    except Exception as e:
        source_conn.rollback()
-        print("删除源表失败"+"imp_开头表,原因"+str(e))
+        logging.error("删除源表失败"+"imp_开头表,原因"+str(e))


    #v_sql ="select  date_format(date_sub(max(sync_date),interval 1 day) , '%Y-%m-%d') pk_maxvalue  from wzx2017.sys_sybnc_date"
@@ -350,14 +353,14 @@ def main():
    v_title = "花费{mtime:.0f}分{stime:.0f}秒时间,同步成功!".format(
        mtime=time_elapsed // 60,
        stime=time_elapsed % 60)
-    print(v_title)
+    logging.info(v_title)

    v_sql = 'insert into sys_sync_date(sync_date,sync_direct,depiction) values(CURRENT_TIMESTAMP,"1","{}") '.format(v_title)
    try:
        target_cursor.execute(v_sql)
        target_conn.commit()
    except Exception as e:
-        print('追加记录至sys_sybnc_date失败？' + ",原因" + str(e))
+        logging.error('追加记录至sys_sybnc_date失败？' + ",原因" + str(e))
        target_conn.rollback()

    v_sql = 'call pro_sync_waitfor_ocrdata(1,@dd);'
@@ -365,7 +368,7 @@ def main():
        target_cursor.execute(v_sql)
        target_conn.commit()
    except Exception as e:
-        print('同步至待ocr 识别完成失败(pro_sync_waitfor_ocrdata)？' + ",原因" + str(e))
+        logging.error('同步至待ocr 识别完成失败(pro_sync_waitfor_ocrdata)？' + ",原因" + str(e))
        target_conn.rollback()

    v_sql = 'select * from sys_sync_tables where sync_direct="2" and del_flag=0 order by 1 '
@@ -393,14 +396,14 @@ def main():
    v_title = "花费{mtime:.0f}分{stime:.0f}秒时间,同步成功!".format(
        mtime=time_elapsed // 60,
        stime=time_elapsed % 60)
-    print(v_title)
+    logging.info(v_title)

    v_sql = 'insert into sys_sync_date(sync_date,sync_direct,depiction) values(CURRENT_TIMESTAMP,"2","{}") '.format(v_title)
    try:
        target_cursor.execute(v_sql)
        target_conn.commit()
    except Exception as e:
-        print('追加记录至sys_sybnc_date失败？' + ",原因" + str(e))
+        logging.error('追加记录至sys_sybnc_date失败？' + ",原因" + str(e))
        target_conn.rollback()

    v_sql = 'call pro_photoocr_update(1,@dd);'
@@ -408,7 +411,7 @@ def main():
        source_cursor.execute(v_sql)
        source_conn.commit()
    except Exception as e:
-        print('ocr识别结果同步完成失败(pro_photoocr_update)？' + ",原因" + str(e))
+        logging.error('ocr识别结果同步完成失败(pro_photoocr_update)？' + ",原因" + str(e))
        source_conn.rollback()

    v_sql = "select a.pk_phhd from imp_zx_phapply a left join zx_phhd b on a.pk_phhd=b.pk_phhd where a.cStatus='2' and b.pk_phhd is null;"
@@ -427,7 +430,7 @@ def main():
            source_conn.commit()
        except Exception as e:
            v_error='(存在待ocr识别但没有成功的案子，自动处理失败' + ",原因" + str(e)+')'
-            print(v_error)
+            logging.error(v_error)
            source_conn.rollback()

    v_sql = "select count(1) jl from zx_phhd where paint_flag in ('1','2');"
@@ -456,7 +459,11 @@ def main():
                                                                                 )
    # v_head=v_mess+v_head
    ## 定义上午11点和中午12点的时间对象
-    v_hour=time.strftime("%H", time.localtime(time.time()))
-    if  ('09'<=v_hour<'10' or '17'<=v_hour<'18') or v_paint>300 :
+    now = datetime.now()
+    v_hour = now.hour
+    v_minute = now.minute
+    if  ((v_hour == 8 and 30 <= v_minute < 45)
+         or (v_hour == 11 and 0 <= v_minute < 15)
+         or (v_hour == 17 and 0 <= v_minute < 15)) or v_paint>300 :
        send_data(html_msg, v_head,v_paint)
    # os._exit(0)
--- a/log/init.py
+++ b/log/init.py
@@ -0,0 +1,70 @@
+import os
+import socket
+
+# 获取主机名，方便区分容器
+HOSTNAME = socket.gethostname()
+# 检测日志文件的路径是否存在，不存在则创建
+LOG_PATHS = [
+    f"log/{HOSTNAME}/error",
+]
+for path in LOG_PATHS:
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+# 配置字典
+LOGGING_CONFIG = {
+    'version': 1,  # 必需，指定配置格式的版本
+    'disable_existing_loggers': False,  # 是否禁用已经存在的logger实例
+
+    # formatters定义了不同格式的日志样式
+    'formatters': {
+        'standard': {
+            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+        },
+    },
+
+    # handlers定义了不同类型的日志处理器
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',  # 控制台处理器
+            'level': 'DEBUG',
+            'formatter': 'standard',
+            'stream': 'ext://sys.stdout',  # 输出到标准输出，默认编码跟随系统，一般为UTF-8
+        },
+        'file': {
+            'class': 'logging.handlers.TimedRotatingFileHandler',  # 文件处理器，支持日志滚动
+            'level': 'INFO',
+            'formatter': 'standard',
+            'filename': f'log/{HOSTNAME}/fcb_photo_review.log',  # 日志文件路径
+            'when': 'midnight',
+            'interval': 1,
+            'backupCount': 14,  # 保留的备份文件数量
+            'encoding': 'utf-8',  # 显式指定文件编码为UTF-8以支持中文
+        },
+        'error': {
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'level': 'INFO',
+            'formatter': 'standard',
+            'filename': f'log/{HOSTNAME}/error/fcb_photo_review_error.log',
+            'when': 'midnight',
+            'interval': 1,
+            'backupCount': 14,
+            'encoding': 'utf-8',
+        },
+    },
+
+    # loggers定义了日志记录器
+    'loggers': {
+        '': {  # 根记录器
+            'handlers': ['console', 'file'],  # 关联的处理器
+            'level': 'DEBUG',  # 根记录器的级别
+            'propagate': False,  # 是否向上级传播日志信息
+        },
+        'error': {
+            'handlers': ['console', 'file', 'error'],
+            'level': 'DEBUG',
+            'propagate': False,
+        },
+    },
+}
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,5 +2,6 @@ pycryptodome==3.21.0
 pymysql==1.1.1
 requests==2.32.3
 sshtunnel==0.4.0
+tenacity==8.5.0
 ufile==3.2.9
 xlrd==2.0.1