优化日志管理

auto_email/__init__.py

@@ -0,0 +1,22 @@
+# 尝试次数
+TRY_TIMES = 3
+# 最小等待时间(秒)
+MIN_WAIT_TIME = 1
+# 最大等待时间(秒)
+MAX_WAIT_TIME = 3
+
+# 程序异常短信配置
+ERROR_EMAIL_CONFIG = {
+    # SMTP服务器地址
+    "smtp_server": "smtp.163.com",
+    # 连接SMTP的端口
+    "port": 994,
+    # 发件人邮箱地址，请确保开启了SMTP邮件服务！
+    "sender": "EchoLiu618@163.com",
+    # 授权码--用于登录第三方邮件客户端的专用密码，不是邮箱密码
+    "authorization_code": "OKPQLIIVLVGRZYVH",
+    # 收件人邮箱地址
+    "receivers": ["1515783401@qq.com"],
+    # 尝试次数
+    "retry_times": 3,
+}

auto_email/error_email.py

@@ -0,0 +1,77 @@
+import datetime
+import logging
+import smtplib
+from email.mime.text import MIMEText
+
+from tenacity import retry, stop_after_attempt, wait_random
+
+from auto_email import ERROR_EMAIL_CONFIG, TRY_TIMES, MIN_WAIT_TIME, MAX_WAIT_TIME
+from log import HOSTNAME
+
+
+@retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME), reraise=True,
+       after=lambda x: logging.warning("发送邮件失败！"))
+def send_email(email_config, massage):
+    smtp_server = email_config["smtp_server"]
+    port = email_config["port"]
+    sender = email_config["sender"]
+    authorization_code = email_config["authorization_code"]
+    receivers = email_config["receivers"]
+    mail = smtplib.SMTP_SSL(smtp_server, port)  # 连接SMTP服务
+    mail.login(sender, authorization_code)  # 登录到SMTP服务
+    mail.sendmail(sender, receivers, massage.as_string())  # 发送邮件
+    mail.quit()
+    logging.info(f"成功发送了一封邮件到[{','.join(receivers)}]")
+
+
+def send_error_email(program_name, error_name, error_detail):
+    """
+    程序出错时发送邮件提醒
+    :param program_name: 运行的程序名
+    :param error_name: 错误名
+    :param error_detail: 错误的详细信息
+    :return:
+    """
+
+    # SMTP 服务器配置
+    sender = ERROR_EMAIL_CONFIG["sender"]
+    receivers = ERROR_EMAIL_CONFIG["receivers"]
+
+    # 获取程序出错的时间
+    error_time = datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d %H:%M:%S:%f")
+    # 邮件内容
+    subject = f"【程序异常提醒】{program_name}({HOSTNAME}) {error_time}"  # 邮件的标题
+    content = f'''<div class="emailcontent" style="width:100%;max-width:720px;text-align:left;margin:0 auto;padding-top:80px;padding-bottom:20px">
+        <div class="emailtitle">
+            <h1 style="color:#fff;background:#51a0e3;line-height:70px;font-size:24px;font-weight:400;padding-left:40px;margin:0">程序运行异常通知</h1>
+            <div class="emailtext" style="background:#fff;padding:20px 32px 20px">
+                <p style="color:#6e6e6e;font-size:13px;line-height:24px">程序：<span style="color:red;">【{program_name}】</span>运行过程中出现异常错误，下面是具体的异常信息，请及时核查处理！</p>
+                <table cellpadding="0" cellspacing="0" border="0" style="width:100%;border-top:1px solid #eee;border-left:1px solid #eee;color:#6e6e6e;font-size:16px;font-weight:normal">
+                    <thead>
+                        <tr>
+                            <th colspan="2" style="padding:10px 0;border-right:1px solid #eee;border-bottom:1px solid #eee;text-align:center;background:#f8f8f8">程序异常详细信息</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        <tr>
+                            <td style="padding:10px 0;border-right:1px solid #eee;border-bottom:1px solid #eee;text-align:center;width:100px">异常简述</td>
+                            <td style="padding:10px 20px 10px 30px;border-right:1px solid #eee;border-bottom:1px solid #eee;line-height:30px">{error_name}</td>
+                        </tr>
+                        <tr>
+                            <td style="padding:10px 0;border-right:1px solid #eee;border-bottom:1px solid #eee;text-align:center">异常详情</td>
+                            <td style="padding:10px 20px 10px 30px;border-right:1px solid #eee;border-bottom:1px solid #eee;line-height:30px">{error_detail}</td>
+                        </tr>
+                    </tbody>
+                </table>
+            </div>
+        </div>
+    </div>
+        '''  # 邮件的正文部分
+    # 实例化一个文本对象
+    massage = MIMEText(content, 'html', 'utf-8')
+    massage['Subject'] = subject  # 标题
+    massage['From'] = sender  # 发件人
+    receivers_str = ','.join(receivers)
+    massage['To'] = receivers_str  # 收件人
+
+    send_email(ERROR_EMAIL_CONFIG, massage)

cron.py

@@ -1,23 +0,0 @@
-from datetime import datetime, timedelta
-from time import sleep, time
-
-import fcbsync_dbphoto_mysql
-
-if __name__ == '__main__':
-    while 1:
-        if datetime.now().hour >= 8 and datetime.now().hour < 18:
-            # 白天间隔10分钟
-            sleep_time = 10 * 60
-        else:
-            # 夜间间隔20分钟
-            sleep_time = 20 * 60
-
-        # 执行同步
-        start_time = time()
-        fcbsync_dbphoto_mysql.main()
-        end_time = time()
-        elapsed_time = end_time - start_time
-        if elapsed_time > 30:
-            print('警告！执行时间超过30秒，请检查程序运行状态！')
-        print(f'下一次同步时间：{datetime.now() + timedelta(seconds=sleep_time)}')
-        sleep(sleep_time)

docker-compose.yml

@@ -6,4 +6,4 @@ services:
       context: .
     container_name: fcb_ai_db_sync
     hostname: fcb_ai_db_sync
-    command: ["cron.py"]
+    command: ["fcb_ai_db_sync.py"]

fcb_ai_db_sync.py

@@ -0,0 +1,38 @@
+import logging.config
+import traceback
+from datetime import datetime, timedelta
+from time import sleep, time
+
+import fcbsync_dbphoto_mysql
+from auto_email.error_email import send_error_email
+from log import LOGGING_CONFIG
+
+if __name__ == '__main__':
+    program_name = 'AI赋能数据库同步脚本'
+    logging.config.dictConfig(LOGGING_CONFIG)
+
+    try:
+        logging.info(f"【{program_name}】开始运行")
+        while 1:
+            if 8 <= datetime.now().hour < 18:
+                # 白天间隔10分钟
+                sleep_time = 10 * 60
+            else:
+                # 夜间间隔20分钟
+                sleep_time = 20 * 60
+
+            # 执行同步
+            start_time = time()
+            fcbsync_dbphoto_mysql.main()
+            end_time = time()
+            elapsed_time = end_time - start_time
+            if elapsed_time > 30:
+                error_message = f'警告！同步脚本执行时间高达{elapsed_time}秒，请检查程序运行状态！'
+                logging.error(error_message)
+                send_error_email(program_name, '超时警告', error_message)
+            logging.info(f'下一次同步时间：{datetime.now() + timedelta(seconds=sleep_time)}')
+            sleep(sleep_time)
+    except Exception as e:
+        error_logger = logging.getLogger("error")
+        error_logger.error(traceback.format_exc())
+        send_error_email(program_name, repr(e), traceback.format_exc())

fcbsync_dbphoto_mysql.py

@@ -2,11 +2,14 @@
 功能：将mysql从源数据表同步目标数据表 主要用于ocr 识别及自动脱敏程序
 """
 import configparser
+import logging
 #import sys
 import os
 import platform
 import re
 import time
+from datetime import datetime
+
 # from tkinter import EXCEPTION
 import requests
 import json
@@ -149,14 +152,14 @@ def  sync_dbtarget(source_cursor,source_conn,target_cursor,target_db,i_sync_fiel
                 target_db.commit()
             except Exception as e:
                 target_db.rollback()
-                print('插入失败imp_'+i_table_name+",原因"+str(e))
+                logging.error('插入失败imp_'+i_table_name+",原因"+str(e))
                 continue
             sql = ""
         j = j + 1
         # 每一千条打印
         if divmod(j, 1000)[1] == 0:
-            print("已经插入"+i_imp_table_name+":" +str(j) + "条")
+            logging.info("已经插入"+i_imp_table_name+":" +str(j) + "条")
-    print("插入"+i_imp_table_name+":" + str(j - 1) + "条")
+    logging.info("插入"+i_imp_table_name+":" + str(j - 1) + "条")
     if v_pk_phhd[1:] != '':
         v_sql ='';
         v_mess='';
@@ -182,10 +185,10 @@ def  sync_dbtarget(source_cursor,source_conn,target_cursor,target_db,i_sync_fiel
             try:
                 source_cursor.execute(v_sql)
                 source_conn.commit()
-                print(v_mess+'成功!'+' from:'+i_table_name+' ,to:'+i_imp_table_name)
+                logging.info(v_mess+'成功!'+' from:'+i_table_name+' ,to:'+i_imp_table_name)
             except Exception as e:
                 source_conn.rollback()
-                print(v_mess+"失败,原因"+str(e))
+                logging.error(v_mess+"失败,原因"+str(e))
 
 def main():
     # path = os.path.abspath(os.path.dirname(__file__))
@@ -297,7 +300,7 @@ def main():
                     target_conn.commit
     except Exception as e:
         target_conn.rollback()
-        print("删除目标表失败"+"imp_开头表,原因"+str(e))
+        logging.error("删除目标表失败"+"imp_开头表,原因"+str(e))
 
     source_cursor = source_conn.cursor()
     try:
@@ -312,7 +315,7 @@ def main():
                     source_conn.commit
     except Exception as e:
         source_conn.rollback()
-        print("删除源表失败"+"imp_开头表,原因"+str(e))
+        logging.error("删除源表失败"+"imp_开头表,原因"+str(e))
 
 
     #v_sql ="select  date_format(date_sub(max(sync_date),interval 1 day) , '%Y-%m-%d') pk_maxvalue  from wzx2017.sys_sybnc_date"
@@ -350,14 +353,14 @@ def main():
     v_title = "花费{mtime:.0f}分{stime:.0f}秒时间,同步成功!".format(
         mtime=time_elapsed // 60,
         stime=time_elapsed % 60)
-    print(v_title)
+    logging.info(v_title)
 
     v_sql = 'insert into sys_sync_date(sync_date,sync_direct,depiction) values(CURRENT_TIMESTAMP,"1","{}") '.format(v_title)
     try:
         target_cursor.execute(v_sql)
         target_conn.commit()
     except Exception as e:
-        print('追加记录至sys_sybnc_date失败？' + ",原因" + str(e))
+        logging.error('追加记录至sys_sybnc_date失败？' + ",原因" + str(e))
         target_conn.rollback()
 
     v_sql = 'call pro_sync_waitfor_ocrdata(1,@dd);'
@@ -365,7 +368,7 @@ def main():
         target_cursor.execute(v_sql)
         target_conn.commit()
     except Exception as e:
-        print('同步至待ocr 识别完成失败(pro_sync_waitfor_ocrdata)？' + ",原因" + str(e))
+        logging.error('同步至待ocr 识别完成失败(pro_sync_waitfor_ocrdata)？' + ",原因" + str(e))
         target_conn.rollback()
 
     v_sql = 'select * from sys_sync_tables where sync_direct="2" and del_flag=0 order by 1 '
@@ -393,14 +396,14 @@ def main():
     v_title = "花费{mtime:.0f}分{stime:.0f}秒时间,同步成功!".format(
         mtime=time_elapsed // 60,
         stime=time_elapsed % 60)
-    print(v_title)
+    logging.info(v_title)
 
     v_sql = 'insert into sys_sync_date(sync_date,sync_direct,depiction) values(CURRENT_TIMESTAMP,"2","{}") '.format(v_title)
     try:
         target_cursor.execute(v_sql)
         target_conn.commit()
     except Exception as e:
-        print('追加记录至sys_sybnc_date失败？' + ",原因" + str(e))
+        logging.error('追加记录至sys_sybnc_date失败？' + ",原因" + str(e))
         target_conn.rollback()
 
     v_sql = 'call pro_photoocr_update(1,@dd);'
@@ -408,7 +411,7 @@ def main():
         source_cursor.execute(v_sql)
         source_conn.commit()
     except Exception as e:
-        print('ocr识别结果同步完成失败(pro_photoocr_update)？' + ",原因" + str(e))
+        logging.error('ocr识别结果同步完成失败(pro_photoocr_update)？' + ",原因" + str(e))
         source_conn.rollback()
 
     v_sql = "select a.pk_phhd from imp_zx_phapply a left join zx_phhd b on a.pk_phhd=b.pk_phhd where a.cStatus='2' and b.pk_phhd is null;"
@@ -427,7 +430,7 @@ def main():
             source_conn.commit()
         except Exception as e:
             v_error='(存在待ocr识别但没有成功的案子，自动处理失败' + ",原因" + str(e)+')'
-            print(v_error)
+            logging.error(v_error)
             source_conn.rollback()
 
     v_sql = "select count(1) jl from zx_phhd where paint_flag in ('1','2');"
@@ -456,7 +459,11 @@ def main():
                                                                                  )
     # v_head=v_mess+v_head
     ## 定义上午11点和中午12点的时间对象
-    v_hour=time.strftime("%H", time.localtime(time.time()))
+    now = datetime.now()
-    if  ('09'<=v_hour<'10' or '17'<=v_hour<'18') or v_paint>300 :
+    v_hour = now.hour
+    v_minute = now.minute
+    if  ((v_hour == 8 and 30 <= v_minute < 45)
+         or (v_hour == 11 and 0 <= v_minute < 15)
+         or (v_hour == 17 and 0 <= v_minute < 15)) or v_paint>300 :
         send_data(html_msg, v_head,v_paint)
     # os._exit(0)

log/__init__.py

@@ -0,0 +1,70 @@
+import os
+import socket
+
+# 获取主机名，方便区分容器
+HOSTNAME = socket.gethostname()
+# 检测日志文件的路径是否存在，不存在则创建
+LOG_PATHS = [
+    f"log/{HOSTNAME}/error",
+]
+for path in LOG_PATHS:
+    if not os.path.exists(path):
+        os.makedirs(path)
+
+# 配置字典
+LOGGING_CONFIG = {
+    'version': 1,  # 必需，指定配置格式的版本
+    'disable_existing_loggers': False,  # 是否禁用已经存在的logger实例
+
+    # formatters定义了不同格式的日志样式
+    'formatters': {
+        'standard': {
+            'format': '%(asctime)s [%(levelname)s] %(name)s: %(message)s',
+            'datefmt': '%Y-%m-%d %H:%M:%S',
+        },
+    },
+
+    # handlers定义了不同类型的日志处理器
+    'handlers': {
+        'console': {
+            'class': 'logging.StreamHandler',  # 控制台处理器
+            'level': 'DEBUG',
+            'formatter': 'standard',
+            'stream': 'ext://sys.stdout',  # 输出到标准输出，默认编码跟随系统，一般为UTF-8
+        },
+        'file': {
+            'class': 'logging.handlers.TimedRotatingFileHandler',  # 文件处理器，支持日志滚动
+            'level': 'INFO',
+            'formatter': 'standard',
+            'filename': f'log/{HOSTNAME}/fcb_photo_review.log',  # 日志文件路径
+            'when': 'midnight',
+            'interval': 1,
+            'backupCount': 14,  # 保留的备份文件数量
+            'encoding': 'utf-8',  # 显式指定文件编码为UTF-8以支持中文
+        },
+        'error': {
+            'class': 'logging.handlers.TimedRotatingFileHandler',
+            'level': 'INFO',
+            'formatter': 'standard',
+            'filename': f'log/{HOSTNAME}/error/fcb_photo_review_error.log',
+            'when': 'midnight',
+            'interval': 1,
+            'backupCount': 14,
+            'encoding': 'utf-8',
+        },
+    },
+
+    # loggers定义了日志记录器
+    'loggers': {
+        '': {  # 根记录器
+            'handlers': ['console', 'file'],  # 关联的处理器
+            'level': 'DEBUG',  # 根记录器的级别
+            'propagate': False,  # 是否向上级传播日志信息
+        },
+        'error': {
+            'handlers': ['console', 'file', 'error'],
+            'level': 'DEBUG',
+            'propagate': False,
+        },
+    },
+}

requirements.txt

@@ -2,5 +2,6 @@ pycryptodome==3.21.0
 pymysql==1.1.1
 requests==2.32.3
 sshtunnel==0.4.0
+tenacity==8.5.0
 ufile==3.2.9
 xlrd==2.0.1