diff --git a/auto_email/__init__.py b/auto_email/__init__.py index 7e31a60..2058bfa 100644 --- a/auto_email/__init__.py +++ b/auto_email/__init__.py @@ -1,3 +1,10 @@ +# 尝试次数 +TRY_TIMES = 3 +# 最小等待时间(秒) +MIN_WAIT_TIME = 1 +# 最大等待时间(秒) +MAX_WAIT_TIME = 3 + # 程序异常短信配置 ERROR_EMAIL_CONFIG = { # SMTP服务器地址 @@ -12,4 +19,4 @@ ERROR_EMAIL_CONFIG = { "receivers": ["1515783401@qq.com"], # 尝试次数 "retry_times": 3, -} \ No newline at end of file +} diff --git a/auto_email/error_email.py b/auto_email/error_email.py index 2f682dd..4a3d4e1 100644 --- a/auto_email/error_email.py +++ b/auto_email/error_email.py @@ -3,7 +3,24 @@ import logging import smtplib from email.mime.text import MIMEText -from auto_email import ERROR_EMAIL_CONFIG +from tenacity import retry, stop_after_attempt, wait_random + +from auto_email import ERROR_EMAIL_CONFIG, TRY_TIMES, MIN_WAIT_TIME, MAX_WAIT_TIME + + +@retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME), reraise=True, + retry_error_callback=lambda x: logging.warning("邮件发送失败!")) +def send_email(email_config, massage): + smtp_server = email_config["smtp_server"] + port = email_config["port"] + sender = email_config["sender"] + authorization_code = email_config["authorization_code"] + receivers = email_config["receivers"] + mail = smtplib.SMTP_SSL(smtp_server, port) # 连接SMTP服务 + mail.login(sender, authorization_code) # 登录到SMTP服务 + mail.sendmail(sender, receivers, massage.as_string()) # 发送邮件 + mail.quit() + logging.info(f"成功发送了一封邮件到[{','.join(receivers)}]") def send_error_email(program_name, error_name, error_detail): @@ -16,12 +33,8 @@ def send_error_email(program_name, error_name, error_detail): """ # SMTP 服务器配置 - smtp_server = ERROR_EMAIL_CONFIG["smtp_server"] - port = ERROR_EMAIL_CONFIG["port"] sender = ERROR_EMAIL_CONFIG["sender"] - authorization_code = ERROR_EMAIL_CONFIG["authorization_code"] receivers = ERROR_EMAIL_CONFIG["receivers"] - retry_times = ERROR_EMAIL_CONFIG["retry_times"] # 获取程序出错的时间 error_time = datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d %H:%M:%S:%f") @@ -60,14 +73,4 @@ def send_error_email(program_name, error_name, error_detail): receivers_str = ','.join(receivers) massage['To'] = receivers_str # 收件人 - for i in range(retry_times): - try: - mail = smtplib.SMTP_SSL(smtp_server, port) # 连接SMTP服务 - mail.login(sender, authorization_code) # 登录到SMTP服务 - mail.sendmail(sender, receivers, massage.as_string()) # 发送邮件 - mail.quit() - logging.info("成功发送了一封邮件到" + receivers_str) - break - except smtplib.SMTPException: - if i == retry_times - 1: - logging.warning("邮件发送失败!") + send_email(ERROR_EMAIL_CONFIG, massage) diff --git a/log/__init__.py b/log/__init__.py index f3dcf97..bf3f6a1 100644 --- a/log/__init__.py +++ b/log/__init__.py @@ -6,6 +6,7 @@ HOSTNAME = socket.gethostname() # 检测日志文件的路径是否存在,不存在则创建 LOG_PATHS = [ f"log/{HOSTNAME}/ucloud", + f"log/{HOSTNAME}/error", ] for path in LOG_PATHS: if not os.path.exists(path): @@ -43,14 +44,24 @@ LOGGING_CONFIG = { 'encoding': 'utf-8', # 显式指定文件编码为UTF-8以支持中文 }, 'ucloud': { - 'class': 'logging.handlers.TimedRotatingFileHandler', # 文件处理器,支持日志滚动 + 'class': 'logging.handlers.TimedRotatingFileHandler', 'level': 'INFO', 'formatter': 'standard', - 'filename': f'log/{HOSTNAME}/ucloud/fcb_photo_review_ucloud.log', # 日志文件路径 + 'filename': f'log/{HOSTNAME}/ucloud/fcb_photo_review_ucloud.log', 'when': 'midnight', 'interval': 1, - 'backupCount': 14, # 保留的备份文件数量 - 'encoding': 'utf-8', # 显式指定文件编码为UTF-8以支持中文 + 'backupCount': 14, + 'encoding': 'utf-8', + }, + 'error': { + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'level': 'INFO', + 'formatter': 'standard', + 'filename': f'log/{HOSTNAME}/error/fcb_photo_review_error.log', + 'when': 'midnight', + 'interval': 1, + 'backupCount': 14, + 'encoding': 'utf-8', }, }, @@ -65,6 +76,11 @@ LOGGING_CONFIG = { 'handlers': ['console', 'ucloud'], 'level': 'DEBUG', 'propagate': False, + }, + 'error': { + 'handlers': ['console', 'file', 'error'], + 'level': 'DEBUG', + 'propagate': False, } }, } diff --git a/photo_mask.py b/photo_mask.py index b04ebfe..e42987e 100644 --- a/photo_mask.py +++ b/photo_mask.py @@ -33,6 +33,7 @@ if __name__ == '__main__': logging.info(f"【{program_name}】开始运行") photo_mask.main() except Exception as e: - logging.error(traceback.format_exc()) + error_logger = logging.getLogger("error") + error_logger.error(traceback.format_exc()) if SEND_ERROR_EMAIL: send_error_email(program_name=program_name, error_name=repr(e), error_detail=traceback.format_exc()) diff --git a/photo_mask/__init__.py b/photo_mask/__init__.py index b08371f..423fa7d 100644 --- a/photo_mask/__init__.py +++ b/photo_mask/__init__.py @@ -9,10 +9,6 @@ PHHD_BATCH_SIZE = 20 SLEEP_MINUTES = 5 # 是否发送异常提醒邮件 SEND_ERROR_EMAIL = True -# 备份原图的尝试次数 -COPY_TRY_TIMES = 3 -# 上传新图的尝试次数 -UPLOAD_TRY_TIMES = 3 """ 关键词配置 diff --git a/photo_mask/photo_mask.py b/photo_mask/photo_mask.py index 9ee735f..6a9f984 100644 --- a/photo_mask/photo_mask.py +++ b/photo_mask/photo_mask.py @@ -7,8 +7,7 @@ from sqlalchemy import update, and_ from db import MysqlSession from db.mysql import ZxPhrec, ZxPhhd -from photo_mask import OCR, PHHD_BATCH_SIZE, SLEEP_MINUTES, COPY_TRY_TIMES, UPLOAD_TRY_TIMES, NAME_KEYS, \ - ID_CARD_NUM_KEYS +from photo_mask import OCR, PHHD_BATCH_SIZE, SLEEP_MINUTES, NAME_KEYS, ID_CARD_NUM_KEYS from ucloud import BUCKET, ufile from util import image_util, util @@ -177,18 +176,12 @@ def photo_mask(pk_phhd, name, id_card_num): # 如果涂抹了要备份以及更新 if is_masked: - for i in range(COPY_TRY_TIMES): - is_copy_success = ufile.copy_file(BUCKET, phrec.cfjaddress, "drg2015", phrec.cfjaddress) - if is_copy_success: - break + ufile.copy_file(BUCKET, phrec.cfjaddress, "drg2015", phrec.cfjaddress) with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as temp_file: cv2.imwrite(temp_file.name, image) try: - for i in range(UPLOAD_TRY_TIMES): - is_upload_success = ufile.upload_file(phrec.cfjaddress, temp_file.name) - if is_upload_success: - break + ufile.upload_file(phrec.cfjaddress, temp_file.name) except Exception as e: logging.error("上传图片出错", exc_info=e) finally: diff --git a/photo_review.py b/photo_review.py index d50effb..3ed5340 100644 --- a/photo_review.py +++ b/photo_review.py @@ -35,6 +35,7 @@ if __name__ == '__main__': logging.info(f"【{program_name}】开始运行") photo_review.main() except Exception as e: - logging.error(traceback.format_exc()) + error_logger = logging.getLogger('error') + error_logger.error(traceback.format_exc()) if SEND_ERROR_EMAIL: send_error_email(program_name, repr(e), traceback.format_exc()) diff --git a/ucloud/ufile.py b/ucloud/ufile.py index 95494dc..6607133 100644 --- a/ucloud/ufile.py +++ b/ucloud/ufile.py @@ -1,7 +1,7 @@ # https://github.com/ucloud/ufile-sdk-python import logging -from tenacity import retry, stop_after_attempt, wait_random, retry_if_exception_type +from tenacity import retry, stop_after_attempt, wait_random, retry_if_exception_type, retry_if_result from ufile import filemanager from ucloud import PUBLIC_KEY, PRIVATE_KEY, UPLOAD_SUFFIX, DOWNLOAD_SUFFIX, BUCKET, PRIVATE_EXPIRES, TRY_TIMES, \ @@ -32,13 +32,13 @@ def get_private_url(key, bucket=BUCKET): @retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME), - retry=retry_if_exception_type(ConnectionError), reraise=True) + retry=retry_if_result(lambda x: x is False)) def copy_file(source_bucket, source_key, target_bucket, target_key): # 复制文件 _, resp = UFILE_HANDLER.copy(target_bucket, target_key, source_bucket, source_key) if resp.status_code == -1: UCLOUD_LOGGER.warning(f"复制{source_key}时uCloud连接失败!") - raise ConnectionError("uCloud连接失败") + return False if resp.status_code != 200: UCLOUD_LOGGER.warning( f"将({source_key})从({source_bucket})拷贝到({target_bucket})失败! status: {resp.status_code} error: {resp.error}" @@ -48,13 +48,13 @@ def copy_file(source_bucket, source_key, target_bucket, target_key): @retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME), - retry=retry_if_exception_type(ConnectionError), reraise=True) + retry=retry_if_result(lambda x: x is False)) def upload_file(key, file_path, bucket=BUCKET): # 普通上传文件至云空间 _, resp = UFILE_HANDLER.putfile(bucket, key, file_path, header=None) if resp.status_code == -1: UCLOUD_LOGGER.warning(f"上传{key}时uCloud连接失败!即将重试...") - raise ConnectionError("uCloud连接失败") + return False if resp.status_code != 200: UCLOUD_LOGGER.warning(f"上传({key})失败! status: {resp.status_code} error: {resp.error}") return False diff --git a/util/image_util.py b/util/image_util.py index 02e93c2..c8d848f 100644 --- a/util/image_util.py +++ b/util/image_util.py @@ -1,13 +1,15 @@ import logging import math import urllib.request -from time import sleep import cv2 import numpy from paddleclas import PaddleClas +from tenacity import retry, stop_after_attempt, wait_random +@retry(stop=stop_after_attempt(3), wait=wait_random(1, 3), reraise=True, + retry_error_callback=lambda e: logging.warning("获取图片失败", exc_info=e)) def read(image_path): """ 从网络或本地读取图片 @@ -15,15 +17,8 @@ def read(image_path): :return: NumPy数组形式的图片 """ if image_path.startswith("http"): - resp = None - for i in range(3): - # 发送HTTP请求并获取图像数据 - try: - resp = urllib.request.urlopen(image_path, timeout=60) - break - except Exception as e: - logging.warning("获取图片失败", exc_info=e) - sleep(3) + # 发送HTTP请求并获取图像数据 + resp = urllib.request.urlopen(image_path, timeout=60) # 将数据读取为字节流 image_data = resp.read() # 将字节流转换为NumPy数组