Echo/fcb_photo_review
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

优化错误日志及重试机制

Browse Source
This commit is contained in:
liuyebo
2024-07-23 09:27:59 +08:00
parent ee86bb4e74
commit bcd9f94daf
9 changed files with 64 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
auto_email/__init__.py
View File

@@ -1,3 +1,10 @@
# 尝试次数
TRY_TIMES = 3
# 最小等待时间(秒)
MIN_WAIT_TIME = 1
# 最大等待时间(秒)
MAX_WAIT_TIME = 3
# 程序异常短信配置
ERROR_EMAIL_CONFIG = {
# SMTP服务器地址

35
auto_email/error_email.py
View File

@@ -3,7 +3,24 @@ import logging
import smtplib
from email.mime.text import MIMEText
from auto_email import ERROR_EMAIL_CONFIG
from tenacity import retry, stop_after_attempt, wait_random
from auto_email import ERROR_EMAIL_CONFIG, TRY_TIMES, MIN_WAIT_TIME, MAX_WAIT_TIME
@retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME), reraise=True,
retry_error_callback=lambda x: logging.warning("邮件发送失败！"))
def send_email(email_config, massage):
smtp_server = email_config["smtp_server"]
port = email_config["port"]
sender = email_config["sender"]
authorization_code = email_config["authorization_code"]
receivers = email_config["receivers"]
mail = smtplib.SMTP_SSL(smtp_server, port) # 连接SMTP服务
mail.login(sender, authorization_code) # 登录到SMTP服务
mail.sendmail(sender, receivers, massage.as_string()) # 发送邮件
mail.quit()
logging.info(f"成功发送了一封邮件到[{','.join(receivers)}]")
def send_error_email(program_name, error_name, error_detail):
@@ -16,12 +33,8 @@ def send_error_email(program_name, error_name, error_detail):
"""
# SMTP 服务器配置
smtp_server = ERROR_EMAIL_CONFIG["smtp_server"]
port = ERROR_EMAIL_CONFIG["port"]
sender = ERROR_EMAIL_CONFIG["sender"]
authorization_code = ERROR_EMAIL_CONFIG["authorization_code"]
receivers = ERROR_EMAIL_CONFIG["receivers"]
retry_times = ERROR_EMAIL_CONFIG["retry_times"]
# 获取程序出错的时间
error_time = datetime.datetime.strftime(datetime.datetime.today(), "%Y-%m-%d %H:%M:%S:%f")
@@ -60,14 +73,4 @@ def send_error_email(program_name, error_name, error_detail):
receivers_str = ','.join(receivers)
massage['To'] = receivers_str # 收件人
for i in range(retry_times):
try:
mail = smtplib.SMTP_SSL(smtp_server, port) # 连接SMTP服务
mail.login(sender, authorization_code) # 登录到SMTP服务
mail.sendmail(sender, receivers, massage.as_string()) # 发送邮件
mail.quit()
logging.info("成功发送了一封邮件到" + receivers_str)
break
except smtplib.SMTPException:
if i == retry_times - 1:
logging.warning("邮件发送失败！")
send_email(ERROR_EMAIL_CONFIG, massage)

24
log/__init__.py
View File

@@ -6,6 +6,7 @@ HOSTNAME = socket.gethostname()
# 检测日志文件的路径是否存在，不存在则创建
LOG_PATHS = [
f"log/{HOSTNAME}/ucloud",
f"log/{HOSTNAME}/error",
]
for path in LOG_PATHS:
if not os.path.exists(path):
@@ -43,14 +44,24 @@ LOGGING_CONFIG = {
'encoding': 'utf-8', # 显式指定文件编码为UTF-8以支持中文
},
'ucloud': {
'class': 'logging.handlers.TimedRotatingFileHandler', # 文件处理器，支持日志滚动
'class': 'logging.handlers.TimedRotatingFileHandler',
'level': 'INFO',
'formatter': 'standard',
'filename': f'log/{HOSTNAME}/ucloud/fcb_photo_review_ucloud.log', # 日志文件路径
'filename': f'log/{HOSTNAME}/ucloud/fcb_photo_review_ucloud.log',
'when': 'midnight',
'interval': 1,
'backupCount': 14, # 保留的备份文件数量
'encoding': 'utf-8', # 显式指定文件编码为UTF-8以支持中文
'backupCount': 14,
'encoding': 'utf-8',
},
'error': {
'class': 'logging.handlers.TimedRotatingFileHandler',
'level': 'INFO',
'formatter': 'standard',
'filename': f'log/{HOSTNAME}/error/fcb_photo_review_error.log',
'when': 'midnight',
'interval': 1,
'backupCount': 14,
'encoding': 'utf-8',
},
},
@@ -65,6 +76,11 @@ LOGGING_CONFIG = {
'handlers': ['console', 'ucloud'],
'level': 'DEBUG',
'propagate': False,
},
'error': {
'handlers': ['console', 'file', 'error'],
'level': 'DEBUG',
'propagate': False,
}
},
}

3
photo_mask.py
View File

@@ -33,6 +33,7 @@ if __name__ == '__main__':
logging.info(f"【{program_name}】开始运行")
photo_mask.main()
except Exception as e:
logging.error(traceback.format_exc())
error_logger = logging.getLogger("error")
error_logger.error(traceback.format_exc())
if SEND_ERROR_EMAIL:
send_error_email(program_name=program_name, error_name=repr(e), error_detail=traceback.format_exc())

4
photo_mask/__init__.py
View File

@@ -9,10 +9,6 @@ PHHD_BATCH_SIZE = 20
SLEEP_MINUTES = 5
# 是否发送异常提醒邮件
SEND_ERROR_EMAIL = True
# 备份原图的尝试次数
COPY_TRY_TIMES = 3
# 上传新图的尝试次数
UPLOAD_TRY_TIMES = 3
"""
关键词配置

13
photo_mask/photo_mask.py
View File

@@ -7,8 +7,7 @@ from sqlalchemy import update, and_
from db import MysqlSession
from db.mysql import ZxPhrec, ZxPhhd
from photo_mask import OCR, PHHD_BATCH_SIZE, SLEEP_MINUTES, COPY_TRY_TIMES, UPLOAD_TRY_TIMES, NAME_KEYS, \
ID_CARD_NUM_KEYS
from photo_mask import OCR, PHHD_BATCH_SIZE, SLEEP_MINUTES, NAME_KEYS, ID_CARD_NUM_KEYS
from ucloud import BUCKET, ufile
from util import image_util, util
@@ -177,18 +176,12 @@ def photo_mask(pk_phhd, name, id_card_num):
# 如果涂抹了要备份以及更新
if is_masked:
for i in range(COPY_TRY_TIMES):
is_copy_success = ufile.copy_file(BUCKET, phrec.cfjaddress, "drg2015", phrec.cfjaddress)
if is_copy_success:
break
ufile.copy_file(BUCKET, phrec.cfjaddress, "drg2015", phrec.cfjaddress)
with tempfile.NamedTemporaryFile(delete=False, suffix=".jpg") as temp_file:
cv2.imwrite(temp_file.name, image)
try:
for i in range(UPLOAD_TRY_TIMES):
is_upload_success = ufile.upload_file(phrec.cfjaddress, temp_file.name)
if is_upload_success:
break
ufile.upload_file(phrec.cfjaddress, temp_file.name)
except Exception as e:
logging.error("上传图片出错", exc_info=e)
finally:

3
photo_review.py
View File

@@ -35,6 +35,7 @@ if __name__ == '__main__':
logging.info(f"【{program_name}】开始运行")
photo_review.main()
except Exception as e:
logging.error(traceback.format_exc())
error_logger = logging.getLogger('error')
error_logger.error(traceback.format_exc())
if SEND_ERROR_EMAIL:
send_error_email(program_name, repr(e), traceback.format_exc())

10
ucloud/ufile.py
View File

@@ -1,7 +1,7 @@
# https://github.com/ucloud/ufile-sdk-python
import logging
from tenacity import retry, stop_after_attempt, wait_random, retry_if_exception_type
from tenacity import retry, stop_after_attempt, wait_random, retry_if_exception_type, retry_if_result
from ufile import filemanager
from ucloud import PUBLIC_KEY, PRIVATE_KEY, UPLOAD_SUFFIX, DOWNLOAD_SUFFIX, BUCKET, PRIVATE_EXPIRES, TRY_TIMES, \
@@ -32,13 +32,13 @@ def get_private_url(key, bucket=BUCKET):
@retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME),
retry=retry_if_exception_type(ConnectionError), reraise=True)
retry=retry_if_result(lambda x: x is False))
def copy_file(source_bucket, source_key, target_bucket, target_key):
# 复制文件
_, resp = UFILE_HANDLER.copy(target_bucket, target_key, source_bucket, source_key)
if resp.status_code == -1:
UCLOUD_LOGGER.warning(f"复制{source_key}时uCloud连接失败!")
raise ConnectionError("uCloud连接失败")
return False
if resp.status_code != 200:
UCLOUD_LOGGER.warning(
f"将({source_key})从({source_bucket})拷贝到({target_bucket})失败! status: {resp.status_code} error: {resp.error}"
@@ -48,13 +48,13 @@ def copy_file(source_bucket, source_key, target_bucket, target_key):
@retry(stop=stop_after_attempt(TRY_TIMES), wait=wait_random(MIN_WAIT_TIME, MAX_WAIT_TIME),
retry=retry_if_exception_type(ConnectionError), reraise=True)
retry=retry_if_result(lambda x: x is False))
def upload_file(key, file_path, bucket=BUCKET):
# 普通上传文件至云空间
_, resp = UFILE_HANDLER.putfile(bucket, key, file_path, header=None)
if resp.status_code == -1:
UCLOUD_LOGGER.warning(f"上传{key}时uCloud连接失败!即将重试...")
raise ConnectionError("uCloud连接失败")
return False
if resp.status_code != 200:
UCLOUD_LOGGER.warning(f"上传({key})失败! status: {resp.status_code} error: {resp.error}")
return False

15
util/image_util.py
View File

@@ -1,13 +1,15 @@
import logging
import math
import urllib.request
from time import sleep
import cv2
import numpy
from paddleclas import PaddleClas
from tenacity import retry, stop_after_attempt, wait_random
@retry(stop=stop_after_attempt(3), wait=wait_random(1, 3), reraise=True,
retry_error_callback=lambda e: logging.warning("获取图片失败", exc_info=e))
def read(image_path):
"""
从网络或本地读取图片
@@ -15,15 +17,8 @@ def read(image_path):
:return: NumPy数组形式的图片
"""
if image_path.startswith("http"):
resp = None
for i in range(3):
# 发送HTTP请求并获取图像数据
try:
resp = urllib.request.urlopen(image_path, timeout=60)
break
except Exception as e:
logging.warning("获取图片失败", exc_info=e)
sleep(3)
# 发送HTTP请求并获取图像数据
resp = urllib.request.urlopen(image_path, timeout=60)
# 将数据读取为字节流
image_data = resp.read()
# 将字节流转换为NumPy数组