Skip to content

优化代码结构和增强解析能力 #72

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
xiaoyaosharenfan wants to merge 6 commits into
base: master
Choose a base branch
from
Open

优化代码结构和增强解析能力 #72

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
a4d6bfd
优化cn2an结构--拆分预处理和数据校验，扩大零后省略数词时的处理范围
xiaoyaosharenfan Oct 7, 2023
66e3ed5
支持k、w等单位
xiaoyaosharenfan Oct 7, 2023
6900315
优化不规范的中文数字表达
xiaoyaosharenfan Oct 7, 2023
ae344b3
支持k、w等单位 + 支持大小写
xiaoyaosharenfan Oct 7, 2023
73a793e
支持千分符号，支持亿的特殊说法，优化小数点处理
xiaoyaosharenfan Oct 7, 2023
1b28c04
优化阿拉伯数字+中文单位的识别，支持1.5千万这种多个单位
xiaoyaosharenfan Oct 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
231 changes: 120 additions & 111 deletions cn2an/cn2an.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
from typing import Union

from proces import preprocess
from functools import reduce

from .an2cn import An2Cn
from .conf import NUMBER_CN2AN, UNIT_CN2AN, STRICT_CN_NUMBER, NORMAL_CN_NUMBER, NUMBER_LOW_AN2CN, UNIT_LOW_AN2CN
from .conf import NUMBER_CN2AN, UNIT_CN2AN, UNIT_EN2AN, STRICT_CN_NUMBER, NORMAL_CN_NUMBER, NUMBER_LOW_AN2CN, UNIT_LOW_AN2CN, CN_NUM_AFTER_INTERTNAL_ZERO


class Cn2An(object):
Expand All @@ -23,7 +24,7 @@ def __init__(self) -> None:
self.ac = An2Cn()
self.mode_list = ["strict", "normal", "smart"]
self.yjf_pattern = re.compile(fr"^.*?[元圆][{self.all_num}]角([{self.all_num}]分)?$")
self.pattern1 = re.compile(fr"^-?\d+(\.\d+)?[{self.all_unit}]?$")
self.an_cu_pattern = re.compile(fr"^-?\d+(\.\d+)?([{self.all_unit}]*)$")
self.ptn_all_num = re.compile(f"^[{self.all_num}]+$")
# "十?" is for special case "十一万三"
self.ptn_speaking_mode = re.compile(f"^([{self.all_num}]{{0,2}}[{self.all_unit}])+[{self.all_num}]$")
Expand All @@ -35,77 +36,69 @@ def cn2an(self, inputs: Union[str, int, float] = None, mode: str = "strict") ->
:param mode: strict 严格，normal 正常，smart 智能
:return: 阿拉伯数字
"""
if inputs is not None or inputs == "":
if mode not in self.mode_list:
raise ValueError(f"mode 仅支持 {str(self.mode_list)} ！")

# 将数字转化为字符串
if not isinstance(inputs, str):
inputs = str(inputs)

# 数据预处理：
# 1. 繁体转简体
# 2. 全角转半角
inputs = preprocess(inputs, pipelines=[
"traditional_to_simplified",
"full_angle_to_half_angle"
])

# 特殊转化 廿
inputs = inputs.replace("廿", "二十")

# 检查输入数据是否有效
sign, integer_data, decimal_data, is_all_num = self.__check_input_data_is_valid(inputs, mode)

# smart 下的特殊情况
if sign == 0:
return integer_data
if not inputs:
raise ValueError("输入数据为空！")
if mode not in self.mode_list:
raise ValueError(f"mode 仅支持 {str(self.mode_list)} ！")

# 将数字转化为字符串
if not isinstance(inputs, str):
inputs = str(inputs)

# 预处理数据
sign, integer_data, decimal_data, mode = self.__preprecess_input_data(inputs, mode)
# smart 下的特殊情况
if sign == 0:
return integer_data

# 检查输入数据是否有效
is_all_num = self.__check_input_data_is_valid(inputs, integer_data, decimal_data, mode)

# 数据转换
if not is_all_num:
if decimal_data is None:
output = self.__integer_convert(integer_data)
else:
if not is_all_num:
if decimal_data is None:
output = self.__integer_convert(integer_data)
else:
output = self.__integer_convert(integer_data) + self.__decimal_convert(decimal_data)
# fix 1 + 0.57 = 1.5699999999999998
output = round(output, len(decimal_data))
else:
if decimal_data is None:
output = self.__direct_convert(integer_data)
else:
output = self.__direct_convert(integer_data) + self.__decimal_convert(decimal_data)
# fix 1 + 0.57 = 1.5699999999999998
output = round(output, len(decimal_data))
output = self.__integer_convert(integer_data) + self.__decimal_convert(decimal_data)
# fix 1 + 0.57 = 1.5699999999999998
output = round(output, len(decimal_data))
else:
raise ValueError("输入数据为空！")

if decimal_data is None:
output = self.__direct_convert(integer_data)
else:
output = self.__direct_convert(integer_data) + self.__decimal_convert(decimal_data)
# fix 1 + 0.57 = 1.5699999999999998
output = round(output, len(decimal_data))

return sign * output

def __get_pattern(self) -> dict:

# 整数严格检查
_0 = "[零]"
_1_9 = "[一二三四五六七八九]"
_10_99 = f"{_1_9}?[十]{_1_9}?"
_10_99 = f"{_1_9}?[十][零]?{_1_9}?"
_1_99 = f"({_10_99}|{_1_9})"
_100_999 = f"({_1_9}[百]([零]{_1_9})?|{_1_9}[百]{_10_99})"
_100_999 = f"({_1_9}[百]([零]{_1_9})?|{_1_9}[百][零]?{_10_99})"
_1_999 = f"({_100_999}|{_1_99})"
_1000_9999 = f"({_1_9}[千]([零]{_1_99})?|{_1_9}[千]{_100_999})"
_1000_9999 = f"({_1_9}[千]([零]{_1_9})?|{_1_9}[千]([零]?{_10_99})?|{_1_9}[千][零]?{_100_999})"
_1_9999 = f"({_1000_9999}|{_1_999})"
_10000_99999999 = f"({_1_9999}[万]([零]{_1_999})?|{_1_9999}[万]{_1000_9999})"
_10000_99999999 = f"({_1_9999}[万]([零]{_1_9})?|{_1_9999}[万]([零]?({_10_99}|{_100_999}))?|{_1_9999}[万][零]?{_1000_9999})"
_1_99999999 = f"({_10000_99999999}|{_1_9999})"
_100000000_9999999999999999 = f"({_1_99999999}[亿]([零]{_1_99999999})?|{_1_99999999}[亿]{_10000_99999999})"
_100000000_9999999999999999 = f"({_1_99999999}[亿]([零]{_1_9})?|{_1_99999999}[亿]([零]?({_10_99}|{_100_999}|{_1000_9999}|{_10000_99999999}))?|{_1_99999999}[亿][零]?{_10000_99999999})"
_1_9999999999999999 = f"({_100000000_9999999999999999}|{_1_99999999})"
str_int_pattern = f"^({_0}|{_1_9999999999999999})$"
nor_int_pattern = f"^({_0}|{_1_9999999999999999})$"

str_dec_pattern = "^[零一二三四五六七八九]{0,15}[一二三四五六七八九]$"
nor_dec_pattern = "^[零一二三四五六七八九]{0,16}$"

for str_num in self.strict_cn_number.keys():
str_int_pattern = str_int_pattern.replace(str_num, self.strict_cn_number[str_num])
str_dec_pattern = str_dec_pattern.replace(str_num, self.strict_cn_number[str_num])
for nor_num in self.normal_cn_number.keys():
nor_int_pattern = nor_int_pattern.replace(nor_num, self.normal_cn_number[nor_num])
nor_dec_pattern = nor_dec_pattern.replace(nor_num, self.normal_cn_number[nor_num])
for str_num, str_cn_num in self.strict_cn_number.items():
str_int_pattern = str_int_pattern.replace(str_num, str_cn_num)
str_dec_pattern = str_dec_pattern.replace(str_num, str_cn_num)
for nor_num, nor_cn_num in self.normal_cn_number.items():
nor_int_pattern = nor_int_pattern.replace(nor_num, nor_cn_num)
nor_dec_pattern = nor_dec_pattern.replace(nor_num, nor_cn_num)

pattern_dict = {
"strict": {
Expand All @@ -124,71 +117,103 @@ def __copy_num(self, num):
for n in num:
cn_num += NUMBER_LOW_AN2CN[int(n)]
return cn_num

def __check_input_data_is_valid(self, check_data: str, mode: str) -> (int, str, str, bool):

def __preprecess_input_data(self, check_data: str, mode: str) -> (int, str, str, str):

# 数据预处理：
# 1. 繁体转简体
# 2. 全角转半角
check_data = preprocess(check_data, pipelines=[
"traditional_to_simplified",
"full_angle_to_half_angle"
])

# 去除 元整、圆整、元正、圆正
stop_words = ["元整", "圆整", "元正", "圆正"]
for word in stop_words:
if check_data[-2:] == word:
check_data = check_data[:-2]

# 去除 元、圆
if mode != "strict":
normal_stop_words = ["圆", "元"]
for word in normal_stop_words:
if check_data[-1] == word:
check_data = check_data[:-1]


# 特殊转化 廿
check_data = check_data.replace("廿", "二十").replace("卅", "三十")

# 支持k、w等单位
for en_unit, cn_unit in UNIT_EN2AN.items():
check_data = check_data.replace(en_unit, cn_unit)

# 去除千分符
check_data = check_data.replace(",", "")

# 亿的特殊说法
check_data = check_data.replace("个亿", "亿")

# 零后省略数词时默认数词为一：一千零十一 一万零百一十一
for cn_num, new_cn_num in CN_NUM_AFTER_INTERTNAL_ZERO.items():
check_data = check_data.replace(cn_num, new_cn_num)

# 处理元角分
result = self.yjf_pattern.search(check_data)
if result:
check_data = check_data.replace("元", "点").replace("角", "").replace("分", "")

# 处理特殊问法：一千零十一 一万零百一十一
if "零十" in check_data:
check_data = check_data.replace("零十", "零一十")
if "零百" in check_data:
check_data = check_data.replace("零百", "零一百")


# 处理完成校验
for data in check_data:
if data not in self.check_key_dict[mode]:
raise ValueError(f"当前为{mode}模式，输入的数据不在转化范围内：{data}！")

# 确定正负号
if check_data[0] == "负":
check_data = check_data[1:]
sign = -1
else:
sign = 1

if "点" in check_data:
split_data = check_data.split("点")
if len(split_data) == 2:
integer_data, decimal_data = split_data
# 将 smart 模式中的阿拉伯数字转化成中文数字
if mode == "smart":
integer_data = re.sub(r"\d+", lambda x: self.ac.an2cn(x.group()), integer_data)
decimal_data = re.sub(r"\d+", lambda x: self.__copy_num(x.group()), decimal_data)
mode = "normal"
else:
raise ValueError("数据中包含不止一个点！")
else:

# 将 smart 模式中的阿拉伯数字转化成中文数字
if "点" not in check_data and mode == "smart":
# 阿拉伯数字+中文单位：10.1千万 = 10.1 * 1000 * 10000
matcher = self.an_cu_pattern.search(check_data)
if matcher and matcher.group() == check_data:
unit = reduce(lambda x,y:x*y, [UNIT_CN2AN[unit_data] for unit_data in matcher.group(2)]) if matcher.group(2) else 1
return 0, float(check_data.replace(matcher.group(2), '')) * unit, None, None

# 拆分整数和小数
split_data = check_data.replace(".", "点").split("点")
if len(split_data) == 1:
integer_data = check_data
decimal_data = None
if mode == "smart":
integer_data = re.sub(r"\d+", lambda x: self.ac.an2cn(x.group()), integer_data)
mode = "normal"
elif len(split_data) == 2:
integer_data, decimal_data = split_data
# 将 smart 模式中的阿拉伯数字转化成中文数字
if mode == "smart":
# 10.1万 10.1
result1 = self.pattern1.search(integer_data)
if result1:
if result1.group() == integer_data:
if integer_data[-1] in UNIT_CN2AN.keys():
output = int(float(integer_data[:-1]) * UNIT_CN2AN[integer_data[-1]])
else:
output = float(integer_data)
return 0, output, None, None

integer_data = re.sub(r"\d+", lambda x: self.ac.an2cn(x.group()), integer_data)
decimal_data = re.sub(r"\d+", lambda x: self.__copy_num(x.group()), decimal_data)
mode = "normal"
else:
raise ValueError("数据中包含不止一个点！")

if mode == "normal":
# 口语模式：一万二，两千三，三百四，十三万六，一百二十五万三
result_speaking_mode = self.ptn_speaking_mode.search(integer_data)
if len(integer_data) >= 3 and result_speaking_mode and result_speaking_mode.group() == integer_data:
# len(integer_data)>=3: because the minimum length of integer_data that can be matched is 3
# to find the last unit
last_unit = result_speaking_mode.groups()[-1][-1]
_unit = UNIT_LOW_AN2CN[UNIT_CN2AN[last_unit] // 10]
integer_data = integer_data + _unit

return sign, integer_data, decimal_data, mode

def __check_input_data_is_valid(self, check_data: str, integer_data: str, decimal_data: str, mode: str) -> bool:

result_int = self.pattern_dict[mode]["int"].search(integer_data)
if result_int:
Expand All @@ -197,9 +222,9 @@ def __check_input_data_is_valid(self, check_data: str, mode: str) -> (int, str,
result_dec = self.pattern_dict[mode]["dec"].search(decimal_data)
if result_dec:
if result_dec.group() == decimal_data:
return sign, integer_data, decimal_data, False
return False
else:
return sign, integer_data, decimal_data, False
return False
else:
if mode == "strict":
raise ValueError(f"不符合格式的数据：{integer_data}")
Expand All @@ -212,28 +237,12 @@ def __check_input_data_is_valid(self, check_data: str, mode: str) -> (int, str,
result_dec = self.pattern_dict[mode]["dec"].search(decimal_data)
if result_dec:
if result_dec.group() == decimal_data:
return sign, integer_data, decimal_data, True
return True
else:
return sign, integer_data, decimal_data, True

# 口语模式：一万二，两千三，三百四，十三万六，一百二十五万三
result_speaking_mode = self.ptn_speaking_mode.search(integer_data)
if len(integer_data) >= 3 and result_speaking_mode and result_speaking_mode.group() == integer_data:
# len(integer_data)>=3: because the minimum length of integer_data that can be matched is 3
# to find the last unit
last_unit = result_speaking_mode.groups()[-1][-1]
_unit = UNIT_LOW_AN2CN[UNIT_CN2AN[last_unit] // 10]
integer_data = integer_data + _unit
if decimal_data is not None:
result_dec = self.pattern_dict[mode]["dec"].search(decimal_data)
if result_dec:
if result_dec.group() == decimal_data:
return sign, integer_data, decimal_data, False
else:
return sign, integer_data, decimal_data, False

return True

raise ValueError(f"不符合格式的数据：{check_data}")

def __integer_convert(self, integer_data: str) -> int:
# 核心
output_integer = 0
Expand Down
19 changes: 19 additions & 0 deletions cn2an/conf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,18 @@
"九": 9,
"玖": 9,
}
UNIT_EN2AN = {
"q": "千",
"Q": "千",
"k": "千",
"K": "千",
"w": "万",
"W": "万",
"e": "亿",
"E": "亿",
"y": "亿",
"Y": "亿",
}
UNIT_CN2AN = {
"十": 10,
"拾": 10,
Expand Down Expand Up @@ -133,3 +145,10 @@
"万": "万",
"亿": "亿",
}
CN_NUM_AFTER_INTERTNAL_ZERO = {
"零十": "零一十",
"零百": "零一百",
"零千": "零一千",
"零万": "零一万",
"零亿": "零一亿",
}