26 lines
971 B
Python
26 lines
971 B
Python
# preprocessing.py
|
||
|
||
import re
|
||
|
||
class Preprocessor:
|
||
def __init__(self, lowercase=False, separate_apostrophes=True, separate_digits=True, separate_punctuation=True):
|
||
self.lowercase = lowercase
|
||
self.separate_apostrophes = separate_apostrophes
|
||
self.separate_punctuation = separate_punctuation
|
||
self.separate_digits = separate_digits
|
||
|
||
def preprocess(self, line: str) -> str:
|
||
if self.lowercase:
|
||
line = line.lower()
|
||
if self.separate_apostrophes:
|
||
# Add spaces around apostrophes
|
||
line = re.sub(r"([’'`])", r" \1 ", line)
|
||
# Add spaces around punctuation (except alphanumeric and apostrophes)
|
||
if self.separate_punctuation:
|
||
line = re.sub(r"([^A-Za-z0-9\s’'`])", r" \1 ", line)
|
||
if self.separate_digits:
|
||
line = re.sub(r"(\d)", r" \1 ", line)
|
||
|
||
# Normalize whitespace
|
||
line = re.sub(r"\s+", " ", line)
|
||
return line.strip() |