42 lines
1.4 KiB
Python
42 lines
1.4 KiB
Python
|
|
import re
|
|
|
|
def normalize_text(text: str) -> str:
|
|
if not isinstance(text, str):
|
|
return ""
|
|
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
text = re.sub(r"[ \t]+$", "", text, flags=re.MULTILINE)
|
|
text = re.sub(r"\n{5,}", "\n\n\n\n", text)
|
|
return text.strip()
|
|
|
|
def encode_structural_whitespace(text: str) -> str:
|
|
text = normalize_text(text)
|
|
if not text:
|
|
return ""
|
|
out_lines = []
|
|
for line in text.split("\n"):
|
|
line = line.replace("\t", " <|tab|> ")
|
|
m = re.match(r"^( +)", line)
|
|
if m:
|
|
n = len(m.group(1))
|
|
rest = line[n:]
|
|
tags = []
|
|
while n >= 4:
|
|
tags.append("<|indent_4|>")
|
|
n -= 4
|
|
while n >= 2:
|
|
tags.append("<|indent_2|>")
|
|
n -= 2
|
|
if n == 1:
|
|
rest = " " + rest
|
|
line = (" ".join(tags) + (" " if tags and rest else "") + rest)
|
|
out_lines.append(line)
|
|
return " <|nl|> ".join(out_lines).strip()
|
|
|
|
def decode_structural_whitespace(text: str) -> str:
|
|
text = text.replace(" <|nl|> ", "\n").replace("<|nl|>", "\n")
|
|
text = text.replace(" <|tab|> ", "\t").replace("<|tab|>", "\t")
|
|
text = text.replace("<|indent_4|> ", " ").replace("<|indent_4|>", " ")
|
|
text = text.replace("<|indent_2|> ", " ").replace("<|indent_2|>", " ")
|
|
return text
|