Skip to content

Commit

Permalink
Merge pull request #100 from hplt-project/sent_final_filter
Browse files Browse the repository at this point in the history
Fix mismatching of sentence final punctuation
  • Loading branch information
XapaJIaMnu authored Nov 16, 2023
2 parents 3ee797f + 61f4671 commit 24014c4
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 0 deletions.
6 changes: 6 additions & 0 deletions opuscleaner/filters/fix_sent_final_punct.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
{
"type": "bilingual",
"description": "Fixes mismatched punctuation at the end of the sentences. Works for latin/cyrillic based languages, WILL BREAK CJK AND ANY LANGUAGE THAT USES NON ENGLISH LIKE SENTENCE ENDING TOKENS.",
"parameters": {},
"command": "./fix_sent_final_punct.py"
}
40 changes: 40 additions & 0 deletions opuscleaner/filters/fix_sent_final_punct.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
import sys

my_punct = {'!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '.', '/', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', '{', '|', '}', '~', '»', '«', '“', '”'}

for line in sys.stdin:
src, trg = line.rstrip("\r\n").split("\t")
if len(src) == 0 or len(trg) == 0:
print(src + '\t' + trg)
continue

# Sometimes we have a space between the final letter and the punctuation, which is wrong except if using french quotes
if len(src) >= 2 and src[-1] in my_punct and src[-2] == " " and src[-1] != '»' and src[-1] != '«':
src = src[:-2] + src[-1]
if len(trg) >= 2 and trg[-1] in my_punct and trg[-2] == " " and trg[-1] != '»' and trg[-1] != '«':
trg = trg[:-2] + trg[-1]
# Sometimes two punctuation marks are swapped...
if len(src) >=2 and len(trg) >= 2 and src[-2] == trg[-1] and src[-1] == trg[-2]:
trg = trg[:-2] + src[-2] + src[-1]
# Sometimes they are swapped with space around eg SPACE». -> .SPACE»
if len(src) >=3 and src[-1] in my_punct and src[-2] == '»' and src[-3] == ' ':
src = src[:-3] + src[-1] + ' ' + src[-2]
if len(trg) >=3 and trg[-1] in my_punct and trg[-2] == '»' and trg[-3] == ' ':
trg = trg[:-3] + trg[-1] + ' ' + trg[-2]


# check for the french quotes special case
if (src[-1] == '»' or src[-1] == '«') and trg[-1] not in my_punct:
trg = trg + '"'
elif (trg[-1] == '»' or trg[-1] == '«') and src[-1] not in my_punct:
src = src + '"'
elif src[-1] in my_punct and trg[-1] not in my_punct:
trg = trg + src[-1]
elif trg[-1] in my_punct and src[-1] not in my_punct:
src = src + trg[-1]
# Final case. Fix mismatched punctuation on the src and trg. EXCEPT in cases like french quotes. And in cases where we have emdash at the front, as it means spech
elif trg[-1] in my_punct and src[-1] in my_punct and src[-1] != trg[-1] and src[-1] != '»' \
and src[-1] != '«' and trg[-1] != '»' and trg[-1] != '«' and src[0] != '–' and trg[0] != '–' and src[0] != '—' and trg[0] != '—':
trg = trg[:-1] + src[-1]
print(src + '\t' + trg)

0 comments on commit 24014c4

Please sign in to comment.