🐛 Handle text with only punctuations & ! at EOL (#38)

🐛 Handle text with only punctuations & ! at EOL

Author: nipunsadvilkar

CHANGELOG.md

@@ -21,3 +21,8 @@
 # v0.1.4
 
 -   ✨ ✅ Handle intermittent punctuations - \#34
+
+# v0.1.5
+
+-   🐛 Handle text with only punctuations - \#36
+-   🐛 Handle exclamation marks at EOL- \#37

pysbd/about.py

@@ -2,7 +2,7 @@
 # https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
 
 __title__ = "pysbd"
-__version__ = "0.1.4"
+__version__ = "0.1.5"
 __summary__ = "pysbd (Python Sentence Boundary Disambiguation) is a rule-based sentence boundary detection that works out-of-the-box across many languages."
 __uri__ = "http://nipunsadvilkar.github.io/"
 __author__ = "Nipun Sadvilkar"

pysbd/lang/common/numbers.py

@@ -7,7 +7,8 @@ class Common(object):
 
     # added special case: r"[。．.！!?].*" to handle intermittent dots, exclamation, etc.
     # TODO: above special cases group can be updated as per developer needs
-    SENTENCE_BOUNDARY_REGEX = r"（(?:[^）])*）(?=\s?[A-Z])|「(?:[^」])*」(?=\s[A-Z])|\((?:[^\)]){2,}\)(?=\s[A-Z])|\'(?:[^\'])*[^,]\'(?=\s[A-Z])|\"(?:[^\"])*[^,]\"(?=\s[A-Z])|\“(?:[^\”])*[^,]\”(?=\s[A-Z])|[。．.！!?].*|\S.*?[。．.！!?？ȸȹ☉☈☇☄]"
+    SENTENCE_BOUNDARY_REGEX = r"（(?:[^）])*）(?=\s?[A-Z])|「(?:[^」])*」(?=\s[A-Z])|\((?:[^\)]){2,}\)(?=\s[A-Z])|\'(?:[^\'])*[^,]\'(?=\s[A-Z])|\"(?:[^\"])*[^,]\"(?=\s[A-Z])|\“(?:[^\”])*[^,]\”(?=\s[A-Z])|[。．.！!?？].*|\S.*?[。．.！!?？ȸȹ☉☈☇☄]"
+
     # # Rubular: http://rubular.com/r/NqCqv372Ix
     QUOTATION_AT_END_OF_SENTENCE_REGEX = r'[!?\.-][\"\'“”]\s{1}[A-Z]'
 

pysbd/lang/standard.py

@@ -39,6 +39,7 @@ class DoublePunctuationRules(object):
     SecondRule = Rule(r'!\?', '☈')
     ThirdRule = Rule(r'\?\?', '☇')
     ForthRule = Rule(r'!!', '☄')
+    DoublePunctuation = r'\?!|!\?|\?\?|!!'
     All = [FirstRule, SecondRule, ThirdRule, ForthRule]
 
 

pysbd/processor.py

@@ -143,8 +143,10 @@ def process_text(self, txt):
             txt += 'ȸ'
         txt = ExclamationWords.apply_rules(txt)
         txt = BetweenPunctuation(txt).replace()
-        txt = Text(txt).apply(*DoublePunctuationRules.All,
-                              Standard.QuestionMarkInQuotationRule,
+        # handle text having only doublepunctuations
+        if not re.match(DoublePunctuationRules.DoublePunctuation, txt):
+            txt = Text(txt).apply(*DoublePunctuationRules.All)
+        txt = Text(txt).apply(Standard.QuestionMarkInQuotationRule,
                               *ExclamationPointRules.All)
         txt = ListItemReplacer(txt).replace_parens()
         txt = self.sentence_boundary_punctuation(txt)
@@ -176,6 +178,8 @@ def sentence_boundary_punctuation(self, txt):
         if hasattr(self.language_module, 'ReplaceNonSentenceBoundaryCommaRule'):
             txt = Text(txt).apply(
                 self.language_module.ReplaceNonSentenceBoundaryCommaRule)
+        # retain exclamation mark if it is an ending character of a given text
+        txt = re.sub(r'&ᓴ&$', '!', txt)
         txt = re.findall(Common.SENTENCE_BOUNDARY_REGEX, txt)
         return txt
 

tests/regression/test_issues.py

@@ -20,6 +20,9 @@
     ('#34', '..', ['..']),
     ('#34', '. . .', ['. . .']),
     ('#34', '! ! !', ['! ! !']),
+    ('#36', '??', ['??']),
+    ('#37', "As an example of a different special-purpose mechanism, we have introduced a methodology for letting donors make their donations to charities conditional on donations by other donors (who, in turn, can make their donations conditional) [70]. We have used this mechanism to collect money for Indian Ocean Tsunami and Hurricane Katrina victims. We have also introduced a more general framework for negotiation when one agent's actions have a direct effect (externality) on the other agents' utilities [69]. Both the charities and externalities methodologies require the solution of NP-hard optimization problems in general, but there are some natural tractable cases as well as effective MIP formulations. Recently, Ghosh and Mahdian [86] at Yahoo! Research extended our charities work, and based on this a web-based system for charitable donations was built at Yahoo!",
+     ['As an example of a different special-purpose mechanism, we have introduced a methodology for letting donors make their donations to charities conditional on donations by other donors (who, in turn, can make their donations conditional) [70].', 'We have used this mechanism to collect money for Indian Ocean Tsunami and Hurricane Katrina victims.', "We have also introduced a more general framework for negotiation when one agent's actions have a direct effect (externality) on the other agents' utilities [69].", 'Both the charities and externalities methodologies require the solution of NP-hard optimization problems in general, but there are some natural tractable cases as well as effective MIP formulations.', 'Recently, Ghosh and Mahdian [86] at Yahoo! Research extended our charities work, and based on this a web-based system for charitable donations was built at Yahoo!'])
 ]
 
 @pytest.mark.parametrize('issue_no,text,expected_sents', TEST_ISSUE_DATA)