✨Add char_span functionality (#40)

✨Add `char_span` functionality

Author: nipunsadvilkar

CHANGELOG.md

@@ -26,3 +26,9 @@
 
 -   🐛 Handle text with only punctuations - \#36
 -   🐛 Handle exclamation marks at EOL- \#37
+
+# v0.2.0
+
+-   ✨Add `char_span` parameter (optional) to get sentence & its (start, end) char offsets from original text
+-   ✨pySBD as a spaCy component example
+-   🐛 Fix double question mark swallow bug - \#39

README.md

@@ -24,6 +24,8 @@ print(seg.segment(text))
 # ['My name is Jonas E. Smith.', 'Please turn to p. 55.']
 ```
 
+-   Use `pysbd` as a [spaCy](https://spacy.io/usage/processing-pipelines) pipeline component. (recommended)</br>Please refer to example [pysbd\_as\_spacy\_component.py](https://github.com/nipunsadvilkar/pySBD/blob/master/examples/pysbd_as_spacy_component.py)
+
 ## Contributing
 
 If you find a text that is incorrectly segmented using pySBD, please submit an issue.

examples/pysbd_as_spacy_component.py

@@ -0,0 +1,29 @@
+"""
+Example of pySBD as a sentencizer component for spaCy
+
+Installation:
+pip install spacy
+"""
+import pysbd
+import spacy
+
+def pysbd_sentence_boundaries(doc):
+    seg = pysbd.Segmenter(language="en", clean=False, char_span=True)
+    sents_char_spans = seg.segment(doc.text)
+    char_spans = [doc.char_span(sent_span.start, sent_span.end) for sent_span in sents_char_spans]
+    start_token_ids = [span[0].idx for span in char_spans if span is not None]
+    for token in doc:
+        token.is_sent_start = True if token.idx in start_token_ids else False
+    return doc
+
+if __name__ == "__main__":
+    text = "My name is Jonas E. Smith.          Please turn to p. 55."
+    nlp = spacy.blank('en')
+
+    # add as a spacy pipeline component
+    nlp.add_pipe(pysbd_sentence_boundaries)
+
+    doc = nlp(text)
+    print('sent_id', 'sentence', sep='\t|\t')
+    for sent_id, sent in enumerate(doc.sents, start=1):
+        print(sent_id, sent.text, sep='\t|\t')

pysbd/abbreviation_replacer.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import re
-from pysbd.rules import Text
+from pysbd.utils import Text
 # TODO: SENTENCE_STARTERS should be lang specific
 from pysbd.lang.standard import Abbreviation, SENTENCE_STARTERS
 from pysbd.lang.common.numbers import (Common, SingleLetterAbbreviationRules,
@@ -93,5 +93,5 @@ def scan_for_replacements(self, txt, am, ind, char_array):
 
 
 if __name__ == "__main__":
-    s = "Here’s the - ahem - official citation: Baker, C., Anderson, Kenneth, Martin, James, & Palen, Leysia."
+    s = "fig. ??"
     print(AbbreviationReplacer(s).replace())

pysbd/about.py

@@ -2,7 +2,7 @@
 # https://python-packaging-user-guide.readthedocs.org/en/latest/single_source_version/
 
 __title__ = "pysbd"
-__version__ = "0.1.5"
+__version__ = "0.2.0"
 __summary__ = "pysbd (Python Sentence Boundary Disambiguation) is a rule-based sentence boundary detection that works out-of-the-box across many languages."
 __uri__ = "http://nipunsadvilkar.github.io/"
 __author__ = "Nipun Sadvilkar"

pysbd/clean/rules.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-from pysbd.rules import Rule
+from pysbd.utils import Rule
 
 
 class CleanRules(object):

pysbd/cleaner.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import re
-from pysbd.rules import Text
+from pysbd.utils import Text
 from pysbd.clean.rules import PDF, HTML, CleanRules as cr
 from pysbd.lang.standard import Abbreviation
 
@@ -10,7 +10,6 @@ class Cleaner(object):
     def __init__(self, text, language='common', doc_type=None):
         self.text = text
         self.language = language
-        # self.language_module = Language.get_language_code(language)
         self.doc_type = doc_type
 
     def clean(self):
@@ -116,7 +115,6 @@ def clean_consecutive_characters(self):
 
 
 if __name__ == "__main__":
-    # text = "Hello world.Today is Tuesday.Mr. Smith went to the store and bought 1,000.That is a lot."
-    text = "• 9. Stop smoking \n• 10. Get some rest \n \nYou have the best chance of having a problem-free pregnancy and a healthy baby if you follow \na few simple guidelines:  \n\n1. Organise your pregnancy care early"
+    text = "Hello world.Today is Tuesday.Mr. Smith went to the store and bought 1,000.That is a lot."
     c = Cleaner(text)
     print(c.clean())

pysbd/exclamation_words.py

@@ -18,6 +18,5 @@ def apply_rules(cls, text):
 
 
 if __name__ == "__main__":
-    # text = "Hello .World work for Yahoo! company"
     text = "\"Dinah'll miss me very much to-night, I should think!\"ȸ"
     print(ExclamationWords.apply_rules(text))

pysbd/lang/common/ellipsis.py

@@ -1,22 +1,24 @@
 # -*- coding: utf-8 -*-
-from pysbd.rules import Rule
+from pysbd.utils import Rule
 
 
 class EllipsisRules(object):
 
+    # below rules aren't similar to original rules of pragmatic segmenter
+    # modification: spaces replaced with same number of symbols
     # Rubular: http://rubular.com/r/i60hCK81fz
-    ThreeConsecutiveRule = Rule(r'\.\.\.(?=\s+[A-Z])', '☏.')
+    ThreeConsecutiveRule = Rule(r'\.\.\.(?=\s+[A-Z])', '☏☏.')
 
     # Rubular: http://rubular.com/r/Hdqpd90owl
-    FourConsecutiveRule = Rule(r'(?<=\S)\.{3}(?=\.\s[A-Z])', 'ƪ')
+    FourConsecutiveRule = Rule(r'(?<=\S)\.{3}(?=\.\s[A-Z])', 'ƪƪƪ')
 
     # Rubular: http://rubular.com/r/YBG1dIHTRu
-    ThreeSpaceRule = Rule(r'(\s\.){3}\s', '♟')
+    ThreeSpaceRule = Rule(r'(\s\.){3}\s', '♟♟♟♟♟♟♟')
 
     # Rubular: http://rubular.com/r/2VvZ8wRbd8
-    FourSpaceRule = Rule(r'(?<=[a-z])(\.\s){3}\.($|\\n)', '♝')
+    FourSpaceRule = Rule(r'(?<=[a-z])(\.\s){3}\.($|\\n)', '♝♝♝♝♝♝♝')
 
-    OtherThreePeriodRule = Rule(r'\.\.\.', 'ƪ')
+    OtherThreePeriodRule = Rule(r'\.\.\.', 'ƪƪƪ')
 
     All = [ThreeSpaceRule, FourSpaceRule, FourConsecutiveRule,
            ThreeConsecutiveRule, OtherThreePeriodRule]

pysbd/lang/common/numbers.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 import re
-from pysbd.rules import Rule
+from pysbd.utils import Rule
 
 
 class Common(object):