Added check for number of processed documents being less than corpus length. Closes #2553

gensim/models/ldamodel.py

@@ -961,6 +961,9 @@ def rho():
             for chunk_no, chunk in enumerate(chunks):
                 reallen += len(chunk)  # keep track of how many documents we've processed so far
 
+                if reallen > lencorpus:
+                    raise RuntimeError("processed size is bigger than corpus length (don't use infinite iterators)")
+
                 if eval_every and ((reallen == lencorpus) or ((chunk_no + 1) % (eval_every * self.numworkers) == 0)):
                     self.log_perplexity(chunk, total_docs=lencorpus)
 

gensim/test/test_ldamodel.py

@@ -508,6 +508,37 @@ def testAlphaAuto(self):
 # endclass TestLdaMulticore
 
 
+class TestLdaIterableTraining(unittest.TestCase, basetmtests.TestBaseTopicModel):
+    """Class for testing infinite generators,
+    for case when size of processed documents
+    goes beyond corpus length."""
+
+    class ProblematicIterable:
+        def __init__(self):
+            self.bag_of_words = [(0, 2), (3, 1), (6, 1), (100, 2)]
+            self.cursor = 0
+
+        def __iter__(self):
+            self.cursor = 0
+            logging.info('TestIterable() __iter__ was called')
+            return self
+
+        def __next__(self):
+            if self.cursor < 11:
+                self.cursor += 1
+                return self.bag_of_words
+            else:
+                logging.info('TestIterable() returned StopIteration')
+                raise StopIteration
+
+    def setUp(self) -> None:
+        self.corpus = self.ProblematicIterable()
+        self.class_ = ldamodel.LdaModel
+
+    def testDoesntTrainBeyondCorpusSize(self):
+        with self.assertRaises(RuntimeError):
+            self.model = self.class_(self.corpus, num_topics=2)
+
 if __name__ == '__main__':
     logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.DEBUG)
     unittest.main()