crowd-kit==0.0.4

Author: Losik

setup.py

@@ -9,7 +9,7 @@
     name='crowd-kit',
     package_dir={PREFIX: 'src'},
     packages=[f'{PREFIX}.{package}' for package in find_packages('src')],
-    version='0.0.3',
+    version='0.0.4',
     description='Python libraries for crowdsourcing',
     license='Apache 2.0',
     author='Vladimir Losev',

src/aggregation/__init__.py

@@ -1,13 +1,13 @@
 from .bradley_terry import BradleyTerry
 from .dawid_skene import DawidSkene
 from .gold_majority_vote import GoldMajorityVote
-from .hrrasa import HRRASA
+from .hrrasa import HRRASA, TextHRRASA
 from .m_msr import MMSR
 from .majority_vote import MajorityVote
 from .noisy_bt import NoisyBradleyTerry
-from .rasa import RASA
+from .rasa import RASA, TextRASA
 from .wawa import Wawa
 from .zero_based_skill import ZeroBasedSkill
 
 __all__ = ['DawidSkene', 'MajorityVote', 'MMSR', 'Wawa', 'GoldMajorityVote', 'ZeroBasedSkill', 'HRRASA', 'RASA',
-           'BradleyTerry', 'NoisyBradleyTerry']
+           'BradleyTerry', 'NoisyBradleyTerry', 'TextRASA', 'TextHRRASA']

src/aggregation/annotations.py

@@ -125,7 +125,7 @@ def manage_docstring(obj):
 
 TASKS_LABEL_PROBAS = Annotation(
     type=pd.DataFrame,
-    title="Tasks' true label probability distributions",
+    title="Tasks' label probability distributions",
     description=textwrap.dedent('''
         A pandas.DataFrame indexed by `task` such that `result.loc[task, label]`
         is the probability of `task`'s true label to be equal to `label`. Each
@@ -135,7 +135,7 @@ def manage_docstring(obj):
 
 TASKS_LABEL_SCORES = Annotation(
     type=pd.DataFrame,
-    title="Tasks' true label scores",
+    title="Tasks' label scores",
     description=textwrap.dedent('''
         A pandas.DataFrame indexed by `task` such that `result.loc[task, label]`
         is the score of `label` for `task`.
@@ -187,9 +187,9 @@ def manage_docstring(obj):
 )
 
 WEIGHTS = Annotation(
-    type=pd.Series,
+    type=pd.DataFrame,
     title='Task weights',
-    description='A pandas.Series indexed by `task` containing task weights'
+    description='A pandas.DataFrame containing `task`, `performer` and `weight`'
 )
 
 

src/aggregation/bradley_terry.py

@@ -37,7 +37,7 @@ def fit(self, data: annotations.PAIRWISE_DATA) -> Annotation(type='BradleyTerry'
         M, unique_labels = self._build_win_matrix(data)
 
         if not unique_labels.size:
-            self.result_ = pd.Series([])
+            self.result_ = pd.Series([], dtype=np.float64)
             return self
 
         T = M.T + M

src/aggregation/closest_to_average.py

@@ -21,19 +21,15 @@ class ClosestToAverage(BaseAggregator):
     distance: Callable[[np.array, np.array], float] = attr.ib()
 
     @manage_docstring
-    def fit(self, data: annotations.EMBEDDED_DATA, skills: annotations.SKILLS = None,
+    def fit(self, data: annotations.EMBEDDED_DATA, aggregated_embeddings: annotations.TASKS_EMBEDDINGS = None,
             true_embeddings: annotations.TASKS_EMBEDDINGS = None) -> Annotation(type='ClosestToAverage', title='self'):
 
         data = data[['task', 'performer', 'output', 'embedding']]
-        if skills is None:
+        if aggregated_embeddings is None:
             avg_embeddings = data.groupby('task')['embedding'].avg()
+            avg_embeddings.update(true_embeddings)
         else:
-            data = data.join(skills.rename('skill'), on='performer')
-            data['weighted_embedding'] = data.skill * data.embedding
-            group = data.groupby('task')
-            avg_embeddings = (group.weighted_embedding.apply(np.sum) / group.skill.sum())
-
-        avg_embeddings.update(true_embeddings)
+            avg_embeddings = aggregated_embeddings
 
         # Calculating distances (scores)
         data = data.join(avg_embeddings.rename('avg_embedding'), on='task')

src/aggregation/dawid_skene.py

@@ -2,6 +2,7 @@
 
 import attr
 import numpy as np
+import pandas as pd
 
 from . import annotations
 from .annotations import manage_docstring, Annotation
@@ -54,16 +55,36 @@ def _e_step(data: annotations.LABELED_DATA, priors: annotations.LABEL_PRIORS, er
         Given performer's answers, labels' prior probabilities and performer's performer's
         errors probabilities matrix estimates tasks' true labels probabilities.
         """
-        joined = data.join(errors, on=['performer', 'label'])
+
+        # We have to multiply lots of probabilities and such products are known to converge
+        # to zero exponentialy fast. To avoid floating-point precision problems we work with
+        # logs of original values
+        joined = data.join(np.log2(errors), on=['performer', 'label'])
         joined.drop(columns=['performer', 'label'], inplace=True)
-        probas = priors * joined.groupby('task', sort=False).prod()
-        return probas.div(probas.sum(axis=1), axis=0)
+        log_likelihoods = np.log2(priors) + joined.groupby('task', sort=False).sum()
+
+        # Exponentiating log_likelihoods 'as is' may still get us beyond our precision.
+        # So we shift every row of log_likelihoods by a constant (which is equivalent to
+        # multiplying likelihoods rows by a constant) so that max log_likelihood in each
+        # row is equal to 0. This trick ensures proper scaling after exponentiating and
+        # does not affect the result of E-step
+        scaled_likelihoods = np.exp2(log_likelihoods.sub(log_likelihoods.max(axis=1), axis=0))
+        return scaled_likelihoods.div(scaled_likelihoods.sum(axis=1), axis=0)
 
     @manage_docstring
     def fit(self, data: annotations.LABELED_DATA) -> Annotation(type='DawidSkene', title='self'):
 
-        # Initialization
         data = data[['task', 'performer', 'label']]
+
+        # Early exit
+        if not data.size:
+            self.probas_ = pd.DataFrame()
+            self.priors_ = pd.Series()
+            self.errors_ = pd.DataFrame()
+            self.labels_ = pd.Series()
+            return self
+
+        # Initialization
         probas = MajorityVote().fit_predict_proba(data)
         priors = probas.mean()
         errors = self._m_step(data, probas)