add support for merge requests

Author: vchrombie

grimoire_elk/enriched/gitlabcomments.py

@@ -157,8 +157,28 @@ def get_time_to_first_attention(self, item):
         reaction_dates = [str_to_datetime(reaction['created_at']) for reaction in item['award_emoji_data']
                           if item['author']['username'] != reaction['user']['username']]
         reaction_dates.extend(comments_dates)
+
         if reaction_dates:
             return min(reaction_dates)
+
+        return None
+
+    def get_time_to_merge_request_response(self, item):
+        """Get the first date at which a review was made on the PR by someone
+        other than the user who created the PR
+        """
+        review_dates = []
+
+        for comment in item['notes_data']:
+            # skip comments of the pull request creator
+            if item['author']['username'] == comment['author']['username']:
+                continue
+
+            review_dates.append(str_to_datetime(comment['created_at']))
+
+        if review_dates:
+            return min(review_dates)
+
         return None
 
     def __get_reactions(self, item):
@@ -266,8 +286,112 @@ def __get_rich_issue(self, item):
         if self.sortinghat:
             item[self.get_field_date()] = rich_issue[self.get_field_date()]
             rich_issue.update(self.get_item_sh(item, self.issue_roles))
+
         return rich_issue
 
+    def __get_rich_merge(self, item):
+        rich_mr = {}
+
+        self.copy_raw_fields(self.RAW_FIELDS_COPY, item, rich_mr)
+        # The real data
+        merge_request = item['data']
+
+        rich_mr['time_to_close_days'] = \
+            get_time_diff_days(merge_request['created_at'], merge_request['closed_at'])
+
+        if merge_request['state'] != 'closed':
+            rich_mr['time_open_days'] = \
+                get_time_diff_days(pull_request['created_at'], datetime_utcnow().replace(tzinfo=None))
+        else:
+            rich_mr['time_open_days'] = rich_mr['time_to_close_days']
+
+        rich_mr['user_login'] = merge_request['author']['username']
+
+        user = merge_request.get('author', None)
+        if user is not None and user:
+            rich_mr['user_name'] = user['name']
+            rich_mr['author_name'] = user['name']
+            rich_mr["user_domain"] = self.get_email_domain(user['email']) if user['email'] else None
+            rich_mr['user_org'] = user.get('company', None)
+            rich_mr['user_location'] = user.get('location', None)
+            rich_mr['user_geolocation'] = None
+        else:
+            rich_mr['user_name'] = None
+            rich_mr["user_domain"] = None
+            rich_mr['user_org'] = None
+            rich_mr['user_location'] = None
+            rich_mr['user_geolocation'] = None
+            rich_mr['author_name'] = None
+
+        merged_by = merge_request.get('merged_by', None)
+        if merged_by is not None and merged_by:
+            rich_mr['merge_author_login'] = merged_by['login']
+            rich_mr['merge_author_name'] = merged_by['name']
+            rich_mr["merge_author_domain"] = self.get_email_domain(merged_by['email']) if merged_by['email'] else None
+            rich_mr['merge_author_org'] = merged_by.get('company', None)
+            rich_mr['merge_author_location'] = merged_by.get('location', None)
+            rich_mr['merge_author_geolocation'] = None
+        else:
+            rich_mr['merge_author_name'] = None
+            rich_mr['merge_author_login'] = None
+            rich_mr["merge_author_domain"] = None
+            rich_mr['merge_author_org'] = None
+            rich_mr['merge_author_location'] = None
+            rich_mr['merge_author_geolocation'] = None
+
+        rich_mr['id'] = merge_request['id']
+        rich_mr['merge_id'] = merge_request['id']
+        rich_mr['merge_id_in_repo'] = merge_request['web_url'].split("/")[-1]
+        rich_mr['repository'] = self.get_project_repository(rich_mr)
+        rich_mr['merge_title'] = merge_request['title']
+        rich_mr['merge_title_analyzed'] = merge_request['title']
+        rich_mr['merge_state'] = merge_request['state']
+        rich_mr['merge_created_at'] = merge_request['created_at']
+        rich_mr['merge_updated_at'] = merge_request['updated_at']
+        rich_mr['merge_status'] = merge_request['merge_status']
+        rich_mr['merge_merged_at'] = merge_request['merged_at']
+        rich_mr['merge_closed_at'] = merge_request['closed_at']
+        rich_mr['url'] = merge_request['web_url']
+        rich_mr['merge_url'] = merge_request['web_url']
+
+        # extract reactions and add it to enriched item
+        # rich_mr.update(self.__get_reactions(merge_request))
+
+        rich_mr['merge_labels'] = merge_request['labels']
+        rich_mr['item_type'] = MERGE_TYPE
+
+        rich_mr['gitlab_repo'] = rich_mr['repository'].replace(GITLAB, '')
+        rich_mr['gitlab_repo'] = re.sub('.git$', '', rich_mr['gitlab_repo'])
+
+        # GMD code development metrics
+        rich_mr['code_merge_duration'] = get_time_diff_days(merge_request['created_at'],
+                                                            merge_request['merged_at'])
+        rich_mr['num_versions'] = len(merge_request['versions_data'])
+
+        rich_mr['time_to_merge_request_response'] = None
+        if merge_request['notes_data'] != 0:
+            min_review_date = self.get_time_to_merge_request_response(pull_request)
+            rich_pr['time_to_merge_request_response'] = \
+                get_time_diff_days(str_to_datetime(pull_request['created_at']), min_review_date)
+
+        if self.prjs_map:
+            rich_mr.update(self.get_item_project(rich_mr))
+
+        if 'project' in item:
+            rich_mr['project'] = item['project']
+
+        rich_mr.update(self.get_grimoire_fields(merge_request['created_at'], MERGE_TYPE))
+
+        # due to backtrack compatibility, `is_gitlabcomments_*` is replaced with `is_gitlab_*`
+        rich_mr.pop('is_gitlabcomments_{}'.format(MERGE_TYPE))
+        rich_mr['is_gitlab_{}'.format(MERGE_TYPE)] = 1
+
+        if self.sortinghat:
+            item[self.get_field_date()] = rich_mr[self.get_field_date()]
+            rich_mr.update(self.get_item_sh(item, self.mr_roles))
+
+        return rich_mr
+
     @metadata
     def get_rich_item(self, item):
         rich_item = {}
@@ -318,9 +442,70 @@ def get_rich_issue_comments(self, comments, eitem):
 
             ecomment['comment_updated_at'] = comment['updated_at']
 
+            # Add id info to allow to coexistence of items of different types in the same index
+            ecomment['id'] = '{}_review_comment_{}'.format(eitem['id'], comment['id'])
+            ecomment.update(self.get_grimoire_fields(comment['updated_at'], ISSUE_COMMENT_TYPE))
+
+            # due to backtrack compatibility, `is_gitlabcomments_*` is replaced with `is_gitlab_*`
+            ecomment.pop('is_gitlabcomments_{}'.format(ISSUE_COMMENT_TYPE))
+            ecomment['is_gitlab_{}'.format(ISSUE_COMMENT_TYPE)] = 1
+            ecomment['is_gitlab_comment'] = 1
+
+            if self.sortinghat:
+                ecomment.update(self.get_item_sh(comment, self.comment_roles, 'updated_at'))
+
+            if self.prjs_map:
+                ecomment.update(self.get_item_project(ecomment))
+
+            if 'project' in eitem:
+                ecomment['project'] = eitem['project']
+
+            self.add_repository_labels(ecomment)
+            self.add_metadata_filter_raw(ecomment)
+            self.add_gelk_metadata(ecomment)
+
+            ecomments.append(ecomment)
+
+        return ecomments
+
+    def get_rich_merge_reviews(self, comments, eitem):
+        ecomments = []
+
+        for comment in comments:
+            ecomment = {}
+
+            self.copy_raw_fields(self.RAW_FIELDS_COPY, eitem, ecomment)
+            # Copy data from the enriched pull
+            ecomment['merge_labels'] = eitem['merge_labels']
+            ecomment['merge_id'] = eitem['merge_id']
+            ecomment['merge_id_in_repo'] = eitem['merge_id_in_repo']
+            ecomment['merge_title'] = eitem['merge_title']
+            ecomment['merge_url'] = eitem['merge_url']
+            ecomment['merge_state'] = eitem['merge_state']
+            ecomment['merge_created_at'] = eitem['merge_created_at']
+            ecomment['merge_updated_at'] = eitem['merge_updated_at']
+            ecomment['merge_merged_at'] = eitem['merge_merged_at']
+            ecomment['merge_closed_at'] = eitem['merge_closed_at']
+            ecomment['merge_status'] = eitem['merge_status']
+            ecomment['gitlab_repo'] = eitem['gitlab_repo']
+            ecomment['repository'] = eitem['repository']
+            ecomment['item_type'] = COMMENT_TYPE
+            ecomment['sub_type'] = REVIEW_COMMENT_TYPE
+
+            # Copy data from the raw comment
+            ecomment['body'] = comment['body'][:self.KEYWORD_MAX_LENGTH]
+            ecomment['body_analyzed'] = comment['body']
+            # ecomment['url'] = comment['html_url']
+
+            # extract reactions and add it to enriched item
+            # ecomment.update(self.__get_reactions(comment))
+
+            ecomment['comment_updated_at'] = comment['updated_at']
+
             # Add id info to allow to coexistence of items of different types in the same index
             ecomment['id'] = '{}_review_comment_{}'.format(eitem['id'], comment['id'])
             ecomment.update(self.get_grimoire_fields(comment['updated_at'], REVIEW_COMMENT_TYPE))
+
             # due to backtrack compatibility, `is_gitlabcomments_*` is replaced with `is_gitlab_*`
             ecomment.pop('is_gitlabcomments_{}'.format(REVIEW_COMMENT_TYPE))
             ecomment['is_gitlab_{}'.format(REVIEW_COMMENT_TYPE)] = 1
@@ -353,6 +538,16 @@ def enrich_issue(self, item, eitem):
 
         return eitems
 
+    def enrich_merge(self, item, eitem):
+        eitems = []
+
+        comments = item['data'].get('notes_data', [])
+        if comments:
+            rich_item_comments = self.get_rich_merge_reviews(comments, eitem)
+            eitems.extend(rich_item_comments)
+
+        return eitems
+
     def enrich_items(self, ocean_backend):
         items_to_enrich = []
         num_items = 0