Merge pull request #451 from scholarly-python-package/develop

Release candidate for v1.7.3

Author: arunkannawadi

.github/workflows/lint.yaml

@@ -2,10 +2,6 @@ name: lint
 
 on:
   workflow_call:
-  push:
-    branches: [main, develop]
-  pull_request:
-    branches: [main, develop]
 
 jobs:
   lint:

.github/workflows/proxytests.yml

@@ -4,15 +4,9 @@
 name: Python package (with methods that need proxy)
 
 on:
-  workflow_run:
-    workflows: ["lint"]
-    branches: [main]
-    types:
-      - completed
   pull_request:
     branches: [main]
 
-
 jobs:
   build:
     runs-on: ubuntu-latest
@@ -37,7 +31,7 @@ jobs:
       #  uses: typilus/[email protected]
       - name: Run unittests
         id: unittests
-        if: ${{ github.event.pull_request.draft == false }}
+        if: ${{ github.event.pull_request.ready_for_review == true }}
         continue-on-error: true
         env:
           CONNECTION_METHOD: ${{ secrets.CONNECTION_METHOD }}
@@ -48,8 +42,6 @@ jobs:
         run: |
           coverage run -m unittest -v test_module.TestScholarlyWithProxy
       - name: Generate coverage report
-        if:
-          "matrix.os == 'macos-latest'"
         run: |
           curl --data-binary @.github/.codecov.yml https://codecov.io/validate | head -n 1
           coverage xml

.github/workflows/pythonpackage.yml

@@ -10,11 +10,6 @@ on:
     branches: [main, develop]
   pull_request:
     branches: [main, develop]
-  workflow_run:
-    workflows: ["lint"]
-#    branches: [main]
-    types:
-      - completed
 
 jobs:
   lint:

scholarly/_scholarly.py

@@ -1,20 +1,24 @@
 """scholarly.py"""
 import requests
+import re
 import os
 import copy
 import csv
 import pprint
-from typing import Dict, List
+import datetime
+import re
+from typing import Dict, List, Union
 from ._navigator import Navigator
 from ._proxy_generator import ProxyGenerator
 from dotenv import find_dotenv, load_dotenv
 from .author_parser import AuthorParser
 from .publication_parser import PublicationParser, _SearchScholarIterator
-from .data_types import Author, AuthorSource, Journal, Publication, PublicationSource
+from .data_types import Author, AuthorSource, CitesPerYear, Journal, Publication, PublicationSource
 
 _AUTHSEARCH = '/citations?hl=en&view_op=search_authors&mauthors={0}'
 _KEYWORDSEARCH = '/citations?hl=en&view_op=search_authors&mauthors=label:{0}'
 _KEYWORDSEARCHBASE = '/citations?hl=en&view_op=search_authors&mauthors={}'
+_KEYWORDSEARCH_PATTERN = "[-: #(),;]+"  # Unallowed characters in the keywords.
 _PUBSEARCH = '/scholar?hl=en&q={0}'
 _CITEDBYSEARCH = '/scholar?hl=en&cites={0}'
 _ORGSEARCH = "/citations?view_op=view_org&hl=en&org={0}"
@@ -155,11 +159,11 @@ def search_pubs(self,
                                   sort_by=sort_by, include_last_year=include_last_year, start_index=start_index)
         return self.__nav.search_publications(url)
 
-    def search_citedby(self, publication_id: int, **kwargs):
+    def search_citedby(self, publication_id: Union[int, str], **kwargs):
         """Searches by Google Scholar publication id and returns a generator of Publication objects.
 
         :param publication_id: Google Scholar publication id
-        :type publication_id: int
+        :type publication_id: int or str
 
         For the remaining parameters, see documentation of `search_pubs`.
         """
@@ -248,20 +252,70 @@ def bibtex(self, object: Publication)->str:
             self.logger.warning("Object not supported for bibtex exportation")
             return
 
+    @staticmethod
+    def _bin_citations_by_year(cites_per_year: CitesPerYear, year_end):
+        years = []
+        y_hi, y_lo = year_end, year_end
+        running_count = 0
+        for y in sorted(cites_per_year, reverse=True):
+            if running_count + cites_per_year[y] <= 1000:
+                running_count += cites_per_year[y]
+                y_lo = y
+            else:
+                running_count = cites_per_year[y]
+                years.append((y_hi, y_lo))
+                y_hi = y
+
+        if running_count > 0:
+            years.append((y_hi, y_lo))
+
+        return years
+
     def citedby(self, object: Publication)->_SearchScholarIterator:
         """Searches Google Scholar for other articles that cite this Publication
         and returns a Publication generator.
 
         :param object: The Publication object for the bibtex exportation
         :type object: Publication
         """
-        if object['container_type'] == "Publication":
-            publication_parser = PublicationParser(self.__nav)
-            return publication_parser.citedby(object)
-        else:
+
+        if object['container_type'] != "Publication":
             self.logger.warning("Object not supported for bibtex exportation")
             return
 
+        if object["num_citations"] <= 1000:
+            return PublicationParser(self.__nav).citedby(object)
+
+        self.logger.debug("Since the paper titled %s has %d citations (>1000), "
+                          "fetching it on an annual basis.", object["bib"]["title"], object["num_citations"])
+
+        year_end = int(datetime.date.today().year)
+
+        if object["source"] == PublicationSource.AUTHOR_PUBLICATION_ENTRY:
+            self.fill(object)
+            years = self._bin_citations_by_year(object.get("cites_per_year", {}), year_end)
+        else:
+            try:
+                year_low = int(object["bib"]["pub_year"])
+            except KeyError:
+                self.logger.warning("Unknown publication year for paper %s, may result in incorrect number "
+                                    "of citedby papers.", object["bib"]["title"])
+                return PublicationParser(self.__nav).citedby(object)
+
+            # Go one year at a time in decreasing order
+            years = zip(range(year_end, year_low-1, -1), range(year_end, year_low-1, -1))
+
+        # Extract cites_id. Note: There could be multiple ones, separated by commas.
+        m = re.search("cites=[\d+,]*", object["citedby_url"])
+        pub_id = m.group()[6:]
+        for y_hi, y_lo in years:
+            sub_citations = self.search_citedby(publication_id=pub_id, year_low=y_lo, year_high=y_hi)
+            if sub_citations.total_results and (sub_citations.total_results > 1000):
+                self.logger.warn("The paper titled %s has %d citations in the year %d. "
+                                 "Due to the limitation in Google Scholar, fetching only 1000 results "
+                                 "from that year.", object["bib"]["title"], sub_citations.total_results, y_lo)
+            yield from sub_citations
+
     def search_author_id(self, id: str, filled: bool = False, sortby: str = "citedby", publication_limit: int = 0)->Author:
         """Search by author id and return a single Author object
         :param sortby: select the order of the citations in the author page. Either by 'citedby' or 'year'. Defaults to 'citedby'.
@@ -321,7 +375,9 @@ def search_keyword(self, keyword: str):
              'source': 'SEARCH_AUTHOR_SNIPPETS',
              'url_picture': 'https://scholar.google.com/citations?view_op=medium_photo&user=lHrs3Y4AAAAJ'}
         """
-        url = _KEYWORDSEARCH.format(requests.utils.quote(keyword))
+
+        reg_keyword = re.sub(_KEYWORDSEARCH_PATTERN, "_", keyword)
+        url = _KEYWORDSEARCH.format(requests.utils.quote(reg_keyword))
         return self.__nav.search_authors(url)
 
     def search_keywords(self, keywords: List[str]):
@@ -355,8 +411,8 @@ def search_keywords(self, keywords: List[str]):
                  'url_picture': 'https://scholar.google.com/citations?view_op=medium_photo&user=_cMw1IUAAAAJ'}
 
         """
-
-        formated_keywords = ['label:'+requests.utils.quote(keyword) for keyword in keywords]
+        reg_keywords = (re.sub(_KEYWORDSEARCH_PATTERN, "_", keyword) for keyword in keywords)
+        formated_keywords = ['label:'+requests.utils.quote(keyword) for keyword in reg_keywords]
         formated_keywords = '+'.join(formated_keywords)
         url = _KEYWORDSEARCHBASE.format(formated_keywords)
         return self.__nav.search_authors(url)

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='scholarly',
-    version='1.7.2',
+    version='1.7.3',
     author='Steven A. Cholewiak, Panos Ipeirotis, Victor Silva, Arun Kannawadi',
     author_email='[email protected], [email protected], [email protected], [email protected]',
     description='Simple access to Google Scholar authors and citations',