Merge pull request #449 from scholarly-python-package/citedby1k

Fetch more than 1000 citations

Author: arunkannawadi

scholarly/_scholarly.py

@@ -5,13 +5,15 @@
 import copy
 import csv
 import pprint
-from typing import Dict, List
+import datetime
+import re
+from typing import Dict, List, Union
 from ._navigator import Navigator
 from ._proxy_generator import ProxyGenerator
 from dotenv import find_dotenv, load_dotenv
 from .author_parser import AuthorParser
 from .publication_parser import PublicationParser, _SearchScholarIterator
-from .data_types import Author, AuthorSource, Journal, Publication, PublicationSource
+from .data_types import Author, AuthorSource, CitesPerYear, Journal, Publication, PublicationSource
 
 _AUTHSEARCH = '/citations?hl=en&view_op=search_authors&mauthors={0}'
 _KEYWORDSEARCH = '/citations?hl=en&view_op=search_authors&mauthors=label:{0}'
@@ -157,11 +159,11 @@ def search_pubs(self,
                                   sort_by=sort_by, include_last_year=include_last_year, start_index=start_index)
         return self.__nav.search_publications(url)
 
-    def search_citedby(self, publication_id: int, **kwargs):
+    def search_citedby(self, publication_id: Union[int, str], **kwargs):
         """Searches by Google Scholar publication id and returns a generator of Publication objects.
 
         :param publication_id: Google Scholar publication id
-        :type publication_id: int
+        :type publication_id: int or str
 
         For the remaining parameters, see documentation of `search_pubs`.
         """
@@ -250,20 +252,70 @@ def bibtex(self, object: Publication)->str:
             self.logger.warning("Object not supported for bibtex exportation")
             return
 
+    @staticmethod
+    def _bin_citations_by_year(cites_per_year: CitesPerYear, year_end):
+        years = []
+        y_hi, y_lo = year_end, year_end
+        running_count = 0
+        for y in sorted(cites_per_year, reverse=True):
+            if running_count + cites_per_year[y] <= 1000:
+                running_count += cites_per_year[y]
+                y_lo = y
+            else:
+                running_count = cites_per_year[y]
+                years.append((y_hi, y_lo))
+                y_hi = y
+
+        if running_count > 0:
+            years.append((y_hi, y_lo))
+
+        return years
+
     def citedby(self, object: Publication)->_SearchScholarIterator:
         """Searches Google Scholar for other articles that cite this Publication
         and returns a Publication generator.
 
         :param object: The Publication object for the bibtex exportation
         :type object: Publication
         """
-        if object['container_type'] == "Publication":
-            publication_parser = PublicationParser(self.__nav)
-            return publication_parser.citedby(object)
-        else:
+
+        if object['container_type'] != "Publication":
             self.logger.warning("Object not supported for bibtex exportation")
             return
 
+        if object["num_citations"] <= 1000:
+            return PublicationParser(self.__nav).citedby(object)
+
+        self.logger.debug("Since the paper titled %s has %d citations (>1000), "
+                          "fetching it on an annual basis.", object["bib"]["title"], object["num_citations"])
+
+        year_end = int(datetime.date.today().year)
+
+        if object["source"] == PublicationSource.AUTHOR_PUBLICATION_ENTRY:
+            self.fill(object)
+            years = self._bin_citations_by_year(object.get("cites_per_year", {}), year_end)
+        else:
+            try:
+                year_low = int(object["bib"]["pub_year"])
+            except KeyError:
+                self.logger.warning("Unknown publication year for paper %s, may result in incorrect number "
+                                    "of citedby papers.", object["bib"]["title"])
+                return PublicationParser(self.__nav).citedby(object)
+
+            # Go one year at a time in decreasing order
+            years = zip(range(year_end, year_low-1, -1), range(year_end, year_low-1, -1))
+
+        # Extract cites_id. Note: There could be multiple ones, separated by commas.
+        m = re.search("cites=[\d+,]*", object["citedby_url"])
+        pub_id = m.group()[6:]
+        for y_hi, y_lo in years:
+            sub_citations = self.search_citedby(publication_id=pub_id, year_low=y_lo, year_high=y_hi)
+            if sub_citations.total_results and (sub_citations.total_results > 1000):
+                self.logger.warn("The paper titled %s has %d citations in the year %d. "
+                                 "Due to the limitation in Google Scholar, fetching only 1000 results "
+                                 "from that year.", object["bib"]["title"], sub_citations.total_results, y_lo)
+            yield from sub_citations
+
     def search_author_id(self, id: str, filled: bool = False, sortby: str = "citedby", publication_limit: int = 0)->Author:
         """Search by author id and return a single Author object
         :param sortby: select the order of the citations in the author page. Either by 'citedby' or 'year'. Defaults to 'citedby'.

test_module.py

@@ -1,6 +1,7 @@
 import unittest
 import os
 import sys
+from collections import Counter
 from scholarly import scholarly, ProxyGenerator
 from scholarly.data_types import Mandate
 from scholarly.publication_parser import PublicationParser
@@ -549,6 +550,16 @@ def test_save_journal_leaderboard(self):
             if os.path.exists(filename):
                 os.remove(filename)
 
+    def test_bin_citations_by_year(self):
+        """Test an internal optimization function to bin cites_per_year
+           while keeping the citation counts less than 1000 per bin.
+        """
+        cpy = {2022: 490, 2021: 340, 2020:327, 2019:298, 2018: 115, 2017: 49, 2016: 20, 2015: 8, 2014: 3, 2013: 1, 2012: 1}
+        years = scholarly._bin_citations_by_year(cpy, 2022)
+        for y_hi, y_lo in years:
+            self.assertLessEqual(y_lo, y_hi)
+            self.assertLessEqual(sum(cpy[y] for y in range(y_lo, y_hi+1)), 1000)
+
 
 class TestScholarlyWithProxy(unittest.TestCase):
     @classmethod
@@ -795,5 +806,41 @@ def test_pubs_custom_url(self):
         self.assertEqual(pub['bib']['pub_year'], '2009')
         self.assertGreaterEqual(pub['num_citations'], 581)
 
+    def check_citedby_1k(self, pub):
+        """A common checking method to check
+        """
+        original_citation_count = pub["num_citations"]
+        # Trigger a different code path
+        if original_citation_count <= 1000:
+            pub["num_citations"] = 1001
+        citations = scholarly.citedby(pub)
+        citation_list = list(citations)
+        self.assertEqual(len(citation_list), original_citation_count)
+        return citation_list
+
+    @unittest.skipIf(os.getenv("CONNECTION_METHOD") in {None, "none", "freeproxy"}, reason="No robust proxy setup")
+    def test_citedby_1k_citations(self):
+        """Test that scholarly can fetch 1000+ citations from an author
+        """
+        author = scholarly.search_author_id('QoX9bu8AAAAJ')
+        scholarly.fill(author, sections=['publications'])
+        pub = [_p for _p in  author['publications'] if _p["author_pub_id"]=="QoX9bu8AAAAJ:L8Ckcad2t8MC"][0]
+        scholarly.fill(pub)
+        citation_list = self.check_citedby_1k(pub)
+
+        yearwise_counter = Counter([c["bib"]["pub_year"] for c in citation_list])
+        for year, count in pub["cites_per_year"].items():
+            self.assertEqual(yearwise_counter.get(str(year), 0), count)
+
+    @unittest.skipIf(os.getenv("CONNECTION_METHOD") in {None, "none", "freeproxy"}, reason="No robust proxy setup")
+    def test_citedby_1k_scholar(self):
+        """Test that scholarly can fetch 1000+ citations from a pub search.
+        """
+        title = "Persistent entanglement in a class of eigenstates of quantum Heisenberg spin glasses"
+        pubs = scholarly.search_pubs(title)
+        pub = next(pubs)
+        self.check_citedby_1k(pub)
+
+
 if __name__ == '__main__':
     unittest.main()