Merge pull request #424 from scholarly-python-package/develop

Releasing v1.7.0

Author: arunkannawadi

.github/workflows/pythonpackage.yml

@@ -5,7 +5,7 @@ name: Python package
 
 on:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "0 0 1,10,20 * *"
   push:
     branches: [main, develop]
   pull_request:
@@ -22,9 +22,9 @@ jobs:
         os: [ubuntu-latest, macos-latest, windows-latest]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
       - name: Lint with flake8

CHANGELOG.md

@@ -1,44 +1,99 @@
 # CHANGELOG
 
+## Changes in v1.7.0
+
+### Features
+- Add a new `citation` entry to `pub` fetched from an author profile with formatted citation entry #423.
+
+### Bugfixes
+- Fix pprint failures on Windows #413.
+- Thoroughly handle 1000 or more publications that are available (or not) according to public access mandates #414.
+- Fix errors in `download_mandates_csv` that may occassionally occur for agencies without a policy link #413.
+
+## Changes in v1.6.3
+
+### Bugfix
+- search_pubs method did not respect include_last_year, which is now fixed #420, #421.
+
+### Enhancement
+- Unit tests involving funding agency mandates are a bit more robust.
+
+## Changes in v1.6.2
+
+### Bugfix
+- Fix an error in the workflow publishing to PyPI.
+
+## Changes in v1.6.1
+
+### Bugfix
+- Handle 1000 or more publications that are available (or not) according to public access mandates #414.
+
+### Enhancement
+- Fetch 20+ coauthors without requiring geckodriver/chrome-driver to be installed #411.
+
+## Changes in v1.6.0
+
+### Features
+- Download table of funding agencies as a CSV file with URL to the funding mandates included
+- Downlad top-ranking journals in general, under sub-categories and in different languages as a CSV file
+
+### Bugfixes
+- #392
+- #394
+
+## Changes in v1.5.1
+
+### Feature
+- Support chromium (chrome-driver) as an alternative to geckodriver #387
+
+### Improvements
+- Firefox/Geckodriver operates in headless mode
+- Increase test coverage to include all public APIs
+- Clean up legacy code and improve coding styles
+- Remove the use of deprecated functions in dependency packages
+
+### Bugfix
+- Stop attempting to reuse a closed webdriver
+
 ## Changes in v1.5.0
-## Features
+### Features
 - Fetch the public access mandates information from a Scholar profile and mark the publications whether or not they satisfy the open-access mandate.
 - Fetch an author's organization identifer from their Scholar profile
 - Search for all authors affiliated with an organization
 - Fetch homepage URL from a Scholar profile
-## Enhancements
+### Enhancements
 - Make `FreeProxies` more robust
 - Stop the misleading traceback error message #313
-## Bugfixes
+### Bugfix
 - Fix bug in exception handling #366
 ---
 ## Changes in v1.4.4
-## Bugfix
+### Bugfix
 - Fix a bug that would have prevented setting up ScraperAPI with exactly 1000 successful requests during the first week of the trial #356
-## Enhancement
+### Enhancement
 - Use FreeProxy instead of premium proxy servers when possible
 ---
 ## Changes in v1.4.3
-## Bugfix
+### Bugfixes
 - Fill the complete title of publications even if it appears truncated
 - Robustly handle exceptions when more than 20 coauthors of a scholar cannot be fetched
  ---
 ## Changes in v1.4.2
-## Bugfix
+### Bugfix
 - ScraperAPI proxy works reliably
  ---
 ## Changes in v1.4.0
-## Features
+### Features
 - Fetch the complete list of coauthors #322
 - Fetch all citeids for a given publication #324
 - Make scholarly objects inherently serializable #325
 - Expose scholarly specific exceptions #327
-## Bugfixes
+### Bugfixes
 - Test Tor on macOS and skip the test if tor is not installed #323
 - Get cites_id and citedby_url without having to fill the publication #328
 ---
 ## Changes in v1.3.0
-## Features
+### Features
 - Make the Author and Publication objects serializable
 - Make `cites_id` a list to allow for multiple values
 - Fetch all (more than 20) coauthors from a Scholar profile

requirements-dev.txt

@@ -1,3 +1,4 @@
 coverage
 flake8
+pandas
 sphinx_rtd_theme

scholarly/_scholarly.py

@@ -18,7 +18,7 @@
 _PUBSEARCH = '/scholar?hl=en&q={0}'
 _CITEDBYSEARCH = '/scholar?hl=en&cites={0}'
 _ORGSEARCH = "/citations?view_op=view_org&hl=en&org={0}"
-_MANDATES_URL = "https://scholar.google.com/citations?view_op=mandates_leaderboard_csv"
+_MANDATES_URL = "https://scholar.google.com/citations?view_op=mandates_leaderboard_csv&hl=en"
 
 
 class _Scholarly:
@@ -434,7 +434,7 @@ def pprint(self, object: Author or Publication)->None:
                     del publication['container_type']
 
         del to_print['container_type']
-        print(pprint.pformat(to_print))
+        print(pprint.pformat(to_print).encode("utf-8"))
 
     def search_org(self, name: str, fromauthor: bool = False) -> list:
         """
@@ -485,7 +485,7 @@ def download_mandates_csv(self, filename: str, overwrite: bool = False,
                               "setting overwrite=True")
         text = self.__nav._get_page(_MANDATES_URL, premium=False)
         if include_links:
-            soup = self.__nav._get_soup("/citations?view_op=mandates_leaderboard")
+            soup = self.__nav._get_soup("/citations?hl=en&view_op=mandates_leaderboard")
             text = text.replace("Funder,", "Funder,Policy,Cached,", 1)
             for agency in soup.find_all("td", class_="gsc_mlt_t"):
                 cached = agency.find("span", class_="gs_a").a["href"]

scholarly/author_parser.py

@@ -132,9 +132,9 @@ def _fill_public_access(self, soup, author):
         not_available = soup.find('div', class_='gsc_rsb_m_na')
         n_available, n_not_available = 0, 0
         if available:
-            n_available = int(available.text.split(" ")[0].replace(",", ""))
+            n_available = int(re.sub("[.,]", "", available.text.split(" ")[0]))
         if not_available:
-            n_not_available = int(not_available.text.split(" ")[0].replace(",", ""))
+            n_not_available = int(re.sub("[.,]", "", not_available.text.split(" ")[0]))
 
         author["public_access"] = PublicAccess(available=n_available,
                                                not_available=n_not_available)

scholarly/data_types.py

@@ -119,6 +119,7 @@ class BibEntry(TypedDict, total=False):
     :param number: NA number of a publication
     :param pages: range of pages
     :param publisher: The publisher's name
+    :param citation: Formatted citation string, usually containing journal name, volume and page numbers (source: AUTHOR_PUBLICATION_ENTRY)
     :param pub_url: url of the website providing the publication
     """
     pub_type: str
@@ -133,6 +134,7 @@ class BibEntry(TypedDict, total=False):
     number: str
     pages: str
     publisher: str
+    citation: str
 
 
 class Mandate(TypedDict, total=False):

scholarly/publication_parser.py

@@ -9,9 +9,9 @@
 _CITATIONPUB = '/citations?hl=en&view_op=view_citation&citation_for_view={0}'
 _SCHOLARPUB = '/scholar?hl=en&oi=bibs&cites={0}'
 _CITATIONPUBRE = r'citation_for_view=([\w-]*:[\w-]*)'
-_BIBCITE = '/scholar?q=info:{0}:scholar.google.com/\
+_BIBCITE = '/scholar?hl=en&q=info:{0}:scholar.google.com/\
 &output=cite&scirp={1}&hl=en'
-_CITEDBYLINK = '/scholar?cites={0}'
+_CITEDBYLINK = '/scholar?hl=en&cites={0}'
 _MANDATES_URL = '/citations?view_op=view_mandate&hl=en&citation_for_view={0}'
 
 _BIB_MAPPING = {
@@ -127,6 +127,13 @@ def _citation_pub(self, __data, publication: Publication):
                 and len(year.text) > 0):
             publication['bib']['pub_year'] = year.text.strip()
 
+        author_citation = __data.find_all('div', class_='gs_gray')
+        try:
+            citation = author_citation[1].text
+        except IndexError:
+            citation = ""
+        publication['bib']['citation'] = citation
+
         return publication
 
     def get_publication(self, __data, pubtype: PublicationSource)->Publication:

setup.py

@@ -5,7 +5,7 @@
 
 setuptools.setup(
     name='scholarly',
-    version='1.6.3',
+    version='1.7.0',
     author='Steven A. Cholewiak, Panos Ipeirotis, Victor Silva, Arun Kannawadi',
     author_email='steven@cholewiak.com, panos@stern.nyu.edu, vsilva@ualberta.ca, arunkannawadi@astro.princeton.edu',
     description='Simple access to Google Scholar authors and citations',