Merge pull request #115 from nexB/auth-gh

Use authenticated requests for GitHub REST API calls

Author: keshav-space

.gitignore

@@ -72,3 +72,6 @@ tcl
 
 # Ignore Jupyter Notebook related temp files
 .ipynb_checkpoints/
+
+# Env Files
+.env

CHANGELOG.rst

@@ -2,6 +2,11 @@ Changelog
 =========
 
 
+v0.5.1
+-------
+- Use authenticated requests for GitHub REST API calls
+
+
 v0.5.0
 -------
 - FetchCode now supports retrieving package info for following generic packages: 

src/fetchcode/package.py

@@ -472,7 +472,9 @@ class UtilLinuxDirectoryListedSource(DirectoryListedSource):
     source_url = "https://mirrors.edge.kernel.org/pub/linux/utils/util-linux/"
     is_nested = True
     # Source archive ex: util-linux-1.2.3.tar.gz
-    source_archive_regex = re.compile(r"^(util-linux-)(?P<version>[\w.-]*)(.tar.gz)$")
+    source_archive_regex = re.compile(
+        r"^(util-linux-|util-linux-ng-)(?P<version>[\w.-]*)(.tar.gz)$"
+    )
     ignored_files_and_dir = []
 
 
@@ -666,7 +668,7 @@ class BareboxDirectoryListedSource(DirectoryListedSource):
 
 
 class LinuxDirectoryListedSource(DirectoryListedSource):
-    source_url = "https://cdn.kernel.org/pub/linux/kernel/"
+    source_url = "https://mirrors.edge.kernel.org/pub/linux/kernel/"
     # Source archive ex: linux-1.2.3.tar.gz
     source_archive_regex = re.compile(r"^(linux-)(?P<version>[\w.-]*)(.tar.gz)$")
     is_nested = True

src/fetchcode/package_util.py

@@ -55,7 +55,7 @@ def get_default_package(cls, purl):
         namespace = purl.namespace
         base_path = "https://api.github.com/repos"
         api_url = f"{base_path}/{namespace}/{name}"
-        response = utils.get_response(api_url)
+        response = utils.get_github_rest(api_url)
         homepage_url = response.get("homepage")
         vcs_url = response.get("git_url")
         github_url = "https://github.com"
@@ -124,6 +124,12 @@ def _get_github_packages(purl, version_regex, ignored_tag_regex, default_package
             version = tag
 
         version = version.strip("Vv").strip()
+        if "+" in version:
+            first, last = version.split("+")
+            first.replace("_", ".")
+            version = f"{first}+{last}"
+        else:
+            version = version.replace("_", ".")
         if not version or not version[0].isdigit():
             continue
 
@@ -227,7 +233,7 @@ def get_package_info(cls, gh_purl):
             package_dict["type"] = "openssl"
             package_dict["namespace"] = None
             package_dict["name"] = "openssl"
-            package_dict["version"] = package_dict["version"].replace("_", ".")
+            package_dict["version"] = package_dict["version"]
 
             yield package_from_dict(package_dict)
 
@@ -251,7 +257,7 @@ def get_package_info(cls, gh_purl):
             package_dict["type"] = "generic"
             package_dict["namespace"] = None
             package_dict["name"] = "erofs-utils"
-            package_dict["version"] = package_dict["version"].replace("_", ".")
+            package_dict["version"] = package_dict["version"]
 
             yield package_from_dict(package_dict)
 
@@ -279,7 +285,7 @@ def get_package_info(cls, gh_purl, package_name):
             package_dict["type"] = "generic"
             package_dict["namespace"] = None
             package_dict["name"] = package_name
-            package_dict["version"] = package_dict["version"].replace("_", ".")
+            package_dict["version"] = package_dict["version"]
 
             yield package_from_dict(package_dict)
 

src/fetchcode/utils.py

@@ -118,13 +118,18 @@ class GraphQLError(Exception):
     pass
 
 
-def github_response(graphql_query):
+def get_github_token():
     gh_token = os.environ.get("GH_TOKEN", None)
     if not gh_token:
         from dotenv import load_dotenv
 
         load_dotenv()
         gh_token = os.environ.get("GH_TOKEN", None)
+    return gh_token
+
+
+def github_response(graphql_query):
+    gh_token = get_github_token()
 
     if not gh_token:
         msg = (
@@ -149,11 +154,22 @@ def github_response(graphql_query):
     return response
 
 
-def get_response(url):
+def get_github_rest(url):
+    headers = None
+    gh_token = get_github_token()
+    if gh_token:
+        headers = {
+            "Authorization": f"Bearer {gh_token}",
+        }
+
+    return get_response(url, headers)
+
+
+def get_response(url, headers=None):
     """
     Generate `Package` object for a `url` string
     """
-    resp = requests.get(url)
+    resp = requests.get(url, headers=headers)
     if resp.status_code == 200:
         return resp.json()