tqdm implemented

Signed-off-by: Finbarrs Oketunji <f@finbarrs.eu>

Author: 0xnu

VERSION

@@ -1 +1 @@
-1.0.5
+1.0.6

amazon_scraper/scraper.py

@@ -4,6 +4,7 @@
 import json
 import random
 import argparse
+from tqdm import tqdm
 
 class AmazonScraper:
     user_agents = [
@@ -52,56 +53,48 @@ def start_scraping(self):
         self.product_writer.writerow(["product_name", "product_images", "number_of_reviews", "price", "product_url", "asin"])
         if self.review:
             self.review_writer.writerow(["product_name", "product_reviews", "product_url", "asin"])
-        for page in range(1, self.pages + 1):
+        for page in tqdm(range(1, self.pages + 1), desc="Scraping Pages"):
             url = self.url + "&page=" + str(page)
             headers = {"User-Agent": random.choice(self.user_agents)}
             response = requests.get(url, headers=headers)
             soup = BeautifulSoup(response.content, "html.parser")
             products = soup.find_all("div", {"class": "sg-col-inner"})
 
             for product in products:
-                # Product name
                 name = product.find("span", {"class": "a-size-medium a-color-base a-text-normal"})
                 if name is not None:
                     name = name.text
                 else:
-                    continue  # Skip if no product name
+                    continue
 
-                # Product images
                 images = product.find_all("img", {"class": "s-image"})
                 if images is not None:
                     images = [image['src'] for image in images]
                 else:
                     images = []
 
-                # Number of Reviews
                 number_of_reviews = product.find("span", {"class": "a-size-base"})
                 if number_of_reviews is not None:
                     number_of_reviews = number_of_reviews.text
                 else:
                     number_of_reviews = ''
 
-                # Price
                 price = product.find("span", {"class": "a-offscreen"})
                 if price is not None:
                     price = price.text
                 else:
                     price = ''
 
-                # Product URL
                 product_url = product.find("a", {"class": "a-link-normal"})
                 if product_url is not None:
                     product_url = f'https://www.amazon.{self.locale}' + product_url['href']
                 else:
                     product_url = ''
 
-                # ASIN
                 asin = product_url.split("/dp/")[1].split("/")[0] if "/dp/" in product_url else ''
 
-                # Write to CSV
                 self.product_writer.writerow([name, ", ".join(images), number_of_reviews, price, product_url, asin])
 
-                # Add to JSON data
                 self.product_json_data.append({
                     "product_name": name,
                     "product_images": images,
@@ -111,7 +104,6 @@ def start_scraping(self):
                     "asin": asin
                 })
 
-                # Product reviews
                 if self.review:
                     product_reviews = []
                     review_url = f'https://www.amazon.{self.locale}/product-reviews/{asin}'
@@ -121,10 +113,8 @@ def start_scraping(self):
                     for review in reviews:
                         product_reviews.append(review.text.strip())
 
-                    # Write to CSV
                     self.review_writer.writerow([name, ", ".join(product_reviews), product_url, asin])
 
-                    # Add to JSON data
                     self.review_json_data.append({
                         "product_name": name,
                         "product_reviews": product_reviews,

amazon_scraper/version.py

@@ -1 +1 @@
-VERSION = "1.0.5"
+VERSION = "1.0.6"

requirements.txt

@@ -1,2 +1,3 @@
 beautifulsoup4
-requests
+requests
+tqdm

setup.py

@@ -5,7 +5,7 @@
 from distutils.core import Extension
 
 NAME = "amazon_scrape"
-VERSION = "1.0.5"
+VERSION = "1.0.6"
 REQUIRES = ["beautifulsoup4", "requests"]
 
 # read the contents of your README file