Bug fix on commit change (#3)

hf_hub_download returns a non existing file path if a file has been previous download and now the repo is on a new commit

Author: agonzc34

include/huggingface_hub.h

@@ -93,7 +93,7 @@ get_model_metadata_from_hf(const std::string &repo, const std::string &file);
 struct DownloadResult
 hf_hub_download(const std::string &repo_id, const std::string &filename,
                 const std::string &cache_dir = "~/.cache/huggingface/hub",
-                bool force_download = false);
+                bool force_download = false, bool verbose = false);
 
 /**
  * @brief Download a file from Hugging Face Hub.

src/huggingface_hub.cpp

@@ -41,6 +41,16 @@ namespace huggingface_hub {
 volatile sig_atomic_t stop_download = 0;
 void handle_sigint(int) { stop_download = 1; }
 
+bool log_verbose = false;
+
+void log_debug(const std::string &message) {
+  if (!log_verbose) {
+    return;
+  }
+  fprintf(stderr, "[DEBUG] %s\n", message.c_str());
+  fflush(stderr);
+}
+
 void log_info(const std::string &message) {
   fprintf(stderr, "[INFO] %s\n", message.c_str());
   fflush(stderr);
@@ -246,11 +256,55 @@ int progress_callback(void *userdata, curl_off_t total, curl_off_t now,
   return 0; // Continue downloading
 }
 
+CURLcode perform_download(std::string url,
+                          std::string blob_incomplete_file_path,
+                          bool force_download, struct FileMetadata metadata) {
+  CURL *curl = curl_easy_init();
+  if (!curl) {
+    return CURLE_FAILED_INIT;
+  }
+
+  std::ofstream file(blob_incomplete_file_path,
+                     std::ios::binary | std::ios::app);
+
+  if (!file.is_open()) {
+    log_error("Error: failed to open file stream!");
+    return CURLE_FAILED_INIT;
+  }
+
+  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());   // Set URL
+  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // Follow redirects
+  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
+                   write_file_data);                // Write data to file
+  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &file); // File stream
+  curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);   // Enable progress callback
+  curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION,
+                   progress_callback); // Progress callback
+  curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &metadata);
+
+  // Resume download if file exists
+  long existing_size = get_file_size(blob_incomplete_file_path);
+  if (existing_size > 0 && !force_download) {
+    curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE,
+                     (curl_off_t)existing_size);
+    log_info("Resuming download from " + std::to_string(existing_size) +
+             " bytes...");
+  }
+
+  fprintf(stderr, "\n"); // New line after progress bar
+  CURLcode res = curl_easy_perform(curl);
+  fprintf(stderr, "\n"); // New line after progress bar
+  curl_easy_cleanup(curl);
+  file.close();
+  return res;
+}
+
 struct DownloadResult hf_hub_download(const std::string &repo_id,
                                       const std::string &filename,
                                       const std::string &cache_dir,
-                                      bool force_download) {
+                                      bool force_download, bool verbose) {
   signal(SIGINT, handle_sigint);
+  log_verbose = verbose;
 
   struct DownloadResult result;
   result.success = true;
@@ -265,13 +319,13 @@ struct DownloadResult hf_hub_download(const std::string &repo_id,
 
   // 2. Create Cache Dir Struct
   std::string cache_model_dir = create_cache_system(cache_dir, repo_id);
-  log_info("Cache directory: " + cache_model_dir);
+  log_debug("Cache directory: " + cache_model_dir);
   log_info("Downloading " + filename + " from " + repo_id);
 
   struct FileMetadata metadata = std::get<struct FileMetadata>(metadata_result);
-  log_info("SHA256: " + metadata.sha256);
-  log_info("Commit: " + metadata.commit);
-  log_info("Size: " + std::to_string(metadata.size) + " bytes");
+  log_debug("SHA256: " + metadata.sha256);
+  log_debug("Commit: " + metadata.commit);
+  log_debug("Size: " + std::to_string(metadata.size) + " bytes");
 
   std::filesystem::path blob_file_path(cache_model_dir + "blobs/" +
                                        metadata.sha256);
@@ -283,8 +337,9 @@ struct DownloadResult hf_hub_download(const std::string &repo_id,
 
   result.path = snapshot_file_path;
 
-  if (std::filesystem::exists(blob_file_path) && !force_download) {
-    log_info("Blob file exists. Skipping download...");
+  if (std::filesystem::exists(snapshot_file_path) &&
+      std::filesystem::exists(blob_file_path) && !force_download) {
+    log_info("Snapshot file exists. Skipping download...");
     return result;
   }
 
@@ -300,77 +355,35 @@ struct DownloadResult hf_hub_download(const std::string &repo_id,
   }
 
   // 3. Download the file
-  CURL *curl = curl_easy_init();
-  if (!curl) {
-    result.success = false;
-    return result;
-  }
-
   std::string url =
       "https://huggingface.co/" + repo_id + "/resolve/main/" + filename;
-
-  std::ofstream file(blob_incomplete_file_path,
-                     std::ios::binary | std::ios::app);
-
-  if (!file.is_open()) {
-    log_error("Failed to open file: " + blob_incomplete_file_path.string());
-    curl_easy_cleanup(curl);
-    result.success = false;
-    return result;
-  }
-
-  curl_easy_setopt(curl, CURLOPT_URL, url.c_str());   // Set URL
-  curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); // Follow redirects
-  curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION,
-                   write_file_data);                // Write data to file
-  curl_easy_setopt(curl, CURLOPT_WRITEDATA, &file); // File stream
-  curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0L);   // Enable progress callback
-  curl_easy_setopt(curl, CURLOPT_XFERINFOFUNCTION,
-                   progress_callback); // Progress callback
-  curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, &metadata);
-
-  // Resume download if file exists
-  long existing_size = get_file_size(blob_incomplete_file_path);
-  if (existing_size > 0 && !force_download) {
-    curl_easy_setopt(curl, CURLOPT_RESUME_FROM_LARGE,
-                     (curl_off_t)existing_size);
-    log_info("Resuming download from " + std::to_string(existing_size) +
-             " bytes...");
-  }
-
-  fprintf(stderr, "\n");
-  CURLcode res = curl_easy_perform(curl);
-  fprintf(stderr, "\n");
-
   std::filesystem::create_directories(snapshot_file_path.parent_path());
 
-  if (stop_download) {
-    log_info("Download interrupted. Exiting...");
-    file.close();
-    curl_easy_cleanup(curl);
-    result.success = false;
-    return result;
-  } else if (res != CURLE_OK) {
-    log_error("CURL request failed: " + std::string(curl_easy_strerror(res)));
-    file.close();
-    curl_easy_cleanup(curl);
-    result.success = false;
-    return result;
+  if (!std::filesystem::exists(blob_file_path) || force_download) {
+    CURLcode res = perform_download(url, blob_incomplete_file_path,
+                                    force_download, metadata);
+    result.success = res == CURLE_OK;
+
+    if (stop_download) {
+      log_info("Download interrupted. Exiting...");
+      return result;
+    } else if (!result.success) {
+      log_error("CURL request failed: " + std::string(curl_easy_strerror(res)));
+      return result;
+    } else {
+      std::filesystem::rename(blob_incomplete_file_path, blob_file_path);
+    }
   }
 
   if (std::filesystem::exists(snapshot_file_path)) {
-    log_info("Snapshot file exists. Deleting...");
+    log_debug("Snapshot file exists. Deleting...");
     std::filesystem::remove(snapshot_file_path);
   }
-  std::filesystem::rename(blob_incomplete_file_path, blob_file_path);
   std::filesystem::create_symlink(blob_file_path, snapshot_file_path);
 
-  file.close();
-  curl_easy_cleanup(curl);
-
   log_info("Downloaded to: " + snapshot_file_path.string());
 
-  result.success = res == CURLE_OK;
+  result.success = true;
   return result;
 }