remove oeis fall-back option (#148)

Author: ckrause

CHANGELOG.md

@@ -5,6 +5,7 @@ To install or update LODA, please follow the [installation instructions](https:/
 ### Bugfixes
 
 * Extend folder locking time out value on Windows
+* Remove fall-back download from OEIS
 
 ### Features
 
@@ -13,7 +14,7 @@ To install or update LODA, please follow the [installation instructions](https:/
 ### Enhancements
 
 * Change default update interval to 3 days
-* Throttle download of OEIS files from API server
+* Throttling and exponential backoff for API server downloads
 
 # v22.5.20
 

src/api_client.cpp

@@ -72,7 +72,7 @@ void ApiClient::postCPUHour() {
   }
 }
 
-bool ApiClient::getOeisFile(const std::string& filename,
+void ApiClient::getOeisFile(const std::string& filename,
                             const std::string& local_path) {
   // throttling
   if (fetched_oeis_files > 2) {
@@ -91,13 +91,26 @@ bool ApiClient::getOeisFile(const std::string& filename,
 
   // fetch file
   const std::string url = BASE_URL + "oeis/" + filename + ".gz";
-  const bool success = WebClient::get(url, local_path + ".gz", false, false);
+  bool success = false;
+  int64_t backoff_delay = OEIS_THROTTLING_SECS;
+  for (int64_t i = 0; i < 5; i++) {
+    if (i > 0) {
+      Log::get().warn("Retrying fetch of " + url);
+    }
+    success = WebClient::get(url, local_path + ".gz", false, false);
+    if (success) {
+      break;
+    }
+    std::this_thread::sleep_for(std::chrono::seconds(backoff_delay));
+    backoff_delay *= 2;
+  }
   if (success) {
     gunzip(local_path + ".gz");
     fetched_oeis_files++;
     last_oeis_time = std::chrono::steady_clock::now();
+  } else {
+    Log::get().error("Error fetching " + url, true);
   }
-  return success;
 }
 
 bool ApiClient::getProgram(int64_t index, const std::string& path) {

src/include/api_client.hpp

@@ -18,7 +18,7 @@ class ApiClient {
 
   void postCPUHour();
 
-  bool getOeisFile(const std::string& filename, const std::string& local_path);
+  void getOeisFile(const std::string& filename, const std::string& local_path);
 
   Program getNextProgram();
 

src/oeis_manager.cpp

@@ -325,12 +325,7 @@ void OeisManager::update() {
     std::string cmd, path;
     for (auto &file : files) {
       path = Setup::getOeisHome() + file;
-      if (!ApiClient::getDefaultInstance().getOeisFile(file, path)) {
-        Log::get().warn("Cannot fetch " + file +
-                        ".gz from API server, falling back to OEIS");
-        WebClient::get("https://oeis.org/" + file + ".gz", path + ".gz");
-        gunzip(path + ".gz");
-      }
+      ApiClient::getDefaultInstance().getOeisFile(file, path);
     }
     // update programs repository using git pull
     auto mode = Setup::getMiningMode();

src/oeis_sequence.cpp

@@ -211,12 +211,7 @@ Sequence OeisSequence::getTerms(int64_t max_num_terms) const {
           big_file.peek() == std::ifstream::traits_type::eof()) {
         ensureDir(path);
         std::remove(path.c_str());
-        if (!ApiClient::getDefaultInstance().getOeisFile(id_str("b") + ".txt",
-                                                         path)) {
-          Log::get().warn(
-              "Cannot fetch b-file from API server, falling back to OEIS");
-          WebClient::get(url_str() + "/" + id_str("b") + ".txt", path);
-        }
+        ApiClient::getDefaultInstance().getOeisFile(id_str("b") + ".txt", path);
         big = loadBFile(id, terms);
       }
     }