Removed redundant iterator constructors

Signed-off-by: Laurent Klock <Laurent.Klock@arhs-cube.com>

Author: klockla

service/src/main/java/crawlercommons/urlfrontier/service/AbstractFrontierService.java

@@ -936,8 +936,9 @@ public void listURLs(
         responseObserver.onCompleted();
     }
 
-    protected abstract Iterator<URLItem> urlIterator(
-            Entry<QueueWithinCrawl, QueueInterface> qentry);
+    protected Iterator<URLItem> urlIterator(Entry<QueueWithinCrawl, QueueInterface> qentry) {
+        return urlIterator(qentry, 0L, Long.MAX_VALUE);
+    }
 
     protected abstract Iterator<URLItem> urlIterator(
             Entry<QueueWithinCrawl, QueueInterface> qentry, long start, long max);

service/src/main/java/crawlercommons/urlfrontier/service/ignite/IgniteService.java

@@ -863,11 +863,4 @@ protected Iterator<URLItem> urlIterator(
             java.util.Map.Entry<QueueWithinCrawl, QueueInterface> qentry, long start, long max) {
         throw new UnsupportedOperationException("Feature not implemented for Ignite backend");
     }
-
-    @Override
-    // TODO Implementation of listURLs for Ignite
-    protected Iterator<URLItem> urlIterator(
-            java.util.Map.Entry<QueueWithinCrawl, QueueInterface> qentry) {
-        throw new UnsupportedOperationException("Feature not implemented for Ignite backend");
-    }
 }

service/src/main/java/crawlercommons/urlfrontier/service/memory/MemoryFrontierService.java

@@ -227,19 +227,15 @@ public Iterator<URLItem> urlIterator(
         return new MemoryURLItemIterator(qentry, start, maxURLs);
     }
 
-    public Iterator<URLItem> urlIterator(Entry<QueueWithinCrawl, QueueInterface> qentry) {
-        return new MemoryURLItemIterator(qentry);
-    }
-
     class MemoryURLItemIterator implements Iterator<URLItem> {
 
         private final org.slf4j.Logger LOG = LoggerFactory.getLogger(MemoryURLItemIterator.class);
 
         private final Entry<QueueWithinCrawl, QueueInterface> qentry;
         private final long start;
         private final long maxURLs;
-        private int pos = 0;
-        private int sent = 0;
+        private long pos = 0;
+        private long sent = 0;
         private URLItem.Builder builder = URLItem.newBuilder();
         private KnownURLItem.Builder knownBuilder = KnownURLItem.newBuilder();
         private Iterator<InternalURL> iter;
@@ -254,14 +250,6 @@ public MemoryURLItemIterator(
             iterCompleted = ((URLQueue) qentry.getValue()).getCompleted().iterator();
         }
 
-        public MemoryURLItemIterator(Entry<QueueWithinCrawl, QueueInterface> qentry) {
-            this.qentry = qentry;
-            this.start = 0;
-            this.maxURLs = Long.MAX_VALUE;
-            iter = ((URLQueue) qentry.getValue()).iterator();
-            iterCompleted = ((URLQueue) qentry.getValue()).getCompleted().iterator();
-        }
-
         @Override
         public boolean hasNext() {
             return sent < maxURLs && (iterCompleted.hasNext() || iter.hasNext());

service/src/main/java/crawlercommons/urlfrontier/service/rocksdb/RocksDBService.java

@@ -861,17 +861,13 @@ public Iterator<URLItem> urlIterator(
         return new RocksDBURLItemIterator(qentry, start, maxURLs);
     }
 
-    public Iterator<URLItem> urlIterator(Entry<QueueWithinCrawl, QueueInterface> qentry) {
-        return new RocksDBURLItemIterator(qentry);
-    }
-
     class RocksDBURLItemIterator implements Iterator<URLItem> {
 
         private final org.slf4j.Logger LOG = LoggerFactory.getLogger(RocksDBURLItemIterator.class);
 
         private final long maxURLs;
-        private int pos = 0;
-        private int sent = 0;
+        private long pos = 0;
+        private long sent = 0;
         private URLItem.Builder builder = URLItem.newBuilder();
         private KnownURLItem.Builder knownBuilder = KnownURLItem.newBuilder();
 
@@ -885,45 +881,35 @@ public RocksDBURLItemIterator(
 
             this.queueID = qentry.getKey();
             this.prefixKey = (queueID.toString() + "_").getBytes(StandardCharsets.UTF_8);
-            this.maxURLs = (int) maxURLs;
+            this.maxURLs = maxURLs;
             this.builder = URLItem.newBuilder();
             this.knownBuilder = KnownURLItem.newBuilder();
 
             this.rocksIterator = rocksDB.newIterator(columnFamilyHandleList.get(0));
             this.rocksIterator.seek(prefixKey);
 
-            // advance to the start position
-            while (rocksIterator.isValid() && pos < start) {
+            if (rocksIterator.isValid() && start == 0L) {
+                // Check if we're not past the seeked queue
                 final String currentKey = new String(rocksIterator.key(), StandardCharsets.UTF_8);
                 final QueueWithinCrawl Qkey = QueueWithinCrawl.parseAndDeNormalise(currentKey);
 
                 if (!queueID.equals(Qkey.getCrawlid(), Qkey.getQueue())) {
                     hasNext = false;
-                    break;
+                    return;
                 }
-                rocksIterator.next();
-                pos++;
-            }
-        }
-
-        public RocksDBURLItemIterator(Entry<QueueWithinCrawl, QueueInterface> qentry) {
-
-            this.queueID = qentry.getKey();
-            this.prefixKey = (queueID.toString() + "_").getBytes(StandardCharsets.UTF_8);
-            this.maxURLs = Long.MAX_VALUE;
-            this.builder = URLItem.newBuilder();
-            this.knownBuilder = KnownURLItem.newBuilder();
-
-            this.rocksIterator = rocksDB.newIterator(columnFamilyHandleList.get(0));
-            this.rocksIterator.seek(prefixKey);
-
-            if (rocksIterator.isValid()) {
-                // Check if we're not past the seeked queue
-                final String currentKey = new String(rocksIterator.key(), StandardCharsets.UTF_8);
-                final QueueWithinCrawl Qkey = QueueWithinCrawl.parseAndDeNormalise(currentKey);
+            } else {
+                // advance to the start position
+                while (rocksIterator.isValid() && pos < start) {
+                    final String currentKey =
+                            new String(rocksIterator.key(), StandardCharsets.UTF_8);
+                    final QueueWithinCrawl Qkey = QueueWithinCrawl.parseAndDeNormalise(currentKey);
 
-                if (!queueID.equals(Qkey.getCrawlid(), Qkey.getQueue())) {
-                    hasNext = false;
+                    if (!queueID.equals(Qkey.getCrawlid(), Qkey.getQueue())) {
+                        hasNext = false;
+                        break;
+                    }
+                    rocksIterator.next();
+                    pos++;
                 }
             }
         }

service/src/main/java/crawlercommons/urlfrontier/service/rocksdb/ShardedRocksDBService.java

@@ -106,11 +106,4 @@ protected Iterator<URLItem> urlIterator(
         throw new UnsupportedOperationException(
                 "Feature not implemented for ShardedRocksDB backend");
     }
-
-    @Override
-    // TODO Implementation of urlIterator for ShardedRocksDB
-    protected Iterator<URLItem> urlIterator(Entry<QueueWithinCrawl, QueueInterface> qentry) {
-        throw new UnsupportedOperationException(
-                "Feature not implemented for ShardedRocksDB backend");
-    }
 }