feat: Respect Cache-Control: max-age header

This change implements support for the `Cache-Control: max-age`
HTTP header to avoid fetching feeds that have not expired.

The `Feed` class now stores `last_checked` and `max_age`
attributes. Before fetching a feed, the application checks if the
cached version is still valid based on these attributes. If the
cache is still fresh, the network request is skipped.

After a successful fetch, the `Cache-Control` header is parsed
from the response, and the `last_checked` and `max_age`
attributes are updated. This ensures that subsequent fetches will
respect the cache duration specified by the feed provider.

This commit adds a test case to verify that the `Cache-Control:
max-age` header is respected.

The test starts a webserver that serves a feed with a `max-age`
value, runs the feed to populate the cache, and then checks that
the feed is not fetched again before the `max-age` expires, and
is fetched again after it expires.

Fixes: rss2email#286

Author: mcepl

rss2email/feed.py

@@ -171,6 +171,8 @@ class Feed (object):
         'etag',
         'modified',
         'seen',
+        'max_age',
+        'last_checked',
         ]
 
     ## saved/loaded from ConfigParser instance
@@ -352,6 +354,8 @@ def reset(self):
         self.etag = None
         self.modified = None
         self.seen = {} # type: Dict[str, Dict[str, Any]]
+        self.max_age = None
+        self.last_checked = None
 
     def _set_name(self, name):
         if not self._name_regexp.match(name):
@@ -369,6 +373,13 @@ def _fetch(self):
         >>> parsed.status
         200
         """
+        if (self.max_age is not None and self.last_checked is not None and
+            _time.time() < self.last_checked + self.max_age):
+            _LOG.info('skipping {}: cache has not expired'.format(self.name))
+            parsed = _feedparser.FeedParserDict()
+            parsed['status'] = 304
+            return parsed
+
         _LOG.info('fetch {}'.format(self))
         if not self.url:
             raise _error.InvalidFeedConfig(setting='url', feed=self)
@@ -937,6 +948,17 @@ def run(self, send=True, clean=False):
             self.modified = None
         parsed = self._fetch()
 
+        if parsed.status != 304:
+            self.last_checked = _time.time()
+            if 'cache-control' in parsed.headers:
+                match = _re.search(r'max-age=(\d+)', parsed.headers['cache-control'])
+                if match:
+                    self.max_age = int(match.group(1))
+                else:
+                    self.max_age = None
+            else:
+                self.max_age = None
+
         if clean and len(parsed.entries) > 0:
             for guid in self.seen:
                 self.seen[guid]['old'] = True

test/test_main.py

@@ -249,6 +249,24 @@ def webserver_for_test_if_fetch(queue, timeout):
     finally:
         httpd.server_close()
 
+def webserver_for_test_cache_control(queue, max_age):
+    class CacheControlHandler(NoLogHandler):
+        def do_GET(self):
+            self.send_response(200)
+            self.send_header('Cache-Control', 'max-age={}'.format(max_age))
+            self.end_headers()
+            with open(_os.path.join(test_dir, 'disqus/feed.rss'), 'rb') as f:
+                self.wfile.write(f.read())
+
+    httpd = http.server.HTTPServer(('', 0), CacheControlHandler)
+    try:
+        port = httpd.server_address[1]
+        queue.put(port)
+        while queue.get() != "stop":
+            httpd.handle_request()
+    finally:
+        httpd.server_close()
+
 class TestFetch(unittest.TestCase):
     "Retrieving feeds from servers"
     def test_delay(self):
@@ -379,6 +397,39 @@ def test_only_new(self):
                 self.assertIn("seen", content["feeds"][0])
         self.assertEqual(queue.get(), "done")
 
+    def test_cache_control(self):
+        "Respects Cache-Control: max-age header"
+        max_age = 5
+        cfg = """[DEFAULT]
+        to = example@example.com"""
+
+        queue = multiprocessing.Queue()
+        webserver_proc = multiprocessing.Process(target=webserver_for_test_cache_control, args=(queue, max_age))
+        webserver_proc.start()
+        port = queue.get()
+
+        with ExecContext(cfg) as ctx:
+            ctx.call("add", 'test', 'http://127.0.0.1:{port}/disqus/feed.rss'.format(port = port))
+
+            # First run, should fetch
+            queue.put("next")
+            p = ctx.call("run", "--no-send")
+            self.assertIn("fetch", p.stderr)
+
+            # Second run, should be cached
+            p = ctx.call("run", "--no-send")
+            self.assertIn("cache has not expired", p.stderr)
+
+            # Wait for cache to expire
+            time.sleep(max_age + 1)
+
+            # Third run, should fetch again
+            queue.put("next")
+            p = ctx.call("run", "--no-send")
+            self.assertIn("fetch", p.stderr)
+
+        queue.put("stop")
+
 
 def webserver_for_test_send(queue):
     httpd = http.server.HTTPServer(('', 0), NoLogHandler)