Using json-rpc instead XMLRPC

Author: mosquito

pypi_server/__init__.py

@@ -9,7 +9,7 @@
     ("Dmitry Orlov", "me@mosquito.su")
 ]
 
-version_info = (0, 3, 30)
+version_info = (0, 4, '0dev')
 
 __version__ = ".".join(map(str, version_info))
 __author__ = ", ".join("{0} <{1}>".format(*author) for author in author_info)

pypi_server/handlers/pypi/proxy/client.py

@@ -1,5 +1,6 @@
 # encoding: utf-8
 import hashlib
+import json
 import logging
 from copy import copy
 from slimurl import URL
@@ -16,6 +17,26 @@
 log = logging.getLogger(__name__)
 
 
+def async_retrying(number, exceptions=(Exception,)):
+    def decorator(func):
+        @coroutine
+        def wrap(*args, **kwargs):
+            last_exc = None
+            for i in range(number):
+                try:
+                    raise Return((yield func(*args, **kwargs)))
+                except Return:
+                    raise
+                except exceptions as e:
+                    log.exception("Error on attempt: %r", i)
+                    last_exc = e
+
+            if last_exc:
+                raise last_exc
+        return wrap
+    return decorator
+
+
 def normalize_package_name(name):
     return name.lower().replace("_", "-").replace(".", "-")
 
@@ -26,19 +47,20 @@ class PYPIClient(object):
     THREAD_POOL = None
     INDEX = None
     XMLRPC = None
+    RPC_URL = None
     LOCK = None
 
     @classmethod
     def configure(cls, backend, thread_pool):
         cls.CLIENT = AsyncHTTPClient(io_loop=IOLoop.current())
         cls.BACKEND = backend
         cls.THREAD_POOL = thread_pool
-        cls.XMLRPC = ServerProxy(
-            str(copy(backend)(path="/pypi")),
-        )
+        cls.RPC_URL = copy(backend)(path="/pypi")
+        cls.XMLRPC = ServerProxy(str(cls.RPC_URL))
         cls.LOCK = Lock()
 
     @classmethod
+    @async_retrying(5)
     @coroutine
     @Cache(HOUR, files_cache=True, ignore_self=True)
     def packages(cls):
@@ -54,6 +76,7 @@ def packages(cls):
             raise Return(index)
 
     @classmethod
+    @async_retrying(5)
     @coroutine
     @Cache(4 * HOUR, files_cache=True, ignore_self=True)
     def search(cls, names, descriptions, operator="or"):
@@ -62,6 +85,7 @@ def search(cls, names, descriptions, operator="or"):
         raise Return(result)
 
     @classmethod
+    @async_retrying(5)
     @coroutine
     def exists(cls, name):
         try:
@@ -76,6 +100,7 @@ def exists(cls, name):
         raise Return(True)
 
     @classmethod
+    @async_retrying(5)
     @coroutine
     def find_real_name(cls, name):
         if not options.pypi_proxy:
@@ -92,6 +117,7 @@ def find_real_name(cls, name):
         raise Return(real_name)
 
     @classmethod
+    @async_retrying(5)
     @coroutine
     @Cache(4 * HOUR, files_cache=True, ignore_self=True)
     def releases(cls, name):
@@ -119,15 +145,20 @@ def releases(cls, name):
         raise Return(set(res))
 
     @classmethod
+    @async_retrying(5)
     @coroutine
     @Cache(MONTH, files_cache=True, ignore_self=True)
     def release_data(cls, name, version):
-        info, files = yield [
-            cls.XMLRPC.release_data(str(name), str(version)),
-            cls.XMLRPC.release_urls(str(name), str(version))
-        ]
+        url = copy(cls.RPC_URL)
+        url.path_append(str(name), str(version), 'json')
+        log.info("Gathering info %s", url)
+
+        response = json.loads((yield cls.CLIENT.fetch(str(url))).body)
+        info = response['info']
+        files = response['urls']
 
         download_url = info.get('download_url')
+
         if download_url and not files:
             try:
                 url = URL(download_url)

pypi_server/handlers/pypi/simple/files.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-from pypi_server.cache import Cache, HOUR
+from pypi_server.cache import Cache
 from pypi_server.timeit import timeit
 from tornado.gen import coroutine, Return
 from tornado.ioloop import IOLoop
@@ -100,8 +100,11 @@ def release_db_save(package, rel, version_info, release_files):
 @timeit
 @coroutine
 def release_fetch(package, rel):
-    version_info, release_files = yield PYPIClient.release_data(package.name, rel)
-    raise Return((yield release_db_save(package, rel, version_info, release_files)))
+    try:
+        version_info, release_files = yield PYPIClient.release_data(package.name, rel)
+        raise Return((yield release_db_save(package, rel, version_info, release_files)))
+    except:
+        log.exception("Error proccess %r %r", package, rel)
 
 
 @route(r'/simple/(?P<package>[\w\.\d\-\_]+)/?')