Version 2.1.1 (#44)

Improves date parsing, making it work even when pt-br locale is not installed on the user's machine

Author: rafpyprog

.gitignore

@@ -1,5 +1,7 @@
 __pycache__/
 .cache/
+.ipynb_checkpoints/
+.pytest_cache/
 .mypy_cache/
 .vscode/
 dist/

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup
 
 
-__version__ = '2.1.0'
+__version__ = "2.1.1"
 
 
 with open('README.rst', 'r', encoding='utf-8') as f:
@@ -19,9 +19,11 @@
 dev_requirements = [
     "bandit==1.6.0",
     "codecov==2.0.15",
+    "mypy==0.910",
     "pytest==4.6.2",
     "pytest-cov==2.7.1",
-    "pytest-mypy>=0.3.2"
+    "pytest-mypy>=0.8.1",
+    "types-requests>=2.25.6"
 ]
 
 setup(

sgs/__version__.py

@@ -1 +1 @@
-__version__ = "2.0.3"
+__version__ = "2.1.1"

sgs/common.py

@@ -1,46 +1,45 @@
 """
 Shared functions.
 """
-from datetime import datetime
+import functools
 import locale
+import os
 import re
+from datetime import datetime
 from typing import Union
-import os
 
 
 LRU_CACHE_SIZE = 32
 MAX_ATTEMPT_NUMBER = 5
 
 
+@functools.lru_cache(maxsize=365)
 def to_datetime(date_string: str, language: str) -> Union[datetime, str]:
     """ Converts a date string to a datetime object """
-    locales = {"pt": "pt_BR.utf-8", "en": "en_US.utf-8"}
-
-    """ correct problem with locale in Windows platform """
-    if os.name == 'nt':
-        locales = {"pt": "Portuguese_Brazil.1252", "en": "Portuguese_Brazil.1252"}
-    
-    locale.setlocale(locale.LC_TIME, locales[language])
-
-    dd_mm_aaaa = "%d/%m/%Y"
-    mmm_aaaa = "%b/%Y"
 
-    formats = [dd_mm_aaaa, mmm_aaaa]
-
-    for fmt in formats:
-        try:
-            date = datetime.strptime(date_string, fmt)
-            break
-        except ValueError:
-            continue
+    yyyy = "[0-9]{4}"
+    mmm_yyyy = r"[a-z]{3}/[0-9]{4}"
+
+    if re.match(yyyy, date_string):
+        date = datetime(int(date_string), 12, 31)
+    elif re.match(mmm_yyyy, date_string):
+        month_text, year_text = date_string.split("/")
+        months = [
+            ("jan", "jan"), ("fev", "feb"), ("mar", "mar"),
+            ("abr", "apr"), ("mai", "may"), ("jun", "jun"),
+            ("jul", "jul"), ("ago", "aug"), ("set", "sep"),
+            ("out", "oct"), ("nov", "nov"), ("dez", "dec")
+        ]
+        for n, month_names in enumerate(months, 1):
+            if month_text in month_names:
+                month_number = n
+                break
+        date = datetime(int(year_text), month_number, 1)
     else:
-        yyyy = "[0-9]{4}"
-        if re.match(yyyy, date_string):
-            year = int(date_string)
-            month = 12
-            day = 31
+        try:
+            day, month, year = [int(date_part) for date_part in date_string.split("/")]
             date = datetime(year, month, day)
-        else:
-            return date_string  # returns original value if cant parse
-
+        except ValueError:
+            # return the original value if we cant parse it
+            return date_string
     return date

sgs/search.py

@@ -90,16 +90,16 @@ def parse_search_response(response, language: str) -> Optional[list]:
             cols["source"]: "source",
         }
         df.rename(columns=col_names, inplace=True)
-        cols = [
+        cols_names = [
             "code",
             "name",
             "unit",
             "frequency",
             "first_value",
             "last_value",
             "source",
-        ]
-        df = df[cols]
+        ]  # type: ignore
+        df = df[cols_names]
     except (IndexError, KeyError):
         return None
     else:
@@ -170,7 +170,7 @@ def search_ts(query: Union[int, str], language: str) -> Optional[list]:
         params["texto"] = query
         params["hdTipoPesquisa"] = 6
 
-    response = session.post(url, params=params, timeout=10)
+    response = session.post(url, params=params, timeout=10)  # type: ignore
     response.raise_for_status()
 
     results = parse_search_response(response, language)

sgs/ts.py

@@ -34,12 +34,12 @@ def time_serie(ts_code: int, start: str, end: str, strict: bool = False) -> pd.S
         2018-01-05    0.026444
         2018-01-08    0.026444
     """
-    
+
     if strict:
         ts_data = api.get_data_with_strict_range(ts_code, start, end)
     else:
         ts_data = api.get_data(ts_code, start, end)
-        
+
     values = []
     index = []
     for i in ts_data:
@@ -48,4 +48,4 @@ def time_serie(ts_code: int, start: str, end: str, strict: bool = False) -> pd.S
 
     # Transform empty strings in null values
     values = [np.nan if value == "" else value for value in values]
-    return pd.Series(values, index, name=ts_code, dtype=np.float)
+    return pd.Series(values, index, name=ts_code, dtype=np.float64)  # type: ignore

tests/test_api.py

@@ -10,7 +10,8 @@ def test_get_data():
     data = api.get_data(4, "02/01/2018", "31/01/2018")
     assert isinstance(data, list)
     assert len(data) == NUMBER_OF_LINES
-    
+
+
 @pytest.mark.api
 def test_get_data_with_strict_range():
     NUMBER_OF_LINES = 0

tests/test_ts.py

@@ -8,23 +8,23 @@
 def test_time_serie():
     ts = time_serie(4, "02/01/2018", "31/01/2018")
     assert len(ts) == 20
-    assert ts.dtype == np.float
+    assert ts.dtype == np.float64
 
 @pytest.mark.ts
 def test_ts_with_null_values():
     # Issue #28
     ts = time_serie(21554, start="31/12/1992", end="01/06/2019")
-    data = ts.loc['1994-04-01']    
+    data = ts.loc['1994-04-01']
     assert np.isnan(data) == True
-   
+
 @pytest.mark.ts
 def test_ts_with_strict_as_false():
     ts = time_serie(20577, "17/08/2019", "18/08/2019")
     assert len(ts) == 1
-    assert ts.dtype == np.float
-    
+    assert ts.dtype == np.float64
+
 @pytest.mark.ts
 def test_ts_with_strict_as_true():
     ts = time_serie(20577, "17/08/2019", "18/08/2019", True)
     assert len(ts) == 0
-    assert ts.dtype == np.float
+    assert ts.dtype == np.float64