#!/usr/bin/env python
# coding: utf-8

# # Country Converter

# The country converter (coco) is a Python package to convert country names into different classifications and between different naming versions. Internally it uses regular expressions to match country names.
# 

# ## Installation

# The package is available as PyPI, use 

# pip install country_converter  -upgrade

# from the command line or use your preferred python package installer.
# The source code is available on github: https://github.com/IndEcol/country_converter

# ## Conversion

# The country converter provides one main class which is used for the conversion:

# In[1]:


import country_converter as coco


# In[2]:


converter = coco.CountryConverter()


# Given a list of countries is a certain classification:

# In[3]:


iso3_codes = ["USA", "VUT", "TKL", "AUT", "AFG", "ALB"]


# This can be converted to any classification provided by:

# In[4]:


converter.convert(names=iso3_codes, src="ISO3", to="name_official")


# or

# In[5]:


converter.convert(names=iso3_codes, src="ISO3", to="continent")


# The parameter "src" specifies the input-, "to" the output format. Possible values for both parameter can be found by:

# In[6]:


converter.valid_class


# Internally, these names are the column header of the underlying pandas dataframe (see below).

# The convert function can also be accessed without initiating the CountryConverter. This can be useful for one time usage. For multiple matches, initiating the CountryConverter avoids that the file providing the matching data gets read in for each conversion.

# In[7]:


converter.convert(names=iso3_codes, src="ISO3", to="ISO2")


# Some of the classifications can be accessed by some shortcuts. For example:

# In[8]:


converter.EU27


# In[9]:


converter.OECDas("ISO2")


# ## Handling missing data

# The return value for non-found entries is be default set to 'not found':

# In[10]:


iso3_codes_missing = ["ABC", "AUT", "XXX"]
converter.convert(iso3_codes_missing, src="ISO3")


# but can also be rest to something else:

# In[11]:


converter.convert(iso3_codes_missing, src="ISO3", not_found="missing")


# Alternativly, the non-found entries can be passed through by passing None to not_found:

# In[12]:


converter.convert(iso3_codes_missing, src="ISO3", not_found=None)


# To extend the underlying dataset, an additional dataframe (or file) can be passed. Note, that all entries below (name_short, name_official, regex, ISO2 and ISO3) must be specified.

# In[13]:


import pandas as pd

add_data = pd.DataFrame.from_dict(
    {
        "name_short": ["xxx country", "abc country"],
        "name_official": ["The XXX country", "The ABC country"],
        "regex": ["xxx country", "abc country"],
        "ISO2": ["xx", "ab"],
        "ISO3": ["xxx", "abc"],
    }
)


# In[14]:


add_data


# In[15]:


extended_converter = coco.CountryConverter(additional_data=add_data)
extended_converter.convert(iso3_codes_missing, src="ISO3", to="name_short")


# Alternatively to a ad hoc dataframe, additional datafiles can be passed. These must have the same format as basic data set. 
# An example can be found here: 
# https://github.com/IndEcol/country_converter/tree/master/tests/custom_data_example.txt
# 
# The custom data example contains the ISO3 code mapping for Romania before 2002 and switches the regex matching for congo between DR Congo and Congo Republic.
# 
# To use is pass the path to the additional country file:

# In[16]:


# extended_converter = coco.CountryConverter(additional_data=path/to/datafile)


# The passed data (file or dataframe) must at least contain the headers 'name_official', 'name_short' and 'regex'. Of course, if the additional data shall be used to a conversion to any other field, these must also be included. 

# Additionally passed data always overwrites the existing one.
# This can be used to adjust coco for datasets with wrong country names. 
# For example, assuming a dataset erroneous switched the ISO2 codes for India (IN) and Indonesia (ID) (therefore assuming 'ID' for India and 'IN' for Indonesia), one can accomedate for that by: 

# In[17]:


switched_converter = coco.CountryConverter(
    additional_data=pd.DataFrame.from_dict(
        {
            "name_short": ["India", "Indonesia"],
            "name_official": ["India", "Indonesia"],
            "regex": ["india", "indonesia"],
            "ISO2": ["ID", "IN"],
            "ISO3": ["IDN", "IND"],
        }
    )
)


# In[18]:


converter.convert("IN", src="ISO2", to="name_short")


# In[19]:


switched_converter.convert("ID", src="ISO2", to="name_short")


# ## Regular expression matching

# The input parameter "src" can be set to "regex" to use regular expression matching for a given country list. For example:

# In[20]:


some_names = [
    "United Rep. of Tanzania",
    "Cape Verde",
    "Burma",
    "Iran (Islamic Republic of)",
    "Korea, Republic of",
    "Dem. People's Rep. of Korea",
]


# In[21]:


coco.convert(names=some_names, src="regex", to="name_short")


# The regular expressions can also be used to match any list of countries to any other. For example: 

# In[22]:


match_these = ["norway", "united_states", "china", "taiwan"]
master_list = [
    "USA",
    "The Swedish Kingdom",
    "Norway is a Kingdom too",
    "Peoples Republic of China",
    "Republic of China",
]

coco.match(match_these, master_list)


# If the regular expression matches several times, all results are given as list and a warning is generated:

# In[23]:


match_these = ["norway", "united_states", "china", "taiwan"]
master_list = [
    "USA",
    "The Swedish Kingdom",
    "Norway is a Kingdom too",
    "Peoples Republic of China",
    "Taiwan, province of china",
    "Republic of China",
]

coco.match(match_these, master_list)


# The parameter "enforce_sublist" can be set to ensure consistent output:

# In[24]:


coco.match(match_these, master_list, enforce_sublist=True)


# You get a warning if one of the names couldn't be found:

# In[25]:


match_these = ["norway", "united_states", "china", "taiwan", "some other country"]
master_list = [
    "USA",
    "The Swedish Kingdom",
    "Norway is a Kingdom too",
    "Peoples Republic of China",
    "Republic of China",
]
coco.match(match_these, master_list)


# And the value for non found countries can be specified: 

# In[26]:


coco.match(match_these, master_list, not_found="its not there")


# This can also be used to pass the not found country to the new classification:

# In[27]:


coco.match(match_these, master_list, not_found=None)


# ## Internals

# Within the new instance, the raw data for the conversion is saved within a pandas dataframe. 
# This dataframe can be accessed directly with:

# In[28]:


converter.data.head()


# This dataframe can be extended in both directions. The only requirement is to provide unique values for name_short, name_official and regex.
# 
# Internally, the data is saved in country_data.txt as tab-separated values (utf-8 encoded).

# Of course, all pandas indexing and matching methods can be used. For example, to get new OECD members since 1995 present in a list:

# In[29]:


some_countries = [
    "Australia",
    "Belgium",
    "Brazil",
    "Bulgaria",
    "Cyprus",
    "Czech Republic",
    "Denmark",
    "Estonia",
    "Finland",
    "France",
    "Germany",
    "Greece",
    "Hungary",
    "India",
    "Indonesia",
    "Ireland",
    "Italy",
    "Japan",
    "Latvia",
    "Lithuania",
    "Luxembourg",
    "Malta",
    "Romania",
    "Russia",
    "Turkey",
    "United Kingdom",
    "United States",
]
converter.data[
    (converter.data.OECD >= 1995) & converter.data.name_short.isin(some_countries)
].name_short


# Further information can be found here: http://pandas.pydata.org/pandas-docs/stable/

# ## Testing

# All regular expressions of the country converter are tested for a unique match to name_short and name_official. 
# Test sets for alternative names found in various databases are also available. 
# 
# The test sets are stored in the ``tests/`` subdirectory. To tests require pytest.
# I recommend to rerun the test if a regular expression is changed. 
# 
# To specify a new test set just add a tab-separated file with headers "name\_short" and "name\_test" and provide name (corresponding to the short name in the main classification file) and the alternative name which should be tested (one pair per row in the file). If the file name starts with "test\_regex\_ " it will be automatically recognised by the test functions.
# 
# Please see the file CONTRIBUTING.rst for further information.

# Konstantin Stadler