Reference

classifier_prediction

ClassifierPredictor

__init__(self, column_name=None, tfidf_path=None, model_path=None, label_encoder_path=None, agro_eco=False, gaul=False, calibrate=True, **kwargs) special

A class that generates predictions based on a trained XGBoost model. Can predict based on Ugandan Regions, GAUL regions, or FAO agro-ecological zones. Optionally predicts gaul regions using an ensemble calibrated classifier.

Examples:

>>> surnames = pd.DataFrame({'names':['Ahimbisibwe', 'Auma', 'Amin', 
             'Makubuya', 'Museveni', 'Oculi', 'Kadaga']})
>>> c = ClassifierPredictor(column_name='names')
>>> predict_xgb = c.predict(surnames, 
                            get_label_names=True, 
                            predict_prob = True,
                            df_out =True)

Parameters:

Name	Type	Description	Default
column_name	str	When passing a pandas dataframe, the name of the columns with surnames. Defaults to None.	None
tfidf_path	Union[str, pathlib.Path]	the path of the joblib dump of the tfidf transformer. Defaults to None.	None
model_path	Union[str, pathlib.Path]	the path of the joblib dump of the trained model. Defaults to None.	None
label_encoder_path	Union[str, pathlib.Path]	the path of the joblib dump of the label encoder. Defaults to None.	None
agro_eco	bool	Whether to predict agro-ecological zones. Defaults to False.	False
gaul	bool	whether to predict gaul regions. Defaults to False.	False
calibrate	bool	whether to predict gaul regions. Defaults to True.	True

Source code in predictor/classifier_prediction.py

def __init__(
    self,
    column_name : str = None,
    tfidf_path : Union[str, Path] =None,
    model_path : Union[str, Path]=None,
    label_encoder_path : Union[str, Path] =None,
    agro_eco : bool=False,
    gaul : bool =False,
    calibrate : bool =True,
    **kwargs
):
    """A class that generates predictions based on a trained XGBoost model. Can predict based on Ugandan Regions, GAUL regions, or FAO agro-ecological zones. Optionally predicts gaul regions using an ensemble calibrated classifier.

    Example:
        >>> surnames = pd.DataFrame({'names':['Ahimbisibwe', 'Auma', 'Amin', 
                     'Makubuya', 'Museveni', 'Oculi', 'Kadaga']})
        >>> c = ClassifierPredictor(column_name='names')
        >>> predict_xgb = c.predict(surnames, 
                                    get_label_names=True, 
                                    predict_prob = True,
                                    df_out =True)

    Args:
        column_name (str, optional): When passing a pandas dataframe, the name of the columns with surnames. Defaults to None.
        tfidf_path (Union[str, Path], optional): the path of the joblib dump of the tfidf transformer. Defaults to None.
        model_path (Union[str, Path], optional): the path of the joblib dump of the trained model. Defaults to None.
        label_encoder_path (Union[str, Path], optional): the path of the joblib dump of the label encoder. Defaults to None.
        agro_eco (bool, optional): Whether to predict agro-ecological zones. Defaults to False.
        gaul (bool, optional): whether to predict gaul regions. Defaults to False.
        calibrate (bool, optional): whether to predict gaul regions. Defaults to True.
    """              
    if agro_eco ==True and gaul==True:

        raise Exception("You can't have agro_eco and gaul both true")


    if tfidf_path is None:

        if agro_eco:
            tfidf_path = Path(
                "predictor",
                "saved_models",
                "tfidf_multilabel_False_nokampala_True_agro_zone_smote_False_opt.joblib",
            )
        elif gaul:
            tfidf_path = Path(
                "predictor",
                "saved_models",
                "tfidf_multilabel_False_nokampala_True_gaul_smote_False_gaul_opt.joblib",
            )

        else:
            tfidf_path = Path(
                "predictor",
                "saved_models",
                "tfidf_multilabel_False_nokampala_True.joblib",
            )
    else:
        tfidf_path = Path(tfidf_path)

    if label_encoder_path is None:

        if agro_eco:
            label_encoder_path = Path(
                "predictor",
                "saved_models",
                "label_encoder_multilabel_False_nokampala_True_agro_zone_smote_False_opt.joblib",
            )
        elif gaul:
            label_encoder_path = Path(
                "predictor",
                "saved_models",
                "label_encoder_multilabel_False_nokampala_True_gaul_smote_False_gaul_opt.joblib",
            )

        else:
            label_encoder_path = Path(
                "predictor",
                "saved_models",
                "label_encoder_multilabel_False_nokampala_True.joblib",
            )
    else:
        label_encoder_path = Path(label_encoder_path)

    if model_path is None:

        if agro_eco:
            model_path = Path(
                "predictor",
                "saved_models",
                "xgb_None_multilabel_False_add_kampala_True_agro_zone_smote_False_opt.joblib",
            )
        elif gaul:
            if calibrate:
                model_path = Path(
                    "predictor",
                    'saved_models',
                    'xgb_None_calibrated_gaul_opt.joblib'
                )
            else:
                model_path = Path(
                    "predictor",
                    "saved_models",
                    "xgb_None_multilabel_False_add_kampala_True_gaul_smote_False_gaul_opt.joblib"
                )

        else:
            model_path = Path(
                "predictor",
                "saved_models",
                "xgb_None_multilabel_False_add_kampala_True.joblib",
            )
    else:
        model_path = Path(model_path)

    self.tfidf_path = tfidf_path
    self.model_path = model_path
    self.label_encoder_path = label_encoder_path

    self.column_name = column_name

load_label_encoder(self)

Loads label encoder. See here for more information.

Returns:

Type	Description
CountVectorizer

Source code in predictor/classifier_prediction.py

def load_label_encoder(self) -> CountVectorizer:
    """Loads label encoder. See [here](https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.LabelEncoder.html)
        for more information.

    Returns:
        CountVectorizer: An sklearn Count Vectorizer object

    """
    return joblib.load(self.label_encoder_path)

load_model(self)

Loads pickled trained classifier. Current one is an XGBoost classifier. See here for more details.

Returns:

Type	Description
Union[xgboost.sklearn.XGBClassifier, sklearn.calibration.CalibratedClassifierCV]	Union[XGBClassifier, CalibratedClassifierCV]: Depending on the option, either a trained XGBoost Classifier or an sklearn calibrated classifier.

Source code in predictor/classifier_prediction.py

def load_model(self) -> Union[XGBClassifier, CalibratedClassifierCV]:
    """Loads pickled trained classifier. Current one is an XGBoost classifier.
        See [here](https://xgboost.readthedocs.io/en/latest/) for more details.

    Returns:
        Union[XGBClassifier, CalibratedClassifierCV]: Depending on the option, either a trained XGBoost Classifier or an sklearn calibrated classifier.
    """
    return joblib.load(self.model_path)

load_tfidf(self)

Loads Tfidf transformer. See here for information on all functionality.

Returns:

Type	Description
TfidfVectorizer

Source code in predictor/classifier_prediction.py

def load_tfidf(self) -> TfidfVectorizer:
    """Loads Tfidf transformer. See [here](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html) for information on all functionality.

    Returns:
        TfidfVectorizer: A sklearn Tfidf Vectorizer Object
    """

    return joblib.load(self.tfidf_path)

predict(self, text=None, get_label_names=False, predict_prob=False, df_out=False)

Predicts origin based on classifier

Keyword Arguments: text (list): List of strings to be predicted (default: {None}) get_label_names (bool): Whether to output the label names after prediction (default: {False}) predict_prob (bool): whether to give probabilities of coming from each region (default: {False}) df_out (bool): whether to output a pandas dataframe (default: {False})

>>> from predictor.classifier_prediction import ClassifierPredictor
>>> # Instantiate predictor
>>> c = ClassifierPredictor()
>>> # Predict
>>> prediction = c.predict(['Auma'])
>>> print(prediction)

Returns:

Type	Description
Union[pandas.core.frame.DataFrame, list]	Union[pd.DataFrame, list]: An object that contains predictions from the model

Source code in predictor/classifier_prediction.py

def predict(self, 
            text : list =None, 
            get_label_names : bool=False, 
            predict_prob : bool=False, 
            df_out : bool=False) -> Union[pd.DataFrame, list]:        
    """Predicts origin based on classifier

    Keyword Arguments:
        text (list): List of strings to be predicted (default: {None})
        get_label_names (bool): Whether to output the label names after prediction (default: {False})
        predict_prob (bool): whether to give probabilities of coming from each region (default: {False})
        df_out (bool): whether to output a pandas dataframe (default: {False})

        >>> from predictor.classifier_prediction import ClassifierPredictor
        >>> # Instantiate predictor
        >>> c = ClassifierPredictor()
        >>> # Predict
        >>> prediction = c.predict(['Auma'])
        >>> print(prediction)

    Returns:
        Union[pd.DataFrame, list]: An object that contains predictions from the model
    """
    labels = self.load_label_encoder()
    raw_text_aux = text.copy(deep = True)
    text = self.process_text(text)
    raw_text = raw_text_aux.assign(processed_text = text).set_index('processed_text')
    X = self.transform_text(text)

    model = self.load_model()

    if get_label_names:
        if predict_prob:
            label_names = labels.classes_
            probs = model.predict_proba(X)
            result = {
                t: {
                    label_name: prob
                    for label_name, prob in zip(label_names, probs[i])
                }
                for t, i in zip(text, range(len(probs)))
            }

        else:
            result = {
                name: prediction
                for name, prediction in zip(
                    text, labels.inverse_transform(model.predict(X))
                )
            }
    else:
        if predict_prob:
            result = model.predict_proba(X)
        else:
            result = model.predict(X)

    if df_out:
        try:
            return (
                pd.DataFrame(result).T
                .sort_index()
                .merge(raw_text, 
                       left_index = True, 
                       right_index = True)
                .reset_index()
                .set_index(self.column_name)
                .rename({'index' : 'processed_surname'}, axis=1)
                )

        except ValueError:
            return pd.DataFrame(result.values(), index = result.keys()).rename({0 : 'prediction'}, axis=1)
    else:
        return result

process_text(self, text)

Pre-processes input text to make it ready for prediction.

Parameters:

Name	Type	Description	Default
text	Union[str, list]	The input string or list of strings that are to be processed	required

Returns:

Type	Description
list

Source code in predictor/classifier_prediction.py

def process_text(self, text: Union[str, list]) -> list:
    """Pre-processes input text to make it ready for prediction.

    Arguments:
        text (str or list-like object): The input string or list of strings that are to be processed

    Returns:
        list: Pre-processed list of strings
    """

    if isinstance(text, str):

        text = [text]
    elif isinstance(text, pd.DataFrame):
        if self.column_name is None:
            raise Exception("Got a dataframe, but did not get `column_name`")
        text = text[self.column_name].tolist()

    processed_text = [i.lower().rstrip().lstrip() for i in text]

    return processed_text

transform_text(self, text=None)

A function that takes a list of surnames or strings and transforms them through the tf-idf transformer

Parameters:

Name	Type	Description	Default
text	str	the text to be transformed. Defaults to None.	None

Exceptions:

Type	Description
NoTextException	Raises if no text was given

Returns:

Type	Description
~Vector

Source code in predictor/classifier_prediction.py

def transform_text(self, text : str = None) -> Vector:
    """A function that takes a list of surnames or strings and transforms them through the tf-idf transformer

    Args:
        text (str, optional): the text to be transformed. Defaults to None.

    Raises:
        NoTextException: Raises if no text was given

    Returns:
        Vector: A sparse matrix of numpy matrix
    """        

    if text is None:
        raise NoTextException("No text was given for transformation.")

    tfidf = self.load_tfidf()

    return tfidf.transform(text)

exceptions

NoTextException

Raises error if text input is not given

table_predictor

TablePredictor

__init__(self, agro_eco=False, gaul=False, table_path=None, column_name=None) special

A class for predicting origin based on the frequency of name occurrences. This is implemented using a simple frequency tables based on the voter registration data.

Parameters:

Name	Type	Description	Default
agro_eco	bool	Whether to predict agro-ecological zones. Defaults to False.	False
gaul	bool	whether to predict GAUL regions. Defaults to False.	False
table_path	Union[str, pathlib.Path]	Path to the CSV file with the frequency table. Defaults to None.	None
column_name	str	name of the column in the dataframe with the surname information. Defaults to None.	None

Source code in predictor/table_predictor.py

def __init__(self, 
             agro_eco : bool = False, 
             gaul : bool=False, 
             table_path : Union[str, Path]  = None, 
             column_name : str = None):
    """A class for predicting origin based on the frequency of name occurrences. This is implemented using a simple frequency tables based on the voter registration data.

    Args:
        agro_eco (bool, optional): Whether to predict agro-ecological zones. Defaults to False.
        gaul (bool, optional): whether to predict GAUL regions. Defaults to False.
        table_path (Union[str, Path], optional): Path to the CSV file with the frequency table. Defaults to None.
        column_name (str, optional): name of the column in the dataframe with the surname information. Defaults to None.

    """        

    if table_path is None:
        if agro_eco:
            table_path = Path('predictor',"table", "agro_zone_predictor.csv")
        elif agro_eco==False and gaul==True:
            table_path = Path('predictor', 'table', 'gaul_predictor.csv')
        elif agro_eco == True and gaul == True:
            raise Exception("Can't have agro_eco and gaul at the same time.")
        else:
            table_path = Path('predictor',"table", "table_predictor.csv")


    self.table_path = table_path

    self.table = self._load_table()

    self.column_name = column_name

predict(self, text, fuzzy=False, n_jobs=1)

Predicts region probability by first trying to find an exact match and then fuzzy matching for any names not matched.

Parameters:

Name	Type	Description	Default
text	list	List of strings to predict	required

Keyword Arguments: n_jobs (int): The number of cores to use for fuzzy-matching. This only applies to fuzzy-matching. If all names are found by exact match, then multi-processing is not used at all. (default: {1})

Returns:

Type	Description
DataFrame	pd.DataFrame: DataFrame of predictions

Source code in predictor/table_predictor.py

def predict(self, text: list, fuzzy : bool = False, n_jobs : int = 1) -> pd.DataFrame:
    """Predicts region probability by first trying to find an exact match and then 
    fuzzy matching for any names not matched.

    Arguments:
        text (list): List of strings to predict

    Keyword Arguments:
        n_jobs (int): The number of cores to use for fuzzy-matching. This
        only applies to fuzzy-matching. If all names are found by exact match, then
        multi-processing is not used at all. (default: {1})

    Returns:
        pd.DataFrame: DataFrame of predictions
    """

    processed_text = self._process_text(text = text)

    # First do exact match
    predicted_df = self._exact_match(text = processed_text, data = self.table)

    if fuzzy:
        # get dict of exact matches in same form as processed text
        predicted_dict = dict(map(reversed, predicted_df['processed_surname'].to_dict().items()))

        if predicted_df.index.size != len(processed_text):
            difference = len(processed_text) - predicted_df.index.unique().size
            print(f"Doing fuzzy matching for {difference} names...")

            # Get names that weren't found
            lost_dict = {k : processed_text[k] for k in set(processed_text) - set(predicted_dict)}

            if n_jobs == 1:
                fuzzy_matches = self._fuzzy_match(text = lost_dict, data = self.table)

            elif n_jobs > 1:
                _fuzzy_lost_dict = partial(self._fuzzy_match, text = lost_dict)

                df_list = self._partition_table(data = self.table, n = n_jobs)

                pool = Pool(processes = n_jobs)
                results = pool.map(_fuzzy_lost_dict, df_list)

                # Now get max probability for each name
                results_df = pd.concat(results)

                max_df = (
                    results_df
                    .groupby(results_df.index)
                    .max()
                    .merge(self.table, left_on = 'name_match', right_index = True)
                )

                predicted_df = predicted_df.append(max_df)

            else:
                raise Exception(f"{n_jobs} n_jobs not allowed as an input")

    return predicted_df

---

Last update: April 20, 2021