Tokenization 指南：字节对编码，WordPiece等方法Python代码详解

Cats are great, but dogs are better!

['Cats', 'are', 'great,', 'but', 'dogs', 'are', 'better!']

['Cats', 'are', 'great', ',', 'but', 'dogs', 'are', 'better', '!']

Cats are great, but dogs are better!

['C', 'a', 't', 's', ' ', 'a', 'r', 'e', ' ', 'g', 'r', 'e', 'a', 't', ',', ' ', 'b', 'u', 't', ' ', 'd', 'o', 'g', 's', ' ', 'a', 'r', 'e', ' ', 'b', 'e', 't', 't', 'e', 'r', '!'`]

from tokenizers.normalizers import NFC, Lowercase, BertNormalizer
 
 # Text to normalize
 text = 'ThÍs is áN ExaMPlé     sÉnteNCE'
 
 # Instantiate normalizer objects
 NFCNorm = NFC()
 LowercaseNorm = Lowercase()
 BertNorm = BertNormalizer()
 
 # Normalize the text
 print(f'NFC:   {NFCNorm.normalize_str(text)}')
 print(f'Lower: {LowercaseNorm.normalize_str(text)}')
 print(f'BERT: {BertNorm.normalize_str(text)}')
 
 #NFC:   ThÍs is áN ExaMPlé     sÉnteNCE
 #Lower: thís is án examplé     séntence
 #BERT: this is an example     sentence

from transformers import FNetTokenizerFast, CamembertTokenizerFast, \
                          BertTokenizerFast
 
 # Text to normalize
 text = 'ThÍs is áN ExaMPlé     sÉnteNCE'
 
 # Instantiate tokenizers
 FNetTokenizer = FNetTokenizerFast.from_pretrained('google/fnet-base')
 CamembertTokenizer = CamembertTokenizerFast.from_pretrained('camembert-base')
 BertTokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
 
 # Normalize the text
 print(f'FNet Output:     \
    {FNetTokenizer.backend_tokenizer.normalizer .normalize_str(text)}')
 
 print(f'CamemBERT Output: \
    {CamembertTokenizer.backend_tokenizer.normalizer.normalize_str(text)}')
 
 print(f'BERT Output:     \
    {BertTokenizer.backend_tokenizer.normalizer.normalize_str(text)}')
     
 #FNet Output:     ThÍs is áN ExaMPlé sÉnteNCE
 #CamemBERT Output: ThÍs is áN ExaMPlé     sÉnteNCE
 #BERT Output:     this is an example     sentence

from tokenizers.pre_tokenizers import WhitespaceSplit, BertPreTokenizer
 
 # Text to normalize 
 text = ("this sentence's content includes: characters, spaces, and " \
        "punctuation.")
 
 # Define helper function to display pre-tokenized output
 def print_pretokenized_str(pre_tokens):
    for pre_token in pre_tokens:
        print(f'"{pre_token[0]}", ', end='')
 
 # Instantiate pre-tokenizers
 wss = WhitespaceSplit()
 bpt = BertPreTokenizer()
 
 # Pre-tokenize the text
 print('Whitespace Pre-Tokenizer:')
 print_pretokenized_str(wss.pre_tokenize_str(text))
 
 #Whitespace Pre-Tokenizer:
 #"this", "sentence's", "content", "includes:", "characters,", "spaces,", 
 #"and", "punctuation.", 
 
 
 print('\n\nBERT Pre-Tokenizer:')
 print_pretokenized_str(bpt.pre_tokenize_str(text))
 
 #BERT Pre-Tokenizer:
 #"this", "sentence", "'", "s", "content", "includes", ":", "characters", 
 #",", "spaces", ",", "and", "punctuation", ".",

from transformers import AutoTokenizer
 
 # Text to pre-tokenize
 text = ("this sentence's content includes: characters, spaces, and " \
        "punctuation.")
 
 # Instatiate the pre-tokenizers
 GPT2_PreTokenizer = AutoTokenizer.from_pretrained('gpt2').backend_tokenizer \
                    .pre_tokenizer
 
 Albert_PreTokenizer = AutoTokenizer.from_pretrained('albert-base-v1') \
                      .backend_tokenizer.pre_tokenizer
 
 # Pre-tokenize the text
 print('GPT-2 Pre-Tokenizer:')
 print_pretokenized_str(GPT2_PreTokenizer.pre_tokenize_str(text))
 
 #GPT-2 Pre-Tokenizer:
 #"this", "Ġsentence", "'s", "Ġcontent", "Ġincludes", ":", "Ġcharacters", ",",
 #"Ġspaces", ",", "Ġand", "Ġpunctuation", ".", 
 
 print('\n\nALBERT Pre-Tokenizer:')
 print_pretokenized_str(Albert_PreTokenizer.pre_tokenize_str(text))
 
 #ALBERT Pre-Tokenizer:
 #"▁this", "▁sentence's", "▁content", "▁includes:", "▁characters,", "▁spaces,",
 #"▁and", "▁punctuation.",

from tokenizers.pre_tokenizers import WhitespaceSplit, BertPreTokenizer
 
 # Text to pre-tokenize
 text = ("this sentence's content includes: characters, spaces, and " \
        "punctuation.")
 
 # Instantiate pre-tokenizer
 bpt = BertPreTokenizer()
 
 # Pre-tokenize the text
 bpt.pre_tokenize_str(example_sentence)

[('this', (0, 4)),
  ('sentence', (5, 13)),
  ("'", (13, 14)),
  ('s', (14, 15)),
  ('content', (16, 23)),
  ('includes', (24, 32)),
  (':', (32, 33)),
  ('characters', (34, 44)),
  (',', (44, 45)),
  ('spaces', (46, 52)),
  (',', (52, 53)),
  ('and', (54, 57)),
  ('punctuation', (58, 69)),
  ('.', (69, 70))]

class TargetVocabularySizeError(Exception):
    def __init__(self, message):
        super().__init__(message)
 
 class BPE:
    '''An implementation of the Byte Pair Encoding tokenizer.'''
 
    def calculate_frequency(self, words):
        ''' Calculate the frequency for each word in a list of words.
 
            Take in a list of words stored as strings and return a list of
            tuples where each tuple contains a string from the words list,
            and an integer representing its frequency count in the list.
 
            Args:
                words (list): A list of words (strings) in any order.
 
            Returns:
                corpus (list[tuple(str, int)]): A list of tuples where the
                  first element is a string of a word in the words list, and
                  the second element is an integer representing the frequency
                  of the word in the list.
        '''
        freq_dict = dict()
 
        for word in words:
            if word not in freq_dict:
                freq_dict[word] = 1
            else:
                freq_dict[word] += 1
 
        corpus = [(word, freq_dict[word]) for word in freq_dict.keys()]
 
        return corpus
 
 
    def create_merge_rule(self, corpus):
        ''' Create a merge rule and add it to the self.merge_rules list.
 
            Args:
                corpus (list[tuple(list, int)]): A list of tuples where the
                    first element is a list of a word in the words list (where
                    the elements are the individual characters (or subwords in
                    later iterations) of the word), and the second element is
                    an integer representing the frequency of the word in the
                    list.
 
            Returns:
                None
        '''
        pair_frequencies = self.find_pair_frequencies(corpus)
        most_frequent_pair = max(pair_frequencies, key=pair_frequencies.get)
        self.merge_rules.append(most_frequent_pair.split(','))
        self.vocabulary.append(most_frequent_pair)
 
 
    def create_vocabulary(self, words):
        ''' Create a list of every unique character in a list of words.
 
            Args:
                words (list): A list of strings containing the words of the
                    input text.
 
            Returns:
                vocabulary (list): A list of every unique character in the list
                    of input words.
        '''
        vocabulary = list(set(''.join(words)))
        return vocabulary
 
    def find_pair_frequencies(self, corpus):
        ''' Find the frequency of each character pair in the corpus.
 
            Loop through the corpus and calculate the frequency of each pair
            of adjacent characters across every word. Return a dictionary of
            each character pair as the keys and the corresponding frequency as
            the values.
 
            Args:
                corpus (list[tuple(list, int)]): A list of tuples where the
                    first element is a list of a word in the words list (where
                    the elements are the individual characters (or subwords in
                    later iterations) of the word), and the second element is
                    an integer representing the frequency of the word in the
                    list.
 
            Returns:
                pair_freq_dict (dict): A dictionary where the keys are the
                    character pairs from the input corpus and the values are an
                    integer representing the frequency of the pair in the
                    corpus.
        '''
        pair_freq_dict = dict()
 
        for word, word_freq in corpus:
            for idx in range(len(word)-1):
 
                char_pair = f'{word[idx]},{word[idx+1]}'
 
                if char_pair not in pair_freq_dict:
                    pair_freq_dict[char_pair] = word_freq
                else:
                    pair_freq_dict[char_pair] += word_freq
 
        return pair_freq_dict
 
 
    def get_merged_chars(self, char_1, char_2):
        ''' Merge the highest score pair and return to the self.merge method.
 
            This method is abstracted so that the BPE class can be used as the
            base class for other Tokenizers, and so the merging method can be
            easily overwritten. For example, in the BPE algorithm the
            characters can simply be concatenated and returned. However in the
            WordPiece algorithm, the # symbols must first be stripped.
 
            Args:
                char_1 (str): The first character in the highest-scoring pair.
                char_2 (str): The second character in the highest-scoring pair.
 
            Returns:
                merged_chars (str): Merged characters.
        '''
        merged_chars = char_1 + char_2
        return merged_chars
 
 
    def initialize_corpus(self, words):
        ''' Split each word into characters and count the word frequency.
 
            Split each word in the input word list on every character. For each
            word, store the split word in a list as the first element inside a
            tuple. Store the frequency count of the word as an integer as the
            second element of the tuple. Create a tuple for every word in this
            fashion and store the tuples in a list called 'corpus', then return
            then corpus list.
 
            Args:
                None
 
            Returns:
                corpus (list[tuple(list, int)]): A list of tuples where the
                    first element is a list of a word in the words list (where
                    the elements are the individual characters of the word),
                    and the second element is an integer representing the
                    frequency of the word in the list.
        '''
        corpus = self.calculate_frequency(words)
        corpus = [([*word], freq) for (word, freq) in corpus]
        return corpus
 
 
    def merge(self, corpus):
        ''' Loop through the corpus and perform the latest merge rule.
 
            Args:
                corpus (list[tuple(list, int)]): A list of tuples where the
                    first element is a list of a word in the words list (where
                    the elements are the individual characters (or subwords in
                    later iterations) of the word), and the second element is
                    an integer representing the frequency of the word in the
                    list.
 
            Returns:
                new_corpus (list[tuple(list, int)]): A modified version of the
                    input argument where the most recent merge rule has been
                    applied to merge the most frequent adjacent characters.
        '''
        merge_rule = self.merge_rules[-1]
        new_corpus = []
 
        for word, word_freq in corpus:
            new_word = []
            idx = 0
 
            while idx < len(word):
                # If a merge pattern has been found
                if (len(word) != 1) and (word[idx] == merge_rule[0]) and\
                (word[idx+1] == merge_rule[1]):
 
                    new_word.append(self.get_merged_chars(word[idx],word[idx+1]))
                    idx += 2
                # If a merge patten has not been found
                else:
                    new_word.append(word[idx])
                    idx += 1
 
            new_corpus.append((new_word, word_freq))
 
        return new_corpus
 
 
    def train(self, words, target_vocab_size):
        ''' Train the model.
 
            Args:
                words (list[str]): A list of words to train the model on.
 
                target_vocab_size (int): The number of words in the vocabulary
                    to be used as the stopping condition when training.
 
            Returns:
                None.
        '''
        self.words = words
        self.target_vocab_size = target_vocab_size
        self.corpus = self.initialize_corpus(self.words)
        self.corpus_history = [self.corpus]
        self.vocabulary = self.create_vocabulary(self.words)
        self.vocabulary_size = len(self.vocabulary)
        self.merge_rules = []
 
        # Iteratively add vocabulary until reaching the target vocabulary size
        if len(self.vocabulary) > self.target_vocab_size:
            raise TargetVocabularySizeError(f'Error: Target vocabulary size \
            must be greater than the initial vocabulary size \
            ({len(self.vocabulary)})')
 
        else:
            while len(self.vocabulary) < self.target_vocab_size:
                try:
                    self.create_merge_rule(self.corpus)
                    self.corpus = self.merge(self.corpus)
                    self.corpus_history.append(self.corpus)
 
                # If no further merging is possible
                except ValueError:
                    print('Exiting: No further merging is possible')
                    break
 
 
    def tokenize(self, text):
        ''' Take in some text and return a list of tokens for that text.
 
            Args:
                text (str): The text to be tokenized.
 
            Returns:
                tokens (list): The list of tokens created from the input text.
        '''
        tokens = [*text]
 
        for merge_rule in self.merge_rules:
 
            new_tokens = []
            idx = 0
 
            while idx < len(tokens):
                # If a merge pattern has been found
                if (len(tokens) != 1) and (tokens[idx] == merge_rule[0]) and \
                    (tokens[idx+1] == merge_rule[1]):
 
                    new_tokens.append(self.get_merged_chars(tokens[idx],
                                                            tokens[idx+1]))
                    idx += 2
                # If a merge patten has not been found
                else:
                    new_tokens.append(tokens[idx])
                    idx += 1
 
            tokens = new_tokens
 
        return tokens

# Training set
 words = ['cat', 'cat', 'cat', 'cat', 'cat',
          'cats', 'cats',
          'eat', 'eat', 'eat', 'eat', 'eat', 'eat', 'eat', 'eat', 'eat', 'eat',
          'eating', 'eating', 'eating',
          'running', 'running',
          'jumping',
          'food', 'food', 'food', 'food', 'food', 'food']
 
 # Instantiate the tokenizer
 bpe = BPE()
 bpe.train(words, 21)
 
 # Print the corpus at each stage of the process, and the merge rule used
 print(f'INITIAL CORPUS:\n{bpe.corpus_history[0]}\n')
 for rule, corpus in list(zip(bpe.merge_rules, bpe.corpus_history[1:])):
    print(f'NEW MERGE RULE: Combine "{rule[0]}" and "{rule[1]}"')
    print(corpus, end='\n\n')

INITIAL CORPUS:
 [(['c', 'a', 't'], 5), (['c', 'a', 't', 's'], 2), (['e', 'a', 't'], 10),
 (['e', 'a', 't', 'i', 'n', 'g'], 3), (['r', 'u', 'n', 'n', 'i', 'n', 'g'], 2), 
 (['j', 'u', 'm', 'p', 'i', 'n', 'g'], 1), (['f', 'o', 'o', 'd'], 6)]
 
 NEW MERGE RULE: Combine "a" and "t"
 [(['c', 'at'], 5), (['c', 'at', 's'], 2), (['e', 'at'], 10), 
 (['e', 'at', 'i', 'n', 'g'], 3), (['r', 'u', 'n', 'n', 'i', 'n', 'g'], 2), 
 (['j', 'u', 'm', 'p', 'i', 'n', 'g'], 1), (['f', 'o', 'o', 'd'], 6)]
 
 NEW MERGE RULE: Combine "e" and "at"
 [(['c', 'at'], 5), (['c', 'at', 's'], 2), (['eat'], 10), 
 (['eat', 'i', 'n', 'g'], 3), (['r', 'u', 'n', 'n', 'i', 'n', 'g'], 2), 
 (['j', 'u', 'm', 'p', 'i', 'n', 'g'], 1), (['f', 'o', 'o', 'd'], 6)]
 
 NEW MERGE RULE: Combine "c" and "at"
 [(['cat'], 5), (['cat', 's'], 2), (['eat'], 10), (['eat', 'i', 'n', 'g'], 3), 
 (['r', 'u', 'n', 'n', 'i', 'n', 'g'], 2), 
 (['j', 'u', 'm', 'p', 'i', 'n', 'g'], 1), (['f', 'o', 'o', 'd'], 6)]
 
 NEW MERGE RULE: Combine "i" and "n"
 [(['cat'], 5), (['cat', 's'], 2), (['eat'], 10), (['eat', 'in', 'g'], 3), 
 (['r', 'u', 'n', 'n', 'in', 'g'], 2), (['j', 'u', 'm', 'p', 'in', 'g'], 1), 
 (['f', 'o', 'o', 'd'], 6)]
 
 NEW MERGE RULE: Combine "in" and "g"
 [(['cat'], 5), (['cat', 's'], 2), (['eat'], 10), (['eat', 'ing'], 3), 
 (['r', 'u', 'n', 'n', 'ing'], 2), (['j', 'u', 'm', 'p', 'ing'], 1), 
 (['f', 'o', 'o', 'd'], 6)]

class WordPiece(BPE):
 
    def add_hashes(self, word):
        ''' Add # symbols to every character in a word except the first.
 
            Take in a word as a string and add # symbols to every character
            except the first. Return the result as a list where each element is
            a character with # symbols in front, except the first character
            which is just the plain character.
 
            Args:
                word (str): The word to add # symbols to.
 
            Returns:
                hashed_word (list): A list of the characters with # symbols
                    (except the first character which is just the plain
                    character).
        '''
        hashed_word = [word[0]]
 
        for char in word[1:]:
            hashed_word.append(f'##{char}')
 
        return hashed_word
 
 
    def create_merge_rule(self, corpus):
        ''' Create a merge rule and add it to the self.merge_rules list.
 
            Args:
                corpus (list[tuple(list, int)]): A list of tuples where the
                    first element is a list of a word in the words list (where
                    the elements are the individual characters (or subwords in
                    later iterations) of the word), and the second element is
                    an integer representing the frequency of the word in the
                    list.
 
            Returns:
                None
        '''
        pair_frequencies = self.find_pair_frequencies(corpus)
        char_frequencies = self.find_char_frequencies(corpus)
        pair_scores = self.find_pair_scores(pair_frequencies, char_frequencies)
 
        highest_scoring_pair = max(pair_scores, key=pair_scores.get)
        self.merge_rules.append(highest_scoring_pair.split(','))
        self.vocabulary.append(highest_scoring_pair)
 
 
    def create_vocabulary(self, words):
        ''' Create a list of every unique character in a list of words.
 
            Unlike the BPE algorithm where each character is stored normally,
            here a distinction is made by characters that begin a word
            (unmarked), and characters that are in the middle or end of a word
            (marked with a '##'). For example, the word 'cat' will be split
            into ['c', '##a', '##t'].
 
            Args:
                words (list): A list of strings containing the words of the
                    input text.
 
            Returns:
                vocabulary (list): A list of every unique character in the list
                    of input words, marked accordingly with ## to denote if the
                    character was featured in the middle/end of a word, instead
                    of as the first character of the word.
        '''
        vocabulary = set()
        for word in words:
            vocabulary.add(word[0])
            for char in word[1:]:
                vocabulary.add(f'##{char}')
 
        # Convert to list so the vocabulary can be appended to later
        vocabulary = list(vocabulary)
        return vocabulary
 
 
    def find_char_frequencies(self, corpus):
        ''' Find the frequency of each character in the corpus.
 
            Loop through the corpus and calculate the frequency of characters.
            Note that 'c' and '##c' are different characters, since the first
            represents a 'c' at the start of a word, and '##c' represents a 'c'
            in the middle/end of a word. Return a dictionary of each character
            pair as the keys and the corresponding frequency as the values.
 
            Args:
                corpus (list[tuple(list, int)]): A list of tuples where the
                    first element is a list of a word in the words list (where
                    the elements are the individual characters (or subwords in
                    later iterations) of the word), and the second element is
                    an integer representing the frequency of the word in the
                    list.
 
            Returns:
                pair_freq_dict (dict): A dictionary where the keys are the
                    characters from the input corpus and the values are an
                    integer representing the frequency.
        '''
        char_frequencies = dict()
 
        for word, word_freq in corpus:
            for char in word:
                if char in char_frequencies:
                    char_frequencies[char] += word_freq
                else:
                    char_frequencies[char] = word_freq
 
        return char_frequencies
 
 
    def find_pair_scores(self, pair_frequencies, char_frequencies):
        ''' Find the pair score for each character pair in the corpus.
 
            Loops through the pair_frequencies dictionary and calculate the
            pair score for each pair of adjacent characters in the corpus.
            Store the scores in a dictionary and return it.
 
            Args:
                pair_frequencies (dict): A dictionary where the keys are the
                    adjacent character pairs in the corpus and the values are
                    the frequencies of each pair.
 
                char_frequencies (dict): A dictionary where the keys are the
                    characters in the corpus and the values are corresponding
                    frequencies.
 
            Returns:
                pair_scores (dict): A dictionary where the keys are the
                    adjacent character pairs in the input corpus and the values
                    are the corresponding pair score.
        '''
        pair_scores = dict()
 
        for pair in pair_frequencies.keys():
            char_1 = pair.split(',')[0]
            char_2 = pair.split(',')[1]
            denominator = (char_frequencies[char_1]*char_frequencies[char_2])
            score = (pair_frequencies[pair]) / denominator
            pair_scores[pair] = score
 
        return pair_scores
 
 
    def get_merged_chars(self, char_1, char_2):
        ''' Merge the highest score pair and return to the self.merge method.
 
            Remove the # symbols as necessary and merge the highest scoring
            pair then return the merged characters to the self.merge method.
 
 
            Args:
                char_1 (str): The first character in the highest-scoring pair.
                char_2 (str): The second character in the highest-scoring pair.
 
            Returns:
                merged_chars (str): Merged characters.
        '''
        if char_2.startswith('##'):
            merged_chars = char_1 + char_2[2:]
        else:
            merged_chars = char_1 + char_2
 
        return merged_chars
 
 
    def initialize_corpus(self, words):
        ''' Split each word into characters and count the word frequency.
 
            Split each word in the input word list on every character. For each
            word, store the split word in a list as the first element inside a
            tuple. Store the frequency count of the word as an integer as the
            second element of the tuple. Create a tuple for every word in this
            fashion and store the tuples in a list called 'corpus', then return
            then corpus list.
 
            Args:
                None.
 
            Returns:
                corpus (list[tuple(list, int)]): A list of tuples where the
                    first element is a list of a word in the words list (where
                    the elements are the individual characters of the word),
                    and the second element is an integer representing the
                    frequency of the word in the list.
        '''
        corpus = self.calculate_frequency(words)
        corpus = [(self.add_hashes(word), freq) for (word, freq) in corpus]
        return corpus
 
    def tokenize(self, text):
        ''' Take in some text and return a list of tokens for that text.
 
            Args:
                text (str): The text to be tokenized.
 
            Returns:
                tokens (list): The list of tokens created from the input text.
        '''
        # Create cleaned vocabulary list without # and commas to check against
        clean_vocabulary = [word.replace('#', '').replace(',', '') 
                            for word in self.vocabulary]
        clean_vocabulary.sort(key=lambda word: len(word))
        clean_vocabulary = clean_vocabulary[::-1]
 
        # Break down the text into the largest tokens first, then smallest
        remaining_string = text
        tokens = []
        keep_checking = True
 
        while keep_checking:
            keep_checking = False
            for vocab in clean_vocabulary:
                if remaining_string.startswith(vocab):
                    tokens.append(vocab)
                    remaining_string = remaining_string[len(vocab):]
                    keep_checking = True
 
        if len(remaining_string) > 0:
            tokens.append(remaining_string)
 
        return tokens

wp = WordPiece()
 wp.train(words, 30)
 
 print(f'INITIAL CORPUS:\n{wp.corpus_history[0]}\n')
 for rule, corpus in list(zip(wp.merge_rules, wp.corpus_history[1:])):
    print(f'NEW MERGE RULE: Combine "{rule[0]}" and "{rule[1]}"')
    print(corpus, end='\n\n')

INITIAL CORPUS:
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##i', '##n', '##g'], 3), 
 (['r', '##u', '##n', '##n', '##i', '##n', '##g'], 2), 
 (['j', '##u', '##m', '##p', '##i', '##n', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "##m" and "##p"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##i', '##n', '##g'], 3), 
 (['r', '##u', '##n', '##n', '##i', '##n', '##g'], 2), 
 (['j', '##u', '##mp', '##i', '##n', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "r" and "##u"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##i', '##n', '##g'], 3), 
 (['ru', '##n', '##n', '##i', '##n', '##g'], 2), 
 (['j', '##u', '##mp', '##i', '##n', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "j" and "##u"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##i', '##n', '##g'], 3), 
 (['ru', '##n', '##n', '##i', '##n', '##g'], 2), 
 (['ju', '##mp', '##i', '##n', '##g'], 1), (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "ju" and "##mp"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##i', '##n', '##g'], 3), 
 (['ru', '##n', '##n', '##i', '##n', '##g'], 2), 
 (['jump', '##i', '##n', '##g'], 1), (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "jump" and "##i"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##i', '##n', '##g'], 3), 
 (['ru', '##n', '##n', '##i', '##n', '##g'], 2), (['jumpi', '##n', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "##i" and "##n"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##in', '##g'], 3), 
 (['ru', '##n', '##n', '##in', '##g'], 2), (['jumpi', '##n', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "ru" and "##n"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##in', '##g'], 3), 
 (['run', '##n', '##in', '##g'], 2), (['jumpi', '##n', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "run" and "##n"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##in', '##g'], 3), 
 (['runn', '##in', '##g'], 2), (['jumpi', '##n', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "jumpi" and "##n"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##in', '##g'], 3), 
 (['runn', '##in', '##g'], 2), (['jumpin', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "runn" and "##in"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##in', '##g'], 3), 
 (['runnin', '##g'], 2), (['jumpin', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "##in" and "##g"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##ing'], 3), 
 (['runnin', '##g'], 2), (['jumpin', '##g'], 1), 
 (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "runnin" and "##g"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##ing'], 3), 
 (['running'], 2), (['jumpin', '##g'], 1), (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "jumpin" and "##g"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##ing'], 3), 
 (['running'], 2), (['jumping'], 1), (['f', '##o', '##o', '##d'], 6)]
 
 NEW MERGE RULE: Combine "f" and "##o"
 [(['c', '##a', '##t'], 5), (['c', '##a', '##t', '##s'], 2), 
 (['e', '##a', '##t'], 10), (['e', '##a', '##t', '##ing'], 3), 
 (['running'], 2), (['jumping'], 1), (['fo', '##o', '##d'], 6)]

print(wp.tokenize('jumper'))
 #['jump', 'e', 'r']

from tokenizers import Tokenizer
 
 tokenizer = Tokenizer.from_pretrained('bert-base-cased')

BertWordPieceTokenizer - The famous Bert tokenizer, using WordPiece
 CharBPETokenizer - The original BPE
 ByteLevelBPETokenizer - The byte level version of the BPE
 SentencePieceBPETokenizer - A BPE implementation compatible with the one used by SentencePiece

# Import a tokenizer
 from tokenizers import BertWordPieceTokenizer, CharBPETokenizer, \
                        ByteLevelBPETokenizer, SentencePieceBPETokenizer
 
 # Instantiate the model
 tokenizer = CharBPETokenizer()
 
 # Train the model
 tokenizer.train(['./path/to/files/1.txt', './path/to/files/2.txt'])
 
 # Tokenize some text
 encoded = tokenizer.encode('I can feel the magic, can you?')
 
 # Save the model
 tokenizer.save('./path/to/directory/my-bpe.tokenizer.json')

from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers, \
                        processors
 
 # Initialize a tokenizer
 tokenizer = Tokenizer(models.BPE())
 
 # Customize pre-tokenization and decoding
 tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space=True)
 tokenizer.decoder = decoders.ByteLevel()
 tokenizer.post_processor = processors.ByteLevel(trim_offsets=True)
 
 # And then train
 trainer = trainers.BpeTrainer(
    vocab_size=20000,
    min_frequency=2,
    initial_alphabet=pre_tokenizers.ByteLevel.alphabet()
 )
 tokenizer.train([
    "./path/to/dataset/1.txt",
    "./path/to/dataset/2.txt",
    "./path/to/dataset/3.txt"
 ], trainer=trainer)
 
 # And Save it
 tokenizer.save("byte-level-bpe.tokenizer.json", pretty=True)