refactor

2 years ago · d65d72af9a
parent 14493cbd1a
commit d65d72af9a
1 changed files with 1 additions and 1 deletions
--- a/codegeex/tokenizer/tokenizer.py
+++ b/codegeex/tokenizer/tokenizer.py
@ -4,7 +4,7 @@ from transformers import AutoTokenizer
 from transformers.models.gpt2 import GPT2TokenizerFast


-def encode_whitespaces(text, start_extra_id: int, max_len: int):
+def encode_whitespaces(text: str, start_extra_id: int, max_len: int):
    """ Encode whitespaces to extra tokens in GPT-J.

    >>> encode_whitespaces('a\\n  b\\n   c', 10, 10)