Token list for Japanese character

Token list for Japanese character

tokenlist_jp(tokens = list())

Arguments

tokens

list

Examples

if (FALSE) {
tokens <- list(purrr::set_names(c(intToUtf8(c(26519, 27278))),
               c(intToUtf8(c(21517, 35422)))))
tokenlist_jp(tokens)
}