计量计费

# 运行前请 pip install tiktoken
from dashscope import get_tokenizer  # dashscope版本 >= 1.14.0

# 获取tokenizer对象，目前只支持通义千问系列模型
tokenizer = get_tokenizer('qwen-turbo')

input_str = '通义千问具有强大的能力。'

# 将字符串切分成token并转换为token id
tokens = tokenizer.encode(input_str)
print(f"经过切分后的token id为：{tokens}。")
# 经过切分后的token id为： [31935, 64559, 99320, 56007, 100629, 104795, 99788, 1773]
print(f"经过切分后共有{len(tokens)}个token")
# 经过切分后共有8个token

# 将token id转化为字符串并打印出来
for i in range(len(tokens)):
    print(f"token id为{tokens[i]}对应的字符串为：{tokenizer.decode(tokens[i])}")

// Copyright (c) Alibaba, Inc. and its affiliates.
// dashscope SDK版本 >= 2.13.0
import java.util.List;
import com.alibaba.dashscope.exception.NoSpecialTokenExists;
import com.alibaba.dashscope.exception.UnSupportedSpecialTokenMode;
import com.alibaba.dashscope.tokenizers.Tokenizer;
import com.alibaba.dashscope.tokenizers.TokenizerFactory;

public class Main {
  public static void testEncodeOrdinary(){
    Tokenizer tokenizer = TokenizerFactory.qwen();
    String prompt ="如果现在要你走十万八千里路，需要多长的时间才能到达？ ";
    // encode string with no special tokens
    List<Integer> ids = tokenizer.encodeOrdinary(prompt);
    System.out.println(ids);
    String decodedString = tokenizer.decode(ids);
    assert decodedString == prompt;
  }

  public static void testEncode() throws NoSpecialTokenExists, UnSupportedSpecialTokenMode{
    Tokenizer tokenizer = TokenizerFactory.qwen();
    String prompt = "<|im_start|>system\nYour are a helpful assistant.<|im_end|>\n<|im_start|>user\nSanFrancisco is a<|im_end|>\n<|im_start|>assistant\n";
    // encode string with special tokens <|im_start|> and <|im_end|>
    List<Integer> ids = tokenizer.encode(prompt, "all");
    // 24 tokens [151644, 8948, 198, 7771, 525, 264, 10950, 17847, 13, 151645, 198, 151644, 872, 198, 23729, 80328, 9464, 374, 264, 151645, 198, 151644, 77091, 198]
    String decodedString = tokenizer.decode(ids);
    System.out.println(ids);
    assert decodedString == prompt;

  }

  public static void main(String[] args) {
      try {
        testEncodeOrdinary();
        testEncode();
      } catch (NoSpecialTokenExists | UnSupportedSpecialTokenMode e) {
        e.printStackTrace();
      }
  }
}

经过切分后的token id为：[31935, 64559, 99320, 56007, 100629, 104795, 99788, 1773]。
经过切分后共有8个token
token id为31935对应的字符串为：通
token id为64559对应的字符串为：义
token id为99320对应的字符串为：千
token id为56007对应的字符串为：问
token id为100629对应的字符串为：具有
token id为104795对应的字符串为：强大的
token id为99788对应的字符串为：能力
token id为1773对应的字符串为：。