tokenizer_config.json 1.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142
  1. {
  2. "add_bos_token": true,
  3. "add_eos_token": false,
  4. "added_tokens_decoder": {
  5. "0": {
  6. "content": "<unk>",
  7. "lstrip": false,
  8. "normalized": false,
  9. "rstrip": false,
  10. "single_word": false,
  11. "special": true
  12. },
  13. "1": {
  14. "content": "<s>",
  15. "lstrip": false,
  16. "normalized": false,
  17. "rstrip": false,
  18. "single_word": false,
  19. "special": true
  20. },
  21. "2": {
  22. "content": "</s>",
  23. "lstrip": false,
  24. "normalized": false,
  25. "rstrip": false,
  26. "single_word": false,
  27. "special": true
  28. }
  29. },
  30. "bos_token": "<s>",
  31. "clean_up_tokenization_spaces": false,
  32. "eos_token": "</s>",
  33. "legacy": true,
  34. "model_max_length": 1000000000000000019884624838656,
  35. "pad_token": null,
  36. "sp_model_kwargs": {},
  37. "spaces_between_special_tokens": false,
  38. "tokenizer_class": "LlamaTokenizer",
  39. "unk_token": "<unk>",
  40. "use_default_system_prompt": false,
  41. "chat_template": "{% for message in messages %}{% if message['role'] == 'user' %}{{'<用户>' + message['content'].strip() + '<AI>'}}{% else %}{{message['content'].strip()}}{% endif %}{% endfor %}"
  42. }