chat_gpt_session.py 4.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104
  1. from bot.session_manager import Session
  2. from common.log import logger
  3. from common import const
  4. """
  5. e.g. [
  6. {"role": "system", "content": "You are a helpful assistant."},
  7. {"role": "user", "content": "Who won the world series in 2020?"},
  8. {"role": "assistant", "content": "The Los Angeles Dodgers won the World Series in 2020."},
  9. {"role": "user", "content": "Where was it played?"}
  10. ]
  11. """
  12. class ChatGPTSession(Session):
  13. def __init__(self, session_id, system_prompt=None, model="gpt-3.5-turbo"):
  14. super().__init__(session_id, system_prompt)
  15. self.model = model
  16. self.reset()
  17. def discard_exceeding(self, max_tokens, cur_tokens=None):
  18. precise = True
  19. try:
  20. cur_tokens = self.calc_tokens()
  21. except Exception as e:
  22. precise = False
  23. if cur_tokens is None:
  24. raise e
  25. logger.debug("Exception when counting tokens precisely for query: {}".format(e))
  26. while cur_tokens > max_tokens:
  27. if len(self.messages) > 2:
  28. self.messages.pop(1)
  29. elif len(self.messages) == 2 and self.messages[1]["role"] == "assistant":
  30. self.messages.pop(1)
  31. if precise:
  32. cur_tokens = self.calc_tokens()
  33. else:
  34. cur_tokens = cur_tokens - max_tokens
  35. break
  36. elif len(self.messages) == 2 and self.messages[1]["role"] == "user":
  37. logger.warn("user message exceed max_tokens. total_tokens={}".format(cur_tokens))
  38. break
  39. else:
  40. logger.debug("max_tokens={}, total_tokens={}, len(messages)={}".format(max_tokens, cur_tokens, len(self.messages)))
  41. break
  42. if precise:
  43. cur_tokens = self.calc_tokens()
  44. else:
  45. cur_tokens = cur_tokens - max_tokens
  46. return cur_tokens
  47. def calc_tokens(self):
  48. return num_tokens_from_messages(self.messages, self.model)
  49. # refer to https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
  50. def num_tokens_from_messages(messages, model):
  51. """Returns the number of tokens used by a list of messages."""
  52. if model in ["wenxin", "xunfei"] or model.startswith(const.GEMINI):
  53. return num_tokens_by_character(messages)
  54. import tiktoken
  55. if model in ["gpt-3.5-turbo-0301", "gpt-35-turbo", "gpt-3.5-turbo-1106", "moonshot", const.LINKAI_35]:
  56. return num_tokens_from_messages(messages, model="gpt-3.5-turbo")
  57. elif model in ["gpt-4-0314", "gpt-4-0613", "gpt-4-32k", "gpt-4-32k-0613", "gpt-3.5-turbo-0613",
  58. "gpt-3.5-turbo-16k", "gpt-3.5-turbo-16k-0613", "gpt-35-turbo-16k", "gpt-4-turbo-preview",
  59. "gpt-4-1106-preview", const.GPT4_TURBO_PREVIEW, const.GPT4_VISION_PREVIEW, const.GPT4_TURBO_01_25,
  60. const.GPT_4o, const.GPT_4O_0806, const.GPT_4o_MINI, const.LINKAI_4o, const.LINKAI_4_TURBO]:
  61. return num_tokens_from_messages(messages, model="gpt-4")
  62. elif model.startswith("claude-3"):
  63. return num_tokens_from_messages(messages, model="gpt-3.5-turbo")
  64. try:
  65. encoding = tiktoken.encoding_for_model(model)
  66. except KeyError:
  67. logger.debug("Warning: model not found. Using cl100k_base encoding.")
  68. encoding = tiktoken.get_encoding("cl100k_base")
  69. if model == "gpt-3.5-turbo":
  70. tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
  71. tokens_per_name = -1 # if there's a name, the role is omitted
  72. elif model == "gpt-4":
  73. tokens_per_message = 3
  74. tokens_per_name = 1
  75. else:
  76. logger.warn(f"num_tokens_from_messages() is not implemented for model {model}. Returning num tokens assuming gpt-3.5-turbo.")
  77. return num_tokens_from_messages(messages, model="gpt-3.5-turbo")
  78. num_tokens = 0
  79. for message in messages:
  80. num_tokens += tokens_per_message
  81. for key, value in message.items():
  82. num_tokens += len(encoding.encode(value))
  83. if key == "name":
  84. num_tokens += tokens_per_name
  85. num_tokens += 3 # every reply is primed with <|start|>assistant<|message|>
  86. return num_tokens
  87. def num_tokens_by_character(messages):
  88. """Returns the number of tokens used by a list of messages."""
  89. tokens = 0
  90. for msg in messages:
  91. tokens += len(msg["content"])
  92. return tokens