test_markdown_format.py 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167
  1. sample = """
  2. [1]: https://baike.baidu.com/item/%E8%B4%A8%E8%83%BD%E6%96%B9%E7%A8%8B/1884527 "质能方程(质能方程式)_百度百科"
  3. [2]: https://www.zhihu.com/question/348249281 "如何理解质能方程 E=mc²? - 知乎"
  4. [3]: https://zhuanlan.zhihu.com/p/32597385 "质能方程的推导与理解 - 知乎 - 知乎专栏"
  5. 你好,这是必应。质能方程是描述质量与能量之间的当量关系的方程[^1^][1]。用tex格式,质能方程可以写成$$E=mc^2$$,其中$E$是能量,$m$是质量,$c$是光速[^2^][2] [^3^][3]。
  6. """
  7. import re
  8. def preprocess_newbing_out(s):
  9. pattern = r"\^(\d+)\^" # 匹配^数字^
  10. pattern2 = r"\[(\d+)\]" # 匹配^数字^
  11. def sub(m):
  12. return "\\[" + m.group(1) + "\\]" # 将匹配到的数字作为替换值
  13. result = re.sub(pattern, sub, s) # 替换操作
  14. if "[1]" in result:
  15. result += (
  16. '<br/><hr style="border-top: dotted 1px #44ac5c;"><br/><small>'
  17. + "<br/>".join(
  18. [
  19. re.sub(pattern2, sub, r)
  20. for r in result.split("\n")
  21. if r.startswith("[")
  22. ]
  23. )
  24. + "</small>"
  25. )
  26. return result
  27. def close_up_code_segment_during_stream(gpt_reply):
  28. """
  29. 在gpt输出代码的中途(输出了前面的```,但还没输出完后面的```),补上后面的```
  30. Args:
  31. gpt_reply (str): GPT模型返回的回复字符串。
  32. Returns:
  33. str: 返回一个新的字符串,将输出代码片段的“后面的```”补上。
  34. """
  35. if "```" not in gpt_reply:
  36. return gpt_reply
  37. if gpt_reply.endswith("```"):
  38. return gpt_reply
  39. # 排除了以上两个情况,我们
  40. segments = gpt_reply.split("```")
  41. n_mark = len(segments) - 1
  42. if n_mark % 2 == 1:
  43. # print('输出代码片段中!')
  44. return gpt_reply + "\n```"
  45. else:
  46. return gpt_reply
  47. import markdown
  48. from latex2mathml.converter import convert as tex2mathml
  49. def markdown_convertion(txt):
  50. """
  51. 将Markdown格式的文本转换为HTML格式。如果包含数学公式,则先将公式转换为HTML格式。
  52. """
  53. pre = '<div class="markdown-body">'
  54. suf = "</div>"
  55. if txt.startswith(pre) and txt.endswith(suf):
  56. # print('警告,输入了已经经过转化的字符串,二次转化可能出问题')
  57. return txt # 已经被转化过,不需要再次转化
  58. markdown_extension_configs = {
  59. "mdx_math": {
  60. "enable_dollar_delimiter": True,
  61. "use_gitlab_delimiters": False,
  62. },
  63. }
  64. find_equation_pattern = r'<script type="math/tex(?:.*?)>(.*?)</script>'
  65. def tex2mathml_catch_exception(content, *args, **kwargs):
  66. try:
  67. content = tex2mathml(content, *args, **kwargs)
  68. except:
  69. content = content
  70. return content
  71. def replace_math_no_render(match):
  72. content = match.group(1)
  73. if "mode=display" in match.group(0):
  74. content = content.replace("\n", "</br>")
  75. return f'<font color="#00FF00">$$</font><font color="#FF00FF">{content}</font><font color="#00FF00">$$</font>'
  76. else:
  77. return f'<font color="#00FF00">$</font><font color="#FF00FF">{content}</font><font color="#00FF00">$</font>'
  78. def replace_math_render(match):
  79. content = match.group(1)
  80. if "mode=display" in match.group(0):
  81. if "\\begin{aligned}" in content:
  82. content = content.replace("\\begin{aligned}", "\\begin{array}")
  83. content = content.replace("\\end{aligned}", "\\end{array}")
  84. content = content.replace("&", " ")
  85. content = tex2mathml_catch_exception(content, display="block")
  86. return content
  87. else:
  88. return tex2mathml_catch_exception(content)
  89. def markdown_bug_hunt(content):
  90. """
  91. 解决一个mdx_math的bug(单$包裹begin命令时多余<script>)
  92. """
  93. content = content.replace(
  94. '<script type="math/tex">\n<script type="math/tex; mode=display">',
  95. '<script type="math/tex; mode=display">',
  96. )
  97. content = content.replace("</script>\n</script>", "</script>")
  98. return content
  99. if ("$" in txt) and ("```" not in txt): # 有$标识的公式符号,且没有代码段```的标识
  100. # convert everything to html format
  101. split = markdown.markdown(text="---")
  102. convert_stage_1 = markdown.markdown(
  103. text=txt,
  104. extensions=["mdx_math", "fenced_code", "tables", "sane_lists"],
  105. extension_configs=markdown_extension_configs,
  106. )
  107. convert_stage_1 = markdown_bug_hunt(convert_stage_1)
  108. # re.DOTALL: Make the '.' special character match any character at all, including a newline; without this flag, '.' will match anything except a newline. Corresponds to the inline flag (?s).
  109. # 1. convert to easy-to-copy tex (do not render math)
  110. convert_stage_2_1, n = re.subn(
  111. find_equation_pattern,
  112. replace_math_no_render,
  113. convert_stage_1,
  114. flags=re.DOTALL,
  115. )
  116. # 2. convert to rendered equation
  117. convert_stage_2_2, n = re.subn(
  118. find_equation_pattern, replace_math_render, convert_stage_1, flags=re.DOTALL
  119. )
  120. # cat them together
  121. return pre + convert_stage_2_1 + f"{split}" + convert_stage_2_2 + suf
  122. else:
  123. return (
  124. pre
  125. + markdown.markdown(
  126. txt, extensions=["fenced_code", "codehilite", "tables", "sane_lists"]
  127. )
  128. + suf
  129. )
  130. sample = preprocess_newbing_out(sample)
  131. sample = close_up_code_segment_during_stream(sample)
  132. sample = markdown_convertion(sample)
  133. with open("tmp.html", "w", encoding="utf8") as f:
  134. f.write(
  135. """
  136. <head>
  137. <title>My Website</title>
  138. <link rel="stylesheet" type="text/css" href="style.css">
  139. </head>
  140. """
  141. )
  142. f.write(sample)