123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185 |
- from .crazy_utils import request_gpt_model_in_new_thread_with_ui_alive
- from toolbox import CatchException, report_exception, promote_file_to_downloadzone
- from toolbox import update_ui, update_ui_lastest_msg, disable_auto_promotion, write_history_to_file
- import logging
- import requests
- import time
- import random
- ENABLE_ALL_VERSION_SEARCH = True
- def get_meta_information(url, chatbot, history):
- import arxiv
- import difflib
- import re
- from bs4 import BeautifulSoup
- from toolbox import get_conf
- from urllib.parse import urlparse
- session = requests.session()
- proxies = get_conf('proxies')
- headers = {
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
- 'Accept-Encoding': 'gzip, deflate, br',
- 'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
- 'Cache-Control':'max-age=0',
- 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
- 'Connection': 'keep-alive'
- }
- try:
- session.proxies.update(proxies)
- except:
- report_exception(chatbot, history,
- a=f"获取代理失败 无代理状态下很可能无法访问OpenAI家族的模型及谷歌学术 建议:检查USE_PROXY选项是否修改。",
- b=f"尝试直接连接")
- yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
- session.headers.update(headers)
- response = session.get(url)
- # 解析网页内容
- soup = BeautifulSoup(response.text, "html.parser")
- def string_similar(s1, s2):
- return difflib.SequenceMatcher(None, s1, s2).quick_ratio()
- if ENABLE_ALL_VERSION_SEARCH:
- def search_all_version(url):
- time.sleep(random.randint(1,5)) # 睡一会防止触发google反爬虫
- response = session.get(url)
- soup = BeautifulSoup(response.text, "html.parser")
- for result in soup.select(".gs_ri"):
- try:
- url = result.select_one(".gs_rt").a['href']
- except:
- continue
- arxiv_id = extract_arxiv_id(url)
- if not arxiv_id:
- continue
- search = arxiv.Search(
- id_list=[arxiv_id],
- max_results=1,
- sort_by=arxiv.SortCriterion.Relevance,
- )
- try: paper = next(search.results())
- except: paper = None
- return paper
- return None
- def extract_arxiv_id(url):
- # 返回给定的url解析出的arxiv_id,如url未成功匹配返回None
- pattern = r'arxiv.org/abs/([^/]+)'
- match = re.search(pattern, url)
- if match:
- return match.group(1)
- else:
- return None
- profile = []
- # 获取所有文章的标题和作者
- for result in soup.select(".gs_ri"):
- title = result.a.text.replace('\n', ' ').replace(' ', ' ')
- author = result.select_one(".gs_a").text
- try:
- citation = result.select_one(".gs_fl > a[href*='cites']").text # 引用次数是链接中的文本,直接取出来
- except:
- citation = 'cited by 0'
- abstract = result.select_one(".gs_rs").text.strip() # 摘要在 .gs_rs 中的文本,需要清除首尾空格
- # 首先在arxiv上搜索,获取文章摘要
- search = arxiv.Search(
- query = title,
- max_results = 1,
- sort_by = arxiv.SortCriterion.Relevance,
- )
- try: paper = next(search.results())
- except: paper = None
-
- is_match = paper is not None and string_similar(title, paper.title) > 0.90
- # 如果在Arxiv上匹配失败,检索文章的历史版本的题目
- if not is_match and ENABLE_ALL_VERSION_SEARCH:
- other_versions_page_url = [tag['href'] for tag in result.select_one('.gs_flb').select('.gs_nph') if 'cluster' in tag['href']]
- if len(other_versions_page_url) > 0:
- other_versions_page_url = other_versions_page_url[0]
- paper = search_all_version('http://' + urlparse(url).netloc + other_versions_page_url)
- is_match = paper is not None and string_similar(title, paper.title) > 0.90
- if is_match:
- # same paper
- abstract = paper.summary.replace('\n', ' ')
- is_paper_in_arxiv = True
- else:
- # different paper
- abstract = abstract
- is_paper_in_arxiv = False
- logging.info('[title]:' + title)
- logging.info('[author]:' + author)
- logging.info('[citation]:' + citation)
- profile.append({
- 'title': title,
- 'author': author,
- 'citation': citation,
- 'abstract': abstract,
- 'is_paper_in_arxiv': is_paper_in_arxiv,
- })
- chatbot[-1] = [chatbot[-1][0], title + f'\n\n是否在arxiv中(不在arxiv中无法获取完整摘要):{is_paper_in_arxiv}\n\n' + abstract]
- yield from update_ui(chatbot=chatbot, history=[]) # 刷新界面
- return profile
- @CatchException
- def 谷歌检索小助手(txt, llm_kwargs, plugin_kwargs, chatbot, history, system_prompt, web_port):
- disable_auto_promotion(chatbot=chatbot)
- # 基本信息:功能、贡献者
- chatbot.append([
- "函数插件功能?",
- "分析用户提供的谷歌学术(google scholar)搜索页面中,出现的所有文章: binary-husky,插件初始化中..."])
- yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
- # 尝试导入依赖,如果缺少依赖,则给出安装建议
- try:
- import arxiv
- import math
- from bs4 import BeautifulSoup
- except:
- report_exception(chatbot, history,
- a = f"解析项目: {txt}",
- b = f"导入软件依赖失败。使用该模块需要额外依赖,安装方法```pip install --upgrade beautifulsoup4 arxiv```。")
- yield from update_ui(chatbot=chatbot, history=history) # 刷新界面
- return
- # 清空历史,以免输入溢出
- history = []
- meta_paper_info_list = yield from get_meta_information(txt, chatbot, history)
- if len(meta_paper_info_list) == 0:
- yield from update_ui_lastest_msg(lastmsg='获取文献失败,可能触发了google反爬虫机制。',chatbot=chatbot, history=history, delay=0)
- return
- batchsize = 5
- for batch in range(math.ceil(len(meta_paper_info_list)/batchsize)):
- if len(meta_paper_info_list[:batchsize]) > 0:
- i_say = "下面是一些学术文献的数据,提取出以下内容:" + \
- "1、英文题目;2、中文题目翻译;3、作者;4、arxiv公开(is_paper_in_arxiv);4、引用数量(cite);5、中文摘要翻译。" + \
- f"以下是信息源:{str(meta_paper_info_list[:batchsize])}"
- inputs_show_user = f"请分析此页面中出现的所有文章:{txt},这是第{batch+1}批"
- gpt_say = yield from request_gpt_model_in_new_thread_with_ui_alive(
- inputs=i_say, inputs_show_user=inputs_show_user,
- llm_kwargs=llm_kwargs, chatbot=chatbot, history=[],
- sys_prompt="你是一个学术翻译,请从数据中提取信息。你必须使用Markdown表格。你必须逐个文献进行处理。"
- )
- history.extend([ f"第{batch+1}批", gpt_say ])
- meta_paper_info_list = meta_paper_info_list[batchsize:]
- chatbot.append(["状态?",
- "已经全部完成,您可以试试让AI写一个Related Works,例如您可以继续输入Write a \"Related Works\" section about \"你搜索的研究领域\" for me."])
- msg = '正常'
- yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
- path = write_history_to_file(history)
- promote_file_to_downloadzone(path, chatbot=chatbot)
- chatbot.append(("完成了吗?", path));
- yield from update_ui(chatbot=chatbot, history=history, msg=msg) # 刷新界面
|