| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455 |
- import fitz
- import io
- from docx import Document
- from dashscope import get_tokenizer # dashscope版本 >= 1.14.0
- from app.service.auth import decode_access_token
- async def get_str_token(input_str):
- # 获取tokenizer对象，目前只支持通义千问系列模型
- tokenizer = get_tokenizer('qwen-turbo')
- # 将字符串切分成token并转换为token id
- tokens = tokenizer.encode(input_str)
- # print(f"经过切分后的token id为：{tokens}。")
- # # 经过切分后的token id为： [31935, 64559, 99320, 56007, 100629, 104795, 99788, 1773]
- # print(f"经过切分后共有{len(tokens)}个token")
- # # 经过切分后共有8个token
- #
- # # 将token id转化为字符串并打印出来
- # for i in range(len(tokens)):
- # print(f"token id为{tokens[i]}对应的字符串为：{tokenizer.decode(tokens[i])}")
- return len(tokens)
- async def read_pdf(pdf_stream):
- text = ""
- with fitz.open(stream=pdf_stream, filetype="pdf") as pdf_document:
- for page in pdf_document:
- text += page.get_text()
- return text
- async def read_word(word_stream):
- # 使用 python-docx 打开 Word 文件流
- doc = Document(io.BytesIO(word_stream))
- # 提取每个段落的文本
- text = ""
- for para in doc.paragraphs:
- text += para.text
- return text
- async def read_file(file, filename, content_type):
- text = ""
- if content_type == "application/pdf" or filename.endswith('.pdf'):
- # 提取 PDF 内容
- text = await read_pdf(file)
- elif content_type == "application/vnd.openxmlformats-officedocument.wordprocessingml.document" or filename.endswith(
- '.docx'):
- text = await read_word(file)
- return await get_str_token(text)
|
