Skip to content

Commit

Permalink
make titles in markdown not be splited with following content (#2971)
Browse files Browse the repository at this point in the history
### What problem does this PR solve?

#2970 
### Type of change

- [ ] Bug Fix (non-breaking change which fixes an issue)
- [x] New Feature (non-breaking change which adds functionality)
  • Loading branch information
KevinHuSh authored Oct 22, 2024
1 parent adb0a93 commit 1fce6ca
Show file tree
Hide file tree
Showing 3 changed files with 12 additions and 4 deletions.
7 changes: 5 additions & 2 deletions rag/app/manual.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,11 @@ def __call__(self, filename, binary=None, from_page=0,
return [(b["text"], b.get("layout_no", ""), self.get_position(b, zoomin))
for i, b in enumerate(self.boxes)], tbls


class Docx(DocxParser):
def __init__(self):
pass

def get_picture(self, document, paragraph):
img = paragraph._element.xpath('.//pic:pic')
if not img:
Expand All @@ -80,6 +82,7 @@ def get_picture(self, document, paragraph):
image = related_part.image
image = Image.open(BytesIO(image.blob))
return image

def concat_img(self, img1, img2):
if img1 and not img2:
return img1
Expand Down Expand Up @@ -160,6 +163,7 @@ def __call__(self, filename, binary=None, from_page=0, to_page=100000, callback=
tbls.append(((None, html), ""))
return ti_list, tbls


def chunk(filename, binary=None, from_page=0, to_page=100000,
lang="Chinese", callback=None, **kwargs):
"""
Expand Down Expand Up @@ -244,6 +248,7 @@ def tag(pn, left, right, top, bottom):
res = tokenize_table(tbls, doc, eng)
res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser))
return res

if re.search(r"\.docx$", filename, re.IGNORECASE):
docx_parser = Docx()
ti_list, tbls = docx_parser(filename, binary,
Expand All @@ -259,8 +264,6 @@ def tag(pn, left, right, top, bottom):
raise NotImplementedError("file type not supported yet(pdf and docx supported)")




if __name__ == "__main__":
import sys

Expand Down
8 changes: 6 additions & 2 deletions rag/app/naive.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,8 +168,12 @@ def __call__(self, filename, binary=None):
sections.append((sec[:int(len(sec) / 2)], ""))
sections.append((sec[int(len(sec) / 2):], ""))
else:
sections.append((sec, ""))
print(tables)
if sections and sections[-1][0].strip().find("#") == 0:
sec_, _ = sections.pop(-1)
sections.append((sec_+"\n"+sec, ""))
else:
sections.append((sec, ""))

for table in tables:
tbls.append(((None, markdown(table, extensions=['markdown.extensions.tables'])), ""))
return sections, tbls
Expand Down
1 change: 1 addition & 0 deletions rag/app/qa.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,7 @@ def chunk(filename, binary=None, lang="Chinese", callback=None, **kwargs):
if sum_question:
res.append(beAdoc(deepcopy(doc), sum_question, markdown(last_answer, extensions=['markdown.extensions.tables']), eng))
return res

elif re.search(r"\.docx$", filename, re.IGNORECASE):
docx_parser = Docx()
qai_list, tbls = docx_parser(filename, binary,
Expand Down

0 comments on commit 1fce6ca

Please sign in to comment.