diff --git a/app/io/document.py b/app/io/document.py new file mode 100644 index 0000000..f7c3fbf --- /dev/null +++ b/app/io/document.py @@ -0,0 +1,72 @@ +# All comments in English. + +from docx import Document +from odf.opendocument import load +from odf.text import P +from bs4 import BeautifulSoup +import markdown + +def normalize_newlines(value): + if value is None: + return "" + return ( + str(value) + .replace("\r\n", "\\n") + .replace("\n\r", "\\n") + .replace("\r", "\\n") + .replace("\n", "\\n") + ) + +def convert_docx(path): + doc = Document(path) + lines = [] + for p in doc.paragraphs: + lines.append(normalize_newlines(p.text)) + return "\n".join(lines) + +def convert_doc(path): + # DOC (binary) requires LibreOffice headless conversion + import subprocess, tempfile, os + tmp = tempfile.mktemp(suffix=".docx") + subprocess.run(["soffice", "--headless", "--convert-to", "docx", "--outdir", "/tmp", path]) + return convert_docx(tmp) + +def convert_odt(path): + doc = load(path) + lines = [] + for p in doc.getElementsByType(P): + text = p.firstChild.data if p.firstChild else "" + lines.append(normalize_newlines(text)) + return "\n".join(lines) + +def convert_txt(path): + with open(path, "r", encoding="utf-8", errors="ignore") as f: + return normalize_newlines(f.read()) + +def convert_md(path): + with open(path, "r", encoding="utf-8", errors="ignore") as f: + raw = f.read() + html = markdown.markdown(raw) + soup = BeautifulSoup(html, "html.parser") + return normalize_newlines(soup.get_text()) + +def convert_html(path): + with open(path, "r", encoding="utf-8", errors="ignore") as f: + raw = f.read() + soup = BeautifulSoup(raw, "html.parser") + return normalize_newlines(soup.get_text()) + +def convert_document(path): + if path.endswith(".docx"): + return convert_docx(path) + if path.endswith(".doc"): + return convert_doc(path) + if path.endswith(".odt"): + return convert_odt(path) + if path.endswith(".txt"): + return convert_txt(path) + if path.endswith(".md"): + return convert_md(path) + if path.endswith(".html") or path.endswith(".htm"): + return convert_html(path) + raise ValueError("Unsupported document format")