Additional Document Format Support

.gitignore

@@ -7,3 +7,4 @@ logs/
 .pageindex/
 dist/
 *.doc_id
+results/

examples/documents/test_doc.txt

@@ -0,0 +1,12 @@
+# Test Document
+
+This is a test document to verify the text conversion feature.
+
+## Section 1
+Here is some text in section 1.
+
+### Subsection 1.1
+More text here.
+
+## Section 2
+And this is section 2.

pageindex/format_converter.py

@@ -0,0 +1,95 @@
+import os
+
+def convert_docx_to_md(docx_path: str) -> str:
+    """
+    Reads a .docx file and converts it to markdown, preserving heading levels.
+    Requires python-docx.
+    """
+    try:
+        import docx
+    except ImportError:
+        raise ImportError("The 'python-docx' package is required to parse .docx files. Please install it.")
+
+    doc = docx.Document(docx_path)
+    md_lines = []
+
+    for para in doc.paragraphs:
+        text = para.text.strip()
+        if not text:
+            continue
+
+        style_name = para.style.name if para.style else ""
+
+        # Check if the style is a heading
+        if style_name.startswith('Heading'):
+            try:
+                # e.g., "Heading 1" -> level 1
+                level_str = style_name.replace('Heading', '').strip()
+                level = int(level_str)
+                # Cap the level at 6 for markdown
+                level = min(level, 6)
+                md_lines.append(f"{'#' * level} {text}\n")
+            except ValueError:
+                # Fallback if parsing fails
+                md_lines.append(f"{text}\n")
+        elif style_name == 'Title':
+            md_lines.append(f"# {text}\n")
+        else:
+            md_lines.append(f"{text}\n")
+
+    return "\n".join(md_lines)
+
+
+def convert_html_to_md(html_path: str) -> str:
+    """
+    Reads an .html file and converts it to markdown using BeautifulSoup and markdownify.
+    """
+    try:
+        from bs4 import BeautifulSoup
+        import markdownify
+    except ImportError:
+        raise ImportError("The 'beautifulsoup4' and 'markdownify' packages are required to parse .html files.")
+
+    with open(html_path, 'r', encoding='utf-8') as f:
+        html_content = f.read()
+
+    soup = BeautifulSoup(html_content, 'html.parser')
+
+    # Remove script and style elements
+    for script in soup(["script", "style", "nav", "footer", "header", "aside"]):
+        script.extract()
+
+    # Convert to markdown with ATX headings (### instead of underlines)
+    md_text = markdownify.markdownify(str(soup), heading_style="ATX")
+
+    # Clean up excessive newlines
+    import re
+    md_text = re.sub(r'\n{3,}', '\n\n', md_text).strip()
+    return md_text
+
+
+def convert_txt_to_md(txt_path: str) -> str:
+    """
+    Reads a plain .txt file. It doesn't need actual conversion, but we wrap it here for consistency.
+    """
+    with open(txt_path, 'r', encoding='utf-8') as f:
+        return f.read()
+
+
+def convert_to_markdown(file_path: str) -> str:
+    """
+    Determines the file type and returns its markdown representation.
+    """
+    ext = os.path.splitext(file_path)[1].lower()
+
+    if ext == '.docx':
+        return convert_docx_to_md(file_path)
+    elif ext in ['.html', '.htm']:
+        return convert_html_to_md(file_path)
+    elif ext == '.txt':
+        return convert_txt_to_md(file_path)
+    elif ext in ['.md', '.markdown']:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return f.read()
+    else:
+        raise ValueError(f"Unsupported file format for conversion: {ext}")

requirements.txt

@@ -9,3 +9,6 @@ python-dotenv==1.2.2
 pyyaml==6.0.2
 regex>=2024.0.0
 sortedcontainers==2.4.0
+python-docx>=1.1.0
+beautifulsoup4>=4.12.0
+markdownify>=0.13.1

run_pageindex.py

@@ -10,6 +10,9 @@
     parser = argparse.ArgumentParser(description='Process PDF or Markdown document and generate structure')
     parser.add_argument('--pdf_path', type=str, help='Path to the PDF file')
     parser.add_argument('--md_path', type=str, help='Path to the Markdown file')
+    parser.add_argument('--docx_path', type=str, help='Path to the DOCX file')
+    parser.add_argument('--html_path', type=str, help='Path to the HTML file')
+    parser.add_argument('--txt_path', type=str, help='Path to the TXT file')
     parser.add_argument('--mode', choices=['flash', 'standard'], default='flash',
                       help='Processing mode (default: flash)')
     parser.add_argument('--flash', action='store_true', default=False,
@@ -58,10 +61,39 @@
         args.mode = 'flash'
 
     # Validate that exactly one file type is specified
-    if not args.pdf_path and not args.md_path:
-        raise ValueError("Either --pdf_path or --md_path must be specified")
-    if args.pdf_path and args.md_path:
-        raise ValueError("Only one of --pdf_path or --md_path can be specified")
+    paths_provided = sum(x is not None for x in [args.pdf_path, args.md_path, args.docx_path, args.html_path, args.txt_path])
+    if paths_provided == 0:
+        raise ValueError("One of --pdf_path, --md_path, --docx_path, --html_path, or --txt_path must be specified")
+    if paths_provided > 1:
+        raise ValueError("Only one file path can be specified")
+
+    if args.docx_path or args.html_path or args.txt_path:
+        from pageindex.format_converter import convert_to_markdown
+        source_path = args.docx_path or args.html_path or args.txt_path
+
+        if args.docx_path and not args.docx_path.lower().endswith('.docx'):
+            raise ValueError("DOCX file must have .docx extension")
+        if args.html_path and not args.html_path.lower().endswith(('.html', '.htm')):
+            raise ValueError("HTML file must have .html or .htm extension")
+        if args.txt_path and not args.txt_path.lower().endswith('.txt'):
+            raise ValueError("TXT file must have .txt extension")
+        if not os.path.isfile(source_path):
+            raise ValueError(f"File not found: {source_path}")
+
+        print(f"Converting {source_path} to Markdown...")
+        md_content = convert_to_markdown(source_path)
+
+        output_dir = './results'
+        os.makedirs(output_dir, exist_ok=True)
+        filename = os.path.splitext(os.path.basename(source_path))[0]
+        tmp_md_path = os.path.join(output_dir, f"{filename}_converted.md")
+
+        with open(tmp_md_path, 'w', encoding='utf-8') as f:
+            f.write(md_content)
+
+        print(f"Converted markdown saved to: {tmp_md_path}")
+        # Route to the markdown logic
+        args.md_path = tmp_md_path
     if args.optimize in ('full', 'merge') and not (args.pdf_path and args.mode == 'flash'):
         raise ValueError("--optimize requires Flash mode with --pdf_path")
     if args.optimize is None: