feat(pdf): export PDFs to DocLang .dclx archives

.gitignore

@@ -7,6 +7,9 @@
 *.snap.new
 __pycache__/
 .venv/
+# maturin develop drops the built extension next to the package
+python/anydoc/_anydoc*.so
+python/anydoc/_anydoc*.pyd
 # maturin build output from the CI python job
 /python/wheels/
 # wasm-pack build output: pkg/ is the npm package, www/pkg/ the demo site copy

Cargo.toml

@@ -19,17 +19,19 @@ include = ["/src", "/examples/convert.rs", "/README.md", "/LICENSE"]
 
 [dev-dependencies]
 insta = "1"
-sha2 = "0.11"
 
 [dependencies]
 cfb = "0.14.0"
 csv = "1.4.0"
 flate2 = "1"
 encoding_rs = "0.8.35"
 log = "0.4"
+lopdf = "0.42"
 pdf-inspector = "1.14.2"
+sha2 = "0.11"
 quick-xml = "0.41.0"
 zip = { version = "9.5.1", default-features = false, features = ["deflate"] }
+weezl = "0.1"
 
 [profile.release]
 lto = "thin"

README.md

@@ -130,6 +130,9 @@ let markdown = anydoc::to_markdown_bytes(&bytes, anydoc::Format::Csv)?;
 
 // Or stop at the document model, which also carries embedded assets:
 let document = anydoc::to_document(&bytes, None)?;
+
+// PDFs (including scanned pages) also export to DocLang `.dclx` archives:
+let dclx = anydoc::to_doclang_bytes(&bytes, anydoc::Format::Pdf, None)?;
 ```
 
 ## OCR
@@ -144,6 +147,14 @@ anydoc reads text-based PDFs locally but does no OCR, so a PDF with scanned or i
 
 Only documents that need OCR leave the machine, and the whole document goes, since Parse has no page selection. If Parse cannot convert it, Node rejects with `code: 'hosted'` and Python raises `HostedError`. `--api-url`, `apiUrl` and `api_url`, else `FIRECRAWL_API_URL`, point at another Parse deployment. `--parse-mode`, `parseMode` and `parse_mode` pick the Parse PDF mode: `auto` (the default) extracts text and falls back to OCR, `fast` extracts text only, `ocr` OCRs every page. The Rust crate has no `ocr` option and never makes network calls.
 
+## DocLang export
+
+The Rust crate can also export PDFs to [DocLang](https://github.com/doclang-project/doclang) `.dclx` archives (DocLang 0.9, OPC ZIP): `to_doclang(path, hints)` and `to_doclang_bytes(bytes, format, hints)` return the archive bytes. Text pages become positioned `<text>`/`<heading>` runs with inline formatting, links, and AcroForm fields; list markers (`•`, `1.`, `a)`, `iv.`) become `<list>`/`<ldiv>`/`<marker>` structure; tables become OTSL `<table>` grids — from `Table`/`TR`/`TD` elements when the PDF is tagged, and from drawn rules (stroked lines and thin `re` rects, including inside Form XObjects) otherwise, with missing interior rules resolving to `lcel`/`ucel`/`xcel` spans. Scanned or image-only pages export as `<picture>` elements backed by extracted raster parts under `pages/` or `assets/` for the supported encodings (JPEG, JPEG 2000, CCITT fax, and Flate/LZW/ASCII85 samples repackaged as PNG) — no OCR required; unsupported encodings still emit a positioned `<picture>` without a raster. The original PDF travels inside the archive as the `/source` part. `DoclangHints` toggles structure detection: `lists`, `tables`, and `pages_with_tables` (restrict the ruled-table detector to 1-indexed pages you know contain tables) — pass `None` for the defaults.
+
+```bash
+cargo run --example convert -- scan.pdf --doclang -o scan.dclx
+```
+
 ## Features
 
 - **One output for every format.** Each format parses into a shared document model and renders through a single Markdown serializer, so escaping, tables, heading anchors, and footnotes behave identically whether the input was a `.doc` from 2003 or a `.pptx` from yesterday.

examples/convert.rs

@@ -1,19 +1,20 @@
-//! Convert a document to Markdown:
-//! `cargo run --example convert -- <file> [-f csv] [-o out.md] [--assets dir]`
+//! Convert a document to Markdown (or DocLang with `--doclang`):
+//! `cargo run --example convert -- <file> [-f csv] [-o out] [--assets dir] [--doclang]`
 
 use std::path::{Path, PathBuf};
 use std::process::ExitCode;
 
 use anydoc::{ConvertError, Format};
 
-const USAGE: &str = "usage: convert <file> [-f csv] [-o out.md] [--assets dir]";
+const USAGE: &str = "usage: convert <file> [-f csv] [-o out] [--assets dir] [--doclang]";
 
 fn main() -> ExitCode {
     let args: Vec<String> = std::env::args().skip(1).collect();
     let mut input: Option<PathBuf> = None;
     let mut output: Option<PathBuf> = None;
     let mut format: Option<Format> = None;
     let mut assets: Option<PathBuf> = None;
+    let mut doclang = false;
     let mut i = 0;
     while i < args.len() {
         match args[i].as_str() {
@@ -34,6 +35,7 @@ fn main() -> ExitCode {
                 i += 1;
                 assets = args.get(i).map(PathBuf::from);
             }
+            "--doclang" => doclang = true,
             other => input = Some(PathBuf::from(other)),
         }
         i += 1;
@@ -43,7 +45,7 @@ fn main() -> ExitCode {
         return ExitCode::FAILURE;
     };
 
-    match run(&input, output.as_deref(), format, assets.as_deref()) {
+    match run(&input, output.as_deref(), format, assets.as_deref(), doclang) {
         Ok(()) => ExitCode::SUCCESS,
         Err(e) => {
             eprintln!("error: {e:#}");
@@ -57,6 +59,7 @@ fn run(
     output: Option<&Path>,
     format: Option<Format>,
     assets: Option<&Path>,
+    doclang: bool,
 ) -> Result<(), ConvertError> {
     let bytes = std::fs::read(input)?;
     // Without -f the format comes from the file content, with the extension as
@@ -73,21 +76,26 @@ fn run(
         };
 
     let start = std::time::Instant::now();
-    let markdown = anydoc::to_markdown_bytes(&bytes, format)?;
+    let payload = if doclang {
+        anydoc::to_doclang_bytes(&bytes, format, None)?
+    } else {
+        anydoc::to_markdown_bytes(&bytes, format)?.into_bytes()
+    };
     let elapsed = start.elapsed().as_secs_f64() * 1000.0;
     eprintln!("converted {} in {}", input.display(), millis(elapsed));
 
     match output {
-        Some(out) => std::fs::write(out, markdown)?,
+        Some(out) => std::fs::write(out, payload)?,
         None => {
             use std::io::Write;
-            let _ = std::io::stdout().write_all(markdown.as_bytes());
+            let _ = std::io::stdout().write_all(&payload);
         }
     }
 
     // Images and embedded objects live on the document model, not in the
-    // Markdown, so they need a second pass to write out.
-    if let Some(dir) = assets {
+    // Markdown, so they need a second pass to write out. DocLang output
+    // carries its rasters inside the archive already.
+    if let Some(dir) = assets.filter(|_| !doclang) {
         let document = anydoc::to_document(&bytes, format)?;
         std::fs::create_dir_all(dir)?;
         let stem = input.file_stem().unwrap_or_default().to_string_lossy();

python/README.md

@@ -58,6 +58,22 @@ markdown = anydoc.to_markdown("scan.pdf", ocr="hosted")
 markdown = anydoc.to_markdown("scan.pdf", ocr="hosted", parse_mode="ocr")
 ```
 
+## DocLang export
+
+`to_doclang` and `to_doclang_bytes` export a PDF to DocLang, returned as the `.dclx` OPC ZIP archive: `document.xml` plus the extracted raster parts and the original PDF as the `source` part. Scanned pages export as `<picture>` elements with extracted rasters, so no OCR is needed. Non-PDF formats raise `UnsupportedError`.
+
+```python
+# From a file path, or from bytes with the format detected:
+dclx = anydoc.to_doclang("report.pdf")
+dclx = anydoc.to_doclang_bytes(pdf_bytes)
+```
+
+`lists` and `tables` steer structure detection (both on by default); `pages_with_tables` restricts the ruled-table detector to those 1-indexed pages.
+
+```python
+dclx = anydoc.to_doclang("report.pdf", lists=False, pages_with_tables=[1, 3])
+```
+
 ## Errors
 
 A conversion raises only when no complete Markdown could come out of the file. The exception type names what went wrong:

python/anydoc/__init__.py

@@ -35,6 +35,8 @@
     format_from_path,
     to_document,
 )
+from anydoc._anydoc import to_doclang as _to_doclang
+from anydoc._anydoc import to_doclang_bytes as _to_doclang_bytes
 from anydoc._anydoc import to_markdown as _to_markdown
 from anydoc._anydoc import to_markdown_bytes as _to_markdown_bytes
 
@@ -109,6 +111,39 @@ def to_markdown_bytes(
     return _parse_hosted(bytes(data), "document.pdf", parse_mode, api_key, api_url)
 
 
+def to_doclang(
+    path: "str | os.PathLike[str]",
+    *,
+    lists: bool = True,
+    tables: bool = True,
+    pages_with_tables: "list[int] | None" = None,
+) -> bytes:
+    """Convert a document file to a DocLang `.dclx` archive (OPC ZIP,
+    returned as bytes). Only PDF has a DocLang exporter; scanned pages
+    export as `<picture>` elements with extracted rasters, so no OCR is
+    needed. The format is detected from the file content; the extension
+    is the fallback. `lists` and `tables` steer structure detection, and
+    `pages_with_tables` restricts the ruled-table detector to those
+    1-indexed pages."""
+    return _to_doclang(path, lists=lists, tables=tables, pages_with_tables=pages_with_tables)
+
+
+def to_doclang_bytes(
+    data: "bytes | bytearray",
+    format: "Format | None" = None,
+    *,
+    lists: bool = True,
+    tables: bool = True,
+    pages_with_tables: "list[int] | None" = None,
+) -> bytes:
+    """Convert an in-memory document to a DocLang `.dclx` archive. Without
+    a format, it is detected from the content. Hints are as for
+    `to_doclang`."""
+    return _to_doclang_bytes(
+        data, format, lists=lists, tables=tables, pages_with_tables=pages_with_tables
+    )
+
+
 _API_URL = "https://api.firecrawl.dev"
 _TIMEOUT_SECONDS = 300
 
@@ -226,6 +261,8 @@ def _version() -> str:
     "format_from_bytes",
     "format_from_extension",
     "format_from_path",
+    "to_doclang",
+    "to_doclang_bytes",
     "to_document",
     "to_markdown",
     "to_markdown_bytes",

python/anydoc/_anydoc.pyi

@@ -71,6 +71,33 @@ def to_markdown_bytes(data: bytes | bytearray, format: Format | None = None) ->
     detected from the content, which signature-less formats (CSV) have to
     name explicitly."""
 
+def to_doclang(
+    path: str | os.PathLike[str],
+    *,
+    lists: bool = True,
+    tables: bool = True,
+    pages_with_tables: list[int] | None = None,
+) -> bytes:
+    """Convert a document file to a DocLang `.dclx` archive (OPC ZIP,
+    returned as bytes). Only PDF has a DocLang exporter; scanned pages
+    export as `<picture>` elements with extracted rasters, so no OCR is
+    needed. The format is detected from the file content; the extension
+    is the fallback. `lists` and `tables` steer structure detection, and
+    `pages_with_tables` restricts the ruled-table detector to those
+    1-indexed pages."""
+
+def to_doclang_bytes(
+    data: bytes | bytearray,
+    format: Format | None = None,
+    *,
+    lists: bool = True,
+    tables: bool = True,
+    pages_with_tables: list[int] | None = None,
+) -> bytes:
+    """Convert an in-memory document to a DocLang `.dclx` archive. Without
+    a format, it is detected from the content. Hints are as for
+    `to_doclang`."""
+
 def to_document(data: bytes | bytearray, format: Format | None = None) -> Document:
     """Parse an in-memory document into the document model, which also
     carries the embedded assets. Without a format, it is detected from the

python/src/lib.rs

@@ -177,6 +177,43 @@ fn to_markdown_bytes(py: Python<'_>, data: Vec<u8>, format: Option<&str>) -> PyR
     py.detach(|| anydoc::to_markdown_bytes(&data, format)).map_err(|e| convert_error(py, e))
 }
 
+/// Convert a document file to a DocLang `.dclx` archive (OPC ZIP, returned
+/// as bytes). Only PDF has a DocLang exporter; scanned pages export as
+/// `<picture>` elements with extracted rasters, so no OCR is needed. The
+/// format is detected from the file content; the extension is the
+/// fallback. `lists` and `tables` steer structure detection, and
+/// `pages_with_tables` restricts the ruled-table detector to those
+/// 1-indexed pages.
+#[pyfunction]
+#[pyo3(signature = (path, *, lists=true, tables=true, pages_with_tables=None))]
+fn to_doclang(
+    py: Python<'_>,
+    path: PathBuf,
+    lists: bool,
+    tables: bool,
+    pages_with_tables: Option<Vec<u32>>,
+) -> PyResult<Vec<u8>> {
+    let hints = anydoc::DoclangHints { lists, tables, pages_with_tables };
+    py.detach(|| anydoc::to_doclang(&path, hints)).map_err(|e| convert_error(py, e))
+}
+
+/// Convert an in-memory document to a DocLang `.dclx` archive. Without a
+/// format, it is detected from the content. Hints are as for `to_doclang`.
+#[pyfunction]
+#[pyo3(signature = (data, format=None, *, lists=true, tables=true, pages_with_tables=None))]
+fn to_doclang_bytes(
+    py: Python<'_>,
+    data: Vec<u8>,
+    format: Option<&str>,
+    lists: bool,
+    tables: bool,
+    pages_with_tables: Option<Vec<u32>>,
+) -> PyResult<Vec<u8>> {
+    let format = format.map(parse_format).transpose()?;
+    let hints = anydoc::DoclangHints { lists, tables, pages_with_tables };
+    py.detach(|| anydoc::to_doclang_bytes(&data, format, hints)).map_err(|e| convert_error(py, e))
+}
+
 /// Parse an in-memory document into the document model, which also carries
 /// the embedded assets. Without a format, it is detected from the content.
 ///
@@ -203,6 +240,8 @@ fn _anydoc(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(format_from_path, m)?)?;
     m.add_function(wrap_pyfunction!(to_markdown, m)?)?;
     m.add_function(wrap_pyfunction!(to_markdown_bytes, m)?)?;
+    m.add_function(wrap_pyfunction!(to_doclang, m)?)?;
+    m.add_function(wrap_pyfunction!(to_doclang_bytes, m)?)?;
     m.add_function(wrap_pyfunction!(to_document, m)?)?;
     m.add_class::<document::Asset>()?;
     m.add_class::<document::Block>()?;

python/tests/test_anydoc.py

@@ -20,6 +20,7 @@
 ENCRYPTED = FIXTURES / "malformed" / "encrypted--errors.odt"
 ZIPBOMB = FIXTURES / "abuse" / "zipbomb--errors.docx"
 MIXED = FIXTURES / "pdf" / "handmade-mixed.pdf"
+TEXT_PDF = FIXTURES / "pdf" / "text.pdf"
 
 HOSTED_MARKDOWN = "# Read by the hosted parser\n"
 
@@ -86,6 +87,32 @@ def test_to_markdown_bytes_detects_the_format_when_none_is_named(self):
             anydoc.to_markdown_bytes(CSV.read_bytes())
         self.assertIn("| --- |", anydoc.to_markdown_bytes(CSV.read_bytes(), "csv"))
 
+    def test_to_doclang_exports_a_dclx_archive(self):
+        dclx = anydoc.to_doclang_bytes(MIXED.read_bytes(), "pdf")
+        self.assertIsInstance(dclx, bytes)
+        with zipfile.ZipFile(io.BytesIO(dclx)) as archive:
+            self.assertIn("document.xml", archive.namelist())
+            xml = archive.read("document.xml").decode()
+        self.assertIn("<doclang", xml)
+        self.assertIn("<picture", xml)  # the mixed fixture's image
+
+    def test_to_doclang_detects_the_format_and_steers_lists(self):
+        with zipfile.ZipFile(io.BytesIO(anydoc.to_doclang(TEXT_PDF))) as archive:
+            xml = archive.read("document.xml").decode()
+        self.assertIn("<list", xml)
+        dclx = anydoc.to_doclang_bytes(TEXT_PDF.read_bytes())
+        with zipfile.ZipFile(io.BytesIO(dclx)) as archive:
+            xml = archive.read("document.xml").decode()
+        self.assertIn("<list", xml)
+        dclx = anydoc.to_doclang_bytes(TEXT_PDF.read_bytes(), "pdf", lists=False)
+        with zipfile.ZipFile(io.BytesIO(dclx)) as archive:
+            xml = archive.read("document.xml").decode()
+        self.assertNotIn("<list", xml)
+
+    def test_to_doclang_rejects_non_pdf_formats(self):
+        with self.assertRaises(anydoc.UnsupportedError):
+            anydoc.to_doclang_bytes(RICH.read_bytes(), "docx")
+
     def test_to_document_exposes_the_document_model(self):
         document = anydoc.to_document(OUTLINE.read_bytes(), "docx")
         heading = next(block for block in document.blocks if block.kind == "heading")