fix(security): resolve Python dependency vulnerabilities (#407)

This pull request primarily updates dependencies across several Python
projects to newer versions, switches some dependencies to local
development mode for easier testing and development, and introduces
improved test utilities for deterministic testing. The updates are aimed
at maintaining compatibility, leveraging new features, and ensuring more
reliable and maintainable test environments.

Dependency updates and improvements:

* Updated multiple dependencies to newer versions across
`libs/admin-api-lib`, `libs/extractor-api-lib`, `libs/rag-core-api`,
`libs/rag-core-lib`, `services/document-extractor`,
`services/mcp-server`, and `services/rag-backend`. Notable updates
include `openai`, `starlette`, `python-multipart`, `pytest`,
`langgraph-checkpoint`, `camelot-py`, `docling`, and others.
[[1]](diffhunk://#diff-9c5aeb0db77c2eec077d07ddc3b3810ae1a4a1e50ee7061fba37a46706c513fbL128-R134)
[[2]](diffhunk://#diff-dede389bcfb615c4b45cd1da7ac14cbe9535305f41f19cce09e321c91a8bb323L118-R126)
[[3]](diffhunk://#diff-dede389bcfb615c4b45cd1da7ac14cbe9535305f41f19cce09e321c91a8bb323L135-R143)
[[4]](diffhunk://#diff-9c4162cc1c16dd4c7ec5e95e79df285e8c0882a1db7ff2892c746a0537d26c36L35-R51)
[[5]](diffhunk://#diff-b19ab043535569caf9345971969d115d6515ae951a21b00a278145a28230fba1L32-R34)
[[6]](diffhunk://#diff-a32cd883126f65652f92c8ecc411d949b7bcf95edb2156c36dc2c1b7063ee690L24-R26)
[[7]](diffhunk://#diff-575f4ba32d7ff340b37eb2f875cb9574553092b79335faadd5f3b6be662b6925L19-R19)
[[8]](diffhunk://#diff-bda9860363f25ca7829f0bc0121455b5cfea15f6ecc4e98d168aba411d9653c9L76-R76)
* Changed `rag-core-lib` dependency to use a local development path
instead of a fixed version in several `pyproject.toml` files,
facilitating local development and testing.
[[1]](diffhunk://#diff-9c5aeb0db77c2eec077d07ddc3b3810ae1a4a1e50ee7061fba37a46706c513fbL116-R116)
[[2]](diffhunk://#diff-dede389bcfb615c4b45cd1da7ac14cbe9535305f41f19cce09e321c91a8bb323L96-R97)
[[3]](diffhunk://#diff-9c4162cc1c16dd4c7ec5e95e79df285e8c0882a1db7ff2892c746a0537d26c36L23-R23)

Testing and development enhancements:

* Added a new `MockSparseEmbeddings` class in
`libs/rag-core-api/tests/mocks/mock_sparse_embeddings.py` to provide
deterministic sparse embeddings for tests, and integrated it into the
test setup to improve test reliability.
[[1]](diffhunk://#diff-4d25f80941e8e49128a0bd00f1a9424f75a689245809af4cd54198dbb70c3a53R1-R16)
[[2]](diffhunk://#diff-596a163e3b70fef32bb64c59bf15f33aeab492eaf41c0cc833bbf2e17439c42aR25-R26)
[[3]](diffhunk://#diff-596a163e3b70fef32bb64c59bf15f33aeab492eaf41c0cc833bbf2e17439c42aR73-R74)
* Updated test infrastructure in `rag_api_test.py` to override the
sparse embedder and reranker with mock implementations and to configure
Qdrant collections for sparse vector support.
[[1]](diffhunk://#diff-596a163e3b70fef32bb64c59bf15f33aeab492eaf41c0cc833bbf2e17439c42aR73-R74)
[[2]](diffhunk://#diff-596a163e3b70fef32bb64c59bf15f33aeab492eaf41c0cc833bbf2e17439c42aR97-R99)

Prompt template formatting:

* Improved formatting in prompt template files by adjusting how
multiline strings are defined, enhancing readability and
maintainability.
[[1]](diffhunk://#diff-f45d2e4f21bc9442444429d1a64091e4c50132f371b05b72374956fa02b1d966L30-R34)
[[2]](diffhunk://#diff-e93a1e7be5b80657d6e435a651dab98960b0d1c1e7a8672f4e3927b4cd0c3fe7L21-R25)

Author: a-klos

libs/admin-api-lib/poetry.lock

@@ -113,7 +113,7 @@ flake8-docstrings = "^1.7.0"
 debugpy = "^1.8.14"
 
 [tool.poetry.dependencies]
-rag-core-lib = "==4.2.0"
+rag-core-lib = {path = "../rag-core-lib", develop = true}
 python = "^3.13"
 uvicorn = "^0.47.0"
 fastapi = "^0.136.0"
@@ -125,13 +125,13 @@ tqdm = "^4.67.1"
 langfuse = "^3.10.1"
 redis = "^6.0.0"
 pyyaml = "^6.0.2"
-python-multipart = "^0.0.30"
+python-multipart = "^0.0.32"
 marshmallow = "^3.26.2"
 protobuf = ">=5.29.6,<6.0.0"
 langchain-experimental = "^0.4.0"
-nltk = "^3.9.2"
-starlette = ">=0.49.1"
-langgraph-checkpoint = ">=3.0.0,<4.0.0"
+nltk = "^3.9.4"
+starlette = ">=1.0.1"
+langgraph-checkpoint = ">=4.0.0,<5.0.0"
 
 [tool.pytest.ini_options]
 log_cli = true

libs/admin-api-lib/pyproject.toml

@@ -93,8 +93,8 @@ uvicorn = "^0.47.0"
 dependency-injector = "^4.46.0"
 pydantic-settings = "^2.9.1"
 boto3 = "^1.38.10"
-rag-core-lib = "==4.2.0"
-python-multipart = "^0.0.30"
+rag-core-lib = {path = "../rag-core-lib", develop = true}
+python-multipart = "^0.0.32"
 oauthlib = "^3.2.2"
 requests-oauthlib = "^2.0.0"
 pdfplumber = "0.11.9"
@@ -115,14 +115,15 @@ langchain-community = "^0.4.1"
 atlassian-python-api = "^4.0.3"
 markdownify = "^1.1.0"
 langchain-core = "^1.0.7"
-camelot-py = {extras = ["cv"], version = "^1.0.0"}
+camelot-py = {extras = ["cv"], version = "^2.0.0"}
 fake-useragent = "^2.2.0"
 pypdfium2 = "4.30.0"
 mammoth = "^1.11.0"
 pypandoc-binary = "^1.15"
-starlette = ">=0.49.1"
+starlette = ">=1.0.1"
 markitdown = {version = "^0.1.3", extras = ["all"]}
-docling = "2.95.0"
+docling = "2.97.0"
+docling-core = ">=2.74.1,<3.0.0"
 torch = [
   { version = "2.10.0", source = "pytorch_cpu", markers = "sys_platform == 'darwin'" },
   { version = "2.10.0+cpu", source = "pytorch_cpu", markers = "sys_platform != 'darwin'" },
@@ -132,13 +133,14 @@ torchvision = [
   { version = "0.25.0+82df5f5", source = "pytorch_cpu", markers = "sys_platform == 'win32' and (platform_machine == 'arm64' or platform_machine == 'ARM64')" },
   { version = "0.25.0+cpu", source = "pytorch_cpu", markers = "sys_platform == 'linux' or (sys_platform == 'win32' and platform_machine != 'arm64' and platform_machine != 'ARM64')" },
 ]
-transformers = ">=4.53.3,<5.0.0"
+transformers = "^5.10.2"
+pypdf = "^6.10.2"
 pypandoc = ">=1.6.2"
 pypandoc_binary = ">=1.6.2"
 
 [tool.poetry.group.test.dependencies]
-pytest = "^8.3.5"
-pytest-asyncio = "^0.26.0"
+pytest = "^9.0.3"
+pytest-asyncio = "^1.0.0"
 coverage = "^7.8.0"
 httpx = "^0.28.1"
 

libs/extractor-api-lib/poetry.lock

@@ -20,7 +20,7 @@ homepage = "https://pypi.org/project/rag-core-api"
 
 [tool.poetry.dependencies]
 python = "^3.13"
-rag-core-lib = "==4.2.0"
+rag-core-lib = {path = "../rag-core-lib", develop = true}
 uvicorn = "^0.47.0"
 langchain-qdrant = "^1.1.0"
 dependency-injector = "^4.46.0"
@@ -32,23 +32,23 @@ datasets = "^4.0.0"
 ragas = "^0.4.0"
 flashrank = "^0.2.10"
 pyyaml = "^6.0.2"
-openai = "^1.77.0"
-langgraph = "^1.0.3"
-pillow = "^12.0.0"
+openai = "^2.26.0"
+langgraph = "^1.1.10"
+pillow = "^12.2.0"
 langchain-ollama = "^1.0.0"
 langchain-community = "^0.4.1"
 fastembed = "^0.8.0"
 langdetect = "^1.0.9"
 langfuse = "^3.10.1"
 marshmallow = "^3.26.2"
-langchain-text-splitters = "^1.0.0"
-starlette = ">=0.49.1"
-langgraph-checkpoint = ">=3.0.0,<4.0.0"
+langchain-text-splitters = "^1.1.2"
+starlette = ">=1.0.1"
+langgraph-checkpoint = ">=4.0.0,<5.0.0"
 
 [tool.poetry.group.test.dependencies]
-pytest = "^8.3.5"
+pytest = "^9.0.3"
 coverage = "^7.8.0"
-pytest-asyncio = "^0.26.0"
+pytest-asyncio = "^1.0.0"
 
 [tool.poetry.group.dev.dependencies]
 debugpy = "^1.8.14"

libs/extractor-api-lib/pyproject.toml

@@ -0,0 +1,16 @@
+"""Provide deterministic sparse embeddings for tests."""
+
+from langchain_qdrant import SparseEmbeddings
+from langchain_qdrant.sparse_embeddings import SparseVector
+
+
+class MockSparseEmbeddings(SparseEmbeddings):
+    """Return a fixed sparse vector without loading an external model."""
+
+    def embed_documents(self, texts: list[str]) -> list[SparseVector]:
+        """Embed documents with a deterministic test vector."""
+        return [SparseVector(indices=[0], values=[1.0]) for _ in texts]
+
+    def embed_query(self, text: str) -> SparseVector:
+        """Embed a query with a deterministic test vector."""
+        return SparseVector(indices=[0], values=[1.0])

libs/poetry.lock

@@ -22,6 +22,8 @@
 
 from mocks.mock_environment_variables import mock_environment_variables
 from mocks.mock_logging_directory import mock_logging_config
+from mocks.mock_reranker import MockReranker
+from mocks.mock_sparse_embeddings import MockSparseEmbeddings
 from mocks import MockLangfuseManager
 
 mock_environment_variables()
@@ -68,6 +70,8 @@ async def adjusted_app() -> AsyncGenerator[FastAPI, None]:
                 embedder=providers.Singleton(FakeEmbeddings, **FakeEmbedderSettings().model_dump()),
             )
         )
+        app.container.sparse_embedder.override(providers.Singleton(MockSparseEmbeddings))
+        app.container.reranker.override(providers.Singleton(MockReranker))
 
         # Override Langfuse with mock implementation
         mock_langfuse = providers.Singleton(MagicMock)
@@ -90,6 +94,9 @@ async def adjusted_app() -> AsyncGenerator[FastAPI, None]:
             client.create_collection(
                 collection_name=collection_name,
                 vectors_config=models.VectorParams(size=FakeEmbedderSettings().size, distance=models.Distance.COSINE),
+                sparse_vectors_config={
+                    "langchain-sparse": models.SparseVectorParams(),
+                },
             )
         yield app
         # Clean up