Merge workspace.toml into pyproject.toml

.github/workflows/ci.yaml

@@ -109,7 +109,7 @@ jobs:
       - name: Install dependencies
         run: |
           # Use UV with system installation for CI (simpler and more reliable)
-          cd ragas && uv pip install --system -e "." --group dev --cache-dir ~/.cache/uv
+          uv pip install --system -e "." --group dev --cache-dir ~/.cache/uv
 
       - name: Run unit tests
         run: |
@@ -171,16 +171,16 @@ jobs:
       - name: Install dependencies
         run: |
           # Use UV with system installation for CI (simpler and more reliable)
-          cd ragas && uv pip install --system -e "." --group dev --cache-dir ~/.cache/uv
+          uv pip install --system -e "." --group dev --cache-dir ~/.cache/uv
 
       - name: Format check (dry run)
         run: |
           # Check if code is properly formatted (without making changes)
           # Note: We use direct commands here instead of the standalone Makefiles
           # to have precise control over CI-specific options like --check for dry-run
           echo "Checking ragas formatting..."
-          ruff format --check ragas/src ragas/tests docs --exclude ragas/src/ragas/_version.py
-          ruff check ragas/src docs ragas/tests --exclude ragas/src/ragas/_version.py
+          ruff format --check ragas/src ragas/tests docs --exclude ragas/src/ragas/_version.py --config pyproject.toml
+          ruff check ragas/src docs ragas/tests --exclude ragas/src/ragas/_version.py --config pyproject.toml
 
       - name: Type check
         run: make type

.readthedocs.yml

@@ -9,6 +9,6 @@ build:
     python: "3.12"
   commands:
     - pip install uv
-    - cd ragas && uv pip install --system -e "." --group docs
+    - uv pip install --system -e "." --group docs
     - if [ -n "$GH_TOKEN" ]; then pip install git+https://${GH_TOKEN}@github.com/squidfunk/mkdocs-material-insiders.git; fi
     - mkdocs build --site-dir $READTHEDOCS_OUTPUT/html

docs/howtos/applications/cost.ipynb

@@ -24,11 +24,12 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain_openai.chat_models import ChatOpenAI\n",
     "from langchain_core.prompt_values import StringPromptValue\n",
+    "from langchain_openai.chat_models import ChatOpenAI\n",
     "\n",
     "# lets import a parser for OpenAI\n",
     "from ragas.cost import get_token_usage_for_openai\n",
@@ -70,9 +71,10 @@
     }
    ],
    "source": [
-    "from ragas import EvaluationDataset\n",
     "from datasets import load_dataset\n",
     "\n",
+    "from ragas import EvaluationDataset\n",
+    "\n",
     "dataset = load_dataset(\"explodinggradients/amnesty_qa\", \"english_v3\")\n",
     "\n",
     "eval_dataset = EvaluationDataset.from_hf_dataset(dataset[\"eval\"])"
@@ -107,9 +109,8 @@
    ],
    "source": [
     "from ragas import evaluate\n",
-    "from ragas.metrics import LLMContextRecall\n",
-    "\n",
     "from ragas.cost import get_token_usage_for_openai\n",
+    "from ragas.metrics import LLMContextRecall\n",
     "\n",
     "result = evaluate(\n",
     "    eval_dataset,\n",
@@ -221,8 +222,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.testset import TestsetGenerator\n",
     "from ragas.llms import llm_factory\n",
+    "from ragas.testset import TestsetGenerator\n",
     "\n",
     "tg = TestsetGenerator(llm=llm_factory(), knowledge_graph=kg)\n",
     "# generating a testset\n",

docs/howtos/customizations/caching.ipynb

@@ -57,6 +57,7 @@
    "outputs": [],
    "source": [
     "from langchain_openai import ChatOpenAI\n",
+    "\n",
     "from ragas.llms import LangchainLLMWrapper\n",
     "\n",
     "cached_llm = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o\"), cache=cacher)"
@@ -70,6 +71,7 @@
    "source": [
     "# if you want to see the cache in action, set the logging level to debug\n",
     "import logging\n",
+    "\n",
     "from ragas.utils import set_logging_level\n",
     "\n",
     "set_logging_level(\"ragas.cache\", logging.DEBUG)"
@@ -88,12 +90,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas import evaluate\n",
-    "from ragas import EvaluationDataset\n",
-    "\n",
-    "from ragas.metrics import FactualCorrectness, AspectCritic\n",
     "from datasets import load_dataset\n",
     "\n",
+    "from ragas import EvaluationDataset, evaluate\n",
+    "from ragas.metrics import AspectCritic, FactualCorrectness\n",
+    "\n",
     "# Define Answer Correctness with AspectCritic\n",
     "answer_correctness = AspectCritic(\n",
     "    name=\"answer_correctness\",\n",

docs/howtos/customizations/metrics/cost.ipynb

@@ -35,11 +35,12 @@
   },
   {
    "cell_type": "code",
+   "execution_count": null,
    "metadata": {},
    "outputs": [],
    "source": [
-    "from langchain_openai.chat_models import ChatOpenAI\n",
     "from langchain_core.prompt_values import StringPromptValue\n",
+    "from langchain_openai.chat_models import ChatOpenAI\n",
     "\n",
     "# lets import a parser for OpenAI\n",
     "from ragas.cost import get_token_usage_for_openai\n",
@@ -75,6 +76,7 @@
    ],
    "source": [
     "from datasets import load_dataset\n",
+    "\n",
     "from ragas import EvaluationDataset\n",
     "from ragas.metrics._aspect_critic import AspectCriticWithReference\n",
     "\n",

docs/howtos/customizations/metrics/modifying-prompts-metrics.ipynb

@@ -165,8 +165,8 @@
    "outputs": [],
    "source": [
     "from ragas.metrics._simple_criteria import (\n",
-    "    SingleTurnSimpleCriteriaWithReferenceInput,\n",
     "    SimpleCriteriaOutput,\n",
+    "    SingleTurnSimpleCriteriaWithReferenceInput,\n",
     ")"
    ]
   },

docs/howtos/customizations/metrics/write_your_own_metric.ipynb

@@ -19,6 +19,7 @@
    "source": [
     "# dataset\n",
     "from datasets import load_dataset\n",
+    "\n",
     "from ragas import EvaluationDataset\n",
     "\n",
     "amnesty_qa = load_dataset(\"explodinggradients/amnesty_qa\", \"english_v3\")\n",
@@ -202,16 +203,16 @@
    "outputs": [],
    "source": [
     "# we are going to create a dataclass that subclasses `MetricWithLLM` and `SingleTurnMetric`\n",
-    "from dataclasses import dataclass, field\n",
-    "\n",
-    "# import the base classes\n",
-    "from ragas.metrics.base import MetricWithLLM, SingleTurnMetric, MetricType\n",
-    "from ragas.metrics import Faithfulness\n",
-    "\n",
     "# import types\n",
     "import typing as t\n",
+    "from dataclasses import dataclass, field\n",
+    "\n",
     "from ragas.callbacks import Callbacks\n",
     "from ragas.dataset_schema import SingleTurnSample\n",
+    "from ragas.metrics import Faithfulness\n",
+    "\n",
+    "# import the base classes\n",
+    "from ragas.metrics.base import MetricType, MetricWithLLM, SingleTurnMetric\n",
     "\n",
     "\n",
     "@dataclass\n",

docs/howtos/customizations/metrics/write_your_own_metric_advanced.ipynb

@@ -68,9 +68,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.prompt import PydanticPrompt\n",
     "from pydantic import BaseModel, Field\n",
     "\n",
+    "from ragas.prompt import PydanticPrompt\n",
+    "\n",
     "\n",
     "class RefusalInput(BaseModel):\n",
     "    user_input: str = Field(description=\"the user request\")\n",
@@ -119,11 +120,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "import typing as t\n",
     "from dataclasses import dataclass, field\n",
-    "from ragas.metrics.base import MetricType\n",
-    "from ragas.messages import AIMessage, HumanMessage, ToolMessage, ToolCall\n",
-    "from ragas import SingleTurnSample, MultiTurnSample\n",
-    "import typing as t"
+    "\n",
+    "from ragas import MultiTurnSample, SingleTurnSample\n",
+    "from ragas.messages import AIMessage, HumanMessage, ToolCall, ToolMessage\n",
+    "from ragas.metrics.base import MetricType"
    ]
   },
   {
@@ -199,6 +201,7 @@
    "outputs": [],
    "source": [
     "from langchain_openai import ChatOpenAI\n",
+    "\n",
     "from ragas.llms.base import LangchainLLMWrapper"
    ]
   },

docs/howtos/customizations/run_config.ipynb

@@ -49,10 +49,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas import EvaluationDataset, SingleTurnSample\n",
-    "from ragas.metrics import Faithfulness\n",
     "from datasets import load_dataset\n",
-    "from ragas import evaluate\n",
+    "\n",
+    "from ragas import EvaluationDataset, SingleTurnSample, evaluate\n",
+    "from ragas.metrics import Faithfulness\n",
     "\n",
     "dataset = load_dataset(\"explodinggradients/amnesty_qa\", \"english_v3\")\n",
     "\n",

docs/howtos/customizations/testgenerator/language_adaptation.ipynb

@@ -55,7 +55,6 @@
    "source": [
     "from langchain_community.document_loaders import DirectoryLoader\n",
     "\n",
-    "\n",
     "path = \"Sample_non_english_corpus/\"\n",
     "loader = DirectoryLoader(path, glob=\"**/*.txt\")\n",
     "docs = loader.load()"
@@ -103,10 +102,10 @@
     }
    ],
    "source": [
-    "from ragas.llms import LangchainLLMWrapper\n",
+    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n",
+    "\n",
     "from ragas.embeddings import LangchainEmbeddingsWrapper\n",
-    "from langchain_openai import ChatOpenAI\n",
-    "from langchain_openai import OpenAIEmbeddings\n",
+    "from ragas.llms import LangchainLLMWrapper\n",
     "\n",
     "generator_llm = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o-mini\"))\n",
     "generator_embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())"

docs/howtos/customizations/testgenerator/persona_generator.ipynb

@@ -68,9 +68,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "from ragas.llms import llm_factory\n",
     "from ragas.testset import TestsetGenerator\n",
     "from ragas.testset.graph import KnowledgeGraph\n",
-    "from ragas.llms import llm_factory\n",
     "\n",
     "# Load the knowledge graph\n",
     "kg = KnowledgeGraph.load(\"../../../../experiments/gitlab_kg.json\")\n",
@@ -191,9 +191,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.testset.persona import generate_personas_from_kg\n",
-    "from ragas.testset.graph import KnowledgeGraph\n",
     "from ragas.llms import llm_factory\n",
+    "from ragas.testset.graph import KnowledgeGraph\n",
+    "from ragas.testset.persona import generate_personas_from_kg\n",
     "\n",
     "kg = KnowledgeGraph.load(\"../../../../experiments/gitlab_kg.json\")\n",
     "llm = llm_factory(\"gpt-4o-mini\")\n",

docs/howtos/customizations/testgenerator/testgen-custom-single-hop.ipynb

@@ -26,7 +26,6 @@
    "source": [
     "from langchain_community.document_loaders import DirectoryLoader\n",
     "\n",
-    "\n",
     "path = \"Sample_Docs_Markdown/\"\n",
     "loader = DirectoryLoader(path, glob=\"**/*.md\")\n",
     "docs = loader.load()"
@@ -58,9 +57,7 @@
     }
    ],
    "source": [
-    "from ragas.testset.graph import KnowledgeGraph\n",
-    "from ragas.testset.graph import Node, NodeType\n",
-    "\n",
+    "from ragas.testset.graph import KnowledgeGraph, Node, NodeType\n",
     "\n",
     "kg = KnowledgeGraph()\n",
     "for doc in docs:\n",
@@ -91,8 +88,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.llms.base import llm_factory\n",
     "from ragas.embeddings.base import embedding_factory\n",
+    "from ragas.llms.base import llm_factory\n",
     "\n",
     "llm = llm_factory()\n",
     "embedding = embedding_factory()"
@@ -119,14 +116,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.testset.transforms import apply_transforms\n",
     "from ragas.testset.transforms import (\n",
     "    HeadlinesExtractor,\n",
     "    HeadlineSplitter,\n",
     "    KeyphrasesExtractor,\n",
+    "    apply_transforms,\n",
     ")\n",
     "\n",
-    "\n",
     "headline_extractor = HeadlinesExtractor(llm=llm)\n",
     "headline_splitter = HeadlineSplitter(min_tokens=300, max_tokens=1000)\n",
     "keyphrase_extractor = KeyphrasesExtractor(\n",
@@ -227,14 +223,15 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.testset.synthesizers.single_hop import (\n",
-    "    SingleHopQuerySynthesizer,\n",
-    ")\n",
     "from dataclasses import dataclass\n",
+    "\n",
     "from ragas.testset.synthesizers.prompts import (\n",
     "    ThemesPersonasInput,\n",
     "    ThemesPersonasMatchingPrompt,\n",
     ")\n",
+    "from ragas.testset.synthesizers.single_hop import (\n",
+    "    SingleHopQuerySynthesizer,\n",
+    ")\n",
     "\n",
     "\n",
     "@dataclass\n",

docs/howtos/customizations/testgenerator/testgen-customisation.ipynb

@@ -73,9 +73,7 @@
     }
    ],
    "source": [
-    "from ragas.testset.graph import KnowledgeGraph\n",
-    "from ragas.testset.graph import Node, NodeType\n",
-    "\n",
+    "from ragas.testset.graph import KnowledgeGraph, Node, NodeType\n",
     "\n",
     "kg = KnowledgeGraph()\n",
     "for doc in docs:\n",
@@ -106,8 +104,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.llms.base import llm_factory\n",
     "from ragas.embeddings.base import embedding_factory\n",
+    "from ragas.llms.base import llm_factory\n",
     "\n",
     "llm = llm_factory()\n",
     "embedding = embedding_factory()"
@@ -136,15 +134,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ragas.testset.transforms import apply_transforms\n",
     "from ragas.testset.transforms import (\n",
     "    HeadlinesExtractor,\n",
     "    HeadlineSplitter,\n",
     "    KeyphrasesExtractor,\n",
     "    OverlapScoreBuilder,\n",
+    "    apply_transforms,\n",
     ")\n",
     "\n",
-    "\n",
     "headline_extractor = HeadlinesExtractor(llm=llm)\n",
     "headline_splitter = HeadlineSplitter(min_tokens=300, max_tokens=1000)\n",
     "keyphrase_extractor = KeyphrasesExtractor(\n",
@@ -244,8 +241,9 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from dataclasses import dataclass\n",
     "import typing as t\n",
+    "from dataclasses import dataclass\n",
+    "\n",
     "from ragas.testset.synthesizers.multi_hop.base import (\n",
     "    MultiHopQuerySynthesizer,\n",
     "    MultiHopScenario,\n",