Merge branch 'FalkorDB:main' into main

Author: liorkesos

.wordlist.txt

@@ -37,4 +37,5 @@ scalable
 APIs
 Embeddings
 NLP
-graphrag
+graphrag
+gemini

README.md

@@ -2,7 +2,7 @@
 [![Dockerhub](https://img.shields.io/docker/pulls/falkordb/falkordb?label=Docker)](https://hub.docker.com/r/falkordb/falkordb/)
 [![pypi](https://badge.fury.io/py/graphrag_sdk.svg)](https://pypi.org/project/graphrag_sdk/)
 [![Discord](https://img.shields.io/discord/1146782921294884966?style=flat-square)](https://discord.gg/6M4QwDXn2w)
-[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md) 
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg)](CODE_OF_CONDUCT.md)
 
 <p align="center">
   <img alt="FalkorDB GraphRAG-SDK README Banner" src="images/FalkorDB GraphRAG-SDK README Banner.png" width="1500">
@@ -34,14 +34,17 @@ pip install graphrag_sdk[litellm]
 
 ### Configure Credentials. See [.env](.env.template) for examples.
 
-* [LiteLLM](https://docs.litellm.ai): A framework supporting inference of large language models, allowing flexibility in deployment and use cases.
+* [LiteLLM](https://docs.litellm.ai): A framework supporting inference of large language models, allowing flexibility in deployment and use cases.  
+  To choose vendor use the prefix "specific_vendor/your_model", for example "gemini/gemini-2.0-flash-exp".
 * [OpenAI](https://openai.com/index/openai-api) Recommended model:`gpt-4o`
 * [Google](https://makersuite.google.com/app/apikey) Recommended model:`gemini-2.0-flash-exp`
 * [Azure-OpenAI](https://ai.azure.com) Recommended model:`gpt-4o`
 * [Ollama](https://ollama.com/) Available only to the Q&A step. Recommended models: `llama3`. Ollama models are suitable for the Q&A step only (after the knowledge graph (KG) created).
 
 
 # How to use
+[![Get started](https://pl-bolts-doc-images.s3.us-east-2.amazonaws.com/app-2/get-started-badge.svg)](https://lightning.ai/muhammadqadora/studios/build-fast-accurate-genai-apps-advanced-rag-with-falkordb)
+[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/FalkorDB/GraphRAG-SDK/blob/main/examples/movies/demo-movies.ipynb)
 
 ### Step 1: Creating Ontologies
 Automate ontology creation from unstructured data or define it manually - See [example](https://github.com/liorkesos/GraphRAG-SDK/blob/main/examples/trip/demo_orchestrator_trip.ipynb)
@@ -65,7 +68,7 @@ urls = ["https://www.rottentomatoes.com/m/side_by_side_2012",
 
 sources = [URL(url) for url in urls]
 
-# Model
+# Model - vendor: gemini, model: gemini-2.0-flash-exp -> gemini/gemini-2.0-flash-exp
 model = LiteModel(model_name="gemini/gemini-2.0-flash-exp")
 
 # Ontology Auto-Detection

examples/movies/demo-movies.ipynb

@@ -50,6 +50,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "# Get your Credentails from google website, see https://aistudio.google.com/\n",
     "os.environ[\"GEMINI_API_KEY\"] = \"\"\n",
     "\n",
     "# Get your Credentails from Falkor website, see https://app.falkordb.cloud\n",

graphrag_sdk/helpers.py

@@ -1,6 +1,6 @@
 import re
-import graphrag_sdk
 import logging
+import graphrag_sdk
 from fix_busted_json import repair_json
 
 logger = logging.getLogger(__name__)
@@ -215,4 +215,4 @@ def validate_cypher_relation_directions(
         except Exception:
             continue
 
-    return errors
+    return errors

graphrag_sdk/kg.py

@@ -78,6 +78,7 @@ def __init__(
         self._name = name
         self._model_config = model_config
         self.sources = set([])
+        self.failed_sources = set([])
 
         if cypher_system_instruction is None:
             cypher_system_instruction = CYPHER_GEN_SYSTEM
@@ -146,38 +147,47 @@ def list_sources(self) -> list[AbstractSource]:
         return [s.source for s in self.sources]
 
     def process_sources(
-        self, sources: list[AbstractSource], instructions: str = None
+        self, sources: list[AbstractSource], instructions: str = None, hide_progress: bool = False
     ) -> None:
         """
         Add entities and relations found in sources into the knowledge-graph
 
         Parameters:
             sources (list[AbstractSource]): list of sources to extract knowledge from
+            instructions (str): instructions to use for extraction
+            hide_progress (bool): hide progress bar
         """
 
         if self.ontology is None:
             raise Exception("Ontology is not defined")
 
         # Create graph with sources
-        self._create_graph_with_sources(sources, instructions)
+        failed_sources = self._create_graph_with_sources(sources, instructions, hide_progress)
 
         # Add processed sources
         for src in sources:
-            self.sources.add(src)
+            if src not in failed_sources:
+                self.sources.add(src)
+                self.failed_sources.discard(src)
+            else:
+                self.failed_sources.add(src)
 
     def _create_graph_with_sources(
-        self, sources: list[AbstractSource] | None = None, instructions: str = None
-    ):
+        self, sources: list[AbstractSource] | None = None, instructions: str = None, hide_progress: bool = False
+    ) -> list[AbstractSource]:
 
         step = ExtractDataStep(
             sources=list(sources),
             ontology=self.ontology,
             model=self._model_config.extract_data,
             graph=self.graph,
+            hide_progress=hide_progress,
         )
 
-        step.run(instructions)
-
+        failed_sources = step.run(instructions)
+        
+        return failed_sources
+        
     def delete(self) -> None:
         """
         Deletes the knowledge graph and any other related resource

graphrag_sdk/ontology.py

@@ -25,7 +25,7 @@ def __init__(self, entities: list[Entity] = None, relations: list[Relation] = No
         """
         Initialize the Ontology class.
 
-        Args:
+        Parameters:
             entities (list[Entity], optional): List of Entity objects. Defaults to None.
             relations (list[Relation], optional): List of Relation objects. Defaults to None.
         """
@@ -37,14 +37,16 @@ def from_sources(
         sources: list[AbstractSource],
         model: GenerativeModel,
         boundaries: Optional[str] = None,
+        hide_progress: bool = False,
     ) -> "Ontology":
         """
         Create an Ontology object from a list of sources.
 
-        Args:
+        Parameters:
             sources (list[AbstractSource]): A list of AbstractSource objects representing the sources.
             boundaries (Optinal[str]): The boundaries for the ontology.
             model (GenerativeModel): The generative model to use.
+            hide_progress (bool): Whether to hide the progress bar.
 
         Returns:
             The created Ontology object.
@@ -53,6 +55,7 @@ def from_sources(
             sources=sources,
             ontology=Ontology(),
             model=model,
+            hide_progress=hide_progress,
         )
 
         return step.run(boundaries=boundaries)
@@ -62,7 +65,7 @@ def from_json(txt: dict | str):
         """
         Creates an Ontology object from a JSON representation.
 
-        Args:
+        Parameters:
             txt (dict | str): The JSON representation of the ontology. It can be either a dictionary or a string.
 
         Returns:
@@ -82,7 +85,7 @@ def from_graph(graph: Graph):
         """
         Creates an Ontology object from a given graph.
 
-        Args:
+        Parameters:
             graph (Graph): The graph object representing the ontology.
 
         Returns:
@@ -114,7 +117,7 @@ def add_relation(self, relation: Relation):
         """
         Adds a relation to the ontology.
 
-        Args:
+        Parameters:
             relation (Relation): The relation to be added.
         """
         self.relations.append(relation)
@@ -135,7 +138,7 @@ def merge_with(self, o: "Ontology"):
         """
         Merges the given ontology `o` with the current ontology.
 
-        Args:
+        Parameters:
             o (Ontology): The ontology to merge with.
 
         Returns:
@@ -256,7 +259,7 @@ def get_entity_with_label(self, label: str):
         """
         Retrieves the entity with the specified label.
 
-        Args:
+        Parameters:
             label (str): The label of the entity to retrieve.
 
         Returns:
@@ -318,7 +321,7 @@ def save_to_graph(self, graph: Graph):
         """
         Saves the entities and relations to the specified graph.
 
-        Args:
+        Parameters:
             graph (Graph): The graph to save the entities and relations to.
         """
         for entity in self.entities: