fix(langchain): update JSONL loader to support complex json structures (#8863)

Co-authored-by: Yury Kozyrev <[email protected]>
Co-authored-by: Hunter Lovell <[email protected]>
Co-authored-by: Hunter Lovell <[email protected]>

Author: 3 people

.changeset/rotten-rockets-allow.md

@@ -0,0 +1,5 @@
+---
+"langchain": patch
+---
+
+update JSONL loader to support complex json structures

langchain/src/document_loaders/fs/json.ts

@@ -168,6 +168,18 @@ export class JSONLinesLoader extends TextLoader {
       .filter(Boolean)
       .map((line) => JSON.parse(line));
     const pointer = jsonpointer.compile(this.pointer);
-    return jsons.map((json) => pointer.get(json));
+    return jsons.map((json) => {
+      const data = pointer.get(json);
+      if (typeof data === "string") {
+        return data;
+      }
+      if (!data) {
+        return "";
+      }
+      if (typeof data === "object") {
+        return JSON.stringify(data);
+      }
+      return "";
+    });
   }
 }

langchain/src/document_loaders/tests/example_data/ml_training.jsonl

@@ -0,0 +1,2 @@
+{"systemInstruction":{"role":"system","parts":[{"text":"You are professional in being an expert\n**There must be only JSON in the output.**"}]},"contents":[{"role":"user","parts":[{"text":"{\"product_id\":\"product1\",\"version\":4,\"supplier\":{\"name\":\"BASF GmbH\"}    }"}]},{"role":"model","parts":[{"text":"{\"is_company\":true }"}]}]}
+{"systemInstruction":{"role":"system","parts":[{"text":"You are professional in being an expert\n**There must be only JSON in the output.**"}]},"contents":[{"role":"user","parts":[{"text":"{\"product_id\":\"product2\",\"version\":4,\"supplier\":{\"name\":\"Martin Burger\"}}"}]},{"role":"model","parts":[{"text":"{\"is_company\":false}"}]}]}

langchain/src/document_loaders/tests/jsonl.test.ts

@@ -20,3 +20,20 @@ test("Test JSON loader from file", async () => {
     })
   );
 });
+
+test("Test JSON loader from complex JSONL file", async () => {
+  const filePath = path.resolve(
+    path.dirname(url.fileURLToPath(import.meta.url)),
+    "./example_data/ml_training.jsonl"
+  );
+  const loader = new JSONLinesLoader(filePath, "/contents");
+  const docs = await loader.load();
+
+  expect(docs.length).toBe(2);
+  expect(docs[0].metadata).toEqual({ source: filePath, line: 1 });
+  const [user, model] = JSON.parse(docs[0].pageContent);
+  const userData = JSON.parse(user.parts[0].text);
+  expect(userData.supplier.name).toEqual("BASF GmbH");
+  const modelData = JSON.parse(model.parts[0].text);
+  expect(modelData.is_company).toBeTruthy();
+});