feat: adding namespaces (#226)

Author: shreyashankar

.github/workflows/docker-ci.yml

@@ -47,6 +47,8 @@ jobs:
         echo "Waiting for container to start..."
         sleep 30
 
+        frontend_healthy=false
+
         # Check container health for up to 3 minutes
         for i in {1..6}; do
           if ! docker ps -q -f name=docetl-test > /dev/null 2>&1; then
@@ -56,8 +58,9 @@ jobs:
           fi
 
           # Try to curl the frontend
-          if curl -s -f http://localhost:3000 > /dev/null; then
+          if curl -s -f http://localhost:3000/playground > /dev/null; then
             echo "Frontend is responding"
+            frontend_healthy=true
             break
           fi
 
@@ -71,6 +74,13 @@ jobs:
           sleep 30
         done
         
+        # Explicitly fail if frontend check never succeeded
+        if [ "$frontend_healthy" = false ]; then
+          echo "Frontend health check failed"
+          docker logs docetl-test
+          exit 1
+        fi
+        
         # If we get here, container is running and healthy
         echo "Container is running successfully"
         

docetl/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.1.7"
+__version__ = "0.2"
 
 from docetl.runner import DSLRunner
 from docetl.builder import Optimizer

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "docetl"
-version = "0.1.7"
+version = "0.2"
 description = "ETL with LLM operations."
 authors = ["Shreya Shankar <[email protected]>"]
 license = "MIT"

tailwind.config.js

@@ -0,0 +1,16 @@
+{
+  theme: {
+    extend: {
+      keyframes: {
+        shake: {
+          '0%, 100%': { transform: 'translateX(0)' },
+          '25%': { transform: 'translateX(-4px)' },
+          '75%': { transform: 'translateX(4px)' },
+        },
+      },
+      animation: {
+        shake: 'shake 0.2s ease-in-out 0s 2',
+      },
+    },
+  },
+} 

website/src/app/api/checkNamespace/route.ts

@@ -0,0 +1,33 @@
+import { NextResponse } from "next/server";
+import fs from "fs";
+import { getNamespaceDir } from "@/app/api/utils";
+import os from "os";
+
+export async function POST(request: Request) {
+  try {
+    const { namespace } = await request.json();
+    const homeDir = process.env.DOCETL_HOME_DIR || os.homedir();
+
+    if (!namespace) {
+      return NextResponse.json(
+        { error: "Namespace parameter is required" },
+        { status: 400 }
+      );
+    }
+
+    const namespaceDir = getNamespaceDir(homeDir, namespace);
+    const exists = fs.existsSync(namespaceDir);
+
+    if (!exists) {
+      fs.mkdirSync(namespaceDir, { recursive: true });
+    }
+
+    return NextResponse.json({ exists });
+  } catch (error) {
+    console.error("Error checking/creating namespace:", error);
+    return NextResponse.json(
+      { error: "Failed to check/create namespace" },
+      { status: 500 }
+    );
+  }
+}

website/src/app/api/getInputOutput/route.ts

@@ -4,8 +4,15 @@ import fs from "fs/promises";
 import os from "os";
 export async function POST(request: Request) {
   try {
-    const { default_model, data, operations, operation_id, name, sample_size } =
-      await request.json();
+    const {
+      default_model,
+      data,
+      operations,
+      operation_id,
+      name,
+      sample_size,
+      namespace,
+    } = await request.json();
 
     if (!name) {
       return NextResponse.json(
@@ -29,7 +36,8 @@ export async function POST(request: Request) {
       operation_id,
       name,
       homeDir,
-      sample_size
+      sample_size,
+      namespace
     );
 
     // Check if inputPath exists

website/src/app/api/getPipelineConfig/route.ts

@@ -10,6 +10,7 @@ export async function POST(request: Request) {
       operation_id,
       name,
       sample_size,
+      namespace,
       system_prompt,
     } = await request.json();
 
@@ -37,6 +38,7 @@ export async function POST(request: Request) {
       name,
       homeDir,
       sample_size,
+      namespace,
       system_prompt
     );
 

website/src/app/api/runPipeline/route.ts

@@ -13,6 +13,7 @@ export async function POST(request: NextRequest) {
   try {
     const formData = await request.formData();
     const files = formData.getAll("files") as File[];
+    const namespace = formData.get("namespace") as string;
 
     if (!files || files.length === 0) {
       return NextResponse.json({ error: "No files provided" }, { status: 400 });
@@ -21,7 +22,7 @@ export async function POST(request: NextRequest) {
     const homeDir = process.env.DOCETL_HOME_DIR || os.homedir();
 
     // Create uploads directory in user's home directory if it doesn't exist
-    const uploadsDir = path.join(homeDir, ".docetl", "documents");
+    const uploadsDir = path.join(homeDir, ".docetl", namespace, "documents");
     await mkdir(uploadsDir, { recursive: true });
 
     const savedFiles = await Promise.all(

website/src/app/api/saveDocuments/route.ts

@@ -8,7 +8,7 @@ export async function POST(request: NextRequest) {
   try {
     const formData = await request.formData();
     const file = formData.get("file") as File;
-
+    const namespace = formData.get("namespace") as string;
     if (!file) {
       return NextResponse.json({ error: "No file uploaded" }, { status: 400 });
     }
@@ -19,7 +19,7 @@ export async function POST(request: NextRequest) {
 
     // Create uploads directory in user's home directory if it doesn't exist
     const homeDir = process.env.DOCETL_HOME_DIR || os.homedir();
-    const uploadDir = path.join(homeDir, ".docetl", "files");
+    const uploadDir = path.join(homeDir, ".docetl", namespace, "files");
     await mkdir(uploadDir, { recursive: true });
 
     // Create full file path