feat: use the duckduckgo search

package.json

@@ -24,6 +24,8 @@
     "@ai-sdk/openai": "^1.1.9",
     "@mendable/firecrawl-js": "^1.16.0",
     "ai": "^4.1.17",
+    "axios": "^1.7.9",
+    "cheerio": "^1.0.0",
     "js-tiktoken": "^1.0.17",
     "lodash-es": "^4.17.21",
     "p-limit": "^6.2.0",

src/deep-research.ts

@@ -1,4 +1,5 @@
 import FirecrawlApp, { SearchResponse } from '@mendable/firecrawl-js';
+import search from './duckduckgo-search';
 import { generateObject } from 'ai';
 import { compact } from 'lodash-es';
 import pLimit from 'p-limit';
@@ -206,12 +207,24 @@ export async function deepResearch({
     serpQueries.map(serpQuery =>
       limit(async () => {
         try {
-          const result = await firecrawl.search(serpQuery.query, {
+          let result = await firecrawl.search(serpQuery.query, {
             timeout: 15000,
             limit: 5,
             scrapeOptions: { formats: ['markdown'] },
           });
 
+          // If the result is empty, use the DuckDuckGo search as a fallback
+          if (result.data.length === 0) {
+            log(
+              `No results found for ${serpQuery.query}, falling back to DuckDuckGo search`,
+            );
+            result = await search(serpQuery.query, {
+              timeout: 15000,
+              limit: 5,
+              scrapeOptions: { formats: ['markdown'] },
+            });
+          }
+
           // Collect URLs from this search
           const newUrls = compact(result.data.map(item => item.url));
           const newBreadth = Math.ceil(breadth / 2);