Merge pull request #22 from dcSpark/nico/duckduckgo-search

Nico/duckduckgo search

apps/shinkai-tool-duckduckgo-search/jest.config.ts

@@ -0,0 +1,6 @@
+/* eslint-disable */
+export default {
+  displayName: '@shinkai_protocol/shinkai-tool-duckduckgo-search',
+  preset: '../../jest.preset.js',
+  coverageDirectory: '../../coverage/apps/shinkai-tool-duckduckgo-search',
+};

apps/shinkai-tool-duckduckgo-search/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@shinkai_protocol/shinkai-tool-duckduckgo-search",
+  "type": "commonjs"
+}

apps/shinkai-tool-duckduckgo-search/project.json

@@ -0,0 +1,35 @@
+{
+  "name": "@shinkai_protocol/shinkai-tool-duckduckgo-search",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "apps/shinkai-tool-duckduckgo-search/src",
+  "projectType": "library",
+  "tags": ["tool"],
+  "targets": {
+    "build": {
+      "executor": "@nx/webpack:webpack",
+      "outputs": ["{options.outputPath}"],
+      "defaultConfiguration": "production",
+      "options": {
+        "compiler": "tsc",
+        "outputPath": "dist/apps/shinkai-tool-duckduckgo-search",
+        "main": "apps/shinkai-tool-duckduckgo-search/src/index.ts",
+        "tsConfig": "apps/shinkai-tool-duckduckgo-search/tsconfig.app.json",
+        "webpackConfig": "apps/shinkai-tool-duckduckgo-search/webpack.config.ts"
+      },
+      "configurations": {
+        "development": {},
+        "production": {}
+      }
+    },
+    "lint": {
+      "executor": "@nx/linter:eslint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": [
+          "apps/shinkai-tool-duckduckgo-search/**/*.ts",
+          "apps/shinkai-tool-duckduckgo-search/package.json"
+        ]
+      }
+    }
+  }
+}

apps/shinkai-tool-duckduckgo-search/src/index.test.ts

@@ -0,0 +1,15 @@
+import { Tool } from '../src/index';
+global.fetch = require('node-fetch');
+
+test('searches DuckDuckGo and gets a response', async () => {
+  const tool = new Tool({});
+  const result = await tool.run({ message: 'best movie of all time' });
+  const message = result.data.message;
+  const searchResults = JSON.parse(message.replace(/^searching: /, ''));
+
+  expect(Array.isArray(searchResults)).toBe(true);
+  expect(searchResults.length).toBeGreaterThan(0);
+  expect(searchResults[0]).toHaveProperty('title');
+  expect(searchResults[0]).toHaveProperty('url'); // Updated from 'href' to 'url'
+  expect(searchResults[0]).toHaveProperty('description'); // Updated from 'body' to 'description'
+});

apps/shinkai-tool-duckduckgo-search/src/index.ts

@@ -0,0 +1,136 @@
+import { BaseTool, RunResult } from '@shinkai_protocol/shinkai-tools-builder';
+import { ToolDefinition } from 'libs/shinkai-tools-builder/src/tool-definition';
+
+type Config = {};
+type Params = {
+  message: string;
+};
+type Result = { message: string };
+
+interface SearchResult {
+  title: string;
+  description: string;
+  url: string;
+}
+
+export class Tool extends BaseTool<Config, Params, Result> {
+  definition: ToolDefinition<Config, Params, Result> = {
+    id: 'shinkai-tool-duckduckgo-search',
+    name: 'Shinkai: DuckDuckGo Search',
+    description: 'Searches the DuckDuckGo search engine',
+    author: 'Shinkai',
+    keywords: ['duckduckgo', 'search', 'shinkai'],
+    configurations: {
+      type: 'object',
+      properties: {},
+      required: [],
+    },
+    parameters: {
+      type: 'object',
+      properties: {
+        message: { type: 'string' },
+      },
+      required: ['message'],
+    },
+    result: {
+      type: 'object',
+      properties: {
+        message: { type: 'string' },
+      },
+      required: ['message'],
+    },
+  };
+
+  private static async getVQD(keywords: string): Promise<string> {
+    const response = await fetch('https://duckduckgo.com', {
+      method: 'POST',
+      body: new URLSearchParams({ q: keywords }),
+    });
+    const text = await response.text();
+    // console.log('DuckDuckGo response HTML:', text);
+
+    // Extract vqd token using a regular expression
+    const vqdMatch = text.match(/vqd=\\?"([^\\"]+)\\?"/);
+    // console.log('vqdMatch: ', vqdMatch);
+    if (!vqdMatch || vqdMatch.length < 2) {
+      throw new Error('Failed to retrieve vqd token');
+    }
+    const vqd = vqdMatch[1];
+    // console.log('vqd: ', vqd);
+    return vqd;
+  }
+
+  private static parseDuckDuckGoResponse(response: string): SearchResult[] {
+    // Regex to extract the JSON content
+    const jsonPattern =
+      /DDG\.inject\('DDG\.Data\.languages\.adLanguages', \{\}\);if \(DDG\.pageLayout\) DDG\.pageLayout\.load\('d',\[(.*?)\]\);DDG\.duckbar\.load\('images'\);DDG\.duckbar\.load\('news'\);/s;
+    const match = response.match(jsonPattern);
+
+    if (!match) {
+      throw new Error('JSON content not found in the response.');
+    }
+
+    // Extracted JSON content as string
+    const jsonString = `[${match[1]}]`;
+
+    // Parse JSON string
+    const jsonData = JSON.parse(jsonString);
+
+    // Extract search results
+    const results: SearchResult[] = jsonData
+    .map((item: any) => ({
+      title: item.t,
+      description: item.a,
+      url: item.u,
+    }))
+    .filter((result: SearchResult) => result.title && result.description && result.url);
+
+    // console.log('results: ', results);
+    // Convert to JSON string
+    return results;
+  }
+
+  private static async textSearch(keywords: string): Promise<any[]> {
+    const vqd = await this.getVQD(keywords);
+    const url = new URL('https://links.duckduckgo.com/d.js');
+    url.searchParams.append('q', keywords);
+    url.searchParams.append('vqd', vqd);
+    url.searchParams.append('kl', 'wt-wt');
+    url.searchParams.append('l', 'wt-wt');
+    url.searchParams.append('p', '');
+    url.searchParams.append('s', '0');
+    url.searchParams.append('df', '');
+    url.searchParams.append('ex', '-1');
+
+    const response = await fetch(url.toString(), {
+      method: 'GET',
+      headers: {
+        'Content-Type': 'application/json',
+      },
+    });
+    const text = await response.text();
+    console.log('DuckDuckGo search response:', text);
+
+    // Parse the response using the custom parser
+    const results = Tool.parseDuckDuckGoResponse(text);
+    if (results.length === 0) {
+      throw new Error('Failed to extract search results');
+    }
+
+    return results;
+  }
+
+  async run(params: Params): Promise<RunResult<Result>> {
+    console.log('run duckduckgo search from js', 4);
+    try {
+      const results = await Tool.textSearch(params.message);
+      return { data: { message: JSON.stringify(results) } };
+    } catch (error) {
+      let errorMessage = 'An unknown error occurred';
+      if (error instanceof Error) {
+        errorMessage = error.message;
+      }
+      return { data: { message: `Error: ${errorMessage}` } };
+    }
+  }
+}

apps/shinkai-tool-duckduckgo-search/tsconfig.app.json

@@ -0,0 +1,4 @@
+{
+  "extends": "./tsconfig.json",
+  "include": ["./src/**/*.ts"]
+}

apps/shinkai-tool-duckduckgo-search/tsconfig.json

@@ -0,0 +1,8 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {},
+  "include": [
+    "./src/**/*.ts",
+    "webpack.config.ts"
+  ]
+}

apps/shinkai-tool-duckduckgo-search/tsconfig.spec.json

@@ -0,0 +1,14 @@
+{
+  "extends": "./tsconfig.json",
+  "compilerOptions": {
+    "outDir": "../../dist/out-tsc",
+    "module": "commonjs",
+    "types": ["jest", "node"]
+  },
+  "include": [
+    "jest.config.ts",
+    "src/**/*.test.ts",
+    "src/**/*.spec.ts",
+    "src/**/*.d.ts"
+  ]
+}

apps/shinkai-tool-duckduckgo-search/webpack.config.ts

@@ -0,0 +1,17 @@
+import * as path from 'path';
+
+import { composePlugins, withNx } from '@nx/webpack';
+import { merge } from 'lodash';
+
+import { withToolWebpackConfig } from '@shinkai_protocol/shinkai-tools-bundler';
+
+module.exports = composePlugins(withNx(), (config, { options, context }) => {
+  return merge(
+    config,
+    withToolWebpackConfig({
+      entry: path.join(__dirname, 'src/index.ts'),
+      outputPath: path.join(__dirname, '../../dist/apps/shinkai-tool-duckduckgo-search'),
+      tsConfigFile: path.join(__dirname, 'tsconfig.app.json'),
+    }),
+  );
+});

libs/shinkai-tools-runner/src/built_in_tools.rs

@@ -87,25 +87,36 @@ lazy_static! {
             )),
         );
         m.insert(
-          "shinkai-tool-ethplorer-tokens",
-          &*Box::leak(Box::new(
-              serde_json::from_str::<ToolDefinition>(include_str!(concat!(
-                  env!("CARGO_MANIFEST_DIR"),
-                  "/tools/shinkai-tool-ethplorer-tokens/definition.json"
-              )))
-              .unwrap(),
-          )),
-        );
-        m.insert(
-          "shinkai-tool-ethplorer-tokens",
-          &*Box::leak(Box::new(
-              serde_json::from_str::<ToolDefinition>(include_str!(concat!(
-                  env!("CARGO_MANIFEST_DIR"),
-                  "/tools/shinkai-tool-ethplorer-tokens/definition.json"
-              )))
-              .unwrap(),
-          )),
+            "shinkai-tool-ethplorer-tokens",
+            &*Box::leak(Box::new(
+                serde_json::from_str::<ToolDefinition>(include_str!(concat!(
+                    env!("CARGO_MANIFEST_DIR"),
+                    "/tools/shinkai-tool-ethplorer-tokens/definition.json"
+                )))
+                .unwrap(),
+            )),
         );
+      // https://bugzilla.mozilla.org/show_bug.cgi?id=1681809
+      //   m.insert(
+      //     "shinkai-tool-token-price",
+      //     &*Box::leak(Box::new(
+      //         serde_json::from_str::<ToolDefinition>(include_str!(concat!(
+      //             env!("CARGO_MANIFEST_DIR"),
+      //             "/tools/shinkai-tool-token-price/definition.json"
+      //         )))
+      //         .unwrap(),
+      //     )),
+      // );
+      m.insert(
+        "shinkai-tool-duckduckgo-search",
+        &*Box::leak(Box::new(
+            serde_json::from_str::<ToolDefinition>(include_str!(concat!(
+                env!("CARGO_MANIFEST_DIR"),
+                "/tools/shinkai-tool-duckduckgo-search/definition.json"
+            )))
+            .unwrap(),
+        )),
+    );
         // ntim: New tools will be inserted here, don't remove this comment
         m
     };