Merge pull request #12 from dcSpark/nico/add_download_page_tool

add downloag_page to markdown

apps/shinkai-tool-download-page/package.json

@@ -0,0 +1,4 @@
+{
+  "name": "@shinkai_protocol/shinkai-tool-download-page",
+  "type": "commonjs"
+}

apps/shinkai-tool-download-page/project.json

@@ -0,0 +1,35 @@
+{
+  "name": "@shinkai_protocol/shinkai-tool-download-page",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "sourceRoot": "apps/shinkai-tool-download-page/src",
+  "projectType": "library",
+  "tags": ["tool"],
+  "targets": {
+    "build": {
+      "executor": "@nx/webpack:webpack",
+      "outputs": ["{options.outputPath}"],
+      "defaultConfiguration": "production",
+      "options": {
+        "compiler": "tsc",
+        "outputPath": "dist/apps/shinkai-tool-download-page",
+        "main": "apps/shinkai-tool-download-page/src/index.ts",
+        "tsConfig": "apps/shinkai-tool-download-page/tsconfig.app.json",
+        "webpackConfig": "apps/shinkai-tool-download-page/webpack.config.ts"
+      },
+      "configurations": {
+        "development": {},
+        "production": {}
+      }
+    },
+    "lint": {
+      "executor": "@nx/linter:eslint",
+      "outputs": ["{options.outputFile}"],
+      "options": {
+        "lintFilePatterns": [
+          "apps/shinkai-tool-download-page/**/*.ts",
+          "apps/shinkai-tool-download-page/package.json"
+        ]
+      }
+    }
+  }
+}

apps/shinkai-tool-download-page/src/index.ts

@@ -0,0 +1,48 @@
+import { BaseTool, RunResult } from '@shinkai_protocol/shinkai-tools-builder';
+import { ToolDefinition } from 'libs/shinkai-tools-builder/src/tool-definition';
+import * as TurndownService from 'turndown';
+
+type Config = {};
+type Params = {
+  url: string;
+};
+
+type Result = { markdown: string };
+
+
+export class Tool extends BaseTool<Config, Params, Result> {
+  definition: ToolDefinition<Config, Params, Result> = {
+    id: 'shinkai-tool-download-page',
+    name: 'Shinkai: Download Page',
+    description: 'Downloads a URL and converts its HTML content to Markdown',
+    author: 'Shinkai',
+    keywords: ['download page', 'url to markdown', 'shinkai'],
+    configurations: {
+      type: 'object',
+      properties: {},
+      required: [],
+    },
+    parameters: {
+      type: 'object',
+      properties: {
+        url: { type: 'string' },
+      },
+      required: ['url'],
+    },
+    result: {
+      type: 'object',
+      properties: {
+        markdown: { type: 'string' },
+      },
+      required: ['markdown'],
+    },
+  };
+
+  async run(params: Params): Promise<RunResult<Result>> {
+    const response = await fetch(params.url);
+    const html = await response.text();
+    const turndownService = new TurndownService();
+    const markdown = turndownService.turndown(html);
+    return Promise.resolve({ data: { markdown } });
+  }
+}

apps/shinkai-tool-download-page/tsconfig.app.json

@@ -0,0 +1,4 @@
+{
+  "extends": "./tsconfig.json",
+  "include": ["./src/**/*.ts"]
+}

apps/shinkai-tool-download-page/tsconfig.json

@@ -0,0 +1,10 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+
+  },
+  "include": [
+    "./src/**/*.ts",
+    "webpack.config.ts"
+  ],
+}

apps/shinkai-tool-download-page/webpack.config.ts

@@ -0,0 +1,17 @@
+import * as path from 'path';
+
+import { composePlugins, withNx } from '@nx/webpack';
+import { merge } from 'lodash';
+
+import { withToolWebpackConfig } from '@shinkai_protocol/shinkai-tools-bundler';
+
+module.exports = composePlugins(withNx(), (config, { options, context }) => {
+  return merge(
+    config,
+    withToolWebpackConfig({
+      entry: path.join(__dirname, 'src/index.ts'),
+      outputPath: path.join(__dirname, '../../dist/apps/shinkai-tool-echo'),
+      tsConfigFile: path.join(__dirname, 'tsconfig.app.json'),
+    }),
+  );
+});

libs/shinkai-tools-runner/src/built_in_tools.rs

@@ -56,6 +56,16 @@ lazy_static! {
                 .unwrap(),
             )),
         );
+        m.insert(
+            "shinkai-tool-download-page",
+            &*Box::leak(Box::new(
+                serde_json::from_str::<ToolDefinition>(include_str!(concat!(
+                    env!("CARGO_MANIFEST_DIR"),
+                    "/tools/shinkai-tool-download-page/definition.json"
+                )))
+                .unwrap(),
+            )),
+        );
         m
     };
 }

package.json

@@ -15,6 +15,7 @@
     "@nx/webpack": "^19.3.1",
     "@types/lodash": "^4.17.6",
     "@types/node": "^20.14.0",
+    "@types/turndown": "^5.0.4",
     "@typescript-eslint/eslint-plugin": "^7.14.1",
     "buffer": "^6.0.3",
     "copyfiles": "^2.4.1",
@@ -42,6 +43,7 @@
   "dependencies": {
     "ajv": "^7.2.4",
     "expr-eval": "^2.0.2",
-    "micro-eth-signer": "^0.10.0"
+    "micro-eth-signer": "^0.10.0",
+    "turndown": "^7.2.0"
   }
 }